From 446cec49e1aa75c4258c07994d0c5ae18ea0c128 Mon Sep 17 00:00:00 2001
From: "mengji.fy" <mengji.fy@taobao.com>
Date: Fri, 17 Mar 2017 18:33:34 +0800
Subject: [PATCH 001/253] [FLINK-6058] fix read DEFAULT_PARALLELISM from
 ContextEnvironment

---
 .../flink/client/program/ContextEnvironment.java      | 11 +++++++++++
 .../api/environment/StreamContextEnvironment.java     |  8 +-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ContextEnvironment.java b/flink-clients/src/main/java/org/apache/flink/client/program/ContextEnvironment.java
index 1ef94cedf2032..91815e1673a33 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/ContextEnvironment.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/ContextEnvironment.java
@@ -23,6 +23,8 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.Plan;
 import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.optimizer.plan.OptimizedPlan;
 import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
 import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
@@ -44,6 +46,8 @@ public class ContextEnvironment extends ExecutionEnvironment {
 	protected final ClassLoader userCodeClassLoader;
 
 	protected final SavepointRestoreSettings savepointSettings;
+
+	private final int defaultParallelism;
 	
 	public ContextEnvironment(ClusterClient remoteConnection, List<URL> jarFiles, List<URL> classpaths,
 				ClassLoader userCodeClassLoader, SavepointRestoreSettings savepointSettings) {
@@ -52,6 +56,9 @@ public ContextEnvironment(ClusterClient remoteConnection, List<URL> jarFiles, Li
 		this.classpathsToAttach = classpaths;
 		this.userCodeClassLoader = userCodeClassLoader;
 		this.savepointSettings = savepointSettings;
+		this.defaultParallelism = GlobalConfiguration.loadConfiguration().getInteger(
+							ConfigConstants.DEFAULT_PARALLELISM_KEY,
+							ConfigConstants.DEFAULT_PARALLELISM);
 	}
 
 	@Override
@@ -104,6 +111,10 @@ public SavepointRestoreSettings getSavepointRestoreSettings() {
 		return savepointSettings;
 	}
 
+	public int getDefaultParallelism() {
+		return this.defaultParallelism;
+	}
+
 	// --------------------------------------------------------------------------------------------
 	
 	static void setAsContext(ContextEnvironmentFactory factory) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
index 49c53478c2d9c..55418133496f0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
@@ -21,12 +21,8 @@
 import org.apache.flink.api.common.JobExecutionResult;
 import org.apache.flink.client.program.ContextEnvironment;
 import org.apache.flink.client.program.DetachedEnvironment;
-
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.streaming.api.graph.StreamGraph;
 import org.apache.flink.util.Preconditions;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,9 +38,7 @@ protected StreamContextEnvironment(ContextEnvironment ctx) {
 		if (ctx.getParallelism() > 0) {
 			setParallelism(ctx.getParallelism());
 		} else {
-			setParallelism(GlobalConfiguration.loadConfiguration().getInteger(
-					ConfigConstants.DEFAULT_PARALLELISM_KEY,
-					ConfigConstants.DEFAULT_PARALLELISM));
+			setParallelism(ctx.getDefaultParallelism());
 		}
 	}
 

From 598536ec75411b5afc547d42779658f76e1ba34b Mon Sep 17 00:00:00 2001
From: WangTaoTheTonic <wangtao111@huawei.com>
Date: Sat, 25 Feb 2017 12:19:43 +0800
Subject: [PATCH 002/253] [FLINK-5904] Make jobmanager.heap.mb and
 taskmanager.heap.mb work in YARN mode

This closes #3414.
---
 .../apache/flink/configuration/ConfigConstants.java |  2 +-
 .../flink/configuration/JobManagerOptions.java      |  5 +++++
 .../flink/configuration/TaskManagerOptions.java     |  5 +++++
 .../flink/yarn/AbstractYarnClusterDescriptor.java   | 13 +++++++++++--
 4 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index de06b59411fa8..a035beb9a2dc7 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -113,7 +113,7 @@ public final class ConfigConstants {
 	public static final String EXECUTION_RETRY_DELAY_KEY = "execution-retries.delay";
 	
 	// -------------------------------- Runtime -------------------------------
-	
+
 	/**
 	 * The config parameter defining the network address to connect to
 	 * for communication with the job manager.
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index d9c9d1815485a..2534ddf75cc11 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -57,6 +57,11 @@ public class JobManagerOptions {
 		key("jobmanager.rpc.port")
 		.defaultValue(6123);
 
+	/** JVM heap size (in megabytes) for the JobManager */
+	public static final ConfigOption<Integer> JOB_MANAGER_HEAP_MEMORY =
+		key("jobmanager.heap.mb")
+		.defaultValue(1024);
+
 	/**
 	 * The maximum number of prior execution attempts kept in history.
 	 */
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
index adfc8e96288e9..9a00a0ff5b60a 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
@@ -39,6 +39,11 @@ public class TaskManagerOptions {
 			key("taskmanager.jvm-exit-on-oom")
 			.defaultValue(false);
 
+	/** JVM heap size (in megabytes) for the TaskManagers */
+	public static final ConfigOption<Integer> TASK_MANAGER_HEAP_MEMORY =
+			key("taskmanager.heap.mb")
+			.defaultValue(1024);
+		   
 	/** Size of memory buffers used by the network stack and the memory manager (in bytes). */
 	public static final ConfigOption<Integer> MEMORY_SEGMENT_SIZE =
 			key("taskmanager.memory.segment-size")
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index b383b59c1de5e..9cb80aa21d5c1 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -26,10 +26,12 @@
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.configuration.SecurityOptions;
 import org.apache.flink.configuration.HighAvailabilityOptions;
+import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
+import org.apache.flink.runtime.jobmanager.JobManagerOptions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -113,9 +115,9 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 	 */
 	private int slots = -1;
 
-	private int jobManagerMemoryMb = 1024;
+	private int jobManagerMemoryMb = JobManagerOptions.JOB_MANAGER_HEAP_MEMORY.defaultValue();
 
-	private int taskManagerMemoryMb = 1024;
+	private int taskManagerMemoryMb = TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.defaultValue();
 
 	private int taskManagerCount = 1;
 
@@ -166,6 +168,13 @@ public AbstractYarnClusterDescriptor() {
 			flinkConfigurationPath = new Path(confFile.getAbsolutePath());
 
 			slots = flinkConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
+			
+			if (flinkConfiguration.containsKey(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY.key())) {
+				jobManagerMemoryMb = flinkConfiguration.getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY);
+			}
+			if (flinkConfiguration.containsKey(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.key())) {
+				taskManagerMemoryMb = flinkConfiguration.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY);
+			}
 		} catch (Exception e) {
 			LOG.debug("Config couldn't be loaded from environment variable.");
 		}

From 3b5dc9dbec8f2542929de6b9eafb7053a1786531 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Tue, 18 Apr 2017 17:24:29 +0200
Subject: [PATCH 003/253] [FLINK-5904] Use proper ConfigOption syntax to
 retrieve config values

---
 .../flink/configuration/JobManagerOptions.java    |  4 +++-
 .../flink/yarn/AbstractYarnClusterDescriptor.java | 15 ++++++---------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index 2534ddf75cc11..d129405e06a60 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -57,7 +57,9 @@ public class JobManagerOptions {
 		key("jobmanager.rpc.port")
 		.defaultValue(6123);
 
-	/** JVM heap size (in megabytes) for the JobManager */
+	/**
+	 * JVM heap size (in megabytes) for the JobManager
+	 */
 	public static final ConfigOption<Integer> JOB_MANAGER_HEAP_MEMORY =
 		key("jobmanager.heap.mb")
 		.defaultValue(1024);
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index 9cb80aa21d5c1..ec7af5a811eae 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -24,6 +24,7 @@
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.SecurityOptions;
 import org.apache.flink.configuration.HighAvailabilityOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
@@ -31,7 +32,6 @@
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
-import org.apache.flink.runtime.jobmanager.JobManagerOptions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -168,15 +168,12 @@ public AbstractYarnClusterDescriptor() {
 			flinkConfigurationPath = new Path(confFile.getAbsolutePath());
 
 			slots = flinkConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
-			
-			if (flinkConfiguration.containsKey(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY.key())) {
-				jobManagerMemoryMb = flinkConfiguration.getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY);
-			}
-			if (flinkConfiguration.containsKey(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.key())) {
-				taskManagerMemoryMb = flinkConfiguration.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY);
-			}
+
+			jobManagerMemoryMb = flinkConfiguration.getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY);
+			taskManagerMemoryMb = flinkConfiguration.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY);
+
 		} catch (Exception e) {
-			LOG.debug("Config couldn't be loaded from environment variable.");
+			LOG.debug("Config couldn't be loaded from environment variable.", e);
 		}
 	}
 

From f27bfe37d6e519e6fd6c3c601b26d774028ec2cd Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Thu, 13 Apr 2017 14:07:29 -0400
Subject: [PATCH 004/253] [hotfix] [gelly] Driver updates

- refactor SimpleDriver to call internal plan method
- add CLI parameters for RMatGraph, AdamicAdar, JaccardIndex
- remove unused data from VertexDegrees
- JaccardIndex now filters on > rather than >=
- handle null in ValueArrayTypeInfo
- add NonForwardingIdentityMapper to GraphUtils
---
 .../java/org/apache/flink/graph/Runner.java   | 14 +++--
 .../flink/graph/drivers/AdamicAdar.java       | 20 ++++--
 .../graph/drivers/ClusteringCoefficient.java  | 30 +++++----
 .../graph/drivers/ConnectedComponents.java    |  7 +--
 .../apache/flink/graph/drivers/EdgeList.java  |  5 +-
 .../org/apache/flink/graph/drivers/HITS.java  |  9 +--
 .../flink/graph/drivers/JaccardIndex.java     | 27 ++++++--
 .../apache/flink/graph/drivers/PageRank.java  |  9 +--
 .../flink/graph/drivers/SimpleDriver.java     | 63 ++++++++++++++++---
 .../flink/graph/drivers/TriangleListing.java  | 32 ++++++----
 .../flink/graph/drivers/input/RMatGraph.java  |  3 +
 .../annotate/directed/VertexDegrees.java      | 30 ++++-----
 .../library/link_analysis/Functions.java      |  4 +-
 .../graph/library/link_analysis/PageRank.java |  5 ++
 .../library/similarity/JaccardIndex.java      |  7 ++-
 .../types/valuearray/ValueArrayTypeInfo.java  | 13 +++-
 .../valuearray/ValueArrayTypeInfoFactory.java |  5 +-
 .../apache/flink/graph/utils/GraphUtils.java  | 27 +++++++-
 .../library/similarity/JaccardIndexTest.java  |  2 +
 19 files changed, 224 insertions(+), 88 deletions(-)

diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java
index 4b6cf42112188..5ffe681cd0fc8 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java
@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.commons.lang3.text.StrBuilder;
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.io.CsvOutputFormat;
 import org.apache.flink.api.java.utils.ParameterTool;
@@ -188,20 +189,21 @@ private static String getAlgorithmUsage(String algorithmName) {
 	public static void main(String[] args) throws Exception {
 		// Set up the execution environment
 		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		ExecutionConfig config = env.getConfig();
 
 		// should not have any non-Flink data types
-		env.getConfig().disableAutoTypeRegistration();
-		env.getConfig().disableForceAvro();
-		env.getConfig().disableForceKryo();
+		config.disableAutoTypeRegistration();
+		config.disableForceAvro();
+		config.disableForceKryo();
 
 		ParameterTool parameters = ParameterTool.fromArgs(args);
-		env.getConfig().setGlobalJobParameters(parameters);
+		config.setGlobalJobParameters(parameters);
 
 		// integration tests run with with object reuse both disabled and enabled
 		if (parameters.has("__disable_object_reuse")) {
-			env.getConfig().disableObjectReuse();
+			config.disableObjectReuse();
 		} else {
-			env.getConfig().enableObjectReuse();
+			config.enableObjectReuse();
 		}
 
 		// Usage
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java
index 742c1ded1e1de..8bf9268c5a871 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java
@@ -20,9 +20,11 @@
 
 import org.apache.commons.lang3.text.StrBuilder;
 import org.apache.commons.lang3.text.WordUtils;
+import org.apache.flink.api.java.DataSet;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.output.CSV;
 import org.apache.flink.graph.drivers.output.Print;
+import org.apache.flink.graph.drivers.parameter.DoubleParameter;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
 import org.apache.flink.graph.library.similarity.AdamicAdar.Result;
 import org.apache.flink.types.CopyableValue;
@@ -33,8 +35,16 @@
  * Driver for {@link org.apache.flink.graph.library.similarity.AdamicAdar}.
  */
 public class AdamicAdar<K extends CopyableValue<K>, VV, EV>
-extends SimpleDriver<Result<K>>
-implements Driver<K, VV, EV>, CSV, Print {
+extends SimpleDriver<K, VV, EV, Result<K>>
+implements CSV, Print {
+
+	private DoubleParameter minRatio = new DoubleParameter(this, "minimum_ratio")
+		.setDefaultValue(0.0)
+		.setMinimumValue(0.0, true);
+
+	private DoubleParameter minScore = new DoubleParameter(this, "minimum_score")
+		.setDefaultValue(0.0)
+		.setMinimumValue(0.0, true);
 
 	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
 		.setDefaultValue(PARALLELISM_DEFAULT);
@@ -61,11 +71,13 @@ public String getLongDescription() {
 	}
 
 	@Override
-	public void plan(Graph<K, VV, EV> graph) throws Exception {
+	protected DataSet<Result<K>> simplePlan(Graph<K, VV, EV> graph) throws Exception {
 		int lp = littleParallelism.getValue().intValue();
 
-		result = graph
+		return graph
 			.run(new org.apache.flink.graph.library.similarity.AdamicAdar<K, VV, EV>()
+				.setMinimumRatio(minRatio.getValue().floatValue())
+				.setMinimumScore(minScore.getValue().floatValue())
 				.setLittleParallelism(lp));
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ClusteringCoefficient.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ClusteringCoefficient.java
index c463c0a2d4585..4958b5afd8ebb 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ClusteringCoefficient.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ClusteringCoefficient.java
@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang3.text.StrBuilder;
 import org.apache.commons.lang3.text.WordUtils;
+import org.apache.flink.api.java.DataSet;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.GraphAnalytic;
 import org.apache.flink.graph.asm.result.PrintableResult;
@@ -43,8 +44,8 @@
  * @see org.apache.flink.graph.library.clustering.undirected.LocalClusteringCoefficient
  */
 public class ClusteringCoefficient<K extends Comparable<K> & CopyableValue<K>, VV, EV>
-extends SimpleDriver<PrintableResult>
-implements Driver<K, VV, EV>, CSV, Hash, Print {
+extends SimpleDriver<K, VV, EV, PrintableResult>
+implements CSV, Hash, Print {
 
 	private static final String DIRECTED = "directed";
 
@@ -85,15 +86,11 @@ public String getLongDescription() {
 	}
 
 	@Override
-	public void plan(Graph<K, VV, EV> graph) throws Exception {
+	protected DataSet<PrintableResult> simplePlan(Graph<K, VV, EV> graph) throws Exception {
 		int lp = littleParallelism.getValue().intValue();
 
 		switch (order.getValue()) {
 			case DIRECTED:
-				result = graph
-					.run(new org.apache.flink.graph.library.clustering.directed.LocalClusteringCoefficient<K, VV, EV>()
-						.setLittleParallelism(lp));
-
 				globalClusteringCoefficient = graph
 					.run(new org.apache.flink.graph.library.clustering.directed.GlobalClusteringCoefficient<K, VV, EV>()
 						.setLittleParallelism(lp));
@@ -101,13 +98,14 @@ public void plan(Graph<K, VV, EV> graph) throws Exception {
 				averageClusteringCoefficient = graph
 					.run(new org.apache.flink.graph.library.clustering.directed.AverageClusteringCoefficient<K, VV, EV>()
 						.setLittleParallelism(lp));
-				break;
 
-			case UNDIRECTED:
-				result = graph
-					.run(new org.apache.flink.graph.library.clustering.undirected.LocalClusteringCoefficient<K, VV, EV>()
+				@SuppressWarnings("unchecked")
+				DataSet<PrintableResult> directedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
+					.run(new org.apache.flink.graph.library.clustering.directed.LocalClusteringCoefficient<K, VV, EV>()
 						.setLittleParallelism(lp));
+				return directedResult;
 
+			case UNDIRECTED:
 				globalClusteringCoefficient = graph
 					.run(new org.apache.flink.graph.library.clustering.undirected.GlobalClusteringCoefficient<K, VV, EV>()
 						.setLittleParallelism(lp));
@@ -115,7 +113,15 @@ public void plan(Graph<K, VV, EV> graph) throws Exception {
 				averageClusteringCoefficient = graph
 					.run(new org.apache.flink.graph.library.clustering.undirected.AverageClusteringCoefficient<K, VV, EV>()
 						.setLittleParallelism(lp));
-				break;
+
+				@SuppressWarnings("unchecked")
+				DataSet<PrintableResult> undirectedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
+					.run(new org.apache.flink.graph.library.clustering.undirected.LocalClusteringCoefficient<K, VV, EV>()
+						.setLittleParallelism(lp));
+				return undirectedResult;
+
+			default:
+				throw new RuntimeException("Unknown order: " + order);
 		}
 	}
 
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ConnectedComponents.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ConnectedComponents.java
index 32263cfe64188..95904d82a319d 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ConnectedComponents.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/ConnectedComponents.java
@@ -78,12 +78,9 @@ public void hash(String executionName) throws Exception {
 
 	@Override
 	public void print(String executionName) throws Exception {
-		Collect<Vertex<K, K>> collector = new Collect<>();
+		List<Vertex<K, K>> results = new Collect<Vertex<K, K>>().run(components).execute(executionName);
 
-		// Refactored due to openjdk7 compile error: https://travis-ci.org/greghogan/flink/builds/200487761
-		List<Vertex<K, K>> records = collector.run(components).execute(executionName);
-
-		for (Vertex<K, K> result : records) {
+		for (Vertex<K, K> result : results) {
 			System.out.println(result);
 		}
 	}
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/EdgeList.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/EdgeList.java
index 524e70ff55837..5da02847f0968 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/EdgeList.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/EdgeList.java
@@ -82,10 +82,7 @@ public void hash(String executionName) throws Exception {
 
 	@Override
 	public void print(String executionName) throws Exception {
-		Collect<Edge<K, EV>> collector = new Collect<>();
-
-		// Refactored due to openjdk7 compile error: https://travis-ci.org/greghogan/flink/builds/200487761
-		List<Edge<K, EV>> records = collector.run(edges).execute(executionName);
+		List<Edge<K, EV>> records = new Collect<Edge<K, EV>>().run(edges).execute(executionName);
 
 		if (hasNullValueEdges(edges)) {
 			for (Edge<K, EV> result : records) {
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/HITS.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/HITS.java
index 6081fea757b80..209cddf5add82 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/HITS.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/HITS.java
@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang3.text.StrBuilder;
 import org.apache.commons.lang3.text.WordUtils;
+import org.apache.flink.api.java.DataSet;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.output.CSV;
 import org.apache.flink.graph.drivers.output.Print;
@@ -30,8 +31,8 @@
  * Driver for {@link org.apache.flink.graph.library.link_analysis.HITS}.
  */
 public class HITS<K, VV, EV>
-extends SimpleDriver<Result<K>>
-implements Driver<K, VV, EV>, CSV, Print {
+extends SimpleDriver<K, VV, EV, Result<K>>
+implements CSV, Print {
 
 	private static final int DEFAULT_ITERATIONS = 10;
 
@@ -59,8 +60,8 @@ public String getLongDescription() {
 	}
 
 	@Override
-	public void plan(Graph<K, VV, EV> graph) throws Exception {
-		result = graph
+	protected DataSet<Result<K>> simplePlan(Graph<K, VV, EV> graph) throws Exception {
+		return graph
 			.run(new org.apache.flink.graph.library.link_analysis.HITS<K, VV, EV>(
 				iterationConvergence.getValue().iterations,
 				iterationConvergence.getValue().convergenceThreshold));
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java
index 1c836ea9e063e..ae0d5f8ccd0ee 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java
@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang3.text.StrBuilder;
 import org.apache.commons.lang3.text.WordUtils;
+import org.apache.flink.api.java.DataSet;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.output.CSV;
 import org.apache.flink.graph.drivers.output.Hash;
@@ -34,8 +35,24 @@
  * Driver for {@link org.apache.flink.graph.library.similarity.JaccardIndex}.
  */
 public class JaccardIndex<K extends CopyableValue<K>, VV, EV>
-extends SimpleDriver<Result<K>>
-implements Driver<K, VV, EV>, CSV, Hash, Print {
+extends SimpleDriver<K, VV, EV, Result<K>>
+implements CSV, Hash, Print {
+
+	private LongParameter minNumerator = new LongParameter(this, "minimum_numerator")
+		.setDefaultValue(0)
+		.setMinimumValue(0);
+
+	private LongParameter minDenominator = new LongParameter(this, "minimum_denominator")
+		.setDefaultValue(1)
+		.setMinimumValue(1);
+
+	private LongParameter maxNumerator = new LongParameter(this, "maximum_numerator")
+		.setDefaultValue(1)
+		.setMinimumValue(0);
+
+	private LongParameter maxDenominator = new LongParameter(this, "maximum_denominator")
+		.setDefaultValue(1)
+		.setMinimumValue(1);
 
 	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
 		.setDefaultValue(PARALLELISM_DEFAULT);
@@ -64,11 +81,13 @@ public String getLongDescription() {
 	}
 
 	@Override
-	public void plan(Graph<K, VV, EV> graph) throws Exception {
+	protected DataSet<Result<K>> simplePlan(Graph<K, VV, EV> graph) throws Exception {
 		int lp = littleParallelism.getValue().intValue();
 
-		result = graph
+		return graph
 			.run(new org.apache.flink.graph.library.similarity.JaccardIndex<K, VV, EV>()
+				.setMinimumScore(minNumerator.getValue().intValue(), minDenominator.getValue().intValue())
+				.setMaximumScore(maxNumerator.getValue().intValue(), maxDenominator.getValue().intValue())
 				.setLittleParallelism(lp));
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/PageRank.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/PageRank.java
index 8cef0772370eb..5d74bdbb8e3b2 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/PageRank.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/PageRank.java
@@ -19,6 +19,7 @@
 package org.apache.flink.graph.drivers;
 
 import org.apache.commons.lang3.text.StrBuilder;
+import org.apache.flink.api.java.DataSet;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.output.CSV;
 import org.apache.flink.graph.drivers.output.Print;
@@ -30,8 +31,8 @@
  * @see org.apache.flink.graph.library.link_analysis.PageRank
  */
 public class PageRank<K, VV, EV>
-extends SimpleDriver<Result<K>>
-implements Driver<K, VV, EV>, CSV, Print {
+extends SimpleDriver<K, VV, EV, Result<K>>
+implements CSV, Print {
 
 	private static final int DEFAULT_ITERATIONS = 10;
 
@@ -64,8 +65,8 @@ public String getLongDescription() {
 	}
 
 	@Override
-	public void plan(Graph<K, VV, EV> graph) throws Exception {
-		result = graph
+	protected DataSet<Result<K>> simplePlan(Graph<K, VV, EV> graph) throws Exception {
+		return graph
 			.run(new org.apache.flink.graph.library.link_analysis.PageRank<K, VV, EV>(
 				dampingFactor.getValue(),
 				iterationConvergence.getValue().iterations,
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/SimpleDriver.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/SimpleDriver.java
index 98bdfc5374b13..5cecca1d9eb1e 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/SimpleDriver.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/SimpleDriver.java
@@ -19,6 +19,7 @@
 package org.apache.flink.graph.drivers;
 
 import org.apache.flink.api.java.DataSet;
+import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.asm.dataset.ChecksumHashCode;
 import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
 import org.apache.flink.graph.asm.dataset.Collect;
@@ -33,30 +34,74 @@
  *
  * @param <R> algorithm's result type
  */
-public abstract class SimpleDriver<R extends PrintableResult>
-extends ParameterizedBase {
+public abstract class SimpleDriver<K, VV, EV, R extends PrintableResult>
+extends ParameterizedBase
+implements Driver<K, VV, EV> {
 
-	protected DataSet<? extends R> result;
+	private DataSet<R> result;
 
+	protected DataSet<R> getResult() {
+		return result;
+	}
+
+	/**
+	 * Plan the algorithm and return the result {@link DataSet}.
+	 *
+	 * @param graph input graph
+	 * @return driver output
+	 * @throws Exception on error
+	 */
+	protected abstract DataSet<R> simplePlan(Graph<K, VV, EV> graph) throws Exception;
+
+	@Override
+	public void plan(Graph<K, VV, EV> graph) throws Exception {
+		result = simplePlan(graph);
+	}
+
+	/**
+	 * Print hash of execution results.
+	 *
+	 * Does *not* implement/override {@code Hash} since {@link Driver}
+	 * implementations designate the appropriate outputs.
+	 *
+	 * @param executionName job name
+	 * @throws Exception on error
+	 */
 	public void hash(String executionName) throws Exception {
 		Checksum checksum = new ChecksumHashCode<R>()
-			.run((DataSet<R>) result)
+			.run(result)
 			.execute(executionName);
 
 		System.out.println(checksum);
 	}
 
+	/**
+	 * Print execution results.
+	 *
+	 * Does *not* implement/override {@code Print} since {@link Driver}
+	 * implementations designate the appropriate outputs.
+	 *
+	 * @param executionName job name
+	 * @throws Exception on error
+	 */
 	public void print(String executionName) throws Exception {
-		Collect<R> collector = new Collect<>();
-
-		// Refactored due to openjdk7 compile error: https://travis-ci.org/greghogan/flink/builds/200487761
-		List<R> records = collector.run((DataSet<R>) result).execute(executionName);
+		List<R> results = new Collect<R>().run(result).execute(executionName);
 
-		for (R result : records) {
+		for (R result : results) {
 			System.out.println(result.toPrintableString());
 		}
 	}
 
+	/**
+	 * Write execution results to file using CSV format.
+	 *
+	 * Does *not* implement/override {@code CSV} since {@link Driver}
+	 * implementations designate the appropriate outputs.
+	 *
+	 * @param filename output filename
+	 * @param lineDelimiter CSV delimiter between lines
+	 * @param fieldDelimiter CSV delimiter between fields
+	 */
 	public void writeCSV(String filename, String lineDelimiter, String fieldDelimiter) {
 		result
 			.writeAsCsv(filename, lineDelimiter, fieldDelimiter)
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
index ca0c1674066e5..5157b8e04c109 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang3.text.StrBuilder;
 import org.apache.commons.lang3.text.WordUtils;
+import org.apache.flink.api.java.DataSet;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.GraphAnalytic;
 import org.apache.flink.graph.asm.result.PrintableResult;
@@ -42,8 +43,8 @@
  * @see org.apache.flink.graph.library.clustering.undirected.TriadicCensus
  */
 public class TriangleListing<K extends Comparable<K> & CopyableValue<K>, VV, EV>
-extends SimpleDriver<PrintableResult>
-implements Driver<K, VV, EV>, CSV, Hash, Print {
+extends SimpleDriver<K, VV, EV, PrintableResult>
+implements CSV, Hash, Print {
 
 	private static final String DIRECTED = "directed";
 
@@ -83,35 +84,40 @@ public String getLongDescription() {
 	}
 
 	@Override
-	public void plan(Graph<K, VV, EV> graph) throws Exception {
+	protected DataSet<PrintableResult> simplePlan(Graph<K, VV, EV> graph) throws Exception {
 		int lp = littleParallelism.getValue().intValue();
 
 		switch (order.getValue()) {
 			case DIRECTED:
-				result = graph
-					.run(new org.apache.flink.graph.library.clustering.directed.TriangleListing<K, VV, EV>()
-						.setSortTriangleVertices(sortTriangleVertices.getValue())
-						.setLittleParallelism(lp));
-
 				if (computeTriadicCensus.getValue()) {
 					triadicCensus = graph
 						.run(new org.apache.flink.graph.library.clustering.directed.TriadicCensus<K, VV, EV>()
 							.setLittleParallelism(lp));
 				}
-				break;
 
-			case UNDIRECTED:
-				result = graph
-					.run(new org.apache.flink.graph.library.clustering.undirected.TriangleListing<K, VV, EV>()
+				@SuppressWarnings("unchecked")
+				DataSet<PrintableResult> directedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
+					.run(new org.apache.flink.graph.library.clustering.directed.TriangleListing<K, VV, EV>()
 						.setSortTriangleVertices(sortTriangleVertices.getValue())
 						.setLittleParallelism(lp));
+				return directedResult;
 
+			case UNDIRECTED:
 				if (computeTriadicCensus.getValue()) {
 					triadicCensus = graph
 						.run(new org.apache.flink.graph.library.clustering.undirected.TriadicCensus<K, VV, EV>()
 							.setLittleParallelism(lp));
 				}
-				break;
+
+				@SuppressWarnings("unchecked")
+				DataSet<PrintableResult> undirectedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
+					.run(new org.apache.flink.graph.library.clustering.undirected.TriangleListing<K, VV, EV>()
+						.setSortTriangleVertices(sortTriangleVertices.getValue())
+						.setLittleParallelism(lp));
+				return undirectedResult;
+
+			default:
+				throw new RuntimeException("Unknown order: " + order);
 		}
 	}
 
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java
index e4e6a4cec2a1a..d64534bec3ac8 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java
@@ -96,6 +96,9 @@ public class RMatGraph<K extends Comparable<K>>
 		.setMinimumValue(0.0, true)
 		.setMaximumValue(2.0, true);
 
+	private LongParameter seed = new LongParameter(this, "seed")
+		.setDefaultValue(JDKRandomGeneratorFactory.DEFAULT_SEED);
+
 	private Simplify simplify = new Simplify(this);
 
 	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/degree/annotate/directed/VertexDegrees.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/degree/annotate/directed/VertexDegrees.java
index a27ca29067c1e..f73d37bc6cca3 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/degree/annotate/directed/VertexDegrees.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/degree/annotate/directed/VertexDegrees.java
@@ -21,11 +21,11 @@
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.operators.Order;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
 import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFieldsFirst;
 import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFieldsSecond;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.graph.Edge;
 import org.apache.flink.graph.EdgeOrder;
@@ -118,7 +118,7 @@ protected boolean mergeConfiguration(GraphAlgorithmWrappingDataSet other) {
 	public DataSet<Vertex<K, Degrees>> runInternal(Graph<K, VV, EV> input)
 			throws Exception {
 		// s, t, bitmask
-		DataSet<Tuple3<K, K, ByteValue>> edgesWithOrder = input.getEdges()
+		DataSet<Tuple2<K, ByteValue>> vertexWithEdgeOrder = input.getEdges()
 			.flatMap(new EmitAndFlipEdge<K, EV>())
 				.setParallelism(parallelism)
 				.name("Emit and flip edge")
@@ -128,9 +128,8 @@ public DataSet<Vertex<K, Degrees>> runInternal(Graph<K, VV, EV> input)
 				.name("Reduce bitmask");
 
 		// s, d(s)
-		DataSet<Vertex<K, Degrees>> vertexDegrees = edgesWithOrder
+		DataSet<Vertex<K, Degrees>> vertexDegrees = vertexWithEdgeOrder
 			.groupBy(0)
-			.sortGroup(1, Order.ASCENDING)
 			.reduceGroup(new DegreeCount<K>())
 				.setParallelism(parallelism)
 				.name("Degree count");
@@ -178,22 +177,22 @@ public void flatMap(Edge<T, TV> value, Collector<Tuple3<T, T, ByteValue>> out)
 	 *
 	 * @param <T> ID type
 	 */
-	@ForwardedFields("0; 1")
+	@ForwardedFields("0")
 	private static final class ReduceBitmask<T>
-	implements GroupReduceFunction<Tuple3<T, T, ByteValue>, Tuple3<T, T, ByteValue>> {
+	implements GroupReduceFunction<Tuple3<T, T, ByteValue>, Tuple2<T, ByteValue>> {
+		private Tuple2<T, ByteValue> output = new Tuple2<>(null, new ByteValue());
+
 		@Override
-		public void reduce(Iterable<Tuple3<T, T, ByteValue>> values, Collector<Tuple3<T, T, ByteValue>> out)
+		public void reduce(Iterable<Tuple3<T, T, ByteValue>> values, Collector<Tuple2<T, ByteValue>> out)
 				throws Exception {
-			Tuple3<T, T, ByteValue> output = null;
-
 			byte bitmask = 0;
 
 			for (Tuple3<T, T, ByteValue> value: values) {
-				output = value;
+				output.f0 = value.f0;
 				bitmask |= value.f2.getValue();
 			}
 
-			output.f2.setValue(bitmask);
+			output.f1.setValue(bitmask);
 			out.collect(output);
 		}
 	}
@@ -203,21 +202,22 @@ public void reduce(Iterable<Tuple3<T, T, ByteValue>> values, Collector<Tuple3<T,
 	 *
 	 * @param <T> ID type
 	 */
+	@ForwardedFields("0")
 	private static class DegreeCount<T>
-	implements GroupReduceFunction<Tuple3<T, T, ByteValue>, Vertex<T, Degrees>> {
+	implements GroupReduceFunction<Tuple2<T, ByteValue>, Vertex<T, Degrees>> {
 		private Vertex<T, Degrees> output = new Vertex<>(null, new Degrees());
 
 		@Override
-		public void reduce(Iterable<Tuple3<T, T, ByteValue>> values, Collector<Vertex<T, Degrees>> out)
+		public void reduce(Iterable<Tuple2<T, ByteValue>> values, Collector<Vertex<T, Degrees>> out)
 				throws Exception {
 			long degree = 0;
 			long outDegree = 0;
 			long inDegree = 0;
 
-			for (Tuple3<T, T, ByteValue> edge : values) {
+			for (Tuple2<T, ByteValue> edge : values) {
 				output.f0 = edge.f0;
 
-				byte bitmask = edge.f2.getValue();
+				byte bitmask = edge.f1.getValue();
 
 				degree++;
 
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/Functions.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/Functions.java
index 5bb2f4c338577..a7d6ef1ab11f9 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/Functions.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/Functions.java
@@ -25,13 +25,15 @@
 
 class Functions {
 
+	private Functions() {}
+
 	/**
 	 * Sum vertices' scores.
 	 *
 	 * @param <T> ID type
 	 */
 	@ForwardedFields("0")
-	static class SumScore<T>
+	protected static final class SumScore<T>
 		implements ReduceFunction<Tuple2<T, DoubleValue>> {
 		@Override
 		public Tuple2<T, DoubleValue> reduce(Tuple2<T, DoubleValue> left, Tuple2<T, DoubleValue> right)
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java
index 747735eba273c..c5c4178275396 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java
@@ -45,6 +45,7 @@
 import org.apache.flink.graph.library.link_analysis.Functions.SumScore;
 import org.apache.flink.graph.library.link_analysis.PageRank.Result;
 import org.apache.flink.graph.utils.GraphUtils;
+import org.apache.flink.graph.utils.GraphUtils.IdentityMapper;
 import org.apache.flink.graph.utils.Murmur3_32;
 import org.apache.flink.graph.utils.proxy.GraphAlgorithmWrappingDataSet;
 import org.apache.flink.types.DoubleValue;
@@ -175,6 +176,10 @@ public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
 			.run(new VertexDegrees<K, VV, EV>()
 				.setParallelism(parallelism));
 
+		// prevent Exception "The dam has been closed." in TempBarrier
+		// for a simplified Graph as in PageRankITCase (see FLINK-5623)
+		vertexDegree = vertexDegree.map(new IdentityMapper<Vertex<K, Degrees>>());
+
 		// vertex count
 		DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);
 
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java
index 2f4516a92be25..bc3cb86c20658 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java
@@ -78,7 +78,7 @@ public class JaccardIndex<K extends CopyableValue<K>, VV, EV>
 
 	private int maximumScoreNumerator = 1;
 
-	private int maximumScoreDenominator = 0;
+	private int maximumScoreDenominator = 1;
 
 	private int littleParallelism = PARALLELISM_DEFAULT;
 
@@ -121,7 +121,7 @@ public JaccardIndex<K, VV, EV> setMinimumScore(int numerator, int denominator) {
 	}
 
 	/**
-	 * Filter out Jaccard Index scores greater than or equal to the given maximum fraction.
+	 * Filter out Jaccard Index scores greater than the given maximum fraction.
 	 *
 	 * @param numerator numerator of the maximum score
 	 * @param denominator denominator of the maximum score
@@ -253,6 +253,7 @@ public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
 	 * number of groups and {@link GenerateGroups} emits each edge into each group.
 	 *
 	 * @param <T> ID type
+	 * @param <ET> edge value type
 	 */
 	@ForwardedFields("0->1; 1->2")
 	private static class GenerateGroupSpans<T, ET>
@@ -439,7 +440,7 @@ public void reduce(Iterable<Tuple3<T, T, IntValue>> values, Collector<Result<T>>
 
 			if (unboundedScores ||
 					(count * minimumScoreDenominator >= distinctNeighbors * minimumScoreNumerator
-						&& count * maximumScoreDenominator < distinctNeighbors * maximumScoreNumerator)) {
+						&& count * maximumScoreDenominator <= distinctNeighbors * maximumScoreNumerator)) {
 				output.f0 = edge.f0;
 				output.f1 = edge.f1;
 				output.f2.setValue(count);
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfo.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfo.java
index ee9b7708bb9fd..4ba8e394fdd99 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfo.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfo.java
@@ -30,6 +30,7 @@
 import org.apache.flink.types.NullValue;
 import org.apache.flink.types.StringValue;
 import org.apache.flink.types.Value;
+import org.apache.flink.util.Preconditions;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -54,7 +55,7 @@ public class ValueArrayTypeInfo<T> extends TypeInformation<ValueArray<T>> implem
 
 	public ValueArrayTypeInfo(TypeInformation<T> valueType) {
 		this.valueType = valueType;
-		this.type = valueType.getTypeClass();
+		this.type = valueType == null ? null : valueType.getTypeClass();
 	}
 
 	@Override
@@ -85,12 +86,16 @@ public boolean isTupleType() {
 
 	@Override
 	public boolean isKeyType() {
+		Preconditions.checkNotNull(type, "TypeInformation type class is required");
+
 		return Comparable.class.isAssignableFrom(type);
 	}
 
 	@Override
 	@SuppressWarnings("unchecked")
 	public TypeSerializer<ValueArray<T>> createSerializer(ExecutionConfig executionConfig) {
+		Preconditions.checkNotNull(type, "TypeInformation type class is required");
+
 		if (IntValue.class.isAssignableFrom(type)) {
 			return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new IntValueArraySerializer();
 		} else if (LongValue.class.isAssignableFrom(type)) {
@@ -107,6 +112,8 @@ public TypeSerializer<ValueArray<T>> createSerializer(ExecutionConfig executionC
 	@SuppressWarnings({ "unchecked", "rawtypes" })
 	@Override
 	public TypeComparator<ValueArray<T>> createComparator(boolean sortOrderAscending, ExecutionConfig executionConfig) {
+		Preconditions.checkNotNull(type, "TypeInformation type class is required");
+
 		if (IntValue.class.isAssignableFrom(type)) {
 			return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new IntValueArrayComparator(sortOrderAscending);
 		} else if (LongValue.class.isAssignableFrom(type)) {
@@ -131,6 +138,8 @@ public Map<String, TypeInformation<?>> getGenericParameters() {
 
 	@Override
 	public int hashCode() {
+		Preconditions.checkNotNull(type, "TypeInformation type class is required");
+
 		return type.hashCode();
 	}
 
@@ -154,6 +163,8 @@ public boolean canEqual(Object obj) {
 
 	@Override
 	public String toString() {
+		Preconditions.checkNotNull(type, "TypeInformation type class is required");
+
 		return "ValueArrayType<" + type.getSimpleName() + ">";
 	}
 }
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfoFactory.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfoFactory.java
index 2145c3dc530c8..1cfb41594f281 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfoFactory.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfoFactory.java
@@ -36,6 +36,9 @@ public class ValueArrayTypeInfoFactory<T> extends TypeInfoFactory<ValueArray<T>>
 
 	@Override
 	public TypeInformation<ValueArray<T>> createTypeInfo(Type t, Map<String, TypeInformation<?>> genericParameters) {
-		return new ValueArrayTypeInfo(genericParameters.get("T"));
+		@SuppressWarnings("unchecked")
+		TypeInformation<ValueArray<T>> typeInfo = new ValueArrayTypeInfo(genericParameters.get("T"));
+
+		return typeInfo;
 	}
 }
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/utils/GraphUtils.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/utils/GraphUtils.java
index 78fb37888435a..52927518a94c0 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/utils/GraphUtils.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/utils/GraphUtils.java
@@ -22,8 +22,10 @@
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.graph.asm.translate.TranslateFunction;
 import org.apache.flink.types.LongValue;
 
 import static org.apache.flink.api.java.typeutils.ValueTypeInfo.LONG_VALUE_TYPE_INFO;
@@ -51,6 +53,7 @@ public static <T> DataSet<LongValue> count(DataSet<T> input) {
 	 *
 	 * @param <T> element type
 	 */
+	@ForwardedFields("*")
 	public static final class IdentityMapper<T>
 	implements MapFunction<T, T> {
 		public T map(T value) {
@@ -58,6 +61,20 @@ public T map(T value) {
 		}
 	}
 
+	/**
+	 * The identity mapper returns the input as output.
+	 *
+	 * This does not forward fields and is used to break an operator chain.
+	 *
+	 * @param <T> element type
+	 */
+	public static final class NonForwardingIdentityMapper<T>
+	implements MapFunction<T, T> {
+		public T map(T value) {
+			return value;
+		}
+	}
+
 	/**
 	 * Map each element to a value.
 	 *
@@ -65,7 +82,7 @@ public T map(T value) {
 	 * @param <O> output type
 	 */
 	public static class MapTo<I, O>
-	implements MapFunction<I, O>, ResultTypeQueryable<O> {
+	implements MapFunction<I, O>, ResultTypeQueryable<O>, TranslateFunction<I, O> {
 		private final O value;
 
 		/**
@@ -78,7 +95,13 @@ public MapTo(O value) {
 		}
 
 		@Override
-		public O map(I o) throws Exception {
+		public O map(I input) throws Exception {
+			return value;
+		}
+
+		@Override
+		public O translate(I input, O reuse)
+				throws Exception {
 			return value;
 		}
 
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/similarity/JaccardIndexTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/similarity/JaccardIndexTest.java
index 128ee70d0e5c0..2443359dbefa7 100644
--- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/similarity/JaccardIndexTest.java
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/similarity/JaccardIndexTest.java
@@ -81,6 +81,8 @@ public void testSimpleGraphWithMaximumScore()
 		String expectedResult =
 			"(0,1,1,4)\n" +
 			"(0,2,1,4)\n" +
+			"(0,3,2,4)\n" +
+			"(1,2,2,4)\n" +
 			"(1,3,1,6)\n" +
 			"(1,4,1,3)\n" +
 			"(1,5,1,3)\n" +

From e8c1693cfe0607c9c550a996f534edfb69012cc5 Mon Sep 17 00:00:00 2001
From: godfreyhe <godfreyhe@163.com>
Date: Wed, 19 Apr 2017 19:43:25 +0800
Subject: [PATCH 005/253] [FLINK-6327] [table] Bug in CommonCalc's
 estimateRowCount() method

This closes #3740.
---
 .../scala/org/apache/flink/table/plan/nodes/CommonCalc.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
index bc25140df725e..96a74704e6edb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
@@ -177,7 +177,7 @@ trait CommonCalc {
 
     if (calcProgram.getCondition != null) {
       // we reduce the result card to push filters down
-      (rowCnt * 0.75).min(1.0)
+      (rowCnt * 0.75).max(1.0)
     } else {
       rowCnt
     }

From bcc233a32ce37ee19c3e51de1f3823c98d24f658 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 19 Apr 2017 18:01:29 +0200
Subject: [PATCH 006/253] [hotfix] [tests] Stabilize
 SystemProcessingTimeServiceTest

---
 .../tasks/SystemProcessingTimeServiceTest.java     | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java
index 50e438c796a96..fb4f0878cb8f7 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java
@@ -20,10 +20,11 @@
 
 import org.apache.flink.core.testutils.OneShotLatch;
 import org.apache.flink.streaming.runtime.operators.TestProcessingTimeServiceTest.ReferenceSettingExceptionHandler;
-
 import org.apache.flink.util.TestLogger;
+
 import org.junit.Test;
 
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -178,7 +179,16 @@ public void onProcessingTime(long timestamp) throws Exception {
 			// this should cancel our future
 			timer.quiesceAndAwaitPending();
 
-			assertTrue(scheduledFuture.isCancelled());
+			// it may be that the cancelled status is not immediately visible after the
+			// termination (not necessary a volatile update), so we need to "get()" the cancellation
+			// exception to be on the safe side
+			try {
+				scheduledFuture.get();
+				fail("scheduled future is not cancelled");
+			}
+			catch (CancellationException ignored) {
+				// expected
+			}
 
 			scheduledFuture = timer.scheduleAtFixedRate(new ProcessingTimeCallback() {
 				@Override

From 220ef9af0b6fe5046ed1d9abadb1fd5abbb46d4c Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 19 Apr 2017 18:21:43 +0200
Subject: [PATCH 007/253] [hotfix] [tests] Stabilize AsyncCallsTest by ensuring
 delayed messages are not executed before their time

---
 .../rpc/akka/AkkaInvocationHandler.java       |  9 ++---
 .../flink/runtime/rpc/akka/AkkaRpcActor.java  | 33 +++++++++++--------
 .../runtime/rpc/akka/messages/RunAsync.java   | 23 ++++++-------
 .../flink/runtime/rpc/AsyncCallsTest.java     |  6 ++--
 4 files changed, 37 insertions(+), 34 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java
index 56505f9770c65..c21accf2e5223 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java
@@ -177,12 +177,13 @@ public void runAsync(Runnable runnable) {
 	}
 
 	@Override
-	public void scheduleRunAsync(Runnable runnable, long delay) {
+	public void scheduleRunAsync(Runnable runnable, long delayMillis) {
 		checkNotNull(runnable, "runnable");
-		checkArgument(delay >= 0, "delay must be zero or greater");
-		
+		checkArgument(delayMillis >= 0, "delay must be zero or greater");
+
 		if (isLocal) {
-			rpcEndpoint.tell(new RunAsync(runnable, delay), ActorRef.noSender());
+			long atTimeNanos = delayMillis == 0 ? 0 : System.nanoTime() + (delayMillis * 1_000_000);
+			rpcEndpoint.tell(new RunAsync(runnable, atTimeNanos), ActorRef.noSender());
 		} else {
 			throw new RuntimeException("Trying to send a Runnable to a remote actor at " +
 				rpcEndpoint.path() + ". This is not supported.");
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
index 99f8211a8c1d1..86cd83eabe17e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
@@ -268,22 +268,27 @@ private void handleRunAsync(RunAsync runAsync) {
 				runAsync.getClass().getName(),
 				runAsync.getClass().getName());
 		}
-		else if (runAsync.getDelay() == 0) {
-			// run immediately
-			try {
-				runAsync.getRunnable().run();
-			} catch (Throwable t) {
-				LOG.error("Caught exception while executing runnable in main thread.", t);
-				ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
-			}
-		}
 		else {
-			// schedule for later. send a new message after the delay, which will then be immediately executed 
-			FiniteDuration delay = new FiniteDuration(runAsync.getDelay(), TimeUnit.MILLISECONDS);
-			RunAsync message = new RunAsync(runAsync.getRunnable(), 0);
+			final long timeToRun = runAsync.getTimeNanos(); 
+			final long delayNanos;
+
+			if (timeToRun == 0 || (delayNanos = timeToRun - System.nanoTime()) <= 0) {
+				// run immediately
+				try {
+					runAsync.getRunnable().run();
+				} catch (Throwable t) {
+					LOG.error("Caught exception while executing runnable in main thread.", t);
+					ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
+				}
+			}
+			else {
+				// schedule for later. send a new message after the delay, which will then be immediately executed 
+				FiniteDuration delay = new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS);
+				RunAsync message = new RunAsync(runAsync.getRunnable(), timeToRun);
 
-			getContext().system().scheduler().scheduleOnce(delay, getSelf(), message,
-					getContext().dispatcher(), ActorRef.noSender());
+				getContext().system().scheduler().scheduleOnce(delay, getSelf(), message,
+						getContext().dispatcher(), ActorRef.noSender());
+			}
 		}
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/messages/RunAsync.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/messages/RunAsync.java
index ce4f9d6ca9065..4b8a0b4c6542a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/messages/RunAsync.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/messages/RunAsync.java
@@ -18,40 +18,37 @@
 
 package org.apache.flink.runtime.rpc.akka.messages;
 
-import java.io.Serializable;
-
 import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkArgument;
 
 /**
  * Message for asynchronous runnable invocations
  */
-public final class RunAsync implements Serializable {
-	private static final long serialVersionUID = -3080595100695371036L;
+public final class RunAsync {
 
 	/** The runnable to be executed. Transient, so it gets lost upon serialization */ 
-	private final transient Runnable runnable;
+	private final Runnable runnable;
 
 	/** The delay after which the runnable should be called */
-	private final long delay;
+	private final long atTimeNanos;
 
 	/**
 	 * Creates a new {@code RunAsync} message.
 	 * 
-	 * @param runnable  The Runnable to run.
-	 * @param delay     The delay in milliseconds. Zero indicates immediate execution.
+	 * @param runnable    The Runnable to run.
+	 * @param atTimeNanos The time (as for System.nanoTime()) when to execute the runnable.
 	 */
-	public RunAsync(Runnable runnable, long delay) {
-		checkArgument(delay >= 0);
+	public RunAsync(Runnable runnable, long atTimeNanos) {
+		checkArgument(atTimeNanos >= 0);
 		this.runnable = checkNotNull(runnable);
-		this.delay = delay;
+		this.atTimeNanos = atTimeNanos;
 	}
 
 	public Runnable getRunnable() {
 		return runnable;
 	}
 
-	public long getDelay() {
-		return delay;
+	public long getTimeNanos() {
+		return atTimeNanos;
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java
index 7affdb942777b..e636d6c7af448 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java
@@ -23,10 +23,10 @@
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.core.testutils.OneShotLatch;
 import org.apache.flink.runtime.akka.AkkaUtils;
-
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
 import org.apache.flink.util.TestLogger;
+
 import org.junit.AfterClass;
 import org.junit.Test;
 
@@ -121,7 +121,7 @@ public void testScheduleWithDelay() throws Exception {
 		final AtomicBoolean concurrentAccess = new AtomicBoolean(false);
 		final OneShotLatch latch = new OneShotLatch();
 
-		final long delay = 200;
+		final long delay = 100;
 
 		TestEndpoint testEndpoint = new TestEndpoint(akkaRpcService, lock);
 		testEndpoint.start();
@@ -161,7 +161,7 @@ public void run() {
 		assertFalse("Rpc Endpoint had concurrent access", testEndpoint.hasConcurrentAccess());
 		assertFalse("Rpc Endpoint had concurrent access", concurrentAccess.get());
 
-		assertTrue("call was not properly delayed", ((stop - start) / 1000000) >= delay);
+		assertTrue("call was not properly delayed", ((stop - start) / 1_000_000) >= delay);
 	}
 
 	// ------------------------------------------------------------------------

From f03c4b46aabf4d9790748dd2d611ee4cd6974f03 Mon Sep 17 00:00:00 2001
From: hamstah <hamstahguru@gmail.com>
Date: Sat, 15 Apr 2017 15:34:14 +0100
Subject: [PATCH 008/253] [FLINK-5646] [docs] Document JAR upload with the REST
 API

This closes #3722
---
 docs/monitoring/rest_api.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/monitoring/rest_api.md b/docs/monitoring/rest_api.md
index a53067d26cad2..c5efcc214a083 100644
--- a/docs/monitoring/rest_api.md
+++ b/docs/monitoring/rest_api.md
@@ -660,6 +660,19 @@ The `savepointPath` points to the external path of the savepoint, which can be u
 
 It is possible to upload, run, and list Flink programs via the REST APIs and web frontend.
 
+#### Upload a new JAR file
+
+Send a `POST` request to `/jars/upload` with your jar file sent as multi-part data under the `jarfile` file.
+Also make sure that the multi-part data includes the `Content-Type` of the file itself, some http libraries do not add the header by default.
+
+The multi-part payload should start like
+
+```
+------BoundaryXXXX
+Content-Disposition: form-data; name="jarfile"; filename="YourFileName.jar"
+Content-Type: application/x-java-archive
+```
+
 #### Run a Program (POST)
 
 Send a `POST` request to `/jars/:jarid/run`. The `jarid` parameter is the file name of the program JAR in the configured web frontend upload directory (configuration key `jobmanager.web.upload.dir`).

From f8dc2b3cd5c93f0233e73f19ebf093cd6baa4997 Mon Sep 17 00:00:00 2001
From: tonycox <anton_solovev@epam.com>
Date: Fri, 27 Jan 2017 12:06:09 +0400
Subject: [PATCH 009/253] [FLINK-5481] [core] Add utility method to easily
 generate RowTypeInfos

This closes #3127
---
 .../flink/api/java/typeutils/Types.java       | 72 +++++++++++++++++++
 .../org/apache/flink/table/api/Types.scala    | 50 +++++++++----
 .../stringexpr/CalcStringExpressionTest.scala |  4 +-
 .../table/expressions/ArrayTypeTest.scala     |  2 +-
 .../expressions/ScalarFunctionsTest.scala     |  6 +-
 .../table/expressions/TemporalTypesTest.scala | 46 ++++++------
 .../UserDefinedScalarFunctionTest.scala       |  8 +--
 .../utils/UserDefinedScalarFunctions.scala    |  2 +-
 8 files changed, 142 insertions(+), 48 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/Types.java

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/Types.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/Types.java
new file mode 100644
index 0000000000000..5159cde1ede19
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/Types.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+
+/**
+ * This class enumerates all supported types of
+ * the BasicTypeInfo, SqlTimeTypeInfo and RowTypeInfo for creation simplifying
+ */
+public class Types {
+
+	public static final BasicTypeInfo<String> STRING = BasicTypeInfo.STRING_TYPE_INFO;
+	public static final BasicTypeInfo<Boolean> BOOLEAN = BasicTypeInfo.BOOLEAN_TYPE_INFO;
+	public static final BasicTypeInfo<Byte> BYTE = BasicTypeInfo.BYTE_TYPE_INFO;
+	public static final BasicTypeInfo<Short> SHORT = BasicTypeInfo.SHORT_TYPE_INFO;
+	public static final BasicTypeInfo<Integer> INT = BasicTypeInfo.INT_TYPE_INFO;
+	public static final BasicTypeInfo<Long> LONG = BasicTypeInfo.LONG_TYPE_INFO;
+	public static final BasicTypeInfo<Float> FLOAT = BasicTypeInfo.FLOAT_TYPE_INFO;
+	public static final BasicTypeInfo<Double> DOUBLE = BasicTypeInfo.DOUBLE_TYPE_INFO;
+	public static final BasicTypeInfo<BigDecimal> DECIMAL = BasicTypeInfo.BIG_DEC_TYPE_INFO;
+
+	public static final SqlTimeTypeInfo<Date> SQL_DATE = SqlTimeTypeInfo.DATE;
+	public static final SqlTimeTypeInfo<Time> SQL_TIME = SqlTimeTypeInfo.TIME;
+	public static final SqlTimeTypeInfo<Timestamp> SQL_TIMESTAMP = SqlTimeTypeInfo.TIMESTAMP;
+
+	/**
+	 * Generates RowTypeInfo with default names (f1, f2 ..).
+	 * same as new RowTypeInfo(types)
+	 *
+	 * @param types of Row fields. e.g. ROW(Types.STRING, Types.INT)
+	 */
+	public static RowTypeInfo ROW(TypeInformation<?>... types) {
+		return new RowTypeInfo(types);
+	}
+
+	/**
+	 * Generates RowTypeInfo.
+	 * same as new RowTypeInfo(types, names)
+	 *
+	 * e.g. ROW(new String[]{"name", "number"}, Types.STRING, Types.INT)
+	 *
+	 * @param fieldNames array of field names
+	 * @param types array of field types
+	 */
+	public static RowTypeInfo ROW(String[] fieldNames, TypeInformation<?>... types) {
+		return new RowTypeInfo(types, fieldNames);
+	}
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
index 939cb67b0212a..b0cb33894d1d9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
@@ -17,29 +17,51 @@
  */
 package org.apache.flink.table.api
 
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, SqlTimeTypeInfo}
+import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
+import org.apache.flink.api.java.typeutils.{Types => JTypes}
 
 /**
   * This class enumerates all supported types of the Table API.
   */
-object Types {
+object Types extends JTypes {
 
-  val STRING = BasicTypeInfo.STRING_TYPE_INFO
-  val BOOLEAN = BasicTypeInfo.BOOLEAN_TYPE_INFO
+  val STRING = JTypes.STRING
+  val BOOLEAN = JTypes.BOOLEAN
 
-  val BYTE = BasicTypeInfo.BYTE_TYPE_INFO
-  val SHORT = BasicTypeInfo.SHORT_TYPE_INFO
-  val INT = BasicTypeInfo.INT_TYPE_INFO
-  val LONG = BasicTypeInfo.LONG_TYPE_INFO
-  val FLOAT = BasicTypeInfo.FLOAT_TYPE_INFO
-  val DOUBLE = BasicTypeInfo.DOUBLE_TYPE_INFO
-  val DECIMAL = BasicTypeInfo.BIG_DEC_TYPE_INFO
+  val BYTE = JTypes.BYTE
+  val SHORT = JTypes.SHORT
+  val INT = JTypes.INT
+  val LONG = JTypes.LONG
+  val FLOAT = JTypes.FLOAT
+  val DOUBLE = JTypes.DOUBLE
+  val DECIMAL = JTypes.DECIMAL
 
-  val DATE = SqlTimeTypeInfo.DATE
-  val TIME = SqlTimeTypeInfo.TIME
-  val TIMESTAMP = SqlTimeTypeInfo.TIMESTAMP
+  val SQL_DATE = JTypes.SQL_DATE
+  val SQL_TIME = JTypes.SQL_TIME
+  val SQL_TIMESTAMP = JTypes.SQL_TIMESTAMP
   val INTERVAL_MONTHS = TimeIntervalTypeInfo.INTERVAL_MONTHS
   val INTERVAL_MILLIS = TimeIntervalTypeInfo.INTERVAL_MILLIS
 
+  /**
+    * Generates RowTypeInfo with default names (f1, f2 ..).
+    * same as ``new RowTypeInfo(types)``
+    *
+    * @param types of Row fields. e.g. ROW(Types.STRING, Types.INT)
+    */
+  def ROW[T: Manifest](types: TypeInformation[_]*) = {
+    JTypes.ROW(types: _*)
+  }
+
+  /**
+    * Generates RowTypeInfo.
+    * same as ``new RowTypeInfo(types, names)``
+    *
+    * @param fields of Row. e.g. ROW(("name", Types.STRING), ("number", Types.INT))
+    */
+  def ROW_NAMED(fields: (String, TypeInformation[_])*) = {
+    val names = fields.toList.map(_._1).toArray
+    val types = fields.toList.map(_._2)
+    JTypes.ROW(names, types: _*)
+  }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/CalcStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/CalcStringExpressionTest.scala
index a5a524197994b..819d6d45bee21 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/CalcStringExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/CalcStringExpressionTest.scala
@@ -357,8 +357,8 @@ class CalcStringExpressionTest {
       .toTable(tEnv, 'a, 'b, 'c, 'd, 'e)
 
     val t1 = t.select('a, 'b, 'c, 'd, 'e, BigDecimal("11.2"), BigDecimal("11.2").bigDecimal,
-        "1984-07-12".cast(Types.DATE), "14:34:24".cast(Types.TIME),
-        "1984-07-12 14:34:24".cast(Types.TIMESTAMP))
+        "1984-07-12".cast(Types.SQL_DATE), "14:34:24".cast(Types.SQL_TIME),
+        "1984-07-12 14:34:24".cast(Types.SQL_TIMESTAMP))
     val t2 = t.select("a, b, c, d, e, 11.2, 11.2," +
       "'1984-07-12'.toDate, '14:34:24'.toTime," +
       "'1984-07-12 14:34:24'.toTimestamp")
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala
index 49cf5725607d5..72b5ab8006280 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala
@@ -347,7 +347,7 @@ class ArrayTypeTest extends ExpressionTestBase {
       Types.INT,
       Types.INT,
       PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO,
-      ObjectArrayTypeInfo.getInfoFor(Types.DATE),
+      ObjectArrayTypeInfo.getInfoFor(Types.SQL_DATE),
       ObjectArrayTypeInfo.getInfoFor(ObjectArrayTypeInfo.getInfoFor(Types.INT)),
       ObjectArrayTypeInfo.getInfoFor(PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO),
       ObjectArrayTypeInfo.getInfoFor(Types.INT),
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
index 9258c025537b5..474043a3d3a35 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
@@ -1150,9 +1150,9 @@ class ScalarFunctionsTest extends ExpressionTestBase {
       Types.DOUBLE,
       Types.INT,
       Types.DECIMAL,
-      Types.DATE,
-      Types.TIME,
-      Types.TIMESTAMP,
+      Types.SQL_DATE,
+      Types.SQL_TIME,
+      Types.SQL_TIMESTAMP,
       Types.INTERVAL_MILLIS,
       Types.INTERVAL_MONTHS,
       Types.BOOLEAN,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala
index 840bec19286fb..58577bebedbc9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala
@@ -44,7 +44,7 @@ class TemporalTypesTest extends ExpressionTestBase {
       "2040-09-11")
 
     testAllApis(
-      "1500-04-30".cast(Types.DATE),
+      "1500-04-30".cast(Types.SQL_DATE),
       "'1500-04-30'.cast(DATE)",
       "CAST('1500-04-30' AS DATE)",
       "1500-04-30")
@@ -61,7 +61,7 @@ class TemporalTypesTest extends ExpressionTestBase {
       "00:00:00")
 
     testAllApis(
-      "1:30:00".cast(Types.TIME),
+      "1:30:00".cast(Types.SQL_TIME),
       "'1:30:00'.cast(TIME)",
       "CAST('1:30:00' AS TIME)",
       "01:30:00")
@@ -78,7 +78,7 @@ class TemporalTypesTest extends ExpressionTestBase {
       "2040-09-11 00:00:00.0")
 
     testAllApis(
-      "1500-04-30 12:00:00".cast(Types.TIMESTAMP),
+      "1500-04-30 12:00:00".cast(Types.SQL_TIMESTAMP),
       "'1500-04-30 12:00:00'.cast(TIMESTAMP)",
       "CAST('1500-04-30 12:00:00' AS TIMESTAMP)",
       "1500-04-30 12:00:00.0")
@@ -168,62 +168,62 @@ class TemporalTypesTest extends ExpressionTestBase {
   @Test
   def testTimePointCasting(): Unit = {
     testAllApis(
-      'f0.cast(Types.TIMESTAMP),
+      'f0.cast(Types.SQL_TIMESTAMP),
       "f0.cast(TIMESTAMP)",
       "CAST(f0 AS TIMESTAMP)",
       "1990-10-14 00:00:00.0")
 
     testAllApis(
-      'f1.cast(Types.TIMESTAMP),
+      'f1.cast(Types.SQL_TIMESTAMP),
       "f1.cast(TIMESTAMP)",
       "CAST(f1 AS TIMESTAMP)",
       "1970-01-01 10:20:45.0")
 
     testAllApis(
-      'f2.cast(Types.DATE),
+      'f2.cast(Types.SQL_DATE),
       "f2.cast(DATE)",
       "CAST(f2 AS DATE)",
       "1990-10-14")
 
     testAllApis(
-      'f2.cast(Types.TIME),
+      'f2.cast(Types.SQL_TIME),
       "f2.cast(TIME)",
       "CAST(f2 AS TIME)",
       "10:20:45")
 
     testAllApis(
-      'f2.cast(Types.TIME),
+      'f2.cast(Types.SQL_TIME),
       "f2.cast(TIME)",
       "CAST(f2 AS TIME)",
       "10:20:45")
 
     testTableApi(
-      'f7.cast(Types.DATE),
+      'f7.cast(Types.SQL_DATE),
       "f7.cast(DATE)",
       "2002-11-09")
 
     testTableApi(
-      'f7.cast(Types.DATE).cast(Types.INT),
+      'f7.cast(Types.SQL_DATE).cast(Types.INT),
       "f7.cast(DATE).cast(INT)",
       "12000")
 
     testTableApi(
-      'f7.cast(Types.TIME),
+      'f7.cast(Types.SQL_TIME),
       "f7.cast(TIME)",
       "00:00:12")
 
     testTableApi(
-      'f7.cast(Types.TIME).cast(Types.INT),
+      'f7.cast(Types.SQL_TIME).cast(Types.INT),
       "f7.cast(TIME).cast(INT)",
       "12000")
 
     testTableApi(
-      'f8.cast(Types.TIMESTAMP),
+      'f8.cast(Types.SQL_TIMESTAMP),
       "f8.cast(TIMESTAMP)",
       "2016-06-27 07:23:33.0")
 
     testTableApi(
-      'f8.cast(Types.TIMESTAMP).cast(Types.LONG),
+      'f8.cast(Types.SQL_TIMESTAMP).cast(Types.LONG),
       "f8.cast(TIMESTAMP).cast(LONG)",
       "1467012213000")
   }
@@ -262,13 +262,13 @@ class TemporalTypesTest extends ExpressionTestBase {
       "false")
 
     testAllApis(
-      'f0.cast(Types.TIMESTAMP) !== 'f2,
+      'f0.cast(Types.SQL_TIMESTAMP) !== 'f2,
       "f0.cast(TIMESTAMP) !== f2",
       "CAST(f0 AS TIMESTAMP) <> f2",
       "true")
 
     testAllApis(
-      'f0.cast(Types.TIMESTAMP) === 'f6,
+      'f0.cast(Types.SQL_TIMESTAMP) === 'f6,
       "f0.cast(TIMESTAMP) === f6",
       "CAST(f0 AS TIMESTAMP) = f6",
       "true")
@@ -558,13 +558,13 @@ class TemporalTypesTest extends ExpressionTestBase {
 
   def typeInfo = {
     new RowTypeInfo(
-      Types.DATE,
-      Types.TIME,
-      Types.TIMESTAMP,
-      Types.DATE,
-      Types.DATE,
-      Types.TIME,
-      Types.TIMESTAMP,
+      Types.SQL_DATE,
+      Types.SQL_TIME,
+      Types.SQL_TIMESTAMP,
+      Types.SQL_DATE,
+      Types.SQL_DATE,
+      Types.SQL_TIME,
+      Types.SQL_TIMESTAMP,
       Types.INT,
       Types.LONG,
       Types.INTERVAL_MONTHS,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala
index 51583c3f21a40..91cce0c8df8cc 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala
@@ -284,7 +284,7 @@ class UserDefinedScalarFunctionTest extends ExpressionTestBase {
       "7591 and 43810000 and 655906210000")
 
     testAllApis(
-      JavaFunc1(Null(Types.TIME), 15, Null(Types.TIMESTAMP)),
+      JavaFunc1(Null(Types.SQL_TIME), 15, Null(Types.SQL_TIMESTAMP)),
       "JavaFunc1(Null(TIME), 15, Null(TIMESTAMP))",
       "JavaFunc1(NULL, 15, NULL)",
       "null and 15 and null")
@@ -337,9 +337,9 @@ class UserDefinedScalarFunctionTest extends ExpressionTestBase {
       Types.STRING,
       Types.BOOLEAN,
       TypeInformation.of(classOf[SimplePojo]),
-      Types.DATE,
-      Types.TIME,
-      Types.TIMESTAMP,
+      Types.SQL_DATE,
+      Types.SQL_TIME,
+      Types.SQL_TIMESTAMP,
       Types.INTERVAL_MONTHS,
       Types.INTERVAL_MILLIS,
       TypeInformation.of(classOf[Seq[String]])
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
index e858187912164..8972a7719ca5f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
@@ -106,7 +106,7 @@ object Func10 extends ScalarFunction {
   }
 
   override def getResultType(signature: Array[Class[_]]): TypeInformation[_] = {
-    Types.TIMESTAMP
+    Types.SQL_TIMESTAMP
   }
 }
 

From eacfeecc40ad37e95b46500102595a42fa611dcb Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 19 Apr 2017 20:05:21 +0200
Subject: [PATCH 010/253] [FLINK-5481] [core] Followups to the typeinfo/Types
 utility   - Move new 'Types' class to 'common' scope, not 'java api'.   -
 Make the method names the same in Scala and Java

---
 .../typeutils => common/typeinfo}/Types.java  | 30 ++++++++++---------
 .../org/apache/flink/table/api/Types.scala    |  6 ++--
 2 files changed, 19 insertions(+), 17 deletions(-)
 rename flink-core/src/main/java/org/apache/flink/api/{java/typeutils => common/typeinfo}/Types.java (71%)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/Types.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java
similarity index 71%
rename from flink-core/src/main/java/org/apache/flink/api/java/typeutils/Types.java
rename to flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java
index 5159cde1ede19..dcbb645b6f1cd 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/Types.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java
@@ -16,11 +16,10 @@
  * limitations under the License.
  */
 
-package org.apache.flink.api.java.typeutils;
+package org.apache.flink.api.common.typeinfo;
 
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
 
 import java.math.BigDecimal;
 import java.sql.Date;
@@ -28,9 +27,9 @@
 import java.sql.Timestamp;
 
 /**
- * This class enumerates all supported types of
- * the BasicTypeInfo, SqlTimeTypeInfo and RowTypeInfo for creation simplifying
+ * This class gives access to the type information of the most most common types.
  */
+@PublicEvolving
 public class Types {
 
 	public static final BasicTypeInfo<String> STRING = BasicTypeInfo.STRING_TYPE_INFO;
@@ -48,25 +47,28 @@ public class Types {
 	public static final SqlTimeTypeInfo<Timestamp> SQL_TIMESTAMP = SqlTimeTypeInfo.TIMESTAMP;
 
 	/**
-	 * Generates RowTypeInfo with default names (f1, f2 ..).
-	 * same as new RowTypeInfo(types)
+	 * Generates a RowTypeInfo with fields of the given types.
+	 * The fields have the default names (f1, f2 ..).
+	 * 
+	 * <p>This method is a shortcut to {@code new RowTypeInfo(types)}.
 	 *
-	 * @param types of Row fields. e.g. ROW(Types.STRING, Types.INT)
+	 * @param types The types of the row fields, e.g., Types.STRING, Types.INT
 	 */
 	public static RowTypeInfo ROW(TypeInformation<?>... types) {
 		return new RowTypeInfo(types);
 	}
 
 	/**
-	 * Generates RowTypeInfo.
-	 * same as new RowTypeInfo(types, names)
-	 *
-	 * e.g. ROW(new String[]{"name", "number"}, Types.STRING, Types.INT)
+	 * Generates a RowTypeInfo with fields of the given types and with given names.
+	 * 
+	 * <p>Example use: {@code ROW_NAMED(new String[]{"name", "number"}, Types.STRING, Types.INT)}.
+	 * 
+	 * <p>This method is identical to {@code new RowTypeInfo(types, names)}.
 	 *
 	 * @param fieldNames array of field names
 	 * @param types array of field types
 	 */
-	public static RowTypeInfo ROW(String[] fieldNames, TypeInformation<?>... types) {
+	public static RowTypeInfo ROW_NAMED(String[] fieldNames, TypeInformation<?>... types) {
 		return new RowTypeInfo(types, fieldNames);
 	}
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
index b0cb33894d1d9..262a452685ca5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
@@ -17,9 +17,9 @@
  */
 package org.apache.flink.table.api
 
-import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeinfo.{Types, TypeInformation}
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
-import org.apache.flink.api.java.typeutils.{Types => JTypes}
+import org.apache.flink.api.common.typeinfo.{Types => JTypes}
 
 /**
   * This class enumerates all supported types of the Table API.
@@ -62,6 +62,6 @@ object Types extends JTypes {
   def ROW_NAMED(fields: (String, TypeInformation[_])*) = {
     val names = fields.toList.map(_._1).toArray
     val types = fields.toList.map(_._2)
-    JTypes.ROW(names, types: _*)
+    JTypes.ROW_NAMED(names, types: _*)
   }
 }

From e7714c7e9b2bb1a9838d41e693de322c4ef74483 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Mon, 20 Mar 2017 18:49:38 +0100
Subject: [PATCH 011/253] [hotfix] [streaming] Propagate 'CancelTaskException'
 in during checkpoint-on-barrier

---
 .../org/apache/flink/streaming/runtime/tasks/StreamTask.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 11e8e0d2c8fbd..7de6c1a2705e3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -546,8 +546,9 @@ public void triggerCheckpointOnBarrier(
 			performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics);
 		}
 		catch (CancelTaskException e) {
-			throw new Exception("Operator " + getName() + " was cancelled while performing checkpoint " +
-				checkpointMetaData.getCheckpointId() + '.');
+			LOG.info("Operator {} was cancelled while performing checkpoint {}.", 
+					getName(), checkpointMetaData.getCheckpointId());
+			throw e;
 		}
 		catch (Exception e) {
 			throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() + " for operator " +

From 5abfd0c4eed5a5e26f4673604f178bfde00edc33 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Mon, 27 Mar 2017 17:41:04 +0200
Subject: [PATCH 012/253] [hotfix] [misc] Minor code cleanups

---
 .../core/io/VersionedIOReadableWritable.java  |  2 +-
 .../runtime/checkpoint/PendingCheckpoint.java | 30 ++++++++++++-------
 .../ContinuousFileMonitoringFunction.java     |  2 +-
 .../runtime/tasks/SourceStreamTaskTest.java   |  6 ++--
 4 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java b/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java
index 31d570ccac203..bad9cef48de0c 100644
--- a/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java
+++ b/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java
@@ -53,7 +53,7 @@ public void read(DataInputView in) throws IOException {
 	 */
 	protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
 		if (!isCompatibleVersion(foundVersion)) {
-			long expectedVersion = getVersion();
+			int expectedVersion = getVersion();
 			throw new VersionMismatchException(
 					"Incompatible version: found " + foundVersion + ", required " + expectedVersion);
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
index b7eb037a10c4b..e1182ae50b5f5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
@@ -62,6 +62,19 @@
  */
 public class PendingCheckpoint {
 
+	/**
+	 * Result of the {@link PendingCheckpoint#acknowledgedTasks} method.
+	 */
+	public enum TaskAcknowledgeResult {
+		SUCCESS, // successful acknowledge of the task
+		DUPLICATE, // acknowledge message is a duplicate
+		UNKNOWN, // unknown task acknowledged
+		DISCARDED // pending checkpoint has been discarded
+	}
+
+	// ------------------------------------------------------------------------
+
+	/** The PendingCheckpoint logs to the same logger as the CheckpointCoordinator */
 	private static final Logger LOG = LoggerFactory.getLogger(CheckpointCoordinator.class);
 
 	private final Object lock = new Object();
@@ -201,7 +214,8 @@ void setStatsCallback(@Nullable PendingCheckpointStats trackerCallback) {
 	}
 
 	/**
-	 * Sets the handle for the canceller to this pending checkoint.
+	 * Sets the handle for the canceller to this pending checkpoint. This method fails
+	 * with an exception if a handle has already been set.
 	 * 
 	 * @return true, if the handle was set, false, if the checkpoint is already disposed;
 	 */
@@ -422,15 +436,7 @@ public TaskAcknowledgeResult acknowledgeTask(
 		}
 	}
 
-	/**
-	 * Result of the {@link PendingCheckpoint#acknowledgedTasks} method.
-	 */
-	public enum TaskAcknowledgeResult {
-		SUCCESS, // successful acknowledge of the task
-		DUPLICATE, // acknowledge message is a duplicate
-		UNKNOWN, // unknown task acknowledged
-		DISCARDED // pending checkpoint has been discarded
-	}
+	
 
 	// ------------------------------------------------------------------------
 	//  Cancellation
@@ -546,7 +552,9 @@ private void reportFailedCheckpoint(Exception cause) {
 		}
 	}
 
-	// --------------------------------------------------------------------------------------------
+	// ------------------------------------------------------------------------
+	//  Utilities
+	// ------------------------------------------------------------------------
 
 	@Override
 	public String toString() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
index a7539d1e343de..8b82d841ed998 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
@@ -83,7 +83,7 @@ public class ContinuousFileMonitoringFunction<OUT>
 	private final int readerParallelism;
 
 	/** The {@link FileInputFormat} to be read. */
-	private FileInputFormat<OUT> format;
+	private final FileInputFormat<OUT> format;
 
 	/** The interval between consecutive path scans. */
 	private final long interval;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
index 1a6fa8f9bdeb5..3676953a4b76e 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
@@ -168,8 +168,10 @@ public void run(SourceContext<Tuple2<Long, Integer>> ctx) {
 				// simulate some work
 				try {
 					Thread.sleep(readDelay);
-				} catch (InterruptedException e) {
-					e.printStackTrace();
+				}
+				catch (InterruptedException e) {
+					// ignore and reset interruption state
+					Thread.currentThread().interrupt();
 				}
 
 				synchronized (lockObject) {

From 9e12dad676f0992ece881fb4b55be60ef6426685 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Mon, 27 Mar 2017 18:23:13 +0200
Subject: [PATCH 013/253] [hotfix] [checkpoints] Rename JobSnapshottingSettings
 to JobCheckpointingSettings

Cleanup to consistently use
  - checkpoint for the overall fault tolerance mechanism and procedure
  - snapshot for an operators state snapshot that is part of a checkpoint
---
 .../jobmanager/JMXJobManagerMetricTest.java      |  4 ++--
 .../checkpoints/CheckpointConfigHandler.java     |  4 ++--
 .../checkpoints/CheckpointConfigHandlerTest.java | 14 +++++++-------
 .../checkpoint/CheckpointStatsTracker.java       | 14 +++++++-------
 .../executiongraph/AccessExecutionGraph.java     |  6 +++---
 .../executiongraph/ArchivedExecutionGraph.java   | 13 ++++++-------
 .../runtime/executiongraph/ExecutionGraph.java   |  7 +++----
 .../executiongraph/ExecutionGraphBuilder.java    |  4 ++--
 .../apache/flink/runtime/jobgraph/JobGraph.java  | 12 ++++++------
 ...ttings.java => JobCheckpointingSettings.java} |  4 ++--
 .../checkpoint/CheckpointStatsTrackerTest.java   | 16 ++++++++--------
 .../checkpoint/CoordinatorShutdownTest.java      |  6 +++---
 .../ArchivedExecutionGraphTest.java              |  4 ++--
 .../ExecutionGraphDeploymentTest.java            |  4 ++--
 ...st.java => JobCheckpointingSettingsTest.java} |  6 +++---
 .../jobmanager/JobManagerHARecoveryTest.java     |  4 ++--
 .../flink/runtime/jobmanager/JobManagerTest.java | 12 ++++++------
 .../flink/runtime/jobmanager/JobSubmitTest.java  |  4 ++--
 .../runtime/jobmanager/JobManagerITCase.scala    |  8 ++++----
 .../api/graph/StreamingJobGraphGenerator.java    |  4 ++--
 .../graph/StreamingJobGraphGeneratorTest.java    |  4 ++--
 21 files changed, 76 insertions(+), 78 deletions(-)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/{JobSnapshottingSettings.java => JobCheckpointingSettings.java} (97%)
 rename flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/tasks/{JobSnapshottingSettingsTest.java => JobCheckpointingSettingsTest.java} (93%)

diff --git a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java
index 1fdac65ee4073..934a6212019ac 100644
--- a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java
+++ b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java
@@ -25,7 +25,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.testingUtils.TestingCluster;
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
 import org.junit.Assert;
@@ -70,7 +70,7 @@ public void testJobManagerJMXMetricAccess() throws Exception {
 			sourceJobVertex.setInvokableClass(BlockingInvokable.class);
 
 			JobGraph jobGraph = new JobGraph("TestingJob", sourceJobVertex);
-			jobGraph.setSnapshotSettings(new JobSnapshottingSettings(
+			jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
 				Collections.<JobVertexID>emptyList(),
 				Collections.<JobVertexID>emptyList(),
 				Collections.<JobVertexID>emptyList(),
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandler.java
index 947b7c3344897..7914c2970c07a 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandler.java
@@ -21,7 +21,7 @@
 import com.fasterxml.jackson.core.JsonGenerator;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
 import org.apache.flink.runtime.webmonitor.handlers.AbstractExecutionGraphRequestHandler;
 import org.apache.flink.runtime.webmonitor.handlers.JsonFactory;
@@ -69,7 +69,7 @@ public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph)
 	private static String createCheckpointConfigJson(AccessExecutionGraph graph) throws IOException {
 		StringWriter writer = new StringWriter();
 		JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
-		JobSnapshottingSettings settings = graph.getJobSnapshottingSettings();
+		JobCheckpointingSettings settings = graph.getJobCheckpointingSettings();
 
 		if (settings == null) {
 			return "{}";
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandlerTest.java
index 9d339f587fcc5..6e48973bc22c2 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandlerTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/checkpoints/CheckpointConfigHandlerTest.java
@@ -24,7 +24,7 @@
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
 import org.apache.flink.runtime.webmonitor.history.ArchivedJson;
 import org.apache.flink.runtime.webmonitor.history.JsonArchivist;
@@ -50,7 +50,7 @@ public void testArchiver() throws IOException {
 
 		AccessExecutionGraph graph = graphAndSettings.graph;
 		when(graph.getJobID()).thenReturn(new JobID());
-		JobSnapshottingSettings settings = graphAndSettings.snapshottingSettings;
+		JobCheckpointingSettings settings = graphAndSettings.snapshottingSettings;
 		ExternalizedCheckpointSettings externalizedSettings = graphAndSettings.externalizedSettings;
 		
 		Collection<ArchivedJson> archives = archivist.archiveJsonWithPath(graph);
@@ -90,7 +90,7 @@ public void testSimpleConfig() throws Exception {
 		GraphAndSettings graphAndSettings = createGraphAndSettings(false, true);
 
 		AccessExecutionGraph graph = graphAndSettings.graph;
-		JobSnapshottingSettings settings = graphAndSettings.snapshottingSettings;
+		JobCheckpointingSettings settings = graphAndSettings.snapshottingSettings;
 
 		CheckpointConfigHandler handler = new CheckpointConfigHandler(mock(ExecutionGraphHolder.class));
 		String json = handler.handleRequest(graph, Collections.<String, String>emptyMap());
@@ -156,7 +156,7 @@ private static GraphAndSettings createGraphAndSettings(boolean externalized, boo
 			? ExternalizedCheckpointSettings.externalizeCheckpoints(true)
 			: ExternalizedCheckpointSettings.none();
 
-		JobSnapshottingSettings settings = new JobSnapshottingSettings(
+		JobCheckpointingSettings settings = new JobCheckpointingSettings(
 			Collections.<JobVertexID>emptyList(),
 			Collections.<JobVertexID>emptyList(),
 			Collections.<JobVertexID>emptyList(),
@@ -169,19 +169,19 @@ private static GraphAndSettings createGraphAndSettings(boolean externalized, boo
 			exactlyOnce);
 
 		AccessExecutionGraph graph = mock(AccessExecutionGraph.class);
-		when(graph.getJobSnapshottingSettings()).thenReturn(settings);
+		when(graph.getJobCheckpointingSettings()).thenReturn(settings);
 
 		return new GraphAndSettings(graph, settings, externalizedSetting);
 	}
 
 	private static class GraphAndSettings {
 		public final AccessExecutionGraph graph;
-		public final JobSnapshottingSettings snapshottingSettings;
+		public final JobCheckpointingSettings snapshottingSettings;
 		public final ExternalizedCheckpointSettings externalizedSettings;
 
 		public GraphAndSettings(
 				AccessExecutionGraph graph,
-				JobSnapshottingSettings snapshottingSettings,
+				JobCheckpointingSettings snapshottingSettings,
 				ExternalizedCheckpointSettings externalizedSettings) {
 			this.graph = graph;
 			this.snapshottingSettings = snapshottingSettings;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java
index c7efb7b88230d..313fe13db1047 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java
@@ -31,7 +31,7 @@
 import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 
 /**
  * Tracker for checkpoint statistics.
@@ -67,7 +67,7 @@ public class CheckpointStatsTracker {
 	private final int totalSubtaskCount;
 
 	/** Snapshotting settings created from the CheckpointConfig. */
-	private final JobSnapshottingSettings jobSnapshottingSettings;
+	private final JobCheckpointingSettings jobCheckpointingSettings;
 
 	/** Checkpoint counts. */
 	private final CheckpointStatsCounts counts = new CheckpointStatsCounts();
@@ -104,19 +104,19 @@ public class CheckpointStatsTracker {
 	 *
 	 * @param numRememberedCheckpoints Maximum number of checkpoints to remember, including in progress ones.
 	 * @param jobVertices Job vertices involved in the checkpoints.
-	 * @param jobSnapshottingSettings Snapshotting settings created from the CheckpointConfig.
+	 * @param jobCheckpointingSettings Snapshotting settings created from the CheckpointConfig.
 	 * @param metricGroup Metric group for exposed metrics
 	 */
 	public CheckpointStatsTracker(
 		int numRememberedCheckpoints,
 		List<ExecutionJobVertex> jobVertices,
-		JobSnapshottingSettings jobSnapshottingSettings,
+		JobCheckpointingSettings jobCheckpointingSettings,
 		MetricGroup metricGroup) {
 
 		checkArgument(numRememberedCheckpoints >= 0, "Negative number of remembered checkpoints");
 		this.history = new CheckpointStatsHistory(numRememberedCheckpoints);
 		this.jobVertices = checkNotNull(jobVertices, "JobVertices");
-		this.jobSnapshottingSettings = checkNotNull(jobSnapshottingSettings);
+		this.jobCheckpointingSettings = checkNotNull(jobCheckpointingSettings);
 
 		// Compute the total subtask count. We do this here in order to only
 		// do it once.
@@ -143,8 +143,8 @@ public CheckpointStatsTracker(
 	 *
 	 * @return The job's snapshotting settings.
 	 */
-	public JobSnapshottingSettings getSnapshottingSettings() {
-		return jobSnapshottingSettings;
+	public JobCheckpointingSettings getSnapshottingSettings() {
+		return jobCheckpointingSettings;
 	}
 
 	/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
index 18c2ec2f40b53..3b064c352b031 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
@@ -24,7 +24,7 @@
 import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.util.SerializedValue;
 
 import java.io.IOException;
@@ -116,12 +116,12 @@ public interface AccessExecutionGraph {
 	CheckpointCoordinator getCheckpointCoordinator();
 
 	/**
-	 * Returns the {@link JobSnapshottingSettings} or <code>null</code> if
+	 * Returns the {@link JobCheckpointingSettings} or <code>null</code> if
 	 * checkpointing is disabled.
 	 *
 	 * @return JobSnapshottingSettings for this execution graph
 	 */
-	JobSnapshottingSettings getJobSnapshottingSettings();
+	JobCheckpointingSettings getJobCheckpointingSettings();
 
 	/**
 	 * Returns a snapshot of the checkpoint statistics or <code>null</code> if
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
index 334b0d004545d..b9db1e7bd70ea 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
@@ -24,7 +24,7 @@
 import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.util.SerializedValue;
 
 import javax.annotation.Nullable;
@@ -81,7 +81,7 @@ public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializabl
 	private final Map<String, SerializedValue<Object>> serializedUserAccumulators;
 
 	@Nullable
-	private final JobSnapshottingSettings jobSnapshottingSettings;
+	private final JobCheckpointingSettings jobCheckpointingSettings;
 
 	@Nullable
 	private final CheckpointStatsSnapshot checkpointStatsSnapshot;
@@ -99,7 +99,7 @@ public ArchivedExecutionGraph(
 			Map<String, SerializedValue<Object>> serializedUserAccumulators,
 			ArchivedExecutionConfig executionConfig,
 			boolean isStoppable,
-			@Nullable JobSnapshottingSettings jobSnapshottingSettings,
+			@Nullable JobCheckpointingSettings jobCheckpointingSettings,
 			@Nullable CheckpointStatsSnapshot checkpointStatsSnapshot) {
 
 		this.jobID = jobID;
@@ -114,7 +114,7 @@ public ArchivedExecutionGraph(
 		this.serializedUserAccumulators = serializedUserAccumulators;
 		this.archivedExecutionConfig = executionConfig;
 		this.isStoppable = isStoppable;
-		this.jobSnapshottingSettings = jobSnapshottingSettings;
+		this.jobCheckpointingSettings = jobCheckpointingSettings;
 		this.checkpointStatsSnapshot = checkpointStatsSnapshot;
 	}
 
@@ -210,9 +210,8 @@ public CheckpointCoordinator getCheckpointCoordinator() {
 		return null;
 	}
 
-	@Override
-	public JobSnapshottingSettings getJobSnapshottingSettings() {
-		return jobSnapshottingSettings;
+	public JobCheckpointingSettings getJobCheckpointingSettings() {
+		return jobCheckpointingSettings;
 	}
 
 	@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index b21b72b53b33a..29b980692b7ac 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -55,7 +55,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.ScheduleMode;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
 import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
 import org.apache.flink.runtime.query.KvStateLocationRegistry;
@@ -419,8 +419,7 @@ public RestartStrategy getRestartStrategy() {
 		return restartStrategy;
 	}
 
-	@Override
-	public JobSnapshottingSettings getJobSnapshottingSettings() {
+	public JobCheckpointingSettings getJobCheckpointingSettings() {
 		if (checkpointStatsTracker != null) {
 			return checkpointStatsTracker.getSnapshottingSettings();
 		} else {
@@ -1477,7 +1476,7 @@ public ArchivedExecutionGraph archive() {
 			serializedUserAccumulators,
 			getArchivedExecutionConfig(),
 			isStoppable(),
-			getJobSnapshottingSettings(),
+			getJobCheckpointingSettings(),
 			getCheckpointStatsSnapshot());
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
index a6455f55994eb..a10c62e31882c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
@@ -42,7 +42,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.StateBackend;
 import org.apache.flink.util.DynamicCodeLoadingException;
@@ -159,7 +159,7 @@ public static ExecutionGraph buildGraph(
 		}
 
 		// configure the state checkpointing
-		JobSnapshottingSettings snapshotSettings = jobGraph.getSnapshotSettings();
+		JobCheckpointingSettings snapshotSettings = jobGraph.getCheckpointingSettings();
 		if (snapshotSettings != null) {
 			List<ExecutionJobVertex> triggerVertices = 
 					idToVertex(snapshotSettings.getVerticesToTrigger(), executionGraph);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
index f6377e5197837..2a8af3735851f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
@@ -28,7 +28,7 @@
 import org.apache.flink.runtime.blob.BlobClient;
 import org.apache.flink.runtime.blob.BlobKey;
 import org.apache.flink.runtime.instance.ActorGateway;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.util.SerializedValue;
 import scala.concurrent.duration.FiniteDuration;
 
@@ -92,7 +92,7 @@ public class JobGraph implements Serializable {
 	private SerializedValue<ExecutionConfig> serializedExecutionConfig;
 
 	/** The settings for the job checkpoints */
-	private JobSnapshottingSettings snapshotSettings;
+	private JobCheckpointingSettings snapshotSettings;
 
 	/** Savepoint restore settings. */
 	private SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.none();
@@ -331,17 +331,17 @@ public int getNumberOfVertices() {
 	 *
 	 * @param settings The snapshot settings, or null, to disable snapshotting.
 	 */
-	public void setSnapshotSettings(JobSnapshottingSettings settings) {
+	public void setSnapshotSettings(JobCheckpointingSettings settings) {
 		this.snapshotSettings = settings;
 	}
 
 	/**
 	 * Gets the settings for asynchronous snapshots. This method returns null, when
-	 * snapshotting is not enabled.
+	 * checkpointing is not enabled.
 	 *
-	 * @return The snapshot settings, or null, if snapshotting is not enabled.
+	 * @return The snapshot settings, or null, if checkpointing is not enabled.
 	 */
-	public JobSnapshottingSettings getSnapshotSettings() {
+	public JobCheckpointingSettings getCheckpointingSettings() {
 		return snapshotSettings;
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettings.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
similarity index 97%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettings.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
index 233aa8887de5f..38130d4c9978a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettings.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
@@ -31,7 +31,7 @@
  * for the asynchronous checkpoints of the JobGraph, such as interval, and which vertices
  * need to participate.
  */
-public class JobSnapshottingSettings implements java.io.Serializable {
+public class JobCheckpointingSettings implements java.io.Serializable {
 	
 	private static final long serialVersionUID = -2593319571078198180L;
 	
@@ -65,7 +65,7 @@ public class JobSnapshottingSettings implements java.io.Serializable {
 	 */
 	private final boolean isExactlyOnce;
 
-	public JobSnapshottingSettings(
+	public JobCheckpointingSettings(
 			List<JobVertexID> verticesToTrigger,
 			List<JobVertexID> verticesToAcknowledge,
 			List<JobVertexID> verticesToConfirm,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
index aaf1774bb6dd3..d66d0be1bb2f3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
@@ -44,7 +44,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.junit.Test;
 
 public class CheckpointStatsTrackerTest {
@@ -58,7 +58,7 @@ public void testGetSnapshottingSettings() throws Exception {
 		when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
 		when(jobVertex.getParallelism()).thenReturn(1);
 
-		JobSnapshottingSettings snapshottingSettings = new JobSnapshottingSettings(
+		JobCheckpointingSettings snapshottingSettings = new JobCheckpointingSettings(
 			Collections.singletonList(new JobVertexID()),
 			Collections.singletonList(new JobVertexID()),
 			Collections.singletonList(new JobVertexID()),
@@ -94,7 +94,7 @@ public void testTrackerWithoutHistory() throws Exception {
 		CheckpointStatsTracker tracker = new CheckpointStatsTracker(
 			0,
 			Collections.singletonList(jobVertex),
-			mock(JobSnapshottingSettings.class),
+			mock(JobCheckpointingSettings.class),
 			new UnregisteredMetricsGroup());
 
 		PendingCheckpointStats pending = tracker.reportPendingCheckpoint(
@@ -142,7 +142,7 @@ public void testCheckpointTracking() throws Exception {
 		CheckpointStatsTracker tracker = new CheckpointStatsTracker(
 			10,
 			Collections.singletonList(jobVertex),
-			mock(JobSnapshottingSettings.class),
+			mock(JobCheckpointingSettings.class),
 			new UnregisteredMetricsGroup());
 
 		// Completed checkpoint
@@ -247,7 +247,7 @@ public void testCreateSnapshot() throws Exception {
 		CheckpointStatsTracker tracker = new CheckpointStatsTracker(
 			10,
 			Collections.singletonList(jobVertex),
-			mock(JobSnapshottingSettings.class),
+			mock(JobCheckpointingSettings.class),
 			new UnregisteredMetricsGroup());
 
 		CheckpointStatsSnapshot snapshot1 = tracker.createSnapshot();
@@ -293,7 +293,7 @@ public void testMetricsRegistration() throws Exception {
 		new CheckpointStatsTracker(
 			0,
 			Collections.singletonList(jobVertex),
-			mock(JobSnapshottingSettings.class),
+			mock(JobCheckpointingSettings.class),
 			metricGroup);
 
 		verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_CHECKPOINTS_METRIC), any(Gauge.class));
@@ -409,7 +409,7 @@ public String getMetricIdentifier(String metricName, CharacterFilter filter) {
 		CheckpointStatsTracker stats = new CheckpointStatsTracker(
 			0,
 			Collections.singletonList(jobVertex),
-			mock(JobSnapshottingSettings.class),
+			mock(JobCheckpointingSettings.class),
 			metricGroup);
 
 		// Make sure to adjust this test if metrics are added/removed
@@ -522,7 +522,7 @@ static CheckpointStatsTracker createTestTracker() {
 		return new CheckpointStatsTracker(
 			0,
 			Collections.singletonList(jobVertex),
-			mock(JobSnapshottingSettings.class),
+			mock(JobCheckpointingSettings.class),
 			new UnregisteredMetricsGroup());
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
index 976da48b2b94b..2e0bd762d3035 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
@@ -28,7 +28,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
@@ -65,7 +65,7 @@ public void testCoordinatorShutsDownOnFailure() {
 			List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
 			
 			JobGraph testGraph = new JobGraph("test job", vertex);
-			testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 
+			testGraph.setSnapshotSettings(new JobCheckpointingSettings(vertexIdList, vertexIdList, vertexIdList, 
 					5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), null, true));
 			
 			ActorGateway jmGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
@@ -124,7 +124,7 @@ public void testCoordinatorShutsDownOnSuccess() {
 			List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
 
 			JobGraph testGraph = new JobGraph("test job", vertex);
-			testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList,
+			testGraph.setSnapshotSettings(new JobCheckpointingSettings(vertexIdList, vertexIdList, vertexIdList,
 					5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), null, true));
 			
 			ActorGateway jmGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
index 077ab53ac1647..f96b62456dcf8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
@@ -41,7 +41,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.SerializedValue;
 import org.junit.BeforeClass;
@@ -116,7 +116,7 @@ public static void setupExecutionGraph() throws Exception {
 		CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
 				0,
 				jobVertices,
-				mock(JobSnapshottingSettings.class),
+				mock(JobCheckpointingSettings.class),
 				new UnregisteredMetricsGroup());
 
 		runtimeGraph.enableCheckpointing(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
index 8d91b844c1a03..866f55c5317a4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
@@ -61,7 +61,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
 import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
 import org.apache.flink.runtime.operators.BatchTask;
@@ -559,7 +559,7 @@ private ExecutionGraph createExecutionGraph(Configuration configuration) throws
 
 		final JobID jobId = new JobID();
 		final JobGraph jobGraph = new JobGraph(jobId, "test");
-		jobGraph.setSnapshotSettings(new JobSnapshottingSettings(
+		jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
 			Collections.<JobVertexID>emptyList(),
 			Collections.<JobVertexID>emptyList(),
 			Collections.<JobVertexID>emptyList(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettingsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettingsTest.java
similarity index 93%
rename from flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettingsTest.java
rename to flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettingsTest.java
index 2508d5ce62a7f..c3524fa2ab247 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettingsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettingsTest.java
@@ -29,14 +29,14 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
-public class JobSnapshottingSettingsTest {
+public class JobCheckpointingSettingsTest {
 
 	/**
 	 * Tests that the settings are actually serializable.
 	 */
 	@Test
 	public void testIsJavaSerializable() throws Exception {
-		JobSnapshottingSettings settings = new JobSnapshottingSettings(
+		JobCheckpointingSettings settings = new JobCheckpointingSettings(
 			Arrays.asList(new JobVertexID(), new JobVertexID()),
 			Arrays.asList(new JobVertexID(), new JobVertexID()),
 			Arrays.asList(new JobVertexID(), new JobVertexID()),
@@ -48,7 +48,7 @@ public void testIsJavaSerializable() throws Exception {
 			new MemoryStateBackend(),
 			false);
 
-		JobSnapshottingSettings copy = CommonTestUtils.createCopySerializable(settings);
+		JobCheckpointingSettings copy = CommonTestUtils.createCopySerializable(settings);
 		assertEquals(settings.getVerticesToAcknowledge(), copy.getVerticesToAcknowledge());
 		assertEquals(settings.getVerticesToConfirm(), copy.getVerticesToConfirm());
 		assertEquals(settings.getVerticesToTrigger(), copy.getVerticesToTrigger());
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
index dcf4722a7313e..6eacaac6d8044 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
@@ -61,7 +61,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
@@ -220,7 +220,7 @@ public void testJobRecoveryWhenLosingLeadership() throws Exception {
 			JobGraph jobGraph = new JobGraph("TestingJob", sourceJobVertex);
 
 			List<JobVertexID> vertexId = Collections.singletonList(sourceJobVertex.getID());
-			jobGraph.setSnapshotSettings(new JobSnapshottingSettings(
+			jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
 					vertexId,
 					vertexId,
 					vertexId,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
index 4dec84bab6777..d7fc71dabb766 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
@@ -48,7 +48,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.jobmanager.JobManagerHARecoveryTest.BlockingStatefulInvokable;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
@@ -821,7 +821,7 @@ public void testCancelWithSavepoint() throws Exception {
 
 			JobGraph jobGraph = new JobGraph("TestingJob", sourceVertex);
 
-			JobSnapshottingSettings snapshottingSettings = new JobSnapshottingSettings(
+			JobCheckpointingSettings snapshottingSettings = new JobCheckpointingSettings(
 					Collections.singletonList(sourceVertex.getID()),
 					Collections.singletonList(sourceVertex.getID()),
 					Collections.singletonList(sourceVertex.getID()),
@@ -947,7 +947,7 @@ public void testCancelWithSavepointNoDirectoriesConfigured() throws Exception {
 
 			JobGraph jobGraph = new JobGraph("TestingJob", sourceVertex);
 
-			JobSnapshottingSettings snapshottingSettings = new JobSnapshottingSettings(
+			JobCheckpointingSettings snapshottingSettings = new JobCheckpointingSettings(
 				Collections.singletonList(sourceVertex.getID()),
 				Collections.singletonList(sourceVertex.getID()),
 				Collections.singletonList(sourceVertex.getID()),
@@ -1053,7 +1053,7 @@ public void testSavepointWithDeactivatedPeriodicCheckpointing() throws Exception
 
 			JobGraph jobGraph = new JobGraph("TestingJob", sourceVertex);
 
-			JobSnapshottingSettings snapshottingSettings = new JobSnapshottingSettings(
+			JobCheckpointingSettings snapshottingSettings = new JobCheckpointingSettings(
 					Collections.singletonList(sourceVertex.getID()),
 					Collections.singletonList(sourceVertex.getID()),
 					Collections.singletonList(sourceVertex.getID()),
@@ -1156,7 +1156,7 @@ public void testSavepointRestoreSettings() throws Exception {
 
 			JobGraph jobGraph = new JobGraph("TestingJob", sourceVertex);
 
-			JobSnapshottingSettings snapshottingSettings = new JobSnapshottingSettings(
+			JobCheckpointingSettings snapshottingSettings = new JobCheckpointingSettings(
 					Collections.singletonList(sourceVertex.getID()),
 					Collections.singletonList(sourceVertex.getID()),
 					Collections.singletonList(sourceVertex.getID()),
@@ -1203,7 +1203,7 @@ public void testSavepointRestoreSettings() throws Exception {
 
 			JobGraph newJobGraph = new JobGraph("NewTestingJob", newSourceVertex);
 
-			JobSnapshottingSettings newSnapshottingSettings = new JobSnapshottingSettings(
+			JobCheckpointingSettings newSnapshottingSettings = new JobCheckpointingSettings(
 					Collections.singletonList(newSourceVertex.getID()),
 					Collections.singletonList(newSourceVertex.getID()),
 					Collections.singletonList(newSourceVertex.getID()),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
index ba5f973b6e900..bdff4019ad970 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
@@ -32,7 +32,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
@@ -228,7 +228,7 @@ private JobGraph createSimpleJobGraph() {
 		List<JobVertexID> vertexIdList = Collections.singletonList(jobVertex.getID());
 
 		JobGraph jg = new JobGraph("test job", jobVertex);
-		jg.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList,
+		jg.setSnapshotSettings(new JobCheckpointingSettings(vertexIdList, vertexIdList, vertexIdList,
 			5000, 5000, 0L, 10, ExternalizedCheckpointSettings.none(), null, true));
 		return jg;
 	}
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
index 5374d01caff21..ce8517ef4ebd8 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
@@ -27,7 +27,7 @@ import org.apache.flink.runtime.checkpoint.{CheckpointCoordinator, CompletedChec
 import org.apache.flink.runtime.client.JobExecutionException
 import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType
-import org.apache.flink.runtime.jobgraph.tasks.{ExternalizedCheckpointSettings, JobSnapshottingSettings}
+import org.apache.flink.runtime.jobgraph.tasks.{ExternalizedCheckpointSettings, JobCheckpointingSettings}
 import org.apache.flink.runtime.jobgraph.{DistributionPattern, JobGraph, JobVertex, ScheduleMode}
 import org.apache.flink.runtime.jobmanager.Tasks._
 import org.apache.flink.runtime.jobmanager.scheduler.{NoResourceAvailableException, SlotSharingGroup}
@@ -827,7 +827,7 @@ class JobManagerITCase(_system: ActorSystem)
           val jobVertex = new JobVertex("Blocking vertex")
           jobVertex.setInvokableClass(classOf[BlockingNoOpInvokable])
           val jobGraph = new JobGraph(jobVertex)
-          jobGraph.setSnapshotSettings(new JobSnapshottingSettings(
+          jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
             java.util.Collections.emptyList(),
             java.util.Collections.emptyList(),
             java.util.Collections.emptyList(),
@@ -887,7 +887,7 @@ class JobManagerITCase(_system: ActorSystem)
           val jobVertex = new JobVertex("Blocking vertex")
           jobVertex.setInvokableClass(classOf[BlockingNoOpInvokable])
           val jobGraph = new JobGraph(jobVertex)
-          jobGraph.setSnapshotSettings(new JobSnapshottingSettings(
+          jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
             java.util.Collections.emptyList(),
             java.util.Collections.emptyList(),
             java.util.Collections.emptyList(),
@@ -955,7 +955,7 @@ class JobManagerITCase(_system: ActorSystem)
           val jobVertex = new JobVertex("Blocking vertex")
           jobVertex.setInvokableClass(classOf[BlockingNoOpInvokable])
           val jobGraph = new JobGraph(jobVertex)
-          jobGraph.setSnapshotSettings(new JobSnapshottingSettings(
+          jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
             java.util.Collections.emptyList(),
             java.util.Collections.emptyList(),
             java.util.Collections.emptyList(),
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 794de5a84e1d2..7d6227358a7c8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -36,7 +36,7 @@
 import org.apache.flink.runtime.jobgraph.ScheduleMode;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
 import org.apache.flink.runtime.operators.util.TaskConfig;
@@ -579,7 +579,7 @@ private void configureCheckpointing() {
 				"exactly-once or at-least-once.");
 		}
 
-		JobSnapshottingSettings settings = new JobSnapshottingSettings(
+		JobCheckpointingSettings settings = new JobCheckpointingSettings(
 				triggerVertices, ackVertices, commitVertices, interval,
 				cfg.getCheckpointTimeout(), cfg.getMinPauseBetweenCheckpoints(),
 				cfg.getMaxConcurrentCheckpoints(),
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
index 6d2fcaace12a8..2c71a07bb1e02 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
@@ -26,7 +26,7 @@
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.DataStreamSink;
 import org.apache.flink.streaming.api.datastream.IterativeStream;
@@ -118,7 +118,7 @@ public void testDisabledCheckpointing() throws Exception {
 		StreamingJobGraphGenerator jobGraphGenerator = new StreamingJobGraphGenerator(streamGraph);
 		JobGraph jobGraph = jobGraphGenerator.createJobGraph();
 
-		JobSnapshottingSettings snapshottingSettings = jobGraph.getSnapshotSettings();
+		JobCheckpointingSettings snapshottingSettings = jobGraph.getCheckpointingSettings();
 		assertEquals(Long.MAX_VALUE, snapshottingSettings.getCheckpointInterval());
 	}
 

From 617241790024cd60c96542dba3e7682ad7ab03e1 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Mon, 27 Mar 2017 19:03:35 +0200
Subject: [PATCH 014/253] [hotfix] [streaming api] Make it clear that
 StreamingJobGraphGenerator is single use.

This reduces the public API of the StreamingJobGraphGenerator to a static createJobGraph() method,
which internally instantiates a generator, creates the job, and drops the generator.
---
 .../streaming/api/graph/StreamGraph.java      |  4 +-
 .../api/graph/StreamingJobGraphGenerator.java | 47 +++++++++++--------
 .../graph/StreamingJobGraphGeneratorTest.java | 13 ++---
 3 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
index cc3ca9ec18b5e..1acfc5b7c646b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
@@ -655,9 +655,7 @@ public JobGraph getJobGraph() {
 							+ "\nThe user can force enable state checkpoints with the reduced guarantees by calling: env.enableCheckpointing(interval,true)");
 		}
 
-		StreamingJobGraphGenerator jobgraphGenerator = new StreamingJobGraphGenerator(this);
-
-		return jobgraphGenerator.createJobGraph();
+		return StreamingJobGraphGenerator.createJobGraph(this);
 	}
 
 	@Override
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 7d6227358a7c8..9eca9a89705e0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -18,6 +18,7 @@
 package org.apache.flink.streaming.api.graph;
 
 import org.apache.commons.lang3.StringUtils;
+
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.operators.ResourceSpec;
 import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
@@ -49,6 +50,7 @@
 import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
 import org.apache.flink.streaming.runtime.tasks.StreamIterationHead;
 import org.apache.flink.streaming.runtime.tasks.StreamIterationTail;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -62,6 +64,9 @@
 import java.util.Map;
 import java.util.Map.Entry;
 
+/**
+ * The StreamingJobGraphGenerator converts a {@link StreamGraph} into a {@link JobGraph}.
+ */
 @Internal
 public class StreamingJobGraphGenerator {
 
@@ -73,32 +78,38 @@ public class StreamingJobGraphGenerator {
 	 */
 	private static final long DEFAULT_RESTART_DELAY = 10000L;
 
-	private StreamGraph streamGraph;
+	// ------------------------------------------------------------------------
+
+	public static JobGraph createJobGraph(StreamGraph streamGraph) {
+		return new StreamingJobGraphGenerator(streamGraph).createJobGraph();
+	}
+
+	// ------------------------------------------------------------------------
+
+	private final StreamGraph streamGraph;
 
-	private Map<Integer, JobVertex> jobVertices;
-	private JobGraph jobGraph;
-	private Collection<Integer> builtVertices;
+	private final Map<Integer, JobVertex> jobVertices;
+	private final JobGraph jobGraph;
+	private final Collection<Integer> builtVertices;
 
-	private List<StreamEdge> physicalEdgesInOrder;
+	private final List<StreamEdge> physicalEdgesInOrder;
 
-	private Map<Integer, Map<Integer, StreamConfig>> chainedConfigs;
+	private final Map<Integer, Map<Integer, StreamConfig>> chainedConfigs;
 
-	private Map<Integer, StreamConfig> vertexConfigs;
-	private Map<Integer, String> chainedNames;
+	private final Map<Integer, StreamConfig> vertexConfigs;
+	private final Map<Integer, String> chainedNames;
 
-	private Map<Integer, ResourceSpec> chainedMinResources;
-	private Map<Integer, ResourceSpec> chainedPreferredResources;
+	private final Map<Integer, ResourceSpec> chainedMinResources;
+	private final Map<Integer, ResourceSpec> chainedPreferredResources;
 
 	private final StreamGraphHasher defaultStreamGraphHasher;
 	private final List<StreamGraphHasher> legacyStreamGraphHashers;
 
-	public StreamingJobGraphGenerator(StreamGraph streamGraph) {
+	private StreamingJobGraphGenerator(StreamGraph streamGraph) {
 		this.streamGraph = streamGraph;
 		this.defaultStreamGraphHasher = new StreamGraphHasherV2();
 		this.legacyStreamGraphHashers = Arrays.asList(new StreamGraphHasherV1(), new StreamGraphUserHashHasher());
-	}
 
-	private void init() {
 		this.jobVertices = new HashMap<>();
 		this.builtVertices = new HashSet<>();
 		this.chainedConfigs = new HashMap<>();
@@ -107,17 +118,15 @@ private void init() {
 		this.chainedMinResources = new HashMap<>();
 		this.chainedPreferredResources = new HashMap<>();
 		this.physicalEdgesInOrder = new ArrayList<>();
-	}
-
-	public JobGraph createJobGraph() {
 
 		jobGraph = new JobGraph(streamGraph.getJobName());
+	}
+
+	private JobGraph createJobGraph() {
 
 		// make sure that all vertices start immediately
 		jobGraph.setScheduleMode(ScheduleMode.EAGER);
 
-		init();
-
 		// Generate deterministic hashes for the nodes in order to identify them across
 		// submission iff they didn't change.
 		Map<Integer, byte[]> hashes = defaultStreamGraphHasher.traverseStreamGraphAndGenerateHashes(streamGraph);
@@ -442,7 +451,7 @@ private void connect(Integer headOfChain, StreamEdge edge) {
 		downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1);
 
 		StreamPartitioner<?> partitioner = edge.getPartitioner();
-		JobEdge jobEdge = null;
+		JobEdge jobEdge;
 		if (partitioner instanceof ForwardPartitioner) {
 			jobEdge = downStreamVertex.connectNewDataSetAsInput(
 				headVertex,
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
index 2c71a07bb1e02..8b5aef59d03e8 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
@@ -61,7 +61,6 @@ public void testParallelismOneNotChained() {
 		DataStream<Tuple2<String, String>> input = env
 				.fromElements("a", "b", "c", "d", "e", "f")
 				.map(new MapFunction<String, Tuple2<String, String>>() {
-					private static final long serialVersionUID = 471891682418382583L;
 
 					@Override
 					public Tuple2<String, String> map(String value) {
@@ -73,8 +72,6 @@ public Tuple2<String, String> map(String value) {
 				.keyBy(0)
 				.map(new MapFunction<Tuple2<String, String>, Tuple2<String, String>>() {
 
-					private static final long serialVersionUID = 3583760206245136188L;
-
 					@Override
 					public Tuple2<String, String> map(Tuple2<String, String> value) {
 						return value;
@@ -82,7 +79,6 @@ public Tuple2<String, String> map(Tuple2<String, String> value) {
 				});
 
 		result.addSink(new SinkFunction<Tuple2<String, String>>() {
-			private static final long serialVersionUID = -5614849094269539342L;
 
 			@Override
 			public void invoke(Tuple2<String, String> value) {}
@@ -115,8 +111,7 @@ public void testDisabledCheckpointing() throws Exception {
 		StreamGraph streamGraph = new StreamGraph(env);
 		assertFalse("Checkpointing enabled", streamGraph.getCheckpointConfig().isCheckpointingEnabled());
 
-		StreamingJobGraphGenerator jobGraphGenerator = new StreamingJobGraphGenerator(streamGraph);
-		JobGraph jobGraph = jobGraphGenerator.createJobGraph();
+		JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
 
 		JobCheckpointingSettings snapshottingSettings = jobGraph.getCheckpointingSettings();
 		assertEquals(Long.MAX_VALUE, snapshottingSettings.getCheckpointInterval());
@@ -137,7 +132,7 @@ public Integer map(Integer value) throws Exception {
 				}
 			})
 			.print();
-		JobGraph jobGraph = new StreamingJobGraphGenerator(env.getStreamGraph()).createJobGraph();
+		JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
 
 		List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
 		JobVertex sourceVertex = verticesSorted.get(0);
@@ -224,7 +219,7 @@ public void invoke(Tuple2<Integer, Integer> value) throws Exception {
 		});
 		sinkMethod.invoke(sink, resource5);
 
-		JobGraph jobGraph = new StreamingJobGraphGenerator(env.getStreamGraph()).createJobGraph();
+		JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
 
 		JobVertex sourceMapFilterVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
 		JobVertex reduceSinkVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
@@ -291,7 +286,7 @@ public void invoke(Integer value) throws Exception {
 		}).disableChaining().name("test_sink");
 		sinkMethod.invoke(sink, resource5);
 
-		JobGraph jobGraph = new StreamingJobGraphGenerator(env.getStreamGraph()).createJobGraph();
+		JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
 
 		for (JobVertex jobVertex : jobGraph.getVertices()) {
 			if (jobVertex.getName().contains("test_source")) {

From 4db9d634603c53d0b4e9352d16e50a9c87abfb4b Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 28 Mar 2017 15:02:50 +0200
Subject: [PATCH 015/253] [FLINK-6339] [streaming api] Delete unused (and
 useless) class ConnectorSource

---
 .../api/functions/source/ConnectorSource.java | 40 -------------------
 1 file changed, 40 deletions(-)
 delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java
deleted file mode 100644
index e4e6c685e4a1d..0000000000000
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.functions.source;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-
-@PublicEvolving
-public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements ResultTypeQueryable<OUT> {
-
-	private static final long serialVersionUID = 1L;
-	
-	protected DeserializationSchema<OUT> schema;
-
-	public ConnectorSource(DeserializationSchema<OUT> schema) {
-		this.schema = schema;
-	}
-
-	@Override
-	public TypeInformation<OUT> getProducedType() {
-		return schema.getProducedType();
-	}
-}

From d977455629463b0e45194916eae000444108d94a Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 28 Mar 2017 15:09:02 +0200
Subject: [PATCH 016/253] [FLINK-6338] [core] Rename SimpleStringUtils to
 StringValueUtils

This more intuitivce name clarifies the role of the class.
This also marks the class as PublicEvolving, as it is not actually used as an internal class.
---
 ...StringUtils.java => StringValueUtils.java} | 17 +++++++++-------
 ...ilsTest.java => StringValueUtilsTest.java} | 20 ++++++++++---------
 2 files changed, 21 insertions(+), 16 deletions(-)
 rename flink-core/src/main/java/org/apache/flink/util/{SimpleStringUtils.java => StringValueUtils.java} (89%)
 rename flink-core/src/test/java/org/apache/flink/util/{SimpleStringUtilsTest.java => StringValueUtilsTest.java} (80%)

diff --git a/flink-core/src/main/java/org/apache/flink/util/SimpleStringUtils.java b/flink-core/src/main/java/org/apache/flink/util/StringValueUtils.java
similarity index 89%
rename from flink-core/src/main/java/org/apache/flink/util/SimpleStringUtils.java
rename to flink-core/src/main/java/org/apache/flink/util/StringValueUtils.java
index 43a93ebd233d2..cffe8b3f61883 100644
--- a/flink-core/src/main/java/org/apache/flink/util/SimpleStringUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/StringValueUtils.java
@@ -20,16 +20,19 @@
 
 import java.io.Serializable;
 
-import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.types.StringValue;
 
 /**
- * Utility class for efficient string operations on strings. All methods in this class are
- * written to be optimized for efficiency and work only on strings whose characters are
- * representable in a single <tt>char</tt>, ie. strings without surrogate characters.
+ * Utility class for efficient operations on {@link StringValue}.
+ * 
+ * <p>All methods in this class are written to be optimized for efficiency and work directly
+ * on the StringValues char arrays, avoiding copies. For simplicity and efficiency, the methods
+ * only apply to strings whose characters are representable in a single <tt>char</tt>,
+ * ie. strings without surrogate characters.
  */
-@Internal
-public final class SimpleStringUtils {
+@PublicEvolving
+public final class StringValueUtils {
 	
 	/**
 	 * Converts the given <code>StringValue</code> into a lower case variant.
@@ -132,5 +135,5 @@ public boolean next(StringValue target) {
 	/**
 	 * Private constructor to prevent instantiation, as this is a utility method encapsulating class.
 	 */
-	private SimpleStringUtils() {}
+	private StringValueUtils() {}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/util/SimpleStringUtilsTest.java b/flink-core/src/test/java/org/apache/flink/util/StringValueUtilsTest.java
similarity index 80%
rename from flink-core/src/test/java/org/apache/flink/util/SimpleStringUtilsTest.java
rename to flink-core/src/test/java/org/apache/flink/util/StringValueUtilsTest.java
index 98f8574ec70af..6b15196f05569 100644
--- a/flink-core/src/test/java/org/apache/flink/util/SimpleStringUtilsTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/StringValueUtilsTest.java
@@ -23,31 +23,33 @@
 import static org.junit.Assert.assertTrue;
 
 import org.apache.flink.types.StringValue;
-import org.apache.flink.util.SimpleStringUtils;
-import org.apache.flink.util.SimpleStringUtils.WhitespaceTokenizer;
-import org.junit.Test;
+import org.apache.flink.util.StringValueUtils.WhitespaceTokenizer;
 
+import org.junit.Test;
 
-public class SimpleStringUtilsTest extends TestLogger {
+/**
+ * Tests for the {@link StringValueUtils}.
+ */
+public class StringValueUtilsTest extends TestLogger {
 
 	@Test
 	public void testToLowerCaseConverting() {
 		StringValue testString = new StringValue("TEST");
-		SimpleStringUtils.toLowerCase(testString);
+		StringValueUtils.toLowerCase(testString);
 		assertEquals(new StringValue("test"), testString);
 	}
 	
 	@Test
 	public void testReplaceNonWordChars() {
 		StringValue testString = new StringValue("TEST123_@");
-		SimpleStringUtils.replaceNonWordChars(testString, '!');
+		StringValueUtils.replaceNonWordChars(testString, '!');
 		assertEquals(new StringValue("TEST123_!"), testString);
 	}
 	
 	@Test
 	public void testTokenizerOnStringWithoutNexToken() {
 		StringValue testString = new StringValue("test");
-		SimpleStringUtils.WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
+		StringValueUtils.WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
 		tokenizer.setStringToTokenize(testString);
 		//first token
 		tokenizer.next(testString);
@@ -58,7 +60,7 @@ public void testTokenizerOnStringWithoutNexToken() {
 	@Test
 	public void testTokenizerOnStringWithNexToken() {
 		StringValue testString = new StringValue("test test");
-		SimpleStringUtils.WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
+		StringValueUtils.WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
 		tokenizer.setStringToTokenize(testString);
 		assertTrue(tokenizer.next(testString));
 	}
@@ -66,7 +68,7 @@ public void testTokenizerOnStringWithNexToken() {
 	@Test
 	public void testTokenizerOnStringOnlyWithDelimiter() {
 		StringValue testString = new StringValue("    ");
-		SimpleStringUtils.WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
+		StringValueUtils.WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
 		tokenizer.setStringToTokenize(testString);
 		assertFalse(tokenizer.next(testString));
 	}

From ebe4dfee9d1f4a6c884205ec6ceded9af756fd02 Mon Sep 17 00:00:00 2001
From: zhangminglei <zml13856086071@163.com>
Date: Wed, 19 Apr 2017 17:43:57 +0800
Subject: [PATCH 017/253] [FLINK-6311] [kinesis] NPE in FlinkKinesisConsumer if
 source was closed before run

This closes #3738.
---
 .../connectors/kinesis/internals/KinesisDataFetcher.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index 46847b32e6389..8f7ca6c40f1e4 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -149,7 +149,7 @@ public class KinesisDataFetcher<T> {
 	private final KinesisProxyInterface kinesis;
 
 	/** Thread that executed runFetcher() */
-	private Thread mainThread;
+	private volatile Thread mainThread;
 
 	/**
 	 * The current number of shards that are actively read by this fetcher.
@@ -408,7 +408,10 @@ public HashMap<KinesisStreamShard, SequenceNumber> snapshotState() {
 	 */
 	public void shutdownFetcher() {
 		running = false;
-		mainThread.interrupt(); // the main thread may be sleeping for the discovery interval
+
+		if (mainThread != null) {
+			mainThread.interrupt(); // the main thread may be sleeping for the discovery interval
+		}
 
 		if (LOG.isInfoEnabled()) {
 			LOG.info("Shutting down the shard consumer threads of subtask {} ...", indexOfThisConsumerSubtask);

From c8255fa0231ed6e92587d7c1f6d38a6429a96487 Mon Sep 17 00:00:00 2001
From: Flavio Pompermaier <f.pompermaier@gmail.com>
Date: Wed, 22 Mar 2017 00:05:25 +0100
Subject: [PATCH 018/253] [FLINK-6103] [core] Improve rename() on
 LocalFileSystem

This closes #3598
---
 .../flink/core/fs/local/LocalFileSystem.java  | 24 +++++++--
 .../core/fs/local/LocalFileSystemTest.java    | 52 ++++++++++++++++++-
 2 files changed, 69 insertions(+), 7 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
index 12aeb7f311a71..0a168de673d2d 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
@@ -20,7 +20,7 @@
 /**
  * This file is based on source code from the Hadoop Project (http://hadoop.apache.org/), licensed by the Apache
  * Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE file distributed with this work for
- * additional information regarding copyright ownership. 
+ * additional information regarding copyright ownership.
  */
 
 package org.apache.flink.core.fs.local;
@@ -43,7 +43,10 @@
 import java.net.InetAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
+import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
@@ -190,7 +193,7 @@ public boolean delete(final Path f, final boolean recursive) throws IOException
 
 	/**
 	 * Deletes the given file or directory.
-	 * 
+	 *
 	 * @param f
 	 *        the file to be deleted
 	 * @return <code>true</code> if all files were deleted successfully, <code>false</code> otherwise
@@ -217,12 +220,13 @@ private boolean delete(final File f) throws IOException {
 
 	/**
 	 * Recursively creates the directory specified by the provided path.
-	 * 
+	 *
 	 * @return <code>true</code>if the directories either already existed or have been created successfully,
 	 *         <code>false</code> otherwise
 	 * @throws IOException
 	 *         thrown if an error occurred while creating the directory/directories
 	 */
+	@Override
 	public boolean mkdirs(final Path f) throws IOException {
 		final File p2f = pathToFile(f);
 
@@ -262,8 +266,18 @@ public FSDataOutputStream create(
 	public boolean rename(final Path src, final Path dst) throws IOException {
 		final File srcFile = pathToFile(src);
 		final File dstFile = pathToFile(dst);
-
-		return srcFile.renameTo(dstFile);
+		//Files.move fails if the destination directory doesn't exist
+		if(dstFile.getParentFile()!= null && !dstFile.getParentFile().exists()){
+			dstFile.getParentFile().mkdirs();
+		}
+		try {
+			Files.move(srcFile.toPath(), dstFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+			return true;
+		}
+		catch (FileAlreadyExistsException | DirectoryNotEmptyException | SecurityException ex) {
+			//Catch the errors that are regular "move failed" exceptions and return false
+			return false;
+		}
 	}
 
 	@Override
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
index ee1deefa0be09..7b7d599ed29ac 100644
--- a/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
@@ -36,6 +36,7 @@
 import org.apache.flink.core.fs.FileStatus;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.fs.FileSystem.WriteMode;
 import org.apache.flink.core.testutils.CommonTestUtils;
 
 import org.apache.flink.util.FileUtils;
@@ -109,7 +110,7 @@ public void testLocalFilesystem() {
 			 */
 
 			// create files.. one ""natively"", one using lfs
-			final FSDataOutputStream lfsoutput1 = lfs.create(pathtotestfile1, false);
+			final FSDataOutputStream lfsoutput1 = lfs.create(pathtotestfile1, WriteMode.NO_OVERWRITE);
 			assertTrue(testfile2.createNewFile());
 
 			// does lfs create files? does lfs recognize created files?
@@ -127,7 +128,7 @@ public void testLocalFilesystem() {
 			try (FileInputStream fisfile1 = new FileInputStream(testfile1)) {
 				assertEquals(testbytestest.length, fisfile1.read(testbytestest));
 			}
-			
+
 			assertArrayEquals(testbytes, testbytestest);
 
 			// does lfs see the correct file length?
@@ -212,4 +213,51 @@ public void testDeletePathIfEmpty() throws IOException {
 		assertTrue(FileUtils.deletePathIfEmpty(fs, directoryPath));
 		assertFalse(fs.exists(directoryPath));
 	}
+
+	@Test
+	public void testRenamePath() throws IOException {
+		File rootDirectory = temporaryFolder.newFolder();
+
+		//create a file /root/src/B/test.csv
+		File srcDirectory  = new File(rootDirectory, "src");
+		srcDirectory  = new File(srcDirectory, "B");
+		assertTrue(srcDirectory.mkdirs());
+		File srcFile = new File(srcDirectory, "test.csv");
+		assertTrue(srcFile.createNewFile());
+
+		//Move/rename B and its content to /root/dst/A
+		File destDirectory  = new File(rootDirectory, "dst");
+		destDirectory  = new File(destDirectory, "B");
+		File destFile  = new File(destDirectory, "test.csv");
+
+		Path srcDirPath = new Path(srcDirectory.toURI());
+		Path srcFilePath = new Path(srcFile.toURI());
+		Path destDirPath = new Path(destDirectory.toURI());
+		Path destFilePath = new Path(destFile.toURI());
+
+		FileSystem fs = FileSystem.getLocalFileSystem();
+
+		// pre-conditions: /root/src/B exists but /root/dst/B does not
+		assertTrue(fs.exists(srcDirPath));
+		assertFalse(fs.exists(destDirPath));
+
+		// do the move/rename: /root/src/B -> /root/dst/
+		assertTrue(fs.rename(srcDirPath, destDirPath));
+		
+		// post-conditions: /root/src/B doens't exists, /root/dst/B/test.csv has been created
+		assertTrue(fs.exists(destFilePath));
+		assertFalse(fs.exists(srcDirPath));
+		
+		//re-create initial situation and test overwrite
+		assertTrue(fs.delete(destDirPath, true));
+		assertTrue(srcDirectory.mkdirs());
+		assertTrue(srcFile.createNewFile());
+		//now use the file as dest: /root/src/B/test.csv -> /root/dst/B/test.csv
+		assertTrue(fs.rename(srcFilePath, destFilePath));
+		
+		// post-conditions: now only the src file has been moved
+		assertFalse(fs.exists(srcFilePath));
+		assertTrue(fs.exists(srcDirPath));
+		assertTrue(fs.exists(destFilePath));
+	}
 }

From 7eb3de1d5aec3183683921ff13d64997b5deb01b Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 20 Apr 2017 16:00:22 +0200
Subject: [PATCH 019/253] [FLINK-6103] [core] Fix conditions and add tests for
 rename() on LocalFileSystem

---
 .../flink/core/fs/local/LocalFileSystem.java  |  17 +-
 .../core/fs/local/LocalFileSystemTest.java    | 260 +++++++++++-------
 2 files changed, 167 insertions(+), 110 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
index 0a168de673d2d..0e3e9f3baa275 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
@@ -43,9 +43,11 @@
 import java.net.InetAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
+import java.nio.file.AccessDeniedException;
 import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.StandardCopyOption;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -266,16 +268,19 @@ public FSDataOutputStream create(
 	public boolean rename(final Path src, final Path dst) throws IOException {
 		final File srcFile = pathToFile(src);
 		final File dstFile = pathToFile(dst);
-		//Files.move fails if the destination directory doesn't exist
-		if(dstFile.getParentFile()!= null && !dstFile.getParentFile().exists()){
-			dstFile.getParentFile().mkdirs();
-		}
+
+		final File dstParent = dstFile.getParentFile();
+
+		// Files.move fails if the destination directory doesn't exist
+		//noinspection ResultOfMethodCallIgnored -- we don't care if the directory existed or was created
+		dstParent.mkdirs();
+
 		try {
 			Files.move(srcFile.toPath(), dstFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
 			return true;
 		}
-		catch (FileAlreadyExistsException | DirectoryNotEmptyException | SecurityException ex) {
-			//Catch the errors that are regular "move failed" exceptions and return false
+		catch (NoSuchFileException | AccessDeniedException | DirectoryNotEmptyException | SecurityException ex) {
+			// catch the errors that are regular "move failed" exceptions and return false
 			return false;
 		}
 	}
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
index 7b7d599ed29ac..2312ee9856ef3 100644
--- a/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
@@ -22,7 +22,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -37,9 +36,9 @@
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.core.fs.FileSystem.WriteMode;
-import org.apache.flink.core.testutils.CommonTestUtils;
-
 import org.apache.flink.util.FileUtils;
+
+import org.junit.Assume;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -51,14 +50,14 @@
 public class LocalFileSystemTest {
 
 	@Rule
-	public TemporaryFolder temporaryFolder = new TemporaryFolder();
+	public final TemporaryFolder temporaryFolder = new TemporaryFolder();
 
 	/**
 	 * This test checks the functionality of the {@link LocalFileSystem} class.
 	 */
 	@Test
-	public void testLocalFilesystem() {
-		final File tempdir = new File(CommonTestUtils.getTempDir(), UUID.randomUUID().toString());
+	public void testLocalFilesystem() throws Exception {
+		final File tempdir = new File(temporaryFolder.getRoot(), UUID.randomUUID().toString());
 
 		final File testfile1 = new File(tempdir, UUID.randomUUID().toString());
 		final File testfile2 = new File(tempdir, UUID.randomUUID().toString());
@@ -66,113 +65,102 @@ public void testLocalFilesystem() {
 		final Path pathtotestfile1 = new Path(testfile1.toURI().getPath());
 		final Path pathtotestfile2 = new Path(testfile2.toURI().getPath());
 
-		try {
-			final LocalFileSystem lfs = new LocalFileSystem();
+		final LocalFileSystem lfs = new LocalFileSystem();
 
-			final Path pathtotmpdir = new Path(tempdir.toURI().getPath());
+		final Path pathtotmpdir = new Path(tempdir.toURI().getPath());
 
-			/*
-			 * check that lfs can see/create/delete/read directories
-			 */
+		/*
+		 * check that lfs can see/create/delete/read directories
+		 */
 
-			// check that dir is not existent yet
-			assertFalse(lfs.exists(pathtotmpdir));
-			assertTrue(tempdir.mkdirs());
+		// check that dir is not existent yet
+		assertFalse(lfs.exists(pathtotmpdir));
+		assertTrue(tempdir.mkdirs());
 
-			// check that local file system recognizes file..
-			assertTrue(lfs.exists(pathtotmpdir));
-			final FileStatus localstatus1 = lfs.getFileStatus(pathtotmpdir);
+		// check that local file system recognizes file..
+		assertTrue(lfs.exists(pathtotmpdir));
+		final FileStatus localstatus1 = lfs.getFileStatus(pathtotmpdir);
 
-			// check that lfs recognizes directory..
-			assertTrue(localstatus1.isDir());
+		// check that lfs recognizes directory..
+		assertTrue(localstatus1.isDir());
 
-			// get status for files in this (empty) directory..
-			final FileStatus[] statusforfiles = lfs.listStatus(pathtotmpdir);
+		// get status for files in this (empty) directory..
+		final FileStatus[] statusforfiles = lfs.listStatus(pathtotmpdir);
 
-			// no files in there.. hence, must be zero
-			assertTrue(statusforfiles.length == 0);
+		// no files in there.. hence, must be zero
+		assertTrue(statusforfiles.length == 0);
 
-			// check that lfs can delete directory..
-			lfs.delete(pathtotmpdir, true);
+		// check that lfs can delete directory..
+		lfs.delete(pathtotmpdir, true);
 
-			// double check that directory is not existent anymore..
-			assertFalse(lfs.exists(pathtotmpdir));
-			assertFalse(tempdir.exists());
+		// double check that directory is not existent anymore..
+		assertFalse(lfs.exists(pathtotmpdir));
+		assertFalse(tempdir.exists());
 
-			// re-create directory..
-			lfs.mkdirs(pathtotmpdir);
+		// re-create directory..
+		lfs.mkdirs(pathtotmpdir);
 
-			// creation successful?
-			assertTrue(tempdir.exists());
+		// creation successful?
+		assertTrue(tempdir.exists());
 
-			/*
-			 * check that lfs can create/read/write from/to files properly and read meta information..
-			 */
+		/*
+		 * check that lfs can create/read/write from/to files properly and read meta information..
+		 */
 
-			// create files.. one ""natively"", one using lfs
-			final FSDataOutputStream lfsoutput1 = lfs.create(pathtotestfile1, WriteMode.NO_OVERWRITE);
-			assertTrue(testfile2.createNewFile());
+		// create files.. one ""natively"", one using lfs
+		final FSDataOutputStream lfsoutput1 = lfs.create(pathtotestfile1, WriteMode.NO_OVERWRITE);
+		assertTrue(testfile2.createNewFile());
 
-			// does lfs create files? does lfs recognize created files?
-			assertTrue(testfile1.exists());
-			assertTrue(lfs.exists(pathtotestfile2));
+		// does lfs create files? does lfs recognize created files?
+		assertTrue(testfile1.exists());
+		assertTrue(lfs.exists(pathtotestfile2));
 
-			// test that lfs can write to files properly
-			final byte[] testbytes = { 1, 2, 3, 4, 5 };
-			lfsoutput1.write(testbytes);
-			lfsoutput1.close();
+		// test that lfs can write to files properly
+		final byte[] testbytes = { 1, 2, 3, 4, 5 };
+		lfsoutput1.write(testbytes);
+		lfsoutput1.close();
 
-			assertEquals(testfile1.length(), 5L);
+		assertEquals(testfile1.length(), 5L);
 
-			byte[] testbytestest = new byte[5];
-			try (FileInputStream fisfile1 = new FileInputStream(testfile1)) {
-				assertEquals(testbytestest.length, fisfile1.read(testbytestest));
-			}
-
-			assertArrayEquals(testbytes, testbytestest);
+		byte[] testbytestest = new byte[5];
+		try (FileInputStream fisfile1 = new FileInputStream(testfile1)) {
+			assertEquals(testbytestest.length, fisfile1.read(testbytestest));
+		}
 
-			// does lfs see the correct file length?
-			assertEquals(lfs.getFileStatus(pathtotestfile1).getLen(), testfile1.length());
+		assertArrayEquals(testbytes, testbytestest);
 
-			// as well, when we call the listStatus (that is intended for directories?)
-			assertEquals(lfs.listStatus(pathtotestfile1)[0].getLen(), testfile1.length());
+		// does lfs see the correct file length?
+		assertEquals(lfs.getFileStatus(pathtotestfile1).getLen(), testfile1.length());
 
-			// test that lfs can read files properly
-			final FileOutputStream fosfile2 = new FileOutputStream(testfile2);
-			fosfile2.write(testbytes);
-			fosfile2.close();
+		// as well, when we call the listStatus (that is intended for directories?)
+		assertEquals(lfs.listStatus(pathtotestfile1)[0].getLen(), testfile1.length());
 
-			testbytestest = new byte[5];
-			final FSDataInputStream lfsinput2 = lfs.open(pathtotestfile2);
-			assertEquals(lfsinput2.read(testbytestest), 5);
-			lfsinput2.close();
-			assertTrue(Arrays.equals(testbytes, testbytestest));
+		// test that lfs can read files properly
+		final FileOutputStream fosfile2 = new FileOutputStream(testfile2);
+		fosfile2.write(testbytes);
+		fosfile2.close();
 
-			// does lfs see two files?
-			assertEquals(lfs.listStatus(pathtotmpdir).length, 2);
+		testbytestest = new byte[5];
+		final FSDataInputStream lfsinput2 = lfs.open(pathtotestfile2);
+		assertEquals(lfsinput2.read(testbytestest), 5);
+		lfsinput2.close();
+		assertTrue(Arrays.equals(testbytes, testbytestest));
 
-			// do we get exactly one blocklocation per file? no matter what start and len we provide
-			assertEquals(lfs.getFileBlockLocations(lfs.getFileStatus(pathtotestfile1), 0, 0).length, 1);
+		// does lfs see two files?
+		assertEquals(lfs.listStatus(pathtotmpdir).length, 2);
 
-			/*
-			 * can lfs delete files / directories?
-			 */
-			assertTrue(lfs.delete(pathtotestfile1, false));
+		// do we get exactly one blocklocation per file? no matter what start and len we provide
+		assertEquals(lfs.getFileBlockLocations(lfs.getFileStatus(pathtotestfile1), 0, 0).length, 1);
 
-			// and can lfs also delete directories recursively?
-			assertTrue(lfs.delete(pathtotmpdir, true));
+		/*
+		 * can lfs delete files / directories?
+		 */
+		assertTrue(lfs.delete(pathtotestfile1, false));
 
-			assertTrue(!tempdir.exists());
+		// and can lfs also delete directories recursively?
+		assertTrue(lfs.delete(pathtotmpdir, true));
 
-		} catch (IOException e) {
-			fail(e.getMessage());
-		}
-		finally {
-			// clean up!
-			testfile1.delete();
-			testfile2.delete();
-			tempdir.delete();
-		}
+		assertTrue(!tempdir.exists());
 	}
 
 	/**
@@ -216,24 +204,23 @@ public void testDeletePathIfEmpty() throws IOException {
 
 	@Test
 	public void testRenamePath() throws IOException {
-		File rootDirectory = temporaryFolder.newFolder();
+		final File rootDirectory = temporaryFolder.newFolder();
 
 		//create a file /root/src/B/test.csv
-		File srcDirectory  = new File(rootDirectory, "src");
-		srcDirectory  = new File(srcDirectory, "B");
+		final File srcDirectory  = new File(new File(rootDirectory, "src"), "B");
 		assertTrue(srcDirectory.mkdirs());
-		File srcFile = new File(srcDirectory, "test.csv");
+
+		final File srcFile = new File(srcDirectory, "test.csv");
 		assertTrue(srcFile.createNewFile());
 
 		//Move/rename B and its content to /root/dst/A
-		File destDirectory  = new File(rootDirectory, "dst");
-		destDirectory  = new File(destDirectory, "B");
-		File destFile  = new File(destDirectory, "test.csv");
+		final File destDirectory  = new File(new File(rootDirectory, "dst"), "B");
+		final File destFile  = new File(destDirectory, "test.csv");
 
-		Path srcDirPath = new Path(srcDirectory.toURI());
-		Path srcFilePath = new Path(srcFile.toURI());
-		Path destDirPath = new Path(destDirectory.toURI());
-		Path destFilePath = new Path(destFile.toURI());
+		final Path srcDirPath = new Path(srcDirectory.toURI());
+		final Path srcFilePath = new Path(srcFile.toURI());
+		final Path destDirPath = new Path(destDirectory.toURI());
+		final Path destFilePath = new Path(destFile.toURI());
 
 		FileSystem fs = FileSystem.getLocalFileSystem();
 
@@ -243,21 +230,86 @@ public void testRenamePath() throws IOException {
 
 		// do the move/rename: /root/src/B -> /root/dst/
 		assertTrue(fs.rename(srcDirPath, destDirPath));
-		
-		// post-conditions: /root/src/B doens't exists, /root/dst/B/test.csv has been created
+
+		// post-conditions: /root/src/B doesn't exists, /root/dst/B/test.csv has been created
 		assertTrue(fs.exists(destFilePath));
 		assertFalse(fs.exists(srcDirPath));
-		
-		//re-create initial situation and test overwrite
-		assertTrue(fs.delete(destDirPath, true));
+
+		// re-create source file and test overwrite
 		assertTrue(srcDirectory.mkdirs());
 		assertTrue(srcFile.createNewFile());
-		//now use the file as dest: /root/src/B/test.csv -> /root/dst/B/test.csv
+
+		// overwrite the destination file
 		assertTrue(fs.rename(srcFilePath, destFilePath));
-		
+
 		// post-conditions: now only the src file has been moved
 		assertFalse(fs.exists(srcFilePath));
 		assertTrue(fs.exists(srcDirPath));
 		assertTrue(fs.exists(destFilePath));
 	}
+
+	@Test
+	public void testRenameNonExistingFile() throws IOException {
+		final FileSystem fs = FileSystem.getLocalFileSystem();
+
+		final File srcFile = new File(temporaryFolder.newFolder(), "someFile.txt");
+		final File destFile  = new File(temporaryFolder.newFolder(), "target");
+
+		final Path srcFilePath = new Path(srcFile.toURI());
+		final Path destFilePath = new Path(destFile.toURI());
+
+		// this cannot succeed because the source file does not exist
+		assertFalse(fs.rename(srcFilePath, destFilePath));
+	}
+
+	@Test
+	public void testRenameFileWithNoAccess() throws IOException {
+		final FileSystem fs = FileSystem.getLocalFileSystem();
+
+		final File srcFile = temporaryFolder.newFile("someFile.txt");
+		final File destFile = new File(temporaryFolder.newFolder(), "target");
+
+		// we need to make the file non-modifiable so that the rename fails
+		Assume.assumeTrue(srcFile.getParentFile().setWritable(false, false));
+		Assume.assumeTrue(srcFile.setWritable(false, false));
+
+		try {
+			final Path srcFilePath = new Path(srcFile.toURI());
+			final Path destFilePath = new Path(destFile.toURI());
+
+			// this cannot succeed because the source folder has no permission to remove the file
+			assertFalse(fs.rename(srcFilePath, destFilePath));
+		}
+		finally {
+			// make sure we give permission back to ensure proper cleanup
+
+			//noinspection ResultOfMethodCallIgnored
+			srcFile.getParentFile().setWritable(true, false);
+			//noinspection ResultOfMethodCallIgnored
+			srcFile.setWritable(true, false);
+		}
+	}
+
+	@Test
+	public void testRenameToNonEmptyTargetDir() throws IOException {
+		final FileSystem fs = FileSystem.getLocalFileSystem();
+
+		// a source folder with a file
+		final File srcFolder = temporaryFolder.newFolder();
+		final File srcFile = new File(srcFolder, "someFile.txt");
+		assertTrue(srcFile.createNewFile());
+
+		// a non-empty destination folder
+		final File dstFolder = temporaryFolder.newFolder();
+		final File dstFile  = new File(dstFolder, "target");
+		assertTrue(dstFile.createNewFile());
+
+		// this cannot succeed because the destination folder is not empty
+		assertFalse(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));
+
+		// retry after deleting the occupying target file
+		assertTrue(dstFile.delete());
+		assertTrue(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));
+		assertTrue(new File(dstFolder, srcFile.getName()).exists());
+	}
 }

From 9c951b99ed1c29c0bc24e3ee9ef403d8c1009256 Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Thu, 20 Apr 2017 08:46:01 -0400
Subject: [PATCH 020/253] [FLINK-5623] [runtime] Fix TempBarrier dam has been
 closed

Properly reset the "pipeline breaker" upon closing.

This closes #3747
---
 .../apache/flink/graph/library/link_analysis/PageRank.java   | 5 -----
 .../java/org/apache/flink/runtime/operators/BatchTask.java   | 1 +
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java
index c5c4178275396..747735eba273c 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/PageRank.java
@@ -45,7 +45,6 @@
 import org.apache.flink.graph.library.link_analysis.Functions.SumScore;
 import org.apache.flink.graph.library.link_analysis.PageRank.Result;
 import org.apache.flink.graph.utils.GraphUtils;
-import org.apache.flink.graph.utils.GraphUtils.IdentityMapper;
 import org.apache.flink.graph.utils.Murmur3_32;
 import org.apache.flink.graph.utils.proxy.GraphAlgorithmWrappingDataSet;
 import org.apache.flink.types.DoubleValue;
@@ -176,10 +175,6 @@ public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
 			.run(new VertexDegrees<K, VV, EV>()
 				.setParallelism(parallelism));
 
-		// prevent Exception "The dam has been closed." in TempBarrier
-		// for a simplified Graph as in PageRankITCase (see FLINK-5623)
-		vertexDegree = vertexDegree.map(new IdentityMapper<Vertex<K, Degrees>>());
-
 		// vertex count
 		DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
index f7480794c4098..87b0a76d0e8e4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
@@ -884,6 +884,7 @@ protected void resetAllInputs() throws Exception {
 					// close the async barrier if there is one
 					if (this.tempBarriers[i] != null) {
 						this.tempBarriers[i].close();
+						this.tempBarriers[i] = null;
 					}
 
 					// recreate the local strategy

From c8ce904233a05cadf79ab3f03b3e4304be054ded Mon Sep 17 00:00:00 2001
From: Marc Tremblay <marcandre.tr@gmail.com>
Date: Fri, 21 Apr 2017 00:22:14 -0700
Subject: [PATCH 021/253] [hotfix] [docs] Typo sick -> sink

This closes #3749
---
 docs/dev/connectors/filesystem_sink.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/dev/connectors/filesystem_sink.md b/docs/dev/connectors/filesystem_sink.md
index 67250f0c83324..d12752f42452d 100644
--- a/docs/dev/connectors/filesystem_sink.md
+++ b/docs/dev/connectors/filesystem_sink.md
@@ -44,7 +44,7 @@ cluster execution.
 #### Bucketing File Sink
 
 The bucketing behaviour as well as the writing can be configured but we will get to that later.
-This is how you can create a bucketing sick which by default, sinks to rolling files that are split by time:
+This is how you can create a bucketing sink which by default, sinks to rolling files that are split by time:
 
 <div class="codetabs" markdown="1">
 <div data-lang="java" markdown="1">

From 7dcfc1361553fd30741bb95ba17bcf5e71018039 Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Sun, 26 Mar 2017 15:46:00 -0400
Subject: [PATCH 022/253] [FLINK-6176] [scripts] [yarn] [mesos] Add JARs to
 CLASSPATH deterministically

Sorts files read from Flink's lib directory and places the distribution
JAR to the end of the CLASSPATH.

This closes #3632
---
 flink-dist/src/main/flink-bin/bin/config.sh   | 18 +++++++++--
 .../flink-bin/mesos-bin/mesos-appmaster.sh    | 17 +---------
 .../flink-bin/mesos-bin/mesos-taskmanager.sh  | 17 +---------
 .../main/flink-bin/yarn-bin/yarn-session.sh   | 17 +---------
 .../yarn/AbstractYarnClusterDescriptor.java   | 32 ++++++++++---------
 5 files changed, 35 insertions(+), 66 deletions(-)

diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index 5ef4ef7218206..dbfdd0e5e4c3e 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -18,16 +18,28 @@
 ################################################################################
 
 constructFlinkClassPath() {
+    local FLINK_DIST
+    local FLINK_CLASSPATH
 
     while read -d '' -r jarfile ; do
-        if [[ $FLINK_CLASSPATH = "" ]]; then
+        if [[ "$jarfile" =~ .*flink-dist.*.jar ]]; then
+            FLINK_DIST="$FLINK_DIST":"$jarfile"
+        elif [[ "$FLINK_CLASSPATH" == "" ]]; then
             FLINK_CLASSPATH="$jarfile";
         else
             FLINK_CLASSPATH="$FLINK_CLASSPATH":"$jarfile"
         fi
-    done < <(find "$FLINK_LIB_DIR" ! -type d -name '*.jar' -print0)
+    done < <(find "$FLINK_LIB_DIR" ! -type d -name '*.jar' -print0 | sort -z)
 
-    echo $FLINK_CLASSPATH
+    if [[ "$FLINK_DIST" == "" ]]; then
+        # write error message to stderr since stdout is stored as the classpath
+        (>&2 echo "[ERROR] Flink distribution jar not found in $FLINK_LIB_DIR.")
+
+        # exit function with empty classpath to force process failure
+        exit 1
+    fi
+
+    echo "$FLINK_CLASSPATH""$FLINK_DIST"
 }
 
 # These are used to mangle paths that are passed to java when using
diff --git a/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster.sh b/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster.sh
index 6fae6c2130eed..67eab9d491a9c 100755
--- a/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster.sh
+++ b/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster.sh
@@ -23,26 +23,11 @@ bin=`cd "$bin"; pwd`
 # get Flink config
 . "$bin"/config.sh
 
-# auxilliary function to construct a lightweight classpath for the
-# Flink AppMaster
-constructAppMasterClassPath() {
-
-    while read -d '' -r jarfile ; do
-        if [[ $CC_CLASSPATH = "" ]]; then
-            CC_CLASSPATH="$jarfile";
-        else
-            CC_CLASSPATH="$CC_CLASSPATH":"$jarfile"
-        fi
-    done < <(find "$FLINK_LIB_DIR" ! -type d -name '*.jar' -print0)
-
-    echo $CC_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS
-}
-
 if [ "$FLINK_IDENT_STRING" = "" ]; then
     FLINK_IDENT_STRING="$USER"
 fi
 
-CC_CLASSPATH=`manglePathList $(constructAppMasterClassPath)`
+CC_CLASSPATH=`manglePathList $(constructFlinkClassPath):$INTERNAL_HADOOP_CLASSPATHS`
 
 log="${FLINK_LOG_DIR}/flink-${FLINK_IDENT_STRING}-mesos-appmaster-${HOSTNAME}.log"
 log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback.xml"
diff --git a/flink-dist/src/main/flink-bin/mesos-bin/mesos-taskmanager.sh b/flink-dist/src/main/flink-bin/mesos-bin/mesos-taskmanager.sh
index 23a301b667b88..ab2f7b1dbd9e5 100755
--- a/flink-dist/src/main/flink-bin/mesos-bin/mesos-taskmanager.sh
+++ b/flink-dist/src/main/flink-bin/mesos-bin/mesos-taskmanager.sh
@@ -23,22 +23,7 @@ bin=`cd "$bin"; pwd`
 # get Flink config
 . "$bin"/config.sh
 
-# auxilliary function to construct a lightweight classpath for the
-# Flink TaskManager
-constructTaskManagerClassPath() {
-
-    while read -d '' -r jarfile ; do
-        if [[ $CC_CLASSPATH = "" ]]; then
-            CC_CLASSPATH="$jarfile";
-        else
-            CC_CLASSPATH="$CC_CLASSPATH":"$jarfile"
-        fi
-    done < <(find "$FLINK_LIB_DIR" ! -type d -name '*.jar' -print0)
-
-    echo $CC_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS
-}
-
-CC_CLASSPATH=`manglePathList $(constructTaskManagerClassPath)`
+CC_CLASSPATH=`manglePathList $(constructFlinkClassPath):$INTERNAL_HADOOP_CLASSPATHS`
 
 log=flink-taskmanager.log
 log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback.xml"
diff --git a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
index 1755f32f4440b..03b1e3ae80f97 100755
--- a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
+++ b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
@@ -29,22 +29,7 @@ fi
 
 JVM_ARGS="$JVM_ARGS -Xmx512m"
 
-# auxilliary function to construct a lightweight classpath for the
-# Flink CLI client
-constructCLIClientClassPath() {
-
-    while read -d '' -r jarfile ; do
-        if [[ $CC_CLASSPATH = "" ]]; then
-            CC_CLASSPATH="$jarfile";
-        else
-            CC_CLASSPATH="$CC_CLASSPATH":"$jarfile"
-        fi
-    done < <(find "$FLINK_LIB_DIR" ! -type d -name '*.jar' -print0)
-
-    echo $CC_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS
-}
-
-CC_CLASSPATH=`manglePathList $(constructCLIClientClassPath)`
+CC_CLASSPATH=`manglePathList $(constructFlinkClassPath):$INTERNAL_HADOOP_CLASSPATHS`
 
 log=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-yarn-session-$HOSTNAME.log
 log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j-yarn-session.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback-yarn.xml"
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index ec7af5a811eae..a5a6c364aab2b 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -23,10 +23,10 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.HighAvailabilityOptions;
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.SecurityOptions;
-import org.apache.flink.configuration.HighAvailabilityOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
@@ -63,10 +63,10 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.ObjectOutputStream;
+import java.io.PrintStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URISyntaxException;
@@ -669,12 +669,11 @@ public ApplicationReport startAppMaster(JobGraph jobGraph, YarnClient yarnClient
 		final Map<String, LocalResource> localResources = new HashMap<>(2 + effectiveShipFiles.size());
 		// list of remote paths (after upload)
 		final List<Path> paths = new ArrayList<>(2 + effectiveShipFiles.size());
-		// classpath assembler
-		final StringBuilder classPathBuilder = new StringBuilder();
 		// ship list that enables reuse of resources for task manager containers
 		StringBuilder envShipFileList = new StringBuilder();
 
 		// upload and register ship files
+		final List<String> classPaths = new ArrayList<>();
 		for (File shipFile : effectiveShipFiles) {
 			LocalResource shipResources = Records.newRecord(LocalResource.class);
 
@@ -693,29 +692,32 @@ public ApplicationReport startAppMaster(JobGraph jobGraph, YarnClient yarnClient
 
 				Files.walkFileTree(shipPath, new SimpleFileVisitor<java.nio.file.Path>() {
 					@Override
-					public FileVisitResult preVisitDirectory(java.nio.file.Path dir, BasicFileAttributes attrs)
+					public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs)
 							throws IOException {
-						super.preVisitDirectory(dir, attrs);
-
-						java.nio.file.Path relativePath = parentPath.relativize(dir);
+						java.nio.file.Path relativePath = parentPath.relativize(file);
 
-						classPathBuilder
-							.append(relativePath)
-							.append(File.separator)
-							.append("*")
-							.append(File.pathSeparator);
+						classPaths.add(relativePath.toString());
 
 						return FileVisitResult.CONTINUE;
 					}
 				});
 			} else {
 				// add files to the classpath
-				classPathBuilder.append(shipFile.getName()).append(File.pathSeparator);
+				classPaths.add(shipFile.getName());
 			}
 
 			envShipFileList.append(remotePath).append(",");
 		}
 
+		// normalize classpath by sorting
+		Collections.sort(classPaths);
+
+		// classpath assembler
+		StringBuilder classPathBuilder = new StringBuilder();
+		for (String classPath : classPaths) {
+			classPathBuilder.append(classPath).append(File.pathSeparator);
+		}
+
 		// Setup jar for ApplicationMaster
 		LocalResource appMasterJar = Records.newRecord(LocalResource.class);
 		LocalResource flinkConf = Records.newRecord(LocalResource.class);

From 417249234167b4f2d7a4ed25f15a531c675bdcc1 Mon Sep 17 00:00:00 2001
From: zcb <2056268568@qq.com>
Date: Thu, 23 Mar 2017 03:44:10 +0800
Subject: [PATCH 023/253] [FLINK-6117] [security] Make setting of
 'zookeeper.sasl.disable' work correctly

This closes #3600
---
 .../apache/flink/configuration/SecurityOptions.java    |  4 ++++
 .../apache/flink/runtime/security/SecurityUtils.java   |  7 +++++++
 .../runtime/security/modules/ZooKeeperModule.java      | 10 ++++++++++
 .../apache/flink/test/util/SecureTestEnvironment.java  |  1 +
 4 files changed, 22 insertions(+)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java
index 95cf0c776d9fb..37631986d43c2 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java
@@ -55,6 +55,10 @@ public class SecurityOptions {
 	//  ZooKeeper Security Options
 	// ------------------------------------------------------------------------
 
+	public static final ConfigOption<Boolean> ZOOKEEPER_SASL_DISABLE =
+		key("zookeeper.sasl.disable")
+			.defaultValue(false);
+
 	public static final ConfigOption<String> ZOOKEEPER_SASL_SERVICE_NAME =
 		key("zookeeper.sasl.service-name")
 			.defaultValue("zookeeper");
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
index d76e7a5cf3011..7a09c32c5a3e6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
@@ -125,6 +125,8 @@ public static class SecurityConfiguration {
 
 		private final org.apache.hadoop.conf.Configuration hadoopConf;
 
+		private final boolean isZkSaslDisable;
+
 		private final boolean useTicketCache;
 
 		private final String keytab;
@@ -164,6 +166,7 @@ public SecurityConfiguration(Configuration flinkConf,
 				org.apache.hadoop.conf.Configuration hadoopConf,
 				List<? extends Class<? extends SecurityModule>> securityModules) {
 			this.hadoopConf = checkNotNull(hadoopConf);
+			this.isZkSaslDisable = flinkConf.getBoolean(SecurityOptions.ZOOKEEPER_SASL_DISABLE);
 			this.keytab = flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB);
 			this.principal = flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL);
 			this.useTicketCache = flinkConf.getBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE);
@@ -175,6 +178,10 @@ public SecurityConfiguration(Configuration flinkConf,
 			validate();
 		}
 
+		public boolean isZkSaslDisable() {
+			return isZkSaslDisable;
+		}
+
 		public String getKeytab() {
 			return keytab;
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/ZooKeeperModule.java b/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/ZooKeeperModule.java
index c0ba4a5a057b7..216bdded8cb72 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/ZooKeeperModule.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/ZooKeeperModule.java
@@ -41,6 +41,8 @@ public class ZooKeeperModule implements SecurityModule {
 	 */
 	private static final String ZK_LOGIN_CONTEXT_NAME = "zookeeper.sasl.clientconfig";
 
+	private String priorSaslEnable;
+
 	private String priorServiceName;
 
 	private String priorLoginContextName;
@@ -48,6 +50,9 @@ public class ZooKeeperModule implements SecurityModule {
 	@Override
 	public void install(SecurityUtils.SecurityConfiguration configuration) throws SecurityInstallException {
 
+		priorSaslEnable = System.getProperty(ZK_ENABLE_CLIENT_SASL, null);
+		System.setProperty(ZK_ENABLE_CLIENT_SASL, String.valueOf(!configuration.isZkSaslDisable()));
+
 		priorServiceName = System.getProperty(ZK_SASL_CLIENT_USERNAME, null);
 		if (!"zookeeper".equals(configuration.getZooKeeperServiceName())) {
 			System.setProperty(ZK_SASL_CLIENT_USERNAME, configuration.getZooKeeperServiceName());
@@ -61,6 +66,11 @@ public void install(SecurityUtils.SecurityConfiguration configuration) throws Se
 
 	@Override
 	public void uninstall() throws SecurityInstallException {
+		if(priorSaslEnable != null) {
+			System.setProperty(ZK_ENABLE_CLIENT_SASL, priorSaslEnable);
+		} else {
+			System.clearProperty(ZK_ENABLE_CLIENT_SASL);
+		}
 		if(priorServiceName != null) {
 			System.setProperty(ZK_SASL_CLIENT_USERNAME, priorServiceName);
 		} else {
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/SecureTestEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/SecureTestEnvironment.java
index febd074f98464..98deee6a98f9e 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/SecureTestEnvironment.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/SecureTestEnvironment.java
@@ -138,6 +138,7 @@ public static void prepare(TemporaryFolder tempFolder) {
 			//ctx.setHadoopConfiguration() for the UGI implementation to work properly.
 			//See Yarn test case module for reference
 			Configuration flinkConfig = GlobalConfiguration.loadConfiguration();
+			flinkConfig.setBoolean(SecurityOptions.ZOOKEEPER_SASL_DISABLE, false);
 			flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, testKeytab);
 			flinkConfig.setBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE, false);
 			flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, testPrincipal);

From af7a2ccead7cfc9fa4b3f5c4d1cee4bdd8637351 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 21 Apr 2017 11:05:00 +0200
Subject: [PATCH 024/253] [misc] Closing commit for outdated pull request

This closes #3592

From ef8919e301318f6da2e3780008e4bdcd54f0f8db Mon Sep 17 00:00:00 2001
From: zjureel <zjureel@gmail.com>
Date: Thu, 30 Mar 2017 11:12:41 +0800
Subject: [PATCH 025/253] [FLINK-6210] [rocksdb] Close RocksDB in
 ListViaMergeSpeedMiniBenchmark && ListViaRangeSpeedMiniBenchmark

This closes #3652
---
 .../ListViaMergeSpeedMiniBenchmark.java       | 73 +++++++-------
 .../ListViaRangeSpeedMiniBenchmark.java       | 99 ++++++++++---------
 2 files changed, 93 insertions(+), 79 deletions(-)

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java
index 2a530e1d903ad..f3e084f550497 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java
@@ -53,52 +53,59 @@ public static void main(String[] args) throws Exception {
 		final String key = "key";
 		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
 
-		final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
-		final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
+		try {
+			final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+			final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
 
-		final int num = 50000;
+			final int num = 50000;
 
-		// ----- insert -----
-		System.out.println("begin insert");
+			// ----- insert -----
+			System.out.println("begin insert");
 
-		final long beginInsert = System.nanoTime();
-		for (int i = 0; i < num; i++) {
-			rocksDB.merge(write_options, keyBytes, valueBytes);
-		}
-		final long endInsert = System.nanoTime();
-		System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
+			final long beginInsert = System.nanoTime();
+			for (int i = 0; i < num; i++) {
+				rocksDB.merge(write_options, keyBytes, valueBytes);
+			}
+			final long endInsert = System.nanoTime();
+			System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
 
-		// ----- read (attempt 1) -----
+			// ----- read (attempt 1) -----
 
-		final byte[] resultHolder = new byte[num * (valueBytes.length + 2)];
-		final long beginGet1 = System.nanoTime();
-		rocksDB.get(keyBytes, resultHolder);
-		final long endGet1 = System.nanoTime();
+			final byte[] resultHolder = new byte[num * (valueBytes.length + 2)];
+			final long beginGet1 = System.nanoTime();
+			rocksDB.get(keyBytes, resultHolder);
+			final long endGet1 = System.nanoTime();
 
-		System.out.println("end get - duration: " + ((endGet1 - beginGet1) / 1_000_000) + " ms");
+			System.out.println("end get - duration: " + ((endGet1 - beginGet1) / 1_000_000) + " ms");
 
-		// ----- read (attempt 2) -----
+			// ----- read (attempt 2) -----
 
-		final long beginGet2 = System.nanoTime();
-		rocksDB.get(keyBytes, resultHolder);
-		final long endGet2 = System.nanoTime();
+			final long beginGet2 = System.nanoTime();
+			rocksDB.get(keyBytes, resultHolder);
+			final long endGet2 = System.nanoTime();
 
-		System.out.println("end get - duration: " + ((endGet2 - beginGet2) / 1_000_000) + " ms");
+			System.out.println("end get - duration: " + ((endGet2 - beginGet2) / 1_000_000) + " ms");
 
-		// ----- compact -----
-		System.out.println("compacting...");
-		final long beginCompact = System.nanoTime();
-		rocksDB.compactRange();
-		final long endCompact = System.nanoTime();
+			// ----- compact -----
+			System.out.println("compacting...");
+			final long beginCompact = System.nanoTime();
+			rocksDB.compactRange();
+			final long endCompact = System.nanoTime();
 
-		System.out.println("end compaction - duration: " + ((endCompact - beginCompact) / 1_000_000) + " ms");
+			System.out.println("end compaction - duration: " + ((endCompact - beginCompact) / 1_000_000) + " ms");
 
-		// ----- read (attempt 3) -----
+			// ----- read (attempt 3) -----
 
-		final long beginGet3 = System.nanoTime();
-		rocksDB.get(keyBytes, resultHolder);
-		final long endGet3 = System.nanoTime();
+			final long beginGet3 = System.nanoTime();
+			rocksDB.get(keyBytes, resultHolder);
+			final long endGet3 = System.nanoTime();
 
-		System.out.println("end get - duration: " + ((endGet3 - beginGet3) / 1_000_000) + " ms");
+			System.out.println("end get - duration: " + ((endGet3 - beginGet3) / 1_000_000) + " ms");
+		} finally {
+			rocksDB.close();
+			options.close();
+			write_options.close();
+			FileUtils.deleteDirectory(rocksDir);
+		}
 	}
 }
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java
index b427ef1f3c56d..f46e2cdb49d1f 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java
@@ -56,61 +56,68 @@ public static void main(String[] args) throws Exception {
 
 		final String key = "key";
 		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
+		
+		try {
 
-		final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
-		final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
+			final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+			final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
 
-		final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4);
+			final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4);
 
-		final Unsafe unsafe = MemoryUtils.UNSAFE;
-		final long offset = unsafe.arrayBaseOffset(byte[].class) + keyTemplate.length - 4;
+			final Unsafe unsafe = MemoryUtils.UNSAFE;
+			final long offset = unsafe.arrayBaseOffset(byte[].class) + keyTemplate.length - 4;
 
-		final int num = 50000;
-		System.out.println("begin insert");
+			final int num = 50000;
+			System.out.println("begin insert");
 
-		final long beginInsert = System.nanoTime();
-		for (int i = 0; i < num; i++) {
-			unsafe.putInt(keyTemplate, offset, i);
-			rocksDB.put(write_options, keyTemplate, valueBytes);
-		}
-		final long endInsert = System.nanoTime();
-		System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
-
-		final byte[] resultHolder = new byte[num * valueBytes.length];
-
-		final long beginGet = System.nanoTime();
-
-		final RocksIterator iterator = rocksDB.newIterator();
-		int pos = 0;
-
-		// seek to start
-		unsafe.putInt(keyTemplate, offset, 0);
-		iterator.seek(keyTemplate);
-
-		// mark end
-		unsafe.putInt(keyTemplate, offset, -1);
-
-		// iterate
-		while (iterator.isValid()) {
-			byte[] currKey = iterator.key();
-			if (samePrefix(keyBytes, currKey)) {
-				byte[] currValue = iterator.value();
-				System.arraycopy(currValue, 0, resultHolder, pos, currValue.length);
-				pos += currValue.length;
-				iterator.next();
+			final long beginInsert = System.nanoTime();
+			for (int i = 0; i < num; i++) {
+				unsafe.putInt(keyTemplate, offset, i);
+				rocksDB.put(write_options, keyTemplate, valueBytes);
 			}
-			else {
-				break;
+			final long endInsert = System.nanoTime();
+			System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
+
+			final byte[] resultHolder = new byte[num * valueBytes.length];
+
+			final long beginGet = System.nanoTime();
+
+			final RocksIterator iterator = rocksDB.newIterator();
+			int pos = 0;
+
+			try {
+				// seek to start
+				unsafe.putInt(keyTemplate, offset, 0);
+				iterator.seek(keyTemplate);
+
+				// mark end
+				unsafe.putInt(keyTemplate, offset, -1);
+
+				// iterate
+				while (iterator.isValid()) {
+					byte[] currKey = iterator.key();
+					if (samePrefix(keyBytes, currKey)) {
+						byte[] currValue = iterator.value();
+						System.arraycopy(currValue, 0, resultHolder, pos, currValue.length);
+						pos += currValue.length;
+						iterator.next();
+					} else {
+						break;
+					}
+				}
+			}finally {
+				iterator.close();
 			}
-		}
 
-		final long endGet = System.nanoTime();
+			final long endGet = System.nanoTime();
 
-		System.out.println("end get - duration: " + ((endGet - beginGet) / 1_000_000) + " ms");
-
-		// WriteOptions and RocksDB ultimately extends AbstractNativeReference, so we need to close resource as well.
-		write_options.close();
-		rocksDB.close();
+			System.out.println("end get - duration: " + ((endGet - beginGet) / 1_000_000) + " ms");
+		} finally {
+			rocksDB.close();
+			options.close();
+			write_options.close();
+			FileUtils.deleteDirectory(rocksDir);
+		}
 	}
 
 	private static boolean samePrefix(byte[] prefix, byte[] key) {

From 474441feec8b40fb8534eac4c4ee70311c5a261c Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 21 Apr 2017 12:19:04 +0200
Subject: [PATCH 026/253] [hotfix] [rocksdb] Convert performance benchmarks to
 unit tests

---
 .../ListViaMergeSpeedMiniBenchmark.java       | 111 ----------
 .../ListViaRangeSpeedMiniBenchmark.java       | 132 ------------
 .../benchmark/RocksDBPerformanceTest.java     | 204 ++++++++++++++++++
 3 files changed, 204 insertions(+), 243 deletions(-)
 delete mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java
 delete mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java
 create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java
deleted file mode 100644
index f3e084f550497..0000000000000
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaMergeSpeedMiniBenchmark.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.contrib.streaming.state.benchmark;
-
-import org.apache.flink.util.FileUtils;
-import org.rocksdb.CompactionStyle;
-import org.rocksdb.Options;
-import org.rocksdb.RocksDB;
-import org.rocksdb.StringAppendOperator;
-import org.rocksdb.WriteOptions;
-
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-
-public class ListViaMergeSpeedMiniBenchmark {
-	
-	public static void main(String[] args) throws Exception {
-		final File rocksDir = new File("/tmp/rdb");
-		FileUtils.deleteDirectory(rocksDir);
-
-		final Options options = new Options()
-				.setCompactionStyle(CompactionStyle.LEVEL)
-				.setLevelCompactionDynamicLevelBytes(true)
-				.setIncreaseParallelism(4)
-				.setUseFsync(false)
-				.setMaxOpenFiles(-1)
-				.setDisableDataSync(true)
-				.setCreateIfMissing(true)
-				.setMergeOperator(new StringAppendOperator());
-
-		final WriteOptions write_options = new WriteOptions()
-				.setSync(false)
-				.setDisableWAL(true);
-
-		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath());
-
-		final String key = "key";
-		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
-
-		try {
-			final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
-			final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
-
-			final int num = 50000;
-
-			// ----- insert -----
-			System.out.println("begin insert");
-
-			final long beginInsert = System.nanoTime();
-			for (int i = 0; i < num; i++) {
-				rocksDB.merge(write_options, keyBytes, valueBytes);
-			}
-			final long endInsert = System.nanoTime();
-			System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
-
-			// ----- read (attempt 1) -----
-
-			final byte[] resultHolder = new byte[num * (valueBytes.length + 2)];
-			final long beginGet1 = System.nanoTime();
-			rocksDB.get(keyBytes, resultHolder);
-			final long endGet1 = System.nanoTime();
-
-			System.out.println("end get - duration: " + ((endGet1 - beginGet1) / 1_000_000) + " ms");
-
-			// ----- read (attempt 2) -----
-
-			final long beginGet2 = System.nanoTime();
-			rocksDB.get(keyBytes, resultHolder);
-			final long endGet2 = System.nanoTime();
-
-			System.out.println("end get - duration: " + ((endGet2 - beginGet2) / 1_000_000) + " ms");
-
-			// ----- compact -----
-			System.out.println("compacting...");
-			final long beginCompact = System.nanoTime();
-			rocksDB.compactRange();
-			final long endCompact = System.nanoTime();
-
-			System.out.println("end compaction - duration: " + ((endCompact - beginCompact) / 1_000_000) + " ms");
-
-			// ----- read (attempt 3) -----
-
-			final long beginGet3 = System.nanoTime();
-			rocksDB.get(keyBytes, resultHolder);
-			final long endGet3 = System.nanoTime();
-
-			System.out.println("end get - duration: " + ((endGet3 - beginGet3) / 1_000_000) + " ms");
-		} finally {
-			rocksDB.close();
-			options.close();
-			write_options.close();
-			FileUtils.deleteDirectory(rocksDir);
-		}
-	}
-}
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java
deleted file mode 100644
index f46e2cdb49d1f..0000000000000
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/ListViaRangeSpeedMiniBenchmark.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.contrib.streaming.state.benchmark;
-
-import org.apache.flink.core.memory.MemoryUtils;
-import org.apache.flink.util.FileUtils;
-import org.rocksdb.CompactionStyle;
-import org.rocksdb.Options;
-import org.rocksdb.RocksDB;
-import org.rocksdb.RocksIterator;
-import org.rocksdb.StringAppendOperator;
-import org.rocksdb.WriteOptions;
-import sun.misc.Unsafe;
-
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-
-public class ListViaRangeSpeedMiniBenchmark {
-	
-	public static void main(String[] args) throws Exception {
-		final File rocksDir = new File("/tmp/rdb");
-		FileUtils.deleteDirectory(rocksDir);
-
-		final Options options = new Options()
-				.setCompactionStyle(CompactionStyle.LEVEL)
-				.setLevelCompactionDynamicLevelBytes(true)
-				.setIncreaseParallelism(4)
-				.setUseFsync(false)
-				.setMaxOpenFiles(-1)
-				.setDisableDataSync(true)
-				.setCreateIfMissing(true)
-				.setMergeOperator(new StringAppendOperator());
-
-		final WriteOptions write_options = new WriteOptions()
-				.setSync(false)
-				.setDisableWAL(true);
-
-		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath());
-
-		final String key = "key";
-		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
-		
-		try {
-
-			final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
-			final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
-
-			final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4);
-
-			final Unsafe unsafe = MemoryUtils.UNSAFE;
-			final long offset = unsafe.arrayBaseOffset(byte[].class) + keyTemplate.length - 4;
-
-			final int num = 50000;
-			System.out.println("begin insert");
-
-			final long beginInsert = System.nanoTime();
-			for (int i = 0; i < num; i++) {
-				unsafe.putInt(keyTemplate, offset, i);
-				rocksDB.put(write_options, keyTemplate, valueBytes);
-			}
-			final long endInsert = System.nanoTime();
-			System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
-
-			final byte[] resultHolder = new byte[num * valueBytes.length];
-
-			final long beginGet = System.nanoTime();
-
-			final RocksIterator iterator = rocksDB.newIterator();
-			int pos = 0;
-
-			try {
-				// seek to start
-				unsafe.putInt(keyTemplate, offset, 0);
-				iterator.seek(keyTemplate);
-
-				// mark end
-				unsafe.putInt(keyTemplate, offset, -1);
-
-				// iterate
-				while (iterator.isValid()) {
-					byte[] currKey = iterator.key();
-					if (samePrefix(keyBytes, currKey)) {
-						byte[] currValue = iterator.value();
-						System.arraycopy(currValue, 0, resultHolder, pos, currValue.length);
-						pos += currValue.length;
-						iterator.next();
-					} else {
-						break;
-					}
-				}
-			}finally {
-				iterator.close();
-			}
-
-			final long endGet = System.nanoTime();
-
-			System.out.println("end get - duration: " + ((endGet - beginGet) / 1_000_000) + " ms");
-		} finally {
-			rocksDB.close();
-			options.close();
-			write_options.close();
-			FileUtils.deleteDirectory(rocksDir);
-		}
-	}
-
-	private static boolean samePrefix(byte[] prefix, byte[] key) {
-		for (int i = 0; i < prefix.length; i++) {
-			if (prefix[i] != key [i]) {
-				return false;
-			}
-		}
-
-		return true;
-	}
-}
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
new file mode 100644
index 0000000000000..011703ef01a6a
--- /dev/null
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state.benchmark;
+
+import org.apache.flink.core.memory.MemoryUtils;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.StringAppendOperator;
+import org.rocksdb.WriteOptions;
+import sun.misc.Unsafe;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+/**
+ * Test that validates that the performance of RocksDB is as expected.
+ * This test guards against the bug filed as 'FLINK-5756'
+ */
+public class RocksDBPerformanceTest extends TestLogger {
+
+	@Rule
+	public final TemporaryFolder TMP = new TemporaryFolder();
+
+	@Test(timeout = 2000)
+	public void testRocksDbMergePerformance() throws Exception {
+		final File rocksDir = TMP.newFolder("rdb");
+
+		final String key = "key";
+		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
+
+		final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+		final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
+
+		final int num = 50000;
+
+		try (
+			final Options options = new Options()
+					.setCompactionStyle(CompactionStyle.LEVEL)
+					.setLevelCompactionDynamicLevelBytes(true)
+					.setIncreaseParallelism(4)
+					.setUseFsync(false)
+					.setMaxOpenFiles(-1)
+					.setDisableDataSync(true)
+					.setCreateIfMissing(true)
+					.setMergeOperator(new StringAppendOperator());
+
+			final WriteOptions write_options = new WriteOptions()
+					.setSync(false)
+					.setDisableWAL(true);
+
+			final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath()))
+		{
+			// ----- insert -----
+			log.info("begin insert");
+
+			final long beginInsert = System.nanoTime();
+			for (int i = 0; i < num; i++) {
+				rocksDB.merge(write_options, keyBytes, valueBytes);
+			}
+			final long endInsert = System.nanoTime();
+			log.info("end insert - duration: {} ms", (endInsert - beginInsert) / 1_000_000);
+
+			// ----- read (attempt 1) -----
+
+			final byte[] resultHolder = new byte[num * (valueBytes.length + 2)];
+			final long beginGet1 = System.nanoTime();
+			rocksDB.get(keyBytes, resultHolder);
+			final long endGet1 = System.nanoTime();
+
+			log.info("end get - duration: {} ms", (endGet1 - beginGet1) / 1_000_000);
+
+			// ----- read (attempt 2) -----
+
+			final long beginGet2 = System.nanoTime();
+			rocksDB.get(keyBytes, resultHolder);
+			final long endGet2 = System.nanoTime();
+
+			log.info("end get - duration: {} ms", (endGet2 - beginGet2) / 1_000_000);
+
+			// ----- compact -----
+			log.info("compacting...");
+			final long beginCompact = System.nanoTime();
+			rocksDB.compactRange();
+			final long endCompact = System.nanoTime();
+
+			log.info("end compaction - duration: {} ms", (endCompact - beginCompact) / 1_000_000);
+
+			// ----- read (attempt 3) -----
+
+			final long beginGet3 = System.nanoTime();
+			rocksDB.get(keyBytes, resultHolder);
+			final long endGet3 = System.nanoTime();
+
+			log.info("end get - duration: {} ms", (endGet3 - beginGet3) / 1_000_000);
+		}
+	}
+
+	@Test(timeout = 2000)
+	public void testRocksDbRangeGetPerformance() throws Exception {
+		final File rocksDir = TMP.newFolder("rdb");
+
+		final String key = "key";
+		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
+
+		final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+		final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
+
+		final int num = 50000;
+
+		try (
+			final Options options = new Options()
+					.setCompactionStyle(CompactionStyle.LEVEL)
+					.setLevelCompactionDynamicLevelBytes(true)
+					.setIncreaseParallelism(4)
+					.setUseFsync(false)
+					.setMaxOpenFiles(-1)
+					.setDisableDataSync(true)
+					.setCreateIfMissing(true)
+					.setMergeOperator(new StringAppendOperator());
+
+			final WriteOptions write_options = new WriteOptions()
+					.setSync(false)
+					.setDisableWAL(true);
+
+			final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath()))
+		{
+			final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4);
+
+			final Unsafe unsafe = MemoryUtils.UNSAFE;
+			final long offset = unsafe.arrayBaseOffset(byte[].class) + keyTemplate.length - 4;
+
+			log.info("begin insert");
+
+			final long beginInsert = System.nanoTime();
+			for (int i = 0; i < num; i++) {
+				unsafe.putInt(keyTemplate, offset, i);
+				rocksDB.put(write_options, keyTemplate, valueBytes);
+			}
+			final long endInsert = System.nanoTime();
+			log.info("end insert - duration: {} ms", (endInsert - beginInsert) / 1_000_000);
+
+			@SuppressWarnings("MismatchedReadAndWriteOfArray")
+			final byte[] resultHolder = new byte[num * valueBytes.length];
+
+			final long beginGet = System.nanoTime();
+
+			int pos = 0;
+
+			try (final RocksIterator iterator = rocksDB.newIterator()) {
+				// seek to start
+				unsafe.putInt(keyTemplate, offset, 0);
+				iterator.seek(keyTemplate);
+
+				// iterate
+				while (iterator.isValid() && samePrefix(keyBytes, iterator.key())) {
+					byte[] currValue = iterator.value();
+					System.arraycopy(currValue, 0, resultHolder, pos, currValue.length);
+					pos += currValue.length;
+					iterator.next();
+				}
+			}
+
+			final long endGet = System.nanoTime();
+
+			log.info("end get - duration: {} ms", (endGet - beginGet) / 1_000_000);
+		}
+	}
+
+
+	private static boolean samePrefix(byte[] prefix, byte[] key) {
+		for (int i = 0; i < prefix.length; i++) {
+			if (prefix[i] != key [i]) {
+				return false;
+			}
+		}
+
+		return true;
+	}
+}

From 85a000386f9f1dc4f5fdcceabdc194296588da42 Mon Sep 17 00:00:00 2001
From: Dawid Wysakowicz <dawid@getindata.com>
Date: Mon, 10 Apr 2017 16:57:48 +0200
Subject: [PATCH 027/253] [FLINK-6290] [CEP] Fix SharedBuffer release when
 having multiple edges between entries

This closes #3706
---
 .../apache/flink/cep/nfa/SharedBuffer.java    |  5 ++--
 .../flink/cep/nfa/SharedBufferTest.java       | 28 +++++++++++++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java
index ccc68849473a1..43c2aca3ad73e 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java
@@ -274,7 +274,6 @@ public void lock(final K key, final V value, final long timestamp) {
 	public void release(final K key, final V value, final long timestamp) {
 		SharedBufferEntry<K, V> entry = get(key, value, timestamp);
 		if (entry != null) {
-			entry.decreaseReferenceCounter();
 			internalRemove(entry);
 		}
 	}
@@ -493,13 +492,13 @@ private void internalRemove(final SharedBufferEntry<K, V> entry) {
 
 		while (!entriesToRemove.isEmpty()) {
 			SharedBufferEntry<K, V> currentEntry = entriesToRemove.pop();
+			currentEntry.decreaseReferenceCounter();
 
 			if (currentEntry.getReferenceCounter() == 0) {
 				currentEntry.remove();
 
-				for (SharedBufferEdge<K, V> edge: currentEntry.getEdges()) {
+				for (SharedBufferEdge<K, V> edge : currentEntry.getEdges()) {
 					if (edge.getTarget() != null) {
-						edge.getTarget().decreaseReferenceCounter();
 						entriesToRemove.push(edge.getTarget());
 					}
 				}
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SharedBufferTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SharedBufferTest.java
index f0a25d266bda2..adc07b32ad156 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SharedBufferTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SharedBufferTest.java
@@ -32,6 +32,7 @@
 import java.util.Collections;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class SharedBufferTest extends TestLogger {
@@ -138,4 +139,31 @@ public void testSharedBufferSerialization() throws IOException, ClassNotFoundExc
 
 		assertEquals(sharedBuffer, copy);
 	}
+
+	@Test
+	public void testClearingSharedBufferWithMultipleEdgesBetweenEntries() {
+		SharedBuffer<String, Event> sharedBuffer = new SharedBuffer<>(Event.createTypeSerializer());
+		int numberEvents = 8;
+		Event[] events = new Event[numberEvents];
+		final long timestamp = 1L;
+
+		for (int i = 0; i < numberEvents; i++) {
+			events[i] = new Event(i + 1, "e" + (i + 1), i);
+		}
+
+		sharedBuffer.put("start", events[1], timestamp, DeweyNumber.fromString("1"));
+		sharedBuffer.put("branching", events[2], timestamp, "start", events[1], timestamp, DeweyNumber.fromString("1.0"));
+		sharedBuffer.put("branching", events[3], timestamp, "start", events[1], timestamp, DeweyNumber.fromString("1.1"));
+		sharedBuffer.put("branching", events[3], timestamp, "branching", events[2], timestamp, DeweyNumber.fromString("1.0.0"));
+		sharedBuffer.put("branching", events[4], timestamp, "branching", events[3], timestamp, DeweyNumber.fromString("1.0.0.0"));
+		sharedBuffer.put("branching", events[4], timestamp, "branching", events[3], timestamp, DeweyNumber.fromString("1.1.0"));
+
+		//simulate IGNORE (next event can point to events[2])
+		sharedBuffer.lock("branching", events[2], timestamp);
+
+		sharedBuffer.release("branching", events[4], timestamp);
+
+		//There should be still events[1] and events[2] in the buffer
+		assertFalse(sharedBuffer.isEmpty());
+	}
 }

From 73ea80a9b0b5be89a9c5024467c7b216c96e5223 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 6 Apr 2017 13:47:33 +0200
Subject: [PATCH 028/253] [FLINK-4769] [metrics] Port metric config parameters
 to ConfigOptions

This closes #3687
---
 .../connectors/kafka/KafkaTestBase.java       |  3 +-
 .../flink/configuration/ConfigConstants.java  | 44 +++++-----
 .../flink/configuration/MetricOptions.java    | 88 +++++++++++++++++++
 .../ScheduledDropwizardReporterTest.java      |  7 +-
 .../DropwizardFlinkHistogramWrapperTest.java  |  3 +-
 .../flink/metrics/jmx/JMXReporterTest.java    |  9 +-
 .../jobmanager/JMXJobManagerMetricTest.java   |  5 +-
 .../metrics/statsd/StatsDReporterTest.java    | 11 +--
 .../metrics/MetricRegistryConfiguration.java  | 24 ++---
 .../runtime/metrics/scope/ScopeFormat.java    | 49 -----------
 .../runtime/metrics/scope/ScopeFormats.java   | 32 +++----
 .../runtime/metrics/MetricRegistryTest.java   | 29 +++---
 .../groups/AbstractMetricGroupTest.java       |  7 +-
 .../metrics/groups/JobManagerGroupTest.java   |  4 +-
 .../groups/JobManagerJobGroupTest.java        | 10 +--
 .../groups/MetricGroupRegistrationTest.java   |  3 +-
 .../metrics/groups/TaskManagerGroupTest.java  |  4 +-
 .../groups/TaskManagerJobGroupTest.java       | 10 +--
 .../metrics/groups/TaskMetricGroupTest.java   | 10 +--
 .../api/operators/AbstractStreamOperator.java |  8 +-
 20 files changed, 196 insertions(+), 164 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java

diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
index 0c6bfa9a1f1ca..a21a2399cf83c 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
@@ -20,6 +20,7 @@
 import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.metrics.jmx.JMXReporter;
 import org.apache.flink.runtime.client.JobExecutionException;
@@ -116,7 +117,7 @@ protected static Configuration getFlinkConfiguration() {
 		flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
 		flinkConfig.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L);
 		flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
-		flinkConfig.setString(ConfigConstants.METRICS_REPORTERS_LIST, "my_reporter");
+		flinkConfig.setString(MetricOptions.REPORTERS_LIST, "my_reporter");
 		flinkConfig.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "my_reporter." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
 		return flinkConfig;
 	}
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index a035beb9a2dc7..61c1b27ecbeed 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -990,21 +990,8 @@ public final class ConfigConstants {
 
 	// ---------------------------- Metrics -----------------------------------
 
-	/**
-	 * The list of named reporters. Names are defined here and per-reporter configs
-	 * are given with the reporter config prefix and the reporter name.
-	 *
-	 * Example:
-	 * <pre>{@code
-	 * metrics.reporters = foo, bar
-	 *
-	 * metrics.reporter.foo.class = org.apache.flink.metrics.reporter.JMXReporter
-	 * metrics.reporter.foo.interval = 10
-	 *
-	 * metrics.reporter.bar.class = org.apache.flink.metrics.graphite.GraphiteReporter
-	 * metrics.reporter.bar.port = 1337
-	 * }</pre>
-	 */
+	/** @deprecated Use {@link MetricOptions#REPORTERS_LIST} instead. */
+	@Deprecated
 	public static final String METRICS_REPORTERS_LIST = "metrics.reporters";
 
 	/**
@@ -1022,28 +1009,36 @@ public final class ConfigConstants {
 	/**	The delimiter used to assemble the metric identifier. This is used as a suffix in an actual reporter config. */
 	public static final String METRICS_REPORTER_SCOPE_DELIMITER = "scope.delimiter";
 
-	/** The delimiter used to assemble the metric identifier. */
+	/** @deprecated Use {@link MetricOptions#SCOPE_DELIMITER} instead. */
+	@Deprecated
 	public static final String METRICS_SCOPE_DELIMITER = "metrics.scope.delimiter";
 
-	/** The scope format string that is applied to all metrics scoped to a JobManager. */
+	/** @deprecated Use {@link MetricOptions#SCOPE_NAMING_JM} instead. */
+	@Deprecated
 	public static final String METRICS_SCOPE_NAMING_JM = "metrics.scope.jm";
 
-	/** The scope format string that is applied to all metrics scoped to a TaskManager. */
+	/** @deprecated Use {@link MetricOptions#SCOPE_NAMING_TM} instead. */
+	@Deprecated
 	public static final String METRICS_SCOPE_NAMING_TM = "metrics.scope.tm";
 
-	/** The scope format string that is applied to all metrics scoped to a job on a JobManager. */
+	/** @deprecated Use {@link MetricOptions#SCOPE_NAMING_JM_JOB} instead. */
+	@Deprecated
 	public static final String METRICS_SCOPE_NAMING_JM_JOB = "metrics.scope.jm.job";
 
-	/** The scope format string that is applied to all metrics scoped to a job on a TaskManager. */
+	/** @deprecated Use {@link MetricOptions#SCOPE_NAMING_TM_JOB} instead. */
+	@Deprecated
 	public static final String METRICS_SCOPE_NAMING_TM_JOB = "metrics.scope.tm.job";
 
-	/** The scope format string that is applied to all metrics scoped to a task. */
+	/** @deprecated Use {@link MetricOptions#SCOPE_NAMING_TASK} instead. */
+	@Deprecated
 	public static final String METRICS_SCOPE_NAMING_TASK = "metrics.scope.task";
 
-	/** The scope format string that is applied to all metrics scoped to an operator. */
+	/** @deprecated Use {@link MetricOptions#SCOPE_NAMING_OPERATOR} instead. */
+	@Deprecated
 	public static final String METRICS_SCOPE_NAMING_OPERATOR = "metrics.scope.operator";
 
-	/** The number of measured latencies to maintain at each operator */
+	/** @deprecated Use {@link MetricOptions#LATENCY_HISTORY_SIZE} instead. */
+	@Deprecated
 	public static final String METRICS_LATENCY_HISTORY_SIZE = "metrics.latency.history-size";
 
 
@@ -1498,7 +1493,8 @@ public final class ConfigConstants {
 
 	// ----------------------------- Metrics ----------------------------
 
-	/** The default number of measured latencies to maintain at each operator */
+	/** @deprecated Use {@link MetricOptions#LATENCY_HISTORY_SIZE} instead. */
+	@Deprecated
 	public static final int DEFAULT_METRICS_LATENCY_HISTORY_SIZE = 128;
 
 	// ----------------------------- Environment Variables ----------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java
new file mode 100644
index 0000000000000..8a328acdb12b6
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import static org.apache.flink.configuration.ConfigOptions.key;
+
+@PublicEvolving
+public class MetricOptions {
+
+	/**
+	 * The list of named reporters. Names are defined here and per-reporter configs
+	 * are given with the reporter config prefix and the reporter name.
+	 *
+	 * Example:
+	 * <pre>{@code
+	 * metrics.reporters = foo, bar
+	 *
+	 * metrics.reporter.foo.class = org.apache.flink.metrics.reporter.JMXReporter
+	 * metrics.reporter.foo.interval = 10
+	 *
+	 * metrics.reporter.bar.class = org.apache.flink.metrics.graphite.GraphiteReporter
+	 * metrics.reporter.bar.port = 1337
+	 * }</pre>
+	 */
+	public static final ConfigOption<String> REPORTERS_LIST =
+		key("metrics.reporters")
+			.noDefaultValue();
+
+	/** The delimiter used to assemble the metric identifier. */
+	public static final ConfigOption<String> SCOPE_DELIMITER =
+		key("metrics.scope.delimiter")
+			.defaultValue(".");
+
+	/** The scope format string that is applied to all metrics scoped to a JobManager. */
+	public static final ConfigOption<String> SCOPE_NAMING_JM =
+		key("metrics.scope.jm")
+			.defaultValue("<host>.jobmanager");
+
+	/** The scope format string that is applied to all metrics scoped to a TaskManager. */
+	public static final ConfigOption<String> SCOPE_NAMING_TM =
+		key("metrics.scope.tm")
+			.defaultValue("<host>.taskmanager.<tm_id>");
+
+	/** The scope format string that is applied to all metrics scoped to a job on a JobManager. */
+	public static final ConfigOption<String> SCOPE_NAMING_JM_JOB =
+		key("metrics.scope.jm.job")
+			.defaultValue("<host>.jobmanager.<job_name>");
+
+	/** The scope format string that is applied to all metrics scoped to a job on a TaskManager. */
+	public static final ConfigOption<String> SCOPE_NAMING_TM_JOB =
+		key("metrics.scope.tm.job")
+			.defaultValue("<host>.taskmanager.<tm_id>.<job_name>");
+
+	/** The scope format string that is applied to all metrics scoped to a task. */
+	public static final ConfigOption<String> SCOPE_NAMING_TASK =
+		key("metrics.scope.task")
+			.defaultValue("<host>.taskmanager.<tm_id>.<job_name>.<task_name>.<subtask_index>");
+
+	/** The scope format string that is applied to all metrics scoped to an operator. */
+	public static final ConfigOption<String> SCOPE_NAMING_OPERATOR =
+		key("metrics.scope.operator")
+			.defaultValue("<host>.taskmanager.<tm_id>.<job_name>.<operator_name>.<subtask_index>");
+
+	/** The number of measured latencies to maintain at each operator */
+	public static final ConfigOption<Integer> LATENCY_HISTORY_SIZE =
+		key("metrics.latency.history-size")
+			.defaultValue(128);
+
+	private MetricOptions() {
+	}
+}
diff --git a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java
index 87c4ccb4c3afa..73e7f0b4f6cbc 100644
--- a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java
+++ b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java
@@ -22,6 +22,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Gauge;
@@ -76,13 +77,13 @@ public void testAddingMetrics() throws NoSuchFieldException, IllegalAccessExcept
 		String taskManagerId = "tas:kMana::ger";
 		String counterName = "testCounter";
 
-		configuration.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+		configuration.setString(MetricOptions.REPORTERS_LIST, "test");
 		configuration.setString(
 				ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX,
 				"org.apache.flink.dropwizard.ScheduledDropwizardReporterTest$TestingScheduledDropwizardReporter");
 
-		configuration.setString(ConfigConstants.METRICS_SCOPE_NAMING_TASK, "<host>.<tm_id>.<job_name>");
-		configuration.setString(ConfigConstants.METRICS_SCOPE_DELIMITER, "_");
+		configuration.setString(MetricOptions.SCOPE_NAMING_TASK, "<host>.<tm_id>.<job_name>");
+		configuration.setString(MetricOptions.SCOPE_DELIMITER, "_");
 
 		MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(configuration);
 
diff --git a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/metrics/DropwizardFlinkHistogramWrapperTest.java b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/metrics/DropwizardFlinkHistogramWrapperTest.java
index 2f2a536cc138a..8f7796ce0bf84 100644
--- a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/metrics/DropwizardFlinkHistogramWrapperTest.java
+++ b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/metrics/DropwizardFlinkHistogramWrapperTest.java
@@ -28,6 +28,7 @@
 import com.codahale.metrics.Timer;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.dropwizard.ScheduledDropwizardReporter;
 import org.apache.flink.metrics.MetricConfig;
 import org.apache.flink.metrics.reporter.MetricReporter;
@@ -96,7 +97,7 @@ public void testDropwizardHistogramWrapperReporting() throws Exception {
 		int size = 10;
 		String histogramMetricName = "histogram";
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "my_reporter");
+		config.setString(MetricOptions.REPORTERS_LIST, "my_reporter");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "my_reporter." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestingReporter.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "my_reporter." + ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, reportingInterval + " MILLISECONDS");
 
diff --git a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/metrics/jmx/JMXReporterTest.java b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/metrics/jmx/JMXReporterTest.java
index 1a9628766101d..85ab897551bff 100644
--- a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/metrics/jmx/JMXReporterTest.java
+++ b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/metrics/jmx/JMXReporterTest.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.metrics.util.TestMeter;
 import org.apache.flink.metrics.reporter.MetricReporter;
@@ -94,7 +95,7 @@ public void testGenerateTable() {
 	@Test
 	public void testPortConflictHandling() throws Exception {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
+		cfg.setString(MetricOptions.REPORTERS_LIST, "test1,test2");
 
 		cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
 		cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1.port", "9020-9035");
@@ -154,7 +155,7 @@ public void testJMXAvailability() throws Exception {
 		Configuration cfg = new Configuration();
 		cfg.setString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter.class.getName());
 
-		cfg.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
+		cfg.setString(MetricOptions.REPORTERS_LIST, "test1,test2");
 
 		cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
 		cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1.port", "9040-9055");
@@ -230,7 +231,7 @@ public void testHistogramReporting() throws Exception {
 
 		try {
 			Configuration config = new Configuration();
-			config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "jmx_test");
+			config.setString(MetricOptions.REPORTERS_LIST, "jmx_test");
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "jmx_test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
 
 			registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
@@ -280,7 +281,7 @@ public void testMeterReporting() throws Exception {
 
 		try {
 			Configuration config = new Configuration();
-			config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "jmx_test");
+			config.setString(MetricOptions.REPORTERS_LIST, "jmx_test");
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "jmx_test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
 
 			registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
diff --git a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java
index 934a6212019ac..6d54db3ad8b71 100644
--- a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java
+++ b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java
@@ -19,6 +19,7 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.metrics.jmx.JMXReporter;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -55,11 +56,11 @@ public void testJobManagerJMXMetricAccess() throws Exception {
 		Deadline deadline = new FiniteDuration(2, TimeUnit.MINUTES).fromNow();
 		Configuration flinkConfiguration = new Configuration();
 
-		flinkConfiguration.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+		flinkConfiguration.setString(MetricOptions.REPORTERS_LIST, "test");
 		flinkConfiguration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
 		flinkConfiguration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.port", "9060-9075");
 
-		flinkConfiguration.setString(ConfigConstants.METRICS_SCOPE_NAMING_JM_JOB, "jobmanager.<job_name>");
+		flinkConfiguration.setString(MetricOptions.SCOPE_NAMING_JM_JOB, "jobmanager.<job_name>");
 
 		TestingCluster flink = new TestingCluster(flinkConfiguration);
 
diff --git a/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java b/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java
index 17fd65ac5e7bd..9215c3f27210f 100644
--- a/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java
+++ b/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java
@@ -21,6 +21,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Histogram;
 import org.apache.flink.metrics.HistogramStatistics;
@@ -75,13 +76,13 @@ public void testAddingMetrics() throws NoSuchFieldException, IllegalAccessExcept
 		String taskManagerId = "tas:kMana::ger";
 		String counterName = "testCounter";
 
-		configuration.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+		configuration.setString(MetricOptions.REPORTERS_LIST, "test");
 		configuration.setString(
 				ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX,
 				"org.apache.flink.metrics.statsd.StatsDReporterTest$TestingStatsDReporter");
 
-		configuration.setString(ConfigConstants.METRICS_SCOPE_NAMING_TASK, "<host>.<tm_id>.<job_name>");
-		configuration.setString(ConfigConstants.METRICS_SCOPE_DELIMITER, "_");
+		configuration.setString(MetricOptions.SCOPE_NAMING_TASK, "<host>.<tm_id>.<job_name>");
+		configuration.setString(MetricOptions.SCOPE_DELIMITER, "_");
 
 		MetricRegistry metricRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(configuration));
 
@@ -145,7 +146,7 @@ public void testStatsDHistogramReporting() throws Exception {
 			int port = receiver.getPort();
 
 			Configuration config = new Configuration();
-			config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+			config.setString(MetricOptions.REPORTERS_LIST, "test");
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, StatsDReporter.class.getName());
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, "1 SECONDS");
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.host", "localhost");
@@ -219,7 +220,7 @@ public void testStatsDMetersReporting() throws Exception {
 			int port = receiver.getPort();
 
 			Configuration config = new Configuration();
-			config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+			config.setString(MetricOptions.REPORTERS_LIST, "test");
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, StatsDReporter.class.getName());
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, "1 SECONDS");
 			config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.host", "localhost");
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java
index 596fc82dfe2e4..3475f049e372c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java
@@ -22,7 +22,7 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.DelegatingConfiguration;
-import org.apache.flink.runtime.metrics.scope.ScopeFormat;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.runtime.metrics.scope.ScopeFormats;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
@@ -101,13 +101,13 @@ public static MetricRegistryConfiguration fromConfiguration(Configuration config
 
 		char delim;
 		try {
-			delim = configuration.getString(ConfigConstants.METRICS_SCOPE_DELIMITER, ".").charAt(0);
+			delim = configuration.getString(MetricOptions.SCOPE_DELIMITER).charAt(0);
 		} catch (Exception e) {
 			LOG.warn("Failed to parse delimiter, using default delimiter.", e);
 			delim = '.';
 		}
 
-		final String definedReporters = configuration.getString(ConfigConstants.METRICS_REPORTERS_LIST, null);
+		final String definedReporters = configuration.getString(MetricOptions.REPORTERS_LIST);
 		List<Tuple2<String, Configuration>> reporterConfigurations;
 
 		if (definedReporters == null) {
@@ -136,18 +136,12 @@ public static MetricRegistryConfiguration fromConfiguration(Configuration config
 	 * @return Scope formats extracted from the given configuration
 	 */
 	static ScopeFormats createScopeConfig(Configuration configuration) {
-		String jmFormat = configuration.getString(
-			ConfigConstants.METRICS_SCOPE_NAMING_JM, ScopeFormat.DEFAULT_SCOPE_JOBMANAGER_GROUP);
-		String jmJobFormat = configuration.getString(
-			ConfigConstants.METRICS_SCOPE_NAMING_JM_JOB, ScopeFormat.DEFAULT_SCOPE_JOBMANAGER_JOB_GROUP);
-		String tmFormat = configuration.getString(
-			ConfigConstants.METRICS_SCOPE_NAMING_TM, ScopeFormat.DEFAULT_SCOPE_TASKMANAGER_GROUP);
-		String tmJobFormat = configuration.getString(
-			ConfigConstants.METRICS_SCOPE_NAMING_TM_JOB, ScopeFormat.DEFAULT_SCOPE_TASKMANAGER_JOB_GROUP);
-		String taskFormat = configuration.getString(
-			ConfigConstants.METRICS_SCOPE_NAMING_TASK, ScopeFormat.DEFAULT_SCOPE_TASK_GROUP);
-		String operatorFormat = configuration.getString(
-			ConfigConstants.METRICS_SCOPE_NAMING_OPERATOR, ScopeFormat.DEFAULT_SCOPE_OPERATOR_GROUP);
+		String jmFormat = configuration.getString(MetricOptions.SCOPE_NAMING_JM);
+		String jmJobFormat = configuration.getString(MetricOptions.SCOPE_NAMING_JM_JOB);
+		String tmFormat = configuration.getString(MetricOptions.SCOPE_NAMING_TM);
+		String tmJobFormat = configuration.getString(MetricOptions.SCOPE_NAMING_TM_JOB);
+		String taskFormat = configuration.getString(MetricOptions.SCOPE_NAMING_TASK);
+		String operatorFormat = configuration.getString(MetricOptions.SCOPE_NAMING_OPERATOR);
 
 		return new ScopeFormats(jmFormat, jmJobFormat, tmFormat, tmJobFormat, taskFormat, operatorFormat);
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormat.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormat.java
index adf0a85d23c36..f9efb882cee49 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormat.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormat.java
@@ -70,46 +70,15 @@ public String filterCharacters(String input) {
 
 	public static final String SCOPE_HOST = asVariable("host");
 
-	// ----- Job Manager ----
-
-	/** The default scope format of the JobManager component: {@code "<host>.jobmanager"} */
-	public static final String DEFAULT_SCOPE_JOBMANAGER_COMPONENT =
-		concat(SCOPE_HOST, "jobmanager");
-
-	/** The default scope format of JobManager metrics: {@code "<host>.jobmanager"} */
-	public static final String DEFAULT_SCOPE_JOBMANAGER_GROUP = DEFAULT_SCOPE_JOBMANAGER_COMPONENT;
-
 	// ----- Task Manager ----
 
 	public static final String SCOPE_TASKMANAGER_ID = asVariable("tm_id");
 
-	/** The default scope format of the TaskManager component: {@code "<host>.taskmanager.<tm_id>"} */
-	public static final String DEFAULT_SCOPE_TASKMANAGER_COMPONENT =
-			concat(SCOPE_HOST, "taskmanager", SCOPE_TASKMANAGER_ID);
-
-	/** The default scope format of TaskManager metrics: {@code "<host>.taskmanager.<tm_id>"} */
-	public static final String DEFAULT_SCOPE_TASKMANAGER_GROUP = DEFAULT_SCOPE_TASKMANAGER_COMPONENT;
-
 	// ----- Job -----
 
 	public static final String SCOPE_JOB_ID = asVariable("job_id");
 	public static final String SCOPE_JOB_NAME = asVariable("job_name");
 
-	/** The default scope format for the job component: {@code "<job_name>"} */
-	public static final String DEFAULT_SCOPE_JOB_COMPONENT = SCOPE_JOB_NAME;
-
-	// ----- Job on Job Manager ----
-
-	/** The default scope format for all job metrics on a jobmanager: {@code "<host>.jobmanager.<job_name>"} */
-	public static final String DEFAULT_SCOPE_JOBMANAGER_JOB_GROUP =
-		concat(DEFAULT_SCOPE_JOBMANAGER_COMPONENT, DEFAULT_SCOPE_JOB_COMPONENT);
-
-	// ----- Job on Task Manager ----
-
-	/** The default scope format for all job metrics on a taskmanager: {@code "<host>.taskmanager.<tm_id>.<job_name>"} */
-	public static final String DEFAULT_SCOPE_TASKMANAGER_JOB_GROUP =
-			concat(DEFAULT_SCOPE_TASKMANAGER_COMPONENT, DEFAULT_SCOPE_JOB_COMPONENT);
-
 	// ----- Task ----
 
 	public static final String SCOPE_TASK_VERTEX_ID = asVariable("task_id");
@@ -118,27 +87,9 @@ public String filterCharacters(String input) {
 	public static final String SCOPE_TASK_ATTEMPT_NUM = asVariable("task_attempt_num");
 	public static final String SCOPE_TASK_SUBTASK_INDEX = asVariable("subtask_index");
 
-	/** Default scope of the task component: {@code "<task_name>.<subtask_index>"} */
-	public static final String DEFAULT_SCOPE_TASK_COMPONENT =
-			concat(SCOPE_TASK_NAME, SCOPE_TASK_SUBTASK_INDEX);
-
-	/** The default scope format for all task metrics:
-	 * {@code "<host>.taskmanager.<tm_id>.<job_name>.<task_name>.<subtask_index>"} */
-	public static final String DEFAULT_SCOPE_TASK_GROUP =
-			concat(DEFAULT_SCOPE_TASKMANAGER_JOB_GROUP, DEFAULT_SCOPE_TASK_COMPONENT);
-
 	// ----- Operator ----
 
 	public static final String SCOPE_OPERATOR_NAME = asVariable("operator_name");
-
-	/** The default scope added by the operator component: "<operator_name>.<subtask_index>" */
-	public static final String DEFAULT_SCOPE_OPERATOR_COMPONENT =
-			concat(SCOPE_OPERATOR_NAME, SCOPE_TASK_SUBTASK_INDEX);
-
-	/** The default scope format for all operator metrics:
-	 * {@code "<host>.taskmanager.<tm_id>.<job_name>.<operator_name>.<subtask_index>"} */
-	public static final String DEFAULT_SCOPE_OPERATOR_GROUP =
-			concat(DEFAULT_SCOPE_TASKMANAGER_JOB_GROUP, DEFAULT_SCOPE_OPERATOR_COMPONENT);
 	
 
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormats.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormats.java
index bbbe6ba6a2b02..bde93beaa3e31 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormats.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/scope/ScopeFormats.java
@@ -18,8 +18,8 @@
 
 package org.apache.flink.runtime.metrics.scope;
 
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
@@ -41,21 +41,21 @@ public class ScopeFormats {
 	 * Creates all default scope formats.
 	 */
 	public ScopeFormats() {
-		this.jobManagerFormat = new JobManagerScopeFormat(ScopeFormat.DEFAULT_SCOPE_JOBMANAGER_COMPONENT);
+		this.jobManagerFormat = new JobManagerScopeFormat(MetricOptions.SCOPE_NAMING_JM.defaultValue());
 
 		this.jobManagerJobFormat = new JobManagerJobScopeFormat(
-				ScopeFormat.DEFAULT_SCOPE_JOBMANAGER_JOB_GROUP, this.jobManagerFormat);
+			MetricOptions.SCOPE_NAMING_JM_JOB.defaultValue(), this.jobManagerFormat);
 
-		this.taskManagerFormat = new TaskManagerScopeFormat(ScopeFormat.DEFAULT_SCOPE_TASKMANAGER_COMPONENT);
+		this.taskManagerFormat = new TaskManagerScopeFormat(MetricOptions.SCOPE_NAMING_TM.defaultValue());
 
 		this.taskManagerJobFormat = new TaskManagerJobScopeFormat(
-				ScopeFormat.DEFAULT_SCOPE_TASKMANAGER_JOB_GROUP, this.taskManagerFormat);
+			MetricOptions.SCOPE_NAMING_TM_JOB.defaultValue(), this.taskManagerFormat);
 
 		this.taskFormat = new TaskScopeFormat(
-				ScopeFormat.DEFAULT_SCOPE_TASK_GROUP, this.taskManagerJobFormat);
+				MetricOptions.SCOPE_NAMING_TASK.defaultValue(), this.taskManagerJobFormat);
 
 		this.operatorFormat = new OperatorScopeFormat(
-				ScopeFormat.DEFAULT_SCOPE_OPERATOR_GROUP, this.taskFormat);
+			MetricOptions.SCOPE_NAMING_OPERATOR.defaultValue(), this.taskFormat);
 	}
 
 	/**
@@ -135,18 +135,12 @@ public OperatorScopeFormat getOperatorFormat() {
 	 * @return The ScopeFormats parsed from the configuration
 	 */
 	public static ScopeFormats fromConfig(Configuration config) {
-		String jmFormat = config.getString(
-				ConfigConstants.METRICS_SCOPE_NAMING_JM, ScopeFormat.DEFAULT_SCOPE_JOBMANAGER_GROUP);
-		String jmJobFormat = config.getString(
-				ConfigConstants.METRICS_SCOPE_NAMING_JM_JOB, ScopeFormat.DEFAULT_SCOPE_JOBMANAGER_JOB_GROUP);
-		String tmFormat = config.getString(
-				ConfigConstants.METRICS_SCOPE_NAMING_TM, ScopeFormat.DEFAULT_SCOPE_TASKMANAGER_GROUP);
-		String tmJobFormat = config.getString(
-				ConfigConstants.METRICS_SCOPE_NAMING_TM_JOB, ScopeFormat.DEFAULT_SCOPE_TASKMANAGER_JOB_GROUP);
-		String taskFormat = config.getString(
-				ConfigConstants.METRICS_SCOPE_NAMING_TASK, ScopeFormat.DEFAULT_SCOPE_TASK_GROUP);
-		String operatorFormat = config.getString(
-				ConfigConstants.METRICS_SCOPE_NAMING_OPERATOR, ScopeFormat.DEFAULT_SCOPE_OPERATOR_GROUP);
+		String jmFormat = config.getString(MetricOptions.SCOPE_NAMING_JM);
+		String jmJobFormat = config.getString(MetricOptions.SCOPE_NAMING_JM_JOB);
+		String tmFormat = config.getString(MetricOptions.SCOPE_NAMING_TM);
+		String tmJobFormat = config.getString(MetricOptions.SCOPE_NAMING_TM_JOB);
+		String taskFormat = config.getString(MetricOptions.SCOPE_NAMING_TASK);
+		String operatorFormat = config.getString(MetricOptions.SCOPE_NAMING_OPERATOR);
 
 		return new ScopeFormats(jmFormat, jmJobFormat, tmFormat, tmJobFormat, taskFormat, operatorFormat);
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryTest.java
index fe29ccb7df92a..b9502b21c90af 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryTest.java
@@ -23,6 +23,7 @@
 import akka.actor.ActorSystem;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Metric;
 import org.apache.flink.metrics.MetricConfig;
@@ -69,7 +70,7 @@ public void testIsShutdown() {
 	public void testReporterInstantiation() {
 		Configuration config = new Configuration();
 
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+		config.setString(MetricOptions.REPORTERS_LIST, "test");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());
 
 		MetricRegistry metricRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
@@ -97,7 +98,7 @@ public void open(MetricConfig config) {
 	public void testMultipleReporterInstantiation() {
 		Configuration config = new Configuration();
 
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1, test2,test3");
+		config.setString(MetricOptions.REPORTERS_LIST, "test1, test2,test3");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter11.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter12.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test3." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter13.class.getName());
@@ -147,7 +148,7 @@ public void open(MetricConfig config) {
 	public void testReporterArgumentForwarding() {
 		Configuration config = new Configuration();
 
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+		config.setString(MetricOptions.REPORTERS_LIST, "test");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter2.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.arg1", "hello");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.arg2", "world");
@@ -172,7 +173,7 @@ public void open(MetricConfig config) {
 	public void testReporterScheduling() throws InterruptedException {
 		Configuration config = new Configuration();
 
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+		config.setString(MetricOptions.REPORTERS_LIST, "test");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter3.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.arg1", "hello");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, "50 MILLISECONDS");
@@ -215,7 +216,7 @@ public void report() {
 	@Test
 	public void testReporterNotifications() {
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
+		config.setString(MetricOptions.REPORTERS_LIST, "test1,test2");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter6.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter7.class.getName());
 
@@ -278,10 +279,10 @@ public void notifyOfRemovedMetric(Metric metric, String metricName, MetricGroup
 	public void testScopeConfig() {
 		Configuration config = new Configuration();
 
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A");
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM_JOB, "B");
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TASK, "C");
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_OPERATOR, "D");
+		config.setString(MetricOptions.SCOPE_NAMING_TM, "A");
+		config.setString(MetricOptions.SCOPE_NAMING_TM_JOB, "B");
+		config.setString(MetricOptions.SCOPE_NAMING_TASK, "C");
+		config.setString(MetricOptions.SCOPE_NAMING_OPERATOR, "D");
 
 		ScopeFormats scopeConfig = MetricRegistryConfiguration.createScopeConfig(config);
 
@@ -294,8 +295,8 @@ public void testScopeConfig() {
 	@Test
 	public void testConfigurableDelimiter() {
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_SCOPE_DELIMITER, "_");
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A.B.C.D.E");
+		config.setString(MetricOptions.SCOPE_DELIMITER, "_");
+		config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B.C.D.E");
 
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
 
@@ -308,7 +309,7 @@ public void testConfigurableDelimiter() {
 	@Test
 	public void testConfigurableDelimiterForReporters() {
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2,test3");
+		config.setString(MetricOptions.REPORTERS_LIST, "test1,test2,test3");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "_");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
@@ -331,7 +332,7 @@ public void testConfigurableDelimiterForReporters() {
 	@Test
 	public void testConfigurableDelimiterForReportersInGroup() {
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2,test3,test4");
+		config.setString(MetricOptions.REPORTERS_LIST, "test1,test2,test3,test4");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "_");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter8.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
@@ -339,7 +340,7 @@ public void testConfigurableDelimiterForReportersInGroup() {
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test3." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "AA");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test3." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter8.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test4." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter8.class.getName());
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A.B");
+		config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B");
 
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
 		List<MetricReporter> reporters = registry.getReporters();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java
index da14bfdd9ba27..04e40aeeebffd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java
@@ -19,6 +19,7 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.metrics.CharacterFilter;
 import org.apache.flink.metrics.Metric;
 import org.apache.flink.metrics.MetricGroup;
@@ -77,8 +78,8 @@ public String filterCharacters(String input) {
 	@Test
 	public void testScopeCachingForMultipleReporters() throws Exception {
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A.B.C.D");
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test1,test2");
+		config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B.C.D");
+		config.setString(MetricOptions.REPORTERS_LIST, "test1,test2");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter2.class.getName());
@@ -166,7 +167,7 @@ public String filterCharacters(String input) {
 	@Test
 	public void testScopeGenerationWithoutReporters() {
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "A.B.C.D");
+		config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B.C.D");
 		MetricRegistry testRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
 
 		try {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerGroupTest.java
index 317d19eadeda4..7834755220460 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerGroupTest.java
@@ -18,8 +18,8 @@
 package org.apache.flink.runtime.metrics.groups;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
@@ -110,7 +110,7 @@ public void testGenerateScopeDefault() {
 	@Test
 	public void testGenerateScopeCustom() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_JM, "constant.<host>.foo.<host>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_JM, "constant.<host>.foo.<host>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 
 		JobManagerMetricGroup group = new JobManagerMetricGroup(registry, "host");
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerJobGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerJobGroupTest.java
index ed4056f6f04b0..d8bd57aa1ff82 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerJobGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/JobManagerJobGroupTest.java
@@ -19,8 +19,8 @@
 package org.apache.flink.runtime.metrics.groups;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
 import org.apache.flink.runtime.metrics.dump.QueryScopeInfo;
@@ -54,8 +54,8 @@ public void testGenerateScopeDefault() {
 	@Test
 	public void testGenerateScopeCustom() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_JM, "abc");
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_JM_JOB, "some-constant.<job_name>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_JM, "abc");
+		cfg.setString(MetricOptions.SCOPE_NAMING_JM_JOB, "some-constant.<job_name>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 
 		JobID jid = new JobID();
@@ -77,8 +77,8 @@ public void testGenerateScopeCustom() {
 	@Test
 	public void testGenerateScopeCustomWildcard() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_JM, "peter");
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_JM_JOB, "*.some-constant.<job_id>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_JM, "peter");
+		cfg.setString(MetricOptions.SCOPE_NAMING_JM_JOB, "*.some-constant.<job_id>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 
 		JobID jid = new JobID();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupRegistrationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupRegistrationTest.java
index f121b59341be5..ace02367ec116 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupRegistrationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupRegistrationTest.java
@@ -19,6 +19,7 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.metrics.Histogram;
@@ -41,7 +42,7 @@ public class MetricGroupRegistrationTest {
 	@Test
 	public void testMetricInstantiation() {
 		Configuration config = new Configuration();
-		config.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
+		config.setString(MetricOptions.REPORTERS_LIST, "test");
 		config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());
 
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java
index 965e8fb9a4234..11f3c0fc7c332 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java
@@ -19,8 +19,8 @@
 package org.apache.flink.runtime.metrics.groups;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.metrics.MetricRegistry;
@@ -161,7 +161,7 @@ public void testGenerateScopeDefault() {
 	@Test
 	public void testGenerateScopeCustom() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "constant.<host>.foo.<host>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TM, "constant.<host>.foo.<host>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 		TaskManagerMetricGroup group = new TaskManagerMetricGroup(registry, "host", "id");
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobGroupTest.java
index f9bef6c80d4b5..f9891cbadc24d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobGroupTest.java
@@ -19,8 +19,8 @@
 package org.apache.flink.runtime.metrics.groups;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 
 import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
@@ -54,8 +54,8 @@ public void testGenerateScopeDefault() {
 	@Test
 	public void testGenerateScopeCustom() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "abc");
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM_JOB, "some-constant.<job_name>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TM, "abc");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TM_JOB, "some-constant.<job_name>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 
 		JobID jid = new JobID();
@@ -76,8 +76,8 @@ public void testGenerateScopeCustom() {
 	@Test
 	public void testGenerateScopeCustomWildcard() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "peter.<tm_id>");
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM_JOB, "*.some-constant.<job_id>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TM, "peter.<tm_id>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TM_JOB, "*.some-constant.<job_id>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 
 		JobID jid = new JobID();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroupTest.java
index eb906d0092d3e..183237a2e58ea 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroupTest.java
@@ -19,8 +19,8 @@
 package org.apache.flink.runtime.metrics.groups;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.metrics.Metric;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
@@ -63,9 +63,9 @@ public void testGenerateScopeDefault() {
 	@Test
 	public void testGenerateScopeCustom() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM, "abc");
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TM_JOB, "def");
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TASK, "<tm_id>.<job_id>.<task_id>.<task_attempt_id>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TM, "abc");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TM_JOB, "def");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TASK, "<tm_id>.<job_id>.<task_id>.<task_attempt_id>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 
 		JobID jid = new JobID();
@@ -90,7 +90,7 @@ public void testGenerateScopeCustom() {
 	@Test
 	public void testGenerateScopeWilcard() {
 		Configuration cfg = new Configuration();
-		cfg.setString(ConfigConstants.METRICS_SCOPE_NAMING_TASK, "*.<task_attempt_id>.<subtask_index>");
+		cfg.setString(MetricOptions.SCOPE_NAMING_TASK, "*.<task_attempt_id>.<subtask_index>");
 		MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(cfg));
 
 		AbstractID executionId = new AbstractID();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 7569170ce03c6..99c3f4ee6f380 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -28,8 +28,8 @@
 import org.apache.flink.api.common.state.StateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.core.fs.FSDataInputStream;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
@@ -177,10 +177,10 @@ public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<S
 			((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseOutputMetricsForTask();
 		}
 		Configuration taskManagerConfig = container.getEnvironment().getTaskManagerInfo().getConfiguration();
-		int historySize = taskManagerConfig.getInteger(ConfigConstants.METRICS_LATENCY_HISTORY_SIZE, ConfigConstants.DEFAULT_METRICS_LATENCY_HISTORY_SIZE);
+		int historySize = taskManagerConfig.getInteger(MetricOptions.LATENCY_HISTORY_SIZE);
 		if (historySize <= 0) {
-			LOG.warn("{} has been set to a value equal or below 0: {}. Using default.", ConfigConstants.METRICS_LATENCY_HISTORY_SIZE, historySize);
-			historySize = ConfigConstants.DEFAULT_METRICS_LATENCY_HISTORY_SIZE;
+			LOG.warn("{} has been set to a value equal or below 0: {}. Using default.", MetricOptions.LATENCY_HISTORY_SIZE, historySize);
+			historySize = MetricOptions.LATENCY_HISTORY_SIZE.defaultValue();
 		}
 
 		latencyGauge = this.metrics.gauge("latency", new LatencyGauge(historySize));

From 19f1cd437a0f6a3a6335bd3ed6c3fc199eada456 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Wed, 5 Apr 2017 23:50:12 +0200
Subject: [PATCH 029/253] [FLINK-5962] [runtime-web] [tests] Replace deprecated
 usage og JsonNode#getValueAsText

This closes #3679
---
 ...CancellationWithSavepointHandlersTest.java | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JobCancellationWithSavepointHandlersTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JobCancellationWithSavepointHandlersTest.java
index 8b948aa6c3b6d..8c2d3fc47873a 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JobCancellationWithSavepointHandlersTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JobCancellationWithSavepointHandlersTest.java
@@ -20,6 +20,8 @@
 
 import akka.dispatch.ExecutionContexts$;
 import akka.dispatch.Futures;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpResponseStatus;
@@ -32,8 +34,6 @@
 import org.apache.flink.runtime.messages.JobManagerMessages.CancelJobWithSavepoint;
 import org.apache.flink.runtime.messages.JobManagerMessages.CancellationSuccess;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.Assert;
 import org.junit.Test;
 import scala.concurrent.ExecutionContext;
@@ -201,9 +201,9 @@ public void testTriggerNewRequest() throws Exception {
 		String json = response.content().toString(Charset.forName("UTF-8"));
 		JsonNode root = new ObjectMapper().readTree(json);
 
-		assertEquals("accepted", root.get("status").getValueAsText());
-		assertEquals("1", root.get("request-id").getValueAsText());
-		assertEquals(location, root.get("location").getValueAsText());
+		assertEquals("accepted", root.get("status").asText());
+		assertEquals("1", root.get("request-id").asText());
+		assertEquals(location, root.get("location").asText());
 
 		// Trigger again
 		response = trigger.handleRequest(params, Collections.<String, String>emptyMap(), jobManager);
@@ -215,9 +215,9 @@ public void testTriggerNewRequest() throws Exception {
 		json = response.content().toString(Charset.forName("UTF-8"));
 		root = new ObjectMapper().readTree(json);
 
-		assertEquals("accepted", root.get("status").getValueAsText());
-		assertEquals("1", root.get("request-id").getValueAsText());
-		assertEquals(location, root.get("location").getValueAsText());
+		assertEquals("accepted", root.get("status").asText());
+		assertEquals("1", root.get("request-id").asText());
+		assertEquals(location, root.get("location").asText());
 
 		// Only single actual request
 		verify(jobManager).ask(eq(new CancelJobWithSavepoint(jobId, "custom-directory")), any(FiniteDuration.class));
@@ -233,8 +233,8 @@ public void testTriggerNewRequest() throws Exception {
 		json = response.content().toString(Charset.forName("UTF-8"));
 		root = new ObjectMapper().readTree(json);
 
-		assertEquals("in-progress", root.get("status").getValueAsText());
-		assertEquals("1", root.get("request-id").getValueAsText());
+		assertEquals("in-progress", root.get("status").asText());
+		assertEquals("1", root.get("request-id").asText());
 
 		// Complete
 		promise.success(new CancellationSuccess(jobId, "_path-savepoint_"));
@@ -249,9 +249,9 @@ public void testTriggerNewRequest() throws Exception {
 
 		root = new ObjectMapper().readTree(json);
 
-		assertEquals("success", root.get("status").getValueAsText());
-		assertEquals("1", root.get("request-id").getValueAsText());
-		assertEquals("_path-savepoint_", root.get("savepoint-path").getValueAsText());
+		assertEquals("success", root.get("status").asText());
+		assertEquals("1", root.get("request-id").asText());
+		assertEquals("_path-savepoint_", root.get("savepoint-path").asText());
 
 		// Query again, keep recent history
 
@@ -265,9 +265,9 @@ public void testTriggerNewRequest() throws Exception {
 
 		root = new ObjectMapper().readTree(json);
 
-		assertEquals("success", root.get("status").getValueAsText());
-		assertEquals("1", root.get("request-id").getValueAsText());
-		assertEquals("_path-savepoint_", root.get("savepoint-path").getValueAsText());
+		assertEquals("success", root.get("status").asText());
+		assertEquals("1", root.get("request-id").asText());
+		assertEquals("_path-savepoint_", root.get("savepoint-path").asText());
 
 		// Query for unknown request
 		params.put("requestId", "9929");
@@ -281,9 +281,9 @@ public void testTriggerNewRequest() throws Exception {
 
 		root = new ObjectMapper().readTree(json);
 
-		assertEquals("failed", root.get("status").getValueAsText());
-		assertEquals("9929", root.get("request-id").getValueAsText());
-		assertEquals("Unknown job/request ID", root.get("cause").getValueAsText());
+		assertEquals("failed", root.get("status").asText());
+		assertEquals("9929", root.get("request-id").asText());
+		assertEquals("Unknown job/request ID", root.get("cause").asText());
 	}
 
 	/**
@@ -327,8 +327,8 @@ public void testFailedCancellation() throws Exception {
 		String json = response.content().toString(Charset.forName("UTF-8"));
 		JsonNode root = new ObjectMapper().readTree(json);
 
-		assertEquals("failed", root.get("status").getValueAsText());
-		assertEquals("1", root.get("request-id").getValueAsText());
-		assertEquals("Test Exception", root.get("cause").getValueAsText());
+		assertEquals("failed", root.get("status").asText());
+		assertEquals("1", root.get("request-id").asText());
+		assertEquals("Test Exception", root.get("cause").asText());
 	}
 }

From d77f4733cda09785ad2311eddaa7b051ee4dafb5 Mon Sep 17 00:00:00 2001
From: Patrick Lucas <me@patricklucas.com>
Date: Fri, 31 Mar 2017 17:00:21 +0200
Subject: [PATCH 030/253] [hotfix] [docs] Remove empty Docker docs page

This will be re-added when we have a coherent plan for Docker support
and after the official Docker images are available.

This closes #3658
---
 docs/setup/docker.md | 29 -----------------------------
 1 file changed, 29 deletions(-)
 delete mode 100644 docs/setup/docker.md

diff --git a/docs/setup/docker.md b/docs/setup/docker.md
deleted file mode 100644
index 8b9757f7a6cbd..0000000000000
--- a/docs/setup/docker.md
+++ /dev/null
@@ -1,29 +0,0 @@
----
-title:  "Docker Setup"
-nav-title: Docker
-nav-parent_id: deployment
-nav-pos: 4
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-* This will be replaced by the TOC
-{:toc}
-
-

From 213f193ca6982e3031f2ee1c9a00222522e8f423 Mon Sep 17 00:00:00 2001
From: rami-alisawi <rami-alisawi@users.noreply.github.com>
Date: Fri, 31 Mar 2017 17:11:29 +0300
Subject: [PATCH 031/253] [FLINK-6236] [docs] Mention that resume from
 savepoint is possible via the web UI

This closes #3657
---
 docs/setup/savepoints.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/setup/savepoints.md b/docs/setup/savepoints.md
index da0784da6bfdb..4bdc43f6cf617 100644
--- a/docs/setup/savepoints.md
+++ b/docs/setup/savepoints.md
@@ -69,6 +69,8 @@ In the above example, the print sink is stateless and hence not part of the save
 
 You can use the [command line client]({{ site.baseurl }}/setup/cli.html#savepoints) to *trigger savepoints*, *cancel a job with a savepoint*, *resume from savepoints*, and *dispose savepoints*.
 
+With Flink >= 1.2.0 it is also possible to *resume from savepoints* using the webui.
+
 ### Triggering Savepoints
 
 When triggering a savepoint, a single savepoint file will be created that contains the checkpoint *meta data*. The actual checkpoint state will be kept around in the configured checkpoint directory. For example with a `FsStateBackend` or `RocksDBStateBackend`:

From 6a302e0434be74cd399b1dc72f6c131539c0fec6 Mon Sep 17 00:00:00 2001
From: Nico Kruber <nico@data-artisans.com>
Date: Mon, 13 Feb 2017 15:33:23 +0100
Subject: [PATCH 032/253] [FLINK-5672] [scripts] Add special cases for a local
 setup in cluster start/stop scripts

This way, if all slaves refer to "localhost", we do not require ssh at all.

This closes #3298
---
 flink-dist/src/main/flink-bin/bin/config.sh   | 33 ++++++++++++++++++-
 .../src/main/flink-bin/bin/start-cluster.sh   | 14 ++------
 .../src/main/flink-bin/bin/stop-cluster.sh    | 14 ++------
 3 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index dbfdd0e5e4c3e..a16529c552dfd 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -410,16 +410,47 @@ readSlaves() {
 
     SLAVES=()
 
+    SLAVES_ALL_LOCALHOST=true
     GOON=true
     while $GOON; do
         read line || GOON=false
         HOST=$( extractHostName $line)
-        if [ -n "$HOST" ]; then
+        if [ -n "$HOST" ] ; then
             SLAVES+=(${HOST})
+            if [ "${HOST}" != "localhost" ] ; then
+                SLAVES_ALL_LOCALHOST=false
+            fi
         fi
     done < "$SLAVES_FILE"
 }
 
+# starts or stops TMs on all slaves
+# TMSlaves start|stop
+TMSlaves() {
+    CMD=$1
+
+    readSlaves
+
+    if [ ${SLAVES_ALL_LOCALHOST} = true ] ; then
+        # all-local setup
+        for slave in ${SLAVES[@]}; do
+            "${FLINK_BIN_DIR}"/taskmanager.sh "${CMD}"
+        done
+    else
+        # non-local setup
+        # Stop TaskManager instance(s) using pdsh (Parallel Distributed Shell) when available
+        command -v pdsh >/dev/null 2>&1
+        if [[ $? -ne 0 ]]; then
+            for slave in ${SLAVES[@]}; do
+                ssh -n $FLINK_SSH_OPTS $slave -- "nohup /bin/bash -l \"${FLINK_BIN_DIR}/taskmanager.sh\" \"${CMD}\" &"
+            done
+        else
+            PDSH_SSH_ARGS="" PDSH_SSH_ARGS_APPEND=$FLINK_SSH_OPTS pdsh -w $(IFS=, ; echo "${SLAVES[*]}") \
+                "nohup /bin/bash -l \"${FLINK_BIN_DIR}/taskmanager.sh\" \"${CMD}\""
+        fi
+    fi
+}
+
 useOffHeapMemory() {
     [[ "`echo ${FLINK_TM_OFFHEAP} | tr '[:upper:]' '[:lower:]'`" == "true" ]]
 }
diff --git a/flink-dist/src/main/flink-bin/bin/start-cluster.sh b/flink-dist/src/main/flink-bin/bin/start-cluster.sh
index 76111897e26a3..5d2f92b4bc0d0 100755
--- a/flink-dist/src/main/flink-bin/bin/start-cluster.sh
+++ b/flink-dist/src/main/flink-bin/bin/start-cluster.sh
@@ -49,15 +49,5 @@ else
 fi
 shopt -u nocasematch
 
-# Start TaskManager instance(s) using pdsh (Parallel Distributed Shell) when available
-readSlaves
-
-command -v pdsh >/dev/null 2>&1
-if [[ $? -ne 0 ]]; then
-    for slave in ${SLAVES[@]}; do
-        ssh -n $FLINK_SSH_OPTS $slave -- "nohup /bin/bash -l \"${FLINK_BIN_DIR}/taskmanager.sh\" start &"
-    done
-else
-    PDSH_SSH_ARGS="" PDSH_SSH_ARGS_APPEND="${FLINK_SSH_OPTS}" pdsh -w $(IFS=, ; echo "${SLAVES[*]}") \
-        "nohup /bin/bash -l \"${FLINK_BIN_DIR}/taskmanager.sh\" start"
-fi
+# Start TaskManager instance(s)
+TMSlaves start
diff --git a/flink-dist/src/main/flink-bin/bin/stop-cluster.sh b/flink-dist/src/main/flink-bin/bin/stop-cluster.sh
index bc86291339d58..e04d2fa0c4d50 100755
--- a/flink-dist/src/main/flink-bin/bin/stop-cluster.sh
+++ b/flink-dist/src/main/flink-bin/bin/stop-cluster.sh
@@ -22,18 +22,8 @@ bin=`cd "$bin"; pwd`
 
 . "$bin"/config.sh
 
-# Stop TaskManager instance(s) using pdsh (Parallel Distributed Shell) when available
-readSlaves
-
-command -v pdsh >/dev/null 2>&1
-if [[ $? -ne 0 ]]; then
-    for slave in ${SLAVES[@]}; do
-        ssh -n $FLINK_SSH_OPTS $slave -- "nohup /bin/bash -l \"${FLINK_BIN_DIR}/taskmanager.sh\" stop &"
-    done
-else
-    PDSH_SSH_ARGS="" PDSH_SSH_ARGS_APPEND=$FLINK_SSH_OPTS pdsh -w $(IFS=, ; echo "${SLAVES[*]}") \
-        "nohup /bin/bash -l \"${FLINK_BIN_DIR}/taskmanager.sh\" stop"
-fi
+# Stop TaskManager instance(s)
+TMSlaves stop
 
 # Stop JobManager instance(s)
 shopt -s nocasematch

From ae757c656e7042483275ac135f0553f42d7104fd Mon Sep 17 00:00:00 2001
From: shijinkui <shijinkui@huawei.com>
Date: Fri, 10 Feb 2017 16:42:29 +0800
Subject: [PATCH 033/253] [FLINK-4562] [table] Move table examples into a
 dedicated module in flink-examples.

This closes #2460.
---
 flink-dist/pom.xml                            |   1 +
 flink-examples/flink-examples-table/pom.xml   | 108 ++++++++++++++++++
 .../table/examples/java/WordCountSQL.java     |   4 +-
 .../table/examples/java/WordCountTable.java   |   4 +-
 .../examples/scala/StreamSQLExample.scala     |   2 +-
 .../examples/scala/StreamTableExample.scala   |   3 +-
 .../examples/scala/TPCHQuery3Table.scala      |   0
 .../table/examples/scala/WordCountSQL.scala   |   0
 .../table/examples/scala/WordCountTable.scala |   0
 flink-examples/pom.xml                        |   1 +
 .../batch/table/AggregationsITCase.scala      |  15 +--
 11 files changed, 124 insertions(+), 14 deletions(-)
 create mode 100644 flink-examples/flink-examples-table/pom.xml
 rename {flink-libraries/flink-table => flink-examples/flink-examples-table}/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java (100%)
 rename {flink-libraries/flink-table => flink-examples/flink-examples-table}/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java (100%)
 rename {flink-libraries/flink-table => flink-examples/flink-examples-table}/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala (100%)
 rename {flink-libraries/flink-table => flink-examples/flink-examples-table}/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala (99%)
 rename {flink-libraries/flink-table => flink-examples/flink-examples-table}/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala (100%)
 rename {flink-libraries/flink-table => flink-examples/flink-examples-table}/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala (100%)
 rename {flink-libraries/flink-table => flink-examples/flink-examples-table}/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala (100%)

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index e42aea6124795..76df09e4f1769 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -338,6 +338,7 @@ under the License.
 								<excludes>
 									<exclude>org.apache.flink:flink-examples-batch</exclude>
 									<exclude>org.apache.flink:flink-examples-streaming</exclude>
+									<exclude>org.apache.flink:flink-examples-table</exclude>
 									<exclude>org.apache.flink:flink-python</exclude>
 									<exclude>org.slf4j:slf4j-log4j12</exclude>
 									<exclude>log4j:log4j</exclude>
diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml
new file mode 100644
index 0000000000000..de050d742b004
--- /dev/null
+++ b/flink-examples/flink-examples-table/pom.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+		 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>org.apache.flink</groupId>
+		<artifactId>flink-examples_2.10</artifactId>
+		<version>1.3-SNAPSHOT</version>
+		<relativePath>..</relativePath>
+	</parent>
+
+	<name>flink-examples-table</name>
+	<artifactId>flink-examples-table_2.10</artifactId>
+	<packaging>jar</packaging>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-table_2.10</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>net.alchim31.maven</groupId>
+				<artifactId>scala-maven-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>scala-compile-first</id>
+						<phase>process-resources</phase>
+						<goals>
+							<goal>add-source</goal>
+							<goal>compile</goal>
+						</goals>
+					</execution>
+					<execution>
+						<id>scala-test-compile</id>
+						<phase>process-test-resources</phase>
+						<goals>
+							<goal>testCompile</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>flink-table-examples_2.10</id>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<shadeTestJar>false</shadeTestJar>
+							<shadedArtifactAttached>false</shadedArtifactAttached>
+							<createDependencyReducedPom>false</createDependencyReducedPom>
+							<finalName>TableExamples</finalName>
+							<outputFile>flink-examples-table-with-dependencies.jar</outputFile>
+							<filters>
+								<filter>
+									<artifact>*:*</artifact>
+									<includes>
+										<include>org.codehaus.commons.compiler.properties</include>
+										<include>org/codehaus/janino/**</include>
+										<include>org/codehaus/commons/**</include>
+										<include>org/apache/calcite/**</include>
+										<include>org/apache/flink/table/**</include>
+										<include>org/apache/flink/shaded/calcite/com/google/common/**</include>
+										<include>org/apache/flink/shaded/calcite/org/eigenbase/util/property/**</include>
+									</includes>
+									<excludes>
+										<exclude>META-INF/*.SF</exclude>
+										<exclude>META-INF/*.DSA</exclude>
+										<exclude>META-INF/*.RSA</exclude>
+									</excludes>
+								</filter>
+							</filters>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java
similarity index 100%
rename from flink-libraries/flink-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java
rename to flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java
index e3b345cac4cda..9e1b45eb6b4b0 100644
--- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java
+++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java
@@ -17,11 +17,11 @@
  */
 package org.apache.flink.table.examples.java;
 
-import org.apache.flink.table.api.Table;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.table.api.java.BatchTableEnvironment;
+import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.api.java.BatchTableEnvironment;
 
 /**
  * Simple example that shows how the Batch SQL API is used in Java.
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java
similarity index 100%
rename from flink-libraries/flink-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java
rename to flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java
index bdb50f53ee126..1ee8c123bb6b9 100644
--- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java
+++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java
@@ -17,11 +17,11 @@
  */
 package org.apache.flink.table.examples.java;
 
-import org.apache.flink.table.api.Table;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.table.api.java.BatchTableEnvironment;
+import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.api.java.BatchTableEnvironment;
 
 /**
   * Simple example for demonstrating the use of the Table API for a Word Count in Java.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala
similarity index 100%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala
rename to flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala
index 6d16722c0b328..2cdd8b8971eb4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala
@@ -18,9 +18,9 @@
 package org.apache.flink.table.examples.scala
 
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.scala._
 import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
 import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala._
 
 /**
   * Simple example for demonstrating the use of SQL on a Stream Table.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala
similarity index 99%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala
rename to flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala
index 6c1467fe220bd..5c5012b57f102 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala
+++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala
@@ -18,9 +18,9 @@
 package org.apache.flink.table.examples.scala
 
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.scala._
 import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
 import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala._
 
 /**
   * Simple example for demonstrating the use of Table API on a Stream Table.
@@ -28,7 +28,6 @@ import org.apache.flink.table.api.TableEnvironment
   * This example shows how to:
   *  - Convert DataStreams to Tables
   *  - Apply union, select, and filter operations
-  *
   */
 object StreamTableExample {
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala
similarity index 100%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala
rename to flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala
similarity index 100%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala
rename to flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala
similarity index 100%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala
rename to flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala
diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml
index d9c906af55e2d..a5939f5d8d3c8 100644
--- a/flink-examples/pom.xml
+++ b/flink-examples/pom.xml
@@ -68,5 +68,6 @@ under the License.
 	<modules>
 		<module>flink-examples-batch</module>
 		<module>flink-examples-streaming</module>
+		<module>flink-examples-table</module>
 	</modules>
 </project>
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
index 22b7f0f230959..5ac09b949b299 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
@@ -25,7 +25,6 @@ import org.apache.flink.table.api.scala._
 import org.apache.flink.api.scala.util.CollectionDataSets
 import org.apache.flink.types.Row
 import org.apache.flink.table.api.TableEnvironment
-import org.apache.flink.table.examples.scala.WordCountTable.{WC => MyWC}
 import org.apache.flink.test.util.TestBaseUtils
 import org.junit._
 import org.junit.runner.RunWith
@@ -156,17 +155,17 @@ class AggregationsITCase(
     val tEnv = TableEnvironment.getTableEnvironment(env, config)
 
     val input = env.fromElements(
-      MyWC("hello", 1),
-      MyWC("hello", 1),
-      MyWC("ciao", 1),
-      MyWC("hola", 1),
-      MyWC("hola", 1))
+      WC("hello", 1),
+      WC("hello", 1),
+      WC("ciao", 1),
+      WC("hola", 1),
+      WC("hola", 1))
     val expr = input.toTable(tEnv)
     val result = expr
       .groupBy('word)
       .select('word, 'frequency.sum as 'frequency)
       .filter('frequency === 2)
-      .toDataSet[MyWC]
+      .toDataSet[WC]
 
     val mappedResult = result.map(w => (w.word, w.frequency * 10)).collect()
     val expected = "(hello,20)\n" + "(hola,20)"
@@ -339,4 +338,6 @@ class AggregationsITCase(
     val results = t.toDataSet[Row].collect()
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
+
+  case class WC(word: String, frequency: Long)
 }

From 874b203c3fe0b38a86c2ef97e543b33554c9f174 Mon Sep 17 00:00:00 2001
From: godfreyhe <godfreyhe@163.com>
Date: Wed, 19 Apr 2017 19:46:18 +0800
Subject: [PATCH 034/253] [FLINK-6326] [table] Add ProjectMergeRule to logical
 optimization rule set.

This closes #3739.
---
 .../table/plan/rules/FlinkRuleSets.scala      |  2 +
 .../table/api/scala/batch/ExplainTest.scala   |  1 +
 .../batch/sql/QueryDecorrelationTest.scala    | 72 +++++++++++--------
 .../scala/batch/sql/SetOperatorsTest.scala    | 20 +++---
 .../scala/batch/sql/WindowAggregateTest.scala | 50 +++++--------
 5 files changed, 77 insertions(+), 68 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
index 94a1abb4abbd4..0bee4e5b03011 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
@@ -49,6 +49,8 @@ object FlinkRuleSets {
     FilterProjectTransposeRule.INSTANCE,
     // push a projection to the children of a join
     ProjectJoinTransposeRule.INSTANCE,
+    // merge projections
+    ProjectMergeRule.INSTANCE,
     // remove identity project
     ProjectRemoveRule.INSTANCE,
     // reorder sort and projection
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
index a323ec98dd3e4..1a6b31491e57a 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
@@ -119,4 +119,5 @@ class ExplainTest
       "../../src/test/scala/resources/testUnion1.out").mkString.replaceAll("\\r\\n", "\n")
     assertEquals(result, source)
   }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
index 7496ff870e2c6..34ae346e3415e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
@@ -44,38 +44,41 @@ class QueryDecorrelationTest extends TableTestBase {
       "DataSetCalc",
       binaryNode(
         "DataSetJoin",
-        binaryNode(
-          "DataSetJoin",
-          unaryNode(
-            "DataSetCalc",
-            batchTableNode(0),
-            term("select", "empno", "ename", "job", "salary", "deptno"),
-            term("where", "<(deptno, 10)")
-          ),
-          unaryNode(
-            "DataSetCalc",
-            batchTableNode(1),
-            term("select", "deptno", "name"),
-            term("where", "<(deptno, 15)")
+        unaryNode(
+          "DataSetCalc",
+          binaryNode(
+            "DataSetJoin",
+            unaryNode(
+              "DataSetCalc",
+              batchTableNode(0),
+              term("select", "empno", "salary", "deptno"),
+              term("where", "<(deptno, 10)")
+            ),
+            unaryNode(
+              "DataSetCalc",
+              batchTableNode(1),
+              term("select", "deptno"),
+              term("where", "<(deptno, 15)")
+            ),
+            term("where", "=(deptno, deptno0)"),
+            term("join", "empno", "salary", "deptno", "deptno0"),
+            term("joinType", "InnerJoin")
           ),
-          term("where", "=(deptno, deptno0)"),
-          term("join", "empno", "ename", "job", "salary", "deptno", "deptno0", "name"),
-          term("joinType", "InnerJoin")
+          term("select", "empno", "salary")
         ),
         unaryNode(
           "DataSetAggregate",
           unaryNode(
             "DataSetCalc",
             batchTableNode(0),
-            term("select", "salary", "empno"),
+            term("select", "empno", "salary"),
             term("where", "IS NOT NULL(empno)")
           ),
           term("groupBy", "empno"),
           term("select", "empno", "AVG(salary) AS EXPR$0")
         ),
         term("where", "AND(=(empno, empno0), >(salary, EXPR$0))"),
-        term("join", "empno", "ename", "job", "salary", "deptno",
-          "deptno0", "name", "empno0", "EXPR$0"),
+        term("join", "empno", "salary", "empno0", "EXPR$0"),
         term("joinType", "InnerJoin")
       ),
       term("select", "empno")
@@ -105,13 +108,25 @@ class QueryDecorrelationTest extends TableTestBase {
           "DataSetCalc",
           binaryNode(
             "DataSetJoin",
-            binaryNode(
-              "DataSetJoin",
-              batchTableNode(0),
-              batchTableNode(1),
-              term("where", "=(deptno, deptno0)"),
-              term("join", "empno", "ename", "job", "salary", "deptno", "deptno0", "name"),
-              term("joinType", "InnerJoin")
+            unaryNode(
+              "DataSetCalc",
+              binaryNode(
+                "DataSetJoin",
+                unaryNode(
+                  "DataSetCalc",
+                  batchTableNode(0),
+                  term("select", "empno", "salary", "deptno")
+                ),
+                unaryNode(
+                  "DataSetCalc",
+                  batchTableNode(1),
+                  term("select", "deptno")
+                ),
+                term("where", "=(deptno, deptno0)"),
+                term("join", "empno", "salary", "deptno", "deptno0"),
+                term("joinType", "InnerJoin")
+              ),
+              term("select", "empno", "salary", "deptno0")
             ),
             unaryNode(
               "DataSetAggregate",
@@ -124,9 +139,8 @@ class QueryDecorrelationTest extends TableTestBase {
               term("groupBy", "deptno"),
               term("select", "deptno", "AVG(salary) AS EXPR$0")
             ),
-            term("where", "AND(=(deptno0, deptno1), >(salary, EXPR$0))"),
-            term("join", "empno", "ename", "job", "salary", "deptno", "deptno0",
-              "name", "deptno1", "EXPR$0"),
+            term("where", "AND(=(deptno0, deptno), >(salary, EXPR$0))"),
+            term("join", "empno", "salary", "deptno0", "deptno", "EXPR$0"),
             term("joinType", "InnerJoin")
           ),
           term("select", "empno")
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SetOperatorsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SetOperatorsTest.scala
index 2f9057d1c5229..ba0326a258508 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SetOperatorsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SetOperatorsTest.scala
@@ -55,18 +55,22 @@ class SetOperatorsTest extends TableTestBase {
         "DataSetJoin",
         batchTableNode(0),
         unaryNode(
-          "DataSetAggregate",
+          "DataSetCalc",
           unaryNode(
-            "DataSetCalc",
-            batchTableNode(1),
-            term("select", "b_long AS b_long3", "true AS $f0"),
-            term("where", "IS NOT NULL(b_long)")
+            "DataSetAggregate",
+            unaryNode(
+              "DataSetCalc",
+              batchTableNode(1),
+              term("select", "b_long AS b_long3", "true AS $f0"),
+              term("where", "IS NOT NULL(b_long)")
+            ),
+            term("groupBy", "b_long3"),
+            term("select", "b_long3", "MIN($f0) AS $f1")
           ),
-          term("groupBy", "b_long3"),
-          term("select", "b_long3", "MIN($f0) AS $f1")
+          term("select", "b_long3")
         ),
         term("where", "=(a_long, b_long3)"),
-        term("join", "a_long", "a_int", "a_string", "b_long3", "$f1"),
+        term("join", "a_long", "a_int", "a_string", "b_long3"),
         term("joinType", "InnerJoin")
       ),
       term("select", "a_int", "a_string")
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
index 25a81c4e038c0..cd6c77df7a44f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
@@ -40,18 +40,14 @@ class WindowAggregateTest extends TableTestBase {
 
     val expected =
       unaryNode(
-        "DataSetCalc",
+        "DataSetWindowAggregate",
         unaryNode(
-          "DataSetWindowAggregate",
-          unaryNode(
-            "DataSetCalc",
-            batchTableNode(0),
-            term("select", "ts, a, b")
-          ),
-          term("window", EventTimeTumblingGroupWindow(Some('w$), 'ts, 7200000.millis)),
-          term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
+          "DataSetCalc",
+          batchTableNode(0),
+          term("select", "ts, a, b")
         ),
-        term("select", "sumA, cntB")
+        term("window", EventTimeTumblingGroupWindow(Some('w$), 'ts, 7200000.millis)),
+        term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
       )
 
     util.verifySql(sqlQuery, expected)
@@ -101,19 +97,15 @@ class WindowAggregateTest extends TableTestBase {
 
     val expected =
       unaryNode(
-        "DataSetCalc",
+        "DataSetWindowAggregate",
         unaryNode(
-          "DataSetWindowAggregate",
-          unaryNode(
-            "DataSetCalc",
-            batchTableNode(0),
-            term("select", "ts, a, b")
-          ),
-          term("window",
-            EventTimeSlidingGroupWindow(Some('w$), 'ts, 5400000.millis, 900000.millis)),
-          term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
+          "DataSetCalc",
+          batchTableNode(0),
+          term("select", "ts, a, b")
         ),
-        term("select", "sumA, cntB")
+        term("window",
+          EventTimeSlidingGroupWindow(Some('w$), 'ts, 5400000.millis, 900000.millis)),
+        term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
       )
 
     util.verifySql(sqlQuery, expected)
@@ -162,18 +154,14 @@ class WindowAggregateTest extends TableTestBase {
 
     val expected =
       unaryNode(
-        "DataSetCalc",
+        "DataSetWindowAggregate",
         unaryNode(
-          "DataSetWindowAggregate",
-          unaryNode(
-            "DataSetCalc",
-            batchTableNode(0),
-            term("select", "ts")
-          ),
-          term("window", EventTimeSessionGroupWindow(Some('w$), 'ts, 1800000.millis)),
-          term("select", "COUNT(*) AS cnt")
+          "DataSetCalc",
+          batchTableNode(0),
+          term("select", "ts")
         ),
-        term("select", "cnt")
+        term("window", EventTimeSessionGroupWindow(Some('w$), 'ts, 1800000.millis)),
+        term("select", "COUNT(*) AS cnt")
       )
 
     util.verifySql(sqlQuery, expected)

From 8d9d52dad9740605a9cc98e4f8a7de3f66d73fcd Mon Sep 17 00:00:00 2001
From: shaoxuan-wang <wshaoxuan@gmail.com>
Date: Tue, 18 Apr 2017 21:45:49 +0800
Subject: [PATCH 035/253] [FLINK-6242] [table] Add code generation for DataSet
 Aggregates

This closes #3735.
---
 .../flink/table/codegen/CodeGenerator.scala   | 224 ++++++-----
 .../plan/nodes/dataset/DataSetAggregate.scala |  11 +-
 .../dataset/DataSetWindowAggregate.scala      |  25 +-
 .../runtime/aggregate/AggregateUtil.scala     | 359 ++++++++++--------
 .../aggregate/DataSetAggFunction.scala        | 109 ++----
 .../aggregate/DataSetFinalAggFunction.scala   | 121 ++----
 .../aggregate/DataSetPreAggFunction.scala     |  93 ++---
 ...tSessionWindowAggReduceGroupFunction.scala | 116 ++----
 ...etSessionWindowAggregatePreProcessor.scala | 145 +++----
 ...lideTimeWindowAggReduceGroupFunction.scala | 177 +++------
 ...tSlideWindowAggReduceCombineFunction.scala |  98 ++---
 ...SetSlideWindowAggReduceGroupFunction.scala | 123 +++---
 ...bleCountWindowAggReduceGroupFunction.scala |  89 ++---
 ...leTimeWindowAggReduceCombineFunction.scala |  88 ++---
 ...mbleTimeWindowAggReduceGroupFunction.scala |  99 ++---
 .../DataSetWindowAggMapFunction.scala         |  64 ++--
 .../aggregate/GeneratedAggregations.scala     |  32 +-
 .../batch/table/AggregationsITCase.scala      |   3 +-
 ...ocessingOverRangeProcessFunctionTest.scala |   8 +-
 19 files changed, 878 insertions(+), 1106 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index c6e3c9a67e82b..510a87069a1fc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -250,57 +250,88 @@ class CodeGenerator(
     * @param aggregates  All aggregate functions
     * @param aggFields   Indexes of the input fields for all aggregate functions
     * @param aggMapping  The mapping of aggregates to output fields
+    * @param partialResults A flag defining whether final or partial results (accumulators) are set
+    *                       to the output row.
     * @param fwdMapping  The mapping of input fields to output fields
+    * @param mergeMapping An optional mapping to specify the accumulators to merge. If not set, we
+    *                     assume that both rows have the accumulators at the same position.
+    * @param constantFlags An optional parameter to define where to set constant boolean flags in
+    *                      the output row.
     * @param outputArity The number of fields in the output row.
     *
     * @return A GeneratedAggregationsFunction
     */
   def generateAggregations(
-     name: String,
-     generator: CodeGenerator,
-     inputType: RelDataType,
-     aggregates: Array[AggregateFunction[_ <: Any]],
-     aggFields: Array[Array[Int]],
-     aggMapping: Array[Int],
-     fwdMapping: Array[(Int, Int)],
-     outputArity: Int)
+      name: String,
+      generator: CodeGenerator,
+      inputType: RelDataType,
+      aggregates: Array[AggregateFunction[_ <: Any]],
+      aggFields: Array[Array[Int]],
+      aggMapping: Array[Int],
+      partialResults: Boolean,
+      fwdMapping: Array[Int],
+      mergeMapping: Option[Array[Int]],
+      constantFlags: Option[Array[(Int, Boolean)]],
+      outputArity: Int)
   : GeneratedAggregationsFunction = {
 
-    def genSetAggregationResults(
-      accTypes: Array[String],
-      aggs: Array[String],
-      aggMapping: Array[Int]): String = {
+    // get unique function name
+    val funcName = newName(name)
+    // register UDAGGs
+    val aggs = aggregates.map(a => generator.addReusableFunction(a))
+    // get java types of accumulators
+    val accTypes = aggregates.map { a =>
+      a.getClass.getMethod("createAccumulator").getReturnType.getCanonicalName
+    }
+
+    // get java types of input fields
+    val javaTypes = inputType.getFieldList
+      .map(f => FlinkTypeFactory.toTypeInfo(f.getType))
+      .map(t => t.getTypeClass.getCanonicalName)
+    // get parameter lists for aggregation functions
+    val parameters = aggFields.map {inFields =>
+      val fields = for (f <- inFields) yield s"(${javaTypes(f)}) input.getField($f)"
+      fields.mkString(", ")
+    }
+
+    def genSetAggregationResults: String = {
 
       val sig: String =
         j"""
-            |  public void setAggregationResults(
-            |    org.apache.flink.types.Row accs,
-            |    org.apache.flink.types.Row output)""".stripMargin
+           |  public final void setAggregationResults(
+           |    org.apache.flink.types.Row accs,
+           |    org.apache.flink.types.Row output)""".stripMargin
 
       val setAggs: String = {
         for (i <- aggs.indices) yield
-          j"""
-             |    org.apache.flink.table.functions.AggregateFunction baseClass$i =
-             |      (org.apache.flink.table.functions.AggregateFunction) ${aggs(i)};
-             |
-             |    output.setField(
-             |      ${aggMapping(i)},
-             |      baseClass$i.getValue((${accTypes(i)}) accs.getField($i)));""".stripMargin
+
+          if (partialResults) {
+            j"""
+               |    output.setField(
+               |      ${aggMapping(i)},
+               |      (${accTypes(i)}) accs.getField($i));""".stripMargin
+          } else {
+            j"""
+               |    org.apache.flink.table.functions.AggregateFunction baseClass$i =
+               |      (org.apache.flink.table.functions.AggregateFunction) ${aggs(i)};
+               |
+               |    output.setField(
+               |      ${aggMapping(i)},
+               |      baseClass$i.getValue((${accTypes(i)}) accs.getField($i)));""".stripMargin
+          }
       }.mkString("\n")
 
-      j"""$sig {
+      j"""
+         |$sig {
          |$setAggs
          |  }""".stripMargin
     }
 
-    def genAccumulate(
-     accTypes: Array[String],
-     aggs: Array[String],
-     parameters: Array[String]): String = {
+    def genAccumulate: String = {
 
       val sig: String =
         j"""
-            |  public void accumulate(
+            |  public final void accumulate(
             |    org.apache.flink.types.Row accs,
             |    org.apache.flink.types.Row input)""".stripMargin
 
@@ -317,14 +348,11 @@ class CodeGenerator(
          |  }""".stripMargin
     }
 
-    def genRetract(
-      accTypes: Array[String],
-      aggs: Array[String],
-      parameters: Array[String]): String = {
+    def genRetract: String = {
 
       val sig: String =
         j"""
-            |  public void retract(
+            |  public final void retract(
             |    org.apache.flink.types.Row accs,
             |    org.apache.flink.types.Row input)""".stripMargin
 
@@ -341,12 +369,11 @@ class CodeGenerator(
          |  }""".stripMargin
     }
 
-    def genCreateAccumulators(
-        aggs: Array[String]): String = {
+    def genCreateAccumulators: String = {
 
       val sig: String =
         j"""
-           |  public org.apache.flink.types.Row createAccumulators()
+           |  public final org.apache.flink.types.Row createAccumulators()
            |    """.stripMargin
       val init: String =
         j"""
@@ -373,22 +400,24 @@ class CodeGenerator(
          |  }""".stripMargin
     }
 
-    def genSetForwardedFields(
-        forwardMapping: Array[(Int, Int)]): String = {
+    def genSetForwardedFields: String = {
 
       val sig: String =
         j"""
-           |  public void setForwardedFields(
+           |  public final void setForwardedFields(
            |    org.apache.flink.types.Row input,
            |    org.apache.flink.types.Row output)
            |    """.stripMargin
+
       val forward: String = {
-        for (i <- forwardMapping.indices) yield
-          j"""
-             |    output.setField(
-             |      ${forwardMapping(i)._1},
-             |      input.getField(${forwardMapping(i)._2}));"""
-            .stripMargin
+        for (i <- fwdMapping.indices if fwdMapping(i) >= 0) yield
+          {
+            j"""
+               |    output.setField(
+               |      $i,
+               |      input.getField(${fwdMapping(i)}));"""
+              .stripMargin
+          }
       }.mkString("\n")
 
       j"""$sig {
@@ -396,20 +425,44 @@ class CodeGenerator(
          |  }""".stripMargin
     }
 
-    def genCreateOutputRow(outputArity: Int): String = {
+    def genSetConstantFlags: String = {
+
+      val sig: String =
+        j"""
+           |  public final void setConstantFlags(org.apache.flink.types.Row output)
+           |    """.stripMargin
+
+      val setFlags: String = if (constantFlags.isDefined) {
+        {
+          for (cf <- constantFlags.get) yield {
+            j"""
+               |    output.setField(${cf._1}, ${if (cf._2) "true" else "false"});"""
+              .stripMargin
+          }
+        }.mkString("\n")
+      } else {
+        ""
+      }
+
+      j"""$sig {
+         |$setFlags
+         |  }""".stripMargin
+    }
+
+    def genCreateOutputRow: String = {
       j"""
-         |  public org.apache.flink.types.Row createOutputRow() {
+         |  public final org.apache.flink.types.Row createOutputRow() {
          |    return new org.apache.flink.types.Row($outputArity);
          |  }""".stripMargin
     }
 
-    def genMergeAccumulatorsPair(
-        accTypes: Array[String],
-        aggs: Array[String]): String = {
+    def genMergeAccumulatorsPair: String = {
+
+      val mapping = mergeMapping.getOrElse(aggs.indices.toArray)
 
       val sig: String =
         j"""
-           |  public org.apache.flink.types.Row mergeAccumulatorsPair(
+           |  public final org.apache.flink.types.Row mergeAccumulatorsPair(
            |    org.apache.flink.types.Row a,
            |    org.apache.flink.types.Row b)
            """.stripMargin
@@ -417,7 +470,7 @@ class CodeGenerator(
         for (i <- aggs.indices) yield
           j"""
              |    ${accTypes(i)} aAcc$i = (${accTypes(i)}) a.getField($i);
-             |    ${accTypes(i)} bAcc$i = (${accTypes(i)}) b.getField($i);
+             |    ${accTypes(i)} bAcc$i = (${accTypes(i)}) b.getField(${mapping(i)});
              |    accList$i.set(0, aAcc$i);
              |    accList$i.set(1, bAcc$i);
              |    a.setField(
@@ -430,75 +483,76 @@ class CodeGenerator(
            |      return a;
            """.stripMargin
 
-      j"""$sig {
+      j"""
+         |$sig {
          |$merge
          |$ret
          |  }""".stripMargin
     }
 
-    def genMergeList(accTypes: Array[String]): String = {
+    def genMergeList: String = {
       {
         for (i <- accTypes.indices) yield
           j"""
-             |    java.util.ArrayList<${accTypes(i)}> accList$i;
+             |    private final java.util.ArrayList<${accTypes(i)}> accList$i =
+             |      new java.util.ArrayList<${accTypes(i)}>(2);
              """.stripMargin
       }.mkString("\n")
     }
 
-    def initMergeList(
-        accTypes: Array[String],
-        aggs: Array[String]): String = {
+    def initMergeList: String = {
       {
         for (i <- accTypes.indices) yield
           j"""
-             |    accList$i = new java.util.ArrayList<${accTypes(i)}>(2);
              |    accList$i.add(${aggs(i)}.createAccumulator());
              |    accList$i.add(${aggs(i)}.createAccumulator());
              """.stripMargin
       }.mkString("\n")
     }
 
-    // get unique function name
-    val funcName = newName(name)
-    // register UDAGGs
-    val aggs = aggregates.map(a => generator.addReusableFunction(a))
-    // get java types of accumulators
-    val accTypes = aggregates.map { a =>
-      a.getClass.getMethod("createAccumulator").getReturnType.getCanonicalName
-    }
+    def genResetAccumulator: String = {
 
-    // get java types of input fields
-    val javaTypes = inputType.getFieldList
-      .map(f => FlinkTypeFactory.toTypeInfo(f.getType))
-      .map(t => t.getTypeClass.getCanonicalName)
-    // get parameter lists for aggregation functions
-    val parameters = aggFields.map {inFields =>
-      val fields = for (f <- inFields) yield s"(${javaTypes(f)}) input.getField($f)"
-      fields.mkString(", ")
+      val sig: String =
+        j"""
+           |  public final void resetAccumulator(
+           |    org.apache.flink.types.Row accs)""".stripMargin
+
+      val reset: String = {
+        for (i <- aggs.indices) yield
+          j"""
+             |    ${aggs(i)}.resetAccumulator(
+             |      ((${accTypes(i)}) accs.getField($i)));""".stripMargin
+      }.mkString("\n")
+
+      j"""$sig {
+         |$reset
+         |  }""".stripMargin
     }
 
     var funcCode =
       j"""
-         |public class $funcName
+         |public final class $funcName
          |  extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations {
          |
          |  ${reuseMemberCode()}
-         |  ${genMergeList(accTypes)}
+         |  $genMergeList
          |  public $funcName() throws Exception {
          |    ${reuseInitCode()}
-         |    ${initMergeList(accTypes, aggs)}
+         |    $initMergeList
          |  }
          |  ${reuseConstructorCode(funcName)}
          |
          """.stripMargin
 
-    funcCode += genSetAggregationResults(accTypes, aggs, aggMapping) + "\n"
-    funcCode += genAccumulate(accTypes, aggs, parameters) + "\n"
-    funcCode += genRetract(accTypes, aggs, parameters) + "\n"
-    funcCode += genCreateAccumulators(aggs) + "\n"
-    funcCode += genSetForwardedFields(fwdMapping) + "\n"
-    funcCode += genCreateOutputRow(outputArity) + "\n"
-    funcCode += genMergeAccumulatorsPair(accTypes, aggs) + "\n"
+    funcCode += genSetAggregationResults + "\n"
+    funcCode += genAccumulate + "\n"
+    funcCode += genRetract + "\n"
+    funcCode += genCreateAccumulators + "\n"
+    funcCode += genSetForwardedFields + "\n"
+    funcCode += genSetConstantFlags + "\n"
+    funcCode += genCreateOutputRow + "\n"
+    funcCode += genMergeAccumulatorsPair + "\n"
+    funcCode += genResetAccumulator + "\n"
     funcCode += "}"
 
     GeneratedAggregationsFunction(funcName, funcCode)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
index 5a4aa5916bdca..b92775c72485f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
@@ -29,6 +29,7 @@ import org.apache.flink.api.java.DataSet
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.table.api.BatchTableEnvironment
 import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.CommonAggregate
 import org.apache.flink.table.runtime.aggregate.{AggregateUtil, DataSetPreAggFunction}
 import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
@@ -89,19 +90,25 @@ class DataSetAggregate(
 
   override def translateToPlan(tableEnv: BatchTableEnvironment): DataSet[Row] = {
 
+    val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(tableEnv)
+
+    val generator = new CodeGenerator(
+      tableEnv.getConfig,
+      false,
+      inputDS.getType)
+
     val (
       preAgg: Option[DataSetPreAggFunction],
       preAggType: Option[TypeInformation[Row]],
       finalAgg: GroupReduceFunction[Row, Row]
       ) = AggregateUtil.createDataSetAggregateFunctions(
+        generator,
         namedAggregates,
         inputType,
         rowRelDataType,
         grouping,
         inGroupingSet)
 
-    val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(tableEnv)
-
     val aggString = aggregationToString(inputType, grouping, getRowType, namedAggregates, Nil)
 
     val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala
index a94deb1caed5f..96c427eed60f6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala
@@ -28,6 +28,7 @@ import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.table.api.BatchTableEnvironment
 import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
 import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.plan.nodes.CommonAggregate
 import org.apache.flink.table.runtime.aggregate.AggregateUtil.{CalcitePair, _}
@@ -109,21 +110,28 @@ class DataSetWindowAggregate(
 
     val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(tableEnv)
 
+    val generator = new CodeGenerator(
+      tableEnv.getConfig,
+      false,
+      inputDS.getType)
+
     // whether identifiers are matched case-sensitively
     val caseSensitive = tableEnv.getFrameworkConfig.getParserConfig.caseSensitive()
 
     window match {
       case EventTimeTumblingGroupWindow(_, _, size) =>
         createEventTimeTumblingWindowDataSet(
+          generator,
           inputDS,
           isTimeInterval(size.resultType),
           caseSensitive)
 
       case EventTimeSessionGroupWindow(_, _, gap) =>
-        createEventTimeSessionWindowDataSet(inputDS, caseSensitive)
+        createEventTimeSessionWindowDataSet(generator, inputDS, caseSensitive)
 
       case EventTimeSlidingGroupWindow(_, _, size, slide) =>
         createEventTimeSlidingWindowDataSet(
+          generator,
           inputDS,
           isTimeInterval(size.resultType),
           asLong(size),
@@ -139,17 +147,20 @@ class DataSetWindowAggregate(
   }
 
   private def createEventTimeTumblingWindowDataSet(
+      generator: CodeGenerator,
       inputDS: DataSet[Row],
       isTimeWindow: Boolean,
       isParserCaseSensitive: Boolean): DataSet[Row] = {
 
     val mapFunction = createDataSetWindowPrepareMapFunction(
+      generator,
       window,
       namedAggregates,
       grouping,
       inputType,
       isParserCaseSensitive)
     val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction(
+      generator,
       window,
       namedAggregates,
       inputType,
@@ -195,6 +206,7 @@ class DataSetWindowAggregate(
   }
 
   private[this] def createEventTimeSessionWindowDataSet(
+      generator: CodeGenerator,
       inputDS: DataSet[Row],
       isParserCaseSensitive: Boolean): DataSet[Row] = {
 
@@ -203,6 +215,7 @@ class DataSetWindowAggregate(
 
     // create mapFunction for initializing the aggregations
     val mapFunction = createDataSetWindowPrepareMapFunction(
+      generator,
       window,
       namedAggregates,
       grouping,
@@ -229,6 +242,7 @@ class DataSetWindowAggregate(
       if (groupingKeys.length > 0) {
         // create groupCombineFunction for combine the aggregations
         val combineGroupFunction = createDataSetWindowAggregationCombineFunction(
+          generator,
           window,
           namedAggregates,
           inputType,
@@ -236,6 +250,7 @@ class DataSetWindowAggregate(
 
         // create groupReduceFunction for calculating the aggregations
         val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction(
+          generator,
           window,
           namedAggregates,
           inputType,
@@ -257,6 +272,7 @@ class DataSetWindowAggregate(
       } else {
         // non-grouping window
         val mapPartitionFunction = createDataSetWindowAggregationMapPartitionFunction(
+          generator,
           window,
           namedAggregates,
           inputType,
@@ -264,6 +280,7 @@ class DataSetWindowAggregate(
 
         // create groupReduceFunction for calculating the aggregations
         val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction(
+          generator,
           window,
           namedAggregates,
           inputType,
@@ -288,6 +305,7 @@ class DataSetWindowAggregate(
 
         // create groupReduceFunction for calculating the aggregations
         val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction(
+          generator,
           window,
           namedAggregates,
           inputType,
@@ -303,6 +321,7 @@ class DataSetWindowAggregate(
       } else {
         // non-grouping window
         val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction(
+          generator,
           window,
           namedAggregates,
           inputType,
@@ -320,6 +339,7 @@ class DataSetWindowAggregate(
   }
 
   private def createEventTimeSlidingWindowDataSet(
+      generator: CodeGenerator,
       inputDS: DataSet[Row],
       isTimeWindow: Boolean,
       size: Long,
@@ -330,6 +350,7 @@ class DataSetWindowAggregate(
     // create MapFunction for initializing the aggregations
     // it aligns the rowtime for pre-tumbling in case of a time-window for partial aggregates
     val mapFunction = createDataSetWindowPrepareMapFunction(
+      generator,
       window,
       namedAggregates,
       grouping,
@@ -365,6 +386,7 @@ class DataSetWindowAggregate(
         // create GroupReduceFunction
         // for pre-tumbling and replicating/omitting the content for each pane
         val prepareReduceFunction = createDataSetSlideWindowPrepareGroupReduceFunction(
+          generator,
           window,
           namedAggregates,
           grouping,
@@ -401,6 +423,7 @@ class DataSetWindowAggregate(
 
     // create GroupReduceFunction for final aggregation and conversion to output row
     val aggregateReduceFunction = createDataSetWindowAggregationGroupReduceFunction(
+      generator,
       window,
       namedAggregates,
       inputType,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index da5715317eea0..2c503c651a89a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -82,7 +82,7 @@ object AggregateUtil {
     val aggregationStateType: RowTypeInfo =
       createDataSetAggregateBufferDataType(Array(), aggregates, inputType)
 
-    val forwardMapping = (0 until inputType.getFieldCount).map(x => (x, x)).toArray
+    val forwardMapping = (0 until inputType.getFieldCount).toArray
     val aggMapping = aggregates.indices.map(x => x + inputType.getFieldCount).toArray
     val outputArity = inputType.getFieldCount + aggregates.length
 
@@ -93,7 +93,10 @@ object AggregateUtil {
       aggregates,
       aggFields,
       aggMapping,
+      partialResults = false,
       forwardMapping,
+      None,
+      None,
       outputArity
     )
 
@@ -153,7 +156,7 @@ object AggregateUtil {
     val aggregationStateType: RowTypeInfo = createAccumulatorRowType(aggregates)
     val inputRowType = FlinkTypeFactory.toInternalRowTypeInfo(inputType).asInstanceOf[RowTypeInfo]
 
-    val forwardMapping = (0 until inputType.getFieldCount).map(x => (x, x)).toArray
+    val forwardMapping = (0 until inputType.getFieldCount).toArray
     val aggMapping = aggregates.indices.map(x => x + inputType.getFieldCount).toArray
     val outputArity = inputType.getFieldCount + aggregates.length
 
@@ -164,7 +167,10 @@ object AggregateUtil {
       aggregates,
       aggFields,
       aggMapping,
+      partialResults = false,
       forwardMapping,
+      None,
+      None,
       outputArity
     )
 
@@ -225,6 +231,7 @@ object AggregateUtil {
     * NOTE: this function is only used for time based window on batch tables.
     */
   def createDataSetWindowPrepareMapFunction(
+      generator: CodeGenerator,
       window: LogicalWindow,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       groupings: Array[Int],
@@ -249,7 +256,7 @@ object AggregateUtil {
         val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
         (timeFieldPos, Some(asLong(size)))
 
-      case EventTimeTumblingGroupWindow(_, time, size) =>
+      case EventTimeTumblingGroupWindow(_, time, _) =>
         val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
         (timeFieldPos, None)
 
@@ -272,10 +279,25 @@ object AggregateUtil {
         throw new UnsupportedOperationException(s"$window is currently not supported on batch")
     }
 
-    new DataSetWindowAggMapFunction(
+    val aggMapping = aggregates.indices.toArray.map(_ + groupings.length)
+    val outputArity = aggregates.length + groupings.length + 1
+
+    val genFunction = generator.generateAggregations(
+      "DataSetAggregatePrepareMapHelper",
+      generator,
+      inputType,
       aggregates,
       aggFieldIndexes,
+      aggMapping,
+      partialResults = true,
       groupings,
+      None,
+      None,
+      outputArity
+    )
+
+    new DataSetWindowAggMapFunction(
+      genFunction,
       timeFieldPos,
       tumbleTimeWindowSize,
       mapReturnType)
@@ -309,6 +331,7 @@ object AggregateUtil {
     * NOTE: this function is only used for sliding windows with partial aggregates on batch tables.
     */
   def createDataSetSlideWindowPrepareGroupReduceFunction(
+      generator: CodeGenerator,
       window: LogicalWindow,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       groupings: Array[Int],
@@ -316,10 +339,10 @@ object AggregateUtil {
       isParserCaseSensitive: Boolean)
     : RichGroupReduceFunction[Row, Row] = {
 
-    val aggregates = transformToAggregateFunctions(
+    val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)._2
+      needRetraction = false)
 
     val returnType: RowTypeInfo = createDataSetAggregateBufferDataType(
       groupings,
@@ -327,13 +350,27 @@ object AggregateUtil {
       inputType,
       Some(Array(BasicTypeInfo.LONG_TYPE_INFO)))
 
+    val keysAndAggregatesArity = groupings.length + namedAggregates.length
+
     window match {
       case EventTimeSlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
         // sliding time-window for partial aggregations
-        new DataSetSlideTimeWindowAggReduceGroupFunction(
+        val genFunction = generator.generateAggregations(
+          "DataSetAggregatePrepareMapHelper",
+          generator,
+          inputType,
           aggregates,
-          groupings.length,
-          returnType.getArity - 1,
+          aggFieldIndexes,
+          aggregates.indices.map(_ + groupings.length).toArray,
+          partialResults = true,
+          groupings,
+          Some(aggregates.indices.map(_ + groupings.length).toArray),
+          None,
+          keysAndAggregatesArity + 1
+        )
+        new DataSetSlideTimeWindowAggReduceGroupFunction(
+          genFunction,
+          keysAndAggregatesArity,
           asLong(size),
           asLong(slide),
           returnType)
@@ -400,6 +437,7 @@ object AggregateUtil {
     * NOTE: this function is only used for window on batch tables.
     */
   def createDataSetWindowAggregationGroupReduceFunction(
+      generator: CodeGenerator,
       window: LogicalWindow,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       inputType: RelDataType,
@@ -414,19 +452,37 @@ object AggregateUtil {
       inputType,
       needRetraction = false)
 
-    // the mapping relation between field index of intermediate aggregate Row and output Row.
-    val groupingOffsetMapping = getGroupKeysMapping(inputType, outputType, groupings)
+    val aggMapping = aggregates.indices.toArray.map(_ + groupings.length)
+
+    val genPreAggFunction = generator.generateAggregations(
+      "GroupingWindowAggregateHelper",
+      generator,
+      inputType,
+      aggregates,
+      aggFieldIndexes,
+      aggMapping,
+      partialResults = true,
+      groupings,
+      Some(aggregates.indices.map(_ + groupings.length).toArray),
+      None,
+      outputType.getFieldCount
+    )
 
-    // the mapping relation between aggregate function index in list and its corresponding
-    // field index in output Row.
-    val aggOffsetMapping = getAggregateMapping(namedAggregates, outputType)
+    val genFinalAggFunction = generator.generateAggregations(
+      "GroupingWindowAggregateHelper",
+      generator,
+      inputType,
+      aggregates,
+      aggFieldIndexes,
+      aggMapping,
+      partialResults = false,
+      groupings.indices.toArray,
+      Some(aggregates.indices.map(_ + groupings.length).toArray),
+      None,
+      outputType.getFieldCount
+    )
 
-    if (groupingOffsetMapping.length != groupings.length ||
-      aggOffsetMapping.length != namedAggregates.length) {
-      throw new TableException(
-        "Could not find output field in input data type " +
-          "or aggregate functions.")
-    }
+    val keysAndAggregatesArity = groupings.length + namedAggregates.length
 
     window match {
       case EventTimeTumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
@@ -435,41 +491,33 @@ object AggregateUtil {
         if (doAllSupportPartialMerge(aggregates)) {
           // for incremental aggregations
           new DataSetTumbleTimeWindowAggReduceCombineFunction(
+            genPreAggFunction,
+            genFinalAggFunction,
             asLong(size),
             startPos,
             endPos,
-            aggregates,
-            groupingOffsetMapping,
-            aggOffsetMapping,
-            outputType.getFieldCount)
+            keysAndAggregatesArity)
         }
         else {
           // for non-incremental aggregations
           new DataSetTumbleTimeWindowAggReduceGroupFunction(
+            genFinalAggFunction,
             asLong(size),
             startPos,
             endPos,
-            aggregates,
-            groupingOffsetMapping,
-            aggOffsetMapping,
             outputType.getFieldCount)
         }
       case EventTimeTumblingGroupWindow(_, _, size) =>
         // tumbling count window
         new DataSetTumbleCountWindowAggReduceGroupFunction(
-          asLong(size),
-          aggregates,
-          groupingOffsetMapping,
-          aggOffsetMapping,
-          outputType.getFieldCount)
+          genFinalAggFunction,
+          asLong(size))
 
       case EventTimeSessionGroupWindow(_, _, gap) =>
         val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
         new DataSetSessionWindowAggReduceGroupFunction(
-          aggregates,
-          groupingOffsetMapping,
-          aggOffsetMapping,
-          outputType.getFieldCount,
+          genFinalAggFunction,
+          keysAndAggregatesArity,
           startPos,
           endPos,
           asLong(gap),
@@ -480,10 +528,9 @@ object AggregateUtil {
         if (doAllSupportPartialMerge(aggregates)) {
           // for partial aggregations
           new DataSetSlideWindowAggReduceCombineFunction(
-            aggregates,
-            groupingOffsetMapping,
-            aggOffsetMapping,
-            outputType.getFieldCount,
+            genPreAggFunction,
+            genFinalAggFunction,
+            keysAndAggregatesArity,
             startPos,
             endPos,
             asLong(size))
@@ -491,10 +538,8 @@ object AggregateUtil {
         else {
           // for non-partial aggregations
           new DataSetSlideWindowAggReduceGroupFunction(
-            aggregates,
-            groupingOffsetMapping,
-            aggOffsetMapping,
-            outputType.getFieldCount,
+            genFinalAggFunction,
+            keysAndAggregatesArity,
             startPos,
             endPos,
             asLong(size))
@@ -502,10 +547,8 @@ object AggregateUtil {
 
       case EventTimeSlidingGroupWindow(_, _, size, _) =>
         new DataSetSlideWindowAggReduceGroupFunction(
-            aggregates,
-            groupingOffsetMapping,
-            aggOffsetMapping,
-            outputType.getFieldCount,
+            genFinalAggFunction,
+            keysAndAggregatesArity,
             None,
             None,
             asLong(size))
@@ -537,15 +580,20 @@ object AggregateUtil {
     *
     */
   def createDataSetWindowAggregationMapPartitionFunction(
+    generator: CodeGenerator,
     window: LogicalWindow,
     namedAggregates: Seq[CalcitePair[AggregateCall, String]],
     inputType: RelDataType,
     groupings: Array[Int]): MapPartitionFunction[Row, Row] = {
 
-    val aggregates = transformToAggregateFunctions(
+    val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)._2
+      needRetraction = false)
+
+    val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
+
+    val keysAndAggregatesArity = groupings.length + namedAggregates.length
 
     window match {
       case EventTimeSessionGroupWindow(_, _, gap) =>
@@ -556,9 +604,23 @@ object AggregateUtil {
             inputType,
             Option(Array(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO)))
 
-        new DataSetSessionWindowAggregatePreProcessor(
+        val genFunction = generator.generateAggregations(
+          "GroupingWindowAggregateHelper",
+          generator,
+          inputType,
           aggregates,
-          groupings,
+          aggFieldIndexes,
+          aggMapping,
+          partialResults = true,
+          groupings.indices.toArray,
+          Some(aggregates.indices.map(_ + groupings.length).toArray),
+          None,
+          groupings.length + aggregates.length + 2
+        )
+
+        new DataSetSessionWindowAggregatePreProcessor(
+          genFunction,
+          keysAndAggregatesArity,
           asLong(gap),
           combineReturnType)
       case _ =>
@@ -585,16 +647,21 @@ object AggregateUtil {
     *
     */
   private[flink] def createDataSetWindowAggregationCombineFunction(
+      generator: CodeGenerator,
       window: LogicalWindow,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       inputType: RelDataType,
       groupings: Array[Int])
     : GroupCombineFunction[Row, Row] = {
 
-    val aggregates = transformToAggregateFunctions(
+    val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)._2
+      needRetraction = false)
+
+    val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
+
+    val keysAndAggregatesArity = groupings.length + namedAggregates.length
 
     window match {
 
@@ -606,9 +673,23 @@ object AggregateUtil {
             inputType,
             Option(Array(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO)))
 
-        new DataSetSessionWindowAggregatePreProcessor(
+        val genFunction = generator.generateAggregations(
+          "GroupingWindowAggregateHelper",
+          generator,
+          inputType,
           aggregates,
-          groupings,
+          aggFieldIndexes,
+          aggMapping,
+          partialResults = true,
+          groupings.indices.toArray,
+          Some(aggregates.indices.map(_ + groupings.length).toArray),
+          None,
+          groupings.length + aggregates.length + 2
+        )
+
+        new DataSetSessionWindowAggregatePreProcessor(
+          genFunction,
+          keysAndAggregatesArity,
           asLong(gap),
           combineReturnType)
 
@@ -625,6 +706,7 @@ object AggregateUtil {
     * respective output type are generated as well.
     */
   private[flink] def createDataSetAggregateFunctions(
+      generator: CodeGenerator,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       inputType: RelDataType,
       outputType: RelDataType,
@@ -645,51 +727,91 @@ object AggregateUtil {
       outputType
     )
 
-    val groupingSetsMapping: Array[(Int, Int)] = if (inGroupingSet) {
-      getGroupingSetsIndicatorMapping(inputType, outputType)
+    val constantFlags: Option[Array[(Int, Boolean)]] =
+    if (inGroupingSet) {
+
+      val groupingSetsMapping = getGroupingSetsIndicatorMapping(inputType, outputType)
+      val nonNullKeysFields = gkeyOutMapping.map(_._1)
+      val flags = for ((in, out) <- groupingSetsMapping) yield
+        (out, !nonNullKeysFields.contains(in))
+      Some(flags)
     } else {
-      Array()
+      None
     }
 
-    if (doAllSupportPartialMerge(aggregates)) {
+    val aggOutFields = aggOutMapping.map(_._1)
 
-      // compute grouping key and aggregation positions
-      val gkeyInFields = gkeyOutMapping.map(_._2)
-      val gkeyOutFields = gkeyOutMapping.map(_._1)
-      val aggOutFields = aggOutMapping.map(_._1)
+    if (doAllSupportPartialMerge(aggregates)) {
 
       // compute preaggregation type
-      val preAggFieldTypes = gkeyInFields
+      val preAggFieldTypes = gkeyOutMapping.map(_._2)
         .map(inputType.getFieldList.get(_).getType)
         .map(FlinkTypeFactory.toTypeInfo) ++ createAccumulatorType(aggregates)
       val preAggRowType = new RowTypeInfo(preAggFieldTypes: _*)
 
+      val genPreAggFunction = generator.generateAggregations(
+        "DataSetAggregatePrepareMapHelper",
+        generator,
+        inputType,
+        aggregates,
+        aggInFields,
+        aggregates.indices.map(_ + groupings.length).toArray,
+        partialResults = true,
+        groupings,
+        None,
+        None,
+        groupings.length + aggregates.length
+      )
+
+      // compute mapping of forwarded grouping keys
+      val gkeyMapping: Array[Int] = if (gkeyOutMapping.nonEmpty) {
+        val gkeyOutFields = gkeyOutMapping.map(_._1)
+        val mapping = Array.fill[Int](gkeyOutFields.max + 1)(-1)
+        gkeyOutFields.zipWithIndex.foreach(m => mapping(m._1) = m._2)
+        mapping
+      } else {
+        new Array[Int](0)
+      }
+
+      val genFinalAggFunction = generator.generateAggregations(
+        "DataSetAggregateFinalHelper",
+        generator,
+        inputType,
+        aggregates,
+        aggInFields,
+        aggOutFields,
+        partialResults = false,
+        gkeyMapping,
+        Some(aggregates.indices.map(_ + groupings.length).toArray),
+        constantFlags,
+        outputType.getFieldCount
+      )
+
       (
-        Some(new DataSetPreAggFunction(
-          aggregates,
-          aggInFields,
-          gkeyInFields
-        )),
+        Some(new DataSetPreAggFunction(genPreAggFunction)),
         Some(preAggRowType),
-        new DataSetFinalAggFunction(
-          aggregates,
-          aggOutFields,
-          gkeyOutFields,
-          groupingSetsMapping,
-          outputType.getFieldCount)
+        new DataSetFinalAggFunction(genFinalAggFunction)
       )
     }
     else {
+      val genFunction = generator.generateAggregations(
+        "DataSetAggregateHelper",
+        generator,
+        inputType,
+        aggregates,
+        aggInFields,
+        aggOutFields,
+        partialResults = false,
+        groupings,
+        None,
+        constantFlags,
+        outputType.getFieldCount
+      )
+
       (
         None,
         None,
-        new DataSetAggFunction(
-          aggregates,
-          aggInFields,
-          aggOutMapping,
-          gkeyOutMapping,
-          groupingSetsMapping,
-          outputType.getFieldCount)
+        new DataSetAggFunction(genFunction)
       )
     }
 
@@ -768,15 +890,12 @@ object AggregateUtil {
       aggregates,
       aggFields,
       aggMapping,
-      Array(),
-      outputArity)
-
-    val aggregateMapping = getAggregateMapping(namedAggregates, outputType)
-
-    if (aggregateMapping.length != namedAggregates.length) {
-      throw new TableException(
-        "Could not find output field in input data type or aggregate functions.")
-    }
+      partialResults = false,
+      Array(), // no fields are forwarded
+      None,
+      None,
+      outputArity
+    )
 
     val aggResultTypes = namedAggregates.map(a => FlinkTypeFactory.toTypeInfo(a.left.getType))
 
@@ -1169,62 +1288,6 @@ object AggregateUtil {
     new RowTypeInfo(aggTypes: _*)
   }
 
-  // Find the mapping between the index of aggregate list and aggregated value index in output Row.
-  private def getAggregateMapping(
-    namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-    outputType: RelDataType): Array[(Int, Int)] = {
-
-    // the mapping relation between aggregate function index in list and its corresponding
-    // field index in output Row.
-    var aggOffsetMapping = ArrayBuffer[(Int, Int)]()
-
-    outputType.getFieldList.zipWithIndex.foreach {
-      case (outputFieldType, outputIndex) =>
-        namedAggregates.zipWithIndex.foreach {
-          case (namedAggCall, aggregateIndex) =>
-            if (namedAggCall.getValue.equals(outputFieldType.getName) &&
-              namedAggCall.getKey.getType.equals(outputFieldType.getType)) {
-              aggOffsetMapping += ((outputIndex, aggregateIndex))
-            }
-        }
-    }
-
-    aggOffsetMapping.toArray
-  }
-
-  // Find the mapping between the index of group key in intermediate aggregate Row and its index
-  // in output Row.
-  private def getGroupKeysMapping(
-    inputDatType: RelDataType,
-    outputType: RelDataType,
-    groupKeys: Array[Int]): Array[(Int, Int)] = {
-
-    // the mapping relation between field index of intermediate aggregate Row and output Row.
-    var groupingOffsetMapping = ArrayBuffer[(Int, Int)]()
-
-    outputType.getFieldList.zipWithIndex.foreach {
-      case (outputFieldType, outputIndex) =>
-        inputDatType.getFieldList.zipWithIndex.foreach {
-          // find the field index in input data type.
-          case (inputFieldType, inputIndex) =>
-            if (outputFieldType.getName.equals(inputFieldType.getName) &&
-              outputFieldType.getType.equals(inputFieldType.getType)) {
-              // as aggregated field in output data type would not have a matched field in
-              // input data, so if inputIndex is not -1, it must be a group key. Then we can
-              // find the field index in buffer data by the group keys index mapping between
-              // input data and buffer data.
-              for (i <- groupKeys.indices) {
-                if (inputIndex == groupKeys(i)) {
-                  groupingOffsetMapping += ((outputIndex, i))
-                }
-              }
-            }
-        }
-    }
-
-    groupingOffsetMapping.toArray
-  }
-
   private def getTimeFieldPosition(
     timeField: Expression,
     inputType: RelDataType,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala
index 867943e701cef..5f459f98a4aa2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala
@@ -21,101 +21,66 @@ import java.lang.Iterable
 
 import org.apache.flink.api.common.functions.RichGroupReduceFunction
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * [[RichGroupReduceFunction]] to compute aggregates that do not support pre-aggregation for batch
   * (DataSet) queries.
   *
-  * @param aggregates The aggregate functions.
-  * @param aggInFields The positions of the aggregation input fields.
-  * @param gkeyOutMapping The mapping of group keys between input and output positions.
-  * @param aggOutMapping  The mapping of aggregates to output positions.
-  * @param groupingSetsMapping The mapping of grouping set keys between input and output positions.
-  * @param finalRowArity The arity of the final resulting row.
+  * @param genAggregations Code-generated [[GeneratedAggregations]]
   */
 class DataSetAggFunction(
-    private val aggregates: Array[AggregateFunction[_ <: Any]],
-    private val aggInFields: Array[Array[Int]],
-    private val aggOutMapping: Array[(Int, Int)],
-    private val gkeyOutMapping: Array[(Int, Int)],
-    private val groupingSetsMapping: Array[(Int, Int)],
-    private val finalRowArity: Int)
-  extends RichGroupReduceFunction[Row, Row] {
-
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(aggInFields)
-  Preconditions.checkNotNull(aggOutMapping)
-  Preconditions.checkNotNull(gkeyOutMapping)
-  Preconditions.checkNotNull(groupingSetsMapping)
+    private val genAggregations: GeneratedAggregationsFunction)
+  extends RichGroupReduceFunction[Row, Row]
+    with Compiler[GeneratedAggregations] {
 
   private var output: Row = _
+  private var accumulators: Row = _
 
-  private var intermediateGKeys: Option[Array[Int]] = None
-  private var accumulators: Array[Accumulator] = _
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    accumulators = new Array(aggregates.length)
-    output = new Row(finalRowArity)
-
-    if (!groupingSetsMapping.isEmpty) {
-      intermediateGKeys = Some(gkeyOutMapping.map(_._1))
-    }
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    output = function.createOutputRow()
+    accumulators = function.createAccumulators()
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
 
-    // create accumulators
-    var i = 0
-    while (i < aggregates.length) {
-      accumulators(i) = aggregates(i).createAccumulator()
-      i += 1
-    }
+    // reset accumulators
+    function.resetAccumulator(accumulators)
 
     val iterator = records.iterator()
 
+    var record: Row = null
     while (iterator.hasNext) {
-      val record = iterator.next()
+      record = iterator.next()
 
       // accumulate
-      i = 0
-      while (i < aggregates.length) {
-        aggregates(i).accumulate(accumulators(i), record.getField(aggInFields(i)(0)))
-        i += 1
-      }
-
-      // check if this record is the last record
-      if (!iterator.hasNext) {
-        // set group keys value to final output
-        i = 0
-        while (i < gkeyOutMapping.length) {
-          val (out, in) = gkeyOutMapping(i)
-          output.setField(out, record.getField(in))
-          i += 1
-        }
-
-        // set agg results to output
-        i = 0
-        while (i < aggOutMapping.length) {
-          val (out, in) = aggOutMapping(i)
-          output.setField(out, aggregates(in).getValue(accumulators(in)))
-          i += 1
-        }
-
-        // set grouping set flags to output
-        if (intermediateGKeys.isDefined) {
-          i = 0
-          while (i < groupingSetsMapping.length) {
-            val (in, out) = groupingSetsMapping(i)
-            output.setField(out, !intermediateGKeys.get.contains(in))
-            i += 1
-          }
-        }
-
-        out.collect(output)
-      }
+      function.accumulate(accumulators, record)
     }
+
+    // set group keys value to final output
+    function.setForwardedFields(record, output)
+
+    // set agg results to output
+    function.setAggregationResults(accumulators, output)
+
+    // set grouping set flags to output
+    function.setConstantFlags(output)
+
+    out.collect(output)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala
index e3db7a28e0a69..9b81992cba24a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala
@@ -19,60 +19,43 @@
 package org.apache.flink.table.runtime.aggregate
 
 import java.lang.Iterable
-import java.util.{ArrayList => JArrayList}
 
 import org.apache.flink.api.common.functions.RichGroupReduceFunction
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * [[RichGroupReduceFunction]] to compute the final result of a pre-aggregated aggregation
   * for batch (DataSet) queries.
   *
-  * @param aggregates The aggregate functions.
-  * @param aggOutFields The positions of the aggregation results in the output
-  * @param gkeyOutFields The positions of the grouping keys in the output
-  * @param groupingSetsMapping The mapping of grouping set keys between input and output positions.
-  * @param finalRowArity The arity of the final resulting row
+  * @param genAggregations Code-generated [[GeneratedAggregations]]
   */
 class DataSetFinalAggFunction(
-    private val aggregates: Array[AggregateFunction[_ <: Any]],
-    private val aggOutFields: Array[Int],
-    private val gkeyOutFields: Array[Int],
-    private val groupingSetsMapping: Array[(Int, Int)],
-    private val finalRowArity: Int)
-  extends RichGroupReduceFunction[Row, Row] {
-
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(aggOutFields)
-  Preconditions.checkNotNull(gkeyOutFields)
-  Preconditions.checkNotNull(groupingSetsMapping)
+    private val genAggregations: GeneratedAggregationsFunction)
+  extends RichGroupReduceFunction[Row, Row]
+    with Compiler[GeneratedAggregations] {
 
   private var output: Row = _
+  private var accumulators: Row = _
 
-  private val intermediateGKeys: Option[Array[Int]] = if (!groupingSetsMapping.isEmpty) {
-    Some(gkeyOutFields)
-  } else {
-    None
-  }
-
-  private val numAggs = aggregates.length
-  private val numGKeys = gkeyOutFields.length
-
-  private val accumulators: Array[JArrayList[Accumulator]] =
-    Array.fill(numAggs)(new JArrayList[Accumulator](2))
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    output = new Row(finalRowArity)
-
-    // init lists with two empty accumulators
-    for (i <- aggregates.indices) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulators(i).add(accumulator)
-      accumulators(i).add(accumulator)
-    }
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    output = function.createOutputRow()
+    accumulators = function.createAccumulators()
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
@@ -80,56 +63,24 @@ class DataSetFinalAggFunction(
     val iterator = records.iterator()
 
     // reset first accumulator
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulators(i).get(0))
-      i += 1
-    }
+    function.resetAccumulator(accumulators)
 
+    var record: Row = null
     while (iterator.hasNext) {
-      val record = iterator.next()
-
+      record = iterator.next()
       // accumulate
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(numGKeys + i).asInstanceOf[Accumulator]
-        accumulators(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulators(i))
-        // insert result into acc list
-        accumulators(i).set(0, retAcc)
-        i += 1
-      }
-
-      // check if this record is the last record
-      if (!iterator.hasNext) {
-        // set group keys value to final output
-        i = 0
-        while (i < gkeyOutFields.length) {
-          output.setField(gkeyOutFields(i), record.getField(i))
-          i += 1
-        }
-
-        // get final aggregate value and set to output.
-        i = 0
-        while (i < aggOutFields.length) {
-          output.setField(aggOutFields(i), aggregates(i).getValue(accumulators(i).get(0)))
-          i += 1
-        }
-
-        // set grouping set flags to output
-        if (intermediateGKeys.isDefined) {
-          i = 0
-          while (i < groupingSetsMapping.length) {
-            val (in, out) = groupingSetsMapping(i)
-            output.setField(out, !intermediateGKeys.get.contains(in))
-            i += 1
-          }
-        }
-
-        out.collect(output)
-      }
+      function.mergeAccumulatorsPair(accumulators, record)
     }
+
+    // set group keys value to final output
+    function.setForwardedFields(record, output)
+
+    // get final aggregate value and set to output.
+    function.setAggregationResults(accumulators, output)
+
+    // set grouping set flags to output
+    function.setConstantFlags(output)
+
+    out.collect(output)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala
index db49a53bcd194..8febe3e820ff4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala
@@ -21,85 +21,64 @@ import java.lang.Iterable
 
 import org.apache.flink.api.common.functions._
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * [[GroupCombineFunction]] and [[MapPartitionFunction]] to compute pre-aggregates for batch
   * (DataSet) queries.
   *
-  * @param aggregates The aggregate functions.
-  * @param aggInFields The positions of the aggregation input fields.
-  * @param groupingKeys The positions of the grouping keys in the input.
+  * @param genAggregations Code-generated [[GeneratedAggregations]]
   */
-class DataSetPreAggFunction(
-    private val aggregates: Array[AggregateFunction[_ <: Any]],
-    private val aggInFields: Array[Array[Int]],
-    private val groupingKeys: Array[Int])
+class DataSetPreAggFunction(genAggregations: GeneratedAggregationsFunction)
   extends AbstractRichFunction
   with GroupCombineFunction[Row, Row]
-  with MapPartitionFunction[Row, Row] {
-
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(aggInFields)
-  Preconditions.checkNotNull(groupingKeys)
+  with MapPartitionFunction[Row, Row]
+  with Compiler[GeneratedAggregations] {
 
   private var output: Row = _
-  private var accumulators: Array[Accumulator] = _
+  private var accumulators: Row = _
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    accumulators = new Array(aggregates.length)
-    output = new Row(groupingKeys.length + aggregates.length)
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    output = function.createOutputRow()
+    accumulators = function.createAccumulators()
   }
 
   override def combine(values: Iterable[Row], out: Collector[Row]): Unit = {
-    preaggregate(values, out)
-  }
-
-  override def mapPartition(values: Iterable[Row], out: Collector[Row]): Unit = {
-    preaggregate(values, out)
-  }
-
-  def preaggregate(records: Iterable[Row], out: Collector[Row]): Unit = {
+    // reset accumulators
+    function.resetAccumulator(accumulators)
 
-    // create accumulators
-    var i = 0
-    while (i < aggregates.length) {
-      accumulators(i) = aggregates(i).createAccumulator()
-      i += 1
-    }
-
-    val iterator = records.iterator()
+    val iterator = values.iterator()
 
+    var record: Row = null
     while (iterator.hasNext) {
-      val record = iterator.next()
-
+      record = iterator.next()
       // accumulate
-      i = 0
-      while (i < aggregates.length) {
-        aggregates(i).accumulate(accumulators(i), record.getField(aggInFields(i)(0)))
-        i += 1
-      }
-      // check if this record is the last record
-      if (!iterator.hasNext) {
-        // set group keys value to output
-        i = 0
-        while (i < groupingKeys.length) {
-          output.setField(i, record.getField(groupingKeys(i)))
-          i += 1
-        }
+      function.accumulate(accumulators, record)
+    }
 
-        // set agg results to output
-        i = 0
-        while (i < accumulators.length) {
-          output.setField(groupingKeys.length + i, accumulators(i))
-          i += 1
-        }
+    // set group keys and accumulators to output
+    function.setAggregationResults(accumulators, output)
+    function.setForwardedFields(record, output)
 
-        out.collect(output)
-      }
-    }
+    out.collect(output)
   }
 
+  override def mapPartition(values: Iterable[Row], out: Collector[Row]): Unit = {
+    combine(values, out)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
index d10857096862f..95699a24849aa 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
@@ -18,13 +18,13 @@
 package org.apache.flink.table.runtime.aggregate
 
 import java.lang.Iterable
-import java.util.{ArrayList => JArrayList}
 
 import org.apache.flink.api.common.functions.RichGroupReduceFunction
 import org.apache.flink.types.Row
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * It wraps the aggregate logic inside of
@@ -40,53 +40,45 @@ import org.apache.flink.util.{Collector, Preconditions}
   *  2. when partial aggregate is supported, the input data structure of reduce is
   * |groupKey1|groupKey2|sum1|count1|sum2|count2|windowStart|windowEnd|
   *
-  * @param aggregates             The aggregate functions.
-  * @param groupKeysMapping       The index mapping of group keys between intermediate aggregate Row
-  *                               and output Row.
-  * @param aggregateMapping       The index mapping between aggregate function list and
-  *                               aggregated value index in output Row.
-  * @param finalRowArity          The output row field count.
+  * @param genAggregations Code-generated [[GeneratedAggregations]]
+  * @param keysAndAggregatesArity    The total arity of keys and aggregates
   * @param finalRowWindowStartPos The relative window-start field position.
   * @param finalRowWindowEndPos   The relative window-end field position.
   * @param gap                    Session time window gap.
   */
 class DataSetSessionWindowAggReduceGroupFunction(
-    aggregates: Array[AggregateFunction[_ <: Any]],
-    groupKeysMapping: Array[(Int, Int)],
-    aggregateMapping: Array[(Int, Int)],
-    finalRowArity: Int,
+    genAggregations: GeneratedAggregationsFunction,
+    keysAndAggregatesArity: Int,
     finalRowWindowStartPos: Option[Int],
     finalRowWindowEndPos: Option[Int],
     gap: Long,
     isInputCombined: Boolean)
-  extends RichGroupReduceFunction[Row, Row] {
+  extends RichGroupReduceFunction[Row, Row]
+    with Compiler[GeneratedAggregations] {
 
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(groupKeysMapping)
+  private var collector: TimeWindowPropertyCollector = _
+  private val intermediateRowWindowStartPos = keysAndAggregatesArity
+  private val intermediateRowWindowEndPos = keysAndAggregatesArity + 1
 
-  private var aggregateBuffer: Row = _
   private var output: Row = _
-  private var collector: TimeWindowPropertyCollector = _
-  private val accumStartPos: Int = groupKeysMapping.length
-  private val intermediateRowArity: Int = accumStartPos + aggregates.length + 2
-  private val intermediateRowWindowStartPos = intermediateRowArity - 2
-  private val intermediateRowWindowEndPos = intermediateRowArity - 1
+  private var accumulators: Row = _
 
-  val accumulatorList: Array[JArrayList[Accumulator]] = Array.fill(aggregates.length) {
-    new JArrayList[Accumulator](2)
-  }
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    aggregateBuffer = new Row(intermediateRowArity)
-    output = new Row(finalRowArity)
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    output = function.createOutputRow()
+    accumulators = function.createAccumulators()
     collector = new TimeWindowPropertyCollector(finalRowWindowStartPos, finalRowWindowEndPos)
-
-    // init lists with two empty accumulators
-    for (i <- aggregates.indices) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulatorList(i).add(accumulator)
-      accumulatorList(i).add(accumulator)
-    }
   }
 
   /**
@@ -105,13 +97,8 @@ class DataSetSessionWindowAggReduceGroupFunction(
     var windowEnd: java.lang.Long = null
     var currentRowTime: java.lang.Long = null
 
-
-    // reset first accumulator in merge list
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-      i += 1
-    }
+    // reset accumulator
+    function.resetAccumulator(accumulators)
 
     val iterator = records.iterator()
 
@@ -125,38 +112,18 @@ class DataSetSessionWindowAggReduceGroupFunction(
         // calculate the current window and open a new window
         if (null != windowEnd) {
           // evaluate and emit the current window's result.
-          doEvaluateAndCollect(out, accumulatorList, windowStart, windowEnd)
-
-          // reset first accumulator in list
-          i = 0
-          while (i < aggregates.length) {
-            aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-            i += 1
-          }
+          doEvaluateAndCollect(out, windowStart, windowEnd)
+          // reset accumulator
+          function.resetAccumulator(accumulators)
         } else {
-          // set group keys value to final output.
-          i = 0
-          while (i < groupKeysMapping.length) {
-            val (after, previous) = groupKeysMapping(i)
-            output.setField(after, record.getField(previous))
-            i += 1
-          }
+          // set keys to output
+          function.setForwardedFields(record, output)
         }
 
         windowStart = record.getField(intermediateRowWindowStartPos).asInstanceOf[Long]
       }
 
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(accumStartPos + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
+      function.mergeAccumulatorsPair(accumulators, record)
 
       windowEnd = if (isInputCombined) {
         // partial aggregate is supported
@@ -167,15 +134,13 @@ class DataSetSessionWindowAggReduceGroupFunction(
       }
     }
     // evaluate and emit the current window's result.
-    doEvaluateAndCollect(out, accumulatorList, windowStart, windowEnd)
+    doEvaluateAndCollect(out, windowStart, windowEnd)
   }
 
   /**
     * Evaluate and emit the data of the current window.
     *
     * @param out             the collection of the aggregate results
-    * @param accumulatorList an array (indexed by aggregate index) of the accumulator lists for
-    *                        each aggregate
     * @param windowStart     the window's start attribute value is the min (rowtime) of all rows
     *                        in the window.
     * @param windowEnd       the window's end property value is max (rowtime) + gap for all rows
@@ -183,18 +148,11 @@ class DataSetSessionWindowAggReduceGroupFunction(
     */
   def doEvaluateAndCollect(
       out: Collector[Row],
-      accumulatorList: Array[JArrayList[Accumulator]],
       windowStart: Long,
       windowEnd: Long): Unit = {
 
-    // merge the accumulators and then get value for the final output
-    var i = 0
-    while (i < aggregateMapping.length) {
-      val (after, previous) = aggregateMapping(i)
-      val agg = aggregates(previous)
-      output.setField(after, agg.getValue(accumulatorList(previous).get(0)))
-      i += 1
-    }
+    // set value for the final output
+    function.setAggregationResults(accumulators, output)
 
     // adds TimeWindow properties to output then emit output
     if (finalRowWindowStartPos.isDefined || finalRowWindowEndPos.isDefined) {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
index acd9e63858b6c..22a2682ba3e8d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
@@ -18,55 +18,55 @@
 package org.apache.flink.table.runtime.aggregate
 
 import java.lang.Iterable
-import java.util.{ArrayList => JArrayList}
 
 import org.apache.flink.api.common.functions.{AbstractRichFunction, GroupCombineFunction, MapPartitionFunction}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.types.Row
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * This wraps the aggregate logic inside of
   * [[org.apache.flink.api.java.operators.GroupCombineOperator]].
   *
-  * @param aggregates          The aggregate functions.
-  * @param groupingKeys        The indexes of the grouping fields.
+  * @param genAggregations Code-generated [[GeneratedAggregations]]
+  * @param keysAndAggregatesArity    The total arity of keys and aggregates
   * @param gap                 Session time window gap.
   * @param intermediateRowType Intermediate row data type.
   */
 class DataSetSessionWindowAggregatePreProcessor(
-    aggregates: Array[AggregateFunction[_ <: Any]],
-    groupingKeys: Array[Int],
+    genAggregations: GeneratedAggregationsFunction,
+    keysAndAggregatesArity: Int,
     gap: Long,
     @transient intermediateRowType: TypeInformation[Row])
   extends AbstractRichFunction
   with MapPartitionFunction[Row,Row]
   with GroupCombineFunction[Row,Row]
-  with ResultTypeQueryable[Row] {
+  with ResultTypeQueryable[Row]
+  with Compiler[GeneratedAggregations] {
 
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(groupingKeys)
+  private var output: Row = _
+  private val rowTimeFieldPos = keysAndAggregatesArity
+  private var accumulators: Row = _
 
-  private var aggregateBuffer: Row = _
-  private val accumStartPos: Int = groupingKeys.length
-  private val rowTimeFieldPos = accumStartPos + aggregates.length
-
-  val accumulatorList: Array[JArrayList[Accumulator]] = Array.fill(aggregates.length) {
-    new JArrayList[Accumulator](2)
-  }
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    aggregateBuffer = new Row(rowTimeFieldPos + 2)
-
-    // init lists with two empty accumulators
-    for (i <- aggregates.indices) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulatorList(i).add(accumulator)
-      accumulatorList(i).add(accumulator)
-    }
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    accumulators = function.createAccumulators()
+    output = function.createOutputRow()
   }
 
   /**
@@ -79,43 +79,13 @@ class DataSetSessionWindowAggregatePreProcessor(
     *
     */
   override def combine(records: Iterable[Row], out: Collector[Row]): Unit = {
-    preProcessing(records, out)
-  }
-
-  /**
-    * Divide window based on the rowtime
-    * (current'rowtime - previous’rowtime > gap), and then merge data (within a unified window)
-    * into an aggregate buffer.
-    *
-    * @param records  Intermediate aggregate Rows.
-    * @return Pre partition intermediate aggregate Row.
-    *
-    */
-  override def mapPartition(records: Iterable[Row], out: Collector[Row]): Unit = {
-    preProcessing(records, out)
-  }
-
-  /**
-    * Intermediate aggregate Rows, divide window based on the rowtime
-    * (current'rowtime - previous’rowtime > gap), and then merge data (within a unified window)
-    * into an aggregate buffer.
-    *
-    * @param records Intermediate aggregate Rows.
-    * @return PreProcessing intermediate aggregate Row.
-    *
-    */
-  private def preProcessing(records: Iterable[Row], out: Collector[Row]): Unit = {
 
     var windowStart: java.lang.Long = null
     var windowEnd: java.lang.Long = null
     var currentRowTime: java.lang.Long = null
 
-    // reset first accumulator in merge list
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-      i += 1
-    }
+    // reset accumulator
+    function.resetAccumulator(accumulators)
 
     val iterator = records.iterator()
 
@@ -128,51 +98,44 @@ class DataSetSessionWindowAggregatePreProcessor(
         // calculate the current window and open a new window.
         if (windowEnd != null) {
           // emit the current window's merged data
-          doCollect(out, accumulatorList, windowStart, windowEnd)
-
-          // reset first value of accumulator list
-          i = 0
-          while (i < aggregates.length) {
-            aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-            i += 1
-          }
+          doCollect(out, windowStart, windowEnd)
+
+          // reset accumulator
+          function.resetAccumulator(accumulators)
         } else {
           // set group keys to aggregateBuffer.
-          i = 0
-          while (i < groupingKeys.length) {
-            aggregateBuffer.setField(i, record.getField(i))
-            i += 1
-          }
+          function.setForwardedFields(record, output)
         }
 
         windowStart = record.getField(rowTimeFieldPos).asInstanceOf[Long]
       }
 
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(accumStartPos + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
+      function.mergeAccumulatorsPair(accumulators, record)
 
       // the current rowtime is the last rowtime of the next calculation.
       windowEnd = currentRowTime + gap
     }
     // emit the merged data of the current window.
-    doCollect(out, accumulatorList, windowStart, windowEnd)
+    doCollect(out, windowStart, windowEnd)
+  }
+
+  /**
+    * Divide window based on the rowtime
+    * (current'rowtime - previous’rowtime > gap), and then merge data (within a unified window)
+    * into an aggregate buffer.
+    *
+    * @param records  Intermediate aggregate Rows.
+    * @return Pre partition intermediate aggregate Row.
+    *
+    */
+  override def mapPartition(records: Iterable[Row], out: Collector[Row]): Unit = {
+    combine(records, out)
   }
 
   /**
     * Emit the merged data of the current window.
     *
     * @param out             the collection of the aggregate results
-    * @param accumulatorList an array (indexed by aggregate index) of the accumulator lists for
-    *                        each aggregate
     * @param windowStart     the window's start attribute value is the min (rowtime)
     *                        of all rows in the window.
     * @param windowEnd       the window's end property value is max (rowtime) + gap
@@ -180,24 +143,18 @@ class DataSetSessionWindowAggregatePreProcessor(
     */
   def doCollect(
       out: Collector[Row],
-      accumulatorList: Array[JArrayList[Accumulator]],
       windowStart: Long,
       windowEnd: Long): Unit = {
 
-    // merge the accumulators into one accumulator
-    var i = 0
-    while (i < aggregates.length) {
-      aggregateBuffer.setField(accumStartPos + i, accumulatorList(i).get(0))
-      i += 1
-    }
+    function.setAggregationResults(accumulators, output)
 
     // intermediate Row WindowStartPos is rowtime pos.
-    aggregateBuffer.setField(rowTimeFieldPos, windowStart)
+    output.setField(rowTimeFieldPos, windowStart)
 
     // intermediate Row WindowEndPos is rowtime pos + 1.
-    aggregateBuffer.setField(rowTimeFieldPos + 1, windowEnd)
+    output.setField(rowTimeFieldPos + 1, windowEnd)
 
-    out.collect(aggregateBuffer)
+    out.collect(output)
   }
 
   override def getProducedType: TypeInformation[Row] = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala
index 422989b0bbeb6..b3a19a443328a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala
@@ -18,16 +18,16 @@
 package org.apache.flink.table.runtime.aggregate
 
 import java.lang.Iterable
-import java.util.{ArrayList => JArrayList}
 
 import org.apache.flink.api.common.functions.{CombineFunction, RichGroupReduceFunction}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * It is used for sliding windows on batch for time-windows. It takes a prepared input row (with
@@ -38,106 +38,78 @@ import org.apache.flink.util.{Collector, Preconditions}
   * it does no final aggregate evaluation. It also includes the logic of
   * [[DataSetSlideTimeWindowAggFlatMapFunction]].
   *
-  * @param aggregates aggregate functions
-  * @param groupingKeysLength number of grouping keys
-  * @param timeFieldPos position of aligned time field
+  * @param genAggregations Code-generated [[GeneratedAggregations]]
+  * @param keysAndAggregatesArity The total arity of keys and aggregates
   * @param windowSize window size of the sliding window
   * @param windowSlide window slide of the sliding window
   * @param returnType return type of this function
   */
 class DataSetSlideTimeWindowAggReduceGroupFunction(
-    private val aggregates: Array[AggregateFunction[_ <: Any]],
-    private val groupingKeysLength: Int,
-    private val timeFieldPos: Int,
+    private val genAggregations: GeneratedAggregationsFunction,
+    private val keysAndAggregatesArity: Int,
     private val windowSize: Long,
     private val windowSlide: Long,
     @transient private val returnType: TypeInformation[Row])
   extends RichGroupReduceFunction[Row, Row]
   with CombineFunction[Row, Row]
-  with ResultTypeQueryable[Row] {
+  with ResultTypeQueryable[Row]
+  with Compiler[GeneratedAggregations] {
 
-  Preconditions.checkNotNull(aggregates)
+  private val timeFieldPos = returnType.getArity - 1
+  private val intermediateWindowStartPos = keysAndAggregatesArity
 
   protected var intermediateRow: Row = _
-  // add one field to store window start
-  protected val intermediateRowArity: Int = groupingKeysLength + aggregates.length + 1
-  protected val accumulatorList: Array[JArrayList[Accumulator]] = Array.fill(aggregates.length) {
-    new JArrayList[Accumulator](2)
-  }
-  private val intermediateWindowStartPos: Int = intermediateRowArity - 1
+  private var accumulators: Row = _
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    intermediateRow = new Row(intermediateRowArity)
-
-    // init lists with two empty accumulators
-    var i = 0
-    while (i < aggregates.length) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulatorList(i).add(accumulator)
-      accumulatorList(i).add(accumulator)
-      i += 1
-    }
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    accumulators = function.createAccumulators()
+    intermediateRow = function.createOutputRow()
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
 
     // reset first accumulator
-    var i = 0
-    while (i < aggregates.length) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulatorList(i).set(0, accumulator)
-      i += 1
-    }
+    function.resetAccumulator(accumulators)
 
     val iterator = records.iterator()
 
+    var record: Row = null
     while (iterator.hasNext) {
-      val record = iterator.next()
+      record = iterator.next()
 
       // accumulate
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(groupingKeysLength + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
+      function.mergeAccumulatorsPair(accumulators, record)
+    }
+
+    val windowStart = record.getField(timeFieldPos).asInstanceOf[Long]
+
+    // adopted from SlidingEventTimeWindows.assignWindows
+    var start: Long = TimeWindow.getWindowStartWithOffset(windowStart, 0, windowSlide)
+
+    // skip preparing output if it is not necessary
+    if (start > windowStart - windowSize) {
+
+      // set group keys and partial accumulated result
+      function.setAggregationResults(accumulators, intermediateRow)
+      function.setForwardedFields(record, intermediateRow)
 
-      // trigger tumbling evaluation
-      if (!iterator.hasNext) {
-        val windowStart = record.getField(timeFieldPos).asInstanceOf[Long]
-
-        // adopted from SlidingEventTimeWindows.assignWindows
-        var start: Long = TimeWindow.getWindowStartWithOffset(windowStart, 0, windowSlide)
-
-        // skip preparing output if it is not necessary
-        if (start > windowStart - windowSize) {
-
-          // set group keys
-          i = 0
-          while (i < groupingKeysLength) {
-            intermediateRow.setField(i, record.getField(i))
-            i += 1
-          }
-
-          // set accumulators
-          i = 0
-          while (i < aggregates.length) {
-            intermediateRow.setField(groupingKeysLength + i, accumulatorList(i).get(0))
-            i += 1
-          }
-
-          // adopted from SlidingEventTimeWindows.assignWindows
-          while (start > windowStart - windowSize) {
-            intermediateRow.setField(intermediateWindowStartPos, start)
-            out.collect(intermediateRow)
-            start -= windowSlide
-          }
-        }
+      // adopted from SlidingEventTimeWindows.assignWindows
+      while (start > windowStart - windowSize) {
+        intermediateRow.setField(intermediateWindowStartPos, start)
+        out.collect(intermediateRow)
+        start -= windowSlide
       }
     }
   }
@@ -145,54 +117,21 @@ class DataSetSlideTimeWindowAggReduceGroupFunction(
   override def combine(records: Iterable[Row]): Row = {
 
     // reset first accumulator
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-      i += 1
-    }
+    function.resetAccumulator(accumulators)
 
     val iterator = records.iterator()
+    var record: Row = null
     while (iterator.hasNext) {
-      val record = iterator.next()
-
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(groupingKeysLength + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
-
-      // check if this record is the last record
-      if (!iterator.hasNext) {
-
-        // set group keys
-        i = 0
-        while (i < groupingKeysLength) {
-          intermediateRow.setField(i, record.getField(i))
-          i += 1
-        }
-
-        // set accumulators
-        i = 0
-        while (i < aggregates.length) {
-          intermediateRow.setField(groupingKeysLength + i, accumulatorList(i).get(0))
-          i += 1
-        }
-
-        intermediateRow.setField(timeFieldPos, record.getField(timeFieldPos))
-
-        return intermediateRow
-      }
+      record = iterator.next()
+      function.mergeAccumulatorsPair(accumulators, record)
     }
+    // set group keys and partial accumulated result
+    function.setAggregationResults(accumulators, intermediateRow)
+    function.setForwardedFields(record, intermediateRow)
+
+    intermediateRow.setField(timeFieldPos, record.getField(timeFieldPos))
 
-    // this code path should never be reached as we return before the loop finishes
-    // we need this to prevent a compiler error
-    throw new IllegalArgumentException("Group is empty. This should never happen.")
+    intermediateRow
   }
 
   override def getProducedType: TypeInformation[Row] = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
index 8eac79d02e9c2..381d4436c75b8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
@@ -20,7 +20,8 @@ package org.apache.flink.table.runtime.aggregate
 import java.lang.Iterable
 
 import org.apache.flink.api.common.functions.CombineFunction
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.GeneratedAggregationsFunction
 import org.apache.flink.types.Row
 
 /**
@@ -30,87 +31,62 @@ import org.apache.flink.types.Row
   *
   * It is used for sliding on batch for both time and count-windows.
   *
-  * @param aggregates aggregate functions.
-  * @param groupKeysMapping index mapping of group keys between intermediate aggregate Row
-  *                         and output Row.
-  * @param aggregateMapping index mapping between aggregate function list and aggregated value
-  *                         index in output Row.
-  * @param finalRowArity output row field count
+  * @param genPreAggregations Code-generated [[GeneratedAggregations]] for partial aggregation.
+  * @param genFinalAggregations Code-generated [[GeneratedAggregations]] for final aggregation.
+  * @param keysAndAggregatesArity The total arity of keys and aggregates
   * @param finalRowWindowStartPos relative window-start position to last field of output row
   * @param finalRowWindowEndPos relative window-end position to last field of output row
   * @param windowSize size of the window, used to determine window-end for output row
   */
 class DataSetSlideWindowAggReduceCombineFunction(
-    aggregates: Array[AggregateFunction[_ <: Any]],
-    groupKeysMapping: Array[(Int, Int)],
-    aggregateMapping: Array[(Int, Int)],
-    finalRowArity: Int,
+    genPreAggregations: GeneratedAggregationsFunction,
+    genFinalAggregations: GeneratedAggregationsFunction,
+    keysAndAggregatesArity: Int,
     finalRowWindowStartPos: Option[Int],
     finalRowWindowEndPos: Option[Int],
     windowSize: Long)
   extends DataSetSlideWindowAggReduceGroupFunction(
-    aggregates,
-    groupKeysMapping,
-    aggregateMapping,
-    finalRowArity,
+    genFinalAggregations,
+    keysAndAggregatesArity,
     finalRowWindowStartPos,
     finalRowWindowEndPos,
     windowSize)
   with CombineFunction[Row, Row] {
 
-  private val intermediateRowArity: Int = groupKeysMapping.length + aggregateMapping.length + 1
-  private val intermediateRow: Row = new Row(intermediateRowArity)
+  private val intermediateRow: Row = new Row(keysAndAggregatesArity + 1)
 
-  override def combine(records: Iterable[Row]): Row = {
-
-    // reset first accumulator
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-      i += 1
-    }
-
-    val iterator = records.iterator()
-    while (iterator.hasNext) {
-      val record = iterator.next()
+  protected var preAggfunction: GeneratedAggregations = _
 
-      // accumulate
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(groupKeysMapping.length + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
+  override def open(config: Configuration): Unit = {
+    super.open(config)
 
-      // check if this record is the last record
-      if (!iterator.hasNext) {
-        // set group keys
-        i = 0
-        while (i < groupKeysMapping.length) {
-          intermediateRow.setField(i, record.getField(i))
-          i += 1
-        }
+    LOG.debug(s"Compiling AggregateHelper: $genPreAggregations.name \n\n " +
+      s"Code:\n$genPreAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genPreAggregations.name,
+      genPreAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    preAggfunction = clazz.newInstance()
+  }
 
-        // set the partial accumulated result
-        i = 0
-        while (i < aggregates.length) {
-          intermediateRow.setField(groupKeysMapping.length + i, accumulatorList(i).get(0))
-          i += 1
-        }
+  override def combine(records: Iterable[Row]): Row = {
 
-        intermediateRow.setField(windowStartPos, record.getField(windowStartPos))
+    // reset accumulator
+    preAggfunction.resetAccumulator(accumulators)
 
-        return intermediateRow
-      }
+    val iterator = records.iterator()
+    var record: Row = null
+    while (iterator.hasNext) {
+      record = iterator.next()
+      preAggfunction.mergeAccumulatorsPair(accumulators, record)
     }
+    // set group keys and partial accumulated result
+    preAggfunction.setAggregationResults(accumulators, intermediateRow)
+    preAggfunction.setForwardedFields(record, intermediateRow)
+
+    intermediateRow.setField(windowStartPos, record.getField(windowStartPos))
 
-    // this code path should never be reached as we return before the loop finishes
-    // we need this to prevent a compiler error
-    throw new IllegalArgumentException("Group is empty. This should never happen.")
+    intermediateRow
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
index d6bc006e38f7a..a221c53be6e82 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
@@ -18,13 +18,13 @@
 package org.apache.flink.table.runtime.aggregate
 
 import java.lang.Iterable
-import java.util.{ArrayList => JArrayList}
 
 import org.apache.flink.api.common.functions.RichGroupReduceFunction
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * It wraps the aggregate logic inside of
@@ -32,109 +32,72 @@ import org.apache.flink.util.{Collector, Preconditions}
   *
   * It is used for sliding on batch for both time and count-windows.
   *
-  * @param aggregates aggregate functions.
-  * @param groupKeysMapping index mapping of group keys between intermediate aggregate Row
-  *                         and output Row.
-  * @param aggregateMapping index mapping between aggregate function list and aggregated value
-  *                         index in output Row.
-  * @param finalRowArity output row field count
+  * @param genAggregations Code-generated [[GeneratedAggregations]]
+  * @param keysAndAggregatesArity The total arity of keys and aggregates
   * @param finalRowWindowStartPos relative window-start position to last field of output row
   * @param finalRowWindowEndPos relative window-end position to last field of output row
   * @param windowSize size of the window, used to determine window-end for output row
   */
 class DataSetSlideWindowAggReduceGroupFunction(
-    aggregates: Array[AggregateFunction[_ <: Any]],
-    groupKeysMapping: Array[(Int, Int)],
-    aggregateMapping: Array[(Int, Int)],
-    finalRowArity: Int,
+    genAggregations: GeneratedAggregationsFunction,
+    keysAndAggregatesArity: Int,
     finalRowWindowStartPos: Option[Int],
     finalRowWindowEndPos: Option[Int],
     windowSize: Long)
-  extends RichGroupReduceFunction[Row, Row] {
-
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(groupKeysMapping)
+  extends RichGroupReduceFunction[Row, Row]
+    with Compiler[GeneratedAggregations] {
 
   private var collector: TimeWindowPropertyCollector = _
+  protected val windowStartPos: Int = keysAndAggregatesArity
+
   private var output: Row = _
-  private val accumulatorStartPos: Int = groupKeysMapping.length
-  protected val windowStartPos: Int = accumulatorStartPos + aggregates.length
+  protected var accumulators: Row = _
 
-  val accumulatorList: Array[JArrayList[Accumulator]] = Array.fill(aggregates.length) {
-    new JArrayList[Accumulator](2)
-  }
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  protected var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    output = new Row(finalRowArity)
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    output = function.createOutputRow()
+    accumulators = function.createAccumulators()
     collector = new TimeWindowPropertyCollector(finalRowWindowStartPos, finalRowWindowEndPos)
-
-    // init lists with two empty accumulators
-    var i = 0
-    while (i < aggregates.length) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulatorList(i).add(accumulator)
-      accumulatorList(i).add(accumulator)
-      i += 1
-    }
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
 
-    // reset first accumulator
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-      i += 1
-    }
+    // reset accumulator
+    function.resetAccumulator(accumulators)
 
     val iterator = records.iterator()
+    var record: Row = null
     while (iterator.hasNext) {
-      val record = iterator.next()
-
-      // accumulate
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(accumulatorStartPos + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
+      record = iterator.next()
+      function.mergeAccumulatorsPair(accumulators, record)
+    }
 
-      // check if this record is the last record
-      if (!iterator.hasNext) {
-        // set group keys value to final output
-        i = 0
-        while (i < groupKeysMapping.length) {
-          val mapping = groupKeysMapping(i)
-          output.setField(mapping._1, record.getField(mapping._2))
-          i += 1
-        }
+    // set group keys value to final output
+    function.setForwardedFields(record, output)
 
-        // get final aggregate value and set to output.
-        i = 0
-        while (i < aggregateMapping.length) {
-          val mapping = aggregateMapping(i)
-          val agg = aggregates(i)
-          val result = agg.getValue(accumulatorList(mapping._2).get(0))
-          output.setField(mapping._1, result)
-          i += 1
-        }
+    // get final aggregate value and set to output
+    function.setAggregationResults(accumulators, output)
 
-        // adds TimeWindow properties to output then emit output
-        if (finalRowWindowStartPos.isDefined || finalRowWindowEndPos.isDefined) {
-          collector.wrappedCollector = out
-          collector.windowStart = record.getField(windowStartPos).asInstanceOf[Long]
-          collector.windowEnd = collector.windowStart + windowSize
+    // adds TimeWindow properties to output then emit output
+    if (finalRowWindowStartPos.isDefined || finalRowWindowEndPos.isDefined) {
+      collector.wrappedCollector = out
+      collector.windowStart = record.getField(windowStartPos).asInstanceOf[Long]
+      collector.windowEnd = collector.windowStart + windowSize
 
-          collector.collect(output)
-        } else {
-          out.collect(output)
-        }
-      }
+      collector.collect(output)
+    } else {
+      out.collect(output)
     }
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala
index 0a525f8038268..0e73f7b34baf2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala
@@ -18,106 +18,69 @@
 package org.apache.flink.table.runtime.aggregate
 
 import java.lang.Iterable
-import java.util.{ArrayList => JArrayList}
 
 import org.apache.flink.api.common.functions.RichGroupReduceFunction
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * It wraps the aggregate logic inside of
   * [[org.apache.flink.api.java.operators.GroupReduceOperator]].
   * It is only used for tumbling count-window on batch.
   *
+  * @param genAggregations  Code-generated [[GeneratedAggregations]]
   * @param windowSize       Tumble count window size
-  * @param aggregates       The aggregate functions.
-  * @param groupKeysMapping The index mapping of group keys between intermediate aggregate Row
-  *                         and output Row.
-  * @param aggregateMapping The index mapping between aggregate function list and aggregated value
-  *                         index in output Row.
-  * @param finalRowArity    The output row field count
   */
 class DataSetTumbleCountWindowAggReduceGroupFunction(
-    private val windowSize: Long,
-    private val aggregates: Array[AggregateFunction[_ <: Any]],
-    private val groupKeysMapping: Array[(Int, Int)],
-    private val aggregateMapping: Array[(Int, Int)],
-    private val finalRowArity: Int)
-  extends RichGroupReduceFunction[Row, Row] {
-
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(groupKeysMapping)
+    private val genAggregations: GeneratedAggregationsFunction,
+    private val windowSize: Long)
+  extends RichGroupReduceFunction[Row, Row]
+    with Compiler[GeneratedAggregations] {
 
   private var output: Row = _
-  private val accumStartPos: Int = groupKeysMapping.length
+  private var accumulators: Row = _
 
-  val accumulatorList: Array[JArrayList[Accumulator]] = Array.fill(aggregates.length) {
-    new JArrayList[Accumulator](2)
-  }
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    output = new Row(finalRowArity)
-
-    // init lists with two empty accumulators
-    for (i <- aggregates.indices) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulatorList(i).add(accumulator)
-      accumulatorList(i).add(accumulator)
-    }
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    output = function.createOutputRow()
+    accumulators = function.createAccumulators()
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
 
     var count: Long = 0
     val iterator = records.iterator()
-    var i = 0
 
     while (iterator.hasNext) {
 
       if (count == 0) {
-        // reset first accumulator
-        i = 0
-        while (i < aggregates.length) {
-          aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-          i += 1
-        }
+        function.resetAccumulator(accumulators)
       }
 
       val record = iterator.next()
       count += 1
 
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(accumStartPos + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
+      accumulators = function.mergeAccumulatorsPair(accumulators, record)
 
       if (windowSize == count) {
         // set group keys value to final output.
-        i = 0
-        while (i < groupKeysMapping.length) {
-          val (after, previous) = groupKeysMapping(i)
-          output.setField(after, record.getField(previous))
-          i += 1
-        }
-
-        // merge the accumulators and then get value for the final output
-        i = 0
-        while (i < aggregateMapping.length) {
-          val (after, previous) = aggregateMapping(i)
-          val agg = aggregates(previous)
-          output.setField(after, agg.getValue(accumulatorList(previous).get(0)))
-          i += 1
-        }
+        function.setForwardedFields(record, output)
 
+        function.setAggregationResults(accumulators, output)
         // emit the output
         out.collect(output)
         count = 0
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceCombineFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceCombineFunction.scala
index 904c76c0ba034..4a459b2f4e100 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceCombineFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceCombineFunction.scala
@@ -20,7 +20,8 @@ package org.apache.flink.table.runtime.aggregate
 import java.lang.Iterable
 
 import org.apache.flink.api.common.functions.CombineFunction
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.GeneratedAggregationsFunction
 import org.apache.flink.types.Row
 
 /**
@@ -29,34 +30,45 @@ import org.apache.flink.types.Row
   * [[org.apache.flink.api.java.operators.GroupCombineOperator]].
   * It is used for tumbling time-window on batch.
   *
-  * @param windowSize       Tumbling time window size
-  * @param windowStartPos   The relative window-start field position to the last field of output row
-  * @param windowEndPos     The relative window-end field position to the last field of output row
-  * @param aggregates       The aggregate functions.
-  * @param groupKeysMapping The index mapping of group keys between intermediate aggregate Row
-  *                         and output Row.
-  * @param aggregateMapping The index mapping between aggregate function list and aggregated value
-  *                         index in output Row.
-  * @param finalRowArity    The output row field count
+  * @param genPreAggregations        Code-generated [[GeneratedAggregations]] for partial aggs.
+  * @param genFinalAggregations        Code-generated [[GeneratedAggregations]] for final aggs.
+  * @param windowSize             Tumbling time window size
+  * @param windowStartPos         The relative window-start field position to the last field of
+  *                               output row
+  * @param windowEndPos           The relative window-end field position to the last field of
+  *                               output row
+  * @param keysAndAggregatesArity The total arity of keys and aggregates
   */
 class DataSetTumbleTimeWindowAggReduceCombineFunction(
+    genPreAggregations: GeneratedAggregationsFunction,
+    genFinalAggregations: GeneratedAggregationsFunction,
     windowSize: Long,
     windowStartPos: Option[Int],
     windowEndPos: Option[Int],
-    aggregates: Array[AggregateFunction[_ <: Any]],
-    groupKeysMapping: Array[(Int, Int)],
-    aggregateMapping: Array[(Int, Int)],
-    finalRowArity: Int)
+    keysAndAggregatesArity: Int)
   extends DataSetTumbleTimeWindowAggReduceGroupFunction(
+    genFinalAggregations,
     windowSize,
     windowStartPos,
     windowEndPos,
-    aggregates,
-    groupKeysMapping,
-    aggregateMapping,
-    finalRowArity)
+    keysAndAggregatesArity)
     with CombineFunction[Row, Row] {
 
+  protected var preAggfunction: GeneratedAggregations = _
+
+  override def open(config: Configuration): Unit = {
+    super.open(config)
+
+    LOG.debug(s"Compiling AggregateHelper: $genPreAggregations.name \n\n " +
+      s"Code:\n$genPreAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genPreAggregations.name,
+      genPreAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    preAggfunction = clazz.newInstance()
+  }
+
   /**
     * For sub-grouped intermediate aggregate Rows, merge all of them into aggregate buffer,
     *
@@ -69,47 +81,21 @@ class DataSetTumbleTimeWindowAggReduceCombineFunction(
     var last: Row = null
     val iterator = records.iterator()
 
-    // reset first accumulator in merge list
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-      i += 1
-    }
+    // reset accumulator
+    preAggfunction.resetAccumulator(accumulators)
 
     while (iterator.hasNext) {
       val record = iterator.next()
-
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(groupKeysMapping.length + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
-
+      preAggfunction.mergeAccumulatorsPair(accumulators, record)
       last = record
     }
 
-    // set the partial merged result to the aggregateBuffer
-    i = 0
-    while (i < aggregates.length) {
-      aggregateBuffer.setField(groupKeysMapping.length + i, accumulatorList(i).get(0))
-      i += 1
-    }
-
-    // set group keys to aggregateBuffer.
-    i = 0
-    while (i < groupKeysMapping.length) {
-      aggregateBuffer.setField(i, last.getField(i))
-      i += 1
-    }
+    // set group keys and partial merged result to aggregateBuffer
+    preAggfunction.setAggregationResults(accumulators, aggregateBuffer)
+    preAggfunction.setForwardedFields(last, aggregateBuffer)
 
     // set the rowtime attribute
-    val rowtimePos = groupKeysMapping.length + aggregates.length
+    val rowtimePos = keysAndAggregatesArity
 
     aggregateBuffer.setField(rowtimePos, last.getField(rowtimePos))
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
index 99e2a0aa006ae..f4a1fc51c9361 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
@@ -18,65 +18,56 @@
 package org.apache.flink.table.runtime.aggregate
 
 import java.lang.Iterable
-import java.util.{ArrayList => JArrayList}
 
 import org.apache.flink.api.common.functions.RichGroupReduceFunction
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
 
 /**
   * It wraps the aggregate logic inside of
   * [[org.apache.flink.api.java.operators.GroupReduceOperator]]. It is used for tumbling time-window
   * on batch.
   *
+  * @param genAggregations  Code-generated [[GeneratedAggregations]]
   * @param windowSize       Tumbling time window size
   * @param windowStartPos   The relative window-start field position to the last field of output row
   * @param windowEndPos     The relative window-end field position to the last field of output row
-  * @param aggregates       The aggregate functions.
-  * @param groupKeysMapping The index mapping of group keys between intermediate aggregate Row
-  *                         and output Row.
-  * @param aggregateMapping The index mapping between aggregate function list and aggregated value
-  *                         index in output Row.
-  * @param finalRowArity    The output row field count
+  * @param keysAndAggregatesArity    The total arity of keys and aggregates
   */
 class DataSetTumbleTimeWindowAggReduceGroupFunction(
+    genAggregations: GeneratedAggregationsFunction,
     windowSize: Long,
     windowStartPos: Option[Int],
     windowEndPos: Option[Int],
-    aggregates: Array[AggregateFunction[_ <: Any]],
-    groupKeysMapping: Array[(Int, Int)],
-    aggregateMapping: Array[(Int, Int)],
-    finalRowArity: Int)
-  extends RichGroupReduceFunction[Row, Row] {
-
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(groupKeysMapping)
+    keysAndAggregatesArity: Int)
+  extends RichGroupReduceFunction[Row, Row]
+    with Compiler[GeneratedAggregations] {
 
   private var collector: TimeWindowPropertyCollector = _
-  protected var aggregateBuffer: Row = _
-  private var output: Row = _
-  private val accumStartPos: Int = groupKeysMapping.length
-  private val rowtimePos: Int = accumStartPos + aggregates.length
-  private val intermediateRowArity: Int = rowtimePos + 1
+  protected var aggregateBuffer: Row = new Row(keysAndAggregatesArity + 1)
 
+  private var output: Row = _
+  protected var accumulators: Row = _
 
-  val accumulatorList: Array[JArrayList[Accumulator]] = Array.fill(aggregates.length) {
-    new JArrayList[Accumulator](2)
-  }
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  protected var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    aggregateBuffer = new Row(intermediateRowArity)
-    output = new Row(finalRowArity)
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getClass.getClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    output = function.createOutputRow()
+    accumulators = function.createAccumulators()
     collector = new TimeWindowPropertyCollector(windowStartPos, windowEndPos)
-
-    // init lists with two empty accumulators
-    for (i <- aggregates.indices) {
-      val accumulator = aggregates(i).createAccumulator()
-      accumulatorList(i).add(accumulator)
-      accumulatorList(i).add(accumulator)
-    }
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
@@ -84,51 +75,23 @@ class DataSetTumbleTimeWindowAggReduceGroupFunction(
     var last: Row = null
     val iterator = records.iterator()
 
-    // reset first accumulator in merge list
-    var i = 0
-    while (i < aggregates.length) {
-      aggregates(i).resetAccumulator(accumulatorList(i).get(0))
-      i += 1
-    }
+    // reset accumulator
+    function.resetAccumulator(accumulators)
 
     while (iterator.hasNext) {
       val record = iterator.next()
-
-      i = 0
-      while (i < aggregates.length) {
-        // insert received accumulator into acc list
-        val newAcc = record.getField(groupKeysMapping.length + i).asInstanceOf[Accumulator]
-        accumulatorList(i).set(1, newAcc)
-        // merge acc list
-        val retAcc = aggregates(i).merge(accumulatorList(i))
-        // insert result into acc list
-        accumulatorList(i).set(0, retAcc)
-        i += 1
-      }
-
+      function.mergeAccumulatorsPair(accumulators, record)
       last = record
     }
 
     // set group keys value to final output.
-    i = 0
-    while (i < groupKeysMapping.length) {
-      val (after, previous) = groupKeysMapping(i)
-      output.setField(after, last.getField(previous))
-      i += 1
-    }
+    function.setForwardedFields(last, output)
 
     // get final aggregate value and set to output.
-    i = 0
-    while (i < aggregateMapping.length) {
-      val (after, previous) = aggregateMapping(i)
-      val agg = aggregates(previous)
-      val result = agg.getValue(accumulatorList(previous).get(0))
-      output.setField(after, result)
-      i += 1
-    }
+    function.setAggregationResults(accumulators, output)
 
     // get window start timestamp
-    val startTs: Long = last.getField(rowtimePos).asInstanceOf[Long]
+    val startTs: Long = last.getField(keysAndAggregatesArity).asInstanceOf[Long]
 
     // set collector and window
     collector.wrappedCollector = out
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala
index 5cc7ada4046e4..d49ed0e13ee39 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala
@@ -25,58 +25,60 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow
-import org.apache.flink.table.functions.AggregateFunction
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.types.Row
-import org.apache.flink.util.Preconditions
-
+import org.slf4j.LoggerFactory
 
 /**
   * This map function only works for windows on batch tables.
   * It appends an (aligned) rowtime field to the end of the output row.
+  *
+  * @param genAggregations      Code-generated [[GeneratedAggregations]]
+  * @param timeFieldPos         Time field position in input row
+  * @param tumbleTimeWindowSize The size of tumble time window
   */
 class DataSetWindowAggMapFunction(
-    private val aggregates: Array[AggregateFunction[_]],
-    private val aggFields: Array[Array[Int]],
-    private val groupingKeys: Array[Int],
-    private val timeFieldPos: Int, // time field position in input row
+    private val genAggregations: GeneratedAggregationsFunction,
+    private val timeFieldPos: Int,
     private val tumbleTimeWindowSize: Option[Long],
     @transient private val returnType: TypeInformation[Row])
-  extends RichMapFunction[Row, Row] with ResultTypeQueryable[Row] {
-
-  Preconditions.checkNotNull(aggregates)
-  Preconditions.checkNotNull(aggFields)
-  Preconditions.checkArgument(aggregates.length == aggFields.length)
+  extends RichMapFunction[Row, Row]
+    with ResultTypeQueryable[Row]
+    with Compiler[GeneratedAggregations] {
 
+  private var accs: Row = _
   private var output: Row = _
-  // add one more arity to store rowtime
-  private val partialRowLength = groupingKeys.length + aggregates.length + 1
-  // rowtime index in the buffer output row
-  private val rowtimeIndex: Int = partialRowLength - 1
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
 
   override def open(config: Configuration) {
-    output = new Row(partialRowLength)
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+                s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getRuntimeContext.getUserCodeClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+
+    accs = function.createAccumulators()
+    output = function.createOutputRow()
   }
 
   override def map(input: Row): Row = {
 
-    var i = 0
-    while (i < aggregates.length) {
-      val agg = aggregates(i)
-      val fieldValue = input.getField(aggFields(i)(0))
-      val accumulator = agg.createAccumulator()
-      agg.accumulate(accumulator, fieldValue)
-      output.setField(groupingKeys.length + i, accumulator)
-      i += 1
-    }
+    function.resetAccumulator(accs)
 
-    i = 0
-    while (i < groupingKeys.length) {
-      output.setField(i, input.getField(groupingKeys(i)))
-      i += 1
-    }
+    function.accumulate(accs, input)
+
+    function.setAggregationResults(accs, output)
+
+    function.setForwardedFields(input, output)
 
     val timeField = input.getField(timeFieldPos)
     val rowtime = getTimestamp(timeField)
+    val rowtimeIndex = output.getArity - 1
     if (tumbleTimeWindowSize.isDefined) {
       // in case of tumble time window, align rowtime to window start to represent the window
       output.setField(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
index 17a1128975c78..bee39faa6f77c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
@@ -27,7 +27,9 @@ import org.apache.flink.types.Row
 abstract class GeneratedAggregations extends Function {
 
   /**
-    * Calculate the results from accumulators, and set the results to the output
+    * Sets the results of the aggregations (partial or final) to the output row.
+    * Final results are computed with the aggregation function.
+    * Partial results are the accumulators themselves.
     *
     * @param accumulators the accumulators (saved in a row) which contains the current
     *                     aggregated results
@@ -36,15 +38,22 @@ abstract class GeneratedAggregations extends Function {
   def setAggregationResults(accumulators: Row, output: Row)
 
   /**
-    * Copies forwarded fields from input row to output row.
+    * Copies forwarded fields, such as grouping keys, from input row to output row.
     *
-    * @param input  input values bundled in a row
-    * @param output output results collected in a row
+    * @param input        input values bundled in a row
+    * @param output       output results collected in a row
     */
   def setForwardedFields(input: Row, output: Row)
 
   /**
-    * Accumulate the input values to the accumulators
+    * Sets constant flags (boolean fields) to an output row.
+    *
+    * @param output The output row to which the constant flags are set.
+    */
+  def setConstantFlags(output: Row)
+
+  /**
+    * Accumulates the input values to the accumulators.
     *
     * @param accumulators the accumulators (saved in a row) which contains the current
     *                     aggregated results
@@ -53,7 +62,7 @@ abstract class GeneratedAggregations extends Function {
   def accumulate(accumulators: Row, input: Row)
 
   /**
-    * Retract the input values from the accumulators
+    * Retracts the input values from the accumulators.
     *
     * @param accumulators the accumulators (saved in a row) which contains the current
     *                     aggregated results
@@ -62,7 +71,7 @@ abstract class GeneratedAggregations extends Function {
   def retract(accumulators: Row, input: Row)
 
   /**
-    * Init the accumulators, and save them to a accumulators Row.
+    * Initializes the accumulators and save them to a accumulators row.
     *
     * @return a row of accumulators which contains the aggregated results
     */
@@ -76,7 +85,7 @@ abstract class GeneratedAggregations extends Function {
   def createOutputRow(): Row
 
   /**
-    * Merges two rows of accumulators into one row
+    * Merges two rows of accumulators into one row.
     *
     * @param a First row of accumulators
     * @param b The other row of accumulators
@@ -84,4 +93,11 @@ abstract class GeneratedAggregations extends Function {
     */
   def mergeAccumulatorsPair(a: Row, b: Row): Row
 
+  /**
+    * Resets all the accumulators.
+    *
+    * @param accumulators the accumulators (saved in a row) which contains the current
+    *                     aggregated results
+    */
+  def resetAccumulator(accumulators: Row)
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
index 5ac09b949b299..48387474817b2 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
@@ -339,5 +339,6 @@ class AggregationsITCase(
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
-  case class WC(word: String, frequency: Long)
 }
+
+case class WC(word: String, frequency: Long)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
index adc84bf0ce4a7..16c493edd5111 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
@@ -159,12 +159,18 @@ class BoundedProcessingOverRangeProcessFunctionTest {
         |    return new org.apache.flink.types.Row(7);
         |  }
         |
-        |  //The test won't use this method
+        |/*******  This test does not use the following methods  *******/
         |  public org.apache.flink.types.Row mergeAccumulatorsPair(
         |    org.apache.flink.types.Row a,
         |    org.apache.flink.types.Row b) {
         |    return null;
         |  }
+        |
+        |  public void resetAccumulator(org.apache.flink.types.Row accs) {
+        |  }
+        |
+        |  public void setConstantFlags(org.apache.flink.types.Row output) {
+        |  }
         |}
       """.stripMargin
 

From e2f75aaacd3dbfe0b9d24fcd1909dc249d1dcb5c Mon Sep 17 00:00:00 2001
From: Ken Geis <geis.ken@gmail.com>
Date: Fri, 21 Apr 2017 11:09:25 -0700
Subject: [PATCH 036/253] [hotfix] [table] Add missing semicolons to Table API
 Java examples.

This closes #3754.
---
 docs/dev/table_api.md | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 2a838c7f1a42d..3d77b85c366f1 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -68,7 +68,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
 
 // register the DataSet cust as table "Customers" with fields derived from the dataset
-tableEnv.registerDataSet("Customers", cust)
+tableEnv.registerDataSet("Customers", cust);
 
 // register the DataSet ord as table "Orders" with fields user, product, and amount
 tableEnv.registerDataSet("Orders", ord, "user, product, amount");
@@ -102,7 +102,7 @@ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironm
 StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
 
 // register the DataStream cust as table "Customers" with fields derived from the datastream
-tableEnv.registerDataStream("Customers", cust)
+tableEnv.registerDataStream("Customers", cust);
 
 // register the DataStream ord as table "Orders" with fields user, product, and amount
 tableEnv.registerDataStream("Orders", ord, "user, product, amount");
@@ -140,10 +140,10 @@ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
 Table custT = tableEnv
   .toTable(custDs, "name, zipcode")
   .where("zipcode = '12345'")
-  .select("name")
+  .select("name");
 
 // register the Table custT as table "custNames"
-tableEnv.registerTable("custNames", custT)
+tableEnv.registerTable("custNames", custT);
 {% endhighlight %}
 </div>
 
@@ -178,10 +178,10 @@ An external table is registered in a `TableEnvironment` using a `TableSource` as
 ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
 
-TableSource custTS = new CsvTableSource("/path/to/file", ...)
+TableSource custTS = new CsvTableSource("/path/to/file", ...);
 
 // register a `TableSource` as external table "Customers"
-tableEnv.registerTableSource("Customers", custTS)
+tableEnv.registerTableSource("Customers", custTS);
 {% endhighlight %}
 </div>
 
@@ -1047,7 +1047,7 @@ The following example shows how to define a window aggregation on a table.
 Table table = input
   .window([Window w].as("w"))  // define window with alias w
   .groupBy("w")  // group the table by window w
-  .select("b.sum")  // aggregate
+  .select("b.sum");  // aggregate
 {% endhighlight %}
 </div>
 
@@ -1070,7 +1070,7 @@ The following example shows how to define a window aggregation with additional g
 Table table = input
   .window([Window w].as("w"))  // define window with alias w
   .groupBy("w, a")  // group the table by attribute a and window w 
-  .select("a, b.sum")  // aggregate
+  .select("a, b.sum");  // aggregate
 {% endhighlight %}
 </div>
 
@@ -1092,7 +1092,7 @@ The `Window` parameter defines how rows are mapped to windows. `Window` is not a
 Table table = input
   .window([Window w].as("w"))  // define window with alias w
   .groupBy("w, a")  // group the table by attribute a and window w 
-  .select("a, w.start, w.end, b.count") // aggregate and add window start and end timestamps
+  .select("a, w.start, w.end, b.count"); // aggregate and add window start and end timestamps
 {% endhighlight %}
 </div>
 
@@ -1144,13 +1144,13 @@ Tumbling windows are defined by using the `Tumble` class as follows:
 <div data-lang="java" markdown="1">
 {% highlight java %}
 // Tumbling Event-time Window
-.window(Tumble.over("10.minutes").on("rowtime").as("w"))
+.window(Tumble.over("10.minutes").on("rowtime").as("w"));
 
 // Tumbling Processing-time Window
-.window(Tumble.over("10.minutes").as("w"))
+.window(Tumble.over("10.minutes").as("w"));
 
 // Tumbling Row-count Window
-.window(Tumble.over("10.rows").as("w"))
+.window(Tumble.over("10.rows").as("w"));
 {% endhighlight %}
 </div>
 
@@ -1211,13 +1211,13 @@ Sliding windows are defined by using the `Slide` class as follows:
 <div data-lang="java" markdown="1">
 {% highlight java %}
 // Sliding Event-time Window
-.window(Slide.over("10.minutes").every("5.minutes").on("rowtime").as("w"))
+.window(Slide.over("10.minutes").every("5.minutes").on("rowtime").as("w"));
 
 // Sliding Processing-time window
-.window(Slide.over("10.minutes").every("5.minutes").as("w"))
+.window(Slide.over("10.minutes").every("5.minutes").as("w"));
 
 // Sliding Row-count window
-.window(Slide.over("10.rows").every("5.rows").as("w"))
+.window(Slide.over("10.rows").every("5.rows").as("w"));
 {% endhighlight %}
 </div>
 
@@ -1273,10 +1273,10 @@ A session window is defined by using the `Session` class as follows:
 <div data-lang="java" markdown="1">
 {% highlight java %}
 // Session Event-time Window
-.window(Session.withGap("10.minutes").on("rowtime").as("w"))
+.window(Session.withGap("10.minutes").on("rowtime").as("w"));
 
 // Session Processing-time Window
-.window(Session.withGap("10.minutes").as("w"))
+.window(Session.withGap("10.minutes").as("w"));
 {% endhighlight %}
 </div>
 
@@ -4952,7 +4952,7 @@ public class HashCode extends ScalarFunction {
 BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
 
 // register the function
-tableEnv.registerFunction("hashCode", new HashCode(10))
+tableEnv.registerFunction("hashCode", new HashCode(10));
 
 // use the function in Java Table API
 myTable.select("string, string.hashCode(), hashCode(string)");
@@ -5194,7 +5194,7 @@ conf.setString("hashcode_factor", "31");
 env.getConfig().setGlobalJobParameters(conf);
 
 // register the function
-tableEnv.registerFunction("hashCode", new HashCode())
+tableEnv.registerFunction("hashCode", new HashCode());
 
 // use the function in Java Table API
 myTable.select("string, string.hashCode(), hashCode(string)");

From 45bd8dd9bc2f1677822b2eb45304d8ae1d3c8b24 Mon Sep 17 00:00:00 2001
From: "xiaogang.sxg" <xiaogang.sxg@alibaba-inc.com>
Date: Mon, 13 Mar 2017 19:23:47 +0800
Subject: [PATCH 037/253] [FLINK-6014] [checkpoint] Allow the registration of
 state objects in checkpoints

---
 .../checkpoint/CheckpointCoordinator.java     | 136 +++++++----
 .../checkpoint/CompletedCheckpoint.java       |  73 +++++-
 .../checkpoint/CompletedCheckpointStore.java  |   9 +-
 .../runtime/checkpoint/PendingCheckpoint.java |  16 +-
 .../StandaloneCompletedCheckpointStore.java   |  18 +-
 .../runtime/checkpoint/SubtaskState.java      |  49 +++-
 .../flink/runtime/checkpoint/TaskState.java   |  30 ++-
 .../ZooKeeperCompletedCheckpointStore.java    |  67 ++++--
 .../runtime/state/CompositeStateHandle.java   |  60 +++++
 .../runtime/state/SharedStateHandle.java      |  39 ++++
 .../runtime/state/SharedStateRegistry.java    | 178 +++++++++++++++
 .../flink/runtime/jobmanager/JobManager.scala |  27 +--
 .../CheckpointCoordinatorFailureTest.java     |  26 ++-
 .../checkpoint/CheckpointCoordinatorTest.java | 211 ++++++++++++++----
 .../CheckpointStateRestoreTest.java           |   4 +-
 .../CompletedCheckpointStoreTest.java         |  78 +++++--
 .../checkpoint/CompletedCheckpointTest.java   |  24 +-
 ...ecutionGraphCheckpointCoordinatorTest.java |   9 +-
 .../checkpoint/PendingCheckpointTest.java     |  22 +-
 ...tandaloneCompletedCheckpointStoreTest.java |  27 ++-
 ...oKeeperCompletedCheckpointStoreITCase.java |  40 +++-
 ...ZooKeeperCompletedCheckpointStoreTest.java |  11 +-
 .../jobmanager/JobManagerHARecoveryTest.java  |  65 +-----
 .../state/SharedStateRegistryTest.java        | 136 +++++++++++
 .../RecoverableCompletedCheckpointStore.java  | 109 +++++++++
 25 files changed, 1176 insertions(+), 288 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 7087540571b08..5309dd4868245 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -21,6 +21,7 @@
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.runtime.checkpoint.savepoint.SavepointLoader;
 import org.apache.flink.runtime.checkpoint.savepoint.SavepointStore;
 import org.apache.flink.runtime.concurrent.ApplyFunction;
 import org.apache.flink.runtime.concurrent.Future;
@@ -36,10 +37,11 @@
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
-import org.apache.flink.runtime.state.StateObject;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.TaskStateHandles;
 
 import org.apache.flink.runtime.taskmanager.DispatcherThreadFactory;
+import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -108,6 +110,9 @@ public class CheckpointCoordinator {
 	/** Completed checkpoints. Implementations can be blocking. Make sure calls to methods
 	 * accessing this don't block the job manager actor and run asynchronously. */
 	private final CompletedCheckpointStore completedCheckpointStore;
+	
+	/** Registry for shared states */
+	private final SharedStateRegistry sharedStateRegistry;
 
 	/** Default directory for persistent checkpoints; <code>null</code> if none configured.
 	 * THIS WILL BE REPLACED BY PROPER STATE-BACKEND METADATA WRITING */
@@ -218,6 +223,7 @@ public CheckpointCoordinator(
 		this.completedCheckpointStore = checkNotNull(completedCheckpointStore);
 		this.checkpointDirectory = checkpointDirectory;
 		this.executor = checkNotNull(executor);
+		this.sharedStateRegistry = new SharedStateRegistry();
 
 		this.recentPendingCheckpoints = new ArrayDeque<>(NUM_GHOST_CHECKPOINT_IDS);
 
@@ -282,7 +288,7 @@ public void shutdown(JobStatus jobStatus) throws Exception {
 				}
 				pendingCheckpoints.clear();
 
-				completedCheckpointStore.shutdown(jobStatus);
+				completedCheckpointStore.shutdown(jobStatus, sharedStateRegistry);
 				checkpointIdCounter.shutdown(jobStatus);
 			}
 		}
@@ -615,7 +621,7 @@ public void receiveDeclineMessage(DeclineCheckpoint message) {
 			throw new IllegalArgumentException("Received DeclineCheckpoint message for job " +
 				message.getJob() + " while this coordinator handles job " + job);
 		}
-
+		
 		final long checkpointId = message.getCheckpointId();
 		final String reason = (message.getReason() != null ? message.getReason().getMessage() : "");
 
@@ -695,7 +701,7 @@ public boolean receiveAcknowledgeMessage(AcknowledgeCheckpoint message) throws C
 		}
 
 		final long checkpointId = message.getCheckpointId();
-
+		
 		synchronized (lock) {
 			// we need to check inside the lock for being shutdown as well, otherwise we
 			// get races and invalid error log messages
@@ -778,49 +784,55 @@ else if (checkpoint != null) {
 	 */
 	private void completePendingCheckpoint(PendingCheckpoint pendingCheckpoint) throws CheckpointException {
 		final long checkpointId = pendingCheckpoint.getCheckpointId();
-		CompletedCheckpoint completedCheckpoint = null;
+		final CompletedCheckpoint completedCheckpoint;
 
 		try {
-			// externalize the checkpoint if required
-			if (pendingCheckpoint.getProps().externalizeCheckpoint()) {
-				completedCheckpoint = pendingCheckpoint.finalizeCheckpointExternalized();
-			} else {
-				completedCheckpoint = pendingCheckpoint.finalizeCheckpointNonExternalized();
-			}
-
-			completedCheckpointStore.addCheckpoint(completedCheckpoint);
-
-			rememberRecentCheckpointId(checkpointId);
-			dropSubsumedCheckpoints(checkpointId);
-		}
-		catch (Exception exception) {
-			// abort the current pending checkpoint if it has not been discarded yet
-			if (!pendingCheckpoint.isDiscarded()) {
-				pendingCheckpoint.abortError(exception);
+			try {
+				// externalize the checkpoint if required
+				if (pendingCheckpoint.getProps().externalizeCheckpoint()) {
+					completedCheckpoint = pendingCheckpoint.finalizeCheckpointExternalized();
+				} else {
+					completedCheckpoint = pendingCheckpoint.finalizeCheckpointNonExternalized();
+				}
+			} catch (Exception e1) {
+				// abort the current pending checkpoint if we fails to finalize the pending checkpoint.
+				if (!pendingCheckpoint.isDiscarded()) {
+					pendingCheckpoint.abortError(e1);
+				}
+	
+				throw new CheckpointException("Could not finalize the pending checkpoint " + checkpointId + '.', e1);
 			}
-
-			if (completedCheckpoint != null) {
+	
+			// the pending checkpoint must be discarded after the finalization
+			Preconditions.checkState(pendingCheckpoint.isDiscarded() && completedCheckpoint != null);
+	
+			try {
+				completedCheckpointStore.addCheckpoint(completedCheckpoint, sharedStateRegistry);
+			} catch (Exception exception) {
 				// we failed to store the completed checkpoint. Let's clean up
-				final CompletedCheckpoint cc = completedCheckpoint;
-
 				executor.execute(new Runnable() {
 					@Override
 					public void run() {
 						try {
-							cc.discard();
+							completedCheckpoint.discardOnFail();
 						} catch (Throwable t) {
-							LOG.warn("Could not properly discard completed checkpoint {}.", cc.getCheckpointID(), t);
+							LOG.warn("Could not properly discard completed checkpoint {}.", completedCheckpoint.getCheckpointID(), t);
 						}
 					}
 				});
+				
+				throw new CheckpointException("Could not complete the pending checkpoint " + checkpointId + '.', exception);
 			}
-
-			throw new CheckpointException("Could not complete the pending checkpoint " + checkpointId + '.', exception);
 		} finally {
 			pendingCheckpoints.remove(checkpointId);
 
 			triggerQueuedRequests();
 		}
+		
+		rememberRecentCheckpointId(checkpointId);
+		
+		// drop those pending checkpoints that are at prior to the completed one
+		dropSubsumedCheckpoints(checkpointId);
 
 		// record the time when this was completed, to calculate
 		// the 'min delay between checkpoints'
@@ -941,7 +953,7 @@ public boolean restoreLatestCheckpointedState(
 			}
 
 			// Recover the checkpoints
-			completedCheckpointStore.recover();
+			completedCheckpointStore.recover(sharedStateRegistry);
 
 			// restore from the latest checkpoint
 			CompletedCheckpoint latest = completedCheckpointStore.getLatestCheckpoint();
@@ -978,6 +990,44 @@ public boolean restoreLatestCheckpointedState(
 		}
 	}
 
+	/**
+	 * Restore the state with given savepoint
+	 * 
+	 * @param savepointPath    Location of the savepoint
+	 * @param allowNonRestored True if allowing checkpoint state that cannot be 
+	 *                         mapped to any job vertex in tasks.
+	 * @param tasks            Map of job vertices to restore. State for these 
+	 *                         vertices is restored via 
+	 *                         {@link Execution#setInitialState(TaskStateHandles)}.
+	 * @param userClassLoader  The class loader to resolve serialized classes in 
+	 *                         legacy savepoint versions. 
+	 */
+	public boolean restoreSavepoint(
+			String savepointPath, 
+			boolean allowNonRestored,
+			Map<JobVertexID, ExecutionJobVertex> tasks,
+			ClassLoader userClassLoader) throws Exception {
+		
+		Preconditions.checkNotNull(savepointPath, "The savepoint path cannot be null.");
+		
+		LOG.info("Starting job from savepoint {} ({})", 
+				savepointPath, (allowNonRestored ? "allowing non restored state" : ""));
+
+		// Load the savepoint as a checkpoint into the system
+		CompletedCheckpoint savepoint = SavepointLoader.loadAndValidateSavepoint(
+				job, tasks, savepointPath, userClassLoader, allowNonRestored);
+
+		completedCheckpointStore.addCheckpoint(savepoint, sharedStateRegistry);
+		
+		// Reset the checkpoint ID counter
+		long nextCheckpointId = savepoint.getCheckpointID() + 1;
+		checkpointIdCounter.setCount(nextCheckpointId);
+		
+		LOG.info("Reset the checkpoint ID to {}.", nextCheckpointId);
+		
+		return restoreLatestCheckpointedState(tasks, true, allowNonRestored);
+	}
+
 	// --------------------------------------------------------------------------------------------
 	//  Accessors
 	// --------------------------------------------------------------------------------------------
@@ -1007,6 +1057,10 @@ public List<CompletedCheckpoint> getSuccessfulCheckpoints() throws Exception {
 	public CompletedCheckpointStore getCheckpointStore() {
 		return completedCheckpointStore;
 	}
+	
+	public SharedStateRegistry getSharedStateRegistry() {
+		return sharedStateRegistry;
+	}
 
 	public CheckpointIDCounter getCheckpointIdCounter() {
 		return checkpointIdCounter;
@@ -1095,24 +1149,30 @@ public void run() {
 	 * @param jobId identifying the job to which the state object belongs
 	 * @param executionAttemptID identifying the task to which the state object belongs
 	 * @param checkpointId of the state object
-	 * @param stateObject to discard asynchronously
+	 * @param subtaskState to discard asynchronously
 	 */
 	private void discardState(
 			final JobID jobId,
 			final ExecutionAttemptID executionAttemptID,
 			final long checkpointId,
-			final StateObject stateObject) {
+			final SubtaskState subtaskState) {
 
-		if (stateObject != null) {
+		if (subtaskState != null) {
 			executor.execute(new Runnable() {
 				@Override
 				public void run() {
 					try {
-						stateObject.discardState();
-					} catch (Throwable throwable) {
-					LOG.warn("Could not properly discard state object of checkpoint {} " +
-						"belonging to task {} of job {}.", checkpointId, executionAttemptID, jobId,
-						throwable);
+						subtaskState.discardSharedStatesOnFail();
+					} catch (Throwable t1) {
+						LOG.warn("Could not properly discard shared states of checkpoint {} " +
+							"belonging to task {} of job {}.", checkpointId, executionAttemptID, jobId, t1);
+					}
+
+					try {
+						subtaskState.discardState();
+					} catch (Throwable t2) {
+						LOG.warn("Could not properly discard state object of checkpoint {} " +
+							"belonging to task {} of job {}.", checkpointId, executionAttemptID, jobId, t2);
 					}
 				}
 			});
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
index 17ce4d51b8bab..58e91e1e0c991 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
@@ -23,10 +23,12 @@
 import org.apache.flink.runtime.checkpoint.CompletedCheckpointStats.DiscardCallback;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.util.ExceptionUtils;
 
+import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -184,22 +186,30 @@ public CheckpointProperties getProperties() {
 		return props;
 	}
 
-	public boolean subsume() throws Exception {
+	public void discardOnFail() throws Exception {
+		discard(null, true);
+	}
+
+	public boolean discardOnSubsume(SharedStateRegistry sharedStateRegistry) throws Exception {
+		Preconditions.checkNotNull(sharedStateRegistry, "The registry cannot be null.");
+
 		if (props.discardOnSubsumed()) {
-			discard();
+			discard(sharedStateRegistry, false);
 			return true;
 		}
 
 		return false;
 	}
 
-	public boolean discard(JobStatus jobStatus) throws Exception {
+	public boolean discardOnShutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
+		Preconditions.checkNotNull(sharedStateRegistry, "The registry cannot be null.");
+
 		if (jobStatus == JobStatus.FINISHED && props.discardOnJobFinished() ||
 				jobStatus == JobStatus.CANCELED && props.discardOnJobCancelled() ||
 				jobStatus == JobStatus.FAILED && props.discardOnJobFailed() ||
 				jobStatus == JobStatus.SUSPENDED && props.discardOnJobSuspended()) {
 
-			discard();
+			discard(sharedStateRegistry, false);
 			return true;
 		} else {
 			if (externalPointer != null) {
@@ -211,7 +221,10 @@ public boolean discard(JobStatus jobStatus) throws Exception {
 		}
 	}
 
-	void discard() throws Exception {
+	private void discard(SharedStateRegistry sharedStateRegistry, boolean failed) throws Exception {
+		Preconditions.checkState(failed || (sharedStateRegistry != null),
+			"The registry must not be null if the complete checkpoint does not fail.");
+
 		try {
 			// collect exceptions and continue cleanup
 			Exception exception = null;
@@ -220,25 +233,31 @@ void discard() throws Exception {
 			if (externalizedMetadata != null) {
 				try {
 					externalizedMetadata.discardState();
-				}
-				catch (Exception e) {
+				} catch (Exception e) {
 					exception = e;
 				}
 			}
 
-			// drop the actual state
+			// In the cases where the completed checkpoint fails, the shared
+			// states have not been registered to the registry. It's the state
+			// handles' responsibility to discard their shared states.
+			if (!failed) {
+				unregisterSharedStates(sharedStateRegistry);
+			} else {
+				discardSharedStatesOnFail();
+			}
+
+			// discard private state objects
 			try {
 				StateUtil.bestEffortDiscardAllStateObjects(taskStates.values());
-			}
-			catch (Exception e) {
+			} catch (Exception e) {
 				exception = ExceptionUtils.firstOrSuppressed(e, exception);
 			}
 
 			if (exception != null) {
 				throw exception;
 			}
-		}
-		finally {
+		} finally {
 			taskStates.clear();
 
 			// to be null-pointer safe, copy reference to stack
@@ -290,6 +309,36 @@ void setDiscardCallback(@Nullable CompletedCheckpointStats.DiscardCallback disca
 		this.discardCallback = discardCallback;
 	}
 
+	/**
+	 * Register all shared states in the given registry. This is method is called
+	 * when the completed checkpoint has been successfully added into the store.
+	 *
+	 * @param sharedStateRegistry The registry where shared states are registered
+	 */
+	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
+		sharedStateRegistry.registerAll(taskStates.values());
+	}
+
+	/**
+	 * Unregister all shared states from the given registry. This is method is
+	 * called when the completed checkpoint is subsumed or the job terminates.
+	 *
+	 * @param sharedStateRegistry The registry where shared states are registered
+	 */
+	private void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
+		sharedStateRegistry.unregisterAll(taskStates.values());
+	}
+
+	/**
+	 * Discard all shared states created in the checkpoint. This method is called
+	 * when the completed checkpoint fails to be added into the store.
+	 */
+	private void discardSharedStatesOnFail() throws Exception {
+		for (TaskState taskState : taskStates.values()) {
+			taskState.discardSharedStatesOnFail();
+		}
+	}
+
 	// --------------------------------------------------------------------------------------------
 
 	@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
index 9c2b19983b109..0ade25c849328 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.checkpoint;
 
 import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 
 import java.util.List;
 
@@ -33,16 +34,16 @@ public interface CompletedCheckpointStore {
 	 * <p>After a call to this method, {@link #getLatestCheckpoint()} returns the latest
 	 * available checkpoint.
 	 */
-	void recover() throws Exception;
+	void recover(SharedStateRegistry sharedStateRegistry) throws Exception;
 
 	/**
 	 * Adds a {@link CompletedCheckpoint} instance to the list of completed checkpoints.
 	 *
 	 * <p>Only a bounded number of checkpoints is kept. When exceeding the maximum number of
 	 * retained checkpoints, the oldest one will be discarded via {@link
-	 * CompletedCheckpoint#discard()}.
+	 * CompletedCheckpoint#discardOnSubsume(SharedStateRegistry)} )}.
 	 */
-	void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception;
+	void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception;
 
 	/**
 	 * Returns the latest {@link CompletedCheckpoint} instance or <code>null</code> if none was
@@ -58,7 +59,7 @@ public interface CompletedCheckpointStore {
 	 *
 	 * @param jobStatus Job state on shut down
 	 */
-	void shutdown(JobStatus jobStatus) throws Exception;
+	void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception;
 
 	/**
 	 * Returns all {@link CompletedCheckpoint} instances.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
index e1182ae50b5f5..6805deaa7cbb7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
@@ -497,6 +497,7 @@ public void abortError(Throwable cause) {
 	}
 
 	private void dispose(boolean releaseState) {
+
 		synchronized (lock) {
 			try {
 				numAcknowledgedTasks = -1;
@@ -504,11 +505,22 @@ private void dispose(boolean releaseState) {
 					executor.execute(new Runnable() {
 						@Override
 						public void run() {
+
+							// discard the shared states that are created in the checkpoint
+							for (TaskState taskState : taskStates.values()) {
+								try {
+									taskState.discardSharedStatesOnFail();
+								} catch (Throwable t) {
+									LOG.warn("Could not properly dispose unreferenced shared states.");
+								}
+							}
+
+							// discard the private states
 							try {
 								StateUtil.bestEffortDiscardAllStateObjects(taskStates.values());
 							} catch (Throwable t) {
-								LOG.warn("Could not properly dispose the pending checkpoint {} of job {}.", 
-										checkpointId, jobId, t);
+								LOG.warn("Could not properly dispose the private states in the pending checkpoint {} of job {}.",
+									checkpointId, jobId, t);
 							} finally {
 								taskStates.clear();
 							}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java
index 6eb5242ce2e26..9f833c31107f6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -56,16 +57,21 @@ public StandaloneCompletedCheckpointStore(int maxNumberOfCheckpointsToRetain) {
 	}
 
 	@Override
-	public void recover() throws Exception {
+	public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
 		// Nothing to do
 	}
 
 	@Override
-	public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception {
-		checkpoints.add(checkpoint);
+	public void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception {
+		
+		checkpoints.addLast(checkpoint);
+
+		checkpoint.registerSharedStates(sharedStateRegistry);
+
 		if (checkpoints.size() > maxNumberOfCheckpointsToRetain) {
 			try {
-				checkpoints.remove().subsume();
+				CompletedCheckpoint checkpointToSubsume = checkpoints.removeFirst();
+				checkpointToSubsume.discardOnSubsume(sharedStateRegistry);
 			} catch (Exception e) {
 				LOG.warn("Fail to subsume the old checkpoint.", e);
 			}
@@ -93,12 +99,12 @@ public int getMaxNumberOfRetainedCheckpoints() {
 	}
 
 	@Override
-	public void shutdown(JobStatus jobStatus) throws Exception {
+	public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
 		try {
 			LOG.info("Shutting down");
 
 			for (CompletedCheckpoint checkpoint : checkpoints) {
-				checkpoint.discard(jobStatus);
+				checkpoint.discardOnShutdown(jobStatus, sharedStateRegistry);
 			}
 		} finally {
 			checkpoints.clear();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
index 97b08fcc651af..e968643602a8c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
@@ -19,11 +19,15 @@
 package org.apache.flink.runtime.checkpoint;
 
 import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.CompositeStateHandle;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.StateObject;
 import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.runtime.state.StreamStateHandle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Arrays;
 
@@ -33,7 +37,9 @@
  * Container for the chained state of one parallel subtask of an operator/task. This is part of the
  * {@link TaskState}.
  */
-public class SubtaskState implements StateObject {
+public class SubtaskState implements CompositeStateHandle {
+
+	private static final Logger LOG = LoggerFactory.getLogger(SubtaskState.class);
 
 	private static final long serialVersionUID = -2394696997971923995L;
 
@@ -130,19 +136,38 @@ public KeyedStateHandle getRawKeyedState() {
 	}
 
 	@Override
-	public long getStateSize() {
-		return stateSize;
+	public void discardState() {
+		try {
+			StateUtil.bestEffortDiscardAllStateObjects(
+				Arrays.asList(
+					legacyOperatorState,
+					managedOperatorState,
+					rawOperatorState,
+					managedKeyedState,
+					rawKeyedState));
+		} catch (Exception e) {
+			LOG.warn("Error while discarding operator states.", e);
+		}
 	}
 
 	@Override
-	public void discardState() throws Exception {
-		StateUtil.bestEffortDiscardAllStateObjects(
-				Arrays.asList(
-						legacyOperatorState,
-						managedOperatorState,
-						rawOperatorState,
-						managedKeyedState,
-						rawKeyedState));
+	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
+		// No shared states
+	}
+
+	@Override
+	public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
+		// No shared states
+	}
+
+	@Override
+	public void discardSharedStatesOnFail() {
+		// No shared states
+	}
+
+	@Override
+	public long getStateSize() {
+		return stateSize;
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -206,7 +231,7 @@ public String toString() {
 				", operatorStateFromBackend=" + managedOperatorState +
 				", operatorStateFromStream=" + rawOperatorState +
 				", keyedStateFromBackend=" + managedKeyedState +
-				", keyedStateHandleFromStream=" + rawKeyedState +
+				", keyedStateFromStream=" + rawKeyedState +
 				", stateSize=" + stateSize +
 				'}';
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
index 76f1c510c50bf..19fe9624746be 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
@@ -19,8 +19,8 @@
 package org.apache.flink.runtime.checkpoint;
 
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.state.StateObject;
-import org.apache.flink.runtime.state.StateUtil;
+import org.apache.flink.runtime.state.CompositeStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.util.Preconditions;
 
 import java.util.Collection;
@@ -35,7 +35,7 @@
  *
  * This class basically groups all non-partitioned state and key-group state belonging to the same job vertex together.
  */
-public class TaskState implements StateObject {
+public class TaskState implements CompositeStateHandle {
 
 	private static final long serialVersionUID = -4845578005863201810L;
 
@@ -124,9 +124,31 @@ public boolean hasNonPartitionedState() {
 
 	@Override
 	public void discardState() throws Exception {
-		StateUtil.bestEffortDiscardAllStateObjects(subtaskStates.values());
+		for (SubtaskState subtaskState : subtaskStates.values()) {
+			subtaskState.discardState();
+		}
+	}
+
+	@Override
+	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
+		for (SubtaskState subtaskState : subtaskStates.values()) {
+			subtaskState.registerSharedStates(sharedStateRegistry);
+		}
+	}
+
+	@Override
+	public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
+		for (SubtaskState subtaskState : subtaskStates.values()) {
+			subtaskState.unregisterSharedStates(sharedStateRegistry);
+		}
 	}
 
+	@Override
+	public void discardSharedStatesOnFail() {
+		for (SubtaskState subtaskState : subtaskStates.values()) {
+			subtaskState.discardSharedStatesOnFail();
+		}
+	}
 
 	@Override
 	public long getStateSize() {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
index af7bcc4f57f73..07546ea4227ff 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
@@ -27,6 +27,7 @@
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
 import org.apache.flink.runtime.state.RetrievableStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper;
 import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore;
 import org.apache.flink.util.FlinkException;
@@ -123,7 +124,7 @@ public ZooKeeperCompletedCheckpointStore(
 		this.checkpointsInZooKeeper = new ZooKeeperStateHandleStore<>(this.client, stateStorage, executor);
 
 		this.checkpointStateHandles = new ArrayDeque<>(maxNumberOfCheckpointsToRetain + 1);
-
+		
 		LOG.info("Initialized in '{}'.", checkpointsPath);
 	}
 
@@ -140,7 +141,7 @@ public boolean requiresExternalizedCheckpoints() {
 	 * that the history of checkpoints is consistent.
 	 */
 	@Override
-	public void recover() throws Exception {
+	public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
 		LOG.info("Recovering checkpoints from ZooKeeper.");
 
 		// Clear local handles in order to prevent duplicates on
@@ -164,8 +165,24 @@ public void recover() throws Exception {
 
 		LOG.info("Found {} checkpoints in ZooKeeper.", numberOfInitialCheckpoints);
 
-		for (Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String> checkpoint : initialCheckpoints) {
-			checkpointStateHandles.add(checkpoint);
+		for (Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String> checkpointStateHandle : initialCheckpoints) {
+
+			CompletedCheckpoint completedCheckpoint = null;
+
+			try {
+				completedCheckpoint = retrieveCompletedCheckpoint(checkpointStateHandle);
+			} catch (Exception e) {
+				LOG.warn("Could not retrieve checkpoint. Removing it from the completed " +
+					"checkpoint store.", e);
+
+				// remove the checkpoint with broken state handle
+				removeBrokenStateHandle(checkpointStateHandle);
+			}
+
+			if (completedCheckpoint != null) {
+				completedCheckpoint.registerSharedStates(sharedStateRegistry);
+				checkpointStateHandles.add(checkpointStateHandle);
+			}
 		}
 	}
 
@@ -175,21 +192,24 @@ public void recover() throws Exception {
 	 * @param checkpoint Completed checkpoint to add.
 	 */
 	@Override
-	public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception {
+	public void addCheckpoint(final CompletedCheckpoint checkpoint, final SharedStateRegistry sharedStateRegistry) throws Exception {
 		checkNotNull(checkpoint, "Checkpoint");
+		
+		final String path = checkpointIdToPath(checkpoint.getCheckpointID());
+		final RetrievableStateHandle<CompletedCheckpoint> stateHandle;
 
 		// First add the new one. If it fails, we don't want to loose existing data.
-		String path = checkpointIdToPath(checkpoint.getCheckpointID());
-
-		final RetrievableStateHandle<CompletedCheckpoint> stateHandle =
-				checkpointsInZooKeeper.add(path, checkpoint);
+		stateHandle = checkpointsInZooKeeper.add(path, checkpoint);
 
 		checkpointStateHandles.addLast(new Tuple2<>(stateHandle, path));
 
+		// Register all shared states in the checkpoint
+		checkpoint.registerSharedStates(sharedStateRegistry);
+
 		// Everything worked, let's remove a previous checkpoint if necessary.
 		while (checkpointStateHandles.size() > maxNumberOfCheckpointsToRetain) {
 			try {
-				removeSubsumed(checkpointStateHandles.removeFirst());
+				removeSubsumed(checkpointStateHandles.removeFirst(), sharedStateRegistry);
 			} catch (Exception e) {
 				LOG.warn("Failed to subsume the old checkpoint", e);
 			}
@@ -261,13 +281,13 @@ public int getMaxNumberOfRetainedCheckpoints() {
 	}
 
 	@Override
-	public void shutdown(JobStatus jobStatus) throws Exception {
+	public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
 		if (jobStatus.isGloballyTerminalState()) {
 			LOG.info("Shutting down");
 
 			for (Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String> checkpoint : checkpointStateHandles) {
 				try {
-					removeShutdown(checkpoint, jobStatus);
+					removeShutdown(checkpoint, jobStatus, sharedStateRegistry);
 				} catch (Exception e) {
 					LOG.error("Failed to discard checkpoint.", e);
 				}
@@ -289,11 +309,19 @@ public void shutdown(JobStatus jobStatus) throws Exception {
 
 	// ------------------------------------------------------------------------
 
-	private void removeSubsumed(final Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String> stateHandleAndPath) throws Exception {
+	private void removeSubsumed(
+		final Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String> stateHandleAndPath,
+		final SharedStateRegistry sharedStateRegistry) throws Exception {
+		
 		Callable<Void> action = new Callable<Void>() {
 			@Override
 			public Void call() throws Exception {
-				stateHandleAndPath.f0.retrieveState().subsume();
+				CompletedCheckpoint completedCheckpoint = retrieveCompletedCheckpoint(stateHandleAndPath);
+				
+				if (completedCheckpoint != null) {
+					completedCheckpoint.discardOnSubsume(sharedStateRegistry);
+				}
+
 				return null;
 			}
 		};
@@ -303,13 +331,18 @@ public Void call() throws Exception {
 
 	private void removeShutdown(
 			final Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String> stateHandleAndPath,
-			final JobStatus jobStatus) throws Exception {
+			final JobStatus jobStatus,
+			final SharedStateRegistry sharedStateRegistry) throws Exception {
 
 		Callable<Void> action = new Callable<Void>() {
 			@Override
 			public Void call() throws Exception {
-				CompletedCheckpoint checkpoint = stateHandleAndPath.f0.retrieveState();
-				checkpoint.discard(jobStatus);
+				CompletedCheckpoint completedCheckpoint = retrieveCompletedCheckpoint(stateHandleAndPath);
+				
+				if (completedCheckpoint != null) {
+					completedCheckpoint.discardOnShutdown(jobStatus, sharedStateRegistry);
+				}
+
 				return null;
 			}
 		};
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java
new file mode 100644
index 0000000000000..2ea5bc9080e12
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+/**
+ * Base of all snapshots that are taken by {@link StateBackend}s and some other
+ * components in tasks.
+ *
+ * <p>Each snapshot is composed of a collection of {@link StateObject}s some of 
+ * which may be referenced by other checkpoints. The shared states will be 
+ * registered at the given {@link SharedStateRegistry} when the handle is
+ * received by the {@link org.apache.flink.runtime.checkpoint.CheckpointCoordinator}
+ * and will be discarded when the checkpoint is discarded.
+ * 
+ * <p>The {@link SharedStateRegistry} is responsible for the discarding of the
+ * shared states. The composite state handle should only delete those private
+ * states in the {@link StateObject#discardState()} method.
+ */
+public interface CompositeStateHandle extends StateObject {
+
+	/**
+	 * Register both created and referenced shared states in the given
+	 * {@link SharedStateRegistry}. This method is called when the checkpoint
+	 * successfully completes or is recovered from failures.
+	 *
+	 * @param stateRegistry The registry where shared states are registered.
+	 */
+	void registerSharedStates(SharedStateRegistry stateRegistry);
+
+	/**
+	 * Unregister both created and referenced shared states in the given
+	 * {@link SharedStateRegistry}. This method is called when the checkpoint is
+	 * subsumed or the job is shut down.
+	 *
+	 * @param stateRegistry The registry where shared states are registered.
+	 */
+	void unregisterSharedStates(SharedStateRegistry stateRegistry);
+
+	/**
+	 * Discard all shared states created in this checkpoint. This method is
+	 * called when the checkpoint fails to complete.
+	 */
+	void discardSharedStatesOnFail() throws Exception;
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
new file mode 100644
index 0000000000000..f856052904952
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+/**
+ * A handle to those states that are referenced by different checkpoints.
+ *
+ * <p> Each shared state handle is identified by a unique key. Two shared states
+ * are considered equal if their keys are identical.
+ *
+ * <p> All shared states are registered at the {@link SharedStateRegistry} once
+ * they are received by the {@link org.apache.flink.runtime.checkpoint.CheckpointCoordinator}
+ * and will be unregistered when the checkpoints are discarded. A shared state
+ * will be discarded once it is not referenced by any checkpoint. A shared state
+ * should not be referenced any more if it has been discarded.
+ */
+public interface SharedStateHandle extends StateObject {
+
+	/**
+	 * Return the identifier of the shared state.
+	 */
+	String getKey();
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
new file mode 100644
index 0000000000000..b5048d0d96208
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A {@code SharedStateRegistry} will be deployed in the 
+ * {@link org.apache.flink.runtime.checkpoint.CheckpointCoordinator} to 
+ * maintain the reference count of {@link SharedStateHandle}s which are shared
+ * among different checkpoints.
+ */
+public class SharedStateRegistry implements Serializable {
+
+	private static Logger LOG = LoggerFactory.getLogger(SharedStateRegistry.class);
+
+	private static final long serialVersionUID = -8357254413007773970L;
+
+	/** All registered state objects */
+	private final Map<String, SharedStateEntry> registeredStates = new HashMap<>();
+
+	/**
+	 * Register the state in the registry
+	 *
+	 * @param state The state to register
+	 * @param isNew True if the shared state is newly created
+	 */
+	public void register(SharedStateHandle state, boolean isNew) {
+		if (state == null) {
+			return;
+		}
+
+		synchronized (registeredStates) {
+			SharedStateEntry entry = registeredStates.get(state.getKey());
+
+			if (isNew) {
+				Preconditions.checkState(entry == null,
+					"The state cannot be created more than once.");
+
+				registeredStates.put(state.getKey(), new SharedStateEntry(state));
+			} else {
+				Preconditions.checkState(entry != null,
+					"The state cannot be referenced if it has not been created yet.");
+
+				entry.increaseReferenceCount();
+			}
+		}
+	}
+
+	/**
+	 * Unregister the state in the registry
+	 *
+	 * @param state The state to unregister
+	 */
+	public void unregister(SharedStateHandle state) {
+		if (state == null) {
+			return;
+		}
+
+		synchronized (registeredStates) {
+			SharedStateEntry entry = registeredStates.get(state.getKey());
+
+			if (entry == null) {
+				throw new IllegalStateException("Cannot unregister an unexisted state.");
+			}
+
+			entry.decreaseReferenceCount();
+
+			// Remove the state from the registry when it's not referenced any more.
+			if (entry.getReferenceCount() == 0) {
+				registeredStates.remove(state.getKey());
+
+				try {
+					entry.getState().discardState();
+				} catch (Exception e) {
+					LOG.warn("Cannot properly discard the state " + entry.getState() + ".", e);
+				}
+			}
+		}
+	}
+
+	/**
+	 * Register given shared states in the registry.
+	 *
+	 * @param stateHandles The shared states to register.
+	 */
+	public void registerAll(Collection<? extends CompositeStateHandle> stateHandles) {
+		if (stateHandles == null) {
+			return;
+		}
+
+		synchronized (registeredStates) {
+			for (CompositeStateHandle stateHandle : stateHandles) {
+				stateHandle.registerSharedStates(this);
+			}
+		}
+	}
+
+
+
+	/**
+	 * Unregister all the shared states referenced by the given.
+	 *
+	 * @param stateHandles The shared states to unregister.
+	 */
+	public void unregisterAll(Collection<? extends CompositeStateHandle> stateHandles) {
+		if (stateHandles == null) {
+			return;
+		}
+
+		synchronized (registeredStates) {
+			for (CompositeStateHandle stateHandle : stateHandles) {
+				stateHandle.unregisterSharedStates(this);
+			}
+		}
+	}
+
+	private static class SharedStateEntry {
+		/** The shared object */
+		private final SharedStateHandle state;
+
+		/** The reference count of the object */
+		private int referenceCount;
+
+		SharedStateEntry(SharedStateHandle value) {
+			this.state = value;
+			this.referenceCount = 1;
+		}
+
+		SharedStateHandle getState() {
+			return state;
+		}
+
+		int getReferenceCount() {
+			return referenceCount;
+		}
+
+		void increaseReferenceCount() {
+			++referenceCount;
+		}
+
+		void decreaseReferenceCount() {
+			--referenceCount;
+		}
+	}
+
+
+	@VisibleForTesting
+	public int getReferenceCount(SharedStateHandle state) {
+		SharedStateEntry entry = registeredStates.get(state.getKey());
+
+		return entry == null ? 0 : entry.getReferenceCount();
+	}
+}
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index f2ecde526ff69..40e2c2a1f25f8 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -1365,27 +1365,12 @@ class JobManager(
                 val savepointPath = savepointSettings.getRestorePath()
                 val allowNonRestored = savepointSettings.allowNonRestoredState()
 
-                log.info(s"Starting job from savepoint '$savepointPath'" +
-                  (if (allowNonRestored) " (allowing non restored state)" else "") + ".")
-
-                  // load the savepoint as a checkpoint into the system
-                  val savepoint: CompletedCheckpoint = SavepointLoader.loadAndValidateSavepoint(
-                    jobId,
-                    executionGraph.getAllVertices,
-                    savepointPath,
-                    executionGraph.getUserClassLoader,
-                    allowNonRestored)
-
-                executionGraph.getCheckpointCoordinator.getCheckpointStore
-                  .addCheckpoint(savepoint)
-
-                // Reset the checkpoint ID counter
-                val nextCheckpointId: Long = savepoint.getCheckpointID + 1
-                log.info(s"Reset the checkpoint ID to $nextCheckpointId")
-                executionGraph.getCheckpointCoordinator.getCheckpointIdCounter
-                  .setCount(nextCheckpointId)
-
-                executionGraph.restoreLatestCheckpointedState(true, allowNonRestored)
+                executionGraph.getCheckpointCoordinator.restoreSavepoint(
+                  savepointPath, 
+                  allowNonRestored,
+                  executionGraph.getAllVertices,
+                  executionGraph.getUserClassLoader
+                )
               } catch {
                 case e: Exception =>
                   jobInfo.notifyClients(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
index 340e2a78607e8..632f2c0678a3d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
@@ -25,6 +25,7 @@
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -38,7 +39,9 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
 @RunWith(PowerMockRunner.class)
@@ -83,12 +86,13 @@ public void testFailingCompletedCheckpointStoreAdd() throws Exception {
 		assertFalse(pendingCheckpoint.isDiscarded());
 
 		final long checkpointId = coord.getPendingCheckpoints().keySet().iterator().next();
-
-		AcknowledgeCheckpoint acknowledgeMessage = new AcknowledgeCheckpoint(jid, executionAttemptId, checkpointId);
-
-		CompletedCheckpoint completedCheckpoint = mock(CompletedCheckpoint.class);
-		PowerMockito.whenNew(CompletedCheckpoint.class).withAnyArguments().thenReturn(completedCheckpoint);
-
+		
+		SubtaskState subtaskState = mock(SubtaskState.class);
+		PowerMockito.when(subtaskState.getLegacyOperatorState()).thenReturn(null);
+		PowerMockito.when(subtaskState.getManagedOperatorState()).thenReturn(null);
+		
+		AcknowledgeCheckpoint acknowledgeMessage = new AcknowledgeCheckpoint(jid, executionAttemptId, checkpointId, new CheckpointMetrics(), subtaskState);
+		
 		try {
 			coord.receiveAcknowledgeMessage(acknowledgeMessage);
 			fail("Expected a checkpoint exception because the completed checkpoint store could not " +
@@ -100,18 +104,20 @@ public void testFailingCompletedCheckpointStoreAdd() throws Exception {
 		// make sure that the pending checkpoint has been discarded after we could not complete it
 		assertTrue(pendingCheckpoint.isDiscarded());
 
-		verify(completedCheckpoint).discard();
+		// make sure that the subtask state has been discarded after we could not complete it.
+		verify(subtaskState, times(1)).discardSharedStatesOnFail();
+		verify(subtaskState).discardState();
 	}
 
 	private static final class FailingCompletedCheckpointStore implements CompletedCheckpointStore {
 
 		@Override
-		public void recover() throws Exception {
+		public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
 			throw new UnsupportedOperationException("Not implemented.");
 		}
 
 		@Override
-		public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception {
+		public void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception {
 			throw new Exception("The failing completed checkpoint store failed again... :-(");
 		}
 
@@ -121,7 +127,7 @@ public CompletedCheckpoint getLatestCheckpoint() throws Exception {
 		}
 
 		@Override
-		public void shutdown(JobStatus jobStatus) throws Exception {
+		public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
 			throw new UnsupportedOperationException("Not implemented.");
 		}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index 117c70d3139fe..fabf3fc33ff8c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -43,11 +43,13 @@
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.TaskStateHandles;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
 import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
+import org.apache.flink.runtime.testutils.RecoverableCompletedCheckpointStore;
 import org.apache.flink.runtime.util.TestByteStreamStateHandleDeepCompare;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.Preconditions;
@@ -86,13 +88,17 @@
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
 
 /**
  * Tests for the checkpoint coordinator.
@@ -545,19 +551,24 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			}
 
 			// acknowledge from one of the tasks
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointId));
+			SubtaskState subtaskState2 = mock(SubtaskState.class);
+			AcknowledgeCheckpoint acknowledgeCheckpoint1 = new AcknowledgeCheckpoint(jid, attemptID2, checkpointId, new CheckpointMetrics(), subtaskState2);
+			coord.receiveAcknowledgeMessage(acknowledgeCheckpoint1);
 			assertEquals(1, checkpoint.getNumberOfAcknowledgedTasks());
 			assertEquals(1, checkpoint.getNumberOfNonAcknowledgedTasks());
 			assertFalse(checkpoint.isDiscarded());
 			assertFalse(checkpoint.isFullyAcknowledged());
+			verify(subtaskState2, never()).registerSharedStates(any(SharedStateRegistry.class));
 
 			// acknowledge the same task again (should not matter)
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointId));
+			coord.receiveAcknowledgeMessage(acknowledgeCheckpoint1);
 			assertFalse(checkpoint.isDiscarded());
 			assertFalse(checkpoint.isFullyAcknowledged());
+			verify(subtaskState2, never()).registerSharedStates(any(SharedStateRegistry.class));
 
 			// acknowledge the other task.
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId));
+			SubtaskState subtaskState1 = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId, new CheckpointMetrics(), subtaskState1));
 
 			// the checkpoint is internally converted to a successful checkpoint and the
 			// pending checkpoint object is disposed
@@ -570,6 +581,12 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			// the canceler should be removed now
 			assertEquals(0, coord.getNumScheduledTasks());
 
+			// validate that the subtasks states have registered their shared states.
+			{
+				verify(subtaskState1, times(1)).registerSharedStates(any(SharedStateRegistry.class));
+				verify(subtaskState2, times(1)).registerSharedStates(any(SharedStateRegistry.class));
+			}
+
 			// validate that the relevant tasks got a confirmation message
 			{
 				verify(vertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId), eq(timestamp), any(CheckpointOptions.class));
@@ -580,7 +597,7 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			assertEquals(jid, success.getJobId());
 			assertEquals(timestamp, success.getTimestamp());
 			assertEquals(checkpoint.getCheckpointId(), success.getCheckpointID());
-			assertTrue(success.getTaskStates().isEmpty());
+			assertEquals(2, success.getTaskStates().size());
 
 			// ---------------
 			// trigger another checkpoint and see that this one replaces the other checkpoint
@@ -602,6 +619,12 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			assertEquals(checkpointIdNew, successNew.getCheckpointID());
 			assertTrue(successNew.getTaskStates().isEmpty());
 
+			// validate that the subtask states in old savepoint have unregister their shared states
+			{
+				verify(subtaskState1, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
+				verify(subtaskState2, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
+			}
+
 			// validate that the relevant tasks got a confirmation message
 			{
 				verify(vertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointIdNew), eq(timestampNew), any(CheckpointOptions.class));
@@ -678,8 +701,6 @@ public void testMultipleConcurrentCheckpoints() {
 			verify(triggerVertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId1), eq(timestamp1), any(CheckpointOptions.class));
 			verify(triggerVertex2.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId1), eq(timestamp1), any(CheckpointOptions.class));
 
-			CheckpointMetaData checkpointMetaData1 = new CheckpointMetaData(checkpointId1, 0L);
-
 			// acknowledge one of the three tasks
 			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId1));
 
@@ -699,8 +720,6 @@ public void testMultipleConcurrentCheckpoints() {
 			}
 			long checkpointId2 = pending2.getCheckpointId();
 
-			CheckpointMetaData checkpointMetaData2 = new CheckpointMetaData(checkpointId2, 0L);
-
 			// trigger messages should have been sent
 			verify(triggerVertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId2), eq(timestamp2), any(CheckpointOptions.class));
 			verify(triggerVertex2.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId2), eq(timestamp2), any(CheckpointOptions.class));
@@ -812,10 +831,9 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			verify(triggerVertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId1), eq(timestamp1), any(CheckpointOptions.class));
 			verify(triggerVertex2.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId1), eq(timestamp1), any(CheckpointOptions.class));
 
-			CheckpointMetaData checkpointMetaData1 = new CheckpointMetaData(checkpointId1, 0L);
-
 			// acknowledge one of the three tasks
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId1));
+			SubtaskState subtaskState1_2 = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId1, new CheckpointMetrics(), subtaskState1_2));
 
 			// start the second checkpoint
 			// trigger the first checkpoint. this should succeed
@@ -839,12 +857,18 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 
 			// we acknowledge one more task from the first checkpoint and the second
 			// checkpoint completely. The second checkpoint should then subsume the first checkpoint
-			CheckpointMetaData checkpointMetaData2= new CheckpointMetaData(checkpointId2, 0L);
 
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId2));
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId2));
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId1));
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId2));
+			SubtaskState subtaskState2_3 = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId2, new CheckpointMetrics(), subtaskState2_3));
+
+			SubtaskState subtaskState2_1 = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId2, new CheckpointMetrics(), subtaskState2_1));
+
+			SubtaskState subtaskState1_1 = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId1, new CheckpointMetrics(), subtaskState1_1));
+
+			SubtaskState subtaskState2_2 = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId2, new CheckpointMetrics(), subtaskState2_2));
 
 			// now, the second checkpoint should be confirmed, and the first discarded
 			// actually both pending checkpoints are discarded, and the second has been transformed
@@ -855,21 +879,47 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			assertEquals(0, coord.getNumberOfPendingCheckpoints());
 			assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
 
+			// validate that all received subtask states in the first checkpoint have been discarded
+			verify(subtaskState1_1, times(1)).discardSharedStatesOnFail();
+			verify(subtaskState1_2, times(1)).discardSharedStatesOnFail();
+			verify(subtaskState1_1, times(1)).discardState();
+			verify(subtaskState1_2, times(1)).discardState();
+
+			// validate that all subtask states in the second checkpoint are not discarded
+			verify(subtaskState2_1, never()).unregisterSharedStates(any(SharedStateRegistry.class));
+			verify(subtaskState2_2, never()).unregisterSharedStates(any(SharedStateRegistry.class));
+			verify(subtaskState2_3, never()).unregisterSharedStates(any(SharedStateRegistry.class));
+			verify(subtaskState2_1, never()).discardState();
+			verify(subtaskState2_2, never()).discardState();
+			verify(subtaskState2_3, never()).discardState();
+
 			// validate the committed checkpoints
 			List<CompletedCheckpoint> scs = coord.getSuccessfulCheckpoints();
 			CompletedCheckpoint success = scs.get(0);
 			assertEquals(checkpointId2, success.getCheckpointID());
 			assertEquals(timestamp2, success.getTimestamp());
 			assertEquals(jid, success.getJobId());
-			assertTrue(success.getTaskStates().isEmpty());
+			assertEquals(3, success.getTaskStates().size());
 
 			// the first confirm message should be out
 			verify(commitVertex.getCurrentExecutionAttempt(), times(1)).notifyCheckpointComplete(eq(checkpointId2), eq(timestamp2));
 
 			// send the last remaining ack for the first checkpoint. This should not do anything
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId1));
+			SubtaskState subtaskState1_3 = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId1, new CheckpointMetrics(), subtaskState1_3));
+			verify(subtaskState1_3, times(1)).discardSharedStatesOnFail();
+			verify(subtaskState1_3, times(1)).discardState();
 
 			coord.shutdown(JobStatus.FINISHED);
+
+			// validate that the states in the second checkpoint have been discarded
+			verify(subtaskState2_1, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
+			verify(subtaskState2_2, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
+			verify(subtaskState2_3, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
+			verify(subtaskState2_1, times(1)).discardState();
+			verify(subtaskState2_2, times(1)).discardState();
+			verify(subtaskState2_3, times(1)).discardState();
+
 		}
 		catch (Exception e) {
 			e.printStackTrace();
@@ -924,7 +974,8 @@ public void testCheckpointTimeoutIsolated() {
 			PendingCheckpoint checkpoint = coord.getPendingCheckpoints().values().iterator().next();
 			assertFalse(checkpoint.isDiscarded());
 
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpoint.getCheckpointId()));
+			SubtaskState subtaskState = mock(SubtaskState.class);
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpoint.getCheckpointId(), new CheckpointMetrics(), subtaskState));
 
 			// wait until the checkpoint must have expired.
 			// we check every 250 msecs conservatively for 5 seconds
@@ -941,6 +992,10 @@ public void testCheckpointTimeoutIsolated() {
 			assertEquals(0, coord.getNumberOfPendingCheckpoints());
 			assertEquals(0, coord.getNumberOfRetainedSuccessfulCheckpoints());
 
+			// validate that the received states have been discarded
+			verify(subtaskState, times(1)).discardSharedStatesOnFail();
+			verify(subtaskState, times(1)).discardState();
+
 			// no confirm message must have been sent
 			verify(commitVertex.getCurrentExecutionAttempt(), times(0)).notifyCheckpointComplete(anyLong(), anyLong());
 
@@ -993,8 +1048,6 @@ public void testHandleMessagesForNonExistingCheckpoints() {
 			// of the vertices that need to be acknowledged.
 			// non of the messages should throw an exception
 
-			CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
-
 			// wrong job id
 			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(new JobID(), ackAttemptID1, checkpointId));
 
@@ -1058,19 +1111,22 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 
 		long checkpointId = pendingCheckpoint.getCheckpointId();
 
-		CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
-
 		SubtaskState triggerSubtaskState = mock(SubtaskState.class);
 
 		// acknowledge the first trigger vertex
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, triggerAttemptId, checkpointId, new CheckpointMetrics(), triggerSubtaskState));
 
+		// verify that the subtask state has registered its shared states at the registry
+		verify(triggerSubtaskState, never()).discardSharedStatesOnFail();
+		verify(triggerSubtaskState, never()).discardState();
+
 		SubtaskState unknownSubtaskState = mock(SubtaskState.class);
 
 		// receive an acknowledge message for an unknown vertex
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), unknownSubtaskState));
 
 		// we should discard acknowledge messages from an unknown vertex belonging to our job
+		verify(unknownSubtaskState, times(1)).discardSharedStatesOnFail();
 		verify(unknownSubtaskState, times(1)).discardState();
 
 		SubtaskState differentJobSubtaskState = mock(SubtaskState.class);
@@ -1079,20 +1135,25 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), differentJobSubtaskState));
 
 		// we should not interfere with different jobs
+		verify(differentJobSubtaskState, never()).discardSharedStatesOnFail();
 		verify(differentJobSubtaskState, never()).discardState();
 
 		// duplicate acknowledge message for the trigger vertex
+		reset(triggerSubtaskState);
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, triggerAttemptId, checkpointId, new CheckpointMetrics(), triggerSubtaskState));
 
 		// duplicate acknowledge messages for a known vertex should not trigger discarding the state
+		verify(triggerSubtaskState, never()).discardSharedStatesOnFail();
 		verify(triggerSubtaskState, never()).discardState();
 
 		// let the checkpoint fail at the first ack vertex
+		reset(triggerSubtaskState);
 		coord.receiveDeclineMessage(new DeclineCheckpoint(jobId, ackAttemptId1, checkpointId));
 
 		assertTrue(pendingCheckpoint.isDiscarded());
 
 		// check that we've cleaned up the already acknowledged state
+		verify(triggerSubtaskState, times(1)).discardSharedStatesOnFail();
 		verify(triggerSubtaskState, times(1)).discardState();
 
 		SubtaskState ackSubtaskState = mock(SubtaskState.class);
@@ -1101,12 +1162,15 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, ackAttemptId2, checkpointId, new CheckpointMetrics(), ackSubtaskState));
 
 		// check that we also cleaned up this state
+		verify(ackSubtaskState, times(1)).discardSharedStatesOnFail();
 		verify(ackSubtaskState, times(1)).discardState();
 
 		// receive an acknowledge message from an unknown job
+		reset(differentJobSubtaskState);
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), differentJobSubtaskState));
 
 		// we should not interfere with different jobs
+		verify(differentJobSubtaskState, never()).discardSharedStatesOnFail();
 		verify(differentJobSubtaskState, never()).discardState();
 
 		SubtaskState unknownSubtaskState2 = mock(SubtaskState.class);
@@ -1115,6 +1179,7 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), unknownSubtaskState2));
 
 		// we should discard acknowledge messages from an unknown vertex belonging to our job
+		verify(unknownSubtaskState2, times(1)).discardSharedStatesOnFail();
 		verify(unknownSubtaskState2, times(1)).discardState();
 	}
 
@@ -1363,12 +1428,11 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		assertFalse(pending.isDiscarded());
 		assertFalse(pending.isFullyAcknowledged());
 		assertFalse(pending.canBeSubsumed());
-		assertTrue(pending instanceof PendingCheckpoint);
-
-		CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
 
 		// acknowledge from one of the tasks
-		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointId));
+		SubtaskState subtaskState2 = mock(SubtaskState.class);
+		AcknowledgeCheckpoint acknowledgeCheckpoint2 = new AcknowledgeCheckpoint(jid, attemptID2, checkpointId, new CheckpointMetrics(), subtaskState2);
+		coord.receiveAcknowledgeMessage(acknowledgeCheckpoint2);
 		assertEquals(1, pending.getNumberOfAcknowledgedTasks());
 		assertEquals(1, pending.getNumberOfNonAcknowledgedTasks());
 		assertFalse(pending.isDiscarded());
@@ -1376,13 +1440,14 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		assertFalse(savepointFuture.isDone());
 
 		// acknowledge the same task again (should not matter)
-		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointId));
+		coord.receiveAcknowledgeMessage(acknowledgeCheckpoint2);
 		assertFalse(pending.isDiscarded());
 		assertFalse(pending.isFullyAcknowledged());
 		assertFalse(savepointFuture.isDone());
 
 		// acknowledge the other task.
-		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId));
+		SubtaskState subtaskState1 = mock(SubtaskState.class);
+		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId, new CheckpointMetrics(), subtaskState1));
 
 		// the checkpoint is internally converted to a successful checkpoint and the
 		// pending checkpoint object is disposed
@@ -1399,11 +1464,17 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 			verify(vertex2.getCurrentExecutionAttempt(), times(1)).notifyCheckpointComplete(eq(checkpointId), eq(timestamp));
 		}
 
+		// validate that the shared states are registered
+		{
+			verify(subtaskState1, times(1)).registerSharedStates(any(SharedStateRegistry.class));
+			verify(subtaskState2, times(1)).registerSharedStates(any(SharedStateRegistry.class));
+		}
+
 		CompletedCheckpoint success = coord.getSuccessfulCheckpoints().get(0);
 		assertEquals(jid, success.getJobId());
 		assertEquals(timestamp, success.getTimestamp());
 		assertEquals(pending.getCheckpointId(), success.getCheckpointID());
-		assertTrue(success.getTaskStates().isEmpty());
+		assertEquals(2, success.getTaskStates().size());
 
 		// ---------------
 		// trigger another checkpoint and see that this one replaces the other checkpoint
@@ -1426,6 +1497,14 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		assertTrue(successNew.getTaskStates().isEmpty());
 		assertTrue(savepointFuture.isDone());
 
+		// validate that the first savepoint does not discard its private states.
+		verify(subtaskState1, never()).discardState();
+		verify(subtaskState2, never()).discardState();
+
+		// Savepoints are not supposed to have any shared state.
+		verify(subtaskState1, never()).unregisterSharedStates(any(SharedStateRegistry.class));
+		verify(subtaskState2, never()).unregisterSharedStates(any(SharedStateRegistry.class));
+
 		// validate that the relevant tasks got a confirmation message
 		{
 			verify(vertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointIdNew), eq(timestampNew), any(CheckpointOptions.class));
@@ -1478,7 +1557,6 @@ public void testSavepointsAreNotSubsumed() throws Exception {
 		// Trigger savepoint and checkpoint
 		Future<CompletedCheckpoint> savepointFuture1 = coord.triggerSavepoint(timestamp, savepointDir);
 		long savepointId1 = counter.getLast();
-		CheckpointMetaData checkpointMetaDataS1 = new CheckpointMetaData(savepointId1, 0L);
 		assertEquals(1, coord.getNumberOfPendingCheckpoints());
 
 		assertTrue(coord.triggerCheckpoint(timestamp + 1, false));
@@ -1488,8 +1566,6 @@ public void testSavepointsAreNotSubsumed() throws Exception {
 		long checkpointId2 = counter.getLast();
 		assertEquals(3, coord.getNumberOfPendingCheckpoints());
 
-		CheckpointMetaData checkpointMetaData2 = new CheckpointMetaData(checkpointId2, 0L);
-
 		// 2nd checkpoint should subsume the 1st checkpoint, but not the savepoint
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId2));
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointId2));
@@ -1505,7 +1581,6 @@ public void testSavepointsAreNotSubsumed() throws Exception {
 
 		Future<CompletedCheckpoint> savepointFuture2 = coord.triggerSavepoint(timestamp + 4, savepointDir);
 		long savepointId2 = counter.getLast();
-		CheckpointMetaData checkpointMetaDataS2 = new CheckpointMetaData(savepointId2, 0L);
 		assertEquals(3, coord.getNumberOfPendingCheckpoints());
 
 		// 2nd savepoint should subsume the last checkpoint, but not the 1st savepoint
@@ -1880,6 +1955,8 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 		ExecutionVertex[] arrayExecutionVertices =
 				allExecutionVertices.toArray(new ExecutionVertex[allExecutionVertices.size()]);
 
+		CompletedCheckpointStore store = new RecoverableCompletedCheckpointStore();
+
 		// set up the coordinator and validate the initial state
 		CheckpointCoordinator coord = new CheckpointCoordinator(
 			jid,
@@ -1892,7 +1969,7 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 			arrayExecutionVertices,
 			arrayExecutionVertices,
 			new StandaloneCheckpointIDCounter(),
-			new StandaloneCompletedCheckpointStore(1),
+			store,
 			null,
 			Executors.directExecutor());
 
@@ -1901,38 +1978,32 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 
 		assertTrue(coord.getPendingCheckpoints().keySet().size() == 1);
 		long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());
-		CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
 
 		List<KeyGroupRange> keyGroupPartitions1 = StateAssignmentOperation.createKeyGroupPartitions(maxParallelism1, parallelism1);
 		List<KeyGroupRange> keyGroupPartitions2 = StateAssignmentOperation.createKeyGroupPartitions(maxParallelism2, parallelism2);
 
 		for (int index = 0; index < jobVertex1.getParallelism(); index++) {
-			ChainedStateHandle<StreamStateHandle> nonPartitionedState = generateStateForVertex(jobVertexID1, index);
-			ChainedStateHandle<OperatorStateHandle> partitionableState = generateChainedPartitionableStateHandle(jobVertexID1, index, 2, 8, false);
-			KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID1, keyGroupPartitions1.get(index), false);
+			SubtaskState subtaskState = mockSubtaskState(jobVertexID1, index, keyGroupPartitions1.get(index));
 
-			SubtaskState checkpointStateHandles = new SubtaskState(nonPartitionedState, partitionableState, null, partitionedKeyGroupState, null);
 			AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
 					jid,
 					jobVertex1.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
 					checkpointId,
 					new CheckpointMetrics(),
-					checkpointStateHandles);
+					subtaskState);
 
 			coord.receiveAcknowledgeMessage(acknowledgeCheckpoint);
 		}
 
 		for (int index = 0; index < jobVertex2.getParallelism(); index++) {
-			ChainedStateHandle<StreamStateHandle> nonPartitionedState = generateStateForVertex(jobVertexID2, index);
-			ChainedStateHandle<OperatorStateHandle> partitionableState = generateChainedPartitionableStateHandle(jobVertexID2, index, 2, 8, false);
-			KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID2, keyGroupPartitions2.get(index), false);
-			SubtaskState checkpointStateHandles = new SubtaskState(nonPartitionedState, partitionableState, null, partitionedKeyGroupState, null);
+			SubtaskState subtaskState = mockSubtaskState(jobVertexID2, index, keyGroupPartitions2.get(index));
+
 			AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
 					jid,
 					jobVertex2.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
 					checkpointId,
 					new CheckpointMetrics(),
-					checkpointStateHandles);
+					subtaskState);
 
 			coord.receiveAcknowledgeMessage(acknowledgeCheckpoint);
 		}
@@ -1941,6 +2012,20 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 
 		assertEquals(1, completedCheckpoints.size());
 
+		// shutdown the store
+		SharedStateRegistry sharedStateRegistry = coord.getSharedStateRegistry();
+		store.shutdown(JobStatus.SUSPENDED, sharedStateRegistry);
+
+		// All shared states should be unregistered once the store is shut down
+		for (CompletedCheckpoint completedCheckpoint : completedCheckpoints) {
+			for (TaskState taskState : completedCheckpoint.getTaskStates().values()) {
+				for (SubtaskState subtaskState : taskState.getStates()) {
+					verify(subtaskState, times(1)).unregisterSharedStates(sharedStateRegistry);
+				}
+			}
+		}
+
+		// restore the store
 		Map<JobVertexID, ExecutionJobVertex> tasks = new HashMap<>();
 
 		tasks.put(jobVertexID1, jobVertex1);
@@ -1948,6 +2033,15 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 
 		coord.restoreLatestCheckpointedState(tasks, true, false);
 
+		// validate that all shared states are registered again after the recovery.
+		for (CompletedCheckpoint completedCheckpoint : completedCheckpoints) {
+			for (TaskState taskState : completedCheckpoint.getTaskStates().values()) {
+				for (SubtaskState subtaskState : taskState.getStates()) {
+					verify(subtaskState, times(2)).registerSharedStates(sharedStateRegistry);
+				}
+			}
+		}
+
 		// verify the restored state
 		verifyStateRestore(jobVertexID1, jobVertex1, keyGroupPartitions1);
 		verifyStateRestore(jobVertexID2, jobVertex2, keyGroupPartitions2);
@@ -2666,6 +2760,26 @@ private static ExecutionVertex mockExecutionVertex(
 		return vertex;
 	}
 
+	static SubtaskState mockSubtaskState(
+		JobVertexID jobVertexID,
+		int index,
+		KeyGroupRange keyGroupRange) throws IOException {
+
+		ChainedStateHandle<StreamStateHandle> nonPartitionedState = generateStateForVertex(jobVertexID, index);
+		ChainedStateHandle<OperatorStateHandle> partitionableState = generateChainedPartitionableStateHandle(jobVertexID, index, 2, 8, false);
+		KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID, keyGroupRange, false);
+
+		SubtaskState subtaskState = mock(SubtaskState.class, withSettings().serializable());
+
+		doReturn(nonPartitionedState).when(subtaskState).getLegacyOperatorState();
+		doReturn(partitionableState).when(subtaskState).getManagedOperatorState();
+		doReturn(null).when(subtaskState).getRawOperatorState();
+		doReturn(partitionedKeyGroupState).when(subtaskState).getManagedKeyedState();
+		doReturn(null).when(subtaskState).getRawKeyedState();
+
+		return subtaskState;
+	}
+
 	public static void verifyStateRestore(
 			JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
 			List<KeyGroupRange> keyGroupPartitions) throws Exception {
@@ -3018,7 +3132,6 @@ public void testCheckpointStatsTrackerRestoreCallback() throws Exception {
 		ExecutionVertex vertex1 = mockExecutionVertex(new ExecutionAttemptID());
 
 		StandaloneCompletedCheckpointStore store = new StandaloneCompletedCheckpointStore(1);
-		store.addCheckpoint(new CompletedCheckpoint(new JobID(), 0, 0, 0, Collections.<JobVertexID, TaskState>emptyMap()));
 
 		// set up the coordinator and validate the initial state
 		CheckpointCoordinator coord = new CheckpointCoordinator(
@@ -3036,6 +3149,10 @@ public void testCheckpointStatsTrackerRestoreCallback() throws Exception {
 			null,
 			Executors.directExecutor());
 
+		store.addCheckpoint(
+			new CompletedCheckpoint(new JobID(), 0, 0, 0, Collections.<JobVertexID, TaskState>emptyMap()),
+			coord.getSharedStateRegistry());
+
 		CheckpointStatsTracker tracker = mock(CheckpointStatsTracker.class);
 		coord.setCheckpointStatsTracker(tracker);
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
index 7e0a7c1859cc6..9e372e161fbfd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
@@ -255,7 +255,7 @@ public void testNonRestoredState() throws Exception {
 		}
 		CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 1, 2, new HashMap<>(checkpointTaskStates));
 
-		coord.getCheckpointStore().addCheckpoint(checkpoint);
+		coord.getCheckpointStore().addCheckpoint(checkpoint, coord.getSharedStateRegistry());
 
 		coord.restoreLatestCheckpointedState(tasks, true, false);
 		coord.restoreLatestCheckpointedState(tasks, true, true);
@@ -273,7 +273,7 @@ public void testNonRestoredState() throws Exception {
 
 		checkpoint = new CompletedCheckpoint(new JobID(), 1, 2, 3, new HashMap<>(checkpointTaskStates));
 
-		coord.getCheckpointStore().addCheckpoint(checkpoint);
+		coord.getCheckpointStore().addCheckpoint(checkpoint, coord.getSharedStateRegistry());
 
 		// (i) Allow non restored state (should succeed)
 		coord.restoreLatestCheckpointedState(tasks, true, true);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
index f77c755797827..aa1726bde5f45 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
@@ -21,13 +21,14 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.messages.CheckpointMessagesTest;
-import org.apache.flink.runtime.state.ChainedStateHandle;
-import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -36,6 +37,9 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 /**
  * Test for basic {@link CompletedCheckpointStore} contract.
@@ -64,7 +68,8 @@ public void testExceptionOnNoRetainedCheckpoints() throws Exception {
 	@Test
 	public void testAddAndGetLatestCheckpoint() throws Exception {
 		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
-
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+		
 		// Empty state
 		assertEquals(0, checkpoints.getNumberOfRetainedCheckpoints());
 		assertEquals(0, checkpoints.getAllCheckpoints().size());
@@ -73,11 +78,11 @@ public void testAddAndGetLatestCheckpoint() throws Exception {
 				createCheckpoint(0), createCheckpoint(1) };
 
 		// Add and get latest
-		checkpoints.addCheckpoint(expected[0]);
+		checkpoints.addCheckpoint(expected[0], sharedStateRegistry);
 		assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
 		verifyCheckpoint(expected[0], checkpoints.getLatestCheckpoint());
 
-		checkpoints.addCheckpoint(expected[1]);
+		checkpoints.addCheckpoint(expected[1], sharedStateRegistry);
 		assertEquals(2, checkpoints.getNumberOfRetainedCheckpoints());
 		verifyCheckpoint(expected[1], checkpoints.getLatestCheckpoint());
 	}
@@ -88,7 +93,8 @@ public void testAddAndGetLatestCheckpoint() throws Exception {
 	 */
 	@Test
 	public void testAddCheckpointMoreThanMaxRetained() throws Exception {
-		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(1);
+		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(1);   
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1),
@@ -96,16 +102,24 @@ public void testAddCheckpointMoreThanMaxRetained() throws Exception {
 		};
 
 		// Add checkpoints
-		checkpoints.addCheckpoint(expected[0]);
+		checkpoints.addCheckpoint(expected[0], sharedStateRegistry);
 		assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
 
 		for (int i = 1; i < expected.length; i++) {
-			checkpoints.addCheckpoint(expected[i]);
+			Collection<TaskState> taskStates = expected[i - 1].getTaskStates().values();
+
+			checkpoints.addCheckpoint(expected[i], sharedStateRegistry);
 
 			// The ZooKeeper implementation discards asynchronously
 			expected[i - 1].awaitDiscard();
 			assertTrue(expected[i - 1].isDiscarded());
 			assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
+
+			for (TaskState taskState : taskStates) {
+				for (SubtaskState subtaskState : taskState.getStates()) {
+					verify(subtaskState, times(1)).unregisterSharedStates(sharedStateRegistry);
+				}
+			}
 		}
 	}
 
@@ -132,6 +146,7 @@ public void testEmptyState() throws Exception {
 	@Test
 	public void testGetAllCheckpoints() throws Exception {
 		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1),
@@ -139,7 +154,7 @@ public void testGetAllCheckpoints() throws Exception {
 		};
 
 		for (TestCompletedCheckpoint checkpoint : expected) {
-			checkpoints.addCheckpoint(checkpoint);
+			checkpoints.addCheckpoint(checkpoint, sharedStateRegistry);
 		}
 
 		List<CompletedCheckpoint> actual = checkpoints.getAllCheckpoints();
@@ -157,6 +172,7 @@ public void testGetAllCheckpoints() throws Exception {
 	@Test
 	public void testDiscardAllCheckpoints() throws Exception {
 		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1),
@@ -164,10 +180,10 @@ public void testDiscardAllCheckpoints() throws Exception {
 		};
 
 		for (TestCompletedCheckpoint checkpoint : expected) {
-			checkpoints.addCheckpoint(checkpoint);
+			checkpoints.addCheckpoint(checkpoint, sharedStateRegistry);
 		}
 
-		checkpoints.shutdown(JobStatus.FINISHED);
+		checkpoints.shutdown(JobStatus.FINISHED, sharedStateRegistry);
 
 		// Empty state
 		assertNull(checkpoints.getLatestCheckpoint());
@@ -203,15 +219,39 @@ protected TestCompletedCheckpoint createCheckpoint(int id, int numberOfStates, C
 		taskGroupStates.put(jvid, taskState);
 
 		for (int i = 0; i < numberOfStates; i++) {
-			ChainedStateHandle<StreamStateHandle> stateHandle = CheckpointCoordinatorTest.generateChainedStateHandle(
-					new CheckpointMessagesTest.MyHandle());
+			SubtaskState subtaskState = CheckpointCoordinatorTest.mockSubtaskState(jvid, i, new KeyGroupRange(i, i));
 
-			taskState.putState(i, new SubtaskState(stateHandle, null, null, null, null));
+			taskState.putState(i, subtaskState);
 		}
 
 		return new TestCompletedCheckpoint(new JobID(), id, 0, taskGroupStates, props);
 	}
 
+	protected void resetCheckpoint(Collection<TaskState> taskStates) {
+		for (TaskState taskState : taskStates) {
+			for (SubtaskState subtaskState : taskState.getStates()) {
+				Mockito.reset(subtaskState);
+			}
+		}
+	}
+
+	protected void verifyCheckpointRegistered(Collection<TaskState> taskStates, SharedStateRegistry sharedStateRegistry) {
+		for (TaskState taskState : taskStates) {
+			for (SubtaskState subtaskState : taskState.getStates()) {
+				verify(subtaskState, times(1)).registerSharedStates(eq(sharedStateRegistry));
+			}
+		}
+	}
+
+	protected void verifyCheckpointDiscarded(Collection<TaskState> taskStates) {
+		for (TaskState taskState : taskStates) {
+			for (SubtaskState subtaskState : taskState.getStates()) {
+				verify(subtaskState, times(1)).discardSharedStatesOnFail();
+				verify(subtaskState, times(1)).discardState();
+			}
+		}
+	}
+
 	private void verifyCheckpoint(CompletedCheckpoint expected, CompletedCheckpoint actual) {
 		assertEquals(expected, actual);
 	}
@@ -241,8 +281,8 @@ public TestCompletedCheckpoint(
 		}
 
 		@Override
-		public boolean subsume() throws Exception {
-			if (super.subsume()) {
+		public boolean discardOnSubsume(SharedStateRegistry sharedStateRegistry) throws Exception {
+			if (super.discardOnSubsume(sharedStateRegistry)) {
 				discard();
 				return true;
 			} else {
@@ -251,8 +291,8 @@ public boolean subsume() throws Exception {
 		}
 
 		@Override
-		public boolean discard(JobStatus jobStatus) throws Exception {
-			if (super.discard(jobStatus)) {
+		public boolean discardOnShutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
+			if (super.discardOnShutdown(jobStatus, sharedStateRegistry)) {
 				discard();
 				return true;
 			} else {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
index b34e9a6ac26f5..0b759d4056273 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
@@ -23,6 +23,8 @@
 import org.apache.flink.core.testutils.CommonTestUtils;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.state.SharedStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
 import org.junit.Rule;
 import org.junit.Test;
@@ -30,10 +32,12 @@
 import org.mockito.Mockito;
 
 import java.io.File;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -61,7 +65,7 @@ public void testDiscard() throws Exception {
 				new FileStateHandle(new Path(file.toURI()), file.length()),
 				file.getAbsolutePath());
 
-		checkpoint.discard(JobStatus.FAILED);
+		checkpoint.discardOnShutdown(JobStatus.FAILED, new SharedStateRegistry());
 
 		assertEquals(false, file.exists());
 	}
@@ -80,10 +84,15 @@ public void testCleanUpOnSubsume() throws Exception {
 		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
 				new JobID(), 0, 0, 1, taskStates, props);
 
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+		checkpoint.registerSharedStates(sharedStateRegistry);
+		verify(state, times(1)).registerSharedStates(sharedStateRegistry);
+
 		// Subsume
-		checkpoint.subsume();
+		checkpoint.discardOnSubsume(sharedStateRegistry);
 
 		verify(state, times(1)).discardState();
+		verify(state, times(1)).unregisterSharedStates(sharedStateRegistry);
 	}
 
 	/**
@@ -112,17 +121,22 @@ public void testCleanUpOnShutdown() throws Exception {
 					new FileStateHandle(new Path(file.toURI()), file.length()),
 					externalPath);
 
-			checkpoint.discard(status);
+			SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+			checkpoint.registerSharedStates(sharedStateRegistry);
+
+			checkpoint.discardOnShutdown(status, sharedStateRegistry);
 			verify(state, times(0)).discardState();
 			assertEquals(true, file.exists());
+			verify(state, times(0)).unregisterSharedStates(sharedStateRegistry);
 
 			// Discard
 			props = new CheckpointProperties(false, false, true, true, true, true, true);
 			checkpoint = new CompletedCheckpoint(
 					new JobID(), 0, 0, 1, new HashMap<>(taskStates), props);
 
-			checkpoint.discard(status);
+			checkpoint.discardOnShutdown(status, sharedStateRegistry);
 			verify(state, times(1)).discardState();
+			verify(state, times(1)).unregisterSharedStates(sharedStateRegistry);
 		}
 	}
 
@@ -146,7 +160,7 @@ public void testCompletedCheckpointStatsCallbacks() throws Exception {
 		CompletedCheckpointStats.DiscardCallback callback = mock(CompletedCheckpointStats.DiscardCallback.class);
 		completed.setDiscardCallback(callback);
 
-		completed.discard(JobStatus.FINISHED);
+		completed.discardOnShutdown(JobStatus.FINISHED, new SharedStateRegistry());
 		verify(callback, times(1)).notifyDiscardedCheckpoint();
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
index 0ab031e7023c2..e7c1c3b1670b1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
@@ -31,6 +31,7 @@
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.SerializedValue;
 
@@ -40,6 +41,8 @@
 import java.net.URL;
 import java.util.Collections;
 
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -59,7 +62,7 @@ public void testShutdownCheckpointCoordinator() throws Exception {
 		graph.fail(new Exception("Test Exception"));
 
 		verify(counter, times(1)).shutdown(JobStatus.FAILED);
-		verify(store, times(1)).shutdown(JobStatus.FAILED);
+		verify(store, times(1)).shutdown(eq(JobStatus.FAILED), any(SharedStateRegistry.class));
 	}
 
 	/**
@@ -75,8 +78,8 @@ public void testSuspendCheckpointCoordinator() throws Exception {
 		graph.suspend(new Exception("Test Exception"));
 
 		// No shutdown
-		verify(counter, times(1)).shutdown(Matchers.eq(JobStatus.SUSPENDED));
-		verify(store, times(1)).shutdown(Matchers.eq(JobStatus.SUSPENDED));
+		verify(counter, times(1)).shutdown(eq(JobStatus.SUSPENDED));
+		verify(store, times(1)).shutdown(eq(JobStatus.SUSPENDED), any(SharedStateRegistry.class));
 	}
 
 	private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
index a15684c6c4b7a..d77fac1bed9dd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
@@ -24,15 +24,19 @@
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.state.SharedStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.mockito.Mock;
 import org.mockito.Mockito;
 
 import java.io.File;
 import java.lang.reflect.Field;
 import java.util.ArrayDeque;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Queue;
@@ -45,7 +49,10 @@
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.powermock.api.mockito.PowerMockito.when;
@@ -184,9 +191,12 @@ public void testCompletionFuture() throws Exception {
 	@SuppressWarnings("unchecked")
 	public void testAbortDiscardsState() throws Exception {
 		CheckpointProperties props = new CheckpointProperties(false, true, false, false, false, false, false);
-		TaskState state = mock(TaskState.class);
 		QueueExecutor executor = new QueueExecutor();
 
+		TaskState state = mock(TaskState.class);
+		doNothing().when(state).registerSharedStates(any(SharedStateRegistry.class));
+		doNothing().when(state).unregisterSharedStates(any(SharedStateRegistry.class));
+
 		String targetDir = tmpFolder.newFolder().getAbsolutePath();
 
 		// Abort declined
@@ -197,6 +207,7 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
+		verify(state, times(1)).discardSharedStatesOnFail();
 
 		// Abort error
 		Mockito.reset(state);
@@ -208,6 +219,7 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
+		verify(state, times(1)).discardSharedStatesOnFail();
 
 		// Abort expired
 		Mockito.reset(state);
@@ -219,6 +231,7 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
+		verify(state, times(1)).discardSharedStatesOnFail();
 
 		// Abort subsumed
 		Mockito.reset(state);
@@ -230,6 +243,7 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
+		verify(state, times(1)).discardSharedStatesOnFail();
 	}
 
 	/**
@@ -340,7 +354,11 @@ private static PendingCheckpoint createPendingCheckpoint(CheckpointProperties pr
 		return createPendingCheckpoint(props, targetDirectory, Executors.directExecutor());
 	}
 
-	private static PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, String targetDirectory, Executor executor) {
+	private static PendingCheckpoint createPendingCheckpoint(
+			CheckpointProperties props,
+			String targetDirectory,
+			Executor executor) {
+
 		Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(ACK_TASKS);
 		return new PendingCheckpoint(
 			new JobID(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
index cc7b2d0855fb4..7a8589719819e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
@@ -19,9 +19,12 @@
 package org.apache.flink.runtime.checkpoint;
 
 import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
@@ -49,15 +52,18 @@ protected CompletedCheckpointStore createCompletedCheckpoints(
 	@Test
 	public void testShutdownDiscardsCheckpoints() throws Exception {
 		CompletedCheckpointStore store = createCompletedCheckpoints(1);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
+		Collection<TaskState> taskStates = checkpoint.getTaskStates().values();
 
-		store.addCheckpoint(checkpoint);
+		store.addCheckpoint(checkpoint, sharedStateRegistry);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
+		verifyCheckpointRegistered(taskStates, sharedStateRegistry);
 
-		store.shutdown(JobStatus.FINISHED);
-
+		store.shutdown(JobStatus.FINISHED, sharedStateRegistry);
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertTrue(checkpoint.isDiscarded());
+		verifyCheckpointDiscarded(taskStates);
 	}
 
 	/**
@@ -67,15 +73,18 @@ public void testShutdownDiscardsCheckpoints() throws Exception {
 	@Test
 	public void testSuspendDiscardsCheckpoints() throws Exception {
 		CompletedCheckpointStore store = createCompletedCheckpoints(1);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
+		Collection<TaskState> taskStates = checkpoint.getTaskStates().values();
 
-		store.addCheckpoint(checkpoint);
+		store.addCheckpoint(checkpoint, sharedStateRegistry);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
+		verifyCheckpointRegistered(taskStates, sharedStateRegistry);
 
-		store.shutdown(JobStatus.SUSPENDED);
-
+		store.shutdown(JobStatus.SUSPENDED, sharedStateRegistry);
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertTrue(checkpoint.isDiscarded());
+		verifyCheckpointDiscarded(taskStates);
 	}
 	
 	/**
@@ -87,14 +96,16 @@ public void testAddCheckpointWithFailedRemove() throws Exception {
 		
 		final int numCheckpointsToRetain = 1;
 		CompletedCheckpointStore store = createCompletedCheckpoints(numCheckpointsToRetain);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		
 		for (long i = 0; i <= numCheckpointsToRetain; ++i) {
 			CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class);
 			doReturn(i).when(checkpointToAdd).getCheckpointID();
-			doThrow(new IOException()).when(checkpointToAdd).subsume();
+			doReturn(Collections.emptyMap()).when(checkpointToAdd).getTaskStates();
+			doThrow(new IOException()).when(checkpointToAdd).discardOnSubsume(sharedStateRegistry);
 			
 			try {
-				store.addCheckpoint(checkpointToAdd);
+				store.addCheckpoint(checkpointToAdd, sharedStateRegistry);
 				
 				// The checkpoint should be in the store if we successfully add it into the store.
 				List<CompletedCheckpoint> addedCheckpoints = store.getAllCheckpoints();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
index 625999a3343c4..607e773cb19c6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
@@ -22,6 +22,7 @@
 import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.state.RetrievableStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper;
 import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
 import org.junit.AfterClass;
@@ -80,22 +81,32 @@ public RetrievableStateHandle<CompletedCheckpoint> store(CompletedCheckpoint sta
 	@Test
 	public void testRecover() throws Exception {
 		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(3);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1), createCheckpoint(2)
 		};
 
 		// Add multiple checkpoints
-		checkpoints.addCheckpoint(expected[0]);
-		checkpoints.addCheckpoint(expected[1]);
-		checkpoints.addCheckpoint(expected[2]);
+		checkpoints.addCheckpoint(expected[0], sharedStateRegistry);
+		checkpoints.addCheckpoint(expected[1], sharedStateRegistry);
+		checkpoints.addCheckpoint(expected[2], sharedStateRegistry);
+
+		verifyCheckpointRegistered(expected[0].getTaskStates().values(), sharedStateRegistry);
+		verifyCheckpointRegistered(expected[1].getTaskStates().values(), sharedStateRegistry);
+		verifyCheckpointRegistered(expected[2].getTaskStates().values(), sharedStateRegistry);
 
 		// All three should be in ZK
 		assertEquals(3, ZooKeeper.getClient().getChildren().forPath(CheckpointsPath).size());
 		assertEquals(3, checkpoints.getNumberOfRetainedCheckpoints());
 
+		resetCheckpoint(expected[0].getTaskStates().values());
+		resetCheckpoint(expected[1].getTaskStates().values());
+		resetCheckpoint(expected[2].getTaskStates().values());
+
 		// Recover
-		checkpoints.recover();
+		SharedStateRegistry newSharedStateRegistry = new SharedStateRegistry();
+		checkpoints.recover(newSharedStateRegistry);
 
 		assertEquals(3, ZooKeeper.getClient().getChildren().forPath(CheckpointsPath).size());
 		assertEquals(3, checkpoints.getNumberOfRetainedCheckpoints());
@@ -106,11 +117,15 @@ public void testRecover() throws Exception {
 		expectedCheckpoints.add(expected[2]);
 		expectedCheckpoints.add(createCheckpoint(3));
 
-		checkpoints.addCheckpoint(expectedCheckpoints.get(2));
+		checkpoints.addCheckpoint(expectedCheckpoints.get(2), newSharedStateRegistry);
 
 		List<CompletedCheckpoint> actualCheckpoints = checkpoints.getAllCheckpoints();
 
 		assertEquals(expectedCheckpoints, actualCheckpoints);
+
+		for (CompletedCheckpoint actualCheckpoint : actualCheckpoints) {
+			verifyCheckpointRegistered(actualCheckpoint.getTaskStates().values(), newSharedStateRegistry);
+		}
 	}
 
 	/**
@@ -121,18 +136,18 @@ public void testShutdownDiscardsCheckpoints() throws Exception {
 		CuratorFramework client = ZooKeeper.getClient();
 
 		CompletedCheckpointStore store = createCompletedCheckpoints(1);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
 
-		store.addCheckpoint(checkpoint);
+		store.addCheckpoint(checkpoint, sharedStateRegistry);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
 		assertNotNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
-		store.shutdown(JobStatus.FINISHED);
-
+		store.shutdown(JobStatus.FINISHED, sharedStateRegistry);
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
-		store.recover();
+		store.recover(sharedStateRegistry);
 
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 	}
@@ -146,19 +161,20 @@ public void testSuspendKeepsCheckpoints() throws Exception {
 		CuratorFramework client = ZooKeeper.getClient();
 
 		CompletedCheckpointStore store = createCompletedCheckpoints(1);
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
 
-		store.addCheckpoint(checkpoint);
+		store.addCheckpoint(checkpoint, sharedStateRegistry);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
 		assertNotNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
-		store.shutdown(JobStatus.SUSPENDED);
+		store.shutdown(JobStatus.SUSPENDED, sharedStateRegistry);
 
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertNotNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
 		// Recover again
-		store.recover();
+		store.recover(sharedStateRegistry);
 
 		CompletedCheckpoint recovered = store.getLatestCheckpoint();
 		assertEquals(checkpoint, recovered);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
index aa2ec851d53d9..1f5731d319bb1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
@@ -27,6 +27,7 @@
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.runtime.state.RetrievableStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper;
 import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore;
 import org.apache.flink.util.TestLogger;
@@ -40,6 +41,7 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.Executor;
@@ -158,7 +160,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
 			stateSotrage,
 			Executors.directExecutor());
 
-		zooKeeperCompletedCheckpointStore.recover();
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+
+		zooKeeperCompletedCheckpointStore.recover(sharedStateRegistry);
 
 		CompletedCheckpoint latestCompletedCheckpoint = zooKeeperCompletedCheckpointStore.getLatestCheckpoint();
 
@@ -222,14 +226,17 @@ public RetrievableStateHandle<CompletedCheckpoint> answer(InvocationOnMock invoc
 			checkpointsPath,
 			stateSotrage,
 			Executors.directExecutor());
+
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		
 		
 		for (long i = 0; i <= numCheckpointsToRetain; ++i) {
 			CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class);
 			doReturn(i).when(checkpointToAdd).getCheckpointID();
+			doReturn(Collections.emptyMap()).when(checkpointToAdd).getTaskStates();
 			
 			try {
-				zooKeeperCompletedCheckpointStore.addCheckpoint(checkpointToAdd);
+				zooKeeperCompletedCheckpointStore.addCheckpoint(checkpointToAdd, sharedStateRegistry);
 				
 				// The checkpoint should be in the store if we successfully add it into the store.
 				List<CompletedCheckpoint> addedCheckpoints = zooKeeperCompletedCheckpointStore.getAllCheckpoints();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
index 6eacaac6d8044..77eb566349a44 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
@@ -71,6 +71,7 @@
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.TaskStateHandles;
 import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
@@ -81,6 +82,7 @@
 import org.apache.flink.runtime.testingUtils.TestingTaskManager;
 import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testutils.RecoverableCompletedCheckpointStore;
 import org.apache.flink.runtime.util.TestByteStreamStateHandleDeepCompare;
 import org.apache.flink.util.InstantiationUtil;
 
@@ -164,7 +166,7 @@ public void testJobRecoveryWhenLosingLeadership() throws Exception {
 			Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
 
 			MySubmittedJobGraphStore mySubmittedJobGraphStore = new MySubmittedJobGraphStore();
-			MyCheckpointStore checkpointStore = new MyCheckpointStore();
+			CompletedCheckpointStore checkpointStore = new RecoverableCompletedCheckpointStore();
 			CheckpointIDCounter checkpointCounter = new StandaloneCheckpointIDCounter();
 			CheckpointRecoveryFactory checkpointStateFactory = new MyCheckpointRecoveryFactory(checkpointStore, checkpointCounter);
 			TestingLeaderElectionService myLeaderElectionService = new TestingLeaderElectionService();
@@ -438,67 +440,6 @@ public void apply(Object o) throws Exception {
 		}
 	}
 
-	/**
-	 * A checkpoint store, which supports shutdown and suspend. You can use this to test HA
-	 * as long as the factory always returns the same store instance.
-	 */
-	static class MyCheckpointStore implements CompletedCheckpointStore {
-
-		private final ArrayDeque<CompletedCheckpoint> checkpoints = new ArrayDeque<>(2);
-
-		private final ArrayDeque<CompletedCheckpoint> suspended = new ArrayDeque<>(2);
-
-		@Override
-		public void recover() throws Exception {
-			checkpoints.addAll(suspended);
-			suspended.clear();
-		}
-
-		@Override
-		public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception {
-			checkpoints.addLast(checkpoint);
-			if (checkpoints.size() > 1) {
-				checkpoints.removeFirst().subsume();
-			}
-		}
-
-		@Override
-		public CompletedCheckpoint getLatestCheckpoint() throws Exception {
-			return checkpoints.isEmpty() ? null : checkpoints.getLast();
-		}
-
-		@Override
-		public void shutdown(JobStatus jobStatus) throws Exception {
-			if (jobStatus.isGloballyTerminalState()) {
-				checkpoints.clear();
-				suspended.clear();
-			} else {
-				suspended.addAll(checkpoints);
-				checkpoints.clear();
-			}
-		}
-
-		@Override
-		public List<CompletedCheckpoint> getAllCheckpoints() throws Exception {
-			return new ArrayList<>(checkpoints);
-		}
-
-		@Override
-		public int getNumberOfRetainedCheckpoints() {
-			return checkpoints.size();
-		}
-
-		@Override
-		public int getMaxNumberOfRetainedCheckpoints() {
-			return 1;
-		}
-
-		@Override
-		public boolean requiresExternalizedCheckpoints() {
-			return false;
-		}
-	}
-
 	static class MyCheckpointRecoveryFactory implements CheckpointRecoveryFactory {
 
 		private final CompletedCheckpointStore store;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
new file mode 100644
index 0000000000000..cb14ff04d10ec
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.runtime.state;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class SharedStateRegistryTest {
+
+	/**
+	 * Validate that all states can be correctly registered at the registry.
+	 */
+	@Test
+	public void testRegistryNormal() {
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+
+		// register one state
+		TestSharedState firstState = new TestSharedState("first");
+		sharedStateRegistry.register(firstState, true);
+		assertEquals(1, sharedStateRegistry.getReferenceCount(firstState));
+
+		// register another state
+		TestSharedState secondState = new TestSharedState("second");
+		sharedStateRegistry.register(secondState, true);
+		assertEquals(1, sharedStateRegistry.getReferenceCount(secondState));
+
+		// register the first state again
+		sharedStateRegistry.register(firstState, false);
+		assertEquals(2, sharedStateRegistry.getReferenceCount(firstState));
+
+		// unregister the second state
+		sharedStateRegistry.unregister(secondState);
+		assertEquals(0, sharedStateRegistry.getReferenceCount(secondState));
+
+		// unregister the first state
+		sharedStateRegistry.unregister(firstState);
+		assertEquals(1, sharedStateRegistry.getReferenceCount(firstState));
+	}
+
+	/**
+	 * Validate that registering a handle referencing uncreated state will throw exception
+	 */
+	@Test(expected = IllegalStateException.class)
+	public void testRegisterWithUncreatedReference() {
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+
+		// register one state
+		TestSharedState state = new TestSharedState("state");
+		sharedStateRegistry.register(state, false);
+	}
+
+	/**
+	 * Validate that registering duplicate creation of the same state will throw exception
+	 */
+	@Test(expected = IllegalStateException.class)
+	public void testRegisterWithDuplicateState() {
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+
+		// register one state
+		TestSharedState state = new TestSharedState("state");
+		sharedStateRegistry.register(state, true);
+		sharedStateRegistry.register(state, true);
+	}
+
+	/**
+	 * Validate that unregister an unexisted key will throw exception
+	 */
+	@Test(expected = IllegalStateException.class)
+	public void testUnregisterWithUnexistedKey() {
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+
+		sharedStateRegistry.unregister(new TestSharedState("unexisted"));
+	}
+
+	private static class TestSharedState implements SharedStateHandle {
+		private static final long serialVersionUID = 4468635881465159780L;
+
+		private String key;
+
+		TestSharedState(String key) {
+			this.key = key;
+		}
+
+		@Override
+		public String getKey() {
+			return key;
+		}
+
+		@Override
+		public void discardState() throws Exception {
+			// nothing to do
+		}
+
+		@Override
+		public long getStateSize() {
+			return key.length();
+		}
+
+		@Override
+		public boolean equals(Object o) {
+			if (this == o) {
+				return true;
+			}
+			if (o == null || getClass() != o.getClass()) {
+				return false;
+			}
+
+			TestSharedState testState = (TestSharedState) o;
+
+			return key.equals(testState.key);
+		}
+
+		@Override
+		public int hashCode() {
+			return key.hashCode();
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
new file mode 100644
index 0000000000000..75b0f6fb7343d
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testutils;
+
+import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
+import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.state.SharedStateRegistry;
+import org.apache.flink.runtime.state.StateObject;
+import org.apache.flink.runtime.state.StateUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A checkpoint store, which supports shutdown and suspend. You can use this to test HA
+ * as long as the factory always returns the same store instance.
+ */
+public class RecoverableCompletedCheckpointStore implements CompletedCheckpointStore {
+
+	private static final Logger LOG = LoggerFactory.getLogger(RecoverableCompletedCheckpointStore.class);
+
+	private final ArrayDeque<CompletedCheckpoint> checkpoints = new ArrayDeque<>(2);
+
+	private final ArrayDeque<CompletedCheckpoint> suspended = new ArrayDeque<>(2);
+
+	@Override
+	public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
+		checkpoints.addAll(suspended);
+		suspended.clear();
+
+		for (CompletedCheckpoint checkpoint : checkpoints) {
+			checkpoint.registerSharedStates(sharedStateRegistry);
+		}
+	}
+
+	@Override
+	public void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception {
+		checkpoints.addLast(checkpoint);
+
+		checkpoint.registerSharedStates(sharedStateRegistry);
+
+		if (checkpoints.size() > 1) {
+			CompletedCheckpoint checkpointToSubsume = checkpoints.removeFirst();
+			checkpointToSubsume.discardOnSubsume(sharedStateRegistry);
+		}
+	}
+
+	@Override
+	public CompletedCheckpoint getLatestCheckpoint() throws Exception {
+		return checkpoints.isEmpty() ? null : checkpoints.getLast();
+	}
+
+	@Override
+	public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
+		if (jobStatus.isGloballyTerminalState()) {
+			checkpoints.clear();
+			suspended.clear();
+		} else {
+			suspended.clear();
+
+			for (CompletedCheckpoint checkpoint : checkpoints) {
+				sharedStateRegistry.unregisterAll(checkpoint.getTaskStates().values());
+				suspended.add(checkpoint);
+			}
+
+			checkpoints.clear();
+		}
+	}
+
+	@Override
+	public List<CompletedCheckpoint> getAllCheckpoints() throws Exception {
+		return new ArrayList<>(checkpoints);
+	}
+
+	@Override
+	public int getNumberOfRetainedCheckpoints() {
+		return checkpoints.size();
+	}
+
+	@Override
+	public int getMaxNumberOfRetainedCheckpoints() {
+		return 1;
+	}
+
+	@Override
+	public boolean requiresExternalizedCheckpoints() {
+		return false;
+	}
+}

From e0900b524fdfccba4673ddbbfb9399ee107167e2 Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Sat, 22 Apr 2017 01:23:09 +0200
Subject: [PATCH 038/253] [FLINK-6014] [checkpoint] Additional review changes

---
 .../AbstractCompletedCheckpointStore.java     |  37 ++++
 .../checkpoint/CheckpointCoordinator.java     |  41 ++---
 .../checkpoint/CompletedCheckpoint.java       | 169 ++++++++++--------
 .../checkpoint/CompletedCheckpointStore.java  |  10 +-
 .../runtime/checkpoint/PendingCheckpoint.java |  12 +-
 .../StandaloneCompletedCheckpointStore.java   |   9 +-
 .../runtime/checkpoint/SubtaskState.java      |   5 -
 .../flink/runtime/checkpoint/TaskState.java   |   7 -
 .../ZooKeeperCompletedCheckpointStore.java    |   8 +-
 .../runtime/state/CompositeStateHandle.java   |  22 +--
 .../runtime/state/SharedStateHandle.java      |   2 +-
 .../runtime/state/SharedStateRegistry.java    |  84 +++++----
 .../flink/runtime/state/StateObject.java      |   6 +-
 .../CheckpointCoordinatorFailureTest.java     |  10 +-
 .../checkpoint/CheckpointCoordinatorTest.java |  23 +--
 .../CheckpointStateRestoreTest.java           |   5 +-
 .../CompletedCheckpointStoreTest.java         |  28 ++-
 ...ecutionGraphCheckpointCoordinatorTest.java |   7 +-
 .../checkpoint/PendingCheckpointTest.java     |   4 -
 ...tandaloneCompletedCheckpointStoreTest.java |  26 ++-
 ...oKeeperCompletedCheckpointStoreITCase.java |  41 ++---
 ...ZooKeeperCompletedCheckpointStoreTest.java |  10 +-
 .../state/SharedStateRegistryTest.java        |  42 +----
 .../RecoverableCompletedCheckpointStore.java  |  13 +-
 24 files changed, 292 insertions(+), 329 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java
new file mode 100644
index 0000000000000..f42fd06f3bfed
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.runtime.state.SharedStateRegistry;
+
+/**
+ * This is the base class that provides implementation of some aspects common for all
+ * {@link CompletedCheckpointStore}s.
+ */
+public abstract class AbstractCompletedCheckpointStore implements CompletedCheckpointStore {
+
+	/**
+	 * Registry for shared states.
+	 */
+	protected final SharedStateRegistry sharedStateRegistry;
+
+	public AbstractCompletedCheckpointStore() {
+		this.sharedStateRegistry = new SharedStateRegistry();
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 5309dd4868245..256321eb854b1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -37,9 +37,7 @@
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
-import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.TaskStateHandles;
-
 import org.apache.flink.runtime.taskmanager.DispatcherThreadFactory;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
@@ -110,9 +108,6 @@ public class CheckpointCoordinator {
 	/** Completed checkpoints. Implementations can be blocking. Make sure calls to methods
 	 * accessing this don't block the job manager actor and run asynchronously. */
 	private final CompletedCheckpointStore completedCheckpointStore;
-	
-	/** Registry for shared states */
-	private final SharedStateRegistry sharedStateRegistry;
 
 	/** Default directory for persistent checkpoints; <code>null</code> if none configured.
 	 * THIS WILL BE REPLACED BY PROPER STATE-BACKEND METADATA WRITING */
@@ -223,7 +218,6 @@ public CheckpointCoordinator(
 		this.completedCheckpointStore = checkNotNull(completedCheckpointStore);
 		this.checkpointDirectory = checkpointDirectory;
 		this.executor = checkNotNull(executor);
-		this.sharedStateRegistry = new SharedStateRegistry();
 
 		this.recentPendingCheckpoints = new ArrayDeque<>(NUM_GHOST_CHECKPOINT_IDS);
 
@@ -288,7 +282,7 @@ public void shutdown(JobStatus jobStatus) throws Exception {
 				}
 				pendingCheckpoints.clear();
 
-				completedCheckpointStore.shutdown(jobStatus, sharedStateRegistry);
+				completedCheckpointStore.shutdown(jobStatus);
 				checkpointIdCounter.shutdown(jobStatus);
 			}
 		}
@@ -732,7 +726,7 @@ public boolean receiveAcknowledgeMessage(AcknowledgeCheckpoint message) throws C
 								"the state handle to avoid lingering state.", message.getCheckpointId(),
 							message.getTaskExecutionId(), message.getJob());
 
-						discardState(message.getJob(), message.getTaskExecutionId(), message.getCheckpointId(), message.getSubtaskState());
+						discardSubtaskState(message.getJob(), message.getTaskExecutionId(), message.getCheckpointId(), message.getSubtaskState());
 
 						break;
 					case DISCARDED:
@@ -741,7 +735,7 @@ public boolean receiveAcknowledgeMessage(AcknowledgeCheckpoint message) throws C
 								"state handle tp avoid lingering state.",
 							message.getCheckpointId(), message.getTaskExecutionId(), message.getJob());
 
-						discardState(message.getJob(), message.getTaskExecutionId(), message.getCheckpointId(), message.getSubtaskState());
+						discardSubtaskState(message.getJob(), message.getTaskExecutionId(), message.getCheckpointId(), message.getSubtaskState());
 				}
 
 				return true;
@@ -767,7 +761,7 @@ else if (checkpoint != null) {
 				}
 
 				// try to discard the state so that we don't have lingering state lying around
-				discardState(message.getJob(), message.getTaskExecutionId(), message.getCheckpointId(), message.getSubtaskState());
+				discardSubtaskState(message.getJob(), message.getTaskExecutionId(), message.getCheckpointId(), message.getSubtaskState());
 
 				return wasPendingCheckpoint;
 			}
@@ -805,16 +799,16 @@ private void completePendingCheckpoint(PendingCheckpoint pendingCheckpoint) thro
 	
 			// the pending checkpoint must be discarded after the finalization
 			Preconditions.checkState(pendingCheckpoint.isDiscarded() && completedCheckpoint != null);
-	
+
 			try {
-				completedCheckpointStore.addCheckpoint(completedCheckpoint, sharedStateRegistry);
+				completedCheckpointStore.addCheckpoint(completedCheckpoint);
 			} catch (Exception exception) {
 				// we failed to store the completed checkpoint. Let's clean up
 				executor.execute(new Runnable() {
 					@Override
 					public void run() {
 						try {
-							completedCheckpoint.discardOnFail();
+							completedCheckpoint.discardOnFailedStoring();
 						} catch (Throwable t) {
 							LOG.warn("Could not properly discard completed checkpoint {}.", completedCheckpoint.getCheckpointID(), t);
 						}
@@ -953,7 +947,7 @@ public boolean restoreLatestCheckpointedState(
 			}
 
 			// Recover the checkpoints
-			completedCheckpointStore.recover(sharedStateRegistry);
+			completedCheckpointStore.recover();
 
 			// restore from the latest checkpoint
 			CompletedCheckpoint latest = completedCheckpointStore.getLatestCheckpoint();
@@ -1017,7 +1011,7 @@ public boolean restoreSavepoint(
 		CompletedCheckpoint savepoint = SavepointLoader.loadAndValidateSavepoint(
 				job, tasks, savepointPath, userClassLoader, allowNonRestored);
 
-		completedCheckpointStore.addCheckpoint(savepoint, sharedStateRegistry);
+		completedCheckpointStore.addCheckpoint(savepoint);
 		
 		// Reset the checkpoint ID counter
 		long nextCheckpointId = savepoint.getCheckpointID() + 1;
@@ -1057,10 +1051,11 @@ public List<CompletedCheckpoint> getSuccessfulCheckpoints() throws Exception {
 	public CompletedCheckpointStore getCheckpointStore() {
 		return completedCheckpointStore;
 	}
-	
-	public SharedStateRegistry getSharedStateRegistry() {
-		return sharedStateRegistry;
-	}
+
+//	@VisibleForTesting
+//	SharedStateRegistry getSharedStateRegistry() {
+//		return sharedStateRegistry;
+//	}
 
 	public CheckpointIDCounter getCheckpointIdCounter() {
 		return checkpointIdCounter;
@@ -1151,7 +1146,7 @@ public void run() {
 	 * @param checkpointId of the state object
 	 * @param subtaskState to discard asynchronously
 	 */
-	private void discardState(
+	private void discardSubtaskState(
 			final JobID jobId,
 			final ExecutionAttemptID executionAttemptID,
 			final long checkpointId,
@@ -1161,12 +1156,6 @@ private void discardState(
 			executor.execute(new Runnable() {
 				@Override
 				public void run() {
-					try {
-						subtaskState.discardSharedStatesOnFail();
-					} catch (Throwable t1) {
-						LOG.warn("Could not properly discard shared states of checkpoint {} " +
-							"belonging to task {} of job {}.", checkpointId, executionAttemptID, jobId, t1);
-					}
 
 					try {
 						subtaskState.discardState();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
index 58e91e1e0c991..79fc31f862344 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
@@ -20,14 +20,12 @@
 
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.checkpoint.CompletedCheckpointStats.DiscardCallback;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.util.ExceptionUtils;
-
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -107,7 +105,7 @@ public class CompletedCheckpoint implements Serializable {
 
 	/** Optional stats tracker callback for discard. */
 	@Nullable
-	private transient volatile DiscardCallback discardCallback;
+	private transient volatile CompletedCheckpointStats.DiscardCallback discardCallback;
 
 	// ------------------------------------------------------------------------
 
@@ -151,7 +149,7 @@ public CompletedCheckpoint(
 		checkArgument((externalPointer == null) == (externalizedMetadata == null),
 				"external pointer without externalized metadata must be both null or both non-null");
 
-		checkArgument(!props.externalizeCheckpoint() || externalPointer != null, 
+		checkArgument(!props.externalizeCheckpoint() || externalPointer != null,
 			"Checkpoint properties require externalized checkpoint, but checkpoint is not externalized");
 
 		this.job = checkNotNull(job);
@@ -186,15 +184,14 @@ public CheckpointProperties getProperties() {
 		return props;
 	}
 
-	public void discardOnFail() throws Exception {
-		discard(null, true);
+	public void discardOnFailedStoring() throws Exception {
+		new UnstoredDiscardStategy().discard();
 	}
 
 	public boolean discardOnSubsume(SharedStateRegistry sharedStateRegistry) throws Exception {
-		Preconditions.checkNotNull(sharedStateRegistry, "The registry cannot be null.");
 
 		if (props.discardOnSubsumed()) {
-			discard(sharedStateRegistry, false);
+			new StoredDiscardStrategy(sharedStateRegistry).discard();
 			return true;
 		}
 
@@ -202,14 +199,13 @@ public boolean discardOnSubsume(SharedStateRegistry sharedStateRegistry) throws
 	}
 
 	public boolean discardOnShutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
-		Preconditions.checkNotNull(sharedStateRegistry, "The registry cannot be null.");
 
 		if (jobStatus == JobStatus.FINISHED && props.discardOnJobFinished() ||
 				jobStatus == JobStatus.CANCELED && props.discardOnJobCancelled() ||
 				jobStatus == JobStatus.FAILED && props.discardOnJobFailed() ||
 				jobStatus == JobStatus.SUSPENDED && props.discardOnJobSuspended()) {
 
-			discard(sharedStateRegistry, false);
+			new StoredDiscardStrategy(sharedStateRegistry).discard();
 			return true;
 		} else {
 			if (externalPointer != null) {
@@ -221,53 +217,6 @@ public boolean discardOnShutdown(JobStatus jobStatus, SharedStateRegistry shared
 		}
 	}
 
-	private void discard(SharedStateRegistry sharedStateRegistry, boolean failed) throws Exception {
-		Preconditions.checkState(failed || (sharedStateRegistry != null),
-			"The registry must not be null if the complete checkpoint does not fail.");
-
-		try {
-			// collect exceptions and continue cleanup
-			Exception exception = null;
-
-			// drop the metadata, if we have some
-			if (externalizedMetadata != null) {
-				try {
-					externalizedMetadata.discardState();
-				} catch (Exception e) {
-					exception = e;
-				}
-			}
-
-			// In the cases where the completed checkpoint fails, the shared
-			// states have not been registered to the registry. It's the state
-			// handles' responsibility to discard their shared states.
-			if (!failed) {
-				unregisterSharedStates(sharedStateRegistry);
-			} else {
-				discardSharedStatesOnFail();
-			}
-
-			// discard private state objects
-			try {
-				StateUtil.bestEffortDiscardAllStateObjects(taskStates.values());
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
-			}
-
-			if (exception != null) {
-				throw exception;
-			}
-		} finally {
-			taskStates.clear();
-
-			// to be null-pointer safe, copy reference to stack
-			DiscardCallback discardCallback = this.discardCallback;
-			if (discardCallback != null) {
-				discardCallback.notifyDiscardedCheckpoint();
-			}
-		}
-	}
-
 	public long getStateSize() {
 		long result = 0L;
 
@@ -319,30 +268,108 @@ public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
 		sharedStateRegistry.registerAll(taskStates.values());
 	}
 
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public String toString() {
+		return String.format("Checkpoint %d @ %d for %s", checkpointID, timestamp, job);
+	}
+
 	/**
-	 * Unregister all shared states from the given registry. This is method is
-	 * called when the completed checkpoint is subsumed or the job terminates.
-	 *
-	 * @param sharedStateRegistry The registry where shared states are registered
+	 * Base class for the discarding strategies of {@link CompletedCheckpoint}.
 	 */
-	private void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
-		sharedStateRegistry.unregisterAll(taskStates.values());
+	private abstract class DiscardStrategy {
+
+		protected Exception storedException;
+
+		public DiscardStrategy() {
+			this.storedException = null;
+		}
+
+		public void discard() throws Exception {
+
+			try {
+				// collect exceptions and continue cleanup
+				storedException = null;
+
+				doDiscardExternalizedMetaData();
+				doDiscardSharedState();
+				doDiscardPrivateState();
+				doReportStoredExceptions();
+			} finally {
+				clearTaskStatesAndNotifyDiscardCompleted();
+			}
+		}
+
+		protected void doDiscardExternalizedMetaData() {
+			// drop the metadata, if we have some
+			if (externalizedMetadata != null) {
+				try {
+					externalizedMetadata.discardState();
+				} catch (Exception e) {
+					storedException = e;
+				}
+			}
+		}
+
+		protected void doDiscardPrivateState() {
+			// discard private state objects
+			try {
+				StateUtil.bestEffortDiscardAllStateObjects(taskStates.values());
+			} catch (Exception e) {
+				storedException = ExceptionUtils.firstOrSuppressed(e, storedException);
+			}
+		}
+
+		protected abstract void doDiscardSharedState();
+
+		protected void doReportStoredExceptions() throws Exception {
+			if (storedException != null) {
+				throw storedException;
+			}
+		}
+
+		protected void clearTaskStatesAndNotifyDiscardCompleted() {
+			taskStates.clear();
+			// to be null-pointer safe, copy reference to stack
+			CompletedCheckpointStats.DiscardCallback discardCallback =
+				CompletedCheckpoint.this.discardCallback;
+
+			if (discardCallback != null) {
+				discardCallback.notifyDiscardedCheckpoint();
+			}
+		}
 	}
 
 	/**
-	 * Discard all shared states created in the checkpoint. This method is called
+	 * Discard all shared states created in the checkpoint. This strategy is applied
 	 * when the completed checkpoint fails to be added into the store.
 	 */
-	private void discardSharedStatesOnFail() throws Exception {
-		for (TaskState taskState : taskStates.values()) {
-			taskState.discardSharedStatesOnFail();
+	private class UnstoredDiscardStategy extends CompletedCheckpoint.DiscardStrategy {
+
+		@Override
+		protected void doDiscardSharedState() {
+			// nothing to do because we did not register any shared state yet. unregistered, new
+			// shared state is then still considered private state and deleted as part of
+			// doDiscardPrivateState().
 		}
 	}
 
-	// --------------------------------------------------------------------------------------------
+	/**
+	 * Unregister all shared states from the given registry. This is strategy is
+	 * applied when the completed checkpoint is subsumed or the job terminates.
+	 */
+	private class StoredDiscardStrategy extends CompletedCheckpoint.DiscardStrategy {
 
-	@Override
-	public String toString() {
-		return String.format("Checkpoint %d @ %d for %s", checkpointID, timestamp, job);
+		SharedStateRegistry sharedStateRegistry;
+
+		public StoredDiscardStrategy(SharedStateRegistry sharedStateRegistry) {
+			this.sharedStateRegistry = Preconditions.checkNotNull(sharedStateRegistry);
+		}
+
+		@Override
+		protected void doDiscardSharedState() {
+			sharedStateRegistry.unregisterAll(taskStates.values());
+		}
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
index 0ade25c849328..82193b5f08da9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
@@ -19,7 +19,6 @@
 package org.apache.flink.runtime.checkpoint;
 
 import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.state.SharedStateRegistry;
 
 import java.util.List;
 
@@ -34,16 +33,15 @@ public interface CompletedCheckpointStore {
 	 * <p>After a call to this method, {@link #getLatestCheckpoint()} returns the latest
 	 * available checkpoint.
 	 */
-	void recover(SharedStateRegistry sharedStateRegistry) throws Exception;
+	void recover() throws Exception;
 
 	/**
 	 * Adds a {@link CompletedCheckpoint} instance to the list of completed checkpoints.
 	 *
 	 * <p>Only a bounded number of checkpoints is kept. When exceeding the maximum number of
-	 * retained checkpoints, the oldest one will be discarded via {@link
-	 * CompletedCheckpoint#discardOnSubsume(SharedStateRegistry)} )}.
+	 * retained checkpoints, the oldest one will be discarded.
 	 */
-	void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception;
+	void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception;
 
 	/**
 	 * Returns the latest {@link CompletedCheckpoint} instance or <code>null</code> if none was
@@ -59,7 +57,7 @@ public interface CompletedCheckpointStore {
 	 *
 	 * @param jobStatus Job state on shut down
 	 */
-	void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception;
+	void shutdown(JobStatus jobStatus) throws Exception;
 
 	/**
 	 * Returns all {@link CompletedCheckpoint} instances.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
index 6805deaa7cbb7..900331bb6c5eb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
@@ -506,16 +506,8 @@ private void dispose(boolean releaseState) {
 						@Override
 						public void run() {
 
-							// discard the shared states that are created in the checkpoint
-							for (TaskState taskState : taskStates.values()) {
-								try {
-									taskState.discardSharedStatesOnFail();
-								} catch (Throwable t) {
-									LOG.warn("Could not properly dispose unreferenced shared states.");
-								}
-							}
-
-							// discard the private states
+							// discard the private states.
+							// unregistered shared states are still considered private at this point.
 							try {
 								StateUtil.bestEffortDiscardAllStateObjects(taskStates.values());
 							} catch (Throwable t) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java
index 9f833c31107f6..f5e1db351fd8c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStore.java
@@ -20,7 +20,6 @@
 
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
-import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,7 +32,7 @@
 /**
  * {@link CompletedCheckpointStore} for JobManagers running in {@link HighAvailabilityMode#NONE}.
  */
-public class StandaloneCompletedCheckpointStore implements CompletedCheckpointStore {
+public class StandaloneCompletedCheckpointStore extends AbstractCompletedCheckpointStore {
 
 	private static final Logger LOG = LoggerFactory.getLogger(StandaloneCompletedCheckpointStore.class);
 
@@ -57,12 +56,12 @@ public StandaloneCompletedCheckpointStore(int maxNumberOfCheckpointsToRetain) {
 	}
 
 	@Override
-	public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void recover() throws Exception {
 		// Nothing to do
 	}
 
 	@Override
-	public void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception {
 		
 		checkpoints.addLast(checkpoint);
 
@@ -99,7 +98,7 @@ public int getMaxNumberOfRetainedCheckpoints() {
 	}
 
 	@Override
-	public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void shutdown(JobStatus jobStatus) throws Exception {
 		try {
 			LOG.info("Shutting down");
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
index e968643602a8c..121ac57f1efbd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
@@ -160,11 +160,6 @@ public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
 		// No shared states
 	}
 
-	@Override
-	public void discardSharedStatesOnFail() {
-		// No shared states
-	}
-
 	@Override
 	public long getStateSize() {
 		return stateSize;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
index 19fe9624746be..4f5f536bcb065 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
@@ -143,13 +143,6 @@ public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
 		}
 	}
 
-	@Override
-	public void discardSharedStatesOnFail() {
-		for (SubtaskState subtaskState : subtaskStates.values()) {
-			subtaskState.discardSharedStatesOnFail();
-		}
-	}
-
 	@Override
 	public long getStateSize() {
 		long result = 0L;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
index 07546ea4227ff..52a4eeaeeaa6a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
@@ -68,7 +68,7 @@
  * checkpoints is consistent. Currently, after recovery we start out with only a single
  * checkpoint to circumvent those situations.
  */
-public class ZooKeeperCompletedCheckpointStore implements CompletedCheckpointStore {
+public class ZooKeeperCompletedCheckpointStore extends AbstractCompletedCheckpointStore {
 
 	private static final Logger LOG = LoggerFactory.getLogger(ZooKeeperCompletedCheckpointStore.class);
 
@@ -141,7 +141,7 @@ public boolean requiresExternalizedCheckpoints() {
 	 * that the history of checkpoints is consistent.
 	 */
 	@Override
-	public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void recover() throws Exception {
 		LOG.info("Recovering checkpoints from ZooKeeper.");
 
 		// Clear local handles in order to prevent duplicates on
@@ -192,7 +192,7 @@ public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
 	 * @param checkpoint Completed checkpoint to add.
 	 */
 	@Override
-	public void addCheckpoint(final CompletedCheckpoint checkpoint, final SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void addCheckpoint(final CompletedCheckpoint checkpoint) throws Exception {
 		checkNotNull(checkpoint, "Checkpoint");
 		
 		final String path = checkpointIdToPath(checkpoint.getCheckpointID());
@@ -281,7 +281,7 @@ public int getMaxNumberOfRetainedCheckpoints() {
 	}
 
 	@Override
-	public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void shutdown(JobStatus jobStatus) throws Exception {
 		if (jobStatus.isGloballyTerminalState()) {
 			LOG.info("Shutting down");
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java
index 2ea5bc9080e12..002b7c315b396 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CompositeStateHandle.java
@@ -28,16 +28,24 @@
  * received by the {@link org.apache.flink.runtime.checkpoint.CheckpointCoordinator}
  * and will be discarded when the checkpoint is discarded.
  * 
- * <p>The {@link SharedStateRegistry} is responsible for the discarding of the
- * shared states. The composite state handle should only delete those private
- * states in the {@link StateObject#discardState()} method.
+ * <p>The {@link SharedStateRegistry} is responsible for the discarding of registered
+ * shared states. Before their first registration through
+ * {@link #registerSharedStates(SharedStateRegistry)}, newly created shared state is still owned by
+ * this handle and considered as private state until it is registered for the first time. Registration
+ * transfers ownership to the {@link SharedStateRegistry}.
+ * The composite state handle should only delete all private states in the
+ * {@link StateObject#discardState()} method.
  */
 public interface CompositeStateHandle extends StateObject {
 
 	/**
-	 * Register both created and referenced shared states in the given
+	 * Register both newly created and already referenced shared states in the given
 	 * {@link SharedStateRegistry}. This method is called when the checkpoint
 	 * successfully completes or is recovered from failures.
+	 * <p>
+	 * After this is completed, newly created shared state is considered as published is no longer
+	 * owned by this handle. This means that it should no longer be deleted as part of calls to
+	 * {@link #discardState()}.
 	 *
 	 * @param stateRegistry The registry where shared states are registered.
 	 */
@@ -51,10 +59,4 @@ public interface CompositeStateHandle extends StateObject {
 	 * @param stateRegistry The registry where shared states are registered.
 	 */
 	void unregisterSharedStates(SharedStateRegistry stateRegistry);
-
-	/**
-	 * Discard all shared states created in this checkpoint. This method is
-	 * called when the checkpoint fails to complete.
-	 */
-	void discardSharedStatesOnFail() throws Exception;
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
index f856052904952..c8c404672b23f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
@@ -35,5 +35,5 @@ public interface SharedStateHandle extends StateObject {
 	/**
 	 * Return the identifier of the shared state.
 	 */
-	String getKey();
+	String getRegistrationKey();
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
index b5048d0d96208..2cb43accfa40b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
@@ -18,13 +18,10 @@
 
 package org.apache.flink.runtime.state;
 
-import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.Serializable;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -33,73 +30,78 @@
  * {@link org.apache.flink.runtime.checkpoint.CheckpointCoordinator} to 
  * maintain the reference count of {@link SharedStateHandle}s which are shared
  * among different checkpoints.
+ *
  */
-public class SharedStateRegistry implements Serializable {
+public class SharedStateRegistry {
 
-	private static Logger LOG = LoggerFactory.getLogger(SharedStateRegistry.class);
+	private static final Logger LOG = LoggerFactory.getLogger(SharedStateRegistry.class);
 
-	private static final long serialVersionUID = -8357254413007773970L;
+	/** All registered state objects by an artificial key */
+	private final Map<String, SharedStateRegistry.SharedStateEntry> registeredStates;
 
-	/** All registered state objects */
-	private final Map<String, SharedStateEntry> registeredStates = new HashMap<>();
+	public SharedStateRegistry() {
+		this.registeredStates = new HashMap<>();
+	}
 
 	/**
-	 * Register the state in the registry
+	 * Register a reference to the given shared state in the registry. This increases the reference
+	 * count for the this shared state by one. Returns the reference count after the update.
 	 *
-	 * @param state The state to register
-	 * @param isNew True if the shared state is newly created
+	 * @param state the shared state for which we register a reference.
+	 * @return the updated reference count for the given shared state.
 	 */
-	public void register(SharedStateHandle state, boolean isNew) {
+	public int register(SharedStateHandle state) {
 		if (state == null) {
-			return;
+			return 0;
 		}
 
 		synchronized (registeredStates) {
-			SharedStateEntry entry = registeredStates.get(state.getKey());
-
-			if (isNew) {
-				Preconditions.checkState(entry == null,
-					"The state cannot be created more than once.");
+			SharedStateRegistry.SharedStateEntry entry =
+				registeredStates.get(state.getRegistrationKey());
 
-				registeredStates.put(state.getKey(), new SharedStateEntry(state));
+			if (entry == null) {
+				SharedStateRegistry.SharedStateEntry stateEntry =
+					new SharedStateRegistry.SharedStateEntry(state);
+				registeredStates.put(state.getRegistrationKey(), stateEntry);
+				return 1;
 			} else {
-				Preconditions.checkState(entry != null,
-					"The state cannot be referenced if it has not been created yet.");
-
 				entry.increaseReferenceCount();
+				return entry.getReferenceCount();
 			}
 		}
 	}
 
 	/**
-	 * Unregister the state in the registry
+	 * Unregister one reference to the given shared state in the registry. This decreases the
+	 * reference count by one. Once the count reaches zero, the shared state is deleted.
 	 *
-	 * @param state The state to unregister
+	 * @param state the shared state for which we unregister a reference.
+	 * @return the reference count for the shared state after the update.
 	 */
-	public void unregister(SharedStateHandle state) {
+	public int unregister(SharedStateHandle state) {
 		if (state == null) {
-			return;
+			return 0;
 		}
 
 		synchronized (registeredStates) {
-			SharedStateEntry entry = registeredStates.get(state.getKey());
+			SharedStateRegistry.SharedStateEntry entry = registeredStates.get(state.getRegistrationKey());
 
-			if (entry == null) {
-				throw new IllegalStateException("Cannot unregister an unexisted state.");
-			}
+			Preconditions.checkState(entry != null, "Cannot unregister a state that is not registered.");
 
 			entry.decreaseReferenceCount();
 
-			// Remove the state from the registry when it's not referenced any more.
-			if (entry.getReferenceCount() == 0) {
-				registeredStates.remove(state.getKey());
+			final int newReferenceCount = entry.getReferenceCount();
 
+			// Remove the state from the registry when it's not referenced any more.
+			if (newReferenceCount <= 0) {
+				registeredStates.remove(state.getRegistrationKey());
 				try {
 					entry.getState().discardState();
 				} catch (Exception e) {
-					LOG.warn("Cannot properly discard the state " + entry.getState() + ".", e);
+					LOG.warn("Cannot properly discard the state {}.", entry.getState(), e);
 				}
 			}
+			return newReferenceCount;
 		}
 	}
 
@@ -108,7 +110,7 @@ public void unregister(SharedStateHandle state) {
 	 *
 	 * @param stateHandles The shared states to register.
 	 */
-	public void registerAll(Collection<? extends CompositeStateHandle> stateHandles) {
+	public void registerAll(Iterable<? extends CompositeStateHandle> stateHandles) {
 		if (stateHandles == null) {
 			return;
 		}
@@ -127,7 +129,7 @@ public void registerAll(Collection<? extends CompositeStateHandle> stateHandles)
 	 *
 	 * @param stateHandles The shared states to unregister.
 	 */
-	public void unregisterAll(Collection<? extends CompositeStateHandle> stateHandles) {
+	public void unregisterAll(Iterable<? extends CompositeStateHandle> stateHandles) {
 		if (stateHandles == null) {
 			return;
 		}
@@ -140,6 +142,7 @@ public void unregisterAll(Collection<? extends CompositeStateHandle> stateHandle
 	}
 
 	private static class SharedStateEntry {
+
 		/** The shared object */
 		private final SharedStateHandle state;
 
@@ -168,10 +171,13 @@ void decreaseReferenceCount() {
 		}
 	}
 
-
-	@VisibleForTesting
 	public int getReferenceCount(SharedStateHandle state) {
-		SharedStateEntry entry = registeredStates.get(state.getKey());
+		if (state == null) {
+			return 0;
+		}
+
+		SharedStateRegistry.SharedStateEntry entry =
+			registeredStates.get(state.getRegistrationKey());
 
 		return entry == null ? 0 : entry.getReferenceCount();
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateObject.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateObject.java
index 7f1dd18e97b4f..3b49df71d5338 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateObject.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateObject.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.runtime.state;
 
+import java.io.Serializable;
+
 /**
  * Base of all handles that represent checkpointed state in some form. The object may hold
  * the (small) state directly, or contain a file path (state is in the file), or contain the
@@ -33,10 +35,10 @@
  * compatibility, they are not stored via {@link java.io.Serializable Java Serialization},
  * but through custom serializers.
  */
-public interface StateObject extends java.io.Serializable {
+public interface StateObject extends Serializable {
 
 	/**
-	 * Discards the state referred to by this handle, to free up resources in
+	 * Discards the state referred to and solemnly owned by this handle, to free up resources in
 	 * the persistent storage. This method is called when the state represented by this
 	 * object will not be used any more.
 	 */
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
index 632f2c0678a3d..90b7fe7113756 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
@@ -25,7 +25,6 @@
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
-import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -39,9 +38,7 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
 @RunWith(PowerMockRunner.class)
@@ -105,19 +102,18 @@ public void testFailingCompletedCheckpointStoreAdd() throws Exception {
 		assertTrue(pendingCheckpoint.isDiscarded());
 
 		// make sure that the subtask state has been discarded after we could not complete it.
-		verify(subtaskState, times(1)).discardSharedStatesOnFail();
 		verify(subtaskState).discardState();
 	}
 
 	private static final class FailingCompletedCheckpointStore implements CompletedCheckpointStore {
 
 		@Override
-		public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
+		public void recover() throws Exception {
 			throw new UnsupportedOperationException("Not implemented.");
 		}
 
 		@Override
-		public void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception {
+		public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception {
 			throw new Exception("The failing completed checkpoint store failed again... :-(");
 		}
 
@@ -127,7 +123,7 @@ public CompletedCheckpoint getLatestCheckpoint() throws Exception {
 		}
 
 		@Override
-		public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
+		public void shutdown(JobStatus jobStatus) throws Exception {
 			throw new UnsupportedOperationException("Not implemented.");
 		}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index fabf3fc33ff8c..24169f2a5b607 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -88,7 +88,6 @@
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.eq;
 import static org.mockito.Mockito.mock;
@@ -880,8 +879,6 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
 
 			// validate that all received subtask states in the first checkpoint have been discarded
-			verify(subtaskState1_1, times(1)).discardSharedStatesOnFail();
-			verify(subtaskState1_2, times(1)).discardSharedStatesOnFail();
 			verify(subtaskState1_1, times(1)).discardState();
 			verify(subtaskState1_2, times(1)).discardState();
 
@@ -907,7 +904,6 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			// send the last remaining ack for the first checkpoint. This should not do anything
 			SubtaskState subtaskState1_3 = mock(SubtaskState.class);
 			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId1, new CheckpointMetrics(), subtaskState1_3));
-			verify(subtaskState1_3, times(1)).discardSharedStatesOnFail();
 			verify(subtaskState1_3, times(1)).discardState();
 
 			coord.shutdown(JobStatus.FINISHED);
@@ -993,7 +989,6 @@ public void testCheckpointTimeoutIsolated() {
 			assertEquals(0, coord.getNumberOfRetainedSuccessfulCheckpoints());
 
 			// validate that the received states have been discarded
-			verify(subtaskState, times(1)).discardSharedStatesOnFail();
 			verify(subtaskState, times(1)).discardState();
 
 			// no confirm message must have been sent
@@ -1117,7 +1112,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, triggerAttemptId, checkpointId, new CheckpointMetrics(), triggerSubtaskState));
 
 		// verify that the subtask state has registered its shared states at the registry
-		verify(triggerSubtaskState, never()).discardSharedStatesOnFail();
 		verify(triggerSubtaskState, never()).discardState();
 
 		SubtaskState unknownSubtaskState = mock(SubtaskState.class);
@@ -1126,7 +1120,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), unknownSubtaskState));
 
 		// we should discard acknowledge messages from an unknown vertex belonging to our job
-		verify(unknownSubtaskState, times(1)).discardSharedStatesOnFail();
 		verify(unknownSubtaskState, times(1)).discardState();
 
 		SubtaskState differentJobSubtaskState = mock(SubtaskState.class);
@@ -1135,7 +1128,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), differentJobSubtaskState));
 
 		// we should not interfere with different jobs
-		verify(differentJobSubtaskState, never()).discardSharedStatesOnFail();
 		verify(differentJobSubtaskState, never()).discardState();
 
 		// duplicate acknowledge message for the trigger vertex
@@ -1143,7 +1135,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, triggerAttemptId, checkpointId, new CheckpointMetrics(), triggerSubtaskState));
 
 		// duplicate acknowledge messages for a known vertex should not trigger discarding the state
-		verify(triggerSubtaskState, never()).discardSharedStatesOnFail();
 		verify(triggerSubtaskState, never()).discardState();
 
 		// let the checkpoint fail at the first ack vertex
@@ -1153,7 +1144,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		assertTrue(pendingCheckpoint.isDiscarded());
 
 		// check that we've cleaned up the already acknowledged state
-		verify(triggerSubtaskState, times(1)).discardSharedStatesOnFail();
 		verify(triggerSubtaskState, times(1)).discardState();
 
 		SubtaskState ackSubtaskState = mock(SubtaskState.class);
@@ -1162,7 +1152,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, ackAttemptId2, checkpointId, new CheckpointMetrics(), ackSubtaskState));
 
 		// check that we also cleaned up this state
-		verify(ackSubtaskState, times(1)).discardSharedStatesOnFail();
 		verify(ackSubtaskState, times(1)).discardState();
 
 		// receive an acknowledge message from an unknown job
@@ -1170,7 +1159,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), differentJobSubtaskState));
 
 		// we should not interfere with different jobs
-		verify(differentJobSubtaskState, never()).discardSharedStatesOnFail();
 		verify(differentJobSubtaskState, never()).discardState();
 
 		SubtaskState unknownSubtaskState2 = mock(SubtaskState.class);
@@ -1179,7 +1167,6 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, new ExecutionAttemptID(), checkpointId, new CheckpointMetrics(), unknownSubtaskState2));
 
 		// we should discard acknowledge messages from an unknown vertex belonging to our job
-		verify(unknownSubtaskState2, times(1)).discardSharedStatesOnFail();
 		verify(unknownSubtaskState2, times(1)).discardState();
 	}
 
@@ -2013,14 +2000,13 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 		assertEquals(1, completedCheckpoints.size());
 
 		// shutdown the store
-		SharedStateRegistry sharedStateRegistry = coord.getSharedStateRegistry();
-		store.shutdown(JobStatus.SUSPENDED, sharedStateRegistry);
+		store.shutdown(JobStatus.SUSPENDED);
 
 		// All shared states should be unregistered once the store is shut down
 		for (CompletedCheckpoint completedCheckpoint : completedCheckpoints) {
 			for (TaskState taskState : completedCheckpoint.getTaskStates().values()) {
 				for (SubtaskState subtaskState : taskState.getStates()) {
-					verify(subtaskState, times(1)).unregisterSharedStates(sharedStateRegistry);
+					verify(subtaskState, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
 				}
 			}
 		}
@@ -2037,7 +2023,7 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 		for (CompletedCheckpoint completedCheckpoint : completedCheckpoints) {
 			for (TaskState taskState : completedCheckpoint.getTaskStates().values()) {
 				for (SubtaskState subtaskState : taskState.getStates()) {
-					verify(subtaskState, times(2)).registerSharedStates(sharedStateRegistry);
+					verify(subtaskState, times(2)).registerSharedStates(any(SharedStateRegistry.class));
 				}
 			}
 		}
@@ -3150,8 +3136,7 @@ public void testCheckpointStatsTrackerRestoreCallback() throws Exception {
 			Executors.directExecutor());
 
 		store.addCheckpoint(
-			new CompletedCheckpoint(new JobID(), 0, 0, 0, Collections.<JobVertexID, TaskState>emptyMap()),
-			coord.getSharedStateRegistry());
+			new CompletedCheckpoint(new JobID(), 0, 0, 0, Collections.<JobVertexID, TaskState>emptyMap()));
 
 		CheckpointStatsTracker tracker = mock(CheckpointStatsTracker.class);
 		coord.setCheckpointStatsTracker(tracker);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
index 9e372e161fbfd..2fc1de5cef3c7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
@@ -30,7 +30,6 @@
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 import org.apache.flink.runtime.state.ChainedStateHandle;
 import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
 import org.apache.flink.runtime.state.StreamStateHandle;
@@ -255,7 +254,7 @@ public void testNonRestoredState() throws Exception {
 		}
 		CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 1, 2, new HashMap<>(checkpointTaskStates));
 
-		coord.getCheckpointStore().addCheckpoint(checkpoint, coord.getSharedStateRegistry());
+		coord.getCheckpointStore().addCheckpoint(checkpoint);
 
 		coord.restoreLatestCheckpointedState(tasks, true, false);
 		coord.restoreLatestCheckpointedState(tasks, true, true);
@@ -273,7 +272,7 @@ public void testNonRestoredState() throws Exception {
 
 		checkpoint = new CompletedCheckpoint(new JobID(), 1, 2, 3, new HashMap<>(checkpointTaskStates));
 
-		coord.getCheckpointStore().addCheckpoint(checkpoint, coord.getSharedStateRegistry());
+		coord.getCheckpointStore().addCheckpoint(checkpoint);
 
 		// (i) Allow non restored state (should succeed)
 		coord.restoreLatestCheckpointedState(tasks, true, true);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
index aa1726bde5f45..4a36dd282c0a6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
@@ -37,6 +37,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -68,7 +69,6 @@ public void testExceptionOnNoRetainedCheckpoints() throws Exception {
 	@Test
 	public void testAddAndGetLatestCheckpoint() throws Exception {
 		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		
 		// Empty state
 		assertEquals(0, checkpoints.getNumberOfRetainedCheckpoints());
@@ -78,11 +78,11 @@ public void testAddAndGetLatestCheckpoint() throws Exception {
 				createCheckpoint(0), createCheckpoint(1) };
 
 		// Add and get latest
-		checkpoints.addCheckpoint(expected[0], sharedStateRegistry);
+		checkpoints.addCheckpoint(expected[0]);
 		assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
 		verifyCheckpoint(expected[0], checkpoints.getLatestCheckpoint());
 
-		checkpoints.addCheckpoint(expected[1], sharedStateRegistry);
+		checkpoints.addCheckpoint(expected[1]);
 		assertEquals(2, checkpoints.getNumberOfRetainedCheckpoints());
 		verifyCheckpoint(expected[1], checkpoints.getLatestCheckpoint());
 	}
@@ -93,8 +93,7 @@ public void testAddAndGetLatestCheckpoint() throws Exception {
 	 */
 	@Test
 	public void testAddCheckpointMoreThanMaxRetained() throws Exception {
-		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(1);   
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(1);
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1),
@@ -102,13 +101,13 @@ public void testAddCheckpointMoreThanMaxRetained() throws Exception {
 		};
 
 		// Add checkpoints
-		checkpoints.addCheckpoint(expected[0], sharedStateRegistry);
+		checkpoints.addCheckpoint(expected[0]);
 		assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
 
 		for (int i = 1; i < expected.length; i++) {
 			Collection<TaskState> taskStates = expected[i - 1].getTaskStates().values();
 
-			checkpoints.addCheckpoint(expected[i], sharedStateRegistry);
+			checkpoints.addCheckpoint(expected[i]);
 
 			// The ZooKeeper implementation discards asynchronously
 			expected[i - 1].awaitDiscard();
@@ -117,7 +116,7 @@ public void testAddCheckpointMoreThanMaxRetained() throws Exception {
 
 			for (TaskState taskState : taskStates) {
 				for (SubtaskState subtaskState : taskState.getStates()) {
-					verify(subtaskState, times(1)).unregisterSharedStates(sharedStateRegistry);
+					verify(subtaskState, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
 				}
 			}
 		}
@@ -146,7 +145,6 @@ public void testEmptyState() throws Exception {
 	@Test
 	public void testGetAllCheckpoints() throws Exception {
 		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1),
@@ -154,7 +152,7 @@ public void testGetAllCheckpoints() throws Exception {
 		};
 
 		for (TestCompletedCheckpoint checkpoint : expected) {
-			checkpoints.addCheckpoint(checkpoint, sharedStateRegistry);
+			checkpoints.addCheckpoint(checkpoint);
 		}
 
 		List<CompletedCheckpoint> actual = checkpoints.getAllCheckpoints();
@@ -172,7 +170,6 @@ public void testGetAllCheckpoints() throws Exception {
 	@Test
 	public void testDiscardAllCheckpoints() throws Exception {
 		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1),
@@ -180,10 +177,10 @@ public void testDiscardAllCheckpoints() throws Exception {
 		};
 
 		for (TestCompletedCheckpoint checkpoint : expected) {
-			checkpoints.addCheckpoint(checkpoint, sharedStateRegistry);
+			checkpoints.addCheckpoint(checkpoint);
 		}
 
-		checkpoints.shutdown(JobStatus.FINISHED, sharedStateRegistry);
+		checkpoints.shutdown(JobStatus.FINISHED);
 
 		// Empty state
 		assertNull(checkpoints.getLatestCheckpoint());
@@ -235,10 +232,10 @@ protected void resetCheckpoint(Collection<TaskState> taskStates) {
 		}
 	}
 
-	protected void verifyCheckpointRegistered(Collection<TaskState> taskStates, SharedStateRegistry sharedStateRegistry) {
+	protected void verifyCheckpointRegistered(Collection<TaskState> taskStates, SharedStateRegistry registry) {
 		for (TaskState taskState : taskStates) {
 			for (SubtaskState subtaskState : taskState.getStates()) {
-				verify(subtaskState, times(1)).registerSharedStates(eq(sharedStateRegistry));
+				verify(subtaskState, times(1)).registerSharedStates(eq(registry));
 			}
 		}
 	}
@@ -246,7 +243,6 @@ protected void verifyCheckpointRegistered(Collection<TaskState> taskStates, Shar
 	protected void verifyCheckpointDiscarded(Collection<TaskState> taskStates) {
 		for (TaskState taskState : taskStates) {
 			for (SubtaskState subtaskState : taskState.getStates()) {
-				verify(subtaskState, times(1)).discardSharedStatesOnFail();
 				verify(subtaskState, times(1)).discardState();
 			}
 		}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
index e7c1c3b1670b1..5fce62ebc2528 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
@@ -31,17 +31,14 @@
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
-import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.SerializedValue;
 
 import org.junit.Test;
-import org.mockito.Matchers;
 
 import java.net.URL;
 import java.util.Collections;
 
-import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -62,7 +59,7 @@ public void testShutdownCheckpointCoordinator() throws Exception {
 		graph.fail(new Exception("Test Exception"));
 
 		verify(counter, times(1)).shutdown(JobStatus.FAILED);
-		verify(store, times(1)).shutdown(eq(JobStatus.FAILED), any(SharedStateRegistry.class));
+		verify(store, times(1)).shutdown(eq(JobStatus.FAILED));
 	}
 
 	/**
@@ -79,7 +76,7 @@ public void testSuspendCheckpointCoordinator() throws Exception {
 
 		// No shutdown
 		verify(counter, times(1)).shutdown(eq(JobStatus.SUSPENDED));
-		verify(store, times(1)).shutdown(eq(JobStatus.SUSPENDED), any(SharedStateRegistry.class));
+		verify(store, times(1)).shutdown(eq(JobStatus.SUSPENDED));
 	}
 
 	private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
index d77fac1bed9dd..2dd1803cd9960 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
@@ -207,7 +207,6 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
-		verify(state, times(1)).discardSharedStatesOnFail();
 
 		// Abort error
 		Mockito.reset(state);
@@ -219,7 +218,6 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
-		verify(state, times(1)).discardSharedStatesOnFail();
 
 		// Abort expired
 		Mockito.reset(state);
@@ -231,7 +229,6 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
-		verify(state, times(1)).discardSharedStatesOnFail();
 
 		// Abort subsumed
 		Mockito.reset(state);
@@ -243,7 +240,6 @@ public void testAbortDiscardsState() throws Exception {
 		// execute asynchronous discard operation
 		executor.runQueuedCommands();
 		verify(state, times(1)).discardState();
-		verify(state, times(1)).discardSharedStatesOnFail();
 	}
 
 	/**
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
index 7a8589719819e..64aeebac41780 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
@@ -30,6 +30,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
 import static org.powermock.api.mockito.PowerMockito.doReturn;
 import static org.powermock.api.mockito.PowerMockito.doThrow;
 import static org.powermock.api.mockito.PowerMockito.mock;
@@ -40,7 +41,7 @@
 public class StandaloneCompletedCheckpointStoreTest extends CompletedCheckpointStoreTest {
 
 	@Override
-	protected CompletedCheckpointStore createCompletedCheckpoints(
+	protected AbstractCompletedCheckpointStore createCompletedCheckpoints(
 			int maxNumberOfCheckpointsToRetain) throws Exception {
 
 		return new StandaloneCompletedCheckpointStore(maxNumberOfCheckpointsToRetain);
@@ -51,16 +52,15 @@ protected CompletedCheckpointStore createCompletedCheckpoints(
 	 */
 	@Test
 	public void testShutdownDiscardsCheckpoints() throws Exception {
-		CompletedCheckpointStore store = createCompletedCheckpoints(1);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+		AbstractCompletedCheckpointStore store = createCompletedCheckpoints(1);
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
 		Collection<TaskState> taskStates = checkpoint.getTaskStates().values();
 
-		store.addCheckpoint(checkpoint, sharedStateRegistry);
+		store.addCheckpoint(checkpoint);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
-		verifyCheckpointRegistered(taskStates, sharedStateRegistry);
+		verifyCheckpointRegistered(taskStates, store.sharedStateRegistry);
 
-		store.shutdown(JobStatus.FINISHED, sharedStateRegistry);
+		store.shutdown(JobStatus.FINISHED);
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertTrue(checkpoint.isDiscarded());
 		verifyCheckpointDiscarded(taskStates);
@@ -72,16 +72,15 @@ public void testShutdownDiscardsCheckpoints() throws Exception {
 	 */
 	@Test
 	public void testSuspendDiscardsCheckpoints() throws Exception {
-		CompletedCheckpointStore store = createCompletedCheckpoints(1);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+		AbstractCompletedCheckpointStore store = createCompletedCheckpoints(1);
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
 		Collection<TaskState> taskStates = checkpoint.getTaskStates().values();
 
-		store.addCheckpoint(checkpoint, sharedStateRegistry);
+		store.addCheckpoint(checkpoint);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
-		verifyCheckpointRegistered(taskStates, sharedStateRegistry);
+		verifyCheckpointRegistered(taskStates, store.sharedStateRegistry);
 
-		store.shutdown(JobStatus.SUSPENDED, sharedStateRegistry);
+		store.shutdown(JobStatus.SUSPENDED);
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertTrue(checkpoint.isDiscarded());
 		verifyCheckpointDiscarded(taskStates);
@@ -96,16 +95,15 @@ public void testAddCheckpointWithFailedRemove() throws Exception {
 		
 		final int numCheckpointsToRetain = 1;
 		CompletedCheckpointStore store = createCompletedCheckpoints(numCheckpointsToRetain);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		
 		for (long i = 0; i <= numCheckpointsToRetain; ++i) {
 			CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class);
 			doReturn(i).when(checkpointToAdd).getCheckpointID();
 			doReturn(Collections.emptyMap()).when(checkpointToAdd).getTaskStates();
-			doThrow(new IOException()).when(checkpointToAdd).discardOnSubsume(sharedStateRegistry);
+			doThrow(new IOException()).when(checkpointToAdd).discardOnSubsume(any(SharedStateRegistry.class));
 			
 			try {
-				store.addCheckpoint(checkpointToAdd, sharedStateRegistry);
+				store.addCheckpoint(checkpointToAdd);
 				
 				// The checkpoint should be in the store if we successfully add it into the store.
 				List<CompletedCheckpoint> addedCheckpoints = store.getAllCheckpoints();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
index 607e773cb19c6..73fcf78298a40 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
@@ -22,7 +22,6 @@
 import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.state.RetrievableStateHandle;
-import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper;
 import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
 import org.junit.AfterClass;
@@ -59,7 +58,7 @@ public void cleanUp() throws Exception {
 	}
 
 	@Override
-	protected CompletedCheckpointStore createCompletedCheckpoints(
+	protected AbstractCompletedCheckpointStore createCompletedCheckpoints(
 			int maxNumberOfCheckpointsToRetain) throws Exception {
 
 		return new ZooKeeperCompletedCheckpointStore(maxNumberOfCheckpointsToRetain,
@@ -80,21 +79,20 @@ public RetrievableStateHandle<CompletedCheckpoint> store(CompletedCheckpoint sta
 	 */
 	@Test
 	public void testRecover() throws Exception {
-		CompletedCheckpointStore checkpoints = createCompletedCheckpoints(3);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+		AbstractCompletedCheckpointStore checkpoints = createCompletedCheckpoints(3);
 
 		TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
 				createCheckpoint(0), createCheckpoint(1), createCheckpoint(2)
 		};
 
 		// Add multiple checkpoints
-		checkpoints.addCheckpoint(expected[0], sharedStateRegistry);
-		checkpoints.addCheckpoint(expected[1], sharedStateRegistry);
-		checkpoints.addCheckpoint(expected[2], sharedStateRegistry);
+		checkpoints.addCheckpoint(expected[0]);
+		checkpoints.addCheckpoint(expected[1]);
+		checkpoints.addCheckpoint(expected[2]);
 
-		verifyCheckpointRegistered(expected[0].getTaskStates().values(), sharedStateRegistry);
-		verifyCheckpointRegistered(expected[1].getTaskStates().values(), sharedStateRegistry);
-		verifyCheckpointRegistered(expected[2].getTaskStates().values(), sharedStateRegistry);
+		verifyCheckpointRegistered(expected[0].getTaskStates().values(), checkpoints.sharedStateRegistry);
+		verifyCheckpointRegistered(expected[1].getTaskStates().values(), checkpoints.sharedStateRegistry);
+		verifyCheckpointRegistered(expected[2].getTaskStates().values(), checkpoints.sharedStateRegistry);
 
 		// All three should be in ZK
 		assertEquals(3, ZooKeeper.getClient().getChildren().forPath(CheckpointsPath).size());
@@ -104,9 +102,8 @@ public void testRecover() throws Exception {
 		resetCheckpoint(expected[1].getTaskStates().values());
 		resetCheckpoint(expected[2].getTaskStates().values());
 
-		// Recover
-		SharedStateRegistry newSharedStateRegistry = new SharedStateRegistry();
-		checkpoints.recover(newSharedStateRegistry);
+		// Recover TODO!!! clear registry!
+		checkpoints.recover();
 
 		assertEquals(3, ZooKeeper.getClient().getChildren().forPath(CheckpointsPath).size());
 		assertEquals(3, checkpoints.getNumberOfRetainedCheckpoints());
@@ -117,14 +114,14 @@ public void testRecover() throws Exception {
 		expectedCheckpoints.add(expected[2]);
 		expectedCheckpoints.add(createCheckpoint(3));
 
-		checkpoints.addCheckpoint(expectedCheckpoints.get(2), newSharedStateRegistry);
+		checkpoints.addCheckpoint(expectedCheckpoints.get(2));
 
 		List<CompletedCheckpoint> actualCheckpoints = checkpoints.getAllCheckpoints();
 
 		assertEquals(expectedCheckpoints, actualCheckpoints);
 
 		for (CompletedCheckpoint actualCheckpoint : actualCheckpoints) {
-			verifyCheckpointRegistered(actualCheckpoint.getTaskStates().values(), newSharedStateRegistry);
+			verifyCheckpointRegistered(actualCheckpoint.getTaskStates().values(), checkpoints.sharedStateRegistry);
 		}
 	}
 
@@ -136,18 +133,17 @@ public void testShutdownDiscardsCheckpoints() throws Exception {
 		CuratorFramework client = ZooKeeper.getClient();
 
 		CompletedCheckpointStore store = createCompletedCheckpoints(1);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
 
-		store.addCheckpoint(checkpoint, sharedStateRegistry);
+		store.addCheckpoint(checkpoint);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
 		assertNotNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
-		store.shutdown(JobStatus.FINISHED, sharedStateRegistry);
+		store.shutdown(JobStatus.FINISHED);
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
-		store.recover(sharedStateRegistry);
+		store.recover();
 
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 	}
@@ -161,20 +157,19 @@ public void testSuspendKeepsCheckpoints() throws Exception {
 		CuratorFramework client = ZooKeeper.getClient();
 
 		CompletedCheckpointStore store = createCompletedCheckpoints(1);
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
 
-		store.addCheckpoint(checkpoint, sharedStateRegistry);
+		store.addCheckpoint(checkpoint);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
 		assertNotNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
-		store.shutdown(JobStatus.SUSPENDED, sharedStateRegistry);
+		store.shutdown(JobStatus.SUSPENDED);
 
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertNotNull(client.checkExists().forPath(CheckpointsPath + "/" + checkpoint.getCheckpointID()));
 
 		// Recover again
-		store.recover(sharedStateRegistry);
+		store.recover();
 
 		CompletedCheckpoint recovered = store.getLatestCheckpoint();
 		assertEquals(checkpoint, recovered);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
index 1f5731d319bb1..66ef232b0d31b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
@@ -27,7 +27,6 @@
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.runtime.state.RetrievableStateHandle;
-import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper;
 import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore;
 import org.apache.flink.util.TestLogger;
@@ -160,9 +159,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
 			stateSotrage,
 			Executors.directExecutor());
 
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
-
-		zooKeeperCompletedCheckpointStore.recover(sharedStateRegistry);
+		zooKeeperCompletedCheckpointStore.recover();
 
 		CompletedCheckpoint latestCompletedCheckpoint = zooKeeperCompletedCheckpointStore.getLatestCheckpoint();
 
@@ -227,16 +224,13 @@ public RetrievableStateHandle<CompletedCheckpoint> answer(InvocationOnMock invoc
 			stateSotrage,
 			Executors.directExecutor());
 
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
-		
-		
 		for (long i = 0; i <= numCheckpointsToRetain; ++i) {
 			CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class);
 			doReturn(i).when(checkpointToAdd).getCheckpointID();
 			doReturn(Collections.emptyMap()).when(checkpointToAdd).getTaskStates();
 			
 			try {
-				zooKeeperCompletedCheckpointStore.addCheckpoint(checkpointToAdd, sharedStateRegistry);
+				zooKeeperCompletedCheckpointStore.addCheckpoint(checkpointToAdd);
 				
 				// The checkpoint should be in the store if we successfully add it into the store.
 				List<CompletedCheckpoint> addedCheckpoints = zooKeeperCompletedCheckpointStore.getAllCheckpoints();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
index cb14ff04d10ec..821bb69912262 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
@@ -34,50 +34,20 @@ public void testRegistryNormal() {
 
 		// register one state
 		TestSharedState firstState = new TestSharedState("first");
-		sharedStateRegistry.register(firstState, true);
-		assertEquals(1, sharedStateRegistry.getReferenceCount(firstState));
+		assertEquals(1, sharedStateRegistry.register(firstState));
 
 		// register another state
 		TestSharedState secondState = new TestSharedState("second");
-		sharedStateRegistry.register(secondState, true);
-		assertEquals(1, sharedStateRegistry.getReferenceCount(secondState));
+		assertEquals(1, sharedStateRegistry.register(secondState));
 
 		// register the first state again
-		sharedStateRegistry.register(firstState, false);
-		assertEquals(2, sharedStateRegistry.getReferenceCount(firstState));
+		assertEquals(2, sharedStateRegistry.register(firstState));
 
 		// unregister the second state
-		sharedStateRegistry.unregister(secondState);
-		assertEquals(0, sharedStateRegistry.getReferenceCount(secondState));
+		assertEquals(0, sharedStateRegistry.unregister(secondState));
 
 		// unregister the first state
-		sharedStateRegistry.unregister(firstState);
-		assertEquals(1, sharedStateRegistry.getReferenceCount(firstState));
-	}
-
-	/**
-	 * Validate that registering a handle referencing uncreated state will throw exception
-	 */
-	@Test(expected = IllegalStateException.class)
-	public void testRegisterWithUncreatedReference() {
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
-
-		// register one state
-		TestSharedState state = new TestSharedState("state");
-		sharedStateRegistry.register(state, false);
-	}
-
-	/**
-	 * Validate that registering duplicate creation of the same state will throw exception
-	 */
-	@Test(expected = IllegalStateException.class)
-	public void testRegisterWithDuplicateState() {
-		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
-
-		// register one state
-		TestSharedState state = new TestSharedState("state");
-		sharedStateRegistry.register(state, true);
-		sharedStateRegistry.register(state, true);
+		assertEquals(1, sharedStateRegistry.unregister(firstState));
 	}
 
 	/**
@@ -100,7 +70,7 @@ private static class TestSharedState implements SharedStateHandle {
 		}
 
 		@Override
-		public String getKey() {
+		public String getRegistrationKey() {
 			return key;
 		}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
index 75b0f6fb7343d..a932c189c66f9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
@@ -18,12 +18,9 @@
 
 package org.apache.flink.runtime.testutils;
 
+import org.apache.flink.runtime.checkpoint.AbstractCompletedCheckpointStore;
 import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
-import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
 import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.state.SharedStateRegistry;
-import org.apache.flink.runtime.state.StateObject;
-import org.apache.flink.runtime.state.StateUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,7 +32,7 @@
  * A checkpoint store, which supports shutdown and suspend. You can use this to test HA
  * as long as the factory always returns the same store instance.
  */
-public class RecoverableCompletedCheckpointStore implements CompletedCheckpointStore {
+public class RecoverableCompletedCheckpointStore extends AbstractCompletedCheckpointStore {
 
 	private static final Logger LOG = LoggerFactory.getLogger(RecoverableCompletedCheckpointStore.class);
 
@@ -44,7 +41,7 @@ public class RecoverableCompletedCheckpointStore implements CompletedCheckpointS
 	private final ArrayDeque<CompletedCheckpoint> suspended = new ArrayDeque<>(2);
 
 	@Override
-	public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void recover() throws Exception {
 		checkpoints.addAll(suspended);
 		suspended.clear();
 
@@ -54,7 +51,7 @@ public void recover(SharedStateRegistry sharedStateRegistry) throws Exception {
 	}
 
 	@Override
-	public void addCheckpoint(CompletedCheckpoint checkpoint, SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception {
 		checkpoints.addLast(checkpoint);
 
 		checkpoint.registerSharedStates(sharedStateRegistry);
@@ -71,7 +68,7 @@ public CompletedCheckpoint getLatestCheckpoint() throws Exception {
 	}
 
 	@Override
-	public void shutdown(JobStatus jobStatus, SharedStateRegistry sharedStateRegistry) throws Exception {
+	public void shutdown(JobStatus jobStatus) throws Exception {
 		if (jobStatus.isGloballyTerminalState()) {
 			checkpoints.clear();
 			suspended.clear();

From 7bdbd3b7a5d11b241c73fe2b9a3d2bd59f016814 Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Fri, 7 Apr 2017 13:34:16 -0400
Subject: [PATCH 039/253] [FLINK-6280] [scripts] Allow logging with Java flags

Evaluate user-defined Java options immediately before starting services
and rotate all log files.

This closes #3701
---
 docs/monitoring/application_profiling.md      | 54 +++++++++++++++++++
 docs/setup/config.md                          |  8 ++-
 flink-dist/src/main/flink-bin/bin/config.sh   | 11 +++-
 .../src/main/flink-bin/bin/flink-daemon.sh    | 11 ++--
 4 files changed, 78 insertions(+), 6 deletions(-)
 create mode 100644 docs/monitoring/application_profiling.md

diff --git a/docs/monitoring/application_profiling.md b/docs/monitoring/application_profiling.md
new file mode 100644
index 0000000000000..65ef45e4fa5fb
--- /dev/null
+++ b/docs/monitoring/application_profiling.md
@@ -0,0 +1,54 @@
+---
+title: "Application Profiling"
+nav-parent_id: monitoring
+nav-pos: 15
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+* ToC
+{:toc}
+
+## Overview of Custom Logging with Apache Flink
+
+Each standalone JobManager, TaskManager, HistoryServer, and ZooKeeper daemon redirects `stdout` and `stderr` to a file
+with a `.out` filename suffix and writes internal logging to a file with a `.log` suffix. Java options configured by the
+user in `env.java.opts`, `env.java.opts.jobmanager`, and `env.java.opts.taskmanager` can likewise define log files with
+use of the script variable `FLINK_LOG_PREFIX` and by enclosing the options in double quotes for late evaluation. Log files
+using `FLINK_LOG_PREFIX` are rotated along with the default `.out` and `.log` files.
+
+# Profiling with Java Flight Recorder
+
+Java Flight Recorder is a profiling and event collection framework built into the Oracle JDK.
+[Java Mission Control](http://www.oracle.com/technetwork/java/javaseproducts/mission-control/java-mission-control-1998576.html)
+is an advanced set of tools that enables efficient and detailed analysis of the extensive of data collected by Java
+Flight Recorder. Example configuration:
+
+~~~
+env.java.opts: "-XX:+UnlockCommercialFeatures -XX:+UnlockDiagnosticVMOptions -XX:+FlightRecorder -XX:+DebugNonSafepoints -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=${FLINK_LOG_PREFIX}.jfr"
+~~~
+
+# Profiling with JITWatch
+
+[JITWatch](https://github.com/AdoptOpenJDK/jitwatch/wiki) is a log analyser and visualizer for the Java HotSpot JIT
+compiler used to inspect inlining decisions, hot methods, bytecode, and assembly. Example configuration:
+
+~~~
+env.java.opts: "-XX:+UnlockDiagnosticVMOptions -XX:+TraceClassLoading -XX:+LogCompilation -XX:LogFile=${FLINK_LOG_PREFIX}.jit -XX:+PrintAssembly"
+~~~
diff --git a/docs/setup/config.md b/docs/setup/config.md
index 6a5c1ff350579..e36f1492d47c1 100644
--- a/docs/setup/config.md
+++ b/docs/setup/config.md
@@ -42,7 +42,11 @@ The configuration files for the TaskManagers can be different, Flink does not as
 
 - `env.java.home`: The path to the Java installation to use (DEFAULT: system's default Java installation, if found). Needs to be specified if the startup scripts fail to automatically resolve the java home directory. Can be specified to point to a specific java installation or version. If this option is not specified, the startup scripts also evaluate the `$JAVA_HOME` environment variable.
 
-- `env.java.opts`: Set custom JVM options. This value is respected by Flink's start scripts, both JobManager and TaskManager, and Flink's YARN client. This can be used to set different garbage collectors or to include remote debuggers into the JVMs running Flink's services. Use `env.java.opts.jobmanager` and `env.java.opts.taskmanager` for JobManager or TaskManager-specific options, respectively.
+- `env.java.opts`: Set custom JVM options. This value is respected by Flink's start scripts, both JobManager and
+TaskManager, and Flink's YARN client. This can be used to set different garbage collectors or to include remote
+debuggers into the JVMs running Flink's services. Enclosing options in double quotes delays parameter substitution
+allowing access to variables from Flink's startup scripts. Use `env.java.opts.jobmanager` and `env.java.opts.taskmanager`
+for JobManager or TaskManager-specific options, respectively.
 
 - `env.java.opts.jobmanager`: JobManager-specific JVM options. These are used in addition to the regular `env.java.opts`.
 
@@ -536,6 +540,8 @@ Previously this key was named `recovery.mode` and the default value was `standal
 
 - `env.log.dir`: (Defaults to the `log` directory under Flink's home) Defines the directory where the Flink logs are saved. It has to be an absolute path.
 
+- `env.log.max`: (Default: `5`) The maximum number of old log files to keep.
+
 - `env.ssh.opts`: Additional command line options passed to SSH clients when starting or stopping JobManager, TaskManager, and Zookeeper services (start-cluster.sh, stop-cluster.sh, start-zookeeper-quorum.sh, stop-zookeeper-quorum.sh).
 
 ### Queryable State
diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index a16529c552dfd..08944818fcbc5 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -356,7 +356,16 @@ extractHostName() {
     echo $SLAVE
 }
 
-# Auxilliary function for log file rotation
+# Auxilliary functions for log file rotation
+rotateLogFilesWithPrefix() {
+    dir=$1
+    prefix=$2
+    while read -r log ; do
+        rotateLogFile $log
+    # find distinct set of log file names, ignoring the rotation number (trailing dot and digit)
+    done < <(find "$dir" ! -type d -path "${prefix}*" | sed -E s/\.[0-9]+$// | sort | uniq)
+}
+
 rotateLogFile() {
     log=$1;
     num=$MAX_LOG_FILE_NUMBER
diff --git a/flink-dist/src/main/flink-bin/bin/flink-daemon.sh b/flink-dist/src/main/flink-bin/bin/flink-daemon.sh
index b25358db73a57..8bb6f0b5ac404 100644
--- a/flink-dist/src/main/flink-bin/bin/flink-daemon.sh
+++ b/flink-dist/src/main/flink-bin/bin/flink-daemon.sh
@@ -78,8 +78,9 @@ fi
 # This allows us to start multiple daemon of each type.
 id=$([ -f "$pid" ] && echo $(wc -l < $pid) || echo "0")
 
-log="${FLINK_LOG_DIR}/flink-${FLINK_IDENT_STRING}-${DAEMON}-${id}-${HOSTNAME}.log"
-out="${FLINK_LOG_DIR}/flink-${FLINK_IDENT_STRING}-${DAEMON}-${id}-${HOSTNAME}.out"
+FLINK_LOG_PREFIX="${FLINK_LOG_DIR}/flink-${FLINK_IDENT_STRING}-${DAEMON}-${id}-${HOSTNAME}"
+log="${FLINK_LOG_PREFIX}.log"
+out="${FLINK_LOG_PREFIX}.out"
 
 log_setting=("-Dlog.file=${log}" "-Dlog4j.configuration=file:${FLINK_CONF_DIR}/log4j.properties" "-Dlogback.configurationFile=file:${FLINK_CONF_DIR}/logback.xml")
 
@@ -96,8 +97,7 @@ case $STARTSTOP in
 
     (start)
         # Rotate log files
-        rotateLogFile $log
-        rotateLogFile $out
+        rotateLogFilesWithPrefix $FLINK_LOG_DIR $FLINK_LOG_PREFIX
 
         # Print a warning if daemons are already running on host
         if [ -f $pid ]; then
@@ -116,6 +116,9 @@ case $STARTSTOP in
           fi
         fi
 
+        # Evaluate user options for local variable expansion
+        FLINK_ENV_JAVA_OPTS=$(eval echo ${FLINK_ENV_JAVA_OPTS})
+
         echo "Starting $DAEMON daemon on host $HOSTNAME."
         $JAVA_RUN $JVM_ARGS ${FLINK_ENV_JAVA_OPTS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}" > "$out" 200<&- 2>&1 < /dev/null &
 

From e5eedc8e4ac7ddb769290cc445365c2ea3a7c002 Mon Sep 17 00:00:00 2001
From: sunjincheng121 <sunjincheng121@gmail.com>
Date: Thu, 13 Apr 2017 17:36:18 +0800
Subject: [PATCH 040/253] [FLINK-6228] [table] Add support for OVER windows to
 streaming Table API.

This closes #3743.
---
 .../{groupWindows.scala => windows.scala}     |  47 +-
 .../flink/table/api/scala/expressionDsl.scala |  27 +-
 .../{groupWindows.scala => windows.scala}     |  43 +-
 .../org/apache/flink/table/api/table.scala    |  62 ++
 .../org/apache/flink/table/api/windows.scala  | 118 +++-
 .../table/expressions/ExpressionParser.scala  |  64 +-
 .../table/expressions/aggregations.scala      |  36 +-
 .../apache/flink/table/expressions/call.scala | 211 ++++++-
 .../table/plan/ProjectionTranslator.scala     |  51 +-
 .../scala/stream/table/OverWindowITCase.scala | 331 ++++++++++
 .../scala/stream/table/OverWindowTest.scala   | 596 ++++++++++++++++++
 .../OverWindowStringExpressionTest.scala      | 151 +++++
 12 files changed, 1713 insertions(+), 24 deletions(-)
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/{groupWindows.scala => windows.scala} (68%)
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/{groupWindows.scala => windows.scala} (73%)
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
similarity index 68%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
index 9c82e9b525333..1e9d65a523336 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
@@ -18,7 +18,8 @@
 
 package org.apache.flink.table.api.java
 
-import org.apache.flink.table.api.{SessionWindow, SlideWithSize, TumblingWindow}
+import org.apache.flink.table.api.{OverWindowPredefined, SessionWindow, SlideWithSize, TumblingWindow}
+import org.apache.flink.table.expressions.{Expression, ExpressionParser}
 
 /**
   * Helper class for creating a tumbling window. Tumbling windows are consecutive, non-overlapping
@@ -82,3 +83,47 @@ object Session {
     */
   def withGap(gap: String): SessionWindow = new SessionWindow(gap)
 }
+
+/**
+  * Helper object for creating a over window.
+  */
+object Over {
+
+  /**
+    * Specifies the time attribute on which rows are grouped.
+    *
+    * For streaming tables call [[orderBy 'rowtime or orderBy 'proctime]] to specify time mode.
+    *
+    * For batch tables, refer to a timestamp or long attribute.
+    */
+  def orderBy(orderBy: String): OverWindowPredefined = {
+    val orderByExpr = ExpressionParser.parseExpression(orderBy)
+    new OverWindowPredefined(Seq[Expression](), orderByExpr)
+  }
+
+  /**
+    * Partitions the elements on some partition keys.
+    *
+    * @param partitionBy some partition keys.
+    * @return A partitionedOver instance that only contains the orderBy method.
+    */
+  def partitionBy(partitionBy: String): PartitionedOver = {
+    val partitionByExpr = ExpressionParser.parseExpressionList(partitionBy).toArray
+    new PartitionedOver(partitionByExpr)
+  }
+}
+
+class PartitionedOver(private val partitionByExpr: Array[Expression]) {
+
+  /**
+    * Specifies the time attribute on which rows are grouped.
+    *
+    * For streaming tables call [[orderBy 'rowtime or orderBy 'proctime]] to specify time mode.
+    *
+    * For batch tables, refer to a timestamp or long attribute.
+    */
+  def orderBy(orderBy: String): OverWindowPredefined = {
+    val orderByExpr = ExpressionParser.parseExpression(orderBy)
+    new OverWindowPredefined(partitionByExpr, orderByExpr)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
index 06d46e3cead4c..fc6cb1016ff8c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
@@ -17,14 +17,15 @@
  */
 package org.apache.flink.table.api.scala
 
+import java.math.{BigDecimal => JBigDecimal}
 import java.sql.{Date, Time, Timestamp}
 
 import org.apache.calcite.avatica.util.DateTimeUtils._
 import org.apache.flink.api.common.typeinfo.{SqlTimeTypeInfo, TypeInformation}
+import org.apache.flink.table.api.{TableException, CurrentRow, CurrentRange, UnboundedRow, UnboundedRange}
 import org.apache.flink.table.expressions.ExpressionUtils.{convertArray, toMilliInterval, toMonthInterval, toRowInterval}
 import org.apache.flink.table.expressions.TimeIntervalUnit.TimeIntervalUnit
 import org.apache.flink.table.expressions._
-import java.math.{BigDecimal => JBigDecimal}
 
 import scala.language.implicitConversions
 
@@ -363,6 +364,23 @@ trait ImplicitExpressionOperations {
     */
   def position(haystack: Expression) = Position(expr, haystack)
 
+  /**
+    * For windowing function to config over window
+    * e.g.:
+    * table
+    *   .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
+    *   .select('c, 'a, 'a.count over 'w, 'a.sum over 'w)
+    */
+  def over(alias: Expression) = {
+    expr match {
+      case _: Aggregation => UnresolvedOverCall(
+        expr.asInstanceOf[Aggregation],
+        alias)
+      case _ => throw new TableException(
+        "The over method can only using with aggregation expression.")
+    }
+  }
+
   /**
     * Replaces a substring of string with a string starting at a position (starting at 1).
     *
@@ -586,6 +604,13 @@ trait ImplicitExpressionOperations {
  * to [[ImplicitExpressionOperations]].
  */
 trait ImplicitExpressionConversions {
+
+  implicit val UNBOUNDED_ROW = UnboundedRow()
+  implicit val UNBOUNDED_RANGE = UnboundedRange()
+
+  implicit val CURRENT_ROW = CurrentRow()
+  implicit val CURRENT_RANGE = CurrentRange()
+
   implicit class WithOperations(e: Expression) extends ImplicitExpressionOperations {
     def expr = e
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
similarity index 73%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/groupWindows.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
index 0e4c1c7606da9..12300276c01b8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/groupWindows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
@@ -19,7 +19,7 @@
 package org.apache.flink.table.api.scala
 
 import org.apache.flink.table.expressions.Expression
-import org.apache.flink.table.api.{SessionWindow, SlideWithSize, TumblingWindow}
+import org.apache.flink.table.api.{OverWindowPredefined, SessionWindow, SlideWithSize, TumblingWindow}
 
 /**
   * Helper object for creating a tumbling window. Tumbling windows are consecutive, non-overlapping
@@ -83,3 +83,44 @@ object Session {
     */
   def withGap(gap: Expression): SessionWindow = new SessionWindow(gap)
 }
+
+/**
+  * Helper object for creating a over window.
+  */
+object Over {
+
+  /**
+    * Specifies the time attribute on which rows are grouped.
+    *
+    * For streaming tables call [[orderBy 'rowtime or orderBy 'proctime]] to specify time mode.
+    *
+    * For batch tables, refer to a timestamp or long attribute.
+    */
+  def orderBy(orderBy: Expression): OverWindowPredefined = {
+    new OverWindowPredefined(Seq[Expression](), orderBy)
+  }
+
+  /**
+    * Partitions the elements on some partition keys.
+    *
+    * @param partitionBy some partition keys.
+    * @return A partitionedOver instance that only contains the orderBy method.
+    */
+  def partitionBy(partitionBy: Expression*): PartitionedOver = {
+    PartitionedOver(partitionBy.toArray)
+  }
+}
+
+case class PartitionedOver(partitionBy: Array[Expression]) {
+
+  /**
+    * Specifies the time attribute on which rows are grouped.
+    *
+    * For streaming tables call [[orderBy 'rowtime or orderBy 'proctime]] to specify time mode.
+    *
+    * For batch tables, refer to a timestamp or long attribute.
+    */
+  def orderBy(orderBy: Expression): OverWindowPredefined = {
+    new OverWindowPredefined(partitionBy, orderBy)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 769e940cd72c8..c688aabfb6a62 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -28,6 +28,7 @@ import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.sinks.TableSink
 
 import _root_.scala.collection.JavaConverters._
+import _root_.scala.annotation.varargs
 
 /**
   * A Table is the core component of the Table API.
@@ -810,6 +811,44 @@ class Table(
     new WindowedTable(this, window)
   }
 
+  /**
+    * Defines over-windows on the records of a table.
+    *
+    * An over-window defines for each record an interval of records over which aggregation
+    * functions can be computed.
+    *
+    * Example:
+    *
+    * {{{
+    *   table
+    *     .window(Over partitionBy 'c orderBy 'rowTime preceding 10.seconds as 'ow)
+    *     .select('c, 'b.count over 'ow, 'e.sum over 'ow)
+    * }}}
+    *
+    * __Note__: Computing over window aggregates on a streaming table is only a parallel operation
+    * if the window is partitioned. Otherwise, the whole stream will be processed by a single
+    * task, i.e., with parallelism 1.
+    *
+    * __Note__: Over-windows for batch tables are currently not supported.
+    *
+    * @param overWindows windows that specify the record interval over which aggregations are
+    *                    computed.
+    * @return An OverWindowedTable to specify the aggregations.
+    */
+  @varargs
+  def window(overWindows: OverWindow*): OverWindowedTable = {
+
+    if (tableEnv.isInstanceOf[BatchTableEnvironment]) {
+      throw TableException("Over-windows for batch tables are currently not supported..")
+    }
+
+    if (overWindows.size != 1) {
+      throw TableException("Over-Windows are currently only supported single window.")
+    }
+
+    new OverWindowedTable(this, overWindows.toArray)
+  }
+
   var tableName: String = _
 
   /**
@@ -928,6 +967,29 @@ class WindowedTable(
 
 }
 
+class OverWindowedTable(
+    private[flink] val table: Table,
+    private[flink] val overWindows: Array[OverWindow]) {
+
+  def select(fields: Expression*): Table = {
+    val expandedFields = expandProjectList(
+      fields,
+      table.logicalPlan,
+      table.tableEnv)
+
+    val expandedOverFields = resolveOverWindows(expandedFields, overWindows, table.tableEnv)
+
+    new Table(
+      table.tableEnv,
+      Project(expandedOverFields.map(UnresolvedAlias), table.logicalPlan).validate(table.tableEnv))
+  }
+
+  def select(fields: String): Table = {
+    val fieldExprs = ExpressionParser.parseExpressionList(fields)
+    select(fieldExprs: _*)
+  }
+}
+
 class WindowGroupedTable(
     private[flink] val table: Table,
     private[flink] val groupKeys: Seq[Expression],
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
index 5ba69343dc371..577d77c377b47 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
@@ -18,8 +18,124 @@
 
 package org.apache.flink.table.api
 
-import org.apache.flink.table.expressions.{Expression, ExpressionParser}
+import org.apache.flink.table.expressions._
 import org.apache.flink.table.plan.logical._
+import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
+import org.apache.flink.table.api.scala.{CURRENT_RANGE, CURRENT_ROW}
+
+/**
+  * Over window is similar to the traditional OVER SQL.
+  */
+case class OverWindow(
+    private[flink] val alias: Expression,
+    private[flink] val partitionBy: Seq[Expression],
+    private[flink] val orderBy: Expression,
+    private[flink] val preceding: Expression,
+    private[flink] val following: Expression)
+
+case class CurrentRow() extends Expression {
+  override private[flink] def resultType = RowIntervalTypeInfo.INTERVAL_ROWS
+  override private[flink] def children = Seq()
+  override def toString = "CURRENT ROW"
+}
+
+case class CurrentRange() extends Expression {
+  override private[flink] def resultType = TimeIntervalTypeInfo.INTERVAL_MILLIS
+  override private[flink] def children = Seq()
+  override def toString = "CURRENT RANGE"
+}
+
+case class UnboundedRow() extends Expression {
+  override private[flink] def resultType = RowIntervalTypeInfo.INTERVAL_ROWS
+  override private[flink] def children = Seq()
+  override def toString = "UNBOUNDED ROW"
+}
+
+case class UnboundedRange() extends Expression {
+  override private[flink] def resultType = TimeIntervalTypeInfo.INTERVAL_MILLIS
+  override private[flink] def children = Seq()
+  override def toString = "UNBOUNDED RANGE"
+}
+
+/**
+  * An over window predefined  specification.
+  */
+class OverWindowPredefined(
+    private val partitionBy: Seq[Expression],
+    private val orderBy: Expression) {
+
+  private[flink] var preceding: Expression = _
+  private[flink] var following: Expression = _
+
+  /**
+    * Assigns an alias for this window that the following `select()` clause can refer to.
+    *
+    * @param alias alias for this over window
+    * @return over window
+    */
+  def as(alias: String): OverWindow = as(ExpressionParser.parseExpression(alias))
+
+  /**
+    * Assigns an alias for this window that the following `select()` clause can refer to.
+    *
+    * @param alias alias for this over window
+    * @return over window
+    */
+  def as(alias: Expression): OverWindow = {
+
+    // set following to CURRENT_ROW / CURRENT_RANGE if not defined
+    if (null == following) {
+      if (preceding.resultType.isInstanceOf[RowIntervalTypeInfo]) {
+        following = CURRENT_ROW
+      } else {
+        following = CURRENT_RANGE
+      }
+    }
+    OverWindow(alias, partitionBy, orderBy, preceding, following)
+  }
+
+  /**
+    * Set the preceding offset (based on time or row-count intervals) for over window.
+    *
+    * @param preceding preceding offset relative to the current row.
+    * @return this over window
+    */
+  def preceding(preceding: String): OverWindowPredefined = {
+    this.preceding(ExpressionParser.parseExpression(preceding))
+  }
+
+  /**
+    * Set the preceding offset (based on time or row-count intervals) for over window.
+    *
+    * @param preceding preceding offset relative to the current row.
+    * @return this over window
+    */
+  def preceding(preceding: Expression): OverWindowPredefined = {
+    this.preceding = preceding
+    this
+  }
+
+  /**
+    * Set the following offset (based on time or row-count intervals) for over window.
+    *
+    * @param following following offset that relative to the current row.
+    * @return this over window
+    */
+  def following(following: String): OverWindowPredefined = {
+    this.following(ExpressionParser.parseExpression(following))
+  }
+
+  /**
+    * Set the following offset (based on time or row-count intervals) for over window.
+    *
+    * @param following following offset that relative to the current row.
+    * @return this over window
+    */
+  def following(following: Expression): OverWindowPredefined = {
+    this.following = following
+    this
+  }
+}
 
 /**
   * A window specification.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
index ed0b16ea0d6f3..1985248a648d5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.expressions
 
 import org.apache.calcite.avatica.util.DateTimeUtils.{MILLIS_PER_DAY, MILLIS_PER_HOUR, MILLIS_PER_MINUTE, MILLIS_PER_SECOND}
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, SqlTimeTypeInfo, TypeInformation}
-import org.apache.flink.table.api.ExpressionParserException
+import org.apache.flink.table.api.{ExpressionParserException, CurrentRow, CurrentRange, UnboundedRow, UnboundedRange}
 import org.apache.flink.table.expressions.ExpressionUtils.{toMilliInterval, toMonthInterval}
 import org.apache.flink.table.expressions.TimeIntervalUnit.TimeIntervalUnit
 import org.apache.flink.table.expressions.TimePointUnit.TimePointUnit
@@ -87,11 +87,17 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val STAR: Keyword = Keyword("*")
   lazy val GET: Keyword = Keyword("get")
   lazy val FLATTEN: Keyword = Keyword("flatten")
+  lazy val OVER: Keyword = Keyword("over")
+  lazy val CURRENT_ROW: Keyword = Keyword("current_row")
+  lazy val CURRENT_RANGE: Keyword = Keyword("current_range")
+  lazy val UNBOUNDED_ROW: Keyword = Keyword("unbounded_row")
+  lazy val UNBOUNDED_RANGE: Keyword = Keyword("unbounded_range")
 
   def functionIdent: ExpressionParser.Parser[String] =
     not(ARRAY) ~ not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~
-      not(SUM) ~ not(START) ~ not(END)~ not(CAST) ~ not(NULL) ~
-      not(IF) ~> super.ident
+      not(SUM) ~ not(START) ~ not(END)~ not(CAST) ~ not(NULL) ~ not(IF) ~
+      not(CURRENT_ROW) ~ not(UNBOUNDED_ROW) ~ not(CURRENT_RANGE) ~ not(UNBOUNDED_RANGE) ~>
+      super.ident
 
   // symbols
 
@@ -166,8 +172,25 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
     dt => Null(dt)
   }
 
+  // OVER constants
+  lazy val currentRange: PackratParser[Expression] = CURRENT_RANGE ^^ {
+    _ => CurrentRange()
+  }
+  lazy val currentRow: PackratParser[Expression] = CURRENT_ROW ^^ {
+    _ => CurrentRow()
+  }
+  lazy val unboundedRange: PackratParser[Expression] = UNBOUNDED_RANGE ^^ {
+    _ => UnboundedRange()
+  }
+  lazy val unboundedRow: PackratParser[Expression] = UNBOUNDED_ROW ^^ {
+    _ => UnboundedRow()
+  }
+  lazy val overConstant: PackratParser[Expression] =
+    currentRange | currentRow | unboundedRange | unboundedRow
+
   lazy val literalExpr: PackratParser[Expression] =
-    numberLiteral | stringLiteralFlink | singleQuoteStringLiteral | boolLiteral | nullLiteral
+    numberLiteral | stringLiteralFlink | singleQuoteStringLiteral | boolLiteral | nullLiteral |
+      overConstant
 
   lazy val fieldReference: PackratParser[NamedExpression] = (STAR | ident) ^^ {
     sym => UnresolvedFieldReference(sym)
@@ -289,12 +312,14 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val suffixFlattening: PackratParser[Expression] =
     composite <~ "." ~ FLATTEN ~ opt("()") ^^ { e => Flattening(e) }
 
+  lazy val suffixAgg: PackratParser[Expression] =
+    suffixSum | suffixMin | suffixMax | suffixCount | suffixAvg
+
   lazy val suffixed: PackratParser[Expression] =
-    suffixTimeInterval | suffixRowInterval | suffixSum | suffixMin | suffixMax | suffixStart |
-      suffixEnd | suffixCount | suffixAvg | suffixCast | suffixAs | suffixTrim |
-      suffixTrimWithoutArgs | suffixIf | suffixAsc | suffixDesc | suffixToDate |
-      suffixToTimestamp | suffixToTime | suffixExtract | suffixFloor | suffixCeil |
-      suffixGet | suffixFlattening |
+    suffixTimeInterval | suffixRowInterval | suffixStart | suffixEnd | suffixAgg |
+      suffixCast | suffixAs | suffixTrim | suffixTrimWithoutArgs | suffixIf | suffixAsc |
+      suffixDesc | suffixToDate | suffixToTimestamp | suffixToTime | suffixExtract |
+      suffixFloor | suffixCeil | suffixGet | suffixFlattening |
       suffixFunctionCall | suffixFunctionCallOneArg // function call must always be at the end
 
   // prefix operators
@@ -375,15 +400,26 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val prefixFlattening: PackratParser[Expression] =
     FLATTEN ~ "(" ~> composite <~ ")" ^^ { e => Flattening(e) }
 
+  lazy val prefixAgg: PackratParser[Expression] =
+    prefixSum | prefixMin | prefixMax | prefixCount | prefixAvg
+
   lazy val prefixed: PackratParser[Expression] =
-    prefixArray | prefixSum | prefixMin | prefixMax | prefixCount | prefixAvg |
-      prefixStart | prefixEnd | prefixCast | prefixAs | prefixTrim | prefixTrimWithoutArgs |
-      prefixIf | prefixExtract | prefixFloor | prefixCeil | prefixGet | prefixFlattening |
-      prefixFunctionCall | prefixFunctionCallOneArg // function call must always be at the end
+    prefixArray | prefixAgg | prefixStart | prefixEnd | prefixCast | prefixAs | prefixTrim |
+      prefixTrimWithoutArgs | prefixIf | prefixExtract | prefixFloor | prefixCeil | prefixGet |
+      prefixFlattening | prefixFunctionCall |
+      prefixFunctionCallOneArg // function call must always be at the end
+
+  // over
+
+  lazy val over: PackratParser[Expression] = suffixAgg ~ OVER ~ fieldReference ^^ {
+    case agg ~ _ ~ windowRef => UnresolvedOverCall(agg, windowRef)
+  } | prefixAgg ~ OVER ~ fieldReference ^^ {
+    case agg ~ _ ~ windowRef => UnresolvedOverCall(agg, windowRef)
+  }
 
   // suffix/prefix composite
 
-  lazy val composite: PackratParser[Expression] = suffixed | prefixed | atom |
+  lazy val composite: PackratParser[Expression] = suffixed | over | prefixed | atom |
     failure("Composite expression expected.")
 
   // unary ops
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
index b2fca883901e8..4ef52099d6c1a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
@@ -18,11 +18,14 @@
 package org.apache.flink.table.expressions
 
 import org.apache.calcite.rex.RexNode
-import org.apache.calcite.sql.fun.SqlStdOperatorTable
+import org.apache.calcite.sql.SqlAggFunction
+import org.apache.calcite.sql.fun._
+import org.apache.calcite.sql.SqlKind._
 import org.apache.calcite.tools.RelBuilder
 import org.apache.calcite.tools.RelBuilder.AggCall
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
 import org.apache.flink.table.typeutils.TypeCheckUtils
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.table.calcite.FlinkTypeFactory
 
 abstract sealed class Aggregation extends UnaryExpression {
 
@@ -35,6 +38,12 @@ abstract sealed class Aggregation extends UnaryExpression {
     * Convert Aggregate to its counterpart in Calcite, i.e. AggCall
     */
   private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall
+
+  /**
+    * Returns the SqlAggFunction for this Aggregation.
+    */
+  private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder): SqlAggFunction
+
 }
 
 case class Sum(child: Expression) extends Aggregation {
@@ -48,6 +57,13 @@ case class Sum(child: Expression) extends Aggregation {
 
   override private[flink] def validateInput() =
     TypeCheckUtils.assertNumericExpr(child.resultType, "sum")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) = {
+    val returnType = relBuilder
+      .getTypeFactory.asInstanceOf[FlinkTypeFactory]
+      .createTypeFromTypeInfo(resultType)
+    new SqlSumAggFunction(returnType)
+  }
 }
 
 case class Min(child: Expression) extends Aggregation {
@@ -61,6 +77,10 @@ case class Min(child: Expression) extends Aggregation {
 
   override private[flink] def validateInput() =
     TypeCheckUtils.assertOrderableExpr(child.resultType, "min")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) = {
+    new SqlMinMaxAggFunction(MIN)
+  }
 }
 
 case class Max(child: Expression) extends Aggregation {
@@ -74,6 +94,10 @@ case class Max(child: Expression) extends Aggregation {
 
   override private[flink] def validateInput() =
     TypeCheckUtils.assertOrderableExpr(child.resultType, "max")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) = {
+    new SqlMinMaxAggFunction(MAX)
+  }
 }
 
 case class Count(child: Expression) extends Aggregation {
@@ -84,6 +108,10 @@ case class Count(child: Expression) extends Aggregation {
   }
 
   override private[flink] def resultType = BasicTypeInfo.LONG_TYPE_INFO
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) = {
+    new SqlCountAggFunction()
+  }
 }
 
 case class Avg(child: Expression) extends Aggregation {
@@ -97,4 +125,8 @@ case class Avg(child: Expression) extends Aggregation {
 
   override private[flink] def validateInput() =
     TypeCheckUtils.assertNumericExpr(child.resultType, "avg")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) = {
+    new SqlAvgAggFunction(AVG)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
index 40db13e8b615a..68ed68827992f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
@@ -17,15 +17,25 @@
  */
 package org.apache.flink.table.expressions
 
-import org.apache.calcite.rex.RexNode
+import java.util
+
+import com.google.common.collect.ImmutableList
+import org.apache.calcite.rex.RexWindowBound._
+import org.apache.calcite.rex.{RexFieldCollation, RexNode, RexWindowBound}
+import org.apache.calcite.sql._
+import org.apache.calcite.sql.`type`.OrdinalReturnTypeInference
+import org.apache.calcite.sql.parser.SqlParserPos
 import org.apache.calcite.tools.RelBuilder
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.table.api.{UnresolvedException, ValidationException}
+import org.apache.flink.table.api._
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
-import org.apache.flink.table.functions.{ScalarFunction, TableFunction}
+import org.apache.flink.table.functions._
 import org.apache.flink.table.plan.logical.{LogicalNode, LogicalTableFunctionCall}
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
+import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
+
+import _root_.scala.collection.JavaConverters._
 
 /**
   * General expression for unresolved function calls. The function can be a built-in
@@ -48,6 +58,201 @@ case class Call(functionName: String, args: Seq[Expression]) extends Expression
     ValidationFailure(s"Unresolved function call: $functionName")
 }
 
+/**
+  * Over call with unresolved alias for over window.
+  *
+  * @param agg The aggregation of the over call.
+  * @param alias The alias of the referenced over window.
+  */
+case class UnresolvedOverCall(agg: Expression, alias: Expression) extends Expression {
+
+  override private[flink] def validateInput() =
+    ValidationFailure("Over window with alias $alias could not be resolved.")
+
+  override private[flink] def resultType = agg.resultType
+
+  override private[flink] def children = Seq()
+}
+
+/**
+  * Over expression for Calcite over transform.
+  *
+  * @param agg            over-agg expression
+  * @param partitionBy    The fields by which the over window is partitioned
+  * @param orderBy        The field by which the over window is sorted
+  * @param preceding      The lower bound of the window
+  * @param following      The upper bound of the window
+  */
+case class OverCall(
+    agg: Expression,
+    partitionBy: Seq[Expression],
+    orderBy: Expression,
+    preceding: Expression,
+    following: Expression) extends Expression {
+
+  override def toString: String = s"$agg OVER (" +
+    s"PARTITION BY (${partitionBy.mkString(", ")}) " +
+    s"ORDER BY $orderBy " +
+    s"PRECEDING $preceding " +
+    s"FOLLOWING $following)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+
+    val rexBuilder = relBuilder.getRexBuilder
+
+    // assemble aggregation
+    val operator: SqlAggFunction = agg.asInstanceOf[Aggregation].getSqlAggFunction()
+    val aggResultType = relBuilder
+      .getTypeFactory.asInstanceOf[FlinkTypeFactory]
+      .createTypeFromTypeInfo(agg.resultType)
+
+    val aggChildName = agg.asInstanceOf[Aggregation].child.asInstanceOf[ResolvedFieldReference].name
+    val aggExprs = List(relBuilder.field(aggChildName).asInstanceOf[RexNode]).asJava
+
+    // assemble order by key
+    val orderKey = orderBy match {
+      case _: RowTime =>
+        new RexFieldCollation(relBuilder.call(EventTimeExtractor), Set[SqlKind]().asJava)
+      case _: ProcTime =>
+        new RexFieldCollation(relBuilder.call(ProcTimeExtractor), Set[SqlKind]().asJava)
+      case _ =>
+        throw new ValidationException("Invalid OrderBy expression.")
+    }
+    val orderKeys = ImmutableList.of(orderKey)
+
+    // assemble partition by keys
+    val partitionKeys = partitionBy.map(_.toRexNode(relBuilder)).asJava
+
+    // assemble bounds
+    val isPhysical: Boolean = preceding.resultType.isInstanceOf[RowIntervalTypeInfo]
+
+    val lowerBound = createBound(relBuilder, preceding, SqlKind.PRECEDING)
+    val upperBound = createBound(relBuilder, following, SqlKind.FOLLOWING)
+
+    // build RexOver
+    rexBuilder.makeOver(
+      aggResultType,
+      operator,
+      aggExprs,
+      partitionKeys,
+      orderKeys,
+      lowerBound,
+      upperBound,
+      isPhysical,
+      true,
+      false)
+  }
+
+  private def createBound(
+    relBuilder: RelBuilder,
+    bound: Expression,
+    sqlKind: SqlKind): RexWindowBound = {
+
+    bound match {
+      case _: UnboundedRow | _: UnboundedRange =>
+        val unbounded = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO)
+        create(unbounded, null)
+      case _: CurrentRow | _: CurrentRange =>
+        val currentRow = SqlWindow.createCurrentRow(SqlParserPos.ZERO)
+        create(currentRow, null)
+      case b: Literal =>
+        val returnType = relBuilder
+          .getTypeFactory.asInstanceOf[FlinkTypeFactory]
+          .createTypeFromTypeInfo(Types.DECIMAL)
+
+        val sqlOperator = new SqlPostfixOperator(
+          sqlKind.name,
+          sqlKind,
+          2,
+          new OrdinalReturnTypeInference(0),
+          null,
+          null)
+
+        val operands: Array[SqlNode] = new Array[SqlNode](1)
+        operands(0) = SqlLiteral.createExactNumeric("1", SqlParserPos.ZERO)
+
+        val node = new SqlBasicCall(sqlOperator, operands, SqlParserPos.ZERO)
+
+        val expressions: util.ArrayList[RexNode] = new util.ArrayList[RexNode]()
+        expressions.add(relBuilder.literal(b.value))
+
+        val rexNode = relBuilder.getRexBuilder.makeCall(returnType, sqlOperator, expressions)
+
+        create(node, rexNode)
+    }
+  }
+
+  override private[flink] def children: Seq[Expression] =
+    Seq(agg) ++ Seq(orderBy) ++ partitionBy ++ Seq(preceding) ++ Seq(following)
+
+  override private[flink] def resultType = agg.resultType
+
+  override private[flink] def validateInput(): ValidationResult = {
+
+    // check that agg expression is aggregation
+    agg match {
+      case _: Aggregation =>
+        ValidationSuccess
+      case _ =>
+        return ValidationFailure(s"OVER can only be applied on an aggregation.")
+    }
+
+    // check partitionBy expression keys are resolved field reference
+    partitionBy.foreach {
+      case r: ResolvedFieldReference if r.resultType.isKeyType  =>
+        ValidationSuccess
+      case r: ResolvedFieldReference =>
+        return ValidationFailure(s"Invalid PartitionBy expression: $r. " +
+          s"Expression must return key type.")
+      case r =>
+        return ValidationFailure(s"Invalid PartitionBy expression: $r. " +
+          s"Expression must be a resolved field reference.")
+    }
+
+    // check preceding is valid
+    preceding match {
+      case _: CurrentRow | _: CurrentRange | _: UnboundedRow | _: UnboundedRange =>
+        ValidationSuccess
+      case Literal(v: Long, _: RowIntervalTypeInfo) if v > 0 =>
+        ValidationSuccess
+      case Literal(_, _: RowIntervalTypeInfo) =>
+        return ValidationFailure("Preceding row interval must be larger than 0.")
+      case Literal(v: Long, _: TimeIntervalTypeInfo[_]) if v >= 0 =>
+        ValidationSuccess
+      case Literal(_, _: TimeIntervalTypeInfo[_]) =>
+        return ValidationFailure("Preceding time interval must be equal or larger than 0.")
+      case Literal(_, _) =>
+        return ValidationFailure("Preceding must be a row interval or time interval literal.")
+    }
+
+    // check following is valid
+    following match {
+      case _: CurrentRow | _: CurrentRange | _: UnboundedRow | _: UnboundedRange =>
+        ValidationSuccess
+      case Literal(v: Long, _: RowIntervalTypeInfo) if v > 0 =>
+        ValidationSuccess
+      case Literal(_, _: RowIntervalTypeInfo) =>
+        return ValidationFailure("Following row interval must be larger than 0.")
+      case Literal(v: Long, _: TimeIntervalTypeInfo[_]) if v >= 0 =>
+        ValidationSuccess
+      case Literal(_, _: TimeIntervalTypeInfo[_]) =>
+        return ValidationFailure("Following time interval must be equal or larger than 0.")
+      case Literal(_, _) =>
+        return ValidationFailure("Following must be a row interval or time interval literal.")
+    }
+
+    // check that preceding and following are of same type
+    (preceding, following) match {
+      case (p: Expression, f: Expression) if p.resultType == f.resultType =>
+        ValidationSuccess
+      case _ =>
+        return ValidationFailure("Preceding and following must be of same interval type.")
+    }
+
+    ValidationSuccess
+  }
+}
+
 /**
   * Expression for calling a user-defined scalar functions.
   *
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
index 94a0aa1cf8142..0d45a37992123 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
@@ -19,8 +19,9 @@
 package org.apache.flink.table.plan
 
 import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.{OverWindow, StreamTableEnvironment, TableEnvironment}
 import org.apache.flink.table.expressions._
+import org.apache.flink.table.functions.{ProcTime, RowTime}
 import org.apache.flink.table.plan.logical.{LogicalNode, Project}
 
 import scala.collection.mutable
@@ -221,6 +222,54 @@ object ProjectionTranslator {
     projectList
   }
 
+  def resolveOverWindows(
+      exprs: Seq[Expression],
+      overWindows: Array[OverWindow],
+      tEnv: TableEnvironment): Seq[Expression] = {
+
+    def resolveOverWindow(unresolvedCall: UnresolvedOverCall): Expression = {
+
+      val overWindow = overWindows.find(_.alias.equals(unresolvedCall.alias))
+      if (overWindow.isDefined) {
+        if (tEnv.isInstanceOf[StreamTableEnvironment]) {
+          val timeIndicator = overWindow.get.orderBy match {
+            case u: UnresolvedFieldReference if u.name.toLowerCase == "rowtime" =>
+              RowTime()
+            case u: UnresolvedFieldReference if u.name.toLowerCase == "proctime" =>
+              ProcTime()
+            case e: Expression => e
+          }
+          OverCall(
+            unresolvedCall.agg,
+            overWindow.get.partitionBy,
+            timeIndicator,
+            overWindow.get.preceding,
+            overWindow.get.following)
+        } else {
+          OverCall(
+            unresolvedCall.agg,
+            overWindow.get.partitionBy,
+            overWindow.get.orderBy,
+            overWindow.get.preceding,
+            overWindow.get.following)
+        }
+      } else {
+        unresolvedCall
+      }
+    }
+
+    val projectList = new ListBuffer[Expression]
+    exprs.foreach {
+      case Alias(u: UnresolvedOverCall, name, _) =>
+        projectList += Alias(resolveOverWindow(u), name)
+      case u: UnresolvedOverCall =>
+        projectList += resolveOverWindow(u)
+      case e: Expression => projectList += e
+    }
+    projectList
+  }
+
+
   /**
     * Extract all field references from the given expressions.
     *
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
new file mode 100644
index 0000000000000..8464ba166119a
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.functions.source.SourceFunction
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.scala.stream.table.OverWindowITCase.RowTimeSourceFunction
+import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamingWithStateTestBase}
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+class OverWindowITCase extends StreamingWithStateTestBase {
+
+  @Test
+  def testProcTimeUnBoundedPartitionedRowOver(): Unit = {
+
+    val data = List(
+      (1L, 1, "Hello"),
+      (2L, 2, "Hello"),
+      (3L, 3, "Hello"),
+      (4L, 4, "Hello"),
+      (5L, 5, "Hello"),
+      (6L, 6, "Hello"),
+      (7L, 7, "Hello World"),
+      (8L, 8, "Hello World"),
+      (20L, 20, "Hello World"))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setParallelism(1)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv, 'a, 'b, 'c)
+
+    val windowedTable = table
+      .window(
+        Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_ROW as 'w)
+      .select('c, 'b.count over 'w as 'mycount)
+      .select('c, 'mycount)
+
+    val results = windowedTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq(
+      "Hello World,1", "Hello World,2", "Hello World,3",
+      "Hello,1", "Hello,2", "Hello,3", "Hello,4", "Hello,5", "Hello,6")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeUnBoundedPartitionedRangeOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
+    env.setParallelism(1)
+
+    val data = Seq(
+      Left(14000005L, (1, 1L, "Hi")),
+      Left(14000000L, (2, 1L, "Hello")),
+      Left(14000002L, (1, 1L, "Hello")),
+      Left(14000002L, (1, 2L, "Hello")),
+      Left(14000002L, (1, 3L, "Hello world")),
+      Left(14000003L, (2, 2L, "Hello world")),
+      Left(14000003L, (2, 3L, "Hello world")),
+      Right(14000020L),
+      Left(14000021L, (1, 4L, "Hello world")),
+      Left(14000022L, (1, 5L, "Hello world")),
+      Left(14000022L, (1, 6L, "Hello world")),
+      Left(14000022L, (1, 7L, "Hello world")),
+      Left(14000023L, (2, 4L, "Hello world")),
+      Left(14000023L, (2, 5L, "Hello world")),
+      Right(14000030L)
+    )
+    val table = env
+      .addSource(new RowTimeSourceFunction[(Int, Long, String)](data))
+      .toTable(tEnv).as('a, 'b, 'c)
+
+    val windowedTable = table
+      .window(Over partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE following
+         CURRENT_RANGE as 'w)
+      .select(
+        'a, 'b, 'c,
+        'b.sum over 'w,
+        'b.count over 'w,
+        'b.avg over 'w,
+        'b.max over 'w,
+        'b.min over 'w)
+
+    val result = windowedTable.toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "1,1,Hello,6,3,2,3,1",
+      "1,2,Hello,6,3,2,3,1",
+      "1,3,Hello world,6,3,2,3,1",
+      "1,1,Hi,7,4,1,3,1",
+      "2,1,Hello,1,1,1,1,1",
+      "2,2,Hello world,6,3,2,3,1",
+      "2,3,Hello world,6,3,2,3,1",
+      "1,4,Hello world,11,5,2,4,1",
+      "1,5,Hello world,29,8,3,7,1",
+      "1,6,Hello world,29,8,3,7,1",
+      "1,7,Hello world,29,8,3,7,1",
+      "2,4,Hello world,15,5,3,5,1",
+      "2,5,Hello world,15,5,3,5,1"
+    )
+
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testProcTimeBoundedPartitionedRangeOver(): Unit = {
+
+    val data = List(
+      (1, 1L, 0, "Hallo", 1L),
+      (2, 2L, 1, "Hallo Welt", 2L),
+      (2, 3L, 2, "Hallo Welt wie", 1L),
+      (3, 4L, 3, "Hallo Welt wie gehts?", 2L),
+      (3, 5L, 4, "ABC", 2L),
+      (3, 6L, 5, "BCD", 3L),
+      (4, 7L, 6, "CDE", 2L),
+      (4, 8L, 7, "DEF", 1L),
+      (4, 9L, 8, "EFG", 1L),
+      (4, 10L, 9, "FGH", 2L),
+      (5, 11L, 10, "GHI", 1L),
+      (5, 12L, 11, "HIJ", 3L),
+      (5, 13L, 12, "IJK", 3L),
+      (5, 14L, 13, "JKL", 2L),
+      (5, 15L, 14, "KLM", 2L))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setParallelism(1)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+
+    val windowedTable = table
+      .window(Over partitionBy 'a orderBy 'proctime preceding 4.rows following CURRENT_ROW as 'w)
+      .select('a, 'c.sum over 'w, 'c.min over 'w)
+    val result = windowedTable.toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "1,0,0",
+      "2,1,1",
+      "2,3,1",
+      "3,3,3",
+      "3,7,3",
+      "3,12,3",
+      "4,6,6",
+      "4,13,6",
+      "4,21,6",
+      "4,30,6",
+      "5,10,10",
+      "5,21,10",
+      "5,33,10",
+      "5,46,10",
+      "5,60,10")
+
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRowOver(): Unit = {
+    val data = Seq(
+      Left((1L, (1L, 1, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((1L, (1L, 1, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((1L, (1L, 1, "Hello"))),
+      Left((3L, (7L, 7, "Hello World"))),
+      Left((1L, (7L, 7, "Hello World"))),
+      Left((1L, (7L, 7, "Hello World"))),
+      Right(2L),
+      Left((3L, (3L, 3, "Hello"))),
+      Left((4L, (4L, 4, "Hello"))),
+      Left((5L, (5L, 5, "Hello"))),
+      Left((6L, (6L, 6, "Hello"))),
+      Left((20L, (20L, 20, "Hello World"))),
+      Right(6L),
+      Left((8L, (8L, 8, "Hello World"))),
+      Left((7L, (7L, 7, "Hello World"))),
+      Right(20L))
+
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setParallelism(1)
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val table = env.addSource[(Long, Int, String)](
+      new RowTimeSourceFunction[(Long, Int, String)](data)).toTable(tEnv).as('a, 'b, 'c)
+
+    val windowedTable = table
+      .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
+      .select('c, 'a, 'a.count over 'w, 'a.sum over 'w)
+
+    val result = windowedTable.toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
+      "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
+      "Hello,3,3,7", "Hello,4,3,9", "Hello,5,3,12",
+      "Hello,6,3,15",
+      "Hello World,7,1,7", "Hello World,7,2,14", "Hello World,7,3,21",
+      "Hello World,7,3,21", "Hello World,8,3,22", "Hello World,20,3,35")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRangeOver(): Unit = {
+    val data = Seq(
+      Left((1500L, (1L, 15, "Hello"))),
+      Left((1600L, (1L, 16, "Hello"))),
+      Left((1000L, (1L, 1, "Hello"))),
+      Left((2000L, (2L, 2, "Hello"))),
+      Right(1000L),
+      Left((2000L, (2L, 2, "Hello"))),
+      Left((2000L, (2L, 3, "Hello"))),
+      Left((3000L, (3L, 3, "Hello"))),
+      Right(2000L),
+      Left((4000L, (4L, 4, "Hello"))),
+      Right(3000L),
+      Left((5000L, (5L, 5, "Hello"))),
+      Right(5000L),
+      Left((6000L, (6L, 6, "Hello"))),
+      Left((6500L, (6L, 65, "Hello"))),
+      Right(7000L),
+      Left((9000L, (6L, 9, "Hello"))),
+      Left((9500L, (6L, 18, "Hello"))),
+      Left((9000L, (6L, 9, "Hello"))),
+      Right(10000L),
+      Left((10000L, (7L, 7, "Hello World"))),
+      Left((11000L, (7L, 17, "Hello World"))),
+      Left((11000L, (7L, 77, "Hello World"))),
+      Right(12000L),
+      Left((14000L, (7L, 18, "Hello World"))),
+      Right(14000L),
+      Left((15000L, (8L, 8, "Hello World"))),
+      Right(17000L),
+      Left((20000L, (20L, 20, "Hello World"))),
+      Right(19000L))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val table = env.addSource[(Long, Int, String)](
+      new RowTimeSourceFunction[(Long, Int, String)](data)).toTable(tEnv).as('a, 'b, 'c)
+
+    val windowedTable = table
+      .window(
+        Over partitionBy 'c orderBy 'rowtime preceding 1.seconds following CURRENT_RANGE as 'w)
+      .select('c, 'b, 'a.count over 'w, 'a.sum over 'w)
+
+    val result = windowedTable.toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
+      "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
+      "Hello,3,4,9",
+      "Hello,4,2,7",
+      "Hello,5,2,9",
+      "Hello,6,2,11", "Hello,65,2,12",
+      "Hello,9,2,12", "Hello,9,2,12", "Hello,18,3,18",
+      "Hello World,7,1,7", "Hello World,17,3,21", "Hello World,77,3,21", "Hello World,18,1,7",
+      "Hello World,8,2,15",
+      "Hello World,20,1,20")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+}
+
+object OverWindowITCase {
+
+  class RowTimeSourceFunction[T](
+      dataWithTimestampList: Seq[Either[(Long, T), Long]]) extends SourceFunction[T] {
+    override def run(ctx: SourceContext[T]): Unit = {
+      dataWithTimestampList.foreach {
+        case Left(t) => ctx.collectWithTimestamp(t._2, t._1)
+        case Right(w) => ctx.emitWatermark(new Watermark(w))
+      }
+    }
+
+    override def cancel(): Unit = ???
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
new file mode 100644
index 0000000000000..26f4eadafe4dc
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.api.scala.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.{Table, ValidationException}
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.utils.TableTestUtil._
+import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase}
+import org.junit.Test
+
+class OverWindowTest extends TableTestBase {
+  private val streamUtil: StreamTableTestUtil = streamTestUtil()
+  val table: Table = streamUtil.addTable[(Int, String, Long)]("MyTable", 'a, 'b, 'c)
+
+  @Test(expected = classOf[ValidationException])
+  def testInvalidWindowAlias(): Unit = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows as 'w)
+      .select('c, 'b.count over 'x)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testOrderBy(): Unit = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'abc preceding 2.rows as 'w)
+      .select('c, 'b.count over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testPrecedingAndFollowingUsingIsLiteral(): Unit = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'rowtime preceding 2 following "xx" as 'w)
+      .select('c, 'b.count over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testPrecedingAndFollowingUsingSameType(): Unit = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_RANGE as 'w)
+      .select('c, 'b.count over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testPartitionByWithUnresolved(): Unit = {
+    val result = table
+      .window(Over partitionBy 'a + 'b orderBy 'rowtime preceding 2.rows as 'w)
+      .select('c, 'b.count over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testPartitionByWithNotKeyType(): Unit = {
+    val table2 = streamUtil.addTable[(Int, String, Either[Long, String])]("MyTable2", 'a, 'b, 'c)
+
+    val result = table2
+      .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows as 'w)
+      .select('c, 'b.count over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testPrecedingValue(): Unit = {
+    val result = table
+      .window(Over orderBy 'rowtime preceding -1.rows as 'w)
+      .select('c, 'b.count over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testFollowingValue(): Unit = {
+    val result = table
+      .window(Over orderBy 'rowtime preceding 1.rows following -2.rows as 'w)
+      .select('c, 'b.count over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
+  @Test
+  def testProcTimeBoundedPartitionedRowsOver() = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'proctime preceding 2.rows following CURRENT_ROW as 'w)
+      .select('c, 'b.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "b", "c", "PROCTIME() AS $2")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "PROCTIME"),
+          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
+          term("select", "b", "c", "PROCTIME", "COUNT(b) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testProcTimeBoundedPartitionedRangeOver() = {
+    val result = table
+      .window(
+        Over partitionBy 'a orderBy 'proctime preceding 2.hours following CURRENT_RANGE as 'w)
+      .select('a, 'c.avg over 'w as 'myAvg)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "PROCTIME() AS $2")
+          ),
+          term("partitionBy", "a"),
+          term("orderBy", "PROCTIME"),
+          term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "PROCTIME",
+            "AVG(c) AS w0$o0"
+          )
+        ),
+        term("select", "a", "w0$o0 AS myAvg")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testProcTimeBoundedNonPartitionedRangeOver() = {
+    val result = table
+      .window(Over orderBy 'proctime preceding 10.second as 'w)
+      .select('a, 'c.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "PROCTIME() AS $2")
+          ),
+          term("orderBy", "PROCTIME"),
+          term("range", "BETWEEN 10000 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "PROCTIME", "COUNT(c) AS w0$o0")
+        ),
+        term("select", "a", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testProcTimeBoundedNonPartitionedRowsOver() = {
+    val result = table
+      .window(Over orderBy 'proctime preceding 2.rows as 'w)
+      .select('c, 'a.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "PROCTIME() AS $2")
+          ),
+          term("orderBy", "PROCTIME"),
+          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testProcTimeUnboundedPartitionedRangeOver() = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_RANGE following
+         CURRENT_RANGE as 'w)
+      .select('a, 'c, 'a.count over 'w, 'a.sum over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "PROCTIME() AS $2")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "PROCTIME"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "PROCTIME",
+            "COUNT(a) AS w0$o0",
+            "SUM(a) AS w0$o1"
+          )
+        ),
+        term(
+          "select",
+          "a",
+          "c",
+          "w0$o0 AS _c2",
+          "w0$o1 AS _c3"
+        )
+      )
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testProcTimeUnboundedPartitionedRowsOver() = {
+    val result = table
+      .window(
+        Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_ROW following CURRENT_ROW as 'w)
+      .select('c, 'a.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "PROCTIME() AS $2")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "PROCTIME"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testProcTimeUnboundedNonPartitionedRangeOver() = {
+    val result = table
+      .window(
+        Over orderBy 'proctime preceding UNBOUNDED_RANGE as 'w)
+      .select('a, 'c, 'a.count over 'w, 'a.sum over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "PROCTIME() AS $2")
+          ),
+          term("orderBy", "PROCTIME"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "PROCTIME",
+            "COUNT(a) AS w0$o0",
+            "SUM(a) AS w0$o1"
+          )
+        ),
+        term(
+          "select",
+          "a",
+          "c",
+          "w0$o0 AS _c2",
+          "w0$o1 AS _c3"
+        )
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testProcTimeUnboundedNonPartitionedRowsOver() = {
+    val result = table
+      .window(Over orderBy 'proctime preceding UNBOUNDED_ROW as 'w)
+      .select('c, 'a.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "PROCTIME() AS $2")
+          ),
+          term("orderBy", "PROCTIME"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRowsOver() = {
+    val result = table
+      .window(
+        Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
+      .select('c, 'b.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "b", "c", "ROWTIME() AS $2")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "ROWTIME"),
+          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
+          term("select", "b", "c", "ROWTIME", "COUNT(b) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRangeOver() = {
+    val result = table
+      .window(
+        Over partitionBy 'a orderBy 'rowtime preceding 2.hours following CURRENT_RANGE as 'w)
+      .select('a, 'c.avg over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "ROWTIME() AS $2")
+          ),
+          term("partitionBy", "a"),
+          term("orderBy", "ROWTIME"),
+          term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "ROWTIME",
+            "AVG(c) AS w0$o0"
+          )
+        ),
+        term("select", "a", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedNonPartitionedRangeOver() = {
+    val result = table
+      .window(Over orderBy 'rowtime preceding 10.second as 'w)
+      .select('a, 'c.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "ROWTIME() AS $2")
+          ),
+          term("orderBy", "ROWTIME"),
+          term("range", "BETWEEN 10000 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "ROWTIME", "COUNT(c) AS w0$o0")
+        ),
+        term("select", "a", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedNonPartitionedRowsOver() = {
+    val result = table
+      .window(Over orderBy 'rowtime preceding 2.rows as 'w)
+      .select('c, 'a.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "ROWTIME() AS $2")
+          ),
+          term("orderBy", "ROWTIME"),
+          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedPartitionedRangeOver() = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'rowtime preceding UNBOUNDED_RANGE following
+         CURRENT_RANGE as 'w)
+      .select('a, 'c, 'a.count over 'w, 'a.sum over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "ROWTIME() AS $2")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "ROWTIME"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "ROWTIME",
+            "COUNT(a) AS w0$o0",
+            "SUM(a) AS w0$o1"
+          )
+        ),
+        term(
+          "select",
+          "a",
+          "c",
+          "w0$o0 AS _c2",
+          "w0$o1 AS _c3"
+        )
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedPartitionedRowsOver() = {
+    val result = table
+      .window(Over partitionBy 'c orderBy 'rowtime preceding UNBOUNDED_ROW following
+         CURRENT_ROW as 'w)
+      .select('c, 'a.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "ROWTIME() AS $2")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "ROWTIME"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedNonPartitionedRangeOver() = {
+    val result = table
+      .window(
+        Over orderBy 'rowtime preceding UNBOUNDED_RANGE as 'w)
+      .select('a, 'c, 'a.count over 'w, 'a.sum over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "ROWTIME() AS $2")
+          ),
+          term("orderBy", "ROWTIME"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "ROWTIME",
+            "COUNT(a) AS w0$o0",
+            "SUM(a) AS w0$o1"
+          )
+        ),
+        term(
+          "select",
+          "a",
+          "c",
+          "w0$o0 AS _c2",
+          "w0$o1 AS _c3"
+        )
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedNonPartitionedRowsOver() = {
+    val result = table
+      .window(Over orderBy 'rowtime preceding UNBOUNDED_ROW as 'w)
+      .select('c, 'a.count over 'w)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "ROWTIME() AS $2")
+          ),
+          term("orderBy", "ROWTIME"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS _c1")
+      )
+
+    streamUtil.verifyTable(result, expected)
+  }
+
+}
+
+object OverWindowTest{
+  case class Pojo(id: Long, name: String)
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
new file mode 100644
index 0000000000000..0a5e0019b1101
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream.table.stringexpr
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.java.{Over => JOver}
+import org.apache.flink.table.api.scala.{Over => SOver, _}
+import org.apache.flink.table.utils.TableTestBase
+import org.junit.Test
+
+class OverWindowStringExpressionTest extends TableTestBase {
+
+  @Test
+  def testPartitionedUnboundedOverRow(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_ROW as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(JOver.partitionBy("a").orderBy("rowtime").preceding("unbounded_row").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testUnboundedOverRow(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver orderBy 'rowtime preceding UNBOUNDED_ROW following CURRENT_ROW as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(JOver.orderBy("rowtime").preceding("unbounded_row").following("current_row").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testPartitionedBoundedOverRow(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver partitionBy('a, 'd) orderBy 'rowtime preceding 10.rows as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(JOver.partitionBy("a, d").orderBy("rowtime").preceding("10.rows").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testBoundedOverRow(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver orderBy 'rowtime preceding 10.rows following CURRENT_ROW as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(JOver.orderBy("rowtime").preceding("10.rows").following("current_row").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testPartitionedUnboundedOverRange(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(JOver.partitionBy("a").orderBy("rowtime").preceding("unbounded_range").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testUnboundedOverRange(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver orderBy 'rowtime preceding UNBOUNDED_RANGE following CURRENT_RANGE as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(
+        JOver.orderBy("rowtime").preceding("unbounded_range").following("current_range").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testPartitionedBoundedOverRange(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver partitionBy('a, 'c) orderBy 'rowtime preceding 10.minutes as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(JOver.partitionBy("a, c").orderBy("rowtime").preceding("10.minutes").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testBoundedOverRange(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+    val resScala = t
+      .window(SOver orderBy 'rowtime preceding 4.hours following CURRENT_RANGE as 'w)
+      .select('a, 'b.sum over 'w)
+    val resJava = t
+      .window(JOver.orderBy("rowtime").preceding("4.hours").following("current_range").as("w"))
+      .select("a, SUM(b) OVER w")
+
+    verifyTableEquals(resScala, resJava)
+  }
+
+
+}

From 57c1ef224530337b6d19813f24bef0c29f8338c8 Mon Sep 17 00:00:00 2001
From: shaoxuan-wang <wshaoxuan@gmail.com>
Date: Tue, 25 Apr 2017 00:28:37 +0800
Subject: [PATCH 041/253] [FLINK-6361] [table] Refactor the AggregateFunction
 interface and built-in aggregates.

This closes #3762.
---
 .../org/apache/flink/table/api/Types.scala    |   3 +-
 .../flink/table/codegen/CodeGenerator.scala   |  60 ++---
 .../table/functions/AggregateFunction.scala   | 157 +++++++------
 .../aggfunctions/AvgAggFunction.scala         | 206 ++++++++----------
 .../aggfunctions/CountAggFunction.scala       |  39 ++--
 .../aggfunctions/MaxAggFunction.scala         |  48 ++--
 .../MaxAggFunctionWithRetract.scala           |  86 ++++----
 .../aggfunctions/MinAggFunction.scala         |  48 ++--
 .../MinAggFunctionWithRetract.scala           |  86 ++++----
 .../aggfunctions/SumAggFunction.scala         |  89 ++++----
 .../SumWithRetractAggFunction.scala           | 107 +++++----
 .../runtime/aggregate/AggregateUtil.scala     |  96 +++++---
 .../aggregate/GeneratedAggregations.scala     |  32 +++
 .../aggfunctions/AggFunctionTestBase.scala    |  62 ++++--
 .../aggfunctions/AvgFunctionTest.scala        |  23 +-
 .../aggfunctions/CountAggFunctionTest.scala   |   6 +-
 .../aggfunctions/MaxAggFunctionTest.scala     |  44 ++--
 .../MaxWithRetractAggFunctionTest.scala       |  47 ++--
 .../aggfunctions/MinAggFunctionTest.scala     |  45 ++--
 .../MinWithRetractAggFunctionTest.scala       |  47 ++--
 .../aggfunctions/SumAggFunctionTest.scala     |  31 +--
 .../SumWithRetractAggFunctionTest.scala       |  31 ++-
 ...ocessingOverRangeProcessFunctionTest.scala |  28 ++-
 23 files changed, 772 insertions(+), 649 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
index 262a452685ca5..d82b9907efccb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
@@ -17,9 +17,8 @@
  */
 package org.apache.flink.table.api
 
-import org.apache.flink.api.common.typeinfo.{Types, TypeInformation}
+import org.apache.flink.api.common.typeinfo.{Types => JTypes, TypeInformation}
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
-import org.apache.flink.api.common.typeinfo.{Types => JTypes}
 
 /**
   * This class enumerates all supported types of the Table API.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 510a87069a1fc..298fb70d742fd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -265,14 +265,16 @@ class CodeGenerator(
       name: String,
       generator: CodeGenerator,
       inputType: RelDataType,
-      aggregates: Array[AggregateFunction[_ <: Any]],
+      aggregates: Array[AggregateFunction[_ <: Any, _ <: Any]],
       aggFields: Array[Array[Int]],
       aggMapping: Array[Int],
       partialResults: Boolean,
       fwdMapping: Array[Int],
       mergeMapping: Option[Array[Int]],
       constantFlags: Option[Array[(Int, Boolean)]],
-      outputArity: Int)
+      outputArity: Int,
+      needRetract: Boolean,
+      needMerge: Boolean)
   : GeneratedAggregationsFunction = {
 
     // get unique function name
@@ -364,9 +366,16 @@ class CodeGenerator(
              |      ${parameters(i)});""".stripMargin
       }.mkString("\n")
 
-      j"""$sig {
-         |$retract
-         |  }""".stripMargin
+      if (needRetract) {
+        j"""
+           |$sig {
+           |$retract
+           |  }""".stripMargin
+      } else {
+        j"""
+           |$sig {
+           |  }""".stripMargin
+      }
     }
 
     def genCreateAccumulators: String = {
@@ -471,11 +480,9 @@ class CodeGenerator(
           j"""
              |    ${accTypes(i)} aAcc$i = (${accTypes(i)}) a.getField($i);
              |    ${accTypes(i)} bAcc$i = (${accTypes(i)}) b.getField(${mapping(i)});
-             |    accList$i.set(0, aAcc$i);
-             |    accList$i.set(1, bAcc$i);
-             |    a.setField(
-             |      $i,
-             |      ${aggs(i)}.merge(accList$i));
+             |    accIt$i.setElement(bAcc$i);
+             |    ${aggs(i)}.merge(aAcc$i, accIt$i);
+             |    a.setField($i, aAcc$i);
              """.stripMargin
       }.mkString("\n")
       val ret: String =
@@ -483,29 +490,27 @@ class CodeGenerator(
            |      return a;
            """.stripMargin
 
-      j"""
-         |$sig {
-         |$merge
-         |$ret
-         |  }""".stripMargin
+      if (needMerge) {
+        j"""
+           |$sig {
+           |$merge
+           |$ret
+           |  }""".stripMargin
+      } else {
+        j"""
+           |$sig {
+           |$ret
+           |  }""".stripMargin
+      }
     }
 
     def genMergeList: String = {
       {
+        val singleIterableClass = "org.apache.flink.table.runtime.aggregate.SingleElementIterable"
         for (i <- accTypes.indices) yield
           j"""
-             |    private final java.util.ArrayList<${accTypes(i)}> accList$i =
-             |      new java.util.ArrayList<${accTypes(i)}>(2);
-             """.stripMargin
-      }.mkString("\n")
-    }
-
-    def initMergeList: String = {
-      {
-        for (i <- accTypes.indices) yield
-          j"""
-             |    accList$i.add(${aggs(i)}.createAccumulator());
-             |    accList$i.add(${aggs(i)}.createAccumulator());
+             |    private final $singleIterableClass<${accTypes(i)}> accIt$i =
+             |      new $singleIterableClass<${accTypes(i)}>();
              """.stripMargin
       }.mkString("\n")
     }
@@ -538,7 +543,6 @@ class CodeGenerator(
          |  $genMergeList
          |  public $funcName() throws Exception {
          |    ${reuseInitCode()}
-         |    $initMergeList
          |  }
          |  ${reuseConstructorCode(funcName)}
          |
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala
index a67ccaae6e21a..7a74112dbc6a8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala
@@ -17,36 +17,100 @@
  */
 package org.apache.flink.table.functions
 
-import java.util.{List => JList}
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.table.api.TableException
-
 /**
   * Base class for User-Defined Aggregates.
   *
-  * @tparam T the type of the aggregation result
+  * The behavior of an [[AggregateFunction]] can be defined by implementing a series of custom
+  * methods. An [[AggregateFunction]] needs at least three methods:
+  *  - createAccumulator,
+  *  - accumulate, and
+  *  - getValue.
+  *
+  *  There are a few other methods that can be optional to have:
+  *  - retract,
+  *  - merge,
+  *  - resetAccumulator, and
+  *  - getAccumulatorType.
+  *
+  * All these methods muse be declared publicly, not static and named exactly as the names
+  * mentioned above. The methods createAccumulator and getValue are defined in the
+  * [[AggregateFunction]] functions, while other methods are explained below.
+  *
+  *
+  * {{{
+  * Processes the input values and update the provided accumulator instance. The method
+  * accumulate can be overloaded with different custom types and arguments. An AggregateFunction
+  * requires at least one accumulate() method.
+  *
+  * @param accumulator           the accumulator which contains the current aggregated results
+  * @param [user defined inputs] the input value (usually obtained from a new arrived data).
+  *
+  * def accumulate(accumulator: ACC, [user defined inputs]): Unit
+  * }}}
+  *
+  *
+  * {{{
+  * Retracts the input values from the accumulator instance. The current design assumes the
+  * inputs are the values that have been previously accumulated. The method retract can be
+  * overloaded with different custom types and arguments. This function must be implemented for
+  * datastream bounded over aggregate.
+  *
+  * @param accumulator           the accumulator which contains the current aggregated results
+  * @param [user defined inputs] the input value (usually obtained from a new arrived data).
+  *
+  * def retract(accumulator: ACC, [user defined inputs]): Unit
+  * }}}
+  *
+  *
+  * {{{
+  * Merges a group of accumulator instances into one accumulator instance. This function must be
+  * implemented for datastream session window grouping aggregate and dataset grouping aggregate.
+  *
+  * @param accumulator  the accumulator which will keep the merged aggregate results. It should
+  *                     be noted that the accumulator may contain the previous aggregated
+  *                     results. Therefore user should not replace or clean this instance in the
+  *                     custom merge method.
+  * @param its          an [[java.lang.Iterable]] pointed to a group of accumulators that will be
+  *                     merged.
+
+  * def merge(accumulator: ACC, its: java.lang.Iterable[ACC]): Unit
+  * }}}
+  *
+  *
+  * {{{
+  * Resets the accumulator for this [[AggregateFunction]]. This function must be implemented for
+  * dataset grouping aggregate.
+  *
+  * @param accumulator  the accumulator which needs to be reset
+
+  * def resetAccumulator(accumulator: ACC): Unit
+  * }}}
+  *
+  *
+  * {{{
+  * Returns the [[org.apache.flink.api.common.typeinfo.TypeInformation]] of the accumulator. This
+  * function is optional and can be implemented if the accumulator type cannot automatically
+  * inferred from the instance returned by createAccumulator method.
+  *
+  * @return  the type information for the accumulator.
+
+  * def getAccumulatorType: TypeInformation[_]
+  * }}}
+  *
+  *
+  * @tparam T   the type of the aggregation result
+  * @tparam ACC base class for aggregate Accumulator. The accumulator is used to keep the aggregated
+  *             values which are needed to compute an aggregation result. AggregateFunction
+  *             represents its state using accumulator, thereby the state of the AggregateFunction
+  *             must be put into the accumulator.
   */
-abstract class AggregateFunction[T] extends UserDefinedFunction {
+abstract class AggregateFunction[T, ACC] extends UserDefinedFunction {
   /**
     * Creates and init the Accumulator for this [[AggregateFunction]].
     *
     * @return the accumulator with the initial value
     */
-  def createAccumulator(): Accumulator
-
-  /**
-    * Retracts the input values from the accumulator instance. The current design assumes the
-    * inputs are the values that have been previously accumulated.
-    *
-    * @param accumulator the accumulator which contains the current
-    *                    aggregated results
-    * @param input       the input value (usually obtained from a new arrived data)
-    */
-  def retract(accumulator: Accumulator, input: Any): Unit = {
-    throw TableException("Retract is an optional method. There is no default implementation. You " +
-                           "must implement one for yourself.")
-  }
+  def createAccumulator(): ACC
 
   /**
     * Called every time when an aggregation result should be materialized.
@@ -58,54 +122,5 @@ abstract class AggregateFunction[T] extends UserDefinedFunction {
     *                    aggregated results
     * @return the aggregation result
     */
-  def getValue(accumulator: Accumulator): T
-
-  /**
-    * Processes the input values and update the provided accumulator instance.
-    *
-    * @param accumulator the accumulator which contains the current
-    *                    aggregated results
-    * @param input       the input value (usually obtained from a new arrived data)
-    */
-  def accumulate(accumulator: Accumulator, input: Any): Unit
-
-  /**
-    * Merges a list of accumulator instances into one accumulator instance.
-    *
-    * IMPORTANT: You may only return a new accumulator instance or the first accumulator of the
-    * input list. If you return another instance, the result of the aggregation function might be
-    * incorrect.
-    *
-    * @param accumulators the [[java.util.List]] of accumulators that will be merged
-    * @return the resulting accumulator
-    */
-  def merge(accumulators: JList[Accumulator]): Accumulator
-
-  /**
-    * Resets the Accumulator for this [[AggregateFunction]].
-    *
-    * @param accumulator the accumulator which needs to be reset
-    */
-  def resetAccumulator(accumulator: Accumulator): Unit
-
-  /**
-    * Returns the [[TypeInformation]] of the accumulator.
-    * This function is optional and can be implemented if the accumulator type cannot automatically
-    * inferred from the instance returned by [[createAccumulator()]].
-    *
-    * @return The type information for the accumulator.
-    */
-  def getAccumulatorType: TypeInformation[_] = null
+  def getValue(accumulator: ACC): T
 }
-
-/**
-  * Base class for aggregate Accumulator. The accumulator is used to keep the
-  * aggregated values which are needed to compute an aggregation result.
-  * The state of the function must be put into the accumulator.
-  *
-  * TODO: We have the plan to have the accumulator and return types of
-  * functions dynamically provided by the users. This needs the refactoring
-  * of the AggregateFunction interface with the code generation. We will remove
-  * the [[Accumulator]] once codeGen for UDAGG is completed (FLINK-5813).
-  */
-trait Accumulator
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala
index 4837139a76b0c..3f4e5db8a7637 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala
@@ -18,15 +18,15 @@
 package org.apache.flink.table.functions.aggfunctions
 
 import java.math.{BigDecimal, BigInteger}
-import java.util.{List => JList}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for Integral Avg aggregate function */
-class IntegralAvgAccumulator extends JTuple2[Long, Long] with Accumulator {
+class IntegralAvgAccumulator extends JTuple2[Long, Long] {
   f0 = 0L //sum
   f1 = 0L //count
 }
@@ -36,57 +36,51 @@ class IntegralAvgAccumulator extends JTuple2[Long, Long] with Accumulator {
   *
   * @tparam T the type for the aggregation result
   */
-abstract class IntegralAvgAggFunction[T] extends AggregateFunction[T] {
+abstract class IntegralAvgAggFunction[T] extends AggregateFunction[T, IntegralAvgAccumulator] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): IntegralAvgAccumulator = {
     new IntegralAvgAccumulator
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: IntegralAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[Number].longValue()
-      val accum = accumulator.asInstanceOf[IntegralAvgAccumulator]
-      accum.f0 += v
-      accum.f1 += 1L
+      acc.f0 += v
+      acc.f1 += 1L
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: IntegralAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[Number].longValue()
-      val accum = accumulator.asInstanceOf[IntegralAvgAccumulator]
-      accum.f0 -= v
-      accum.f1 -= 1L
+      acc.f0 -= v
+      acc.f1 -= 1L
     }
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val accum = accumulator.asInstanceOf[IntegralAvgAccumulator]
-    if (accum.f1 == 0) {
+  override def getValue(acc: IntegralAvgAccumulator): T = {
+    if (acc.f1 == 0) {
       null.asInstanceOf[T]
     } else {
-      resultTypeConvert(accum.f0 / accum.f1)
+      resultTypeConvert(acc.f0 / acc.f1)
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[IntegralAvgAccumulator]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[IntegralAvgAccumulator]
-      ret.f1 += a.f1
-      ret.f0 += a.f0
-      i += 1
+  def merge(acc: IntegralAvgAccumulator, its: JIterable[IntegralAvgAccumulator]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
+      acc.f1 += a.f1
+      acc.f0 += a.f0
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[IntegralAvgAccumulator].f0 = 0L
-    accumulator.asInstanceOf[IntegralAvgAccumulator].f1 = 0L
+  def resetAccumulator(acc: IntegralAvgAccumulator): Unit = {
+    acc.f0 = 0L
+    acc.f1 = 0L
   }
 
-  override def getAccumulatorType: TypeInformation[_] = {
+  def getAccumulatorType: TypeInformation[_] = {
     new TupleTypeInfo(
       new IntegralAvgAccumulator().getClass,
       BasicTypeInfo.LONG_TYPE_INFO,
@@ -126,7 +120,7 @@ class IntAvgAggFunction extends IntegralAvgAggFunction[Int] {
 
 /** The initial accumulator for Big Integral Avg aggregate function */
 class BigIntegralAvgAccumulator
-  extends JTuple2[BigInteger, Long] with Accumulator {
+  extends JTuple2[BigInteger, Long] {
   f0 = BigInteger.ZERO //sum
   f1 = 0L //count
 }
@@ -136,57 +130,52 @@ class BigIntegralAvgAccumulator
   *
   * @tparam T the type for the aggregation result
   */
-abstract class BigIntegralAvgAggFunction[T] extends AggregateFunction[T] {
+abstract class BigIntegralAvgAggFunction[T]
+  extends AggregateFunction[T, BigIntegralAvgAccumulator] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): BigIntegralAvgAccumulator = {
     new BigIntegralAvgAccumulator
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: BigIntegralAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[Long]
-      val a = accumulator.asInstanceOf[BigIntegralAvgAccumulator]
-      a.f0 = a.f0.add(BigInteger.valueOf(v))
-      a.f1 += 1L
+      acc.f0 = acc.f0.add(BigInteger.valueOf(v))
+      acc.f1 += 1L
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: BigIntegralAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[Long]
-      val a = accumulator.asInstanceOf[BigIntegralAvgAccumulator]
-      a.f0 = a.f0.subtract(BigInteger.valueOf(v))
-      a.f1 -= 1L
+      acc.f0 = acc.f0.subtract(BigInteger.valueOf(v))
+      acc.f1 -= 1L
     }
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[BigIntegralAvgAccumulator]
-    if (a.f1 == 0) {
+  override def getValue(acc: BigIntegralAvgAccumulator): T = {
+    if (acc.f1 == 0) {
       null.asInstanceOf[T]
     } else {
-      resultTypeConvert(a.f0.divide(BigInteger.valueOf(a.f1)))
+      resultTypeConvert(acc.f0.divide(BigInteger.valueOf(acc.f1)))
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[BigIntegralAvgAccumulator]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[BigIntegralAvgAccumulator]
-      ret.f1 += a.f1
-      ret.f0 = ret.f0.add(a.f0)
-      i += 1
+  def merge(acc: BigIntegralAvgAccumulator, its: JIterable[BigIntegralAvgAccumulator]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
+      acc.f1 += a.f1
+      acc.f0 = acc.f0.add(a.f0)
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[BigIntegralAvgAccumulator].f0 = BigInteger.ZERO
-    accumulator.asInstanceOf[BigIntegralAvgAccumulator].f1 = 0
+  def resetAccumulator(acc: BigIntegralAvgAccumulator): Unit = {
+    acc.f0 = BigInteger.ZERO
+    acc.f1 = 0
   }
 
-  override def getAccumulatorType: TypeInformation[_] = {
+  def getAccumulatorType: TypeInformation[_] = {
     new TupleTypeInfo(
       new BigIntegralAvgAccumulator().getClass,
       BasicTypeInfo.BIG_INT_TYPE_INFO,
@@ -212,7 +201,7 @@ class LongAvgAggFunction extends BigIntegralAvgAggFunction[Long] {
 }
 
 /** The initial accumulator for Floating Avg aggregate function */
-class FloatingAvgAccumulator extends JTuple2[Double, Long] with Accumulator {
+class FloatingAvgAccumulator extends JTuple2[Double, Long] {
   f0 = 0 //sum
   f1 = 0L //count
 }
@@ -222,57 +211,51 @@ class FloatingAvgAccumulator extends JTuple2[Double, Long] with Accumulator {
   *
   * @tparam T the type for the aggregation result
   */
-abstract class FloatingAvgAggFunction[T] extends AggregateFunction[T] {
+abstract class FloatingAvgAggFunction[T] extends AggregateFunction[T, FloatingAvgAccumulator] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): FloatingAvgAccumulator = {
     new FloatingAvgAccumulator
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: FloatingAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[Number].doubleValue()
-      val accum = accumulator.asInstanceOf[FloatingAvgAccumulator]
-      accum.f0 += v
-      accum.f1 += 1L
+      acc.f0 += v
+      acc.f1 += 1L
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: FloatingAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[Number].doubleValue()
-      val accum = accumulator.asInstanceOf[FloatingAvgAccumulator]
-      accum.f0 -= v
-      accum.f1 -= 1L
+      acc.f0 -= v
+      acc.f1 -= 1L
     }
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val accum = accumulator.asInstanceOf[FloatingAvgAccumulator]
-    if (accum.f1 == 0) {
+  override def getValue(acc: FloatingAvgAccumulator): T = {
+    if (acc.f1 == 0) {
       null.asInstanceOf[T]
     } else {
-      resultTypeConvert(accum.f0 / accum.f1)
+      resultTypeConvert(acc.f0 / acc.f1)
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[FloatingAvgAccumulator]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[FloatingAvgAccumulator]
-      ret.f1 += a.f1
-      ret.f0 += a.f0
-      i += 1
+  def merge(acc: FloatingAvgAccumulator, its: JIterable[FloatingAvgAccumulator]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
+      acc.f1 += a.f1
+      acc.f0 += a.f0
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[FloatingAvgAccumulator].f0 = 0
-    accumulator.asInstanceOf[FloatingAvgAccumulator].f1 = 0L
+  def resetAccumulator(acc: FloatingAvgAccumulator): Unit = {
+    acc.f0 = 0
+    acc.f1 = 0L
   }
 
-  override def getAccumulatorType: TypeInformation[_] = {
+  def getAccumulatorType: TypeInformation[_] = {
     new TupleTypeInfo(
       new FloatingAvgAccumulator().getClass,
       BasicTypeInfo.DOUBLE_TYPE_INFO,
@@ -304,8 +287,7 @@ class DoubleAvgAggFunction extends FloatingAvgAggFunction[Double] {
 }
 
 /** The initial accumulator for Big Decimal Avg aggregate function */
-class DecimalAvgAccumulator
-  extends JTuple2[BigDecimal, Long] with Accumulator {
+class DecimalAvgAccumulator extends JTuple2[BigDecimal, Long] {
   f0 = BigDecimal.ZERO //sum
   f1 = 0L //count
 }
@@ -313,57 +295,51 @@ class DecimalAvgAccumulator
 /**
   * Base class for built-in Big Decimal Avg aggregate function
   */
-class DecimalAvgAggFunction extends AggregateFunction[BigDecimal] {
+class DecimalAvgAggFunction extends AggregateFunction[BigDecimal, DecimalAvgAccumulator] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): DecimalAvgAccumulator = {
     new DecimalAvgAccumulator
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: DecimalAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[BigDecimal]
-      val accum = accumulator.asInstanceOf[DecimalAvgAccumulator]
-      accum.f0 = accum.f0.add(v)
-      accum.f1 += 1L
+      acc.f0 = acc.f0.add(v)
+      acc.f1 += 1L
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: DecimalAvgAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[BigDecimal]
-      val accum = accumulator.asInstanceOf[DecimalAvgAccumulator]
-      accum.f0 = accum.f0.subtract(v)
-      accum.f1 -= 1L
+      acc.f0 = acc.f0.subtract(v)
+      acc.f1 -= 1L
     }
   }
 
-  override def getValue(accumulator: Accumulator): BigDecimal = {
-    val a = accumulator.asInstanceOf[DecimalAvgAccumulator]
-    if (a.f1 == 0) {
+  override def getValue(acc: DecimalAvgAccumulator): BigDecimal = {
+    if (acc.f1 == 0) {
       null.asInstanceOf[BigDecimal]
     } else {
-      a.f0.divide(BigDecimal.valueOf(a.f1))
+      acc.f0.divide(BigDecimal.valueOf(acc.f1))
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[DecimalAvgAccumulator]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[DecimalAvgAccumulator]
-      ret.f0 = ret.f0.add(a.f0)
-      ret.f1 += a.f1
-      i += 1
+  def merge(acc: DecimalAvgAccumulator, its: JIterable[DecimalAvgAccumulator]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
+      acc.f0 = acc.f0.add(a.f0)
+      acc.f1 += a.f1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[DecimalAvgAccumulator].f0 = BigDecimal.ZERO
-    accumulator.asInstanceOf[DecimalAvgAccumulator].f1 = 0L
+  def resetAccumulator(acc: DecimalAvgAccumulator): Unit = {
+    acc.f0 = BigDecimal.ZERO
+    acc.f1 = 0L
   }
 
-  override def getAccumulatorType: TypeInformation[_] = {
+  def getAccumulatorType: TypeInformation[_] = {
     new TupleTypeInfo(
       new DecimalAvgAccumulator().getClass,
       BasicTypeInfo.BIG_DEC_TYPE_INFO,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala
index 231337a117a51..77341cd761583 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala
@@ -17,58 +17,55 @@
  */
 package org.apache.flink.table.functions.aggfunctions
 
-import java.util.{List => JList}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple1 => JTuple1}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for count aggregate function */
-class CountAccumulator extends JTuple1[Long] with Accumulator {
+class CountAccumulator extends JTuple1[Long] {
   f0 = 0L //count
 }
 
 /**
   * built-in count aggregate function
   */
-class CountAggFunction extends AggregateFunction[Long] {
+class CountAggFunction extends AggregateFunction[Long, CountAccumulator] {
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: CountAccumulator, value: Any): Unit = {
     if (value != null) {
-      accumulator.asInstanceOf[CountAccumulator].f0 += 1L
+      acc.f0 += 1L
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: CountAccumulator, value: Any): Unit = {
     if (value != null) {
-      accumulator.asInstanceOf[CountAccumulator].f0 -= 1L
+      acc.f0 -= 1L
     }
   }
 
-  override def getValue(accumulator: Accumulator): Long = {
-    accumulator.asInstanceOf[CountAccumulator].f0
+  override def getValue(acc: CountAccumulator): Long = {
+    acc.f0
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[CountAccumulator]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      ret.f0 += accumulators.get(i).asInstanceOf[CountAccumulator].f0
-      i += 1
+  def merge(acc: CountAccumulator, its: JIterable[CountAccumulator]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      acc.f0 += iter.next().f0
     }
-    ret
   }
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): CountAccumulator = {
     new CountAccumulator
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[CountAccumulator].f0 = 0L
+  def resetAccumulator(acc: CountAccumulator): Unit = {
+    acc.f0 = 0L
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo((new CountAccumulator).getClass, BasicTypeInfo.LONG_TYPE_INFO)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
index 2e666fa0ec926..96ee8d18a1044 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
@@ -18,69 +18,65 @@
 package org.apache.flink.table.functions.aggfunctions
 
 import java.math.BigDecimal
-import java.util.{List => JList}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for Max aggregate function */
-class MaxAccumulator[T] extends JTuple2[T, Boolean] with Accumulator
+class MaxAccumulator[T] extends JTuple2[T, Boolean]
 
 /**
   * Base class for built-in Max aggregate function
   *
   * @tparam T the type for the aggregation result
   */
-abstract class MaxAggFunction[T](implicit ord: Ordering[T]) extends AggregateFunction[T] {
+abstract class MaxAggFunction[T](implicit ord: Ordering[T])
+  extends AggregateFunction[T, MaxAccumulator[T]] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): MaxAccumulator[T] = {
     val acc = new MaxAccumulator[T]
     acc.f0 = getInitValue
     acc.f1 = false
     acc
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: MaxAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[MaxAccumulator[T]]
-      if (!a.f1 || ord.compare(a.f0, v) < 0) {
-        a.f0 = v
-        a.f1 = true
+      if (!acc.f1 || ord.compare(acc.f0, v) < 0) {
+        acc.f0 = v
+        acc.f1 = true
       }
     }
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[MaxAccumulator[T]]
-    if (a.f1) {
-      a.f0
+  override def getValue(acc: MaxAccumulator[T]): T = {
+    if (acc.f1) {
+      acc.f0
     } else {
       null.asInstanceOf[T]
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0)
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[MaxAccumulator[T]]
+  def merge(acc: MaxAccumulator[T], its: JIterable[MaxAccumulator[T]]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
       if (a.f1) {
-        accumulate(ret.asInstanceOf[MaxAccumulator[T]], a.f0)
+        accumulate(acc, a.f0)
       }
-      i += 1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[MaxAccumulator[T]].f0 = getInitValue
-    accumulator.asInstanceOf[MaxAccumulator[T]].f1 = false
+  def resetAccumulator(acc: MaxAccumulator[T]): Unit = {
+    acc.f0 = getInitValue
+    acc.f1 = false
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       new MaxAccumulator[T].getClass,
       getValueTypeInfo,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
index 14ceba2303674..6f18739f38bdd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
@@ -18,15 +18,16 @@
 package org.apache.flink.table.functions.aggfunctions
 
 import java.math.BigDecimal
-import java.util.{HashMap => JHashMap, List => JList}
+import java.util.{HashMap => JHashMap}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.{MapTypeInfo, TupleTypeInfo}
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for Max with retraction aggregate function */
-class MaxWithRetractAccumulator[T] extends JTuple2[T, JHashMap[T, Long]] with Accumulator
+class MaxWithRetractAccumulator[T] extends JTuple2[T, JHashMap[T, Long]]
 
 /**
   * Base class for built-in Max with retraction aggregate function
@@ -34,110 +35,105 @@ class MaxWithRetractAccumulator[T] extends JTuple2[T, JHashMap[T, Long]] with Ac
   * @tparam T the type for the aggregation result
   */
 abstract class MaxWithRetractAggFunction[T](implicit ord: Ordering[T])
-  extends AggregateFunction[T] {
+  extends AggregateFunction[T, MaxWithRetractAccumulator[T]] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): MaxWithRetractAccumulator[T] = {
     val acc = new MaxWithRetractAccumulator[T]
     acc.f0 = getInitValue //max
     acc.f1 = new JHashMap[T, Long]() //store the count for each value
     acc
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: MaxWithRetractAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[MaxWithRetractAccumulator[T]]
 
-      if (a.f1.size() == 0 || (ord.compare(a.f0, v) < 0)) {
-        a.f0 = v
+      if (acc.f1.size() == 0 || (ord.compare(acc.f0, v) < 0)) {
+        acc.f0 = v
       }
 
-      if (!a.f1.containsKey(v)) {
-        a.f1.put(v, 1L)
+      if (!acc.f1.containsKey(v)) {
+        acc.f1.put(v, 1L)
       } else {
-        var count = a.f1.get(v)
+        var count = acc.f1.get(v)
         count += 1L
-        a.f1.put(v, count)
+        acc.f1.put(v, count)
       }
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: MaxWithRetractAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[MaxWithRetractAccumulator[T]]
 
-      var count = a.f1.get(v)
+      var count = acc.f1.get(v)
       count -= 1L
       if (count == 0) {
         //remove the key v from the map if the number of appearance of the value v is 0
-        a.f1.remove(v)
+        acc.f1.remove(v)
         //if the total count is 0, we could just simply set the f0(max) to the initial value
-        if (a.f1.size() == 0) {
-          a.f0 = getInitValue
+        if (acc.f1.size() == 0) {
+          acc.f0 = getInitValue
           return
         }
         //if v is the current max value, we have to iterate the map to find the 2nd biggest
         // value to replace v as the max value
-        if (v == a.f0) {
-          val iterator = a.f1.keySet().iterator()
+        if (v == acc.f0) {
+          val iterator = acc.f1.keySet().iterator()
           var key = iterator.next()
-          a.f0 = key
+          acc.f0 = key
           while (iterator.hasNext()) {
             key = iterator.next()
-            if (ord.compare(a.f0, key) < 0) {
-              a.f0 = key
+            if (ord.compare(acc.f0, key) < 0) {
+              acc.f0 = key
             }
           }
         }
       } else {
-        a.f1.put(v, count)
+        acc.f1.put(v, count)
       }
     }
 
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[MaxWithRetractAccumulator[T]]
-    if (a.f1.size() != 0) {
-      a.f0
+  override def getValue(acc: MaxWithRetractAccumulator[T]): T = {
+    if (acc.f1.size() != 0) {
+      acc.f0
     } else {
       null.asInstanceOf[T]
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[MaxWithRetractAccumulator[T]]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[MaxWithRetractAccumulator[T]]
+  def merge(acc: MaxWithRetractAccumulator[T],
+      its: JIterable[MaxWithRetractAccumulator[T]]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
       if (a.f1.size() != 0) {
         // set max element
-        if (ord.compare(ret.f0, a.f0) < 0) {
-          ret.f0 = a.f0
+        if (ord.compare(acc.f0, a.f0) < 0) {
+          acc.f0 = a.f0
         }
         // merge the count for each key
         val iterator = a.f1.keySet().iterator()
         while (iterator.hasNext()) {
           val key = iterator.next()
-          if (ret.f1.containsKey(key)) {
-            ret.f1.put(key, ret.f1.get(key) + a.f1.get(key))
+          if (acc.f1.containsKey(key)) {
+            acc.f1.put(key, acc.f1.get(key) + a.f1.get(key))
           } else {
-            ret.f1.put(key, a.f1.get(key))
+            acc.f1.put(key, a.f1.get(key))
           }
         }
       }
-      i += 1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[MaxWithRetractAccumulator[T]].f0 = getInitValue
-    accumulator.asInstanceOf[MaxWithRetractAccumulator[T]].f1.clear()
+  def resetAccumulator(acc: MaxWithRetractAccumulator[T]): Unit = {
+    acc.f0 = getInitValue
+    acc.f1.clear()
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       new MaxWithRetractAccumulator[T].getClass,
       getValueTypeInfo,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala
index 75a8ebce07945..88d7afd20c8d2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala
@@ -18,69 +18,65 @@
 package org.apache.flink.table.functions.aggfunctions
 
 import java.math.BigDecimal
-import java.util.{List => JList}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for Min aggregate function */
-class MinAccumulator[T] extends JTuple2[T, Boolean] with Accumulator
+class MinAccumulator[T] extends JTuple2[T, Boolean]
 
 /**
   * Base class for built-in Min aggregate function
   *
   * @tparam T the type for the aggregation result
   */
-abstract class MinAggFunction[T](implicit ord: Ordering[T]) extends AggregateFunction[T] {
+abstract class MinAggFunction[T](implicit ord: Ordering[T])
+  extends AggregateFunction[T, MinAccumulator[T]] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): MinAccumulator[T] = {
     val acc = new MinAccumulator[T]
     acc.f0 = getInitValue
     acc.f1 = false
     acc
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: MinAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[MinAccumulator[T]]
-      if (!a.f1 || ord.compare(a.f0, v) > 0) {
-        a.f0 = v
-        a.f1 = true
+      if (!acc.f1 || ord.compare(acc.f0, v) > 0) {
+        acc.f0 = v
+        acc.f1 = true
       }
     }
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[MinAccumulator[T]]
-    if (a.f1) {
-      a.f0
+  override def getValue(acc: MinAccumulator[T]): T = {
+    if (acc.f1) {
+      acc.f0
     } else {
       null.asInstanceOf[T]
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0)
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[MinAccumulator[T]]
+  def merge(acc: MinAccumulator[T], its: JIterable[MinAccumulator[T]]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
       if (a.f1) {
-        accumulate(ret.asInstanceOf[MinAccumulator[T]], a.f0)
+        accumulate(acc, a.f0)
       }
-      i += 1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[MinAccumulator[T]].f0 = getInitValue
-    accumulator.asInstanceOf[MinAccumulator[T]].f1 = false
+  def resetAccumulator(acc: MinAccumulator[T]): Unit = {
+    acc.f0 = getInitValue
+    acc.f1 = false
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       new MinAccumulator[T].getClass,
       getValueTypeInfo,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionWithRetract.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionWithRetract.scala
index 6f2c3a163538d..2d3348b2bc8b0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionWithRetract.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionWithRetract.scala
@@ -18,15 +18,16 @@
 package org.apache.flink.table.functions.aggfunctions
 
 import java.math.BigDecimal
-import java.util.{HashMap => JHashMap, List => JList}
+import java.util.{HashMap => JHashMap}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.{MapTypeInfo, TupleTypeInfo}
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for Min with retraction aggregate function */
-class MinWithRetractAccumulator[T] extends JTuple2[T, JHashMap[T, Long]] with Accumulator
+class MinWithRetractAccumulator[T] extends JTuple2[T, JHashMap[T, Long]]
 
 /**
   * Base class for built-in Min with retraction aggregate function
@@ -34,110 +35,105 @@ class MinWithRetractAccumulator[T] extends JTuple2[T, JHashMap[T, Long]] with Ac
   * @tparam T the type for the aggregation result
   */
 abstract class MinWithRetractAggFunction[T](implicit ord: Ordering[T])
-  extends AggregateFunction[T] {
+  extends AggregateFunction[T, MinWithRetractAccumulator[T]] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): MinWithRetractAccumulator[T] = {
     val acc = new MinWithRetractAccumulator[T]
     acc.f0 = getInitValue //min
     acc.f1 = new JHashMap[T, Long]() //store the count for each value
     acc
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: MinWithRetractAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[MinWithRetractAccumulator[T]]
 
-      if (a.f1.size() == 0 || (ord.compare(a.f0, v) > 0)) {
-        a.f0 = v
+      if (acc.f1.size() == 0 || (ord.compare(acc.f0, v) > 0)) {
+        acc.f0 = v
       }
 
-      if (!a.f1.containsKey(v)) {
-        a.f1.put(v, 1L)
+      if (!acc.f1.containsKey(v)) {
+        acc.f1.put(v, 1L)
       } else {
-        var count = a.f1.get(v)
+        var count = acc.f1.get(v)
         count += 1L
-        a.f1.put(v, count)
+        acc.f1.put(v, count)
       }
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: MinWithRetractAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[MinWithRetractAccumulator[T]]
 
-      var count = a.f1.get(v)
+      var count = acc.f1.get(v)
       count -= 1L
       if (count == 0) {
         //remove the key v from the map if the number of appearance of the value v is 0
-        a.f1.remove(v)
+        acc.f1.remove(v)
         //if the total count is 0, we could just simply set the f0(min) to the initial value
-        if (a.f1.size() == 0) {
-          a.f0 = getInitValue
+        if (acc.f1.size() == 0) {
+          acc.f0 = getInitValue
           return
         }
         //if v is the current min value, we have to iterate the map to find the 2nd smallest
         // value to replace v as the min value
-        if (v == a.f0) {
-          val iterator = a.f1.keySet().iterator()
+        if (v == acc.f0) {
+          val iterator = acc.f1.keySet().iterator()
           var key = iterator.next()
-          a.f0 = key
+          acc.f0 = key
           while (iterator.hasNext()) {
             key = iterator.next()
-            if (ord.compare(a.f0, key) > 0) {
-              a.f0 = key
+            if (ord.compare(acc.f0, key) > 0) {
+              acc.f0 = key
             }
           }
         }
       } else {
-        a.f1.put(v, count)
+        acc.f1.put(v, count)
       }
     }
 
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[MinWithRetractAccumulator[T]]
-    if (a.f1.size() != 0) {
-      a.f0
+  override def getValue(acc: MinWithRetractAccumulator[T]): T = {
+    if (acc.f1.size() != 0) {
+      acc.f0
     } else {
       null.asInstanceOf[T]
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[MinWithRetractAccumulator[T]]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[MinWithRetractAccumulator[T]]
+  def merge(acc: MinWithRetractAccumulator[T],
+      its: JIterable[MinWithRetractAccumulator[T]]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
       if (a.f1.size() != 0) {
         // set min element
-        if (ord.compare(ret.f0, a.f0) > 0) {
-          ret.f0 = a.f0
+        if (ord.compare(acc.f0, a.f0) > 0) {
+          acc.f0 = a.f0
         }
         // merge the count for each key
         val iterator = a.f1.keySet().iterator()
         while (iterator.hasNext()) {
           val key = iterator.next()
-          if (ret.f1.containsKey(key)) {
-            ret.f1.put(key, ret.f1.get(key) + a.f1.get(key))
+          if (acc.f1.containsKey(key)) {
+            acc.f1.put(key, acc.f1.get(key) + a.f1.get(key))
           } else {
-            ret.f1.put(key, a.f1.get(key))
+            acc.f1.put(key, a.f1.get(key))
           }
         }
       }
-      i += 1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[MinWithRetractAccumulator[T]].f0 = getInitValue
-    accumulator.asInstanceOf[MinWithRetractAccumulator[T]].f1.clear()
+  def resetAccumulator(acc: MinWithRetractAccumulator[T]): Unit = {
+    acc.f0 = getInitValue
+    acc.f1.clear()
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       new MinWithRetractAccumulator[T].getClass,
       getValueTypeInfo,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala
index 8ee986217e230..55996acaddb63 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala
@@ -18,70 +18,65 @@
 package org.apache.flink.table.functions.aggfunctions
 
 import java.math.BigDecimal
-import java.util.{List => JList}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for Sum aggregate function */
-class SumAccumulator[T] extends JTuple2[T, Boolean] with Accumulator
+class SumAccumulator[T] extends JTuple2[T, Boolean]
 
 /**
   * Base class for built-in Sum aggregate function
   *
   * @tparam T the type for the aggregation result
   */
-abstract class SumAggFunction[T: Numeric] extends AggregateFunction[T] {
+abstract class SumAggFunction[T: Numeric] extends AggregateFunction[T, SumAccumulator[T]] {
 
   private val numeric = implicitly[Numeric[T]]
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): SumAccumulator[T] = {
     val acc = new SumAccumulator[T]()
     acc.f0 = numeric.zero //sum
     acc.f1 = false
     acc
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(accumulator: SumAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[SumAccumulator[T]]
-      a.f0 = numeric.plus(v, a.f0)
-      a.f1 = true
+      accumulator.f0 = numeric.plus(v, accumulator.f0)
+      accumulator.f1 = true
     }
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[SumAccumulator[T]]
-    if (a.f1) {
-      a.f0
+  override def getValue(accumulator: SumAccumulator[T]): T = {
+    if (accumulator.f1) {
+      accumulator.f0
     } else {
       null.asInstanceOf[T]
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[SumAccumulator[T]]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[SumAccumulator[T]]
+  def merge(acc: SumAccumulator[T], its: JIterable[SumAccumulator[T]]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
       if (a.f1) {
-        ret.f0 = numeric.plus(ret.f0, a.f0)
-        ret.f1 = true
+        acc.f0 = numeric.plus(acc.f0, a.f0)
+        acc.f1 = true
       }
-      i += 1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[SumAccumulator[T]].f0 = numeric.zero
-    accumulator.asInstanceOf[SumAccumulator[T]].f1 = false
+  def resetAccumulator(acc: SumAccumulator[T]): Unit = {
+    acc.f0 = numeric.zero
+    acc.f1 = false
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       (new SumAccumulator).getClass,
       getValueTypeInfo,
@@ -134,7 +129,7 @@ class DoubleSumAggFunction extends SumAggFunction[Double] {
 }
 
 /** The initial accumulator for Big Decimal Sum aggregate function */
-class DecimalSumAccumulator extends JTuple2[BigDecimal, Boolean] with Accumulator {
+class DecimalSumAccumulator extends JTuple2[BigDecimal, Boolean] {
   f0 = BigDecimal.ZERO
   f1 = false
 }
@@ -142,49 +137,45 @@ class DecimalSumAccumulator extends JTuple2[BigDecimal, Boolean] with Accumulato
 /**
   * Built-in Big Decimal Sum aggregate function
   */
-class DecimalSumAggFunction extends AggregateFunction[BigDecimal] {
+class DecimalSumAggFunction extends AggregateFunction[BigDecimal, DecimalSumAccumulator] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): DecimalSumAccumulator = {
     new DecimalSumAccumulator
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: DecimalSumAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[BigDecimal]
-      val accum = accumulator.asInstanceOf[DecimalSumAccumulator]
-      accum.f0 = accum.f0.add(v)
-      accum.f1 = true
+      acc.f0 = acc.f0.add(v)
+      acc.f1 = true
     }
   }
 
-  override def getValue(accumulator: Accumulator): BigDecimal = {
-    if (!accumulator.asInstanceOf[DecimalSumAccumulator].f1) {
+  override def getValue(acc: DecimalSumAccumulator): BigDecimal = {
+    if (!acc.f1) {
       null.asInstanceOf[BigDecimal]
     } else {
-      accumulator.asInstanceOf[DecimalSumAccumulator].f0
+      acc.f0
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[DecimalSumAccumulator]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[DecimalSumAccumulator]
+  def merge(acc: DecimalSumAccumulator, its: JIterable[DecimalSumAccumulator]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
       if (a.f1) {
-        ret.f0 = ret.f0.add(a.f0)
-        ret.f1 = true
+        acc.f0 = acc.f0.add(a.f0)
+        acc.f1 = true
       }
-      i += 1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[DecimalSumAccumulator].f0 = BigDecimal.ZERO
-    accumulator.asInstanceOf[DecimalSumAccumulator].f1 = false
+  def resetAccumulator(acc: DecimalSumAccumulator): Unit = {
+    acc.f0 = BigDecimal.ZERO
+    acc.f1 = false
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       (new DecimalSumAccumulator).getClass,
       BasicTypeInfo.BIG_DEC_TYPE_INFO,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunction.scala
index 928be1154ca9d..7f68d117f6f57 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunction.scala
@@ -18,77 +18,73 @@
 package org.apache.flink.table.functions.aggfunctions
 
 import java.math.BigDecimal
-import java.util.{List => JList}
+import java.lang.{Iterable => JIterable}
 
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 
 /** The initial accumulator for Sum with retract aggregate function */
-class SumWithRetractAccumulator[T] extends JTuple2[T, Long] with Accumulator
+class SumWithRetractAccumulator[T] extends JTuple2[T, Long]
 
 /**
   * Base class for built-in Sum with retract aggregate function
   *
   * @tparam T the type for the aggregation result
   */
-abstract class SumWithRetractAggFunction[T: Numeric] extends AggregateFunction[T] {
+abstract class SumWithRetractAggFunction[T: Numeric]
+  extends AggregateFunction[T, SumWithRetractAccumulator[T]] {
 
   private val numeric = implicitly[Numeric[T]]
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): SumWithRetractAccumulator[T] = {
     val acc = new SumWithRetractAccumulator[T]()
     acc.f0 = numeric.zero //sum
     acc.f1 = 0L //total count
     acc
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: SumWithRetractAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[SumWithRetractAccumulator[T]]
-      a.f0 = numeric.plus(a.f0, v)
-      a.f1 += 1
+      acc.f0 = numeric.plus(acc.f0, v)
+      acc.f1 += 1
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: SumWithRetractAccumulator[T], value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[T]
-      val a = accumulator.asInstanceOf[SumWithRetractAccumulator[T]]
-      a.f0 = numeric.minus(a.f0, v)
-      a.f1 -= 1
+      acc.f0 = numeric.minus(acc.f0, v)
+      acc.f1 -= 1
     }
   }
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[SumWithRetractAccumulator[T]]
-    if (a.f1 > 0) {
-      a.f0
+  override def getValue(acc: SumWithRetractAccumulator[T]): T = {
+    if (acc.f1 > 0) {
+      acc.f0
     } else {
       null.asInstanceOf[T]
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[SumWithRetractAccumulator[T]]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[SumWithRetractAccumulator[T]]
-      ret.f0 = numeric.plus(ret.f0, a.f0)
-      ret.f1 += a.f1
-      i += 1
+  def merge(acc: SumWithRetractAccumulator[T],
+      its: JIterable[SumWithRetractAccumulator[T]]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
+      acc.f0 = numeric.plus(acc.f0, a.f0)
+      acc.f1 += a.f1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[SumWithRetractAccumulator[T]].f0 = numeric.zero
-    accumulator.asInstanceOf[SumWithRetractAccumulator[T]].f1 = 0L
+  def resetAccumulator(acc: SumWithRetractAccumulator[T]): Unit = {
+    acc.f0 = numeric.zero
+    acc.f1 = 0L
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       (new SumWithRetractAccumulator).getClass,
       getValueTypeInfo,
@@ -141,7 +137,7 @@ class DoubleSumWithRetractAggFunction extends SumWithRetractAggFunction[Double]
 }
 
 /** The initial accumulator for Big Decimal Sum with retract aggregate function */
-class DecimalSumWithRetractAccumulator extends JTuple2[BigDecimal, Long] with Accumulator {
+class DecimalSumWithRetractAccumulator extends JTuple2[BigDecimal, Long] {
   f0 = BigDecimal.ZERO
   f1 = 0L
 }
@@ -149,56 +145,53 @@ class DecimalSumWithRetractAccumulator extends JTuple2[BigDecimal, Long] with Ac
 /**
   * Built-in Big Decimal Sum with retract aggregate function
   */
-class DecimalSumWithRetractAggFunction extends AggregateFunction[BigDecimal] {
+class DecimalSumWithRetractAggFunction
+  extends AggregateFunction[BigDecimal, DecimalSumWithRetractAccumulator] {
 
-  override def createAccumulator(): Accumulator = {
+  override def createAccumulator(): DecimalSumWithRetractAccumulator = {
     new DecimalSumWithRetractAccumulator
   }
 
-  override def accumulate(accumulator: Accumulator, value: Any): Unit = {
+  def accumulate(acc: DecimalSumWithRetractAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[BigDecimal]
-      val accum = accumulator.asInstanceOf[DecimalSumWithRetractAccumulator]
-      accum.f0 = accum.f0.add(v)
-      accum.f1 += 1L
+      acc.f0 = acc.f0.add(v)
+      acc.f1 += 1L
     }
   }
 
-  override def retract(accumulator: Accumulator, value: Any): Unit = {
+  def retract(acc: DecimalSumWithRetractAccumulator, value: Any): Unit = {
     if (value != null) {
       val v = value.asInstanceOf[BigDecimal]
-      val accum = accumulator.asInstanceOf[DecimalSumWithRetractAccumulator]
-      accum.f0 = accum.f0.subtract(v)
-      accum.f1 -= 1L
+      acc.f0 = acc.f0.subtract(v)
+      acc.f1 -= 1L
     }
   }
 
-  override def getValue(accumulator: Accumulator): BigDecimal = {
-    if (accumulator.asInstanceOf[DecimalSumWithRetractAccumulator].f1 == 0) {
+  override def getValue(acc: DecimalSumWithRetractAccumulator): BigDecimal = {
+    if (acc.f1 == 0) {
       null.asInstanceOf[BigDecimal]
     } else {
-      accumulator.asInstanceOf[DecimalSumWithRetractAccumulator].f0
+      acc.f0
     }
   }
 
-  override def merge(accumulators: JList[Accumulator]): Accumulator = {
-    val ret = accumulators.get(0).asInstanceOf[DecimalSumWithRetractAccumulator]
-    var i: Int = 1
-    while (i < accumulators.size()) {
-      val a = accumulators.get(i).asInstanceOf[DecimalSumWithRetractAccumulator]
-      ret.f0 = ret.f0.add(a.f0)
-      ret.f1 += a.f1
-      i += 1
+  def merge(acc: DecimalSumWithRetractAccumulator,
+      its: JIterable[DecimalSumWithRetractAccumulator]): Unit = {
+    val iter = its.iterator()
+    while (iter.hasNext) {
+      val a = iter.next()
+      acc.f0 = acc.f0.add(a.f0)
+      acc.f1 += a.f1
     }
-    ret
   }
 
-  override def resetAccumulator(accumulator: Accumulator): Unit = {
-    accumulator.asInstanceOf[DecimalSumWithRetractAccumulator].f0 = BigDecimal.ZERO
-    accumulator.asInstanceOf[DecimalSumWithRetractAccumulator].f1 = 0L
+  def resetAccumulator(acc: DecimalSumWithRetractAccumulator): Unit = {
+    acc.f0 = BigDecimal.ZERO
+    acc.f1 = 0L
   }
 
-  override def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo(
       (new DecimalSumWithRetractAccumulator).getClass,
       BasicTypeInfo.BIG_DEC_TYPE_INFO,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 2c503c651a89a..a82f38312ce16 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.table.runtime.aggregate
 
+import java.lang.reflect.Method
 import java.util
 
 import org.apache.calcite.rel.`type`._
@@ -73,11 +74,12 @@ object AggregateUtil {
     isPartitioned: Boolean,
     isRowsClause: Boolean): ProcessFunction[Row, Row] = {
 
+    val needRetract = false
     val (aggFields, aggregates) =
       transformToAggregateFunctions(
         namedAggregates.map(_.getKey),
         inputType,
-        needRetraction = false)
+        needRetract)
 
     val aggregationStateType: RowTypeInfo =
       createDataSetAggregateBufferDataType(Array(), aggregates, inputType)
@@ -97,7 +99,9 @@ object AggregateUtil {
       forwardMapping,
       None,
       None,
-      outputArity
+      outputArity,
+      needRetract,
+      needMerge = false
     )
 
     if (isRowTimeType) {
@@ -147,11 +151,12 @@ object AggregateUtil {
     isRowsClause: Boolean,
     isRowTimeType: Boolean): ProcessFunction[Row, Row] = {
 
+    val needRetract = true
     val (aggFields, aggregates) =
       transformToAggregateFunctions(
         namedAggregates.map(_.getKey),
         inputType,
-        needRetraction = true)
+        needRetract)
 
     val aggregationStateType: RowTypeInfo = createAccumulatorRowType(aggregates)
     val inputRowType = FlinkTypeFactory.toInternalRowTypeInfo(inputType).asInstanceOf[RowTypeInfo]
@@ -171,7 +176,9 @@ object AggregateUtil {
       forwardMapping,
       None,
       None,
-      outputArity
+      outputArity,
+      needRetract,
+      needMerge = false
     )
 
     if (isRowTimeType) {
@@ -239,10 +246,11 @@ object AggregateUtil {
       isParserCaseSensitive: Boolean)
   : MapFunction[Row, Row] = {
 
+    val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)
+      needRetract)
 
     val mapReturnType: RowTypeInfo =
       createDataSetAggregateBufferDataType(
@@ -293,7 +301,9 @@ object AggregateUtil {
       groupings,
       None,
       None,
-      outputArity
+      outputArity,
+      needRetract,
+      needMerge = false
     )
 
     new DataSetWindowAggMapFunction(
@@ -339,10 +349,11 @@ object AggregateUtil {
       isParserCaseSensitive: Boolean)
     : RichGroupReduceFunction[Row, Row] = {
 
+    val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)
+      needRetract)
 
     val returnType: RowTypeInfo = createDataSetAggregateBufferDataType(
       groupings,
@@ -366,7 +377,9 @@ object AggregateUtil {
           groupings,
           Some(aggregates.indices.map(_ + groupings.length).toArray),
           None,
-          keysAndAggregatesArity + 1
+          keysAndAggregatesArity + 1,
+          needRetract,
+          needMerge = true
         )
         new DataSetSlideTimeWindowAggReduceGroupFunction(
           genFunction,
@@ -447,10 +460,11 @@ object AggregateUtil {
       isInputCombined: Boolean = false)
     : RichGroupReduceFunction[Row, Row] = {
 
+    val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)
+      needRetract)
 
     val aggMapping = aggregates.indices.toArray.map(_ + groupings.length)
 
@@ -465,7 +479,9 @@ object AggregateUtil {
       groupings,
       Some(aggregates.indices.map(_ + groupings.length).toArray),
       None,
-      outputType.getFieldCount
+      outputType.getFieldCount,
+      needRetract,
+      needMerge = true
     )
 
     val genFinalAggFunction = generator.generateAggregations(
@@ -479,7 +495,9 @@ object AggregateUtil {
       groupings.indices.toArray,
       Some(aggregates.indices.map(_ + groupings.length).toArray),
       None,
-      outputType.getFieldCount
+      outputType.getFieldCount,
+      needRetract,
+      needMerge = true
     )
 
     val keysAndAggregatesArity = groupings.length + namedAggregates.length
@@ -586,10 +604,11 @@ object AggregateUtil {
     inputType: RelDataType,
     groupings: Array[Int]): MapPartitionFunction[Row, Row] = {
 
+    val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)
+      needRetract)
 
     val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
 
@@ -615,7 +634,9 @@ object AggregateUtil {
           groupings.indices.toArray,
           Some(aggregates.indices.map(_ + groupings.length).toArray),
           None,
-          groupings.length + aggregates.length + 2
+          groupings.length + aggregates.length + 2,
+          needRetract,
+          needMerge = true
         )
 
         new DataSetSessionWindowAggregatePreProcessor(
@@ -654,10 +675,11 @@ object AggregateUtil {
       groupings: Array[Int])
     : GroupCombineFunction[Row, Row] = {
 
+    val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)
+      needRetract)
 
     val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
 
@@ -684,7 +706,9 @@ object AggregateUtil {
           groupings.indices.toArray,
           Some(aggregates.indices.map(_ + groupings.length).toArray),
           None,
-          groupings.length + aggregates.length + 2
+          groupings.length + aggregates.length + 2,
+          needRetract,
+          needMerge = true
         )
 
         new DataSetSessionWindowAggregatePreProcessor(
@@ -715,10 +739,11 @@ object AggregateUtil {
         Option[TypeInformation[Row]],
         RichGroupReduceFunction[Row, Row]) = {
 
+    val needRetract = false
     val (aggInFields, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
       inputType,
-      needRetraction = false)
+      needRetract)
 
     val (gkeyOutMapping, aggOutMapping) = getOutputMappings(
       namedAggregates,
@@ -760,7 +785,9 @@ object AggregateUtil {
         groupings,
         None,
         None,
-        groupings.length + aggregates.length
+        groupings.length + aggregates.length,
+        needRetract,
+        needMerge = false
       )
 
       // compute mapping of forwarded grouping keys
@@ -784,7 +811,9 @@ object AggregateUtil {
         gkeyMapping,
         Some(aggregates.indices.map(_ + groupings.length).toArray),
         constantFlags,
-        outputType.getFieldCount
+        outputType.getFieldCount,
+        needRetract,
+        needMerge = true
       )
 
       (
@@ -805,7 +834,9 @@ object AggregateUtil {
         groupings,
         None,
         constantFlags,
-        outputType.getFieldCount
+        outputType.getFieldCount,
+        needRetract,
+        needMerge = false
       )
 
       (
@@ -874,11 +905,12 @@ object AggregateUtil {
       outputType: RelDataType)
     : (DataStreamAggFunction[Row, Row, Row], RowTypeInfo, RowTypeInfo) = {
 
+    val needRetract = false
     val (aggFields, aggregates) =
       transformToAggregateFunctions(
         namedAggregates.map(_.getKey),
         inputType,
-        needRetraction = false)
+        needRetract)
 
     val aggMapping = aggregates.indices.toArray
     val outputArity = aggregates.length
@@ -894,7 +926,9 @@ object AggregateUtil {
       Array(), // no fields are forwarded
       None,
       None,
-      outputArity
+      outputArity,
+      needRetract,
+      needMerge = true
     )
 
     val aggResultTypes = namedAggregates.map(a => FlinkTypeFactory.toTypeInfo(a.left.getType))
@@ -926,7 +960,7 @@ object AggregateUtil {
     * Return true if all aggregates can be partially merged. False otherwise.
     */
   private[flink] def doAllSupportPartialMerge(
-      aggregateList: Array[TableAggregateFunction[_ <: Any]]): Boolean = {
+      aggregateList: Array[TableAggregateFunction[_ <: Any, _ <: Any]]): Boolean = {
     aggregateList.forall(ifMethodExistInFunction("merge", _))
   }
 
@@ -1033,11 +1067,11 @@ object AggregateUtil {
       aggregateCalls: Seq[AggregateCall],
       inputType: RelDataType,
       needRetraction: Boolean)
-  : (Array[Array[Int]], Array[TableAggregateFunction[_ <: Any]]) = {
+  : (Array[Array[Int]], Array[TableAggregateFunction[_ <: Any, _ <: Any]]) = {
 
     // store the aggregate fields of each aggregate function, by the same order of aggregates.
     val aggFieldIndexes = new Array[Array[Int]](aggregateCalls.size)
-    val aggregates = new Array[TableAggregateFunction[_ <: Any]](aggregateCalls.size)
+    val aggregates = new Array[TableAggregateFunction[_ <: Any, _ <: Any]](aggregateCalls.size)
 
     // create aggregate function instances by function type and aggregate field data type.
     aggregateCalls.zipWithIndex.foreach { case (aggregateCall, index) =>
@@ -1232,12 +1266,18 @@ object AggregateUtil {
   }
 
   private def createAccumulatorType(
-      aggregates: Array[TableAggregateFunction[_]]): Seq[TypeInformation[_]] = {
+      aggregates: Array[TableAggregateFunction[_, _]]): Seq[TypeInformation[_]] = {
 
     val aggTypes: Seq[TypeInformation[_]] =
       aggregates.map {
         agg =>
-          val accType = agg.getAccumulatorType
+          val accType = try {
+            val method: Method = agg.getClass.getMethod("getAccumulatorType")
+            method.invoke(agg).asInstanceOf[TypeInformation[_]]
+          } catch {
+            case _: NoSuchMethodException => null
+            case ite: Throwable => throw new TableException("Unexpected exception:", ite)
+          }
           if (accType != null) {
             accType
           } else {
@@ -1259,7 +1299,7 @@ object AggregateUtil {
 
   private def createDataSetAggregateBufferDataType(
       groupings: Array[Int],
-      aggregates: Array[TableAggregateFunction[_]],
+      aggregates: Array[TableAggregateFunction[_, _]],
       inputType: RelDataType,
       windowKeyTypes: Option[Array[TypeInformation[_]]] = None): RowTypeInfo = {
 
@@ -1281,7 +1321,7 @@ object AggregateUtil {
   }
 
   private[flink] def createAccumulatorRowType(
-      aggregates: Array[TableAggregateFunction[_]]): RowTypeInfo = {
+      aggregates: Array[TableAggregateFunction[_, _]]): RowTypeInfo = {
 
     val aggTypes: Seq[TypeInformation[_]] = createAccumulatorType(aggregates)
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
index bee39faa6f77c..5f48e091996e5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
@@ -101,3 +101,35 @@ abstract class GeneratedAggregations extends Function {
     */
   def resetAccumulator(accumulators: Row)
 }
+
+class SingleElementIterable[T] extends java.lang.Iterable[T] {
+
+  class SingleElementIterator extends java.util.Iterator[T] {
+
+    var element: T = _
+    var newElement: Boolean = false
+
+    override def hasNext: Boolean = newElement
+
+    override def next(): T = {
+      if (newElement) {
+        newElement = false
+        element
+      } else {
+        throw new java.util.NoSuchElementException
+      }
+    }
+
+    override def remove(): Unit = new java.lang.UnsupportedOperationException
+  }
+
+  val it = new SingleElementIterator
+
+  def setElement(element: T): Unit = it.element = element
+
+  override def iterator(): java.util.Iterator[T] = {
+    it.newElement = true
+    it
+  }
+}
+
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AggFunctionTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AggFunctionTestBase.scala
index cb1137f10428f..39b9ec3e80a8c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AggFunctionTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AggFunctionTestBase.scala
@@ -17,9 +17,10 @@
  */
 package org.apache.flink.table.functions.aggfunctions
 
+import java.lang.reflect.Method
 import java.math.BigDecimal
 import java.util.{ArrayList => JArrayList, List => JList}
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
 import org.junit.Assert.assertEquals
 import org.junit.Test
@@ -29,14 +30,18 @@ import org.junit.Test
   *
   * @tparam T the type for the aggregation result
   */
-abstract class AggFunctionTestBase[T] {
+abstract class AggFunctionTestBase[T, ACC] {
   def inputValueSets: Seq[Seq[_]]
 
   def expectedResults: Seq[T]
 
-  def aggregator: AggregateFunction[T]
+  def aggregator: AggregateFunction[T, ACC]
 
-  def supportRetraction: Boolean = true
+  val accType = aggregator.getClass.getMethod("createAccumulator").getReturnType
+
+  def accumulateFunc: Method = aggregator.getClass.getMethod("accumulate", accType, classOf[Any])
+
+  def retractFunc: Method = null
 
   @Test
   // test aggregate and retract functions without partial merge
@@ -47,52 +52,55 @@ abstract class AggFunctionTestBase[T] {
       val result = aggregator.getValue(accumulator)
       validateResult[T](expected, result)
 
-      if (supportRetraction) {
+      if (ifMethodExistInFunction("retract", aggregator)) {
         retractVals(accumulator, vals)
         val expectedAccum = aggregator.createAccumulator()
         //The two accumulators should be exactly same
-        validateResult[Accumulator](expectedAccum, accumulator)
+        validateResult[ACC](expectedAccum, accumulator)
       }
     }
   }
 
   @Test
-  // test aggregate functions with partial merge
   def testAggregateWithMerge(): Unit = {
 
     if (ifMethodExistInFunction("merge", aggregator)) {
+      val mergeFunc =
+        aggregator.getClass.getMethod("merge", accType, classOf[java.lang.Iterable[ACC]])
       // iterate over input sets
       for ((vals, expected) <- inputValueSets.zip(expectedResults)) {
         //equally split the vals sequence into two sequences
         val (firstVals, secondVals) = vals.splitAt(vals.length / 2)
 
         //1. verify merge with accumulate
-        val accumulators: JList[Accumulator] = new JArrayList[Accumulator]()
-        accumulators.add(accumulateVals(firstVals))
+        val accumulators: JList[ACC] = new JArrayList[ACC]()
         accumulators.add(accumulateVals(secondVals))
 
-        val accumulator = aggregator.merge(accumulators)
-        val result = aggregator.getValue(accumulator)
+        val acc = accumulateVals(firstVals)
+
+        mergeFunc.invoke(aggregator, acc.asInstanceOf[Object], accumulators)
+        val result = aggregator.getValue(acc)
         validateResult[T](expected, result)
 
         //2. verify merge with accumulate & retract
-        if (supportRetraction) {
-          retractVals(accumulator, vals)
+        if (ifMethodExistInFunction("retract", aggregator)) {
+          retractVals(acc, vals)
           val expectedAccum = aggregator.createAccumulator()
           //The two accumulators should be exactly same
-          validateResult[Accumulator](expectedAccum, accumulator)
+          validateResult[ACC](expectedAccum, acc)
         }
       }
 
       // iterate over input sets
       for ((vals, expected) <- inputValueSets.zip(expectedResults)) {
         //3. test partial merge with an empty accumulator
-        val accumulators: JList[Accumulator] = new JArrayList[Accumulator]()
-        accumulators.add(accumulateVals(vals))
+        val accumulators: JList[ACC] = new JArrayList[ACC]()
         accumulators.add(aggregator.createAccumulator())
 
-        val accumulator = aggregator.merge(accumulators)
-        val result = aggregator.getValue(accumulator)
+        val acc = accumulateVals(vals)
+
+        mergeFunc.invoke(aggregator, acc.asInstanceOf[Object], accumulators)
+        val result = aggregator.getValue(acc)
         validateResult[T](expected, result)
       }
     }
@@ -103,13 +111,14 @@ abstract class AggFunctionTestBase[T] {
   def testResetAccumulator(): Unit = {
 
     if (ifMethodExistInFunction("resetAccumulator", aggregator)) {
+      val resetAccFunc = aggregator.getClass.getMethod("resetAccumulator", accType)
       // iterate over input sets
       for ((vals, expected) <- inputValueSets.zip(expectedResults)) {
         val accumulator = accumulateVals(vals)
-        aggregator.resetAccumulator(accumulator)
+        resetAccFunc.invoke(aggregator, accumulator.asInstanceOf[Object])
         val expectedAccum = aggregator.createAccumulator()
         //The accumulator after reset should be exactly same as the new accumulator
-        validateResult[Accumulator](expectedAccum, accumulator)
+        validateResult[ACC](expectedAccum, accumulator)
       }
     }
   }
@@ -130,13 +139,18 @@ abstract class AggFunctionTestBase[T] {
     }
   }
 
-  private def accumulateVals(vals: Seq[_]): Accumulator = {
+  private def accumulateVals(vals: Seq[_]): ACC = {
     val accumulator = aggregator.createAccumulator()
-    vals.foreach(v => aggregator.accumulate(accumulator, v))
+    vals.foreach(
+      v =>
+        accumulateFunc.invoke(aggregator, accumulator.asInstanceOf[Object], v.asInstanceOf[Object])
+    )
     accumulator
   }
 
-  private def retractVals(accumulator:Accumulator, vals: Seq[_]) = {
-    vals.foreach(v => aggregator.retract(accumulator, v))
+  private def retractVals(accumulator:ACC, vals: Seq[_]) = {
+    vals.foreach(
+      v => retractFunc.invoke(aggregator, accumulator.asInstanceOf[Object], v.asInstanceOf[Object])
+    )
   }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
index a388acf2e2eea..fd510ff65db8a 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
@@ -25,7 +25,7 @@ import org.apache.flink.table.functions.AggregateFunction
   *
   * @tparam T the type for the aggregation result
   */
-abstract class AvgAggFunctionTestBase[T: Numeric] extends AggFunctionTestBase[T] {
+abstract class AvgAggFunctionTestBase[T: Numeric, ACC] extends AggFunctionTestBase[T, ACC] {
 
   private val numeric: Numeric[T] = implicitly[Numeric[T]]
 
@@ -91,9 +91,11 @@ abstract class AvgAggFunctionTestBase[T: Numeric] extends AggFunctionTestBase[T]
     numeric.fromInt(3),
     null.asInstanceOf[T]
   )
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
-class ByteAvgAggFunctionTest extends AvgAggFunctionTestBase[Byte] {
+class ByteAvgAggFunctionTest extends AvgAggFunctionTestBase[Byte, IntegralAvgAccumulator] {
 
   override def minVal = (Byte.MinValue + 1).toByte
 
@@ -102,7 +104,7 @@ class ByteAvgAggFunctionTest extends AvgAggFunctionTestBase[Byte] {
   override def aggregator = new ByteAvgAggFunction()
 }
 
-class ShortAvgAggFunctionTest extends AvgAggFunctionTestBase[Short] {
+class ShortAvgAggFunctionTest extends AvgAggFunctionTestBase[Short, IntegralAvgAccumulator] {
 
   override def minVal = (Short.MinValue + 1).toShort
 
@@ -111,7 +113,7 @@ class ShortAvgAggFunctionTest extends AvgAggFunctionTestBase[Short] {
   override def aggregator = new ShortAvgAggFunction()
 }
 
-class IntAvgAggFunctionTest extends AvgAggFunctionTestBase[Int] {
+class IntAvgAggFunctionTest extends AvgAggFunctionTestBase[Int, IntegralAvgAccumulator] {
 
   override def minVal = Int.MinValue + 1
 
@@ -120,7 +122,7 @@ class IntAvgAggFunctionTest extends AvgAggFunctionTestBase[Int] {
   override def aggregator = new IntAvgAggFunction()
 }
 
-class LongAvgAggFunctionTest extends AvgAggFunctionTestBase[Long] {
+class LongAvgAggFunctionTest extends AvgAggFunctionTestBase[Long, BigIntegralAvgAccumulator] {
 
   override def minVal = Long.MinValue + 1
 
@@ -129,7 +131,7 @@ class LongAvgAggFunctionTest extends AvgAggFunctionTestBase[Long] {
   override def aggregator = new LongAvgAggFunction()
 }
 
-class FloatAvgAggFunctionTest extends AvgAggFunctionTestBase[Float] {
+class FloatAvgAggFunctionTest extends AvgAggFunctionTestBase[Float, FloatingAvgAccumulator] {
 
   override def minVal = Float.MinValue
 
@@ -138,7 +140,7 @@ class FloatAvgAggFunctionTest extends AvgAggFunctionTestBase[Float] {
   override def aggregator = new FloatAvgAggFunction()
 }
 
-class DoubleAvgAggFunctionTest extends AvgAggFunctionTestBase[Double] {
+class DoubleAvgAggFunctionTest extends AvgAggFunctionTestBase[Double, FloatingAvgAccumulator] {
 
   override def minVal = Float.MinValue
 
@@ -147,7 +149,7 @@ class DoubleAvgAggFunctionTest extends AvgAggFunctionTestBase[Double] {
   override def aggregator = new DoubleAvgAggFunction()
 }
 
-class DecimalAvgAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalAvgAggFunctionTest extends AggFunctionTestBase[BigDecimal, DecimalAvgAccumulator] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -182,5 +184,8 @@ class DecimalAvgAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
     null
   )
 
-  override def aggregator: AggregateFunction[BigDecimal] = new DecimalAvgAggFunction()
+  override def aggregator: AggregateFunction[BigDecimal, DecimalAvgAccumulator] =
+    new DecimalAvgAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala
index d5f09b2e78172..6830b8f769776 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala
@@ -23,7 +23,7 @@ import org.apache.flink.table.functions.AggregateFunction
 /**
   * Test case for built-in count aggregate function
   */
-class CountAggFunctionTest extends AggFunctionTestBase[Long] {
+class CountAggFunctionTest extends AggFunctionTestBase[Long, CountAccumulator] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq("a", "b", null, "c", null, "d", "e", null, "f"),
@@ -32,5 +32,7 @@ class CountAggFunctionTest extends AggFunctionTestBase[Long] {
 
   override def expectedResults: Seq[Long] = Seq(6L, 0L)
 
-  override def aggregator: AggregateFunction[Long] = new CountAggFunction()
+  override def aggregator: AggregateFunction[Long, CountAccumulator] = new CountAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionTest.scala
index 38ea993543d49..54c2a7db6f6ef 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionTest.scala
@@ -25,7 +25,7 @@ import org.apache.flink.table.functions.AggregateFunction
   *
   * @tparam T the type for the aggregation result
   */
-abstract class MaxAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T] {
+abstract class MaxAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T, MaxAccumulator[T]] {
 
   private val numeric: Numeric[T] = implicitly[Numeric[T]]
 
@@ -61,8 +61,6 @@ abstract class MaxAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T] {
     maxVal,
     null.asInstanceOf[T]
   )
-
-  override def supportRetraction: Boolean = false
 }
 
 class ByteMaxAggFunctionTest extends MaxAggFunctionTest[Byte] {
@@ -71,7 +69,8 @@ class ByteMaxAggFunctionTest extends MaxAggFunctionTest[Byte] {
 
   override def maxVal = (Byte.MaxValue - 1).toByte
 
-  override def aggregator: AggregateFunction[Byte] = new ByteMaxAggFunction()
+  override def aggregator: AggregateFunction[Byte, MaxAccumulator[Byte]] =
+    new ByteMaxAggFunction()
 }
 
 class ShortMaxAggFunctionTest extends MaxAggFunctionTest[Short] {
@@ -80,7 +79,8 @@ class ShortMaxAggFunctionTest extends MaxAggFunctionTest[Short] {
 
   override def maxVal = (Short.MaxValue - 1).toShort
 
-  override def aggregator: AggregateFunction[Short] = new ShortMaxAggFunction()
+  override def aggregator: AggregateFunction[Short, MaxAccumulator[Short]] =
+    new ShortMaxAggFunction()
 }
 
 class IntMaxAggFunctionTest extends MaxAggFunctionTest[Int] {
@@ -89,7 +89,8 @@ class IntMaxAggFunctionTest extends MaxAggFunctionTest[Int] {
 
   override def maxVal = Int.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Int] = new IntMaxAggFunction()
+  override def aggregator: AggregateFunction[Int, MaxAccumulator[Int]] =
+    new IntMaxAggFunction()
 }
 
 class LongMaxAggFunctionTest extends MaxAggFunctionTest[Long] {
@@ -98,7 +99,8 @@ class LongMaxAggFunctionTest extends MaxAggFunctionTest[Long] {
 
   override def maxVal = Long.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Long] = new LongMaxAggFunction()
+  override def aggregator: AggregateFunction[Long, MaxAccumulator[Long]] =
+    new LongMaxAggFunction()
 }
 
 class FloatMaxAggFunctionTest extends MaxAggFunctionTest[Float] {
@@ -107,7 +109,8 @@ class FloatMaxAggFunctionTest extends MaxAggFunctionTest[Float] {
 
   override def maxVal = Float.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Float] = new FloatMaxAggFunction()
+  override def aggregator: AggregateFunction[Float, MaxAccumulator[Float]] =
+    new FloatMaxAggFunction()
 }
 
 class DoubleMaxAggFunctionTest extends MaxAggFunctionTest[Double] {
@@ -116,10 +119,11 @@ class DoubleMaxAggFunctionTest extends MaxAggFunctionTest[Double] {
 
   override def maxVal = Double.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Double] = new DoubleMaxAggFunction()
+  override def aggregator: AggregateFunction[Double, MaxAccumulator[Double]] =
+    new DoubleMaxAggFunction()
 }
 
-class BooleanMaxAggFunctionTest extends AggFunctionTestBase[Boolean] {
+class BooleanMaxAggFunctionTest extends AggFunctionTestBase[Boolean, MaxAccumulator[Boolean]] {
 
   override def inputValueSets: Seq[Seq[Boolean]] = Seq(
     Seq(
@@ -155,12 +159,12 @@ class BooleanMaxAggFunctionTest extends AggFunctionTestBase[Boolean] {
     null.asInstanceOf[Boolean]
   )
 
-  override def aggregator: AggregateFunction[Boolean] = new BooleanMaxAggFunction()
-
-  override def supportRetraction: Boolean = false
+  override def aggregator: AggregateFunction[Boolean, MaxAccumulator[Boolean]] =
+    new BooleanMaxAggFunction()
 }
 
-class DecimalMaxAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalMaxAggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, MaxAccumulator[BigDecimal]] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -188,12 +192,11 @@ class DecimalMaxAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
     null
   )
 
-  override def aggregator: AggregateFunction[BigDecimal] = new DecimalMaxAggFunction()
-
-  override def supportRetraction: Boolean = false
+  override def aggregator: AggregateFunction[BigDecimal, MaxAccumulator[BigDecimal]] =
+    new DecimalMaxAggFunction()
 }
 
-class StringMaxAggFunctionTest extends AggFunctionTestBase[String] {
+class StringMaxAggFunctionTest extends AggFunctionTestBase[String, MaxAccumulator[String]] {
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
       new String("a"),
@@ -221,7 +224,6 @@ class StringMaxAggFunctionTest extends AggFunctionTestBase[String] {
     new String("household")
   )
 
-  override def aggregator: AggregateFunction[String] = new StringMaxAggFunction()
-
-  override def supportRetraction: Boolean = false
+  override def aggregator: AggregateFunction[String, MaxAccumulator[String]] =
+    new StringMaxAggFunction()
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxWithRetractAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxWithRetractAggFunctionTest.scala
index 8bd5c6feb52cb..c11ae41faab1c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxWithRetractAggFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MaxWithRetractAggFunctionTest.scala
@@ -25,7 +25,8 @@ import org.apache.flink.table.functions.AggregateFunction
   *
   * @tparam T the type for the aggregation result
   */
-abstract class MaxWithRetractAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T] {
+abstract class MaxWithRetractAggFunctionTest[T: Numeric]
+  extends AggFunctionTestBase[T, MaxWithRetractAccumulator[T]] {
 
   private val numeric: Numeric[T] = implicitly[Numeric[T]]
 
@@ -61,6 +62,8 @@ abstract class MaxWithRetractAggFunctionTest[T: Numeric] extends AggFunctionTest
     maxVal,
     null.asInstanceOf[T]
   )
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
 class ByteMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Byte] {
@@ -69,7 +72,8 @@ class ByteMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[By
 
   override def maxVal = (Byte.MaxValue - 1).toByte
 
-  override def aggregator: AggregateFunction[Byte] = new ByteMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Byte, MaxWithRetractAccumulator[Byte]] =
+    new ByteMaxWithRetractAggFunction()
 }
 
 class ShortMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Short] {
@@ -78,7 +82,8 @@ class ShortMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[S
 
   override def maxVal = (Short.MaxValue - 1).toShort
 
-  override def aggregator: AggregateFunction[Short] = new ShortMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Short, MaxWithRetractAccumulator[Short]] =
+    new ShortMaxWithRetractAggFunction()
 }
 
 class IntMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Int] {
@@ -87,7 +92,8 @@ class IntMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Int
 
   override def maxVal = Int.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Int] = new IntMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Int, MaxWithRetractAccumulator[Int]] =
+    new IntMaxWithRetractAggFunction()
 }
 
 class LongMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Long] {
@@ -96,7 +102,8 @@ class LongMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Lo
 
   override def maxVal = Long.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Long] = new LongMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Long, MaxWithRetractAccumulator[Long]] =
+    new LongMaxWithRetractAggFunction()
 }
 
 class FloatMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Float] {
@@ -105,7 +112,8 @@ class FloatMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[F
 
   override def maxVal = Float.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Float] = new FloatMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Float, MaxWithRetractAccumulator[Float]] =
+    new FloatMaxWithRetractAggFunction()
 }
 
 class DoubleMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[Double] {
@@ -114,10 +122,12 @@ class DoubleMaxWithRetractAggFunctionTest extends MaxWithRetractAggFunctionTest[
 
   override def maxVal = Double.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Double] = new DoubleMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Double, MaxWithRetractAccumulator[Double]] =
+    new DoubleMaxWithRetractAggFunction()
 }
 
-class BooleanMaxWithRetractAggFunctionTest extends AggFunctionTestBase[Boolean] {
+class BooleanMaxWithRetractAggFunctionTest
+  extends AggFunctionTestBase[Boolean, MaxWithRetractAccumulator[Boolean]] {
 
   override def inputValueSets: Seq[Seq[Boolean]] = Seq(
     Seq(
@@ -153,10 +163,14 @@ class BooleanMaxWithRetractAggFunctionTest extends AggFunctionTestBase[Boolean]
     null.asInstanceOf[Boolean]
   )
 
-  override def aggregator: AggregateFunction[Boolean] = new BooleanMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Boolean, MaxWithRetractAccumulator[Boolean]] =
+    new BooleanMaxWithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
-class DecimalMaxWithRetractAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalMaxWithRetractAggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, MaxWithRetractAccumulator[BigDecimal]] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -184,10 +198,14 @@ class DecimalMaxWithRetractAggFunctionTest extends AggFunctionTestBase[BigDecima
     null
   )
 
-  override def aggregator: AggregateFunction[BigDecimal] = new DecimalMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[BigDecimal, MaxWithRetractAccumulator[BigDecimal]] =
+    new DecimalMaxWithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
-class StringMaxWithRetractAggFunctionTest extends AggFunctionTestBase[String] {
+class StringMaxWithRetractAggFunctionTest
+  extends AggFunctionTestBase[String, MaxWithRetractAccumulator[String]] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -216,5 +234,8 @@ class StringMaxWithRetractAggFunctionTest extends AggFunctionTestBase[String] {
     "x"
   )
 
-  override def aggregator: AggregateFunction[String] = new StringMaxWithRetractAggFunction()
+  override def aggregator: AggregateFunction[String, MaxWithRetractAccumulator[String]] =
+    new StringMaxWithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionTest.scala
index 84e541a498dd5..6f41bd2d0ac0e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunctionTest.scala
@@ -25,7 +25,7 @@ import org.apache.flink.table.functions.AggregateFunction
   *
   * @tparam T the type for the aggregation result
   */
-abstract class MinAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T] {
+abstract class MinAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T, MinAccumulator[T]] {
 
   private val numeric: Numeric[T] = implicitly[Numeric[T]]
 
@@ -61,8 +61,6 @@ abstract class MinAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T] {
     minVal,
     null.asInstanceOf[T]
   )
-
-  override def supportRetraction: Boolean = false
 }
 
 class ByteMinAggFunctionTest extends MinAggFunctionTest[Byte] {
@@ -71,7 +69,8 @@ class ByteMinAggFunctionTest extends MinAggFunctionTest[Byte] {
 
   override def maxVal = (Byte.MaxValue - 1).toByte
 
-  override def aggregator: AggregateFunction[Byte] = new ByteMinAggFunction()
+  override def aggregator: AggregateFunction[Byte, MinAccumulator[Byte]] =
+    new ByteMinAggFunction()
 }
 
 class ShortMinAggFunctionTest extends MinAggFunctionTest[Short] {
@@ -80,7 +79,8 @@ class ShortMinAggFunctionTest extends MinAggFunctionTest[Short] {
 
   override def maxVal = (Short.MaxValue - 1).toShort
 
-  override def aggregator: AggregateFunction[Short] = new ShortMinAggFunction()
+  override def aggregator: AggregateFunction[Short, MinAccumulator[Short]] =
+    new ShortMinAggFunction()
 }
 
 class IntMinAggFunctionTest extends MinAggFunctionTest[Int] {
@@ -89,7 +89,8 @@ class IntMinAggFunctionTest extends MinAggFunctionTest[Int] {
 
   override def maxVal = Int.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Int] = new IntMinAggFunction()
+  override def aggregator: AggregateFunction[Int, MinAccumulator[Int]] =
+    new IntMinAggFunction()
 }
 
 class LongMinAggFunctionTest extends MinAggFunctionTest[Long] {
@@ -98,7 +99,8 @@ class LongMinAggFunctionTest extends MinAggFunctionTest[Long] {
 
   override def maxVal = Long.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Long] = new LongMinAggFunction()
+  override def aggregator: AggregateFunction[Long, MinAccumulator[Long]] =
+    new LongMinAggFunction()
 }
 
 class FloatMinAggFunctionTest extends MinAggFunctionTest[Float] {
@@ -107,7 +109,8 @@ class FloatMinAggFunctionTest extends MinAggFunctionTest[Float] {
 
   override def maxVal = Float.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Float] = new FloatMinAggFunction()
+  override def aggregator: AggregateFunction[Float, MinAccumulator[Float]] =
+    new FloatMinAggFunction()
 }
 
 class DoubleMinAggFunctionTest extends MinAggFunctionTest[Double] {
@@ -116,10 +119,11 @@ class DoubleMinAggFunctionTest extends MinAggFunctionTest[Double] {
 
   override def maxVal = Double.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Double] = new DoubleMinAggFunction()
+  override def aggregator: AggregateFunction[Double, MinAccumulator[Double]] =
+    new DoubleMinAggFunction()
 }
 
-class BooleanMinAggFunctionTest extends AggFunctionTestBase[Boolean] {
+class BooleanMinAggFunctionTest extends AggFunctionTestBase[Boolean, MinAccumulator[Boolean]] {
 
   override def inputValueSets: Seq[Seq[Boolean]] = Seq(
     Seq(
@@ -155,12 +159,12 @@ class BooleanMinAggFunctionTest extends AggFunctionTestBase[Boolean] {
     null.asInstanceOf[Boolean]
   )
 
-  override def aggregator: AggregateFunction[Boolean] = new BooleanMinAggFunction()
-
-  override def supportRetraction: Boolean = false
+  override def aggregator: AggregateFunction[Boolean, MinAccumulator[Boolean]] =
+    new BooleanMinAggFunction()
 }
 
-class DecimalMinAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalMinAggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, MinAccumulator[BigDecimal]] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -188,12 +192,12 @@ class DecimalMinAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
     null
   )
 
-  override def aggregator: AggregateFunction[BigDecimal] = new DecimalMinAggFunction()
-
-  override def supportRetraction: Boolean = false
+  override def aggregator: AggregateFunction[BigDecimal, MinAccumulator[BigDecimal]] =
+    new DecimalMinAggFunction()
 }
 
-class StringMinAggFunctionTest extends AggFunctionTestBase[String] {
+class StringMinAggFunctionTest
+  extends AggFunctionTestBase[String, MinAccumulator[String]] {
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
       new String("a"),
@@ -221,7 +225,6 @@ class StringMinAggFunctionTest extends AggFunctionTestBase[String] {
     new String("1House")
   )
 
-  override def aggregator: AggregateFunction[String] = new StringMinAggFunction()
-
-  override def supportRetraction: Boolean = false
+  override def aggregator: AggregateFunction[String, MinAccumulator[String]] =
+    new StringMinAggFunction()
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinWithRetractAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinWithRetractAggFunctionTest.scala
index 9da3828d5d0ba..e13e69bcc805b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinWithRetractAggFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/MinWithRetractAggFunctionTest.scala
@@ -25,7 +25,8 @@ import org.apache.flink.table.functions.AggregateFunction
   *
   * @tparam T the type for the aggregation result
   */
-abstract class MinWithRetractAggFunctionTest[T: Numeric] extends AggFunctionTestBase[T] {
+abstract class MinWithRetractAggFunctionTest[T: Numeric]
+  extends AggFunctionTestBase[T, MinWithRetractAccumulator[T]] {
 
   private val numeric: Numeric[T] = implicitly[Numeric[T]]
 
@@ -61,6 +62,8 @@ abstract class MinWithRetractAggFunctionTest[T: Numeric] extends AggFunctionTest
     minVal,
     null.asInstanceOf[T]
   )
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
 class ByteMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Byte] {
@@ -69,7 +72,8 @@ class ByteMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[By
 
   override def maxVal = (Byte.MaxValue - 1).toByte
 
-  override def aggregator: AggregateFunction[Byte] = new ByteMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Byte, MinWithRetractAccumulator[Byte]] =
+    new ByteMinWithRetractAggFunction()
 }
 
 class ShortMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Short] {
@@ -78,7 +82,8 @@ class ShortMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[S
 
   override def maxVal = (Short.MaxValue - 1).toShort
 
-  override def aggregator: AggregateFunction[Short] = new ShortMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Short, MinWithRetractAccumulator[Short]] =
+    new ShortMinWithRetractAggFunction()
 }
 
 class IntMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Int] {
@@ -87,7 +92,8 @@ class IntMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Int
 
   override def maxVal = Int.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Int] = new IntMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Int, MinWithRetractAccumulator[Int]] =
+    new IntMinWithRetractAggFunction()
 }
 
 class LongMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Long] {
@@ -96,7 +102,8 @@ class LongMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Lo
 
   override def maxVal = Long.MaxValue - 1
 
-  override def aggregator: AggregateFunction[Long] = new LongMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Long, MinWithRetractAccumulator[Long]] =
+    new LongMinWithRetractAggFunction()
 }
 
 class FloatMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Float] {
@@ -105,7 +112,8 @@ class FloatMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[F
 
   override def maxVal = Float.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Float] = new FloatMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Float, MinWithRetractAccumulator[Float]] =
+    new FloatMinWithRetractAggFunction()
 }
 
 class DoubleMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[Double] {
@@ -114,10 +122,12 @@ class DoubleMinWithRetractAggFunctionTest extends MinWithRetractAggFunctionTest[
 
   override def maxVal = Double.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Double] = new DoubleMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Double, MinWithRetractAccumulator[Double]] =
+    new DoubleMinWithRetractAggFunction()
 }
 
-class BooleanMinWithRetractAggFunctionTest extends AggFunctionTestBase[Boolean] {
+class BooleanMinWithRetractAggFunctionTest
+  extends AggFunctionTestBase[Boolean, MinWithRetractAccumulator[Boolean]] {
 
   override def inputValueSets: Seq[Seq[Boolean]] = Seq(
     Seq(
@@ -153,10 +163,14 @@ class BooleanMinWithRetractAggFunctionTest extends AggFunctionTestBase[Boolean]
     null.asInstanceOf[Boolean]
   )
 
-  override def aggregator: AggregateFunction[Boolean] = new BooleanMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[Boolean, MinWithRetractAccumulator[Boolean]] =
+    new BooleanMinWithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
-class DecimalMinWithRetractAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalMinWithRetractAggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, MinWithRetractAccumulator[BigDecimal]] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -184,10 +198,14 @@ class DecimalMinWithRetractAggFunctionTest extends AggFunctionTestBase[BigDecima
     null
   )
 
-  override def aggregator: AggregateFunction[BigDecimal] = new DecimalMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[BigDecimal, MinWithRetractAccumulator[BigDecimal]] =
+    new DecimalMinWithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
-class StringMinWithRetractAggFunctionTest extends AggFunctionTestBase[String] {
+class StringMinWithRetractAggFunctionTest
+  extends AggFunctionTestBase[String, MinWithRetractAccumulator[String]] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -216,5 +234,8 @@ class StringMinWithRetractAggFunctionTest extends AggFunctionTestBase[String] {
     "e"
   )
 
-  override def aggregator: AggregateFunction[String] = new StringMinWithRetractAggFunction()
+  override def aggregator: AggregateFunction[String, MinWithRetractAccumulator[String]] =
+    new StringMinWithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunctionTest.scala
index cd691876c080f..6816a5e5812dd 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunctionTest.scala
@@ -26,7 +26,8 @@ import org.apache.flink.table.functions.AggregateFunction
   *
   * @tparam T the type for the aggregation result
   */
-abstract class SumAggFunctionTestBase[T: Numeric] extends AggFunctionTestBase[T] {
+abstract class SumAggFunctionTestBase[T: Numeric]
+  extends AggFunctionTestBase[T, SumAccumulator[T]] {
 
   private val numeric: Numeric[T] = implicitly[Numeric[T]]
 
@@ -63,54 +64,59 @@ abstract class SumAggFunctionTestBase[T: Numeric] extends AggFunctionTestBase[T]
     numeric.fromInt(2),
     null.asInstanceOf[T]
   )
-
-  override def supportRetraction: Boolean = false
 }
 
 class ByteSumAggFunctionTest extends SumAggFunctionTestBase[Byte] {
 
   override def maxVal = (Byte.MaxValue / 2).toByte
 
-  override def aggregator: AggregateFunction[Byte] = new ByteSumAggFunction
+  override def aggregator: AggregateFunction[Byte, SumAccumulator[Byte]] =
+    new ByteSumAggFunction
 }
 
 class ShortSumAggFunctionTest extends SumAggFunctionTestBase[Short] {
 
   override def maxVal = (Short.MaxValue / 2).toShort
 
-  override def aggregator: AggregateFunction[Short] = new ShortSumAggFunction
+  override def aggregator: AggregateFunction[Short, SumAccumulator[Short]] =
+    new ShortSumAggFunction
 }
 
 class IntSumAggFunctionTest extends SumAggFunctionTestBase[Int] {
 
   override def maxVal = Int.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Int] = new IntSumAggFunction
+  override def aggregator: AggregateFunction[Int, SumAccumulator[Int]] =
+    new IntSumAggFunction
 }
 
 class LongSumAggFunctionTest extends SumAggFunctionTestBase[Long] {
 
   override def maxVal = Long.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Long] = new LongSumAggFunction
+  override def aggregator: AggregateFunction[Long, SumAccumulator[Long]] =
+    new LongSumAggFunction
 }
 
 class FloatSumAggFunctionTest extends SumAggFunctionTestBase[Float] {
 
   override def maxVal = 12345.6789f
 
-  override def aggregator: AggregateFunction[Float] = new FloatSumAggFunction
+  override def aggregator: AggregateFunction[Float, SumAccumulator[Float]] =
+    new FloatSumAggFunction
 }
 
 class DoubleSumAggFunctionTest extends SumAggFunctionTestBase[Double] {
 
   override def maxVal = 12345.6789d
 
-  override def aggregator: AggregateFunction[Double] = new DoubleSumAggFunction
+  override def aggregator: AggregateFunction[Double, SumAccumulator[Double]] =
+    new DoubleSumAggFunction
 }
 
 
-class DecimalSumAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalSumAggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, DecimalSumAccumulator] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -143,9 +149,8 @@ class DecimalSumAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
     null
   )
 
-  override def aggregator: AggregateFunction[BigDecimal] = new DecimalSumAggFunction()
-
-  override def supportRetraction: Boolean = false
+  override def aggregator: AggregateFunction[BigDecimal, DecimalSumAccumulator] =
+    new DecimalSumAggFunction()
 }
 
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunctionTest.scala
index 72af358ba065b..8b6daba863416 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/SumWithRetractAggFunctionTest.scala
@@ -26,7 +26,8 @@ import org.apache.flink.table.functions.AggregateFunction
   *
   * @tparam T the type for the aggregation result
   */
-abstract class SumWithRetractAggFunctionTestBase[T: Numeric] extends AggFunctionTestBase[T] {
+abstract class SumWithRetractAggFunctionTestBase[T: Numeric]
+  extends AggFunctionTestBase[T, SumWithRetractAccumulator[T]] {
 
   private val numeric: Numeric[T] = implicitly[Numeric[T]]
 
@@ -63,52 +64,61 @@ abstract class SumWithRetractAggFunctionTestBase[T: Numeric] extends AggFunction
     numeric.fromInt(2),
     null.asInstanceOf[T]
   )
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
 class ByteSumWithRetractAggFunctionTest extends SumWithRetractAggFunctionTestBase[Byte] {
 
   override def maxVal = (Byte.MaxValue / 2).toByte
 
-  override def aggregator: AggregateFunction[Byte] = new ByteSumWithRetractAggFunction
+  override def aggregator: AggregateFunction[Byte, SumWithRetractAccumulator[Byte]] =
+    new ByteSumWithRetractAggFunction
 }
 
 class ShortSumWithRetractAggFunctionTest extends SumWithRetractAggFunctionTestBase[Short] {
 
   override def maxVal = (Short.MaxValue / 2).toShort
 
-  override def aggregator: AggregateFunction[Short] = new ShortSumWithRetractAggFunction
+  override def aggregator: AggregateFunction[Short, SumWithRetractAccumulator[Short]] =
+    new ShortSumWithRetractAggFunction
 }
 
 class IntSumWithRetractAggFunctionTest extends SumWithRetractAggFunctionTestBase[Int] {
 
   override def maxVal = Int.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Int] = new IntSumWithRetractAggFunction
+  override def aggregator: AggregateFunction[Int, SumWithRetractAccumulator[Int]] =
+    new IntSumWithRetractAggFunction
 }
 
 class LongSumWithRetractAggFunctionTest extends SumWithRetractAggFunctionTestBase[Long] {
 
   override def maxVal = Long.MaxValue / 2
 
-  override def aggregator: AggregateFunction[Long] = new LongSumWithRetractAggFunction
+  override def aggregator: AggregateFunction[Long, SumWithRetractAccumulator[Long]] =
+    new LongSumWithRetractAggFunction
 }
 
 class FloatSumWithRetractAggFunctionTest extends SumWithRetractAggFunctionTestBase[Float] {
 
   override def maxVal = 12345.6789f
 
-  override def aggregator: AggregateFunction[Float] = new FloatSumWithRetractAggFunction
+  override def aggregator: AggregateFunction[Float, SumWithRetractAccumulator[Float]] =
+    new FloatSumWithRetractAggFunction
 }
 
 class DoubleSumWithRetractAggFunctionTest extends SumWithRetractAggFunctionTestBase[Double] {
 
   override def maxVal = 12345.6789d
 
-  override def aggregator: AggregateFunction[Double] = new DoubleSumWithRetractAggFunction
+  override def aggregator: AggregateFunction[Double, SumWithRetractAccumulator[Double]] =
+    new DoubleSumWithRetractAggFunction
 }
 
 
-class DecimalSumWithRetractAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalSumWithRetractAggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, DecimalSumWithRetractAccumulator] {
 
   override def inputValueSets: Seq[Seq[_]] = Seq(
     Seq(
@@ -141,7 +151,10 @@ class DecimalSumWithRetractAggFunctionTest extends AggFunctionTestBase[BigDecima
     null
   )
 
-  override def aggregator: AggregateFunction[BigDecimal] = new DecimalSumWithRetractAggFunction()
+  override def aggregator: AggregateFunction[BigDecimal, DecimalSumWithRetractAccumulator] =
+    new DecimalSumWithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
 }
 
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
index 16c493edd5111..5e3e995221ec5 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
@@ -51,7 +51,7 @@ class BoundedProcessingOverRangeProcessFunctionTest {
 
     val aggregates =
       Array(new LongMinWithRetractAggFunction,
-            new LongMaxWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_]]]
+            new LongMaxWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
     val aggregationStateType: RowTypeInfo = AggregateUtil.createAccumulatorRowType(aggregates)
 
     val funcCode =
@@ -71,9 +71,9 @@ class BoundedProcessingOverRangeProcessFunctionTest {
         |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
         |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
         |    "MuTG9uZ01pbldpdGhSZXRyYWN0QWdnRnVuY3Rpb26oIdX_DaMPxQIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
-        |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWluV2l0aFJldHJhY3RBZ2dGdW5jdGlvbkDcXxs1apkP" +
+        |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWluV2l0aFJldHJhY3RBZ2dGdW5jdGlvbq_ZGuzxtA_S" +
         |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
-        |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uSa3YqbzCL3QCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
+        |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
         |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
         |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
         |
@@ -81,9 +81,9 @@ class BoundedProcessingOverRangeProcessFunctionTest {
         |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
         |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
         |    "MuTG9uZ01heFdpdGhSZXRyYWN0QWdnRnVuY3Rpb25RmsI8azNGXwIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
-        |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWF4V2l0aFJldHJhY3RBZ2dGdW5jdGlvbu4_w_gPePlO" +
+        |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWF4V2l0aFJldHJhY3RBZ2dGdW5jdGlvbvnwowlX0_Qf" +
         |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
-        |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uSa3YqbzCL3QCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
+        |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
         |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
         |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
         |  }
@@ -95,12 +95,14 @@ class BoundedProcessingOverRangeProcessFunctionTest {
         |    org.apache.flink.table.functions.AggregateFunction baseClass0 =
         |      (org.apache.flink.table.functions.AggregateFunction) fmin;
         |    output.setField(5, baseClass0.getValue(
-        |      (org.apache.flink.table.functions.Accumulator) accs.getField(0)));
+        |      (org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+        |      accs.getField(0)));
         |
         |    org.apache.flink.table.functions.AggregateFunction baseClass1 =
         |      (org.apache.flink.table.functions.AggregateFunction) fmax;
         |    output.setField(6, baseClass1.getValue(
-        |      (org.apache.flink.table.functions.Accumulator) accs.getField(1)));
+        |      (org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+        |      accs.getField(1)));
         |  }
         |
         |  public void accumulate(
@@ -108,11 +110,13 @@ class BoundedProcessingOverRangeProcessFunctionTest {
         |    org.apache.flink.types.Row input) {
         |
         |    fmin.accumulate(
-        |      ((org.apache.flink.table.functions.Accumulator) accs.getField(0)),
+        |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+        |      accs.getField(0)),
         |      (java.lang.Long) input.getField(4));
         |
         |    fmax.accumulate(
-        |      ((org.apache.flink.table.functions.Accumulator) accs.getField(1)),
+        |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+        |      accs.getField(1)),
         |      (java.lang.Long) input.getField(4));
         |  }
         |
@@ -121,11 +125,13 @@ class BoundedProcessingOverRangeProcessFunctionTest {
         |    org.apache.flink.types.Row input) {
         |
         |    fmin.retract(
-        |      ((org.apache.flink.table.functions.Accumulator) accs.getField(0)),
+        |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+        |      accs.getField(0)),
         |      (java.lang.Long) input.getField(4));
         |
         |    fmax.retract(
-        |      ((org.apache.flink.table.functions.Accumulator) accs.getField(1)),
+        |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+        |      accs.getField(1)),
         |      (java.lang.Long) input.getField(4));
         |  }
         |

From 38f7d78435168e6a0ba45e285955a79d3e2e7e70 Mon Sep 17 00:00:00 2001
From: xccui <xingcanc@gmail.com>
Date: Tue, 25 Apr 2017 14:06:45 +0800
Subject: [PATCH 042/253] [FLINK-6368] [table] Grouping keys in stream
 aggregations have wrong order

This closes #3768.
---
 .../datastream/DataStreamAggregate.scala      |  7 +++--
 .../DataStreamAggregateITCase.scala           | 27 +++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
index 62bcfc80d909f..187773da7b55a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
@@ -101,7 +101,6 @@ class DataStreamAggregate(
 
   override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
 
-    val groupingKeys = grouping.indices.toArray
     val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
 
     val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType)
@@ -124,15 +123,15 @@ class DataStreamAggregate(
       inputDS.getType)
 
     // grouped / keyed aggregation
-    if (groupingKeys.length > 0) {
+    if (grouping.length > 0) {
       val windowFunction = AggregateUtil.createAggregationGroupWindowFunction(
         window,
-        groupingKeys.length,
+        grouping.length,
         namedAggregates.size,
         rowRelDataType.getFieldCount,
         namedProperties)
 
-      val keyedStream = inputDS.keyBy(groupingKeys: _*)
+      val keyedStream = inputDS.keyBy(grouping: _*)
       val windowedStream =
         createKeyedWindowedStream(window, keyedStream)
           .asInstanceOf[WindowedStream[Row, Tuple, DataStreamWindow]]
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala
index 85a237331e2e2..dcd3c6cfce1d5 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala
@@ -213,6 +213,33 @@ class DataStreamAggregateITCase extends StreamingMultipleProgramsTestBase {
       "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.003")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
+
+  @Test
+  def testEventTimeGroupWindowWithoutExplicitTimeField(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env
+      .fromCollection(data)
+      .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+      .map(t => (t._2, t._6))
+    val table = stream.toTable(tEnv, 'int, 'string)
+
+    val windowedTable = table
+      .window(Slide over 3.milli every 10.milli on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, 'int.count, 'w.start, 'w.end)
+
+    val results = windowedTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+    val expected = Seq(
+      "Hallo,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.003",
+      "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.003")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
 }
 
 object DataStreamAggregateITCase {

From 2cf24edad6490e8f93992b3d9998edeb6d2d6395 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Tue, 25 Apr 2017 20:12:33 +0200
Subject: [PATCH 043/253] [hotfix] [tests] Harden JobManagerRegistrationTest

The problem was that the started JobManagers try to instantiate a MetricQueryService
actor with a fixed name. Thus, if the previous JobManager was not shutdown properly,
then there was still a MetricQueryServie actor around having the exact same name
with which you try to start a new actor.

The problem is solved twofold:

1. by properly stopping the instantiated actors
2. by starting the JobManager without a MetricsRegistry
---
 .../JobManagerRegistrationTest.scala          | 279 +++++++++++-------
 .../runtime/testingUtils/TestingUtils.scala   |  56 +++-
 2 files changed, 221 insertions(+), 114 deletions(-)

diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
index 76d95917461c9..4f8eab2bb8eb8 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
@@ -22,7 +22,7 @@ import java.net.InetAddress
 import java.util.concurrent.{Executors, ScheduledExecutorService}
 
 import akka.actor._
-import akka.testkit.{ImplicitSender, TestKit}
+import akka.testkit.{ImplicitSender, TestKit, TestProbe}
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.runtime.akka.AkkaUtils
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager
@@ -33,6 +33,7 @@ import org.apache.flink.runtime.jobmanager.JobManagerRegistrationTest.PlainForwa
 import org.apache.flink.runtime.messages.JobManagerMessages.LeaderSessionMessage
 import org.apache.flink.runtime.messages.RegistrationMessages.{AcknowledgeRegistration, AlreadyRegistered, RegisterTaskManager}
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation
+import org.apache.flink.runtime.testingUtils.TestingUtils
 import org.apache.flink.runtime.testutils.TestingResourceManager
 import org.apache.flink.runtime.util.LeaderRetrievalUtils
 import org.junit.Assert.{assertNotEquals, assertNotNull}
@@ -62,132 +63,186 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
   "The JobManager" should {
 
     "assign a TaskManager a unique instance ID" in {
-      val jm = startTestingJobManager(_system)
-      val rm = startTestingResourceManager(_system, jm.actor())
-
-      val tm1 = _system.actorOf(Props(new PlainForwardingActor(testActor)))
-      val tm2 = _system.actorOf(Props(new PlainForwardingActor(testActor)))
-
-      val resourceId1 = ResourceID.generate()
-      val resourceId2 = ResourceID.generate()
-      
-      val connectionInfo1 = new TaskManagerLocation(resourceId1, InetAddress.getLocalHost, 10000)
-      val connectionInfo2 = new TaskManagerLocation(resourceId2, InetAddress.getLocalHost, 10001)
-
-      val hardwareDescription = HardwareDescription.extractFromSystem(10)
-
-      var id1: InstanceID = null
-      var id2: InstanceID = null
-
-      // task manager 1
-      within(10 seconds) {
-        jm.tell(
-          RegisterTaskManager(
-            resourceId1,
-            connectionInfo1,
-            hardwareDescription,
-            1),
-          new AkkaActorGateway(tm1, HighAvailabilityServices.DEFAULT_LEADER_ID))
-
-        val response = expectMsgType[LeaderSessionMessage]
-        response match {
-          case LeaderSessionMessage(_, AcknowledgeRegistration(id, _)) => id1 = id
-          case _ => fail("Wrong response message: " + response)
-        }
-      }
 
-      // task manager 2
-      within(10 seconds) {
-        jm.tell(
-          RegisterTaskManager(
-            resourceId2,
-            connectionInfo2,
-            hardwareDescription,
-            1),
-          new AkkaActorGateway(tm2, HighAvailabilityServices.DEFAULT_LEADER_ID))
-
-        val response = expectMsgType[LeaderSessionMessage]
-        response match {
-          case LeaderSessionMessage(leaderSessionID, AcknowledgeRegistration(id, _)) => id2 = id
-          case _ => fail("Wrong response message: " + response)
-        }
+      var jmOption: Option[ActorGateway] = None
+      var rmOption: Option[ActorGateway] = None
+      var tm1Option: Option[ActorRef] = None
+      var tm2Option: Option[ActorRef] = None
+
+      try {
+        val jm = startTestingJobManager(_system)
+        jmOption = Some(jm)
+
+        val rm = startTestingResourceManager(_system, jm.actor())
+        rmOption = Some(rm)
+
+        val probe = TestProbe()
+        val tm1 = _system.actorOf(Props(new PlainForwardingActor(probe.ref)))
+        tm1Option = Some(tm1)
+
+        val tm2 = _system.actorOf(Props(new PlainForwardingActor(probe.ref)))
+        tm2Option = Some(tm2)
+
+        val resourceId1 = ResourceID.generate()
+        val resourceId2 = ResourceID.generate()
+
+        val connectionInfo1 = new TaskManagerLocation(resourceId1, InetAddress.getLocalHost, 10000)
+        val connectionInfo2 = new TaskManagerLocation(resourceId2, InetAddress.getLocalHost, 10001)
+
+        val hardwareDescription = HardwareDescription.extractFromSystem(10)
+
+        var id1: InstanceID = null
+        var id2: InstanceID = null
+
+        // task manager 1
+        within(10 seconds) {
+         jm.tell(
+           RegisterTaskManager(
+             resourceId1,
+             connectionInfo1,
+             hardwareDescription,
+             1),
+           new AkkaActorGateway(tm1, HighAvailabilityServices.DEFAULT_LEADER_ID))
+
+         val response = probe.expectMsgType[LeaderSessionMessage]
+         response match {
+           case LeaderSessionMessage(_, AcknowledgeRegistration(id, _)) => id1 = id
+           case _ => fail("Wrong response message: " + response)
+         }
+       }
+
+        // task manager 2
+        within(10 seconds) {
+         jm.tell(
+           RegisterTaskManager(
+             resourceId2,
+             connectionInfo2,
+             hardwareDescription,
+             1),
+           new AkkaActorGateway(tm2, HighAvailabilityServices.DEFAULT_LEADER_ID))
+
+         val response = probe.expectMsgType[LeaderSessionMessage]
+         response match {
+           case LeaderSessionMessage(leaderSessionID, AcknowledgeRegistration(id, _)) => id2 = id
+           case _ => fail("Wrong response message: " + response)
+         }
+       }
+
+        assertNotNull(id1)
+        assertNotNull(id2)
+        assertNotEquals(id1, id2)
+      } finally {
+        jmOption.foreach(TestingUtils.stopActorGracefully)
+        rmOption.foreach(TestingUtils.stopActorGracefully)
+        tm1Option.foreach(TestingUtils.stopActorGracefully)
+        tm2Option.foreach(TestingUtils.stopActorGracefully)
       }
-
-      assertNotNull(id1)
-      assertNotNull(id2)
-      assertNotEquals(id1, id2)
     }
 
     "handle repeated registration calls" in {
 
-      val jm = startTestingJobManager(_system)
-      val rm = startTestingResourceManager(_system, jm.actor())
-
-      val selfGateway = new AkkaActorGateway(testActor, HighAvailabilityServices.DEFAULT_LEADER_ID)
-
-      val resourceID = ResourceID.generate()
-      val connectionInfo = new TaskManagerLocation(resourceID, InetAddress.getLocalHost, 1)
-      val hardwareDescription = HardwareDescription.extractFromSystem(10)
-
-      within(20 seconds) {
-        jm.tell(
-          RegisterTaskManager(
-            resourceID,
-            connectionInfo,
-            hardwareDescription,
-            1),
-          selfGateway)
-
-        jm.tell(
-          RegisterTaskManager(
-            resourceID,
-            connectionInfo,
-            hardwareDescription,
-            1),
-          selfGateway)
-
-        jm.tell(
-          RegisterTaskManager(
-            resourceID,
-            connectionInfo,
-            hardwareDescription,
-            1),
-          selfGateway)
-
-        expectMsgType[LeaderSessionMessage] match {
-          case LeaderSessionMessage(
-            HighAvailabilityServices.DEFAULT_LEADER_ID,
-            AcknowledgeRegistration(_, _)) =>
-          case m => fail("Wrong message type: " + m)
-        }
-
-        expectMsgType[LeaderSessionMessage] match {
-          case LeaderSessionMessage(
-            HighAvailabilityServices.DEFAULT_LEADER_ID,
-            AlreadyRegistered(_, _)) =>
-          case m => fail("Wrong message type: " + m)
-        }
-
-        expectMsgType[LeaderSessionMessage] match {
-          case LeaderSessionMessage(
-            HighAvailabilityServices.DEFAULT_LEADER_ID,
-            AlreadyRegistered(_, _)) =>
-          case m => fail("Wrong message type: " + m)
+      var jmOption: Option[ActorGateway] = None
+      var rmOption: Option[ActorGateway] = None
+
+      try {
+        val probe = TestProbe()
+
+        val jm = startTestingJobManager(_system)
+        jmOption = Some(jm)
+        val rm = startTestingResourceManager(_system, jm.actor())
+        rmOption = Some(rm)
+
+        val selfGateway = new AkkaActorGateway(probe.ref, HighAvailabilityServices.DEFAULT_LEADER_ID)
+
+        val resourceID = ResourceID.generate()
+        val connectionInfo = new TaskManagerLocation(resourceID, InetAddress.getLocalHost, 1)
+        val hardwareDescription = HardwareDescription.extractFromSystem(10)
+
+        within(20 seconds) {
+          jm.tell(
+            RegisterTaskManager(
+              resourceID,
+              connectionInfo,
+              hardwareDescription,
+              1),
+              selfGateway)
+
+          jm.tell(
+            RegisterTaskManager(
+              resourceID,
+              connectionInfo,
+              hardwareDescription,
+              1),
+              selfGateway)
+
+          jm.tell(
+            RegisterTaskManager(
+              resourceID,
+              connectionInfo,
+              hardwareDescription,
+              1),
+              selfGateway)
+
+          probe.expectMsgType[LeaderSessionMessage] match {
+            case LeaderSessionMessage(
+              HighAvailabilityServices.DEFAULT_LEADER_ID,
+              AcknowledgeRegistration(_, _)) =>
+            case m => fail("Wrong message type: " + m)
+          }
+
+          probe.expectMsgType[LeaderSessionMessage] match {
+            case LeaderSessionMessage(
+              HighAvailabilityServices.DEFAULT_LEADER_ID,
+              AlreadyRegistered(_, _)) =>
+            case m => fail("Wrong message type: " + m)
+          }
+
+          probe.expectMsgType[LeaderSessionMessage] match {
+            case LeaderSessionMessage(
+              HighAvailabilityServices.DEFAULT_LEADER_ID,
+              AlreadyRegistered(_, _)) =>
+            case m => fail("Wrong message type: " + m)
+          }
         }
+      } finally {
+        jmOption.foreach(TestingUtils.stopActorGracefully)
+        rmOption.foreach(TestingUtils.stopActorGracefully)
       }
     }
   }
 
   private def startTestingJobManager(system: ActorSystem): ActorGateway = {
-    val (jm: ActorRef, _) = JobManager.startJobManagerActors(
-      new Configuration(),
-      _system,
+    val config = new Configuration()
+
+    val components = JobManager.createJobManagerComponents(
+      config,
       executor,
       executor,
-      None,
-      None,
+      None)
+
+    // Start the JobManager without a MetricRegistry so that we don't start the MetricQueryService.
+    // The problem of the MetricQueryService is that it starts an actor with a fixed name. Thus,
+    // if there exists already one of these actors (e.g. JobManager has not been properly shutdown),
+    // then this will fail the JobManager creation
+    val props = JobManager.getJobManagerProps(
       classOf[JobManager],
-      classOf[MemoryArchivist])
+      config,
+      executor,
+      executor,
+      components._1,
+      components._2,
+      components._3,
+      ActorRef.noSender,
+      components._4,
+      components._5,
+      components._8,
+      components._9,
+      components._10,
+      components._11,
+      None)
+
+    val jm = _system.actorOf(props)
+
     new AkkaActorGateway(jm, HighAvailabilityServices.DEFAULT_LEADER_ID)
   }
 
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index 49baff11d3578..f58faf2a3cbd6 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -23,7 +23,7 @@ import java.util.{Collections, UUID}
 import java.util.concurrent._
 
 import akka.actor.{ActorRef, ActorSystem, Kill, Props}
-import akka.pattern.ask
+import akka.pattern.{Patterns, ask}
 import com.google.common.util.concurrent.MoreExecutors
 import com.typesafe.config.ConfigFactory
 import grizzled.slf4j.Logger
@@ -60,6 +60,8 @@ object TestingUtils {
   
   val TESTING_DURATION = 2 minute
 
+  val TESTING_TIMEOUT = 1 minute
+
   val DEFAULT_AKKA_ASK_TIMEOUT = "200 s"
 
   def getDefaultTestingActorSystemConfigString: String = {
@@ -340,7 +342,7 @@ object TestingUtils {
     }
   }
 
-  /** Stops the given actro by sending it a Kill message
+  /** Stops the given actor by sending it a Kill message
     *
     * @param actorGateway
     */
@@ -350,6 +352,56 @@ object TestingUtils {
     }
   }
 
+  def stopActorGracefully(actor: ActorRef): Unit = {
+    val gracefulStopFuture = Patterns.gracefulStop(actor, TestingUtils.TESTING_TIMEOUT)
+
+    Await.result(gracefulStopFuture, TestingUtils.TESTING_TIMEOUT)
+  }
+
+  def stopActorGracefully(actorGateway: ActorGateway): Unit = {
+    stopActorGracefully(actorGateway.actor())
+  }
+
+  def stopActorsGracefully(actors: ActorRef*): Unit = {
+    val gracefulStopFutures = actors.flatMap{
+      actor =>
+        Option(actor) match {
+          case Some(actorRef) => Some(Patterns.gracefulStop(actorRef, TestingUtils.TESTING_TIMEOUT))
+          case None => None
+        }
+    }
+
+    implicit val executionContext = defaultExecutionContext
+
+    val globalStopFuture = scala.concurrent.Future.sequence(gracefulStopFutures)
+
+    Await.result(globalStopFuture, TestingUtils.TESTING_TIMEOUT)
+  }
+
+  def stopActorsGracefully(actors: java.util.List[ActorRef]): Unit = {
+    import scala.collection.JavaConverters._
+
+    stopActorsGracefully(actors.asScala: _*)
+  }
+
+  def stopActorGatewaysGracefully(actorGateways: ActorGateway*): Unit = {
+    val actors = actorGateways.flatMap {
+      actorGateway =>
+        Option(actorGateway) match {
+          case Some(actorGateway) => Some(actorGateway.actor())
+          case None => None
+        }
+    }
+
+    stopActorsGracefully(actors: _*)
+  }
+
+  def stopActorGatewaysGracefully(actorGateways: java.util.List[ActorGateway]): Unit = {
+    import scala.collection.JavaConverters._
+
+    stopActorGatewaysGracefully(actorGateways.asScala: _*)
+  }
+
   /** Creates a testing JobManager using the default recovery mode (standalone)
     *
     * @param actorSystem The ActorSystem to use

From 21a58fb618f5ad8365dedd4ff42b30d598da325a Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Wed, 22 Mar 2017 09:09:02 +0100
Subject: [PATCH 044/253] [FLINK-6155] Introduce an endpoint id for
 RpcEndpoints

An endpoint id allows to assign a unique name to a RpcEndpoint. This name can be used
to look up the rpc endpoint within the RpcService.

Remove component endpoint name methods from the HighAvailabilityServices

Remove JobMaster endpoint id

This closes #3596.
---
 .../EmbeddedNonHaServices.java                |  6 ---
 .../HighAvailabilityServices.java             |  6 ---
 .../highavailability/NonHaServices.java       | 10 -----
 .../highavailability/ZookeeperHaServices.java | 10 -----
 .../runtime/jobmaster/JobManagerRunner.java   |  2 +-
 .../flink/runtime/jobmaster/JobMaster.java    |  7 +++-
 .../runtime/minicluster/MiniCluster.java      |  2 +
 .../resourcemanager/ResourceManager.java      |  5 ++-
 .../ResourceManagerRunner.java                |  2 +
 .../StandaloneResourceManager.java            |  2 +
 .../apache/flink/runtime/rpc/RpcEndpoint.java | 30 +++++++++++++--
 .../flink/runtime/rpc/RpcServiceUtils.java    | 25 +++++++++++-
 .../runtime/rpc/akka/AkkaRpcService.java      |  2 +-
 .../runtime/taskexecutor/TaskExecutor.java    | 38 ++++++++++---------
 .../taskexecutor/TaskManagerRunner.java       |  2 +-
 .../clusterframework/ResourceManagerTest.java |  4 +-
 .../TestingHighAvailabilityServices.java      |  5 ---
 .../runtime/jobmaster/JobMasterTest.java      |  2 +-
 .../ResourceManagerHATest.java                |  8 +++-
 .../ResourceManagerJobMasterTest.java         |  5 ++-
 .../ResourceManagerTaskExecutorTest.java      |  2 +
 .../slotmanager/SlotProtocolTest.java         |  9 ++++-
 .../taskexecutor/TaskExecutorITCase.java      |  7 +++-
 .../taskexecutor/TaskExecutorTest.java        | 18 ++++-----
 .../YarnFlinkApplicationMasterRunner.java     |  5 ++-
 .../flink/yarn/YarnResourceManager.java       |  2 +
 .../yarn/configuration/YarnConfigOptions.java |  2 +-
 .../AbstractYarnNonHaServices.java            | 12 ------
 .../YarnPreConfiguredMasterNonHaServices.java |  3 +-
 ...YarnPreConfiguredMasterHaServicesTest.java | 30 ---------------
 30 files changed, 135 insertions(+), 128 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
index a417599784b67..8bf81ebbcab3a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
@@ -42,12 +42,6 @@ public EmbeddedNonHaServices() {
 
 	// ------------------------------------------------------------------------
 
-	@Override
-	public String getResourceManagerEndpointName() {
-		// dynamic actor name
-		return null;
-	}
-
 	@Override
 	public LeaderRetrievalService getResourceManagerLeaderRetriever() {
 		return resourceManagerLeaderService.createLeaderRetrievalService();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index 416920475309f..3de75d5cd01c5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -56,12 +56,6 @@ public interface HighAvailabilityServices extends AutoCloseable {
 	 */
 	UUID DEFAULT_LEADER_ID = new UUID(0, 0);
 
-	// ------------------------------------------------------------------------
-	//  Endpoint Naming
-	// ------------------------------------------------------------------------
-
-	String getResourceManagerEndpointName();
-
 	// ------------------------------------------------------------------------
 	//  Services
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
index d644fb9916f75..beb596326271e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
@@ -53,16 +53,6 @@ public NonHaServices(String resourceManagerAddress) {
 		this.resourceManagerAddress = checkNotNull(resourceManagerAddress);
 	}
 
-	// ------------------------------------------------------------------------
-	//  Names
-	// ------------------------------------------------------------------------
-
-	@Override
-	public String getResourceManagerEndpointName() {
-		return RESOURCE_MANAGER_RPC_ENDPOINT_NAME;
-	}
-
-
 	// ------------------------------------------------------------------------
 	//  Services
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
index 741f9e6ba2f35..4d0db0a33ad6f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
@@ -111,16 +111,6 @@ public ZookeeperHaServices(CuratorFramework client, Executor executor, Configura
 	//  Services
 	// ------------------------------------------------------------------------
 
-	@Override
-	public String getResourceManagerEndpointName() {
-		// since the resource manager name must be dynamic, we return null here
-		return null;
-	}
-
-	// ------------------------------------------------------------------------
-	//  Services
-	// ------------------------------------------------------------------------
-
 	@Override
 	public LeaderRetrievalService getResourceManagerLeaderRetriever() {
 		return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index 33ee29df1228e..d7fae35d3a7d9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -188,10 +188,10 @@ public JobManagerRunner(
 
 			// now start the JobManager
 			this.jobManager = new JobMaster(
+				rpcService,
 				resourceId,
 				jobGraph,
 				configuration,
-				rpcService,
 				haServices,
 				heartbeatServices,
 				jobManagerServices.executorService,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 080b48e02008c..6fe8cb3c0d13a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -64,6 +64,7 @@
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.OnCompletionActions;
 import org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException;
 import org.apache.flink.runtime.jobmanager.slots.AllocatedSlot;
@@ -87,6 +88,7 @@
 import org.apache.flink.runtime.rpc.RpcEndpoint;
 import org.apache.flink.runtime.rpc.RpcMethod;
 import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.RpcServiceUtils;
 import org.apache.flink.runtime.rpc.StartStoppable;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
@@ -188,10 +190,10 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
 	// ------------------------------------------------------------------------
 
 	public JobMaster(
+			RpcService rpcService,
 			ResourceID resourceId,
 			JobGraph jobGraph,
 			Configuration configuration,
-			RpcService rpcService,
 			HighAvailabilityServices highAvailabilityService,
 			HeartbeatServices heartbeatServices,
 			ScheduledExecutorService executor,
@@ -202,7 +204,8 @@ public JobMaster(
 			OnCompletionActions jobCompletionActions,
 			FatalErrorHandler errorHandler,
 			ClassLoader userCodeLoader) throws Exception {
-		super(rpcService);
+
+		super(rpcService, RpcServiceUtils.createRandomName(JobManager.JOB_MANAGER_NAME()));
 
 		this.resourceId = checkNotNull(resourceId);
 		this.jobGraph = checkNotNull(jobGraph);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 9d5f9d12a0b27..03fbef5d44357 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -26,6 +26,7 @@
 import org.apache.flink.configuration.UnmodifiableConfiguration;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
@@ -524,6 +525,7 @@ protected ResourceManagerRunner[] startResourceManagers(
 
 			resourceManagerRunners[i] = new ResourceManagerRunner(
 				ResourceID.generate(),
+				FlinkResourceManager.RESOURCE_MANAGER_NAME + '_' + i,
 				configuration,
 				resourceManagerRpcServices[i],
 				haServices,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 5467177e1b6fe..c0ff41292de9f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -131,6 +131,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
 
 	public ResourceManager(
 			RpcService rpcService,
+			String resourceManagerEndpointId,
 			ResourceID resourceId,
 			ResourceManagerConfiguration resourceManagerConfiguration,
 			HighAvailabilityServices highAvailabilityServices,
@@ -140,7 +141,7 @@ public ResourceManager(
 			JobLeaderIdService jobLeaderIdService,
 			FatalErrorHandler fatalErrorHandler) {
 
-		super(rpcService);
+		super(rpcService, resourceManagerEndpointId);
 
 		this.resourceId = checkNotNull(resourceId);
 		this.resourceManagerConfiguration = checkNotNull(resourceManagerConfiguration);
@@ -162,6 +163,8 @@ public ResourceManager(
 		infoMessageListeners = new ConcurrentHashMap<>(8);
 	}
 
+
+
 	// ------------------------------------------------------------------------
 	//  RPC lifecycle methods
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
index 3a8baa628d53f..c6d64db167e80 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
@@ -46,6 +46,7 @@ public class ResourceManagerRunner implements FatalErrorHandler {
 
 	public ResourceManagerRunner(
 			final ResourceID resourceId,
+			final String resourceManagerEndpointId,
 			final Configuration configuration,
 			final RpcService rpcService,
 			final HighAvailabilityServices highAvailabilityServices,
@@ -70,6 +71,7 @@ public ResourceManagerRunner(
 
 		this.resourceManager = new StandaloneResourceManager(
 			rpcService,
+			resourceManagerEndpointId,
 			resourceId,
 			resourceManagerConfiguration,
 			highAvailabilityServices,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
index fd5a001f23fbb..389ca5c82666f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
@@ -39,6 +39,7 @@ public class StandaloneResourceManager extends ResourceManager<ResourceID> {
 
 	public StandaloneResourceManager(
 			RpcService rpcService,
+			String resourceManagerEndpointId,
 			ResourceID resourceId,
 			ResourceManagerConfiguration resourceManagerConfiguration,
 			HighAvailabilityServices highAvailabilityServices,
@@ -49,6 +50,7 @@ public StandaloneResourceManager(
 			FatalErrorHandler fatalErrorHandler) {
 		super(
 			rpcService,
+			resourceManagerEndpointId,
 			resourceId,
 			resourceManagerConfiguration,
 			highAvailabilityServices,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
index f30e345fdc2c1..40b9568377f8a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
@@ -26,6 +26,7 @@
 import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
@@ -63,6 +64,9 @@ public abstract class RpcEndpoint<C extends RpcGateway> {
 	/** RPC service to be used to start the RPC server and to obtain rpc gateways */
 	private final RpcService rpcService;
 
+	/** Unique identifier for this rpc endpoint */
+	private final String endpointId;
+
 	/** Class of the self gateway */
 	private final Class<C> selfGatewayType;
 
@@ -79,10 +83,12 @@ public abstract class RpcEndpoint<C extends RpcGateway> {
 	/**
 	 * Initializes the RPC endpoint.
 	 * 
-	 * @param rpcService The RPC server that dispatches calls to this RPC endpoint. 
+	 * @param rpcService The RPC server that dispatches calls to this RPC endpoint.
+	 * @param endpointId Unique identifier for this endpoint
 	 */
-	protected RpcEndpoint(final RpcService rpcService) {
+	protected RpcEndpoint(final RpcService rpcService, final String endpointId) {
 		this.rpcService = checkNotNull(rpcService, "rpcService");
+		this.endpointId = checkNotNull(endpointId, "endpointId");
 
 		// IMPORTANT: Don't change order of selfGatewayType and self because rpcService.startServer
 		// requires that selfGatewayType has been initialized
@@ -92,6 +98,15 @@ protected RpcEndpoint(final RpcService rpcService) {
 		this.mainThreadExecutor = new MainThreadExecutor((MainThreadExecutable) self);
 	}
 
+	/**
+	 * Initializes the RPC endpoint with a random endpoint id.
+	 *
+	 * @param rpcService The RPC server that dispatches calls to this RPC endpoint.
+	 */
+	protected RpcEndpoint(final RpcService rpcService) {
+		this(rpcService, UUID.randomUUID().toString());
+	}
+
 	/**
 	 * Returns the class of the self gateway type.
 	 *
@@ -100,7 +115,16 @@ protected RpcEndpoint(final RpcService rpcService) {
 	public final Class<C> getSelfGatewayType() {
 		return selfGatewayType;
 	}
-	
+
+	/**
+	 * Returns the rpc endpoint's identifier.
+	 *
+	 * @return Rpc endpoint's identifier.
+	 */
+	public String getEndpointId() {
+		return endpointId;
+	}
+
 	// ------------------------------------------------------------------------
 	//  Start & Shutdown
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
index 018c3ed4a8921..e555e7f509ab7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
@@ -28,6 +28,7 @@
 import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
 import org.apache.flink.util.NetUtils;
 
+import org.apache.flink.util.Preconditions;
 import org.jboss.netty.channel.ChannelException;
 
 import org.slf4j.Logger;
@@ -35,6 +36,7 @@
 
 import java.io.IOException;
 import java.net.UnknownHostException;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static org.apache.flink.util.Preconditions.checkArgument;
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -47,6 +49,8 @@ public class RpcServiceUtils {
 
 	private static final Logger LOG = LoggerFactory.getLogger(RpcServiceUtils.class);
 
+	private static final AtomicLong nextNameOffset = new AtomicLong(0L);
+
 	// ------------------------------------------------------------------------
 	//  RPC instantiation
 	// ------------------------------------------------------------------------
@@ -105,7 +109,7 @@ public static RpcService createRpcService(String hostname, int port, Configurati
 	 * @param hostname     The hostname or address where the target RPC service is listening.
 	 * @param port         The port where the target RPC service is listening.
 	 * @param endpointName The name of the RPC endpoint.
-	 * @param config       Teh configuration from which to deduce further settings.
+	 * @param config       The configuration from which to deduce further settings.
 	 *
 	 * @return The RPC URL of the specified RPC endpoint.
 	 */
@@ -144,6 +148,25 @@ public static String getRpcUrl(String hostname, int port, String endpointName, b
 		return String.format("%s://flink@%s/user/%s", protocol, hostPort, endpointName);
 	}
 
+	/**
+	 * Creates a random name of the form prefix_X, where X is an increasing number.
+	 *
+	 * @param prefix Prefix string to prepend to the monotonically increasing name offset number
+	 * @return A random name of the form prefix_X where X is an increasing number
+	 */
+	public static String createRandomName(String prefix) {
+		Preconditions.checkNotNull(prefix, "Prefix must not be null.");
+
+		long nameOffset;
+
+		// obtain the next name offset by incrementing it atomically
+		do {
+			nameOffset = nextNameOffset.get();
+		} while (!nextNameOffset.compareAndSet(nameOffset, nameOffset + 1L));
+
+		return prefix + '_' + nameOffset;
+	}
+
 	// ------------------------------------------------------------------------
 
 	/** This class is not meant to be instantiated */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
index f5ccdbb2010a2..d182b03d83199 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
@@ -187,7 +187,7 @@ public <C extends RpcGateway, S extends RpcEndpoint<C>> C startServer(S rpcEndpo
 
 		synchronized (lock) {
 			checkState(!stopped, "RpcService is stopped");
-			actorRef = actorSystem.actorOf(akkaRpcActorProps);
+			actorRef = actorSystem.actorOf(akkaRpcActorProps, rpcEndpoint.getEndpointId());
 			actors.add(actorRef);
 		}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 6ad0bd980daff..d04cabbac93ae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -67,6 +67,7 @@
 import org.apache.flink.runtime.rpc.RpcEndpoint;
 import org.apache.flink.runtime.rpc.RpcMethod;
 import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.RpcServiceUtils;
 import org.apache.flink.runtime.taskexecutor.exceptions.CheckpointException;
 import org.apache.flink.runtime.taskexecutor.exceptions.PartitionException;
 import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
@@ -85,6 +86,7 @@
 import org.apache.flink.runtime.taskmanager.CheckpointResponder;
 import org.apache.flink.runtime.taskmanager.Task;
 import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.apache.flink.runtime.taskmanager.TaskManager;
 import org.apache.flink.runtime.taskmanager.TaskManagerActions;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.util.ExceptionUtils;
@@ -167,24 +169,24 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
 	// ------------------------------------------------------------------------
 
 	public TaskExecutor(
-		TaskManagerConfiguration taskManagerConfiguration,
-		TaskManagerLocation taskManagerLocation,
-		RpcService rpcService,
-		MemoryManager memoryManager,
-		IOManager ioManager,
-		NetworkEnvironment networkEnvironment,
-		HighAvailabilityServices haServices,
-		HeartbeatServices heartbeatServices,
-		MetricRegistry metricRegistry,
-		TaskManagerMetricGroup taskManagerMetricGroup,
-		BroadcastVariableManager broadcastVariableManager,
-		FileCache fileCache,
-		TaskSlotTable taskSlotTable,
-		JobManagerTable jobManagerTable,
-		JobLeaderService jobLeaderService,
-		FatalErrorHandler fatalErrorHandler) {
-
-		super(rpcService);
+			RpcService rpcService,
+			TaskManagerConfiguration taskManagerConfiguration,
+			TaskManagerLocation taskManagerLocation,
+			MemoryManager memoryManager,
+			IOManager ioManager,
+			NetworkEnvironment networkEnvironment,
+			HighAvailabilityServices haServices,
+			HeartbeatServices heartbeatServices,
+			MetricRegistry metricRegistry,
+			TaskManagerMetricGroup taskManagerMetricGroup,
+			BroadcastVariableManager broadcastVariableManager,
+			FileCache fileCache,
+			TaskSlotTable taskSlotTable,
+			JobManagerTable jobManagerTable,
+			JobLeaderService jobLeaderService,
+			FatalErrorHandler fatalErrorHandler) {
+
+		super(rpcService, RpcServiceUtils.createRandomName(TaskManager.TASK_MANAGER_NAME()));
 
 		checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index c99eb91640dc4..2be8ff1663763 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -123,9 +123,9 @@ public TaskManagerRunner(
 		TaskExecutorMetricsInitializer.instantiateStatusMetrics(taskManagerMetricGroup, taskManagerServices.getNetworkEnvironment());
 
 		this.taskManager = new TaskExecutor(
+			rpcService,
 			taskManagerConfiguration,
 			taskManagerServices.getTaskManagerLocation(),
-			rpcService,
 			taskManagerServices.getMemoryManager(),
 			taskManagerServices.getIOManager(),
 			taskManagerServices.getNetworkEnvironment(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
index 72925bb3cf621..3464129f388b9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
@@ -394,7 +394,9 @@ public void testHeartbeatTimeoutWithTaskExecutor() throws Exception {
 
 		try {
 			final StandaloneResourceManager resourceManager = new StandaloneResourceManager(
-				rpcService, resourceManagerResourceID,
+				rpcService,
+				FlinkResourceManager.RESOURCE_MANAGER_NAME,
+				resourceManagerResourceID,
 				resourceManagerConfiguration,
 				highAvailabilityServices,
 				heartbeatServices,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index 3f9865c83b9c2..37a86c714d241 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -155,11 +155,6 @@ public BlobStore createBlobStore() throws IOException {
 		return new VoidBlobStore();
 	}
 
-	@Override
-	public String getResourceManagerEndpointName() {
-		throw new UnsupportedOperationException();
-	}
-
 	// ------------------------------------------------------------------------
 	//  Shutdown
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
index ee8f51d59e707..8b9b800644896 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
@@ -92,10 +92,10 @@ public void testHeartbeatTimeoutWithTaskManager() throws Exception {
 
 		try {
 			final JobMaster jobMaster = new JobMaster(
+				rpc,
 				jmResourceId,
 				jobGraph,
 				new Configuration(),
-				rpc,
 				haServices,
 				heartbeatServices,
 				Executors.newScheduledThreadPool(1),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
index c8e209d12e329..49bc5703825e2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.resourcemanager;
 
 import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
@@ -28,6 +29,7 @@
 import org.apache.flink.runtime.rpc.RpcService;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
 import org.apache.flink.runtime.util.TestingFatalErrorHandler;
+import org.apache.flink.util.TestLogger;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -38,7 +40,7 @@
 /**
  * resourceManager HA test, including grant leadership and revoke leadership
  */
-public class ResourceManagerHATest {
+public class ResourceManagerHATest extends TestLogger {
 
 	@Test
 	public void testGrantAndRevokeLeadership() throws Exception {
@@ -68,7 +70,9 @@ public void testGrantAndRevokeLeadership() throws Exception {
 
 		final ResourceManager resourceManager =
 			new StandaloneResourceManager(
-				rpcService, rmResourceId,
+				rpcService,
+				FlinkResourceManager.RESOURCE_MANAGER_NAME,
+				rmResourceId,
 				resourceManagerConfiguration,
 				highAvailabilityServices,
 				heartbeatServices,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
index 32b40ac18d826..b6b86141ab4d6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
@@ -217,7 +218,9 @@ private ResourceManager createAndStartResourceManager(
 			Time.minutes(5L));
 
 		ResourceManager resourceManager = new StandaloneResourceManager(
-			rpcService, rmResourceId,
+			rpcService,
+			FlinkResourceManager.RESOURCE_MANAGER_NAME,
+			rmResourceId,
 			resourceManagerConfiguration,
 			highAvailabilityServices,
 			heartbeatServices,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
index cb0a4141906bc..bae7086cd6dc8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.resourcemanager;
 
 import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
@@ -165,6 +166,7 @@ private StandaloneResourceManager createAndStartResourceManager(TestingLeaderEle
 		StandaloneResourceManager resourceManager =
 			new StandaloneResourceManager(
 				rpcService,
+				FlinkResourceManager.RESOURCE_MANAGER_NAME,
 				resourceManagerResourceID,
 				resourceManagerConfiguration,
 				highAvailabilityServices,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 68aff42b88779..4d2309a1e5f63 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -19,6 +19,7 @@
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
@@ -237,7 +238,9 @@ public void testSlotAvailableRequest() throws Exception {
 
 		ResourceManager<ResourceID> resourceManager =
 			Mockito.spy(new StandaloneResourceManager(
-				testRpcService, rmResourceId,
+				testRpcService,
+				FlinkResourceManager.RESOURCE_MANAGER_NAME,
+				rmResourceId,
 				resourceManagerConfiguration,
 				testingHaServices,
 				heartbeatServices,
@@ -324,7 +327,9 @@ public SpiedResourceManager(
 				JobLeaderIdService jobLeaderIdService,
 				FatalErrorHandler fatalErrorHandler) {
 			super(
-				rpcService, resourceId,
+				rpcService,
+				FlinkResourceManager.RESOURCE_MANAGER_NAME,
+				resourceId,
 				resourceManagerConfiguration,
 				highAvailabilityServices,
 				heartbeatServices,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
index fdcd2824ba07d..fa326b5735a25 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
@@ -22,6 +22,7 @@
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
@@ -121,7 +122,9 @@ public void testSlotAllocation() throws Exception {
 		final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
 
 		ResourceManager<ResourceID> resourceManager = new StandaloneResourceManager(
-			rpcService, rmResourceId,
+			rpcService,
+			FlinkResourceManager.RESOURCE_MANAGER_NAME,
+			rmResourceId,
 			resourceManagerConfiguration,
 			testingHAServices,
 			heartbeatServices,
@@ -131,9 +134,9 @@ public void testSlotAllocation() throws Exception {
 			testingFatalErrorHandler);
 
 		TaskExecutor taskExecutor = new TaskExecutor(
+			rpcService,
 			taskManagerConfiguration,
 			taskManagerLocation,
-			rpcService,
 			memoryManager,
 			ioManager,
 			networkEnvironment,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 330d4fb1a7ce5..bc6fe6801de1f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -169,9 +169,9 @@ public HeartbeatManagerImpl<Void, Void> answer(InvocationOnMock invocation) thro
 
 		try {
 			final TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				tmConfig,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
@@ -275,9 +275,9 @@ public HeartbeatManagerImpl<Void, Void> answer(InvocationOnMock invocation) thro
 
 		try {
 			final TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerConfiguration,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
@@ -344,9 +344,9 @@ public void testImmediatelyRegistersIfLeaderIsKnown() throws Exception {
 			final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
 
 			TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerServicesConfiguration,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
@@ -427,9 +427,9 @@ public void testTriggerRegistrationOnLeaderChange() throws Exception {
 			final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
 
 			TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerServicesConfiguration,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
@@ -559,9 +559,9 @@ public void testTaskSubmission() throws Exception {
 			final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
 
 			TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerConfiguration,
 				mock(TaskManagerLocation.class),
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				networkEnvironment,
@@ -676,9 +676,9 @@ public void testJobLeaderDetection() throws Exception {
 
 		try {
 			TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerConfiguration,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
@@ -790,9 +790,9 @@ public void testSlotAcceptance() throws Exception {
 
 		try {
 			TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerConfiguration,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
@@ -867,9 +867,9 @@ public void testRejectAllocationRequestsForOutOfSyncSlots() throws Exception {
 			final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
 
 			TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerServicesConfiguration,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
@@ -1011,9 +1011,9 @@ public void testSubmitTaskBeforeAcceptSlot() throws Exception {
 
 		try {
 			final TaskExecutor taskManager = new TaskExecutor(
+				rpc,
 				taskManagerConfiguration,
 				taskManagerLocation,
-				rpc,
 				mock(MemoryManager.class),
 				mock(IOManager.class),
 				mock(NetworkEnvironment.class),
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
index 4c935f16f690a..65d12b5010bee 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
@@ -26,6 +26,7 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.ApplicationStatus;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
@@ -199,7 +200,9 @@ private ResourceManager<?> createResourceManager(Configuration config) throws Ex
 			commonRpcService.getScheduledExecutor());
 
 		return new YarnResourceManager(
-			commonRpcService, ResourceID.generate(),
+			commonRpcService,
+			FlinkResourceManager.RESOURCE_MANAGER_NAME,
+			ResourceID.generate(),
 			config,
 			ENV,
 			resourceManagerConfiguration,
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
index f8cf2750da0e1..74359c8f9fc8f 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
@@ -108,6 +108,7 @@ public class YarnResourceManager extends ResourceManager<ResourceID> implements
 
 	public YarnResourceManager(
 			RpcService rpcService,
+			String resourceManagerEndpointId,
 			ResourceID resourceId,
 			Configuration flinkConfig,
 			Map<String, String> env,
@@ -120,6 +121,7 @@ public YarnResourceManager(
 			FatalErrorHandler fatalErrorHandler) {
 		super(
 			rpcService,
+			resourceManagerEndpointId,
 			resourceId,
 			resourceManagerConfiguration,
 			highAvailabilityServices,
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
index c3902d34d8fcd..071bb7d06378e 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
@@ -39,7 +39,7 @@ public class YarnConfigOptions {
 	 * The port where the application master RPC system is listening.
 	 */
 	public static final ConfigOption<Integer> APP_MASTER_RPC_PORT =
-			key("yarn.appmaster.rpc.address")
+			key("yarn.appmaster.rpc.port")
 			.defaultValue(-1);
 
 	// ------------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/AbstractYarnNonHaServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/AbstractYarnNonHaServices.java
index 7aa481fd8b552..f1656aa45468f 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/AbstractYarnNonHaServices.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/AbstractYarnNonHaServices.java
@@ -38,9 +38,6 @@
  */
 public abstract class AbstractYarnNonHaServices extends YarnHighAvailabilityServices {
 
-	/** The constant name of the ResourceManager RPC endpoint */
-	protected static final String RESOURCE_MANAGER_RPC_ENDPOINT_NAME = "resource_manager";
-
 	// ------------------------------------------------------------------------
 
 	/**
@@ -61,15 +58,6 @@ protected AbstractYarnNonHaServices(
 		super(config, hadoopConf);
 	}
 
-	// ------------------------------------------------------------------------
-	//  Names
-	// ------------------------------------------------------------------------
-
-	@Override
-	public String getResourceManagerEndpointName() {
-		return RESOURCE_MANAGER_RPC_ENDPOINT_NAME;
-	}
-
 	// ------------------------------------------------------------------------
 	//  Services
 	// ------------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java
index eb4b77e21d65b..72d780f4595b7 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java
@@ -21,6 +21,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
@@ -107,7 +108,7 @@ public YarnPreConfiguredMasterNonHaServices(
 			}
 
 			this.resourceManagerRpcUrl = RpcServiceUtils.getRpcUrl(
-					rmHost, rmPort, RESOURCE_MANAGER_RPC_ENDPOINT_NAME, config);
+					rmHost, rmPort, FlinkResourceManager.RESOURCE_MANAGER_NAME, config);
 
 			// all well!
 			successful = true;
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
index a13deacffb031..c7301029b768f 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
@@ -23,8 +23,6 @@
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.leaderelection.LeaderContender;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.util.TestLogger;
 import org.apache.flink.yarn.configuration.YarnConfigOptions;
 
@@ -40,14 +38,9 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import static org.mockito.Mockito.*;
-
-
 public class YarnPreConfiguredMasterHaServicesTest extends TestLogger {
 
 	@ClassRule
@@ -94,29 +87,6 @@ public void initConfig() {
 	//  Tests
 	// ------------------------------------------------------------------------
 
-	@Test
-	public void testConstantResourceManagerName() throws Exception {
-		final Configuration flinkConfig = new Configuration();
-		flinkConfig.setString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS, "localhost");
-		flinkConfig.setInteger(YarnConfigOptions.APP_MASTER_RPC_PORT, 1427);
-
-		YarnHighAvailabilityServices services1 = new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
-		YarnHighAvailabilityServices services2 = new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
-
-		try {
-			String rmName1 = services1.getResourceManagerEndpointName();
-			String rmName2 = services2.getResourceManagerEndpointName();
-
-			assertNotNull(rmName1);
-			assertNotNull(rmName2);
-			assertEquals(rmName1, rmName2);
-		}
-		finally {
-			services1.closeAndCleanupAllData();
-			services2.closeAndCleanupAllData();
-		}
-	}
-
 	@Test
 	public void testMissingRmConfiguration() throws Exception {
 		final Configuration flinkConfig = new Configuration();

From 698f17d215955b346b0a7c7ab436a4d5dee6987f Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Tue, 25 Apr 2017 15:10:27 -0400
Subject: [PATCH 045/253] [hotfix] Fix Scalastyle for broken build

---
 .../flink/runtime/jobmanager/JobManagerRegistrationTest.scala | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
index 4f8eab2bb8eb8..6be8bcc049ad8 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
@@ -152,7 +152,9 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
         val rm = startTestingResourceManager(_system, jm.actor())
         rmOption = Some(rm)
 
-        val selfGateway = new AkkaActorGateway(probe.ref, HighAvailabilityServices.DEFAULT_LEADER_ID)
+        val selfGateway = new AkkaActorGateway(
+          probe.ref,
+          HighAvailabilityServices.DEFAULT_LEADER_ID)
 
         val resourceID = ResourceID.generate()
         val connectionInfo = new TaskManagerLocation(resourceID, InetAddress.getLocalHost, 1)

From a15499aa197e2446fe889939ba2164a8531c6353 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Fri, 17 Mar 2017 15:32:23 +0100
Subject: [PATCH 046/253] [FLINK-6107] Custom checkstyle for
 flink-streaming-java

This is based on the Apache Beam checkstyle with some custom
modifications for making it fit the exisiting Flink code base. The most
notable change is that Flink uses Tabs for indentation while Beam uses
spaces.

This adds a lot of checks that are commented out because they require
non-trivial changes to the code base.

There are some trivial code changes for good practices checkstyle rules
that where only broken in very few places.
---
 flink-streaming-java/pom.xml                  |  35 ++
 .../collector/selector/OutputSelector.java    |   2 +-
 .../streaming/api/datastream/DataStream.java  |   2 +-
 .../source/ContinuousFileReaderOperator.java  |   2 +-
 .../MessageAcknowledgingSourceBase.java       |   4 +-
 .../windowing/delta/DeltaFunction.java        |   2 +-
 .../windowing/delta/extractor/Extractor.java  |   2 +-
 .../flink/streaming/api/graph/StreamEdge.java |   8 +-
 .../api/graph/StreamGraphGenerator.java       |   2 +-
 .../flink/streaming/api/graph/StreamNode.java |   2 +-
 .../api/operators/StreamSourceContexts.java   |   2 +-
 .../streaming/api/operators/Triggerable.java  |   2 +-
 .../runtime/operators/windowing/KeyMap.java   |   4 +-
 .../operators/windowing/WindowOperator.java   |   2 +-
 .../runtime/tasks/ProcessingTimeCallback.java |   2 +-
 .../util/typeutils/FieldAccessor.java         |   4 +-
 .../util/typeutils/FieldAccessorFactory.java  |   6 +-
 tools/maven/strict-checkstyle.xml             | 550 ++++++++++++++++++
 18 files changed, 609 insertions(+), 24 deletions(-)
 create mode 100644 tools/maven/strict-checkstyle.xml

diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml
index 950c923fc4d16..72c2133722a28 100644
--- a/flink-streaming-java/pom.xml
+++ b/flink-streaming-java/pom.xml
@@ -123,6 +123,41 @@ under the License.
 					</execution>
 				</executions>
 			</plugin>
+
+			<!-- We have more strict checkstyle rules than the rest of the project -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-checkstyle-plugin</artifactId>
+				<version>2.17</version>
+				<dependencies>
+					<dependency>
+						<groupId>com.puppycrawl.tools</groupId>
+						<artifactId>checkstyle</artifactId>
+						<version>6.19</version>
+					</dependency>
+				</dependencies>
+				<configuration>
+					<configLocation>../tools/maven/strict-checkstyle.xml</configLocation>
+					<suppressionsLocation>../tools/maven/suppressions.xml</suppressionsLocation>
+					<logViolationsToConsole>true</logViolationsToConsole>
+					<failOnViolation>true</failOnViolation>
+				</configuration>
+				<executions>
+					<!--
+					Execute checkstyle after compilation but before tests.
+
+					This ensures that any parsing or type checking errors are from
+					javac, so they look as expected. Beyond that, we want to
+					fail as early as possible.
+					-->
+					<execution>
+						<phase>test-compile</phase>
+						<goals>
+							<goal>check</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
 		</plugins>
 	</build>
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java
index 6632da48a129f..126effef275bf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java
@@ -42,5 +42,5 @@ public interface OutputSelector<OUT> extends Serializable {
 	 * @param value
 	 *            Output object for which the output selection should be made.
 	 */
-	public Iterable<String> select(OUT value);
+	Iterable<String> select(OUT value);
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 71ef048d6ef1d..fc70416e7d9d9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -1147,7 +1147,7 @@ public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {
 
 		// configure the type if needed
 		if (sinkFunction instanceof InputTypeConfigurable) {
-			((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig() );
+			((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
 		}
 
 		StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java
index b86d97c476a64..6e8d01d321271 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java
@@ -367,7 +367,7 @@ public void run() {
 
 		private List<TimestampedFileInputSplit> getReaderState() throws IOException {
 			List<TimestampedFileInputSplit> snapshot = new ArrayList<>(this.pendingSplits.size());
-			if (currentSplit != null ) {
+			if (currentSplit != null) {
 				if (this.format instanceof CheckpointableInputFormat && this.isSplitOpen) {
 					Serializable formatState =
 						((CheckpointableInputFormat<TimestampedFileInputSplit, Serializable>) this.format).getCurrentState();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java
index 035b7ddd1c956..e835070d0fde7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java
@@ -178,9 +178,9 @@ public void close() throws Exception {
 
 	/**
 	 * This method must be implemented to acknowledge the given set of IDs back to the message queue.
-	 * @param UIds The list od IDs to acknowledge.
+	 * @param uIds The list od IDs to acknowledge.
 	 */
-	protected abstract void acknowledgeIDs(long checkpointId, List<UId> UIds);
+	protected abstract void acknowledgeIDs(long checkpointId, List<UId> uIds);
 
 	/**
 	 * Adds an ID to be stored with the current checkpoint.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
index b94e453c07cf6..d9295c42a72ad 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
@@ -42,6 +42,6 @@ public interface DeltaFunction<DATA> extends Serializable {
 	 *            the new data point.
 	 * @return the delta between the two given points.
 	 */
-	public double getDelta(DATA oldDataPoint, DATA newDataPoint);
+	double getDelta(DATA oldDataPoint, DATA newDataPoint);
 
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
index a94f09a2fc893..0b7e4da00056d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
@@ -41,6 +41,6 @@ public interface Extractor<FROM, TO> extends Serializable {
 	 *            the input data
 	 * @return the extracted/converted data
 	 */
-	public TO extract(FROM in);
+	TO extract(FROM in);
 
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
index 8e1c36165ca67..94e624ff4e9b4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
@@ -34,15 +34,15 @@ public class StreamEdge implements Serializable {
 
 	private static final long serialVersionUID = 1L;
 
-	final private String edgeId;
+	private final String edgeId;
 
-	final private StreamNode sourceVertex;
-	final private StreamNode targetVertex;
+	private final StreamNode sourceVertex;
+	private final StreamNode targetVertex;
 
 	/**
 	 * The type number of the input for co-tasks.
 	 */
-	final private int typeNumber;
+	private final int typeNumber;
 
 	/**
 	 * A list of output names that the target vertex listens to (if there is
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
index f9eec4f10caf7..3b9a506cfb4db 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
@@ -369,7 +369,7 @@ private <T> Collection<Integer> transformFeedback(FeedbackTransformation<T> iter
 			iterate.getParallelism(),
 			iterate.getMaxParallelism(),
 			iterate.getMinResources(),
-			iterate.getPreferredResources()	);
+			iterate.getPreferredResources());
 
 		StreamNode itSource = itSourceAndSink.f0;
 		StreamNode itSink = itSourceAndSink.f1;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
index e4164bad65e91..71f2a4a4e1bb5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
@@ -40,7 +40,7 @@ public class StreamNode implements Serializable {
 
 	private static final long serialVersionUID = 1L;
 
-	transient private StreamExecutionEnvironment env;
+	private transient StreamExecutionEnvironment env;
 
 	private final int id;
 	private Integer parallelism = null;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
index e4d051c01f085..36d10d3c98004 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
@@ -336,7 +336,7 @@ protected boolean allowWatermark(Watermark mark) {
 	 * toggles the status. ACTIVE status resumes as soon as some record or watermark is collected
 	 * again.
 	 */
-	private static abstract class WatermarkContext<T> implements SourceFunction.SourceContext<T> {
+	private abstract static class WatermarkContext<T> implements SourceFunction.SourceContext<T> {
 
 		protected final ProcessingTimeService timeService;
 		protected final Object checkpointLock;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Triggerable.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Triggerable.java
index 36e9ad1952459..48b18b33057fa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Triggerable.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Triggerable.java
@@ -36,5 +36,5 @@ public interface Triggerable<K, N> {
 	/**
 	 * Invoked when a processing-time timer fires.
 	 */
-	void onProcessingTime(InternalTimer<K, N> timer) throws Exception ;
+	void onProcessingTime(InternalTimer<K, N> timer) throws Exception;
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
index 52c321c731a39..b0f4f46949bb2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
@@ -606,7 +606,7 @@ public int compare(KeyMap<?, ?> o1, KeyMap<?, ?> o2) {
 	 *
 	 * @param <V> The type created by the factory.
 	 */
-	public static interface LazyFactory<V> {
+	public interface LazyFactory<V> {
 
 		/**
 		 * The factory method; creates the value.
@@ -625,7 +625,7 @@ public static interface LazyFactory<V> {
 	 * @param <K> The type of the key.
 	 * @param <V> The type of the value.
 	 */
-	public static interface TraversalEvaluator<K, V> {
+	public interface TraversalEvaluator<K, V> {
 
 		/**
 		 * Called whenever the traversal starts with a new key.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index 3d407167b5cf1..70692517960d9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -310,7 +310,7 @@ public long getCurrentProcessingTime() {
 
 			final TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>(
 					typedTuple,
-					new TypeSerializer[] {windowSerializer, windowSerializer} );
+					new TypeSerializer[] {windowSerializer, windowSerializer});
 
 			final ListStateDescriptor<Tuple2<W, W>> mergingSetsStateDescriptor =
 					new ListStateDescriptor<>("merging-window-set", tupleSerializer);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
index 035939f7b3bd7..7588c3d54c9cb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
@@ -36,5 +36,5 @@ public interface ProcessingTimeCallback {
 	 * 
 	 * @param timestamp The timestamp for which the trigger event was scheduled.
 	 */
-	void onProcessingTime(long timestamp) throws Exception ;
+	void onProcessingTime(long timestamp) throws Exception;
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
index 2828308589c2e..21f477ae2ce13 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
@@ -92,7 +92,7 @@ public TypeInformation<F> getFieldType() {
 	 * This is when the entire record is considered as a single field. (eg. field 0 of a basic type, or a
 	 * field of a POJO that is itself some composite type but is not further decomposed)
 	 */
-	final static class SimpleFieldAccessor<T> extends FieldAccessor<T, T> {
+	static final class SimpleFieldAccessor<T> extends FieldAccessor<T, T> {
 
 		private static final long serialVersionUID = 1L;
 
@@ -113,7 +113,7 @@ public T set(T record, T fieldValue) {
 		}
 	}
 
-	final static class ArrayFieldAccessor<T, F> extends FieldAccessor<T, F> {
+	static final class ArrayFieldAccessor<T, F> extends FieldAccessor<T, F> {
 
 		private static final long serialVersionUID = 1L;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
index 6dbeedd6ab41c..1bfce5ba5468d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
@@ -198,9 +198,9 @@ public static <T, F> FieldAccessor<T, F> getAccessor(TypeInformation<T> typeInfo
 
 	// --------------------------------------------------------------------------------------------------
 
-	private final static String REGEX_FIELD = "[\\p{L}\\p{Digit}_\\$]*"; // This can start with a digit (because of Tuples)
-	private final static String REGEX_NESTED_FIELDS = "("+REGEX_FIELD+")(\\.(.+))?";
-	private final static String REGEX_NESTED_FIELDS_WILDCARD = REGEX_NESTED_FIELDS
+	private  static final String REGEX_FIELD = "[\\p{L}\\p{Digit}_\\$]*"; // This can start with a digit (because of Tuples)
+	private static final String REGEX_NESTED_FIELDS = "("+REGEX_FIELD+")(\\.(.+))?";
+	private static final String REGEX_NESTED_FIELDS_WILDCARD = REGEX_NESTED_FIELDS
 		+"|\\"+ Keys.ExpressionKeys.SELECT_ALL_CHAR
 		+"|\\"+ Keys.ExpressionKeys.SELECT_ALL_CHAR_SCALA;
 
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
new file mode 100644
index 0000000000000..ba2f4be6efe0c
--- /dev/null
+++ b/tools/maven/strict-checkstyle.xml
@@ -0,0 +1,550 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE module PUBLIC
+    "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+    "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+
+<!--
+This is a checkstyle configuration file. For descriptions of
+what the following rules do, please see the checkstyle configuration
+page at http://checkstyle.sourceforge.net/config.html.
+
+This file is based on the checkstyle file of Apache Beam.
+-->
+
+<module name="Checker">
+
+  <!--<module name="FileTabCharacter">-->
+    <!--&lt;!&ndash; Checks that there are no tab characters in the file. &ndash;&gt;-->
+  <!--</module>-->
+
+  <!--<module name="NewlineAtEndOfFile">-->
+    <!--&lt;!&ndash; windows can use \n\r vs \n, so enforce the most used one ie UNIx style &ndash;&gt;-->
+    <!--<property name="lineSeparator" value="lf" />-->
+  <!--</module>-->
+
+  <module name="RegexpSingleline">
+    <!-- Checks that TODOs don't have stuff in parenthesis, e.g., username. -->
+    <property name="format" value="((//.*)|(\*.*))TODO\(" />
+    <property name="message" value="TODO comments must not include usernames." />
+    <property name="severity" value="error" />
+  </module>
+
+  <!--<module name="RegexpSingleline">-->
+    <!--<property name="format" value="\s+$"/>-->
+    <!--<property name="message" value="Trailing whitespace"/>-->
+    <!--<property name="severity" value="error"/>-->
+  <!--</module>-->
+
+  <module name="RegexpSingleline">
+    <property name="format" value="Throwables.propagate\("/>
+    <property name="message" value="Throwables.propagate is deprecated"/>
+    <property name="severity" value="error"/>
+  </module>
+
+  <!-- Prevent *Tests.java as tools may not pick them up -->
+  <!--<module name="RegexpOnFilename">-->
+    <!--<property name="fileNamePattern" value=".*Tests\.java$" />-->
+  <!--</module>-->
+
+  <!-- Allow use of comment to suppress javadocstyle -->
+  <module name="SuppressionCommentFilter">
+    <property name="offCommentFormat" value="CHECKSTYLE.OFF\: ([\w\|]+)"/>
+    <property name="onCommentFormat" value="CHECKSTYLE.ON\: ([\w\|]+)"/>
+    <property name="checkFormat" value="$1"/>
+  </module>
+  <module name="SuppressionFilter">
+    <property name="file" value="${checkstyle.suppressions.file}" default="suppressions.xml" />
+  </module>
+
+  <!-- Check that every module has a package-info.java -->
+  <!--<module name="JavadocPackage"/>-->
+
+  <!--
+
+  FLINK CUSTOM CHECKS
+
+  -->
+
+  <module name="FileLength">
+    <property name="max" value="2500"/>
+  </module>
+
+  <!-- All Java AST specific tests live under TreeWalker module. -->
+  <module name="TreeWalker">
+
+    <!--
+
+    FLINK CUSTOM CHECKS
+
+    -->
+
+    <module name="RegexpSinglelineJava">
+      <property name="format" value="^\t* +\t*\S" />
+      <property name="message"
+                value="Line has leading space characters; indentation should be performed with tabs only." />
+      <property name="ignoreComments" value="true" />
+    </module>
+
+    <!-- Prohibit T.getT() methods for standard boxed types -->
+    <module name="Regexp">
+      <property name="format" value="Boolean\.getBoolean"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use System.getProperties() to get system properties."/>
+    </module>
+
+    <module name="Regexp">
+      <property name="format" value="Integer\.getInteger"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use System.getProperties() to get system properties."/>
+    </module>
+
+    <module name="Regexp">
+      <property name="format" value="Long\.getLong"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use System.getProperties() to get system properties."/>
+    </module>
+
+    <!--
+
+    IllegalImport cannot blacklist classes so we have to fall back to Regexp.
+
+    -->
+
+    <!-- forbid use of commons lang validate -->
+    <module name="Regexp">
+      <property name="format" value="org\.apache\.commons\.lang3\.Validate"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use Guava Checks instead of Commons Validate. Please refer to the coding guidelines."/>
+    </module>
+    <!-- forbid the use of google.common.base.Preconditions -->
+    <module name="Regexp">
+      <property name="format" value="import com\.google\.common\.base\.Preconditions"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use Flink's Preconditions instead of Guava's Preconditions"/>
+    </module>
+    <!-- forbid the use of com.google.common.annotations.VisibleForTesting -->
+    <module name="Regexp">
+      <property name="format" value="import com\.google\.common\.annotations\.VisibleForTesting"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use Flink's VisibleForTesting instead of Guava's VisibleForTesting"/>
+    </module>
+    <module name="Regexp">
+      <property name="format" value="import static com\.google\.common\.base\.Preconditions"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use Flink's Preconditions instead of Guava's Preconditions"/>
+    </module>
+    <!-- forbid the use of org.apache.commons.lang.SerializationUtils -->
+    <module name="Regexp">
+      <property name="format" value="org\.apache\.commons\.lang\.SerializationUtils"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use Flink's InstantiationUtil instead of common's SerializationUtils"/>
+    </module>
+    <module name="Regexp">
+      <property name="format" value="org\.apache\.commons\.lang3\.SerializationUtils"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use Flink's InstantiationUtil instead of common's SerializationUtils"/>
+    </module>
+    <module name="Regexp">
+      <property name="format" value="org\.apache\.commons\.lang\."/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use commons-lang3 instead of commons-lang."/>
+    </module>
+    <module name="Regexp">
+      <property name="format" value="org\.codehaus\.jettison"/>
+      <property name="illegalPattern" value="true"/>
+      <property name="message" value="Use com.fasterxml.jackson instead of jettison."/>
+    </module>
+
+    <!-- Enforce Java-style array declarations -->
+    <module name="ArrayTypeStyle" />
+
+    <module name="TodoComment">
+      <!-- Checks that disallowed strings are not used in comments.  -->
+      <property name="format" value="(FIXME)|(XXX)|(@author)" />
+    </module>
+
+    <!--
+
+    IMPORT CHECKS
+
+    -->
+
+    <module name="RedundantImport">
+      <!-- Checks for redundant import statements. -->
+      <property name="severity" value="error"/>
+      <message key="import.redundancy"
+               value="Redundant import {0}."/>
+    </module>
+
+    <!--<module name="ImportOrder">-->
+      <!--&lt;!&ndash; Checks for out of order import statements. &ndash;&gt;-->
+      <!--<property name="severity" value="error"/>-->
+      <!--&lt;!&ndash; This ensures that static imports go first. &ndash;&gt;-->
+      <!--<property name="option" value="top"/>-->
+      <!--<property name="sortStaticImportsAlphabetically" value="true"/>-->
+      <!--<property name="tokens" value="STATIC_IMPORT, IMPORT"/>-->
+      <!--<message key="import.ordering"-->
+               <!--value="Import {0} appears after other imports that it should precede"/>-->
+    <!--</module>-->
+
+    <module name="AvoidStarImport">
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="IllegalImport">
+      <property name="illegalPkgs" value="org.apache.flink.shaded, autovalue.shaded, avro.shaded, com.google.api.client.repackaged, com.google.appengine.repackaged, io.netty.util.internal"/>
+    </module>
+
+    <module name="RedundantModifier">
+      <!-- Checks for redundant modifiers on various symbol definitions.
+        See: http://checkstyle.sourceforge.net/config_modifier.html#RedundantModifier
+      -->
+      <property name="tokens" value="METHOD_DEF, VARIABLE_DEF, ANNOTATION_FIELD_DEF, INTERFACE_DEF, CLASS_DEF, ENUM_DEF"/>
+    </module>
+
+    <!--
+        IllegalImport cannot blacklist classes, and c.g.api.client.util is used for some shaded
+        code and some useful code. So we need to fall back to Regexp.
+    -->
+    <!--<module name="RegexpSinglelineJava">-->
+      <!--<property name="format" value="com\.google\.api\.client\.util\.(ByteStreams|Charsets|Collections2|Joiner|Lists|Maps|Objects|Preconditions|Sets|Strings|Throwables)"/>-->
+    <!--</module>-->
+
+    <!--
+         Require static importing from Preconditions.
+    -->
+    <module name="RegexpSinglelineJava">
+      <property name="format" value="^import com.google.common.base.Preconditions;$"/>
+      <property name="message" value="Static import functions from Guava Preconditions"/>
+    </module>
+
+    <module name="UnusedImports">
+      <property name="severity" value="error"/>
+      <property name="processJavadoc" value="true"/>
+      <message key="import.unused"
+               value="Unused import: {0}."/>
+    </module>
+
+    <!--
+
+    JAVADOC CHECKS
+
+    -->
+
+    <!-- Checks for Javadoc comments.                     -->
+    <!-- See http://checkstyle.sf.net/config_javadoc.html -->
+    <!--<module name="JavadocMethod">-->
+      <!--<property name="scope" value="protected"/>-->
+      <!--<property name="severity" value="error"/>-->
+      <!--<property name="allowMissingJavadoc" value="true"/>-->
+      <!--<property name="allowMissingParamTags" value="true"/>-->
+      <!--<property name="allowMissingReturnTag" value="true"/>-->
+      <!--<property name="allowMissingThrowsTags" value="true"/>-->
+      <!--<property name="allowThrowsTagsForSubclasses" value="true"/>-->
+      <!--<property name="allowUndeclaredRTE" value="true"/>-->
+    <!--</module>-->
+
+    <!--&lt;!&ndash; Check that paragraph tags are used correctly in Javadoc. &ndash;&gt;-->
+    <!--<module name="JavadocParagraph"/>-->
+
+    <!--<module name="JavadocType">-->
+      <!--<property name="scope" value="protected"/>-->
+      <!--<property name="severity" value="error"/>-->
+      <!--<property name="allowMissingParamTags" value="true"/>-->
+    <!--</module>-->
+
+    <!--<module name="JavadocStyle">-->
+      <!--<property name="severity" value="error"/>-->
+      <!--<property name="checkHtml" value="true"/>-->
+    <!--</module>-->
+
+    <!--
+
+    NAMING CHECKS
+
+    -->
+
+    <!-- Item 38 - Adhere to generally accepted naming conventions -->
+
+    <module name="PackageName">
+      <!-- Validates identifiers for package names against the
+        supplied expression. -->
+      <!-- Here the default checkstyle rule restricts package name parts to
+        seven characters, this is not in line with common practice at Google.
+      -->
+      <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]{1,})*$"/>
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="TypeNameCheck">
+      <!-- Validates static, final fields against the
+      expression "^[A-Z][a-zA-Z0-9]*$". -->
+      <metadata name="altname" value="TypeName"/>
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="ConstantNameCheck">
+      <!-- Validates non-private, static, final fields against the supplied
+      public/package final fields "^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$". -->
+      <metadata name="altname" value="ConstantName"/>
+      <property name="applyToPublic" value="true"/>
+      <property name="applyToProtected" value="true"/>
+      <property name="applyToPackage" value="true"/>
+      <property name="applyToPrivate" value="false"/>
+      <property name="format" value="^([A-Z][A-Z0-9]*(_[A-Z0-9]+)*|FLAG_.*)$"/>
+      <message key="name.invalidPattern"
+               value="Variable ''{0}'' should be in ALL_CAPS (if it is a constant) or be private (otherwise)."/>
+      <property name="severity" value="error"/>
+    </module>
+
+    <!--<module name="StaticVariableNameCheck">-->
+      <!--&lt;!&ndash; Validates static, non-final fields against the supplied-->
+      <!--expression "^[a-z][a-zA-Z0-9]*_?$". &ndash;&gt;-->
+      <!--<metadata name="altname" value="StaticVariableName"/>-->
+      <!--<property name="applyToPublic" value="true"/>-->
+      <!--<property name="applyToProtected" value="true"/>-->
+      <!--<property name="applyToPackage" value="true"/>-->
+      <!--<property name="applyToPrivate" value="true"/>-->
+      <!--<property name="format" value="^[a-z][a-zA-Z0-9]*_?$"/>-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <!--<module name="MemberNameCheck">-->
+      <!--&lt;!&ndash; Validates non-static members against the supplied expression. &ndash;&gt;-->
+      <!--<metadata name="altname" value="MemberName"/>-->
+      <!--<property name="applyToPublic" value="true"/>-->
+      <!--<property name="applyToProtected" value="true"/>-->
+      <!--<property name="applyToPackage" value="true"/>-->
+      <!--<property name="applyToPrivate" value="true"/>-->
+      <!--<property name="format" value="^[a-z][a-zA-Z0-9]*$"/>-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <module name="MethodNameCheck">
+      <!-- Validates identifiers for method names. -->
+      <metadata name="altname" value="MethodName"/>
+      <property name="format" value="^[a-z][a-zA-Z0-9]*(_[a-zA-Z0-9]+)*$"/>
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="ParameterName">
+      <!-- Validates identifiers for method parameters against the
+        expression "^[a-z][a-zA-Z0-9]*$". -->
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="LocalFinalVariableName">
+      <!-- Validates identifiers for local final variables against the
+        expression "^[a-z][a-zA-Z0-9]*$". -->
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="LocalVariableName">
+      <!-- Validates identifiers for local variables against the
+        expression "^[a-z][a-zA-Z0-9]*$". -->
+      <property name="severity" value="error"/>
+    </module>
+
+    <!-- Type parameters must be either one of the four blessed letters
+    T, K, V, W, X or else be capital-case terminated with a T,
+    such as MyGenericParameterT -->
+    <!--<module name="ClassTypeParameterName">-->
+      <!--<property name="format" value="^(((T|K|V|W|X)[0-9]*)|([A-Z][a-z][a-zA-Z]*T))$"/>-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <!--<module name="MethodTypeParameterName">-->
+      <!--<property name="format" value="^(((T|K|V|W|X)[0-9]*)|([A-Z][a-z][a-zA-Z]*T))$"/>-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <!--<module name="InterfaceTypeParameterName">-->
+      <!--<property name="format" value="^(((T|K|V|W|X)[0-9]*)|([A-Z][a-z][a-zA-Z]*T))$"/>-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <!--
+
+    LENGTH and CODING CHECKS
+
+    -->
+
+    <!--<module name="LineLength">-->
+      <!--&lt;!&ndash; Checks if a line is too long. &ndash;&gt;-->
+      <!--<property name="max" value="100"/>-->
+      <!--<property name="severity" value="error"/>-->
+
+      <!--&lt;!&ndash;-->
+        <!--The default ignore pattern exempts the following elements:-->
+          <!-- - import statements-->
+          <!-- - long URLs inside comments-->
+      <!--&ndash;&gt;-->
+
+      <!--<property name="ignorePattern"-->
+          <!--value="^(package .*;\s*)|(import .*;\s*)|( *\* .*https?://.*)$"/>-->
+    <!--</module>-->
+
+    <!--<module name="LeftCurly">-->
+      <!--&lt;!&ndash; Checks for placement of the left curly brace ('{'). &ndash;&gt;-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <!--<module name="RightCurly">-->
+      <!--&lt;!&ndash; Checks right curlies on CATCH, ELSE, and TRY blocks are on-->
+      <!--the same line. e.g., the following example is fine:-->
+      <!--<pre>-->
+        <!--if {-->
+          <!--...-->
+        <!--} else-->
+      <!--</pre>-->
+      <!--&ndash;&gt;-->
+      <!--&lt;!&ndash; This next example is not fine:-->
+      <!--<pre>-->
+        <!--if {-->
+          <!--...-->
+        <!--}-->
+        <!--else-->
+      <!--</pre>-->
+      <!--&ndash;&gt;-->
+      <!--<property name="option" value="same"/>-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <!-- Checks for braces around if and else blocks -->
+    <module name="NeedBraces">
+      <property name="severity" value="error"/>
+      <property name="tokens" value="LITERAL_IF, LITERAL_ELSE, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO"/>
+    </module>
+
+    <module name="UpperEll">
+      <!-- Checks that long constants are defined with an upper ell.-->
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="FallThrough">
+      <!-- Warn about falling through to the next case statement.  Similar to
+      javac -Xlint:fallthrough, but the check is suppressed if a single-line comment
+      on the last non-blank line preceding the fallen-into case contains 'fall through' (or
+      some other variants that we don't publicized to promote consistency).
+      -->
+      <property name="reliefPattern"
+       value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on"/>
+      <property name="severity" value="error"/>
+    </module>
+
+    <!-- Checks for over-complicated boolean expressions. -->
+    <module name="SimplifyBooleanExpression"/>
+
+    <!-- Detects empty statements (standalone ";" semicolon). -->
+    <module name="EmptyStatement"/>
+
+    <!--
+
+    MODIFIERS CHECKS
+
+    -->
+
+    <module name="ModifierOrder">
+      <!-- Warn if modifier order is inconsistent with JLS3 8.1.1, 8.3.1, and
+           8.4.3.  The prescribed order is:
+           public, protected, private, abstract, static, final, transient, volatile,
+           synchronized, native, strictfp
+        -->
+      <property name="severity" value="error"/>
+    </module>
+
+
+    <!--
+
+    WHITESPACE CHECKS
+
+    -->
+
+    <!--<module name="WhitespaceAround">-->
+      <!--&lt;!&ndash; Checks that various tokens are surrounded by whitespace.-->
+           <!--This includes most binary operators and keywords followed-->
+           <!--by regular or curly braces.-->
+      <!--&ndash;&gt;-->
+      <!--<property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR,-->
+        <!--BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN,-->
+        <!--EQUAL, GE, GT, LAND, LE, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,-->
+        <!--LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,-->
+        <!--LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS,-->
+        <!--MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION,-->
+        <!--SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN"/>-->
+      <!--<property name="severity" value="error"/>-->
+    <!--</module>-->
+
+    <!--<module name="WhitespaceAfter">-->
+      <!--&lt;!&ndash; Checks that commas, semicolons and typecasts are followed by-->
+           <!--whitespace.-->
+      <!--&ndash;&gt;-->
+      <!--<property name="tokens" value="COMMA, SEMI, TYPECAST"/>-->
+    <!--</module>-->
+
+    <module name="NoWhitespaceAfter">
+      <!-- Checks that there is no whitespace after various unary operators.
+           Linebreaks are allowed.
+      -->
+      <property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS,
+        UNARY_PLUS"/>
+      <property name="allowLineBreaks" value="true"/>
+      <property name="severity" value="error"/>
+    </module>
+
+    <module name="NoWhitespaceBefore">
+      <!-- Checks that there is no whitespace before various unary operators.
+           Linebreaks are allowed.
+      -->
+      <property name="tokens" value="SEMI, DOT, POST_DEC, POST_INC"/>
+      <property name="allowLineBreaks" value="true"/>
+      <property name="severity" value="error"/>
+    </module>
+
+    <!--<module name="OperatorWrap">-->
+      <!--&lt;!&ndash; Checks that operators like + and ? appear at newlines rather than-->
+           <!--at the end of the previous line.-->
+      <!--&ndash;&gt;-->
+      <!--<property name="option" value="NL"/>-->
+      <!--<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL,-->
+        <!--GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD,-->
+        <!--NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR "/>-->
+    <!--</module>-->
+
+    <module name="OperatorWrap">
+      <!-- Checks that assignment operators are at the end of the line. -->
+      <property name="option" value="eol"/>
+      <property name="tokens" value="ASSIGN"/>
+    </module>
+
+    <module name="ParenPad">
+      <!-- Checks that there is no whitespace before close parens or after
+           open parens.
+      -->
+      <property name="severity" value="error"/>
+    </module>
+
+    <!-- Required to support SuppressWarningsComment -->
+    <module name="FileContentsHolder"/>
+
+  </module>
+</module>
+

From 2fe6079089dd87a25e4038e429a79f119126630f Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 11:21:17 +0100
Subject: [PATCH 047/253] [FLINK-6107] Enable Javadoc checks in streaming
 checkstyle

---
 .../MultiplexingStreamRecordSerializer.java   |   4 +
 .../streamrecord/StreamRecordSerializer.java  |   3 +-
 .../streaming/api/TimeCharacteristic.java     |  64 +++---
 .../api/checkpoint/Checkpointed.java          |   5 +-
 .../CheckpointedAsynchronously.java           |   3 +-
 .../api/checkpoint/CheckpointedFunction.java  |  15 +-
 .../api/checkpoint/CheckpointedRestoring.java |   2 +-
 .../api/checkpoint/ListCheckpointed.java      |  11 +-
 .../collector/selector/DirectedOutput.java    |   5 +-
 .../selector/OutputSelectorWrapper.java       |  28 ---
 .../api/datastream/AllWindowedStream.java     |  78 +++----
 .../api/datastream/AsyncDataStream.java       |   8 +-
 .../api/datastream/CoGroupedStreams.java      |  23 +-
 .../api/datastream/ConnectedStreams.java      |   4 +-
 .../streaming/api/datastream/DataStream.java  | 101 ++++----
 .../api/datastream/DataStreamSink.java        |  11 +-
 .../api/datastream/IterativeStream.java       |  15 +-
 .../api/datastream/JoinedStreams.java         |  17 +-
 .../streaming/api/datastream/KeyedStream.java |  28 ++-
 .../SingleOutputStreamOperator.java           |  56 ++---
 .../api/datastream/StreamProjection.java      |   6 +-
 .../api/datastream/WindowedStream.java        |  87 +++----
 .../api/environment/CheckpointConfig.java     |  18 +-
 .../Flip6LocalStreamEnvironment.java          |   2 +-
 .../environment/LocalStreamEnvironment.java   |   2 +-
 .../environment/RemoteStreamEnvironment.java  |  13 +-
 .../environment/StreamContextEnvironment.java |   5 +
 .../StreamExecutionEnvironment.java           | 217 +++++++++---------
 .../environment/StreamPlanEnvironment.java    |   4 +
 .../AscendingTimestampExtractor.java          |   2 +-
 .../api/functions/TimestampExtractor.java     |  12 +-
 .../aggregation/AggregationFunction.java      |   7 +-
 .../aggregation/ComparableAggregator.java     |   4 +
 .../api/functions/aggregation/Comparator.java |   3 +
 .../functions/aggregation/SumAggregator.java  |   3 +
 .../functions/aggregation/SumFunction.java    |  17 +-
 .../functions/async/RichAsyncFunction.java    |   7 +-
 .../async/collector/AsyncCollector.java       |  11 +-
 .../api/functions/co/CoProcessFunction.java   |   4 +-
 .../api/functions/sink/RichSinkFunction.java  |   3 +
 .../api/functions/sink/SocketClientSink.java  |  10 +-
 .../api/functions/sink/WriteFormat.java       |   2 +-
 .../ContinuousFileMonitoringFunction.java     |   5 +-
 .../source/FileMonitoringFunction.java        |   6 +
 .../functions/source/FileReadFunction.java    |   3 +
 .../source/FromElementsFunction.java          |  15 +-
 .../source/FromIteratorFunction.java          |   3 +
 .../FromSplittableIteratorFunction.java       |   3 +
 .../source/InputFormatSourceFunction.java     |   3 +
 .../MessageAcknowledgingSourceBase.java       |  30 +--
 ...ipleIdsMessageAcknowledgingSourceBase.java |   8 +-
 .../source/SocketTextStreamFunction.java      |  19 +-
 .../api/functions/source/SourceFunction.java  |   2 -
 .../source/TimestampedFileInputSplit.java     |  13 +-
 .../AscendingTimestampExtractor.java          |   2 +-
 .../AggregateApplyAllWindowFunction.java      |   2 +-
 .../AggregateApplyWindowFunction.java         |   2 +-
 .../windowing/FoldApplyAllWindowFunction.java |   6 +-
 .../FoldApplyProcessAllWindowFunction.java    |   7 +-
 .../FoldApplyProcessWindowFunction.java       |   7 +-
 .../windowing/FoldApplyWindowFunction.java    |   6 +-
 .../PassThroughAllWindowFunction.java         |   5 +-
 .../windowing/PassThroughWindowFunction.java  |   5 +-
 .../windowing/ProcessAllWindowFunction.java   |   2 +-
 .../windowing/ProcessWindowFunction.java      |   2 +-
 .../ReduceApplyAllWindowFunction.java         |   6 +-
 .../ReduceApplyProcessAllWindowFunction.java  |   7 +-
 .../ReduceApplyProcessWindowFunction.java     |   7 +-
 .../windowing/ReduceApplyWindowFunction.java  |   6 +-
 .../ReduceIterableAllWindowFunction.java      |  48 ----
 .../ReduceIterableWindowFunction.java         |  48 ----
 .../windowing/RichAllWindowFunction.java      |   6 +-
 .../windowing/delta/CosineDistance.java       |   5 +-
 .../windowing/delta/EuclideanDistance.java    |   5 +-
 .../delta/ExtractionAwareDeltaFunction.java   |   4 +-
 .../delta/extractor/ArrayFromTuple.java       |   3 -
 .../delta/extractor/ConcatenatedExtract.java  |   3 -
 .../windowing/delta/extractor/Extractor.java  |   2 +-
 .../delta/extractor/FieldFromArray.java       |   6 +-
 .../delta/extractor/FieldFromTuple.java       |  11 +-
 .../delta/extractor/FieldsFromArray.java      |   4 +-
 .../delta/extractor/FieldsFromTuple.java      |   7 +-
 .../streaming/api/graph/JSONGenerator.java    |   3 +
 .../streaming/api/graph/StreamConfig.java     |   4 +
 .../streaming/api/graph/StreamGraph.java      |   8 +-
 .../api/graph/StreamGraphGenerator.java       |  50 ++--
 .../api/graph/StreamGraphHasherV2.java        |   9 +-
 .../api/operators/AbstractStreamOperator.java |  60 +++--
 .../operators/AbstractUdfStreamOperator.java  |   5 +-
 .../operators/HeapInternalTimerService.java   |   2 +-
 .../api/operators/KeyedProcessOperator.java   |   4 +
 .../api/operators/ProcessOperator.java        |   4 +
 .../streaming/api/operators/StreamFilter.java |   3 +
 .../api/operators/StreamFlatMap.java          |   3 +
 .../api/operators/StreamGroupedFold.java      |   4 +
 .../api/operators/StreamGroupedReduce.java    |   5 +
 .../streaming/api/operators/StreamMap.java    |   3 +
 .../api/operators/StreamOperator.java         |  15 +-
 .../api/operators/StreamProject.java          |   5 +-
 .../streaming/api/operators/StreamSink.java   |   3 +
 .../api/operators/StreamSourceContexts.java   |  19 +-
 .../operators/StreamingRuntimeContext.java    |  11 +-
 .../operators/async/AsyncWaitOperator.java    |  22 +-
 .../api/operators/async/Emitter.java          |   8 +-
 .../queue/OrderedStreamElementQueue.java      |  10 +-
 .../async/queue/StreamElementQueueEntry.java  |   1 -
 .../async/queue/StreamRecordQueueEntry.java   |   4 +-
 .../queue/UnorderedStreamElementQueue.java    |  14 +-
 .../api/operators/co/CoProcessOperator.java   |   4 +
 .../api/operators/co/CoStreamFlatMap.java     |   4 +
 .../api/operators/co/CoStreamMap.java         |   4 +
 .../operators/co/KeyedCoProcessOperator.java  |   4 +
 .../CoFeedbackTransformation.java             |  11 +-
 .../FeedbackTransformation.java               |  19 +-
 .../OneInputTransformation.java               |   2 +-
 .../PartitionTransformation.java              |   5 +-
 .../transformations/SelectTransformation.java |   5 +-
 .../transformations/SinkTransformation.java   |   2 +-
 .../transformations/SourceTransformation.java |   2 +-
 .../transformations/SplitTransformation.java  |   7 +-
 .../transformations/StreamTransformation.java |  56 +++--
 .../TwoInputTransformation.java               |   2 +-
 .../transformations/UnionTransformation.java  |   8 +-
 .../streaming/api/watermark/Watermark.java    |  12 +-
 .../assigners/BaseAlignedWindowAssigner.java  |   5 +-
 .../assigners/EventTimeSessionWindows.java    |   5 +-
 .../windowing/assigners/GlobalWindows.java    |   7 +-
 .../assigners/MergingWindowAssigner.java      |   2 +-
 .../ProcessingTimeSessionWindows.java         |   5 +-
 .../assigners/SlidingEventTimeWindows.java    |   3 +-
 .../SlidingProcessingTimeWindows.java         |   5 +-
 .../assigners/TumblingEventTimeWindows.java   |   3 +-
 .../TumblingProcessingTimeWindows.java        |   5 +-
 .../windowing/assigners/WindowAssigner.java   |   3 +-
 .../api/windowing/evictors/CountEvictor.java  |   2 +-
 .../api/windowing/evictors/DeltaEvictor.java  |   5 +-
 .../api/windowing/evictors/Evictor.java       |  12 +-
 .../api/windowing/evictors/TimeEvictor.java   |   2 +-
 .../streaming/api/windowing/time/Time.java    |   6 +-
 .../api/windowing/triggers/DeltaTrigger.java  |   3 +-
 .../windowing/triggers/EventTimeTrigger.java  |   3 +-
 .../windowing/triggers/PurgingTrigger.java    |   7 +-
 .../api/windowing/windows/GlobalWindow.java   |   9 +-
 .../api/windowing/windows/TimeWindow.java     |   2 +-
 .../api/windowing/windows/Window.java         |   5 +-
 .../streaming/runtime/io/BarrierBuffer.java   |  45 ++--
 .../streaming/runtime/io/BarrierTracker.java  |  24 +-
 .../runtime/io/BlockingQueueBroker.java       |  10 +-
 .../streaming/runtime/io/BufferSpiller.java   |  40 ++--
 .../runtime/io/StreamInputProcessor.java      |  15 +-
 .../runtime/io/StreamRecordWriter.java        |  10 +-
 .../runtime/io/StreamTwoInputProcessor.java   |  15 +-
 .../streaming/runtime/io/StreamingReader.java |  30 ---
 ...ctAlignedProcessingTimeWindowOperator.java |   3 +
 .../windowing/AbstractKeyedTimePanes.java     |   4 +-
 .../windowing/AccumulatingKeyedTimePanes.java |   7 +-
 ...umulatingProcessingTimeWindowOperator.java |   3 +
 .../windowing/AggregatingKeyedTimePanes.java  |   7 +-
 ...gregatingProcessingTimeWindowOperator.java |   3 +
 .../windowing/EvictingWindowOperator.java     |   6 +-
 .../runtime/operators/windowing/KeyMap.java   |  34 +--
 .../operators/windowing/TimestampedValue.java |   8 +-
 .../operators/windowing/WindowOperator.java   |  15 +-
 .../functions/InternalWindowFunction.java     |   5 +
 .../ConfigurableStreamPartitioner.java        |   2 +-
 .../partitioner/GlobalPartitioner.java        |   2 +-
 .../partitioner/StreamPartitioner.java        |   3 +
 .../runtime/streamrecord/LatencyMarker.java   |   8 +-
 .../streamrecord/StreamElementSerializer.java |   6 +-
 .../runtime/streamrecord/StreamRecord.java    |   6 +-
 .../streamstatus/StatusWatermarkValve.java    |  28 ++-
 .../runtime/streamstatus/StreamStatus.java    |  10 +-
 .../runtime/tasks/AsynchronousException.java  |   2 +-
 .../runtime/tasks/OneInputStreamTask.java     |   3 +
 .../runtime/tasks/OperatorChain.java          |   1 -
 .../runtime/tasks/ProcessingTimeCallback.java |   8 +-
 .../runtime/tasks/SourceStreamTask.java       |   4 +-
 .../runtime/tasks/StreamIterationHead.java    |   4 +
 .../runtime/tasks/StreamIterationTail.java    |   4 +
 .../streaming/runtime/tasks/StreamTask.java   |  65 +++---
 .../tasks/SystemProcessingTimeService.java    |   7 +-
 .../runtime/tasks/TimerException.java         |   2 +-
 .../runtime/tasks/TwoInputStreamTask.java     |   3 +
 .../streaming/runtime/tasks/package-info.java |   4 +-
 .../TypeInformationSerializationSchema.java   |  14 +-
 .../util/typeutils/FieldAccessor.java         |  13 +-
 .../api/operators/StreamProjectTest.java      |   2 +-
 .../windowing/EvictingWindowOperatorTest.java |  11 +-
 .../operators/windowing/TimeWindowTest.java   |   2 +-
 .../tasks/OneInputStreamTaskTestHarness.java  |   2 +-
 .../tasks/SourceStreamTaskStoppingTest.java   |   2 +-
 .../runtime/tasks/SourceStreamTaskTest.java   |   2 +-
 .../runtime/tasks/StreamMockEnvironment.java  |   2 +-
 .../tasks/TwoInputStreamTaskTestHarness.java  |   2 +-
 .../streaming/util/EvenOddOutputSelector.java |   2 +-
 .../flink/streaming/util/NoOpIntMap.java      |   2 +-
 .../flink/streaming/util/TestHarnessUtil.java |   2 +-
 .../TwoInputStreamOperatorTestHarness.java    |   2 +-
 tools/maven/strict-checkstyle.xml             |  42 ++--
 199 files changed, 1139 insertions(+), 1198 deletions(-)
 delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelectorWrapper.java
 delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java
 delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java
 delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamingReader.java

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
index a0f5a6011e4b6..db9977b5cdd84 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
@@ -29,6 +29,10 @@
 
 import static java.util.Objects.requireNonNull;
 
+/**
+ * Legacy multiplexing {@link TypeSerializer} for stream records, watermarks and other stream
+ * elements.
+ */
 public class MultiplexingStreamRecordSerializer<T> extends TypeSerializer<StreamElement> {
 
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
index 0235ab88803d9..885ece58b1ae8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
@@ -31,8 +31,7 @@
  * Serializer for {@link StreamRecord}. This version ignores timestamps and only deals with
  * the element.
  *
- * <p>
- * {@link MultiplexingStreamRecordSerializer} is a version that deals with timestamps and also
+ * <p>{@link MultiplexingStreamRecordSerializer} is a version that deals with timestamps and also
  * multiplexes {@link org.apache.flink.streaming.api.watermark.Watermark Watermarks} in the same
  * stream with {@link StreamRecord StreamRecords}.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java
index df2e4b9d1354e..9ad5879272ac3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java
@@ -33,10 +33,11 @@ public enum TimeCharacteristic {
 	 * to determine the current time of the data stream. Processing-time windows trigger based
 	 * on wall-clock time and include whatever elements happen to have arrived at the operator at
 	 * that point in time.
-	 * <p>
-	 * Using processing time for window operations results in general in quite non-deterministic results,
-	 * because the contents of the windows depends on the speed in which elements arrive. It is, however,
-	 * the cheapest method of forming windows and the method that introduces the least latency.
+	 *
+	 * <p>Using processing time for window operations results in general in quite non-deterministic
+	 * results, because the contents of the windows depends on the speed in which elements arrive.
+	 * It is, however, the cheapest method of forming windows and the method that introduces the
+	 * least latency.
 	 */
 	ProcessingTime,
 
@@ -45,41 +46,42 @@ public enum TimeCharacteristic {
 	 * when the element enters the Flink streaming data flow. Operations like windows group the
 	 * elements based on that time, meaning that processing speed within the streaming dataflow
 	 * does not affect windowing, but only the speed at which sources receive elements.
-	 * <p>
-	 * Ingestion time is often a good compromise between processing time and event time.
+	 *
+	 * <p>Ingestion time is often a good compromise between processing time and event time.
 	 * It does not need and special manual form of watermark generation, and events are typically
 	 * not too much out-or-order when they arrive at operators; in fact, out-of-orderness can 
-	 * only be introduced by streaming shuffles or split/join/union operations. The fact that elements
-	 * are not very much out-of-order means that the latency increase is moderate, compared to event
+	 * only be introduced by streaming shuffles or split/join/union operations. The fact that
+	 * elements are not very much out-of-order means that the latency increase is moderate,
+	 * compared to event
 	 * time.
 	 */
 	IngestionTime,
 
 	/**
 	 * Event time means that the time of each individual element in the stream (also called event)
-	 * is determined by the event's individual custom timestamp. These timestamps either exist in the
-	 * elements from before they entered the Flink streaming dataflow, or are user-assigned at the sources.
-	 * The big implication of this is that it allows for elements to arrive in the sources and in
-	 * all operators out of order, meaning that elements with earlier timestamps may arrive after
-	 * elements with later timestamps.
-	 * <p>
-	 * Operators that window or order data with respect to event time must buffer data until they can
-	 * be sure that all timestamps for a certain time interval have been received. This is handled by
-	 * the so called "time watermarks".
-	 * <p>
-	 * Operations based on event time are very predictable - the result of windowing operations
-	 * is typically identical no matter when the window is executed and how fast the streams operate.
-	 * At the same time, the buffering and tracking of event time is also costlier than operating
-	 * with processing time, and typically also introduces more latency. The amount of extra
-	 * cost depends mostly on how much out of order the elements arrive, i.e., how long the time span
-	 * between the arrival of early and late elements is. With respect to the "time watermarks", this
-	 * means that the cost typically depends on how early or late the watermarks can be generated
-	 * for their timestamp.
-	 * <p>
-	 * In relation to {@link #IngestionTime}, the event time is similar, but refers the the event's
-	 * original time, rather than the time assigned at the data source. Practically, that means that
-	 * event time has generally more meaning, but also that it takes longer to determine that all
-	 * elements for a certain time have arrived.
+	 * is determined by the event's individual custom timestamp. These timestamps either exist in
+	 * the elements from before they entered the Flink streaming dataflow, or are user-assigned at
+	 * the sources. The big implication of this is that it allows for elements to arrive in the
+	 * sources and in all operators out of order, meaning that elements with earlier timestamps may
+	 * arrive after elements with later timestamps.
+	 *
+	 * <p>Operators that window or order data with respect to event time must buffer data until they
+	 * can be sure that all timestamps for a certain time interval have been received. This is
+	 * handled by the so called "time watermarks".
+	 *
+	 * <p>Operations based on event time are very predictable - the result of windowing operations
+	 * is typically identical no matter when the window is executed and how fast the streams
+	 * operate. At the same time, the buffering and tracking of event time is also costlier than
+	 * operating with processing time, and typically also introduces more latency. The amount of
+	 * extra cost depends mostly on how much out of order the elements arrive, i.e., how long the
+	 * time span between the arrival of early and late elements is. With respect to the
+	 * "time watermarks", this means that the cost typically depends on how early or late the
+	 * watermarks can be generated for their timestamp.
+	 *
+	 * <p>In relation to {@link #IngestionTime}, the event time is similar, but refers the the
+	 * event's original time, rather than the time assigned at the data source. Practically, that
+	 * means that event time has generally more meaning, but also that it takes longer to determine
+	 * that all elements for a certain time have arrived.
 	 */
 	EventTime
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
index dd93462ea21a1..2479e0a72b1da 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
@@ -28,14 +28,11 @@
  * and return a snapshot of their state, which will be checkpointed.
  * 
  * <h1>Deprecation and Replacement</h1>
- *
  * The short cut replacement for this interface is via {@link ListCheckpointed} and works
  * as shown in the example below. The {@code ListCheckpointed} interface returns a list of
  * elements (
- * 
- * 
  *
- * <pre>{@code
+ * <p><pre>{@code
  * public class ExampleFunction<T> implements MapFunction<T, T>, ListCheckpointed<Integer> {
  *
  *     private int count;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
index 4bafd908e3530..4be522ca42ac3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
@@ -29,12 +29,11 @@
  * this interface has not been implemented.
  * 
  * <h1>Deprecation and Replacement</h1>
- * 
  * The shortcut replacement for this interface is via {@link ListCheckpointed} and works
  * as shown in the example below. Please refer to the JavaDocs of {@link ListCheckpointed} for
  * a more detailed description of how to use the new interface.
  * 
- * <pre>{@code
+ * <p><pre>{@code
  * public class ExampleFunction<T> implements MapFunction<T, T>, ListCheckpointed<Integer> {
  * 
  *     private int count;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java
index 51ac5db237028..556e95744152e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java
@@ -36,7 +36,6 @@
  * abstraction represented by this interface.
  * 
  * <h1>Initialization</h1>
- * 
  * The {@link CheckpointedFunction#initializeState(FunctionInitializationContext)} is called when
  * the parallel instance of the transformation function is created during distributed execution.
  * The method gives access to the {@link FunctionInitializationContext} which in turn gives access
@@ -44,13 +43,13 @@
  * 
  * <p>The {@code OperatorStateStore} and {@code KeyedStateStore} give access to the data structures
  * in which state should be stored for Flink to transparently manage and checkpoint it, such as
- * {@link org.apache.flink.api.common.state.ValueState} or {@link org.apache.flink.api.common.state.ListState}.
-
- * <p><i>Note:</i> The {@code KeyedStateStore} can only be used when the transformation supports
+ * {@link org.apache.flink.api.common.state.ValueState} or
+ * {@link org.apache.flink.api.common.state.ListState}.
+ *
+ * <p><b>Note:</b> The {@code KeyedStateStore} can only be used when the transformation supports
  * <i>keyed state</i>, i.e., when it is applied on a keyed stream (after a {@code keyBy(...)}).
  * 
  * <h1>Snapshot</h1>
- * 
  * The {@link CheckpointedFunction#snapshotState(FunctionSnapshotContext)} is called whenever a
  * checkpoint takes a state snapshot of the transformation function. Inside this method, functions typically
  * make sure that the checkpointed data structures (obtained in the initialization phase) are up
@@ -61,7 +60,6 @@
  * external systems.
  *
  * <h1>Example</h1>
- * 
  * The code example below illustrates how to use this interface for a function that keeps counts
  * of events per key and per parallel partition (parallel instance of the transformation function
  * during distributed execution).
@@ -69,7 +67,7 @@
  * adding up the counters of partitions that get merged on scale-down. Note that this is a
  * toy example, but should illustrate the basic skeleton for a stateful function.
  * 
- * <pre>{@code
+ * <p><pre>{@code
  * public class MyFunction<T> implements MapFunction<T, T>, CheckpointedFunction {
  *
  *     private ReducingState<Long> countPerKey;
@@ -112,18 +110,15 @@
  * <hr>
  * 
  * <h1><a name="shortcuts">Shortcuts</a></h1>
- * 
  * There are various ways that transformation functions can use state without implementing the
  * full-fledged {@code CheckpointedFunction} interface:
  * 
  * <h4>Operator State</h4>
- *
  * Checkpointing some state that is part of the function object itself is possible in a simpler way
  * by directly implementing the {@link ListCheckpointed} interface. 
  * That mechanism is similar to the previously used {@link Checkpointed} interface.
  * 
  * <h4>Keyed State</h4>
- * 
  * Access to keyed state is possible via the {@link RuntimeContext}'s methods:
  * <pre>{@code
  * public class CountPerKeyFunction<T> extends RichMapFunction<T, T> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
index 1fd652fe9926b..cfaa505f0847b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
@@ -35,7 +35,7 @@ public interface CheckpointedRestoring<T extends Serializable> {
 	 * Restores the state of the function or operator to that of a previous checkpoint.
 	 * This method is invoked when a function is executed as part of a recovery run.
 	 *
-	 * Note that restoreState() is called before open().
+	 * <p>Note that restoreState() is called before open().
 	 *
 	 * @param state The state to be restored.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
index 84a9700cf8f3e..ab18c5891fc35 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
@@ -38,9 +38,8 @@
  * of the state objects, or have multiple named states.
  * 
  * <h2>State Redistribution</h2>
- * 
  * State redistribution happens when the parallelism of the operator is changed.
- * State redistribution of <i>operator state<i> (to which category the state handled by this
+ * State redistribution of <i>operator state</i> (to which category the state handled by this
  * interface belongs) always goes through a checkpoint, so it appears
  * to the transformation functions like a failure/recovery combination, where recovery happens
  * with a different parallelism.
@@ -59,7 +58,7 @@
  * +----+----+   +----+----+   +----+
  * </pre>
  * 
- * Recovering the checkpoint with <i>parallelism = 5</i> yields the following state assignment:
+ * <p>Recovering the checkpoint with <i>parallelism = 5</i> yields the following state assignment:
  * <pre>
  * func_1   func_2   func_3   func_4   func_5
  * +----+   +----+   +----+   +----+   +----+
@@ -76,10 +75,8 @@
  * </pre>
  * 
  * <h2>Example</h2>
- * 
  * The following example illustrates how to implement a {@code MapFunction} that counts all elements
- * passing through it, keeping the total count accurate under re-scaling  (changes or parallelism).
- * 
+ * passing through it, keeping the total count accurate under re-scaling  (changes or parallelism):
  * <pre>{@code
  * public class CountingFunction<T> implements MapFunction<T, Tuple2<T, Long>>, ListCheckpointed<Long> {
  * 
@@ -122,7 +119,7 @@ public interface ListCheckpointed<T extends Serializable> {
 	 * the {@link ListCheckpointed class docs} for an illustration how list-style state
 	 * redistribution works.
 	 * 
-	 * <p> As special case, the returned list may be null or empty (if the operator has no state)
+	 * <p>As special case, the returned list may be null or empty (if the operator has no state)
 	 * or it may contain a single element (if the operator state is indivisible).
 	 *
 	 * @param checkpointId The ID of the checkpoint - a unique and monotonously increasing value. 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
index 6339506a2e30b..a709d45e82640 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
@@ -35,7 +35,10 @@
 import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.XORShiftRandom;
 
-
+/**
+ * Wrapping {@link Output} that forwards to other {@link Output Outputs } based on a list of
+ * {@link OutputSelector OutputSelectors}.
+ */
 public class DirectedOutput<OUT> implements Output<StreamRecord<OUT>> {
 	
 	protected final OutputSelector<OUT>[] outputSelectors;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelectorWrapper.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelectorWrapper.java
deleted file mode 100644
index 24f6018f5842f..0000000000000
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelectorWrapper.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector.selector;
-
-import java.io.Serializable;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-@PublicEvolving
-public interface OutputSelectorWrapper<OUT> extends Serializable {
-
-	void sendOutputs(OUT record);
-}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
index 50f0f85a75329..7c0ebd22b8c8d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
@@ -75,15 +75,13 @@
  * {@link org.apache.flink.streaming.api.windowing.assigners.WindowAssigner}. Window emission
  * is triggered based on a {@link org.apache.flink.streaming.api.windowing.triggers.Trigger}.
  *
- * <p>
- * If an {@link org.apache.flink.streaming.api.windowing.evictors.Evictor} is specified it will be
+ * <p>If an {@link org.apache.flink.streaming.api.windowing.evictors.Evictor} is specified it will be
  * used to evict elements from the window after
  * evaluation was triggered by the {@code Trigger} but before the actual evaluation of the window.
  * When using an evictor window performance will degrade significantly, since
  * pre-aggregation of window results cannot be used.
  *
- * <p>
- * Note that the {@code AllWindowedStream} is purely and API construct, during runtime
+ * <p>Note that the {@code AllWindowedStream} is purely and API construct, during runtime
  * the {@code AllWindowedStream} will be collapsed together with the
  * operation over the window into one single operation.
  *
@@ -93,10 +91,10 @@
 @Public
 public class AllWindowedStream<T, W extends Window> {
 
-	/** The keyed data stream that is windowed by this stream */
+	/** The keyed data stream that is windowed by this stream. */
 	private final KeyedStream<T, Byte> input;
 
-	/** The window assigner */
+	/** The window assigner. */
 	private final WindowAssigner<? super T, W> windowAssigner;
 
 	/** The trigger that is used for window evaluation/emission. */
@@ -171,8 +169,7 @@ public AllWindowedStream<T, W> sideOutputLateData(OutputTag<T> outputTag) {
 	/**
 	 * Sets the {@code Evictor} that should be used to evict elements from a window before emission.
 	 *
-	 * <p>
-	 * Note: When using an evictor window performance will degrade significantly, since
+	 * <p>Note: When using an evictor window performance will degrade significantly, since
 	 * incremental aggregation of window results cannot be used.
 	 */
 	@PublicEvolving
@@ -195,8 +192,7 @@ public AllWindowedStream<T, W> evictor(Evictor<? super T, ? super W> evictor) {
 	 * of the window for each key individually. The output of the reduce function is interpreted
 	 * as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * This window will try and incrementally aggregate data as much as the window policies permit.
+	 * <p>This window will try and incrementally aggregate data as much as the window policies permit.
 	 * For example, tumbling time windows can aggregate the data, meaning that only one element per
 	 * key is stored. Sliding time windows will aggregate on the granularity of the slide interval,
 	 * so a few elements are stored per key (one per slide interval).
@@ -227,8 +223,7 @@ public SingleOutputStreamOperator<T> reduce(ReduceFunction<T> function) {
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -251,8 +246,7 @@ public <R> SingleOutputStreamOperator<R> reduce(
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -326,8 +320,7 @@ public <R> SingleOutputStreamOperator<R> reduce(ReduceFunction<T> reduceFunction
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The process window function.
@@ -349,8 +342,7 @@ public <R> SingleOutputStreamOperator<R> reduce(
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The process window function.
@@ -796,8 +788,7 @@ public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R>
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -821,8 +812,7 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunctio
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -904,8 +894,7 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue,
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -929,8 +918,7 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunctio
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -1016,8 +1004,7 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue,
 	 * evaluation of the window. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Not that this function requires that all data in the windows is buffered until the window
+	 * <p>Not that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The window function.
@@ -1036,8 +1023,7 @@ public <R> SingleOutputStreamOperator<R> apply(AllWindowFunction<T, R, W> functi
 	 * evaluation of the window. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Not that this function requires that all data in the windows is buffered until the window
+	 * <p>Not that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The window function.
@@ -1054,8 +1040,7 @@ public <R> SingleOutputStreamOperator<R> apply(AllWindowFunction<T, R, W> functi
 	 * evaluation of the window. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Not that this function requires that all data in the windows is buffered until the window
+	 * <p>Not that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The process window function.
@@ -1075,8 +1060,7 @@ public <R> SingleOutputStreamOperator<R> process(ProcessAllWindowFunction<T, R,
 	 * evaluation of the window. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Not that this function requires that all data in the windows is buffered until the window
+	 * <p>Not that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The process window function.
@@ -1146,8 +1130,7 @@ private <R> SingleOutputStreamOperator<R> apply(InternalWindowFunction<Iterable<
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -1169,8 +1152,7 @@ public <R> SingleOutputStreamOperator<R> apply(ReduceFunction<T> reduceFunction,
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -1246,8 +1228,7 @@ public <R> SingleOutputStreamOperator<R> apply(ReduceFunction<T> reduceFunction,
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -1270,8 +1251,7 @@ public <R> SingleOutputStreamOperator<R> apply(R initialValue, FoldFunction<T, R
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -1364,10 +1344,8 @@ public SingleOutputStreamOperator<T> sum(int positionToSum) {
 	 * Applies an aggregation that sums every window of the pojo data stream at
 	 * the given field for every window.
 	 *
-	 * <p>
-	 * A field expression is either
-	 * the name of a public field or a getter method with parentheses of the
-	 * stream's underlying type. A dot can be used to drill down into objects,
+	 * <p>A field expression is either the name of a public field or a getter method with
+	 * parentheses of the stream's underlying type. A dot can be used to drill down into objects,
 	 * as in {@code "field1.getInnerField2()" }.
 	 *
 	 * @param field The field to sum
@@ -1392,11 +1370,9 @@ public SingleOutputStreamOperator<T> min(int positionToMin) {
 	 * Applies an aggregation that that gives the minimum value of the pojo data
 	 * stream at the given field expression for every window.
 	 *
-	 * <p>
-	 * A field
-	 * expression is either the name of a public field or a getter method with
-	 * parentheses of the {@link DataStream}S underlying type. A dot can be used
-	 * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+	 * <p>A field expression is either the name of a public field or a getter method with
+	 * parentheses of the {@link DataStream}S underlying type. A dot can be used to drill down into
+	 * objects, as in {@code "field1.getInnerField2()" }.
 	 *
 	 * @param field The field expression based on which the aggregation will be applied.
 	 * @return The transformed DataStream.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java
index 8132d284a09b0..8461d2c859b01 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java
@@ -28,8 +28,8 @@
 
 /**
  * A helper class to apply {@link AsyncFunction} to a data stream.
- * <p>
- * <pre>{@code
+ *
+ * <p><pre>{@code
  * DataStream<String> input = ...
  * AsyncFunction<String, Tuple<String, String>> asyncFunc = ...
  *
@@ -40,6 +40,10 @@
 
 @PublicEvolving
 public class AsyncDataStream {
+
+	/**
+	 * Output mode for asynchronous operations.
+	 */
 	public enum OutputMode { ORDERED, UNORDERED }
 
 	private static final int DEFAULT_QUEUE_CAPACITY = 100;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
index 62d032da60024..5c0dbf901e57f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
@@ -47,17 +47,13 @@
  *{@code CoGroupedStreams} represents two {@link DataStream DataStreams} that have been co-grouped.
  * A streaming co-group operation is evaluated over elements in a window.
  *
- * <p>
- * To finalize co-group operation you also need to specify a {@link KeySelector} for
+ * <p>To finalize co-group operation you also need to specify a {@link KeySelector} for
  * both the first and second input and a {@link WindowAssigner}.
  *
- * <p>
- * Note: Right now, the groups are being built in memory so you need to ensure that they don't
+ * <p>Note: Right now, the groups are being built in memory so you need to ensure that they don't
  * get too big. Otherwise the JVM might crash.
  *
- * <p>
- * Example:
- *
+ * <p>Example:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> one = ...;
  * DataStream<Tuple2<String, Integer>> two = ...;
@@ -72,14 +68,15 @@
 @Public
 public class CoGroupedStreams<T1, T2> {
 
-	/** The first input stream */
+	/** The first input stream. */
 	private final DataStream<T1> input1;
 
-	/** The second input stream */
+	/** The second input stream. */
 	private final DataStream<T2> input2;
 
 	/**
-	 * Creates new CoGroped data streams, which are the first step towards building a streaming co-group.
+	 * Creates new CoGroped data streams, which are the first step towards building a streaming
+	 * co-group.
 	 * 
 	 * @param input1 The first data stream.
 	 * @param input2 The second data stream.
@@ -209,10 +206,10 @@ public WithWindow<T1, T2, KEY, W> trigger(Trigger<? super TaggedUnion<T1, T2>, ?
 		}
 
 		/**
-		 * Sets the {@code Evictor} that should be used to evict elements from a window before emission.
+		 * Sets the {@code Evictor} that should be used to evict elements from a window before
+		 * emission.
 		 *
-		 * <p>
-		 * Note: When using an evictor window performance will degrade significantly, since
+		 * <p>Note: When using an evictor window performance will degrade significantly, since
 		 * pre-aggregation of window results cannot be used.
 		 */
 		@PublicEvolving
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
index 9fe3a4c6cc190..e884140383f64 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
@@ -90,7 +90,7 @@ public DataStream<IN2> getSecondInput() {
 	}
 
 	/**
-	 * Gets the type of the first input
+	 * Gets the type of the first input.
 	 *
 	 * @return The type of the first input
 	 */
@@ -99,7 +99,7 @@ public TypeInformation<IN1> getType1() {
 	}
 
 	/**
-	 * Gets the type of the second input
+	 * Gets the type of the second input.
 	 *
 	 * @return The type of the second input
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index fc70416e7d9d9..43a1aead44ddd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -186,7 +186,7 @@ protected <F> F clean(F f) {
 
 	/**
 	 * Returns the {@link StreamExecutionEnvironment} that was used to create this
-	 * {@link DataStream}
+	 * {@link DataStream}.
 	 *
 	 * @return The Execution Environment
 	 */
@@ -253,7 +253,6 @@ public <R> ConnectedStreams<T, R> connect(DataStream<R> dataStream) {
 	}
 
 	/**
-	 *
 	 * It creates a new {@link KeyedStream} that uses the provided key for partitioning
 	 * its operator states. 
 	 *
@@ -304,8 +303,8 @@ private KeyedStream<T, Tuple> keyBy(Keys<T> keys) {
 	/**
 	 * Partitions a tuple DataStream on the specified key fields using a custom partitioner.
 	 * This method takes the key position to partition on, and a partitioner that accepts the key type.
-	 * <p>
-	 * Note: This method works only on single field keys.
+	 *
+	 * <p>Note: This method works only on single field keys.
 	 *
 	 * @param partitioner The partitioner to assign partitions to keys.
 	 * @param field The field index on which the DataStream is to partitioned.
@@ -319,8 +318,8 @@ public <K> DataStream<T> partitionCustom(Partitioner<K> partitioner, int field)
 	/**
 	 * Partitions a POJO DataStream on the specified key fields using a custom partitioner.
 	 * This method takes the key expression to partition on, and a partitioner that accepts the key type.
-	 * <p>
-	 * Note: This method works only on single field keys.
+	 *
+	 * <p>Note: This method works only on single field keys.
 	 *
 	 * @param partitioner The partitioner to assign partitions to keys.
 	 * @param field The expression for the field on which the DataStream is to partitioned.
@@ -336,8 +335,8 @@ public <K> DataStream<T> partitionCustom(Partitioner<K> partitioner, String fiel
 	 * Partitions a DataStream on the key returned by the selector, using a custom partitioner.
 	 * This method takes the key selector to get the key to partition on, and a partitioner that
 	 * accepts the key type.
-	 * <p>
-	 * Note: This method works only on single field keys, i.e. the selector cannot return tuples
+	 *
+	 * <p>Note: This method works only on single field keys, i.e. the selector cannot return tuples
 	 * of fields.
 	 *
 	 * @param partitioner
@@ -453,16 +452,16 @@ public DataStream<T> global() {
 	 * input of the iteration and treat the input and feedback streams as a
 	 * {@link ConnectedStreams} be calling
 	 * {@link IterativeStream#withFeedbackType(TypeInformation)}
-	 * <p>
-	 * A common usage pattern for streaming iterations is to use output
+	 *
+	 * <p>A common usage pattern for streaming iterations is to use output
 	 * splitting to send a part of the closing data stream to the head. Refer to
 	 * {@link #split(OutputSelector)} for more information.
-	 * <p>
-	 * The iteration edge will be partitioned the same way as the first input of
+	 *
+	 * <p>The iteration edge will be partitioned the same way as the first input of
 	 * the iteration head unless it is changed in the
 	 * {@link IterativeStream#closeWith(DataStream)} call.
-	 * <p>
-	 * By default a DataStream with iteration will never terminate, but the user
+	 *
+	 * <p>By default a DataStream with iteration will never terminate, but the user
 	 * can use the maxWaitTime parameter to set a max waiting time for the
 	 * iteration head. If no data received in the set time, the stream
 	 * terminates.
@@ -485,16 +484,16 @@ public IterativeStream<T> iterate() {
 	 * input of the iteration and treat the input and feedback streams as a
 	 * {@link ConnectedStreams} be calling
 	 * {@link IterativeStream#withFeedbackType(TypeInformation)}
-	 * <p>
-	 * A common usage pattern for streaming iterations is to use output
+	 *
+	 * <p>A common usage pattern for streaming iterations is to use output
 	 * splitting to send a part of the closing data stream to the head. Refer to
 	 * {@link #split(OutputSelector)} for more information.
-	 * <p>
-	 * The iteration edge will be partitioned the same way as the first input of
+	 *
+	 * <p>The iteration edge will be partitioned the same way as the first input of
 	 * the iteration head unless it is changed in the
 	 * {@link IterativeStream#closeWith(DataStream)} call.
-	 * <p>
-	 * By default a DataStream with iteration will never terminate, but the user
+	 *
+	 * <p>By default a DataStream with iteration will never terminate, but the user
 	 * can use the maxWaitTime parameter to set a max waiting time for the
 	 * iteration head. If no data received in the set time, the stream
 	 * terminates.
@@ -634,8 +633,7 @@ public SingleOutputStreamOperator<T> filter(FilterFunction<T> filter) {
 	 * Initiates a Project transformation on a {@link Tuple} {@link DataStream}.<br>
 	 * <b>Note: Only Tuple DataStreams can be projected.</b>
 	 *
-	 * <p>
-	 * The transformation projects each Tuple of the DataSet onto a (sub)set of
+	 * <p>The transformation projects each Tuple of the DataSet onto a (sub)set of
 	 * fields.
 	 *
 	 * @param fieldIndexes
@@ -671,13 +669,11 @@ public <T2> JoinedStreams<T, T2> join(DataStream<T2> otherStream) {
 	/**
 	 * Windows this {@code DataStream} into tumbling time windows.
 	 *
-	 * <p>
-	 * This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
+	 * <p>This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
 	 * {@code .window(TumblingProcessingTimeWindows.of(size))} depending on the time characteristic
 	 * set using
 	 *
-	 * <p>
-	 * Note: This operation can be inherently non-parallel since all elements have to pass through
+	 * <p>Note: This operation can be inherently non-parallel since all elements have to pass through
 	 * the same operator instance. (Only for special cases, such as aligned time windows is
 	 * it possible to perform this operation in parallel).
 	 *
@@ -696,14 +692,12 @@ public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size) {
 	/**
 	 * Windows this {@code DataStream} into sliding time windows.
 	 *
-	 * <p>
-	 * This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
+	 * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
 	 * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time characteristic
 	 * set using
 	 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 	 *
-	 * <p>
-	 * Note: This operation can be inherently non-parallel since all elements have to pass through
+	 * <p>Note: This operation can be inherently non-parallel since all elements have to pass through
 	 * the same operator instance. (Only for special cases, such as aligned time windows is
 	 * it possible to perform this operation in parallel).
 	 *
@@ -720,8 +714,7 @@ public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size, Time slide) {
 	/**
 	 * Windows this {@code DataStream} into tumbling count windows.
 	 *
-	 * <p>
-	 * Note: This operation can be inherently non-parallel since all elements have to pass through
+	 * <p>Note: This operation can be inherently non-parallel since all elements have to pass through
 	 * the same operator instance. (Only for special cases, such as aligned time windows is
 	 * it possible to perform this operation in parallel).
 	 *
@@ -734,8 +727,7 @@ public AllWindowedStream<T, GlobalWindow> countWindowAll(long size) {
 	/**
 	 * Windows this {@code DataStream} into sliding count windows.
 	 *
-	 * <p>
-	 * Note: This operation can be inherently non-parallel since all elements have to pass through
+	 * <p>Note: This operation can be inherently non-parallel since all elements have to pass through
 	 * the same operator instance. (Only for special cases, such as aligned time windows is
 	 * it possible to perform this operation in parallel).
 	 *
@@ -754,13 +746,11 @@ public AllWindowedStream<T, GlobalWindow> countWindowAll(long size, long slide)
 	 * {@link org.apache.flink.streaming.api.windowing.assigners.WindowAssigner}. The grouping of
 	 * elements is done both by key and by window.
 	 *
-	 * <p>
-	 * A {@link org.apache.flink.streaming.api.windowing.triggers.Trigger} can be defined to specify
+	 * <p>A {@link org.apache.flink.streaming.api.windowing.triggers.Trigger} can be defined to specify
 	 * when windows are evaluated. However, {@code WindowAssigners} have a default {@code Trigger}
 	 * that is used if a {@code Trigger} is not specified.
 	 *
-	 * <p>
-	 * Note: This operation can be inherently non-parallel since all elements have to pass through
+	 * <p>Note: This operation can be inherently non-parallel since all elements have to pass through
 	 * the same operator instance. (Only for special cases, such as aligned time windows is
 	 * it possible to perform this operation in parallel).
 	 *
@@ -780,8 +770,7 @@ public <W extends Window> AllWindowedStream<T, W> windowAll(WindowAssigner<? sup
 	 * Extracts a timestamp from an element and assigns it as the internal timestamp of that element.
 	 * The internal timestamps are, for example, used to to event-time window operations.
 	 *
-	 * <p>
-	 * If you know that the timestamps are strictly increasing you can use an
+	 * <p>If you know that the timestamps are strictly increasing you can use an
 	 * {@link AscendingTimestampExtractor}. Otherwise,
 	 * you should provide a {@link TimestampExtractor} that also implements
 	 * {@link TimestampExtractor#getCurrentWatermark()} to keep track of watermarks.
@@ -902,9 +891,7 @@ public SingleOutputStreamOperator<T> assignTimestampsAndWatermarks(
 	/**
 	 * Writes a DataStream to the standard output stream (stdout).
 	 *
-	 * <p>
-	 * For each element of the DataStream the result of
-	 * {@link Object#toString()} is written.
+	 * <p>For each element of the DataStream the result of {@link Object#toString()} is written.
 	 *
 	 * @return The closed DataStream.
 	 */
@@ -917,9 +904,7 @@ public DataStreamSink<T> print() {
 	/**
 	 * Writes a DataStream to the standard output stream (stderr).
 	 *
-	 * <p>
-	 * For each element of the DataStream the result of
-	 * {@link Object#toString()} is written.
+	 * <p>For each element of the DataStream the result of {@link Object#toString()} is written.
 	 *
 	 * @return The closed DataStream.
 	 */
@@ -932,9 +917,7 @@ public DataStreamSink<T> printToErr() {
 	/**
 	 * Writes a DataStream to the file specified by path in text format.
 	 *
-	 * <p>
-	 * For every element of the DataStream the result of {@link Object#toString()}
-	 * is written.
+	 * <p>For every element of the DataStream the result of {@link Object#toString()} is written.
 	 *
 	 * @param path
 	 *            The path pointing to the location the text file is written to.
@@ -950,9 +933,7 @@ public DataStreamSink<T> writeAsText(String path) {
 	/**
 	 * Writes a DataStream to the file specified by path in text format.
 	 *
-	 * <p>
-	 * For every element of the DataStream the result of {@link Object#toString()}
-	 * is written.
+	 * <p>For every element of the DataStream the result of {@link Object#toString()} is written.
 	 *
 	 * @param path
 	 *            The path pointing to the location the text file is written to
@@ -973,8 +954,7 @@ public DataStreamSink<T> writeAsText(String path, WriteMode writeMode) {
 	/**
 	 * Writes a DataStream to the file specified by the path parameter.
 	 *
-	 * <p>
-	 * For every field of an element of the DataStream the result of {@link Object#toString()}
+	 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 	 * is written. This method can only be used on data streams of tuples.
 	 *
 	 * @param path
@@ -991,8 +971,7 @@ public DataStreamSink<T> writeAsCsv(String path) {
 	/**
 	 * Writes a DataStream to the file specified by the path parameter.
 	 *
-	 * <p>
-	 * For every field of an element of the DataStream the result of {@link Object#toString()}
+	 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 	 * is written. This method can only be used on data streams of tuples.
 	 *
 	 * @param path
@@ -1012,8 +991,7 @@ public DataStreamSink<T> writeAsCsv(String path, WriteMode writeMode) {
 	 * Writes a DataStream to the file specified by the path parameter. The
 	 * writing is performed periodically every millis milliseconds.
 	 *
-	 * <p>
-	 * For every field of an element of the DataStream the result of {@link Object#toString()}
+	 * <p>For every field of an element of the DataStream the result of {@link Object#toString()}
 	 * is written. This method can only be used on data streams of tuples.
 	 *
 	 * @param path
@@ -1073,9 +1051,10 @@ public DataStreamSink<T> writeToSocket(String hostName, int port, SerializationS
 	/**
 	 * Writes the dataStream into an output, described by an OutputFormat.
 	 *
-	 * The output is not participating in Flink's checkpointing!
+	 * <p>The output is not participating in Flink's checkpointing!
 	 *
-	 * For writing to a file system periodically, the use of the "flink-connector-filesystem" is recommended.
+	 * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem"
+	 * is recommended.
 	 *
 	 * @param format The output format
 	 * @return The closed DataStream
@@ -1121,7 +1100,7 @@ public <R> SingleOutputStreamOperator<R> transform(String operatorName, TypeInfo
 	}
 
 	/**
-	 * Internal function for setting the partitioner for the DataStream
+	 * Internal function for setting the partitioner for the DataStream.
 	 *
 	 * @param partitioner
 	 *            Partitioner to set.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index b0e78d7a17afa..d098c986e79ab 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -80,19 +80,17 @@ public DataStreamSink<T> uid(String uid) {
 
 	/**
 	 * Sets an user provided hash for this operator. This will be used AS IS the create the JobVertexID.
-	 * <p/>
+	 *
 	 * <p>The user provided hash is an alternative to the generated hashes, that is considered when identifying an
 	 * operator through the default hash mechanics fails (e.g. because of changes between Flink versions).
-	 * <p/>
+	 *
 	 * <p><strong>Important</strong>: this should be used as a workaround or for trouble shooting. The provided hash
 	 * needs to be unique per transformation and job. Otherwise, job submission will fail. Furthermore, you cannot
 	 * assign user-specified hash to intermediate nodes in an operator chain and trying so will let your job fail.
 	 *
-	 * <p>
-	 * A use case for this is in migration between Flink versions or changing the jobs in a way that changes the
+	 * <p>A use case for this is in migration between Flink versions or changing the jobs in a way that changes the
 	 * automatically generated hashes. In this case, providing the previous hashes directly through this method (e.g.
 	 * obtained from old logs) can help to reestablish a lost mapping from states to their target operator.
-	 * <p/>
 	 *
 	 * @param uidHash The user provided hash for this operator. This will become the JobVertexID, which is shown in the
 	 *                 logs and web ui.
@@ -158,8 +156,7 @@ private DataStreamSink<T> setResources(ResourceSpec resources) {
 	 * Turns off chaining for this operator so thread co-location will not be
 	 * used as an optimization.
 	 *
-	 * <p>
-	 * Chaining can be turned off for the whole
+	 * <p>Chaining can be turned off for the whole
 	 * job by {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#disableOperatorChaining()}
 	 * however it is not advised for performance considerations.
 	 *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
index 069872540d4e5..c6426a68def74 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
@@ -53,8 +53,7 @@ protected IterativeStream(DataStream<T> dataStream, long maxWaitTime) {
 	 * Closes the iteration. This method defines the end of the iterative
 	 * program part that will be fed back to the start of the iteration.
 	 *
-	 * <p>
-	 * A common usage pattern for streaming iterations is to use output
+	 * <p>A common usage pattern for streaming iterations is to use output
 	 * splitting to send a part of the closing data stream to the head. Refer to
 	 * {@link DataStream#split(org.apache.flink.streaming.api.collector.selector.OutputSelector)}
 	 * for more information.
@@ -86,8 +85,7 @@ public DataStream<T> closeWith(DataStream<T> feedbackStream) {
 	 * co-transformations on the input and feedback stream, as in a
 	 * {@link ConnectedStreams}.
 	 *
-	 * <p>
-	 * For type safety the user needs to define the feedback type
+	 * <p>For type safety the user needs to define the feedback type
 	 * 
 	 * @param feedbackTypeString
 	 *            String describing the type information of the feedback stream.
@@ -102,8 +100,7 @@ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(String feedbackTypeS
 	 * co-transformations on the input and feedback stream, as in a
 	 * {@link ConnectedStreams}.
 	 *
-	 * <p>
-	 * For type safety the user needs to define the feedback type
+	 * <p>For type safety the user needs to define the feedback type
 	 * 
 	 * @param feedbackTypeClass
 	 *            Class of the elements in the feedback stream.
@@ -118,8 +115,7 @@ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(Class<F> feedbackTyp
 	 * co-transformations on the input and feedback stream, as in a
 	 * {@link ConnectedStreams}.
 	 *
-	 * <p>
-	 * For type safety the user needs to define the feedback type
+	 * <p>For type safety the user needs to define the feedback type
 	 * 
 	 * @param feedbackType
 	 *            The type information of the feedback stream.
@@ -135,8 +131,7 @@ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(TypeInformation<F> f
 	 * iteration and the feedback of the iteration are connected as in a
 	 * {@link ConnectedStreams}.
 	 *
-	 * <p>
-	 * The user can distinguish between the two inputs using co-transformation,
+	 * <p>The user can distinguish between the two inputs using co-transformation,
 	 * thus eliminating the need for mapping the inputs and outputs to a common
 	 * type.
 	 * 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
index ed1cbd7ed7206..40059cf385611 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
@@ -39,17 +39,13 @@
  *{@code JoinedStreams} represents two {@link DataStream DataStreams} that have been joined.
  * A streaming join operation is evaluated over elements in a window.
  *
- * <p>
- * To finalize the join operation you also need to specify a {@link KeySelector} for
+ * <p>To finalize the join operation you also need to specify a {@link KeySelector} for
  * both the first and second input and a {@link WindowAssigner}.
  *
- * <p>
- * Note: Right now, the the join is being evaluated in memory so you need to ensure that the number
+ * <p>Note: Right now, the the join is being evaluated in memory so you need to ensure that the number
  * of elements per key does not get too high. Otherwise the JVM might crash.
  *
- * <p>
- * Example:
- *
+ * <p>Example:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> one = ...;
  * DataStream<Tuple2<String, Integer>> twp = ...;
@@ -64,10 +60,10 @@
 @Public
 public class JoinedStreams<T1, T2> {
 
-	/** The first input stream */
+	/** The first input stream. */
 	private final DataStream<T1> input1;
 
-	/** The second input stream */
+	/** The second input stream. */
 	private final DataStream<T2> input2;
 
 	/**
@@ -206,8 +202,7 @@ public WithWindow<T1, T2, KEY, W> trigger(Trigger<? super TaggedUnion<T1, T2>, ?
 		/**
 		 * Sets the {@code Evictor} that should be used to evict elements from a window before emission.
 		 *
-		 * <p>
-		 * Note: When using an evictor window performance will degrade significantly, since
+		 * <p>Note: When using an evictor window performance will degrade significantly, since
 		 * pre-aggregation of window results cannot be used.
 		 */
 		@PublicEvolving
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
index 860aac6a5aa3f..27545c220a9cf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
@@ -79,9 +79,8 @@
  * {@code DataStream} are also possible on a {@code KeyedStream}, with the exception of
  * partitioning methods such as shuffle, forward and keyBy.
  *
- * <p>
- * Reduce-style operations, such as {@link #reduce}, {@link #sum} and {@link #fold} work on elements
- * that have the same key.
+ * <p>Reduce-style operations, such as {@link #reduce}, {@link #sum} and {@link #fold} work on
+ * elements that have the same key.
  *
  * @param <T> The type of the elements in the Keyed Stream.
  * @param <KEY> The type of the key in the Keyed Stream.
@@ -89,10 +88,12 @@
 @Public
 public class KeyedStream<T, KEY> extends DataStream<T> {
 
-	/** The key selector that can get the key by which the stream if partitioned from the elements */
+	/**
+	 * The key selector that can get the key by which the stream if partitioned from the elements.
+	 */
 	private final KeySelector<T, KEY> keySelector;
 
-	/** The type of the key by which the stream is partitioned */
+	/** The type of the key by which the stream is partitioned. */
 	private final TypeInformation<KEY> keyType;
 	
 	/**
@@ -313,8 +314,7 @@ public <R> SingleOutputStreamOperator<R> process(
 	/**
 	 * Windows this {@code KeyedStream} into tumbling time windows.
 	 *
-	 * <p>
-	 * This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
+	 * <p>This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
 	 * {@code .window(TumblingProcessingTimeWindows.of(size))} depending on the time characteristic
 	 * set using
 	 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
@@ -332,10 +332,9 @@ public WindowedStream<T, KEY, TimeWindow> timeWindow(Time size) {
 	/**
 	 * Windows this {@code KeyedStream} into sliding time windows.
 	 *
-	 * <p>
-	 * This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
-	 * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time characteristic
-	 * set using
+	 * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
+	 * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time
+	 * characteristic set using
 	 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 	 *
 	 * @param size The size of the window.
@@ -374,10 +373,9 @@ public WindowedStream<T, KEY, GlobalWindow> countWindow(long size, long slide) {
 	 * over a key grouped stream. Elements are put into windows by a {@link WindowAssigner}. The
 	 * grouping of elements is done both by key and by window.
 	 *
-	 * <p>
-	 * A {@link org.apache.flink.streaming.api.windowing.triggers.Trigger} can be defined to specify
-	 * when windows are evaluated. However, {@code WindowAssigners} have a default {@code Trigger}
-	 * that is used if a {@code Trigger} is not specified.
+	 * <p>A {@link org.apache.flink.streaming.api.windowing.triggers.Trigger} can be defined to
+	 * specify when windows are evaluated. However, {@code WindowAssigners} have a default
+	 * {@code Trigger} that is used if a {@code Trigger} is not specified.
 	 *
 	 * @param assigner The {@code WindowAssigner} that assigns elements to windows.
 	 * @return The trigger windows data stream.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 9c5d9f072b82e..5f063ded12100 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -101,23 +101,26 @@ public SingleOutputStreamOperator<T> uid(String uid) {
 	}
 
 	/**
-	 * Sets an user provided hash for this operator. This will be used AS IS the create the JobVertexID.
-	 * <p/>
-	 * <p>The user provided hash is an alternative to the generated hashes, that is considered when identifying an
-	 * operator through the default hash mechanics fails (e.g. because of changes between Flink versions).
-	 * <p/>
-	 * <p><strong>Important</strong>: this should be used as a workaround or for trouble shooting. The provided hash
-	 * needs to be unique per transformation and job. Otherwise, job submission will fail. Furthermore, you cannot
-	 * assign user-specified hash to intermediate nodes in an operator chain and trying so will let your job fail.
+	 * Sets an user provided hash for this operator. This will be used AS IS the create the
+	 * JobVertexID.
+	 *
+	 * <p>The user provided hash is an alternative to the generated hashes, that is considered when
+	 * identifying an operator through the default hash mechanics fails (e.g. because of changes
+	 * between Flink versions).
 	 *
-	 * <p>
-	 * A use case for this is in migration between Flink versions or changing the jobs in a way that changes the
-	 * automatically generated hashes. In this case, providing the previous hashes directly through this method (e.g.
-	 * obtained from old logs) can help to reestablish a lost mapping from states to their target operator.
+	 * <p><strong>Important</strong>: this should be used as a workaround or for trouble shooting.
+	 * The provided hash needs to be unique per transformation and job. Otherwise, job submission
+	 * will fail. Furthermore, you cannot assign user-specified hash to intermediate nodes in an
+	 * operator chain and trying so will let your job fail.
+	 *
+	 * <p>A use case for this is in migration between Flink versions or changing the jobs in a way
+	 * that changes the automatically generated hashes. In this case, providing the previous hashes
+	 * directly through this method (e.g. obtained from old logs) can help to reestablish a lost
+	 * mapping from states to their target operator.
 	 * <p/>
 	 *
-	 * @param uidHash The user provided hash for this operator. This will become the JobVertexID, which is shown in the
-	 *                 logs and web ui.
+	 * @param uidHash The user provided hash for this operator. This will become the JobVertexID,
+	 *                  which is shown in the logs and web ui.
 	 * @return The operator with the user provided hash.
 	 */
 	@PublicEvolving
@@ -148,7 +151,7 @@ public SingleOutputStreamOperator<T> setParallelism(int parallelism) {
 	/**
 	 * Sets the maximum parallelism of this operator.
 	 *
-	 * The maximum parallelism specifies the upper bound for dynamic scaling. It also defines the
+	 * <p>The maximum parallelism specifies the upper bound for dynamic scaling. It also defines the
 	 * number of key groups used for partitioned state.
 	 *
 	 * @param maxParallelism Maximum parallelism
@@ -253,11 +256,12 @@ private SingleOutputStreamOperator<T> setChainingStrategy(ChainingStrategy strat
 	}
 
 	/**
-	 * Turns off chaining for this operator so thread co-location will not be
-	 * used as an optimization.
-	 * <p> Chaining can be turned off for the whole
-	 * job by {@link StreamExecutionEnvironment#disableOperatorChaining()}
-	 * however it is not advised for performance considerations.
+	 * Turns off chaining for this operator so thread co-location will not be used as an
+	 * optimization.
+	 *
+	 * <p>Chaining can be turned off for the whole job by
+	 * {@link StreamExecutionEnvironment#disableOperatorChaining()} however it is not advised for
+	 * performance considerations.
 	 * 
 	 * @return The operator with chaining disabled
 	 */
@@ -358,13 +362,11 @@ public SingleOutputStreamOperator<T> returns(TypeInformation<T> typeInfo) {
 	/**
 	 * Adds a type information hint about the return type of this operator. 
 	 * 
-	 * <p>
-	 * Type hints are important in cases where the Java compiler
-	 * throws away generic type information necessary for efficient execution.
+	 * <p>Type hints are important in cases where the Java compiler throws away generic type
+	 * information necessary for efficient execution.
 	 * 
-	 * <p>
-	 * This method takes a type information string that will be parsed. A type information string can contain the following
-	 * types:
+	 * <p>This method takes a type information string that will be parsed. A type information string
+	 * can contain the following types:
 	 *
 	 * <ul>
 	 * <li>Basic types such as <code>Integer</code>, <code>String</code>, etc.
@@ -383,7 +385,7 @@ public SingleOutputStreamOperator<T> returns(TypeInformation<T> typeInfo) {
 	 * <li>Enum types such as <code>Enum&lt;org.my.CustomEnum&gt;</code></li>
 	 * </ul>
 	 *
-	 * Example:
+	 * <p>Example:
 	 * <code>"Tuple2&lt;String,Tuple2&lt;Integer,org.my.MyJob$Pojo&lt;word=String&gt;&gt;&gt;"</code>
 	 *
 	 * @param typeInfoString
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
index 8537318c56f91..e1e56fda2422b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
@@ -49,6 +49,10 @@
 import org.apache.flink.streaming.api.operators.StreamProject;
 import org.apache.flink.util.Preconditions;
 
+/**
+ * The result of {@link DataStream#project(int...)}. This can be used to add more fields to the
+ * projection.
+ */
 @PublicEvolving
 public class StreamProjection<IN> {
 
@@ -77,7 +81,7 @@ protected StreamProjection(DataStream<IN> dataStream, int[] fieldIndexes) {
 
 	/**
 	 * Chooses a projectTupleX according to the length of
-	 * {@link org.apache.flink.streaming.api.datastream.StreamProjection#fieldIndexes}
+	 * {@link org.apache.flink.streaming.api.datastream.StreamProjection#fieldIndexes}.
 	 *
 	 * @return The projected DataStream.
 	 * @see org.apache.flink.api.java.operators.ProjectOperator.Projection
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
index 334851ee15861..6104b35df5b07 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
@@ -85,20 +85,17 @@
  * {@link org.apache.flink.streaming.api.windowing.assigners.WindowAssigner}. Window emission
  * is triggered based on a {@link org.apache.flink.streaming.api.windowing.triggers.Trigger}.
  *
- * <p>
- * The windows are conceptually evaluated for each key individually, meaning windows can trigger at
- * different points for each key.
+ * <p>The windows are conceptually evaluated for each key individually, meaning windows can trigger
+ * at different points for each key.
  *
- * <p>
- * If an {@link Evictor} is specified it will be used to evict elements from the window after
+ * <p>If an {@link Evictor} is specified it will be used to evict elements from the window after
  * evaluation was triggered by the {@code Trigger} but before the actual evaluation of the window.
  * When using an evictor window performance will degrade significantly, since
  * incremental aggregation of window results cannot be used.
  *
- * <p>
- * Note that the {@code WindowedStream} is purely and API construct, during runtime
- * the {@code WindowedStream} will be collapsed together with the
- * {@code KeyedStream} and the operation over the window into one single operation.
+ * <p>Note that the {@code WindowedStream} is purely and API construct, during runtime the
+ * {@code WindowedStream} will be collapsed together with the {@code KeyedStream} and the operation
+ * over the window into one single operation.
  *
  * @param <T> The type of elements in the stream.
  * @param <K> The type of the key by which elements are grouped.
@@ -107,10 +104,10 @@
 @Public
 public class WindowedStream<T, K, W extends Window> {
 
-	/** The keyed data stream that is windowed by this stream */
+	/** The keyed data stream that is windowed by this stream. */
 	private final KeyedStream<T, K> input;
 
-	/** The window assigner */
+	/** The window assigner. */
 	private final WindowAssigner<? super T, W> windowAssigner;
 
 	/** The trigger that is used for window evaluation/emission. */
@@ -189,8 +186,7 @@ public WindowedStream<T, K, W> sideOutputLateData(OutputTag<T> outputTag) {
 	/**
 	 * Sets the {@code Evictor} that should be used to evict elements from a window before emission.
 	 *
-	 * <p>
-	 * Note: When using an evictor window performance will degrade significantly, since
+	 * <p>Note: When using an evictor window performance will degrade significantly, since
 	 * incremental aggregation of window results cannot be used.
 	 */
 	@PublicEvolving
@@ -216,11 +212,10 @@ public WindowedStream<T, K, W> evictor(Evictor<? super T, ? super W> evictor) {
 	 * of the window for each key individually. The output of the reduce function is interpreted
 	 * as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * This window will try and incrementally aggregate data as much as the window policies permit.
-	 * For example, tumbling time windows can aggregate the data, meaning that only one element per
-	 * key is stored. Sliding time windows will aggregate on the granularity of the slide interval,
-	 * so a few elements are stored per key (one per slide interval).
+	 * <p>This window will try and incrementally aggregate data as much as the window policies
+	 * permit. For example, tumbling time windows can aggregate the data, meaning that only one
+	 * element per key is stored. Sliding time windows will aggregate on the granularity of the
+	 * slide interval, so a few elements are stored per key (one per slide interval).
 	 * Custom windows may not be able to incrementally aggregate, or may need to store extra values
 	 * in an aggregation tree.
 	 *
@@ -254,8 +249,7 @@ public SingleOutputStreamOperator<T> reduce(ReduceFunction<T> function) {
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -271,8 +265,7 @@ public <R> SingleOutputStreamOperator<R> reduce(ReduceFunction<T> reduceFunction
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -292,8 +285,7 @@ public <R> SingleOutputStreamOperator<R> reduce(
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -318,8 +310,7 @@ private <R> SingleOutputStreamOperator<R> reduce(
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -535,8 +526,7 @@ public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R>
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -560,8 +550,7 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunctio
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -1084,8 +1073,7 @@ public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Not that this function requires that all data in the windows is buffered until the window
+	 * <p>Not that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The window function.
@@ -1103,8 +1091,7 @@ public <R> SingleOutputStreamOperator<R> apply(WindowFunction<T, R, K, W> functi
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Note that this function requires that all data in the windows is buffered until the window
+	 * <p>Note that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The window function.
@@ -1122,8 +1109,7 @@ public <R> SingleOutputStreamOperator<R> apply(WindowFunction<T, R, K, W> functi
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Not that this function requires that all data in the windows is buffered until the window
+	 * <p>Not that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The window function.
@@ -1142,8 +1128,7 @@ public <R> SingleOutputStreamOperator<R> process(ProcessWindowFunction<T, R, K,
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Not that this function requires that all data in the windows is buffered until the window
+	 * <p>Not that this function requires that all data in the windows is buffered until the window
 	 * is evaluated, as the function provides no means of incremental aggregation.
 	 *
 	 * @param function The window function.
@@ -1221,8 +1206,7 @@ private <R> SingleOutputStreamOperator<R> apply(InternalWindowFunction<Iterable<
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -1244,8 +1228,7 @@ public <R> SingleOutputStreamOperator<R> apply(ReduceFunction<T> reduceFunction,
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given reducer.
+	 * <p>Arriving data is incrementally aggregated using the given reducer.
 	 *
 	 * @param reduceFunction The reduce function that is used for incremental aggregation.
 	 * @param function The window function.
@@ -1321,8 +1304,7 @@ public <R> SingleOutputStreamOperator<R> apply(ReduceFunction<T> reduceFunction,
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -1345,8 +1327,7 @@ public <R> SingleOutputStreamOperator<R> apply(R initialValue, FoldFunction<T, R
 	 * evaluation of the window for each key individually. The output of the window function is
 	 * interpreted as a regular non-windowed stream.
 	 *
-	 * <p>
-	 * Arriving data is incrementally aggregated using the given fold function.
+	 * <p>Arriving data is incrementally aggregated using the given fold function.
 	 *
 	 * @param initialValue The initial value of the fold.
 	 * @param foldFunction The fold function that is used for incremental aggregation.
@@ -1434,13 +1415,11 @@ public SingleOutputStreamOperator<T> sum(int positionToSum) {
 	}
 
 	/**
-	 * Applies an aggregation that sums every window of the pojo data stream at
-	 * the given field for every window.
+	 * Applies an aggregation that sums every window of the pojo data stream at the given field for
+	 * every window.
 	 *
-	 * <p>
-	 * A field expression is either
-	 * the name of a public field or a getter method with parentheses of the
-	 * stream's underlying type. A dot can be used to drill down into objects,
+	 * <p>A field expression is either the name of a public field or a getter method with
+	 * parentheses of the stream's underlying type. A dot can be used to drill down into objects,
 	 * as in {@code "field1.getInnerField2()" }.
 	 *
 	 * @param field The field to sum
@@ -1465,9 +1444,7 @@ public SingleOutputStreamOperator<T> min(int positionToMin) {
 	 * Applies an aggregation that that gives the minimum value of the pojo data
 	 * stream at the given field expression for every window.
 	 *
-	 * <p>
-	 * A field
-	 * expression is either the name of a public field or a getter method with
+	 * <p>A field * expression is either the name of a public field or a getter method with
 	 * parentheses of the {@link DataStream}S underlying type. A dot can be used
 	 * to drill down into objects, as in {@code "field1.getInnerField2()" }.
 	 *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
index eb7833a636e32..1cb48a26a1292 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
@@ -35,16 +35,16 @@ public class CheckpointConfig implements java.io.Serializable {
 
 	private static final long serialVersionUID = -750378776078908147L;
 
-	/** The default checkpoint mode: exactly once */
+	/** The default checkpoint mode: exactly once. */
 	public static final CheckpointingMode DEFAULT_MODE = CheckpointingMode.EXACTLY_ONCE;
 
-	/** The default timeout of a checkpoint attempt: 10 minutes */
+	/** The default timeout of a checkpoint attempt: 10 minutes. */
 	public static final long DEFAULT_TIMEOUT = 10 * 60 * 1000;
 
-	/** The default minimum pause to be made between checkpoints: none */
+	/** The default minimum pause to be made between checkpoints: none. */
 	public static final long DEFAULT_MIN_PAUSE_BETWEEN_CHECKPOINTS = 0;
 
-	/** The default limit of concurrently happening checkpoints: one */
+	/** The default limit of concurrently happening checkpoints: one. */
 	public static final int DEFAULT_MAX_CONCURRENT_CHECKPOINTS = 1;
 
 	// ------------------------------------------------------------------------
@@ -52,19 +52,19 @@ public class CheckpointConfig implements java.io.Serializable {
 	/** Checkpointing mode (exactly-once vs. at-least-once). */
 	private CheckpointingMode checkpointingMode = DEFAULT_MODE;
 
-	/** Periodic checkpoint triggering interval */
+	/** Periodic checkpoint triggering interval. */
 	private long checkpointInterval = -1; // disabled
 
-	/** Maximum time checkpoint may take before being discarded */
+	/** Maximum time checkpoint may take before being discarded. */
 	private long checkpointTimeout = DEFAULT_TIMEOUT;
 
-	/** Minimal pause between checkpointing attempts */
+	/** Minimal pause between checkpointing attempts. */
 	private long minPauseBetweenCheckpoints = DEFAULT_MIN_PAUSE_BETWEEN_CHECKPOINTS;
 
-	/** Maximum number of checkpoint attempts in progress at the same time */
+	/** Maximum number of checkpoint attempts in progress at the same time. */
 	private int maxConcurrentCheckpoints = DEFAULT_MAX_CONCURRENT_CHECKPOINTS;
 
-	/** Flag to force checkpointing in iterative jobs */
+	/** Flag to force checkpointing in iterative jobs. */
 	private boolean forceCheckpointing;
 
 	/** Cleanup behaviour for persistent checkpoints. */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
index 61ca55f514668..0a4209fdb4a58 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
@@ -46,7 +46,7 @@ public class Flip6LocalStreamEnvironment extends StreamExecutionEnvironment {
 
 	private static final Logger LOG = LoggerFactory.getLogger(Flip6LocalStreamEnvironment.class);
 
-	/** The configuration to use for the mini cluster */
+	/** The configuration to use for the mini cluster. */
 	private final Configuration conf;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
index 566beba6a266d..584955997dd63 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
@@ -48,7 +48,7 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment {
 
 	private static final Logger LOG = LoggerFactory.getLogger(LocalStreamEnvironment.class);
 	
-	/** The configuration to use for the local cluster */
+	/** The configuration to use for the local cluster. */
 	private final Configuration conf;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
index 333f9c08c73a5..28a06756f28ec 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
@@ -41,24 +41,27 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * A {@link StreamExecutionEnvironment} for executing on a cluster.
+ */
 @Public
 public class RemoteStreamEnvironment extends StreamExecutionEnvironment {
 	
 	private static final Logger LOG = LoggerFactory.getLogger(RemoteStreamEnvironment.class);
 
-	/** The hostname of the JobManager */
+	/** The hostname of the JobManager. */
 	private final String host;
 
-	/** The port of the JobManager main actor system */
+	/** The port of the JobManager main actor system. */
 	private final int port;
 
-	/** The configuration used to parametrize the client that connects to the remote cluster */
+	/** The configuration used to parametrize the client that connects to the remote cluster. */
 	private final Configuration clientConfiguration;
 
-	/** The jar files that need to be attached to each job */
+	/** The jar files that need to be attached to each job. */
 	private final List<URL> jarFiles;
 	
-	/** The classpaths that need to be attached to each job */
+	/** The classpaths that need to be attached to each job. */
 	private final List<URL> globalClasspaths;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
index 49c53478c2d9c..52d2c4b2995f0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
@@ -30,6 +30,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Special {@link StreamExecutionEnvironment} that will be used in cases where the CLI client or
+ * testing utilities create a {@link StreamExecutionEnvironment} that should be used when
+ * {@link StreamExecutionEnvironment#getExecutionEnvironment()} is called.
+ */
 @PublicEvolving
 public class StreamContextEnvironment extends StreamExecutionEnvironment {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index e8366168f384c..354eb4ab0f19c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -100,27 +100,29 @@
 @Public
 public abstract class StreamExecutionEnvironment {
 
-	/** The default name to use for a streaming job if no other name has been specified */
+	/** The default name to use for a streaming job if no other name has been specified. */
 	public static final String DEFAULT_JOB_NAME = "Flink Streaming Job";
 
-	/** The time characteristic that is used if none other is set */
+	/** The time characteristic that is used if none other is set. */
 	private static final TimeCharacteristic DEFAULT_TIME_CHARACTERISTIC = TimeCharacteristic.ProcessingTime;
 
-	/** The default buffer timeout (max delay of records in the network stack) */
+	/** The default buffer timeout (max delay of records in the network stack). */
 	private static final long DEFAULT_NETWORK_BUFFER_TIMEOUT = 100L;
 
-	/** The environment of the context (local by default, cluster if invoked through command line) */
+	/**
+	 * The environment of the context (local by default, cluster if invoked through command line).
+	 */
 	private static StreamExecutionEnvironmentFactory contextEnvironmentFactory;
 
-	/** The default parallelism used when creating a local environment */
+	/** The default parallelism used when creating a local environment. */
 	private static int defaultLocalParallelism = Runtime.getRuntime().availableProcessors();
 
 	// ------------------------------------------------------------------------
 
-	/** The execution configuration for this environment */
+	/** The execution configuration for this environment. */
 	private final ExecutionConfig config = new ExecutionConfig();
 
-	/** Settings that control the checkpointing behavior */
+	/** Settings that control the checkpointing behavior. */
 	private final CheckpointConfig checkpointCfg = new CheckpointConfig();
 
 	protected final List<StreamTransformation<?>> transformations = new ArrayList<>();
@@ -129,10 +131,10 @@ public abstract class StreamExecutionEnvironment {
 
 	protected boolean isChainingEnabled = true;
 
-	/** The state backend used for storing k/v state and state snapshots */
+	/** The state backend used for storing k/v state and state snapshots. */
 	private AbstractStateBackend defaultStateBackend;
 
-	/** The time characteristic used by the data streams */
+	/** The time characteristic used by the data streams. */
 	private TimeCharacteristic timeCharacteristic = DEFAULT_TIME_CHARACTERISTIC;
 
 
@@ -168,12 +170,14 @@ public StreamExecutionEnvironment setParallelism(int parallelism) {
 	}
 
 	/**
-	 * Sets the maximum degree of parallelism defined for the program. The upper limit (inclusive) is Short.MAX_VALUE.
+	 * Sets the maximum degree of parallelism defined for the program. The upper limit (inclusive)
+	 * is Short.MAX_VALUE.
 	 *
-	 * The maximum degree of parallelism specifies the upper limit for dynamic scaling. It also
+	 * <p>The maximum degree of parallelism specifies the upper limit for dynamic scaling. It also
 	 * defines the number of key groups used for partitioned state.
 	 *
-	 * @param maxParallelism Maximum degree of parallelism to be used for the program., with 0 < maxParallelism <= 2^15 - 1
+	 * @param maxParallelism Maximum degree of parallelism to be used for the program.,
+	 *              with 0 < maxParallelism <= 2^15 - 1
 	 */
 	public StreamExecutionEnvironment setMaxParallelism(int maxParallelism) {
 		Preconditions.checkArgument(maxParallelism > 0 &&
@@ -200,7 +204,7 @@ public int getParallelism() {
 	/**
 	 * Gets the maximum degree of parallelism defined for the program.
 	 *
-	 * The maximum degree of parallelism specifies the upper limit for dynamic scaling. It also
+	 * <p>The maximum degree of parallelism specifies the upper limit for dynamic scaling. It also
 	 * defines the number of key groups used for partitioned state.
 	 *
 	 * @return Maximum degree of parallelism
@@ -214,15 +218,12 @@ public int getMaxParallelism() {
 	 * output buffers. By default the output buffers flush frequently to provide
 	 * low latency and to aid smooth developer experience. Setting the parameter
 	 * can result in three logical modes:
-	 * <p>
+	 *
 	 * <ul>
-	 * <li>
-	 * A positive integer triggers flushing periodically by that integer</li>
-	 * <li>
-	 * 0 triggers flushing after every record thus minimizing latency</li>
-	 * <li>
-	 * -1 triggers flushing only when the output buffer is full thus maximizing
-	 * throughput</li>
+	 *   <li>A positive integer triggers flushing periodically by that integer</li>
+	 *   <li>0 triggers flushing after every record thus minimizing latency</li>
+	 *   <li>-1 triggers flushing only when the output buffer is full thus maximizing
+	 *      throughput</li>
 	 * </ul>
 	 *
 	 * @param timeoutMillis
@@ -292,12 +293,12 @@ public CheckpointConfig getCheckpointConfig() {
 	 * {@link CheckpointingMode#EXACTLY_ONCE} guarantees.
 	 *
 	 * <p>The job draws checkpoints periodically, in the given interval. The state will be
-	 * stored in the configured state backend.</p>
+	 * stored in the configured state backend.
 	 *
 	 * <p>NOTE: Checkpointing iterative streaming dataflows in not properly supported at
 	 * the moment. For that reason, iterative jobs will not be started if used
 	 * with enabled checkpointing. To override this mechanism, use the
-	 * {@link #enableCheckpointing(long, CheckpointingMode, boolean)} method.</p>
+	 * {@link #enableCheckpointing(long, CheckpointingMode, boolean)} method.
 	 *
 	 * @param interval Time interval between state checkpoints in milliseconds.
 	 */
@@ -313,12 +314,12 @@ public StreamExecutionEnvironment enableCheckpointing(long interval) {
 	 *
 	 * <p>The job draws checkpoints periodically, in the given interval. The system uses the
 	 * given {@link CheckpointingMode} for the checkpointing ("exactly once" vs "at least once").
-	 * The state will be stored in the configured state backend.</p>
+	 * The state will be stored in the configured state backend.
 	 *
 	 * <p>NOTE: Checkpointing iterative streaming dataflows in not properly supported at
 	 * the moment. For that reason, iterative jobs will not be started if used
 	 * with enabled checkpointing. To override this mechanism, use the
-	 * {@link #enableCheckpointing(long, CheckpointingMode, boolean)} method.</p>
+	 * {@link #enableCheckpointing(long, CheckpointingMode, boolean)} method.
 	 *
 	 * @param interval
 	 *             Time interval between state checkpoints in milliseconds.
@@ -337,11 +338,11 @@ public StreamExecutionEnvironment enableCheckpointing(long interval, Checkpointi
 	 * dataflow will be restarted from the latest completed checkpoint.
 	 *
 	 * <p>The job draws checkpoints periodically, in the given interval. The state will be
-	 * stored in the configured state backend.</p>
+	 * stored in the configured state backend.
 	 *
 	 * <p>NOTE: Checkpointing iterative streaming dataflows in not properly supported at
 	 * the moment. If the "force" parameter is set to true, the system will execute the
-	 * job nonetheless.</p>
+	 * job nonetheless.
 	 *
 	 * @param interval
 	 *            Time interval between state checkpoints in millis.
@@ -370,12 +371,12 @@ public StreamExecutionEnvironment enableCheckpointing(long interval, Checkpointi
 	 * {@link CheckpointingMode#EXACTLY_ONCE} guarantees.
 	 *
 	 * <p>The job draws checkpoints periodically, in the default interval. The state will be
-	 * stored in the configured state backend.</p>
+	 * stored in the configured state backend.
 	 *
 	 * <p>NOTE: Checkpointing iterative streaming dataflows in not properly supported at
 	 * the moment. For that reason, iterative jobs will not be started if used
 	 * with enabled checkpointing. To override this mechanism, use the
-	 * {@link #enableCheckpointing(long, CheckpointingMode, boolean)} method.</p>
+	 * {@link #enableCheckpointing(long, CheckpointingMode, boolean)} method.
 	 *
 	 * @deprecated Use {@link #enableCheckpointing(long)} instead.
 	 */
@@ -520,8 +521,8 @@ public int getNumberOfExecutionRetries() {
 
 	/**
 	 * Adds a new Kryo default serializer to the Runtime.
-	 * <p>
-	 * Note that the serializer instance must be serializable (as defined by
+	 *
+	 * <p>Note that the serializer instance must be serializable (as defined by
 	 * java.io.Serializable), because it may be distributed to the worker nodes
 	 * by java serialization.
 	 *
@@ -548,8 +549,8 @@ public void addDefaultKryoSerializer(Class<?> type, Class<? extends Serializer<?
 
 	/**
 	 * Registers the given type with a Kryo Serializer.
-	 * <p>
-	 * Note that the serializer instance must be serializable (as defined by
+	 *
+	 * <p>Note that the serializer instance must be serializable (as defined by
 	 * java.io.Serializable), because it may be distributed to the worker nodes
 	 * by java serialization.
 	 *
@@ -564,7 +565,7 @@ public <T extends Serializer<?> & Serializable>void registerTypeWithKryoSerializ
 
 	/**
 	 * Registers the given Serializer via its class as a serializer for the
-	 * given type at the KryoSerializer
+	 * given type at the KryoSerializer.
 	 *
 	 * @param type
 	 * 		The class of the types serialized with the given serializer.
@@ -607,8 +608,7 @@ public void registerType(Class<?> type) {
 	 * Sets the time characteristic for all streams create from this environment, e.g., processing
 	 * time, event time, or ingestion time.
 	 *
-	 * <p>
-	 * If you set the characteristic to IngestionTime of EventTime this will set a default
+	 * <p>If you set the characteristic to IngestionTime of EventTime this will set a default
 	 * watermark update interval of 200 ms. If this is not applicable for your application
 	 * you should change it using {@link ExecutionConfig#setAutoWatermarkInterval(long)}.
 	 *
@@ -660,15 +660,15 @@ public DataStreamSource<Long> generateSequence(long from, long to) {
 	}
 
 	/**
-	 * Creates a new data stream that contains the given elements. The elements must all be of the same type, for
-	 * example, all of the {@link String} or {@link Integer}.
-	 * <p>
-	 * The framework will try and determine the exact type from the elements. In case of generic elements, it may be
-	 * necessary to manually supply the type information via {@link #fromCollection(java.util.Collection,
-	 * org.apache.flink.api.common.typeinfo.TypeInformation)}.
-	 * <p>
-	 * Note that this operation will result in a non-parallel data stream source, i.e. a data stream source with a
-	 * degree of parallelism one.
+	 * Creates a new data stream that contains the given elements. The elements must all be of the
+	 * same type, for example, all of the {@link String} or {@link Integer}.
+	 *
+	 * <p>The framework will try and determine the exact type from the elements. In case of generic
+	 * elements, it may be necessary to manually supply the type information via
+	 * {@link #fromCollection(java.util.Collection, org.apache.flink.api.common.typeinfo.TypeInformation)}.
+	 *
+	 * <p>Note that this operation will result in a non-parallel data stream source, i.e. a data
+	 * stream source with a degree of parallelism one.
 	 *
 	 * @param data
 	 * 		The array of elements to create the data stream from.
@@ -733,10 +733,10 @@ public final <OUT> DataStreamSource<OUT> fromElements(Class<OUT> type, OUT... da
 	 *
 	 * <p>The framework will try and determine the exact type from the collection elements. In case of generic
 	 * elements, it may be necessary to manually supply the type information via
-	 * {@link #fromCollection(java.util.Collection, org.apache.flink.api.common.typeinfo.TypeInformation)}.</p>
+	 * {@link #fromCollection(java.util.Collection, org.apache.flink.api.common.typeinfo.TypeInformation)}.
 	 *
 	 * <p>Note that this operation will result in a non-parallel data stream source, i.e. a data stream source with a
-	 * parallelism one.</p>
+	 * parallelism one.
 	 *
 	 * @param data
 	 * 		The collection of elements to create the data stream from.
@@ -772,7 +772,7 @@ public <OUT> DataStreamSource<OUT> fromCollection(Collection<OUT> data) {
 	 * Creates a data stream from the given non-empty collection.
 	 *
 	 * <p>Note that this operation will result in a non-parallel data stream source,
-	 * i.e., a data stream source with a parallelism one.</p>
+	 * i.e., a data stream source with a parallelism one.
 	 *
 	 * @param data
 	 * 		The collection of elements to create the data stream from
@@ -803,10 +803,10 @@ public <OUT> DataStreamSource<OUT> fromCollection(Collection<OUT> data, TypeInfo
 	 *
 	 * <p>Because the iterator will remain unmodified until the actual execution happens,
 	 * the type of data returned by the iterator must be given explicitly in the form of the type
-	 * class (this is due to the fact that the Java compiler erases the generic type information).</p>
+	 * class (this is due to the fact that the Java compiler erases the generic type information).
 	 *
 	 * <p>Note that this operation will result in a non-parallel data stream source, i.e.,
-	 * a data stream source with a parallelism of one.</p>
+	 * a data stream source with a parallelism of one.
 	 *
 	 * @param data
 	 * 		The iterator of elements to create the data stream from
@@ -828,10 +828,10 @@ public <OUT> DataStreamSource<OUT> fromCollection(Iterator<OUT> data, Class<OUT>
 	 * the type of data returned by the iterator must be given explicitly in the form of the type
 	 * information. This method is useful for cases where the type is generic.
 	 * In that case, the type class (as given in
-	 * {@link #fromCollection(java.util.Iterator, Class)} does not supply all type information.</p>
+	 * {@link #fromCollection(java.util.Iterator, Class)} does not supply all type information.
 	 *
 	 * <p>Note that this operation will result in a non-parallel data stream source, i.e.,
-	 * a data stream source with a parallelism one.</p>
+	 * a data stream source with a parallelism one.
 	 *
 	 * @param data
 	 * 		The iterator of elements to create the data stream from
@@ -849,12 +849,13 @@ public <OUT> DataStreamSource<OUT> fromCollection(Iterator<OUT> data, TypeInform
 	}
 
 	/**
-	 * Creates a new data stream that contains elements in the iterator. The iterator is splittable, allowing the
-	 * framework to create a parallel data stream source that returns the elements in the iterator.
+	 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
+	 * allowing the framework to create a parallel data stream source that returns the elements in
+	 * the iterator.
 	 *
-	 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data returned by the
-	 * iterator must be given explicitly in the form of the type class (this is due to the fact that the Java compiler
-	 * erases the generic type information).</p>
+	 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
+	 * of data returned by the iterator must be given explicitly in the form of the type class
+	 * (this is due to the fact that the Java compiler erases the generic type information).
 	 *
 	 * @param iterator
 	 * 		The iterator that produces the elements of the data stream
@@ -869,13 +870,16 @@ public <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT
 	}
 
 	/**
-	 * Creates a new data stream that contains elements in the iterator. The iterator is splittable, allowing the
-	 * framework to create a parallel data stream source that returns the elements in the iterator.
-	 * <p>
-	 * Because the iterator will remain unmodified until the actual execution happens, the type of data returned by the
-	 * iterator must be given explicitly in the form of the type information. This method is useful for cases where the
-	 * type is generic. In that case, the type class (as given in {@link #fromParallelCollection(org.apache.flink.util.SplittableIterator,
-	 * Class)} does not supply all type information.
+	 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
+	 * allowing the framework to create a parallel data stream source that returns the elements in
+	 * the iterator.
+	 *
+	 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
+	 * of data returned by the iterator must be given explicitly in the form of the type
+	 * information. This method is useful for cases where the type is generic. In that case, the
+	 * type class (as given in
+	 * {@link #fromParallelCollection(org.apache.flink.util.SplittableIterator, Class)} does not
+	 * supply all type information.
 	 *
 	 * @param iterator
 	 * 		The iterator that produces the elements of the data stream
@@ -897,15 +901,15 @@ private <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OU
 	}
 
 	/**
-	 * Reads the given file line-by-line and creates a data stream that contains a string with the contents of each such
-	 * line. The file will be read with the system's default character set.
+	 * Reads the given file line-by-line and creates a data stream that contains a string with the
+	 * contents of each such line. The file will be read with the system's default character set.
 	 *
-	 * <p>
-	 * <b> NOTES ON CHECKPOINTING: </b> The source monitors the path, creates the
-	 * {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits} to be processed,
-	 * forwards them to the downstream {@link ContinuousFileReaderOperator readers} to read the actual data,
-	 * and exits, without waiting for the readers to finish reading. This implies that no more checkpoint
-	 * barriers are going to be forwarded after the source exits, thus having no checkpoints after that point.
+	 * <p><b>NOTES ON CHECKPOINTING: </b> The source monitors the path, creates the
+	 * {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits} to be processed, forwards
+	 * them to the downstream {@link ContinuousFileReaderOperator readers} to read the actual data,
+	 * and exits, without waiting for the readers to finish reading. This implies that no more
+	 * checkpoint barriers are going to be forwarded after the source exits, thus having no
+	 * checkpoints after that point.
 	 *
 	 * @param filePath
 	 * 		The path of the file, as a URI (e.g., "file:///some/local/file" or "hdfs://host:port/file/path").
@@ -916,11 +920,11 @@ public DataStreamSource<String> readTextFile(String filePath) {
 	}
 
 	/**
-	 * Reads the given file line-by-line and creates a data stream that contains a string with the contents of each such
-	 * line. The {@link java.nio.charset.Charset} with the given name will be used to read the files.
+	 * Reads the given file line-by-line and creates a data stream that contains a string with the
+	 * contents of each such line. The {@link java.nio.charset.Charset} with the given name will be
+	 * used to read the files.
 	 *
-	 * <p>
-	 * <b> NOTES ON CHECKPOINTING: </b> The source monitors the path, creates the
+	 * <p><b>NOTES ON CHECKPOINTING: </b> The source monitors the path, creates the
 	 * {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits} to be processed,
 	 * forwards them to the downstream {@link ContinuousFileReaderOperator readers} to read the actual data,
 	 * and exits, without waiting for the readers to finish reading. This implies that no more checkpoint
@@ -947,16 +951,14 @@ public DataStreamSource<String> readTextFile(String filePath, String charsetName
 	/**
 	 * Reads the contents of the user-specified {@code filePath} based on the given {@link FileInputFormat}.
 	 *
-	 * <p>
-	 * Since all data streams need specific information about their types, this method needs to determine the
+	 * <p>Since all data streams need specific information about their types, this method needs to determine the
 	 * type of the data produced by the input format. It will attempt to determine the data type by reflection,
 	 * unless the input format implements the {@link org.apache.flink.api.java.typeutils.ResultTypeQueryable} interface.
 	 * In the latter case, this method will invoke the
 	 * {@link org.apache.flink.api.java.typeutils.ResultTypeQueryable#getProducedType()} method to determine data
 	 * type produced by the input format.
 	 *
-	 * <p>
-	 * <b> NOTES ON CHECKPOINTING: </b> The source monitors the path, creates the
+	 * <p><b>NOTES ON CHECKPOINTING: </b> The source monitors the path, creates the
 	 * {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits} to be processed,
 	 * forwards them to the downstream {@link ContinuousFileReaderOperator readers} to read the actual data,
 	 * and exits, without waiting for the readers to finish reading. This implies that no more checkpoint
@@ -976,11 +978,10 @@ public <OUT> DataStreamSource<OUT> readFile(FileInputFormat<OUT> inputFormat,
 	}
 
 	/**
-	 *
 	 * Reads the contents of the user-specified {@code filePath} based on the given {@link FileInputFormat}. Depending
 	 * on the provided {@link FileProcessingMode}.
-	 * <p>
-	 * See {@link #readFile(FileInputFormat, String, FileProcessingMode, long)}
+	 *
+	 * <p>See {@link #readFile(FileInputFormat, String, FileProcessingMode, long)}
 	 *
 	 * @param inputFormat
 	 * 		The input format used to create the data stream
@@ -1028,16 +1029,14 @@ public <OUT> DataStreamSource<OUT> readFile(FileInputFormat<OUT> inputFormat,
 	 * can specify a custom {@link FilePathFilter}. As a default implementation you can use
 	 * {@link FilePathFilter#createDefaultFilter()}.
 	 *
-	 * <p>
-	 * Since all data streams need specific information about their types, this method needs to determine the
+	 * <p>Since all data streams need specific information about their types, this method needs to determine the
 	 * type of the data produced by the input format. It will attempt to determine the data type by reflection,
 	 * unless the input format implements the {@link org.apache.flink.api.java.typeutils.ResultTypeQueryable} interface.
 	 * In the latter case, this method will invoke the
 	 * {@link org.apache.flink.api.java.typeutils.ResultTypeQueryable#getProducedType()} method to determine data
 	 * type produced by the input format.
 	 *
-	 * <p>
-	 * <b> NOTES ON CHECKPOINTING: </b> If the {@code watchType} is set to {@link FileProcessingMode#PROCESS_ONCE},
+	 * <p><b>NOTES ON CHECKPOINTING: </b> If the {@code watchType} is set to {@link FileProcessingMode#PROCESS_ONCE},
 	 * the source monitors the path <b>once</b>, creates the {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits}
 	 * to be processed, forwards them to the downstream {@link ContinuousFileReaderOperator readers} to read the actual data,
 	 * and exits, without waiting for the readers to finish reading. This implies that no more checkpoint barriers
@@ -1108,8 +1107,7 @@ public DataStream<String> readFileStream(String filePath, long intervalMillis, F
 	 * the user can specify a custom {@link FilePathFilter}. As a default implementation you can use
 	 * {@link FilePathFilter#createDefaultFilter()}.
 	 *
-	 * <p>
-	 *  <b> NOTES ON CHECKPOINTING: </b> If the {@code watchType} is set to {@link FileProcessingMode#PROCESS_ONCE},
+	 * <p><b>NOTES ON CHECKPOINTING: </b> If the {@code watchType} is set to {@link FileProcessingMode#PROCESS_ONCE},
 	 * the source monitors the path <b>once</b>, creates the {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits}
 	 * to be processed, forwards them to the downstream {@link ContinuousFileReaderOperator readers} to read the actual data,
 	 * and exits, without waiting for the readers to finish reading. This implies that no more checkpoint barriers
@@ -1148,8 +1146,8 @@ public <OUT> DataStreamSource<OUT> readFile(FileInputFormat<OUT> inputFormat,
 	 * Creates a new data stream that contains the strings received infinitely from a socket. Received strings are
 	 * decoded by the system's default character set. On the termination of the socket server connection retries can be
 	 * initiated.
-	 * <p>
-	 * Let us note that the socket itself does not report on abort and as a consequence retries are only initiated when
+	 *
+	 * <p>Let us note that the socket itself does not report on abort and as a consequence retries are only initiated when
 	 * the socket was gracefully terminated.
 	 *
 	 * @param hostname
@@ -1177,8 +1175,8 @@ public DataStreamSource<String> socketTextStream(String hostname, int port, char
 	 * Creates a new data stream that contains the strings received infinitely from a socket. Received strings are
 	 * decoded by the system's default character set. On the termination of the socket server connection retries can be
 	 * initiated.
-	 * <p>
-	 * Let us note that the socket itself does not report on abort and as a consequence retries are only initiated when
+	 *
+	 * <p>Let us note that the socket itself does not report on abort and as a consequence retries are only initiated when
 	 * the socket was gracefully terminated.
 	 *
 	 * @param hostname
@@ -1259,16 +1257,15 @@ public DataStreamSource<String> socketTextStream(String hostname, int port) {
 
 	/**
 	 * Generic method to create an input data stream with {@link org.apache.flink.api.common.io.InputFormat}.
-	 * <p>
-	 * Since all data streams need specific information about their types, this method needs to determine the
+	 *
+	 * <p>Since all data streams need specific information about their types, this method needs to determine the
 	 * type of the data produced by the input format. It will attempt to determine the data type by reflection,
 	 * unless the input format implements the {@link org.apache.flink.api.java.typeutils.ResultTypeQueryable} interface.
 	 * In the latter case, this method will invoke the
 	 * {@link org.apache.flink.api.java.typeutils.ResultTypeQueryable#getProducedType()} method to determine data
 	 * type produced by the input format.
 	 *
-	 * <p>
-	 * <b> NOTES ON CHECKPOINTING: </b> In the case of a {@link FileInputFormat}, the source
+	 * <p><b>NOTES ON CHECKPOINTING: </b> In the case of a {@link FileInputFormat}, the source
 	 * (which executes the {@link ContinuousFileMonitoringFunction}) monitors the path, creates the
 	 * {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits} to be processed, forwards
 	 * them to the downstream {@link ContinuousFileReaderOperator} to read the actual data, and exits,
@@ -1288,13 +1285,12 @@ public <OUT> DataStreamSource<OUT> createInput(InputFormat<OUT, ?> inputFormat)
 
 	/**
 	 * Generic method to create an input data stream with {@link org.apache.flink.api.common.io.InputFormat}.
-	 * <p>
-	 * The data stream is typed to the given TypeInformation. This method is intended for input formats
+	 *
+	 * <p>The data stream is typed to the given TypeInformation. This method is intended for input formats
 	 * where the return type cannot be determined by reflection analysis, and that do not implement the
 	 * {@link org.apache.flink.api.java.typeutils.ResultTypeQueryable} interface.
 	 *
-	 * <p>
-	 * <b> NOTES ON CHECKPOINTING: </b> In the case of a {@link FileInputFormat}, the source
+	 * <p><b>NOTES ON CHECKPOINTING: </b> In the case of a {@link FileInputFormat}, the source
 	 * (which executes the {@link ContinuousFileMonitoringFunction}) monitors the path, creates the
 	 * {@link org.apache.flink.core.fs.FileInputSplit FileInputSplits} to be processed, forwards
 	 * them to the downstream {@link ContinuousFileReaderOperator} to read the actual data, and exits,
@@ -1364,8 +1360,7 @@ private <OUT> DataStreamSource<OUT> createFileInput(FileInputFormat<OUT> inputFo
 	/**
 	 * Adds a Data Source to the streaming topology.
 	 *
-	 * <p>
-	 * By default sources have a parallelism of 1. To enable parallel execution, the user defined source should
+	 * <p>By default sources have a parallelism of 1. To enable parallel execution, the user defined source should
 	 * implement {@link org.apache.flink.streaming.api.functions.source.ParallelSourceFunction} or extend {@link
 	 * org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction}. In these cases the resulting source
 	 * will have the parallelism of the environment. To change this afterwards call {@link
@@ -1466,7 +1461,7 @@ public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, Strin
 	/**
 	 * Casts the source function into a SourceFunction implementing the StoppableFunction.
 	 *
-	 * This method should only be used if the source function was checked to implement the
+	 * <p>This method should only be used if the source function was checked to implement the
 	 * {@link StoppableFunction} interface.
 	 *
 	 * @param sourceFunction Source function to cast
@@ -1483,8 +1478,8 @@ private <OUT, T extends SourceFunction<OUT> & StoppableFunction> T cast2Stoppabl
 	 * Triggers the program execution. The environment will execute all parts of
 	 * the program that have resulted in a "sink" operation. Sink operations are
 	 * for example printing results or forwarding them to a message queue.
-	 * <p>
-	 * The program execution will be logged and displayed with a generated
+	 *
+	 * <p>The program execution will be logged and displayed with a generated
 	 * default name.
 	 *
 	 * @return The result of the job execution, containing elapsed time and accumulators.
@@ -1498,8 +1493,8 @@ public JobExecutionResult execute() throws Exception {
 	 * Triggers the program execution. The environment will execute all parts of
 	 * the program that have resulted in a "sink" operation. Sink operations are
 	 * for example printing results or forwarding them to a message queue.
-	 * <p>
-	 * The program execution will be logged and displayed with the provided name
+	 *
+	 * <p>The program execution will be logged and displayed with the provided name
 	 *
 	 * @param jobName
 	 * 		Desired name of the job
@@ -1550,12 +1545,10 @@ public <F> F clean(F f) {
 	 * Adds an operator to the list of operators that should be executed when calling
 	 * {@link #execute}.
 	 *
-	 * <p>
-	 * When calling {@link #execute()} only the operators that where previously added to the list
+	 * <p>When calling {@link #execute()} only the operators that where previously added to the list
 	 * are executed.
 	 *
-	 * <p>
-	 * This is not meant to be used by users. The API methods that create operators must call
+	 * <p>This is not meant to be used by users. The API methods that create operators must call
 	 * this method.
 	 */
 	@Internal
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
index b1521f529877c..50f2ed3fe7fb0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
@@ -26,6 +26,10 @@
 import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.streaming.api.graph.StreamGraph;
 
+/**
+ * A special {@link StreamExecutionEnvironment} that is used in the web frontend when generating
+ * a user-inspectable graph of a streaming job.
+ */
 @PublicEvolving
 public class StreamPlanEnvironment extends StreamExecutionEnvironment {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java
index d13e4618e09b9..4b6d52ae35fd9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java
@@ -25,7 +25,7 @@
  * ascending. In this case, the local watermarks for the streams are easy to generate, because
  * they strictly follow the timestamps.
  *
- * <b>Note:</b> This is just a deprecated stub class. The actual code for this has moved to
+ * <p><b>Note:</b> This is just a deprecated stub class. The actual code for this has moved to
  * {@link org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor}.
  *
  * @param <T> The type of the elements that this function can extract timestamps from
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java
index b220d077ac7c9..2dc909160ec0a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java
@@ -24,14 +24,12 @@
 /**
  * Interface for user functions that extract timestamps from elements.
  *
- * <p>
- * The extractor must also keep track of the current watermark. The system will periodically
- * retrieve this watermark using {@link #getCurrentWatermark()} and submit it throughout the topology.
+ * <p>The extractor must also keep track of the current watermark. The system will periodically
+ * retrieve this watermark using {@link #getCurrentWatermark()} and submit it throughout the
+ * topology.
  *
- * <p>
- * Note: If you know that timestamps are monotonically increasing you can use
- * {@link AscendingTimestampExtractor}. This will
- * keep track of watermarks.
+ * <p>Note: If you know that timestamps are monotonically increasing you can use
+ * {@link AscendingTimestampExtractor}. This will keep track of watermarks.
  *
  * @param <T> The type of the elements that this function can extract timestamps from
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java
index 57c48a62dac41..e6ae93f24c4fb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/AggregationFunction.java
@@ -20,12 +20,17 @@
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.ReduceFunction;
 
+/**
+ * Holder class for aggregation types that can be used on a windowed stream or keyed stream.
+ */
 @Internal
 public abstract class AggregationFunction<T> implements ReduceFunction<T> {
 	private static final long serialVersionUID = 1L;
 
+	/**
+	 * Aggregation types that can be used on a windowed stream or keyed stream.
+	 */
 	public enum AggregationType {
 		SUM, MIN, MAX, MINBY, MAXBY,
 	}
-
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java
index c634434dbb512..3216e3a711163 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java
@@ -23,6 +23,10 @@
 import org.apache.flink.streaming.util.typeutils.FieldAccessor;
 import org.apache.flink.streaming.util.typeutils.FieldAccessorFactory;
 
+/**
+ * An {@link AggregationFunction} that computes values based on comparisons of
+ * {@link Comparable Comparables}.
+ */
 @Internal
 public class ComparableAggregator<T> extends AggregationFunction<T> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/Comparator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/Comparator.java
index 14e1764240e59..9202a499057f4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/Comparator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/Comparator.java
@@ -22,6 +22,9 @@
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.functions.aggregation.AggregationFunction.AggregationType;
 
+/**
+ * Internal comparator implementation, for use with {@link ComparableAggregator}.
+ */
 @Internal
 public abstract class Comparator implements Serializable {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
index 5e1378ed463d7..58b5981fc8967 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
@@ -26,6 +26,9 @@
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.streaming.util.typeutils.FieldAccessor;
 
+/**
+ * An {@link AggregationFunction} that sums up fields.
+ */
 @Internal
 public class SumAggregator<T> extends AggregationFunction<T> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java
index 124b726b7e58d..a177c580c8159 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java
@@ -21,8 +21,11 @@
 
 import java.io.Serializable;
 
+/**
+ * Internal function for summing up contents of fields. This is used with {@link SumAggregator}.
+ */
 @Internal
-public abstract class SumFunction implements Serializable{
+public abstract class SumFunction implements Serializable {
 
 	private static final long serialVersionUID = 1L;
 
@@ -48,7 +51,7 @@ public static SumFunction getForClass(Class<?> clazz) {
 		}
 	}
 
-	public static class IntSum extends SumFunction {
+	static class IntSum extends SumFunction {
 		private static final long serialVersionUID = 1L;
 
 		@Override
@@ -57,7 +60,7 @@ public Object add(Object value1, Object value2) {
 		}
 	}
 
-	public static class LongSum extends SumFunction {
+	static class LongSum extends SumFunction {
 		private static final long serialVersionUID = 1L;
 
 		@Override
@@ -66,7 +69,7 @@ public Object add(Object value1, Object value2) {
 		}
 	}
 
-	public static class DoubleSum extends SumFunction {
+	static class DoubleSum extends SumFunction {
 
 		private static final long serialVersionUID = 1L;
 
@@ -76,7 +79,7 @@ public Object add(Object value1, Object value2) {
 		}
 	}
 
-	public static class ShortSum extends SumFunction {
+	static class ShortSum extends SumFunction {
 		private static final long serialVersionUID = 1L;
 
 		@Override
@@ -85,7 +88,7 @@ public Object add(Object value1, Object value2) {
 		}
 	}
 
-	public static class FloatSum extends SumFunction {
+	static class FloatSum extends SumFunction {
 		private static final long serialVersionUID = 1L;
 
 		@Override
@@ -94,7 +97,7 @@ public Object add(Object value1, Object value2) {
 		}
 	}
 
-	public static class ByteSum extends SumFunction {
+	static class ByteSum extends SumFunction {
 		private static final long serialVersionUID = 1L;
 
 		@Override
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java
index 8c788b48dace6..552f009b72b55 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java
@@ -57,11 +57,10 @@
  * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
  * {@link RichFunction#close()}.
  *
- * <p>
- * State related apis in {@link RuntimeContext} are not supported yet because the key may get
+ * <p>State related apis in {@link RuntimeContext} are not supported yet because the key may get
  * changed while accessing states in the working thread.
- * <p>
- * {@link IterationRuntimeContext#getIterationAggregator(String)} is not supported since the
+ *
+ * <p>{@link IterationRuntimeContext#getIterationAggregator(String)} is not supported since the
  * aggregator may be modified by multiple threads.
  *
  * @param <IN> The type of the input elements.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java
index 25078ae89bcb4..964c13ab4e8b2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java
@@ -31,19 +31,18 @@
 public interface AsyncCollector<OUT> {
 	/**
 	 * Set result.
-	 * <p>
-	 * Note that it should be called for exactly one time in the user code.
+	 *
+	 * <p>Note that it should be called for exactly one time in the user code.
 	 * Calling this function for multiple times will cause data lose.
-	 * <p>
-	 * Put all results in a {@link Collection} and then issue
-	 * {@link AsyncCollector#collect(Collection)}.
+	 *
+	 * <p>Put all results in a {@link Collection} and then emit output.
 	 *
 	 * @param result A list of results.
 	 */
 	void collect(Collection<OUT> result);
 
 	/**
-	 * Set error
+	 * Set error.
 	 *
 	 * @param error A Throwable object.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java
index 8811e32608eda..4285cc3ec0498 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java
@@ -51,7 +51,7 @@ public abstract class CoProcessFunction<IN1, IN2, OUT> extends AbstractRichFunct
 	/**
 	 * This method is called for each element in the first of the connected streams.
 	 *
-	 * This function can output zero or more elements using the {@link Collector} parameter
+	 * <p>This function can output zero or more elements using the {@link Collector} parameter
 	 * and also update internal state or set timers using the {@link Context} parameter.
 	 * 
 	 * @param value The stream element
@@ -68,7 +68,7 @@ public abstract class CoProcessFunction<IN1, IN2, OUT> extends AbstractRichFunct
 	/**
 	 * This method is called for each element in the second of the connected streams.
 	 *
-	 * This function can output zero or more elements using the {@link Collector} parameter
+	 * <p>This function can output zero or more elements using the {@link Collector} parameter
 	 * and also update internal state or set timers using the {@link Context} parameter.
 	 * 
 	 * @param value The stream element
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/RichSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/RichSinkFunction.java
index 643c06ca39ef8..64c38b9affb9e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/RichSinkFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/RichSinkFunction.java
@@ -20,6 +20,9 @@
 import org.apache.flink.annotation.Public;
 import org.apache.flink.api.common.functions.AbstractRichFunction;
 
+/**
+ * A {@link org.apache.flink.api.common.functions.RichFunction} version of {@link SinkFunction}.
+ */
 @Public
 public abstract class RichSinkFunction<IN> extends AbstractRichFunction implements SinkFunction<IN> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
index 807735dfbe657..0d0669a778360 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
@@ -34,11 +34,11 @@
 
 /**
  * Socket client that acts as a streaming sink. The data is sent to a Socket as a byte array.
- * <p>
- * The sink can be set to retry message sends after the sending failed.
- * <p>
- * The sink can be set to 'autoflush', in which case the socket stream is flushed after every message. This
- * significantly reduced throughput, but also decreases message latency.
+ *
+ * <p>The sink can be set to retry message sends after the sending failed.
+ *
+ * <p>The sink can be set to 'autoflush', in which case the socket stream is flushed after every
+ * message. This significantly reduced throughput, but also decreases message latency.
  *
  * @param <IN> data to be written into the Socket.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java
index d445be018377b..65915af19dae8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java
@@ -35,7 +35,7 @@ public abstract class WriteFormat<IN> implements Serializable {
 
 	/**
 	 * Writes the contents of tupleList to the file specified by path.
-	 * 
+	 *
 	 * @param path
 	 *            is the path to the location where the tuples are written
 	 * @param tupleList
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
index 8b82d841ed998..869413592f908 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
@@ -56,10 +56,10 @@
  *     <li>Assigning them to downstream tasks for further processing.</li>
  * </ol>
  *
- * The splits to be read are forwarded to the downstream {@link ContinuousFileReaderOperator}
+ * <p>The splits to be read are forwarded to the downstream {@link ContinuousFileReaderOperator}
  * which can have parallelism greater than one.
  *
- * <b>IMPORTANT NOTE: </b> Splits are forwarded downstream for reading in ascending modification time order,
+ * <p><b>IMPORTANT NOTE: </b> Splits are forwarded downstream for reading in ascending modification time order,
  * based on the modification time of the files they belong to.
  */
 @Internal
@@ -72,6 +72,7 @@ public class ContinuousFileMonitoringFunction<OUT>
 
 	/**
 	 * The minimum interval allowed between consecutive path scans.
+	 *
 	 * <p><b>NOTE:</b> Only applicable to the {@code PROCESS_CONTINUOUSLY} mode.
 	 */
 	public static final long MIN_MONITORING_INTERVAL = 1L;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
index ff941f13e5028..8cee0bcec0bb0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
@@ -33,6 +33,9 @@
 import java.util.Map;
 
 /**
+ * A {@link SourceFunction} that monitors a directory and sends events downstream when it detects
+ * new files. Used together with {@link FileReadFunction}.
+ *
  * @deprecated Internal class deprecated in favour of {@link ContinuousFileMonitoringFunction}.
  */
 @Internal
@@ -42,6 +45,9 @@ public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Lon
 
 	private static final Logger LOG = LoggerFactory.getLogger(FileMonitoringFunction.class);
 
+	/**
+	 * The watch type of the {@code FileMonitoringFunction}.
+	 */
 	public enum WatchType {
 		ONLY_NEW_FILES, // Only new files will be processed.
 		REPROCESS_WITH_APPENDED, // When some files are appended, all contents
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java
index ff8a29549aae0..172f7a3c900e3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java
@@ -30,6 +30,9 @@
 import org.apache.flink.util.Collector;
 
 /**
+ * This is used together with {@link FileMonitoringFunction} to read from files that the
+ * monitoring functions discovers.
+ *
  * @deprecated Internal class deprecated in favour of {@link ContinuousFileMonitoringFunction}.
  */
 @Internal
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
index e294cae22c6b0..68a17508cadd0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
@@ -45,8 +45,7 @@
  * That way, any object transport using Java serialization will not be affected by the serializability
  * of the elements.</p>
  *
- * <p>
- * <b>NOTE:</b> This source has a parallelism of 1.
+ * <p><b>NOTE:</b> This source has a parallelism of 1.
  *
  * @param <T> The type of elements returned by this function.
  */
@@ -55,22 +54,22 @@ public class FromElementsFunction<T> implements SourceFunction<T>, CheckpointedF
 	
 	private static final long serialVersionUID = 1L;
 
-	/** The (de)serializer to be used for the data elements */
+	/** The (de)serializer to be used for the data elements. */
 	private final TypeSerializer<T> serializer;
 	
-	/** The actual data elements, in serialized form */
+	/** The actual data elements, in serialized form. */
 	private final byte[] elementsSerialized;
 	
-	/** The number of serialized elements */
+	/** The number of serialized elements. */
 	private final int numElements;
 
-	/** The number of elements emitted already */
+	/** The number of elements emitted already. */
 	private volatile int numElementsEmitted;
 
-	/** The number of elements to skip initially */
+	/** The number of elements to skip initially. */
 	private volatile int numElementsToSkip;
 	
-	/** Flag to make the source cancelable */
+	/** Flag to make the source cancelable. */
 	private volatile boolean isRunning = true;
 
 	private transient ListState<Integer> checkpointedState;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java
index e721bf3542c56..a8b527f471e9d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java
@@ -21,6 +21,9 @@
 
 import java.util.Iterator;
 
+/**
+ * A {@link SourceFunction} that reads elements from an {@link Iterator} and emits them.
+ */
 @PublicEvolving
 public class FromIteratorFunction<T> implements SourceFunction<T> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java
index dc97b1a7698a1..db6c8a22d37f3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java
@@ -23,6 +23,9 @@
 
 import java.util.Iterator;
 
+/**
+ * A {@link SourceFunction} that reads elements from an {@link SplittableIterator} and emits them.
+ */
 @PublicEvolving
 public class FromSplittableIteratorFunction<T> extends RichParallelSourceFunction<T> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
index c067ca7785288..9592c889f5e30 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
@@ -32,6 +32,9 @@
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 
+/**
+ * A {@link SourceFunction} that reads data using an {@link InputFormat}.
+ */
 @Internal
 public class InputFormatSourceFunction<OUT> extends RichParallelSourceFunction<OUT> {
 	private static final long serialVersionUID = 1L;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java
index e835070d0fde7..3da1ec34b4626 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java
@@ -44,27 +44,27 @@
 /**
  * Abstract base class for data sources that receive elements from a message queue and
  * acknowledge them back by IDs.
- * <p>
- * The mechanism for this source assumes that messages are identified by a unique ID.
+ *
+ * <p>The mechanism for this source assumes that messages are identified by a unique ID.
  * When messages are taken from the message queue, the message must not be dropped immediately,
  * but must be retained until acknowledged. Messages that are not acknowledged within a certain
  * time interval will be served again (to a different connection, established by the recovered source).
- * <p>
- * Note that this source can give no guarantees about message order in the case of failures,
+ *
+ * <p>Note that this source can give no guarantees about message order in the case of failures,
  * because messages that were retrieved but not yet acknowledged will be returned later again, after
  * a set of messages that was not retrieved before the failure.
- * <p>
- * Internally, this source gathers the IDs of elements it emits. Per checkpoint, the IDs are stored and
+ *
+ * <p>Internally, this source gathers the IDs of elements it emits. Per checkpoint, the IDs are stored and
  * acknowledged when the checkpoint is complete. That way, no message is acknowledged unless it is certain
  * that it has been successfully processed throughout the topology and the updates to any state caused by
  * that message are persistent.
- * <p>
- * All messages that are emitted and successfully processed by the streaming program will eventually be
+ *
+ * <p>All messages that are emitted and successfully processed by the streaming program will eventually be
  * acknowledged. In corner cases, the source may receive certain IDs multiple times, if a
  * failure occurs while acknowledging. To cope with this situation, an additional Set stores all
  * processed IDs. IDs are only removed after they have been acknowledged.
- * <p>
- * A typical way to use this base in a source function is by implementing a run() method as follows:
+ *
+ * <p>A typical way to use this base in a source function is by implementing a run() method as follows:
  * <pre>{@code
  * public void run(SourceContext<Type> ctx) throws Exception {
  *     while (running) {
@@ -91,13 +91,16 @@ public abstract class MessageAcknowledgingSourceBase<Type, UId>
 
 	private static final Logger LOG = LoggerFactory.getLogger(MessageAcknowledgingSourceBase.class);
 
-	/** Serializer used to serialize the IDs for checkpoints */
+	/** Serializer used to serialize the IDs for checkpoints. */
 	private final TypeSerializer<UId> idSerializer;
 
-	/** The list gathering the IDs of messages emitted during the current checkpoint */
+	/** The list gathering the IDs of messages emitted during the current checkpoint. */
 	private transient List<UId> idsForCurrentCheckpoint;
 
-	/** The list with IDs from checkpoints that were triggered, but not yet completed or notified of completion */
+	/**
+	 * The list with IDs from checkpoints that were triggered, but not yet completed or notified of
+	 * completion.
+	 */
 	protected transient ArrayDeque<Tuple2<Long, List<UId>>> pendingCheckpoints;
 
 	/**
@@ -178,6 +181,7 @@ public void close() throws Exception {
 
 	/**
 	 * This method must be implemented to acknowledge the given set of IDs back to the message queue.
+	 *
 	 * @param uIds The list od IDs to acknowledge.
 	 */
 	protected abstract void acknowledgeIDs(long checkpointId, List<UId> uIds);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java
index 188fdce15be20..965dc02223c16 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java
@@ -37,10 +37,12 @@
  * acknowledge them back by IDs. In contrast to {@link MessageAcknowledgingSourceBase}, this source
  * handles two types of ids:
  *
- * 1) Session ids
- * 2) Unique message ids
+ * <ol>
+ *   <li>Session ids
+ *   <li>Unique message ids
+ * </ol>
  *
- * Session ids are used to acknowledge messages in a session. When a checkpoint is restored,
+ * <p>Session ids are used to acknowledge messages in a session. When a checkpoint is restored,
  * unacknowledged messages are redelivered. Duplicates are detected using the unique message ids
  * which are checkpointed.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
index d1fea1e0be123..8d77b15fbac20 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
@@ -32,14 +32,15 @@
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
- * A source function that reads strings from a socket. The source will read bytes from the socket stream
- * and convert them to characters, each byte individually. When the delimiter character is received,
- * the function will output the current string, and begin a new string.
- * <p>
- * The function strips trailing <i>carriage return</i> characters (\r) when the delimiter is the
+ * A source function that reads strings from a socket. The source will read bytes from the socket
+ * stream and convert them to characters, each byte individually. When the delimiter character is
+ * received, the function will output the current string, and begin a new string.
+ *
+ * <p>The function strips trailing <i>carriage return</i> characters (\r) when the delimiter is the
  * newline character (\n).
- * <p>
- * The function can be set to reconnect to the server socket in case that the stream is closed on the server side.
+ *
+ * <p>The function can be set to reconnect to the server socket in case that the stream is closed on
+ * the server side.
  */
 @PublicEvolving
 public class SocketTextStreamFunction implements SourceFunction<String> {
@@ -48,10 +49,10 @@ public class SocketTextStreamFunction implements SourceFunction<String> {
 	
 	private static final Logger LOG = LoggerFactory.getLogger(SocketTextStreamFunction.class);
 
-	/** Default delay between successive connection attempts */
+	/** Default delay between successive connection attempts. */
 	private static final int DEFAULT_CONNECTION_RETRY_SLEEP = 500;
 
-	/** Default connection timeout when connecting to the server socket (infinite) */
+	/** Default connection timeout when connecting to the server socket (infinite). */
 	private static final int CONNECTION_TIMEOUT_TIME = 0;
 	
 	
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
index fc7f793ddfeb3..ba18df442c941 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
@@ -69,7 +69,6 @@
  *
  *
  * <h3>Timestamps and watermarks:</h3>
- * 
  * Sources may assign timestamps to elements and may manually emit watermarks.
  * However, these are only interpreted if the streaming program runs on 
  * {@link TimeCharacteristic#EventTime}. On other time characteristics 
@@ -77,7 +76,6 @@
  * the watermarks from the source function are ignored.
  *
  * <h3>Gracefully Stopping Functions</h3>
- * 
  * Functions may additionally implement the {@link org.apache.flink.api.common.functions.StoppableFunction}
  * interface. "Stopping" a function, in contrast to "canceling" means a graceful exit that leaves the
  * state and the emitted elements in a consistent state.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
index 2a0be98bcb062..07eb1d9c49d16 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
@@ -66,13 +66,12 @@ public TimestampedFileInputSplit(long modificationTime, int num, Path file, long
 	}
 
 	/**
-	 * Sets the state of the split. This information is used when
-	 * restoring from a checkpoint and allows to resume reading the
-	 * underlying file from the point we left off.
-	 * <p>
-	 * This is applicable to {@link org.apache.flink.api.common.io.FileInputFormat FileInputFormats}
-	 * that implement the {@link org.apache.flink.api.common.io.CheckpointableInputFormat
-	 * CheckpointableInputFormat} interface.
+	 * Sets the state of the split. This information is used when restoring from a checkpoint and
+	 * allows to resume reading the underlying file from the point we left off.
+	 *
+	 * <p>* This is applicable to
+	 * {@link org.apache.flink.api.common.io.FileInputFormat FileInputFormats} that implement the
+	 * {@link org.apache.flink.api.common.io.CheckpointableInputFormat} interface.
 	 * */
 	public void setSplitState(Serializable state) {
 		this.splitState = state;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
index 0a138f27c908a..253608f383e9c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
@@ -42,7 +42,7 @@ public abstract class AscendingTimestampExtractor<T> implements AssignerWithPeri
 	/** The current timestamp. */
 	private long currentTimestamp = Long.MIN_VALUE;
 
-	/** Handler that is called when timestamp monotony is violated */
+	/** Handler that is called when timestamp monotony is violated. */
 	private MonotonyViolationHandler violationHandler = new LoggingHandler();
 	
 	
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java
index 929e336e481a8..e20b8789c7109 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
index 73e1f0f85e0f1..912a5c03be53d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
index 0efffb9fdf118..2514355fe5231 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -35,6 +35,10 @@
 import java.io.IOException;
 import java.util.Collections;
 
+/**
+ * Internal {@link AllWindowFunction} that is used for implementing a fold on a window configuration
+ * that only allows {@link AllWindowFunction} and cannot directly execute a {@link FoldFunction}.
+ */
 @Internal
 public class FoldApplyAllWindowFunction<W extends Window, T, ACC, R>
 	extends WrappingFunction<AllWindowFunction<ACC, R, W>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
index 8e8e52c131936..125da03f27357 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -36,6 +36,11 @@
 import java.io.IOException;
 import java.util.Collections;
 
+/**
+ * Internal {@link ProcessAllWindowFunction} that is used for implementing a fold on a window
+ * configuration that only allows {@link ProcessAllWindowFunction} and cannot directly execute a
+ * {@link FoldFunction}.
+ */
 @Internal
 public class FoldApplyProcessAllWindowFunction<W extends Window, T, ACC, R>
 	extends RichProcessAllWindowFunction<T, R, W>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
index 073a2f38849b3..9a72cab9d13ea 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -36,6 +36,11 @@
 import java.io.IOException;
 import java.util.Collections;
 
+/**
+ * Internal {@link ProcessWindowFunction} that is used for implementing a fold on a window
+ * configuration that only allows {@link ProcessWindowFunction} and cannot directly execute a
+ * {@link FoldFunction}.
+ */
 @Internal
 public class FoldApplyProcessWindowFunction<K, W extends Window, T, ACC, R>
 	extends RichProcessWindowFunction<T, R, K, W>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
index 9e916f1bc78eb..21b9c3b6c6884 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -35,6 +35,10 @@
 import java.io.IOException;
 import java.util.Collections;
 
+/**
+ * Internal {@link WindowFunction} that is used for implementing a fold on a window configuration
+ * that only allows {@link WindowFunction} and cannot directly execute a {@link FoldFunction}.
+ */
 @Internal
 public class FoldApplyWindowFunction<K, W extends Window, T, ACC, R>
 	extends WrappingFunction<WindowFunction<ACC, R, K, W>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughAllWindowFunction.java
index 443564488f5aa..fb1ff819d812b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughAllWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -21,6 +21,9 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
+/**
+ * A {@link AllWindowFunction} that just emits each input element.
+ */
 @Internal
 public class PassThroughAllWindowFunction<W extends Window, T> implements AllWindowFunction<T, T, W> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughWindowFunction.java
index 319acb68ef7f5..edc495c6b8815 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/PassThroughWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -21,6 +21,9 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
+/**
+ * A {@link WindowFunction} that just emits each input element.
+ */
 @Internal
 public class PassThroughWindowFunction<K, W extends Window, T> implements WindowFunction<T, T, K, W> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java
index f49aa27fd1215..4d247a71f76b9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java
@@ -57,7 +57,7 @@ public abstract class ProcessAllWindowFunction<IN, OUT, W extends Window> implem
 	public void clear(Context context) throws Exception {}
 
 	/**
-	 * The context holding window metadata
+	 * The context holding window metadata.
 	 */
 	public abstract class Context {
 		/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
index bcefaf75e1c55..980a94ac97ec7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
@@ -59,7 +59,7 @@ public abstract class ProcessWindowFunction<IN, OUT, KEY, W extends Window> impl
 	public void clear(Context context) throws Exception {}
 
 	/**
-	 * The context holding window metadata
+	 * The context holding window metadata.
 	 */
 	public abstract class Context implements java.io.Serializable {
 		/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java
index 46a645678e157..5a635b77c7996 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -25,6 +25,10 @@
 
 import java.util.Collections;
 
+/**
+ * Internal {@link AllWindowFunction} that is used for implementing a fold on a window configuration
+ * that only allows {@link AllWindowFunction} and cannot directly execute a {@link ReduceFunction}.
+ */
 @Internal
 public class ReduceApplyAllWindowFunction<W extends Window, T, R>
 	extends WrappingFunction<AllWindowFunction<T, R, W>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
index 4c54c94c32968..847a79352ed59 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -27,6 +27,11 @@
 
 import java.util.Collections;
 
+/**
+ * Internal {@link ProcessAllWindowFunction} that is used for implementing a fold on a window
+ * configuration that only allows {@link AllWindowFunction} and cannot directly execute a
+ * {@link ReduceFunction}.
+ */
 @Internal
 public class ReduceApplyProcessAllWindowFunction<W extends Window, T, R>
 	extends RichProcessAllWindowFunction<T, R, W> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
index 1af783a26f059..0fdce83880290 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -27,6 +27,11 @@
 
 import java.util.Collections;
 
+/**
+ * Internal {@link ProcessWindowFunction} that is used for implementing a fold on a window
+ * configuration that only allows {@link AllWindowFunction} and cannot directly execute a
+ * {@link ReduceFunction}.
+ */
 @Internal
 public class ReduceApplyProcessWindowFunction<K, W extends Window, T, R>
 	extends RichProcessWindowFunction<T, R, K, W> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java
index 6e1ba279c9946..c8174641cc55e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -25,6 +25,10 @@
 
 import java.util.Collections;
 
+/**
+ * Internal {@link WindowFunction} that is used for implementing a fold on a window configuration
+ * that only allows {@link WindowFunction} and cannot directly execute a {@link ReduceFunction}.
+ */
 @Internal
 public class ReduceApplyWindowFunction<K, W extends Window, T, R>
 	extends WrappingFunction<WindowFunction<T, R, K, W>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java
deleted file mode 100644
index 8ec5809ac93c5..0000000000000
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableAllWindowFunction.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.api.functions.windowing;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.windows.Window;
-import org.apache.flink.util.Collector;
-
-@Internal
-public class ReduceIterableAllWindowFunction<W extends Window, T> implements AllWindowFunction<T, T, W> {
-	private static final long serialVersionUID = 1L;
-
-	private final ReduceFunction<T> reduceFunction;
-
-	public ReduceIterableAllWindowFunction(ReduceFunction<T> reduceFunction) {
-		this.reduceFunction = reduceFunction;
-	}
-
-	@Override
-	public void apply(W window, Iterable<T> input, Collector<T> out) throws Exception {
-
-		T curr = null;
-		for (T val: input) {
-			if (curr == null) {
-				curr = val;
-			} else {
-				curr = reduceFunction.reduce(curr, val);
-			}
-		}
-		out.collect(curr);
-	}
-}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java
deleted file mode 100644
index afb021950dcf4..0000000000000
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceIterableWindowFunction.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.api.functions.windowing;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.windows.Window;
-import org.apache.flink.util.Collector;
-
-@Internal
-public class ReduceIterableWindowFunction<K, W extends Window, T> implements WindowFunction<T, T, K, W> {
-	private static final long serialVersionUID = 1L;
-
-	private final ReduceFunction<T> reduceFunction;
-
-	public ReduceIterableWindowFunction(ReduceFunction<T> reduceFunction) {
-		this.reduceFunction = reduceFunction;
-	}
-
-	@Override
-	public void apply(K k, W window, Iterable<T> input, Collector<T> out) throws Exception {
-
-		T curr = null;
-		for (T val: input) {
-			if (curr == null) {
-				curr = val;
-			} else {
-				curr = reduceFunction.reduce(curr, val);
-			}
-		}
-		out.collect(curr);
-	}
-}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichAllWindowFunction.java
index 643680f1273ba..b0df6d83c9e68 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichAllWindowFunction.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -21,6 +21,10 @@
 import org.apache.flink.api.common.functions.AbstractRichFunction;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 
+/**
+ * A {@link org.apache.flink.api.common.functions.RichFunction} version of
+ * {@link AllWindowFunction}.
+ */
 @Public
 public abstract class RichAllWindowFunction<IN, OUT, W extends Window> extends AbstractRichFunction implements AllWindowFunction<IN, OUT, W> {
 	private static final long serialVersionUID = 1L;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java
index 5b1e806b73962..a8fc60077a89f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java
@@ -24,7 +24,7 @@
  * This delta function calculates the cosine distance between two given vectors.
  * The cosine distance is defined as: cosineDistance=1-cosineSimilarity
  * 
- * Cosine similarity: http://en.wikipedia.org/wiki/Cosine_similarity
+ * <p>Cosine similarity: http://en.wikipedia.org/wiki/Cosine_similarity
  * 
  * @param <DATA>
  *            The input data type. This delta function works with a double[],
@@ -35,9 +35,6 @@
 @PublicEvolving
 public class CosineDistance<DATA> extends ExtractionAwareDeltaFunction<DATA, double[]> {
 
-	/**
-	 * auto-generated id
-	 */
 	private static final long serialVersionUID = -1217813582965151599L;
 
 	public CosineDistance() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java
index 91020b80f6104..c4b416ce3083a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java
@@ -24,7 +24,7 @@
  * This delta function calculates the euclidean distance between two given
  * points.
  * 
- * Euclidean distance: http://en.wikipedia.org/wiki/Euclidean_distance
+ * <p>Euclidean distance: http://en.wikipedia.org/wiki/Euclidean_distance
  * 
  * @param <DATA>
  *            The input data type. This delta function works with a double[],
@@ -43,9 +43,6 @@ public EuclideanDistance(Extractor<DATA, double[]> converter) {
 		super(converter);
 	}
 
-	/**
-	 * auto-generated version id
-	 */
 	private static final long serialVersionUID = 3119432599634512359L;
 
 	@Override
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java
index 6ec150c90cfaf..535d513f47b61 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java
@@ -37,10 +37,8 @@
 @PublicEvolving
 public abstract class ExtractionAwareDeltaFunction<DATA, TO> implements DeltaFunction<DATA> {
 
-	/**
-	 * Generated Version ID
-	 */
 	private static final long serialVersionUID = 6927486219702689554L;
+	
 	private Extractor<DATA, TO> converter;
 
 	public ExtractionAwareDeltaFunction(Extractor<DATA, TO> converter) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java
index aa48aa502487f..16d0935ac79fc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java
@@ -27,9 +27,6 @@
 @Internal
 public class ArrayFromTuple implements Extractor<Tuple, Object[]> {
 
-	/**
-	 * Auto generated version id
-	 */
 	private static final long serialVersionUID = -6076121226427616818L;
 	int[] order = null;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java
index b1f8f752180f4..6fd9a67dc1c9e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java
@@ -34,9 +34,6 @@
 @Internal
 public class ConcatenatedExtract<FROM, OVER, TO> implements Extractor<FROM, TO> {
 
-	/**
-	 * auto-generated id
-	 */
 	private static final long serialVersionUID = -7807197760725651752L;
 
 	private Extractor<FROM, OVER> e1;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
index 0b7e4da00056d..76d95b9202ba1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
@@ -35,7 +35,7 @@
 public interface Extractor<FROM, TO> extends Serializable {
 
 	/**
-	 * Extracts/Converts the given input to an object of the output type
+	 * Extracts/Converts the given input to an object of the output type.
 	 * 
 	 * @param in
 	 *            the input data
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
index 11afaba3eceb0..c44f32ba12407 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
@@ -30,14 +30,12 @@
 @Internal
 public class FieldFromArray<OUT> implements Extractor<Object, OUT> {
 
-	/**
-	 * Auto-gernated version id
-	 */
 	private static final long serialVersionUID = -5161386546695574359L;
+	
 	private int fieldId = 0;
 
 	/**
-	 * Extracts the first field (id 0) from the array
+	 * Extracts the first field (id 0) from the array.
 	 */
 	public FieldFromArray() {
 		// noting to do => will use default 0
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java
index c6cf1e1eacef1..468b878ddb3ed 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java
@@ -29,18 +29,9 @@
 @Internal
 public class FieldFromTuple<OUT> implements Extractor<Tuple, OUT> {
 
-	/**
-	 * Auto-gernated version id
-	 */
 	private static final long serialVersionUID = -5161386546695574359L;
-	private int fieldId = 0;
 
-	/**
-	 * Extracts the first field (id 0) from the tuple
-	 */
-	public FieldFromTuple() {
-		// noting to do => will use default 0
-	}
+	private int fieldId = 0;
 
 	/**
 	 * Extracts the field with the given id from the tuple.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
index 9ac96e128e12f..afc2b4ddc59f1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
@@ -33,10 +33,8 @@
 @Internal
 public class FieldsFromArray<OUT> implements Extractor<Object, OUT[]> {
 
-	/**
-	 * Auto-generated version id
-	 */
 	private static final long serialVersionUID = 8075055384516397670L;
+	
 	private int[] order;
 	private Class<OUT> clazz;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java
index 21a264104c45c..2321878b8f9b1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java
@@ -21,16 +21,13 @@
 import org.apache.flink.api.java.tuple.Tuple;
 
 /**
- * Extracts one or more fields of the type Double from a tuple and puts them
- * into a new double[]
+ * Extracts one or more fields of the type Double from a tuple and puts them into a new double[].
  */
 @Internal
 public class FieldsFromTuple implements Extractor<Tuple, double[]> {
 
-	/**
-	 * auto generated version id
-	 */
 	private static final long serialVersionUID = -2554079091050273761L;
+
 	int[] indexes;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java
index c9c27d041b9f3..d4e4873a2c7a9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java
@@ -30,6 +30,9 @@
 import org.apache.sling.commons.json.JSONException;
 import org.apache.sling.commons.json.JSONObject;
 
+/**
+ * Helper class for generating a JSON representation from a {@link StreamGraph}.
+ */
 @Internal
 public class JSONGenerator {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
index 93a6387e23c4b..be591f756cc8b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
@@ -41,6 +41,10 @@
 import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.Preconditions;
 
+/**
+ * Internal configuration for a {@link StreamOperator}. This is created and populated by the
+ * {@link StreamingJobGraphGenerator}.
+ */
 @Internal
 public class StreamConfig implements Serializable {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
index 1acfc5b7c646b..157023ef8f598 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
@@ -277,7 +277,7 @@ protected StreamNode addNode(Integer vertexID,
 	 * Adds a new virtual node that is used to connect a downstream vertex to only the outputs
 	 * with the selected names.
 	 *
-	 * When adding an edge from the virtual node to a downstream node the connection will be made
+	 * <p>When adding an edge from the virtual node to a downstream node the connection will be made
 	 * to the original node, only with the selected names given here.
 	 *
 	 * @param originalId ID of the node that should be connected to.
@@ -329,10 +329,10 @@ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, Outp
 	}
 
 	/**
-	 * Adds a new virtual node that is used to connect a downstream vertex to an input with a certain
-	 * partitioning.
+	 * Adds a new virtual node that is used to connect a downstream vertex to an input with a
+	 * certain partitioning.
 	 *
-	 * When adding an edge from the virtual node to a downstream node the connection will be made
+	 * <p>When adding an edge from the virtual node to a downstream node the connection will be made
 	 * to the original node, but with the partitioning given here.
 	 *
 	 * @param originalId ID of the node that should be connected to.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
index 3b9a506cfb4db..be80371239935 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
@@ -49,15 +49,13 @@
  * A generator that generates a {@link StreamGraph} from a graph of
  * {@link StreamTransformation StreamTransformations}.
  *
- * <p>
- * This traverses the tree of {@code StreamTransformations} starting from the sinks. At each
+ * <p>This traverses the tree of {@code StreamTransformations} starting from the sinks. At each
  * transformation we recursively transform the inputs, then create a node in the {@code StreamGraph}
  * and add edges from the input Nodes to our newly created node. The transformation methods
  * return the IDs of the nodes in the StreamGraph that represent the input transformation. Several
  * IDs can be returned to be able to deal with feedback transformations and unions.
  *
- * <p>
- * Partitioning, split/select and union don't create actual nodes in the {@code StreamGraph}. For
+ * <p>Partitioning, split/select and union don't create actual nodes in the {@code StreamGraph}. For
  * these, we create a virtual node in the {@code StreamGraph} that holds the specific property, i.e.
  * partitioning, selector and so on. When an edge is created from a virtual node to a downstream
  * node the {@code StreamGraph} resolved the id of the original node and creates an edge
@@ -67,7 +65,7 @@
  *     Map-1 -&gt; HashPartition-2 -&gt; Map-3
  * </pre>
  *
- * where the numbers represent transformation IDs. We first recurse all the way down. {@code Map-1}
+ * <p>where the numbers represent transformation IDs. We first recurse all the way down. {@code Map-1}
  * is transformed, i.e. we create a {@code StreamNode} with ID 1. Then we transform the
  * {@code HashPartition}, for this, we create virtual node of ID 4 that holds the property
  * {@code HashPartition}. This transformation returns the ID 4. Then we transform the {@code Map-3}.
@@ -137,8 +135,7 @@ private StreamGraph generateInternal(List<StreamTransformation<?>> transformatio
 	/**
 	 * Transforms one {@code StreamTransformation}.
 	 *
-	 * <p>
-	 * This checks whether we already transformed it and exits early in that case. If not it
+	 * <p>This checks whether we already transformed it and exits early in that case. If not it
 	 * delegates to one of the transformation specific methods.
 	 */
 	private Collection<Integer> transform(StreamTransformation<?> transform) {
@@ -215,8 +212,7 @@ private Collection<Integer> transform(StreamTransformation<?> transform) {
 	/**
 	 * Transforms a {@code UnionTransformation}.
 	 *
-	 * <p>
-	 * This is easy, we only have to transform the inputs and return all the IDs in a list so
+	 * <p>This is easy, we only have to transform the inputs and return all the IDs in a list so
 	 * that downstream operations can connect to all upstream nodes.
 	 */
 	private <T> Collection<Integer> transformUnion(UnionTransformation<T> union) {
@@ -233,8 +229,7 @@ private <T> Collection<Integer> transformUnion(UnionTransformation<T> union) {
 	/**
 	 * Transforms a {@code PartitionTransformation}.
 	 *
-	 * <p>
-	 * For this we create a virtual node in the {@code StreamGraph} that holds the partition
+	 * <p>For this we create a virtual node in the {@code StreamGraph} that holds the partition
 	 * property. @see StreamGraphGenerator
 	 */
 	private <T> Collection<Integer> transformPartition(PartitionTransformation<T> partition) {
@@ -254,8 +249,7 @@ private <T> Collection<Integer> transformPartition(PartitionTransformation<T> pa
 	/**
 	 * Transforms a {@code SplitTransformation}.
 	 *
-	 * <p>
-	 * We add the output selector to previously transformed nodes.
+	 * <p>We add the output selector to previously transformed nodes.
 	 */
 	private <T> Collection<Integer> transformSplit(SplitTransformation<T> split) {
 
@@ -278,8 +272,8 @@ private <T> Collection<Integer> transformSplit(SplitTransformation<T> split) {
 	/**
 	 * Transforms a {@code SelectTransformation}.
 	 *
-	 * <p>
-	 * For this we create a virtual node in the {@code StreamGraph} holds the selected names.
+	 * <p>For this we create a virtual node in the {@code StreamGraph} holds the selected names.
+	 * 
 	 * @see org.apache.flink.streaming.api.graph.StreamGraphGenerator
 	 */
 	private <T> Collection<Integer> transformSelect(SelectTransformation<T> select) {
@@ -305,8 +299,7 @@ private <T> Collection<Integer> transformSelect(SelectTransformation<T> select)
 	/**
 	 * Transforms a {@code SideOutputTransformation}.
 	 *
-	 * <p>
-	 * For this we create a virtual node in the {@code StreamGraph} that holds the side-output
+	 * <p>For this we create a virtual node in the {@code StreamGraph} that holds the side-output
 	 * {@link org.apache.flink.util.OutputTag}.
 	 *
 	 * @see org.apache.flink.streaming.api.graph.StreamGraphGenerator
@@ -334,13 +327,12 @@ private <T> Collection<Integer> transformSideOutput(SideOutputTransformation<T>
 	/**
 	 * Transforms a {@code FeedbackTransformation}.
 	 *
-	 * <p>
-	 * This will recursively transform the input and the feedback edges. We return the concatenation
-	 * of the input IDs and the feedback IDs so that downstream operations can be wired to both.
+	 * <p>This will recursively transform the input and the feedback edges. We return the
+	 * concatenation of the input IDs and the feedback IDs so that downstream operations can be
+	 * wired to both.
 	 *
-	 * <p>
-	 * This is responsible for creating the IterationSource and IterationSink which
-	 * are used to feed back the elements.
+	 * <p>This is responsible for creating the IterationSource and IterationSink which are used to
+	 * feed back the elements.
 	 */
 	private <T> Collection<Integer> transformFeedback(FeedbackTransformation<T> iterate) {
 
@@ -411,13 +403,11 @@ private <T> Collection<Integer> transformFeedback(FeedbackTransformation<T> iter
 	/**
 	 * Transforms a {@code CoFeedbackTransformation}.
 	 *
-	 * <p>
-	 * This will only transform feedback edges, the result of this transform will be wired
+	 * <p>This will only transform feedback edges, the result of this transform will be wired
 	 * to the second input of a Co-Transform. The original input is wired directly to the first
 	 * input of the downstream Co-Transform.
 	 *
-	 * <p>
-	 * This is responsible for creating the IterationSource and IterationSink which
+	 * <p>This is responsible for creating the IterationSource and IterationSink which
 	 * are used to feed back the elements.
 	 */
 	private <F> Collection<Integer> transformCoFeedback(CoFeedbackTransformation<F> coIterate) {
@@ -531,8 +521,7 @@ private <T> Collection<Integer> transformSink(SinkTransformation<T> sink) {
 	/**
 	 * Transforms a {@code OneInputTransformation}.
 	 *
-	 * <p>
-	 * This recursively transforms the inputs, creates a new {@code StreamNode} in the graph and
+	 * <p>This recursively transforms the inputs, creates a new {@code StreamNode} in the graph and
 	 * wired the inputs to this new node.
 	 */
 	private <IN, OUT> Collection<Integer> transformOneInputTransform(OneInputTransformation<IN, OUT> transform) {
@@ -571,8 +560,7 @@ private <IN, OUT> Collection<Integer> transformOneInputTransform(OneInputTransfo
 	/**
 	 * Transforms a {@code TwoInputTransformation}.
 	 *
-	 * <p>
-	 * This recusively transforms the inputs, creates a new {@code StreamNode} in the graph and
+	 * <p>This recusively transforms the inputs, creates a new {@code StreamNode} in the graph and
 	 * wired the inputs to this new node.
 	 */
 	private <IN1, IN2, OUT> Collection<Integer> transformTwoInputTransform(TwoInputTransformation<IN1, IN2, OUT> transform) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
index d5084ed70866c..c778b20da1d04 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
@@ -59,17 +59,15 @@ public class StreamGraphHasherV2 implements StreamGraphHasher {
 	 * StreamGraph}. The hash is used as the {@link JobVertexID} in order to
 	 * identify nodes across job submissions if they didn't change.
 	 *
-	 * <p>
 	 * <p>The complete {@link StreamGraph} is traversed. The hash is either
 	 * computed from the transformation's user-specified id (see
 	 * {@link StreamTransformation#getUid()}) or generated in a deterministic way.
 	 *
-	 * <p>
 	 * <p>The generated hash is deterministic with respect to:
 	 * <ul>
-	 * <li>node-local properties (like parallelism, UDF, node ID),
-	 * <li>chained output nodes, and
-	 * <li>input nodes hashes
+	 *   <li>node-local properties (like parallelism, UDF, node ID),
+	 *   <li>chained output nodes, and
+	 *   <li>input nodes hashes
 	 * </ul>
 	 *
 	 * @return A map from {@link StreamNode#id} to hash as 16-byte array.
@@ -282,7 +280,6 @@ private byte[] generateDeterministicHash(
 	 * attributes are taken into account). The hasher encapsulates the current
 	 * state of the hash.
 	 *
-	 * <p>
 	 * <p>The specified ID is local to this node. We cannot use the
 	 * {@link StreamNode#id}, because it is incremented in a static counter.
 	 * Therefore, the IDs for identical jobs will otherwise be different.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 99c3f4ee6f380..e5e5dfc605c1c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -98,7 +98,7 @@ public abstract class AbstractStreamOperator<OUT>
 
 	private static final long serialVersionUID = 1L;
 	
-	/** The logger used by the operator class and its subclasses */
+	/** The logger used by the operator class and its subclasses. */
 	protected static final Logger LOG = LoggerFactory.getLogger(AbstractStreamOperator.class);
 
 	// ----------- configuration properties -------------
@@ -108,41 +108,53 @@ public abstract class AbstractStreamOperator<OUT>
 
 	// ---------------- runtime fields ------------------
 
-	/** The task that contains this operator (and other operators in the same chain) */
+	/** The task that contains this operator (and other operators in the same chain). */
 	private transient StreamTask<?, ?> container;
 	
 	protected transient StreamConfig config;
 
 	protected transient Output<StreamRecord<OUT>> output;
 
-	/** The runtime context for UDFs */
+	/** The runtime context for UDFs. */
 	private transient StreamingRuntimeContext runtimeContext;
 
 	// ----------------- general state -------------------
 
-	/** The factory that give this operator access to checkpoint storage */
+	/** The factory that give this operator access to checkpoint storage. */
 	private transient CheckpointStreamFactory checkpointStreamFactory;
 
 	// ---------------- key/value state ------------------
 
-	/** key selector used to get the key for the state. Non-null only is the operator uses key/value state */
+	/**
+	 * {@code KeySelector} for extracting a key from an element being processed. This is used to
+	 * scope keyed state to a key. This is null if the operator is not a keyed operator.
+	 *
+	 * <p>This is for elements from the first input.
+	 */
 	private transient KeySelector<?, ?> stateKeySelector1;
+
+	/**
+	 * {@code KeySelector} for extracting a key from an element being processed. This is used to
+	 * scope keyed state to a key. This is null if the operator is not a keyed operator.
+	 *
+	 * <p>This is for elements from the second input.
+	 */
 	private transient KeySelector<?, ?> stateKeySelector2;
 
 	/** Backend for keyed state. This might be empty if we're not on a keyed stream. */
 	private transient AbstractKeyedStateBackend<?> keyedStateBackend;
 
-	/** Keyed state store view on the keyed backend */
+	/** Keyed state store view on the keyed backend. */
 	private transient DefaultKeyedStateStore keyedStateStore;
 
 	// ---------------- operator state ------------------
 
-	/** Operator state backend / store */
+	/** Operator state backend / store. */
 	private transient OperatorStateBackend operatorStateBackend;
 
 	// --------------- Metrics ---------------------------
 
-	/** Metric group for the operator */
+	/** Metric group for the operator. */
 	protected transient MetricGroup metrics;
 
 	protected transient LatencyGauge latencyGauge;
@@ -326,7 +338,7 @@ private void initOperatorState(Collection<OperatorStateHandle> operatorStateHand
 	 * {@link OneInputStreamOperator#processElement(StreamRecord)}, or
 	 * {@link TwoInputStreamOperator#processElement1(StreamRecord)} and
 	 * {@link TwoInputStreamOperator#processElement2(StreamRecord)}.
-
+	 *
 	 * <p>The method is expected to flush all remaining buffered data. Exceptions during this flushing
 	 * of buffered should be propagated, in order to cause the operation to be recognized asa failed,
 	 * because the last data items are not processed properly.
@@ -340,7 +352,7 @@ public void close() throws Exception {}
 	 * This method is called at the very end of the operator's life, both in the case of a successful
 	 * completion of the operation, and in the case of a failure and canceling.
 	 *
-	 * This method is expected to make a thorough effort to release all resources
+	 * <p>This method is expected to make a thorough effort to release all resources
 	 * that the operator has acquired.
 	 */
 	@Override
@@ -614,10 +626,6 @@ protected <N, S extends State, T> S getOrCreateKeyedState(
 
 	/**
 	 * Creates a partitioned state handle, using the state backend configured for this task.
-	 * 
-	 * TODO: NOTE: This method does a lot of work caching / retrieving states just to update the namespace.
-	 *       This method should be removed for the sake of namespaces being lazily fetched from the keyed
-	 *       state backend, or being set on the state directly.
 	 *
 	 * @throws IllegalStateException Thrown, if the key/value state was already initialized.
 	 * @throws Exception Thrown, if the state backend cannot create the key/value state.
@@ -627,6 +635,13 @@ protected <S extends State, N> S getPartitionedState(
 			TypeSerializer<N> namespaceSerializer,
 			StateDescriptor<S, ?> stateDescriptor) throws Exception {
 
+
+		/*
+	    TODO: NOTE: This method does a lot of work caching / retrieving states just to update the namespace.
+	    This method should be removed for the sake of namespaces being lazily fetched from the keyed
+	    state backend, or being set on the state directly.
+	    */
+		
 		if (keyedStateStore != null) {
 			return keyedStateBackend.getPartitionedState(namespace, namespaceSerializer, stateDescriptor);
 		} else {
@@ -780,23 +795,25 @@ public Map<String, HashMap<String, Double>> getValue() {
 	}
 
 	/**
-	 * Identifier for a latency source
+	 * Identifier for a latency source.
 	 */
 	private static class LatencySourceDescriptor {
 		/**
-		 * A unique ID identifying a logical source in Flink
+		 * A unique ID identifying a logical source in Flink.
 		 */
 		private final int vertexID;
 
 		/**
-		 * Identifier for parallel subtasks of a logical source
+		 * Identifier for parallel subtasks of a logical source.
 		 */
 		private final int subtaskIndex;
 
 		/**
+		 * Creates a {@code LatencySourceDescriptor} from a given {@code LatencyMarker}.
 		 *
 		 * @param marker The latency marker to extract the LatencySourceDescriptor from.
-		 * @param ignoreSubtaskIndex Set to true to ignore the subtask index, to treat the latencies from all the parallel instances of a source as the same.
+		 * @param ignoreSubtaskIndex Set to true to ignore the subtask index, to treat the latencies
+		 *      from all the parallel instances of a source as the same.
 		 * @return A LatencySourceDescriptor for the given marker.
 		 */
 		public static LatencySourceDescriptor of(LatencyMarker marker, boolean ignoreSubtaskIndex) {
@@ -846,6 +863,9 @@ public String toString() {
 		}
 	}
 
+	/**
+	 * Wrapping {@link Output} that updates metrics on the number of emitted elements.
+	 */
 	public class CountingOutput implements Output<StreamRecord<OUT>> {
 		private final Output<StreamRecord<OUT>> output;
 		private final Counter numRecordsOut;
@@ -891,8 +911,8 @@ public void close() {
 	 * Returns an {@link InternalWatermarkCallbackService} which  allows to register a
 	 * {@link OnWatermarkCallback} and multiple keys, for which
 	 * the callback will be invoked every time a new {@link Watermark} is received.
-	 * <p>
-	 * <b>NOTE: </b> This service is only available to <b>keyed</b> operators.
+	 *
+	 * <p><b>NOTE: </b> This service is only available to <b>keyed</b> operators.
 	 */
 	public <K> InternalWatermarkCallbackService<K> getInternalWatermarkCallbackService() {
 		checkTimerServiceInitialization();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
index 19559e176c2db..550add18d5b18 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
@@ -63,10 +63,10 @@ public abstract class AbstractUdfStreamOperator<OUT, F extends Function>
 	private static final long serialVersionUID = 1L;
 	
 	
-	/** the user function */
+	/** The user function. */
 	protected final F userFunction;
 	
-	/** Flag to prevent duplicate function.close() calls in close() and dispose() */
+	/** Flag to prevent duplicate function.close() calls in close() and dispose(). */
 	private transient boolean functionsClosed = false;
 	
 	public AbstractUdfStreamOperator(F userFunction) {
@@ -205,7 +205,6 @@ public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig exec
 	// ------------------------------------------------------------------------
 
 	/**
-	 * 
 	 * Since the streaming API does not implement any parametrization of functions via a
 	 * configuration, the config returned here is actually empty.
 	 * 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
index 742e11994637e..a6d66e82db668 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
@@ -59,7 +59,7 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
 	private final PriorityQueue<InternalTimer<K, N>> eventTimeTimersQueue;
 
 	/**
-	 * Information concerning the local key-group range
+	 * Information concerning the local key-group range.
 	 */
 	private final KeyGroupsList localKeyGroupRange;
 	private final int totalKeyGroups;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
index 97bc8ce7651b0..a46897c974109 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
@@ -30,6 +30,10 @@
 import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkState;
 
+/**
+ * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing keyed
+ * {@link ProcessFunction ProcessFunctions}.
+ */
 @Internal
 public class KeyedProcessOperator<K, IN, OUT>
 		extends AbstractUdfStreamOperator<OUT, ProcessFunction<IN, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java
index f73b610288b6e..f08c1ee897888 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java
@@ -27,6 +27,10 @@
 
 import static org.apache.flink.util.Preconditions.checkState;
 
+/**
+ * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing
+ * {@link ProcessFunction ProcessFunctions}.
+ */
 @Internal
 public class ProcessOperator<IN, OUT>
 		extends AbstractUdfStreamOperator<OUT, ProcessFunction<IN, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFilter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFilter.java
index 2df95caf76db3..22f9d9d83ab65 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFilter.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFilter.java
@@ -21,6 +21,9 @@
 import org.apache.flink.api.common.functions.FilterFunction;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A {@link StreamOperator} for executing {@link FilterFunction FilterFunctions}.
+ */
 @Internal
 public class StreamFilter<IN> extends AbstractUdfStreamOperator<IN, FilterFunction<IN>> implements OneInputStreamOperator<IN, IN> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFlatMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFlatMap.java
index c3ad26084fc36..563325b206c62 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFlatMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamFlatMap.java
@@ -21,6 +21,9 @@
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A {@link StreamOperator} for executing {@link FlatMapFunction FlatMapFunctions}.
+ */
 @Internal
 public class StreamFlatMap<IN, OUT>
 		extends AbstractUdfStreamOperator<OUT, FlatMapFunction<IN, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
index 76a18d849fd86..f984a330d8ceb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
@@ -32,6 +32,10 @@
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A {@link StreamOperator} for executing a {@link FoldFunction} on a
+ * {@link org.apache.flink.streaming.api.datastream.KeyedStream}.
+ */
 @Internal
 public class StreamGroupedFold<IN, OUT, KEY>
 		extends AbstractUdfStreamOperator<OUT, FoldFunction<IN, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java
index 156f3361a6b3f..2fd3f543eedfc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java
@@ -24,6 +24,11 @@
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A {@link StreamOperator} for executing a {@link ReduceFunction} on a
+ * {@link org.apache.flink.streaming.api.datastream.KeyedStream}.
+ */
+
 @Internal
 public class StreamGroupedReduce<IN> extends AbstractUdfStreamOperator<IN, ReduceFunction<IN>>
 		implements OneInputStreamOperator<IN, IN> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamMap.java
index 6755bc01077cc..5286d2c3f551d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamMap.java
@@ -21,6 +21,9 @@
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A {@link StreamOperator} for executing {@link MapFunction MapFunctions}.
+ */
 @Internal
 public class StreamMap<IN, OUT>
 		extends AbstractUdfStreamOperator<OUT, MapFunction<IN, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index 006e910509fbd..b0b770322918e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -34,10 +34,10 @@
  * {@link org.apache.flink.streaming.api.operators.TwoInputStreamOperator} to create operators
  * that process elements.
  * 
- * <p> The class {@link org.apache.flink.streaming.api.operators.AbstractStreamOperator}
+ * <p>The class {@link org.apache.flink.streaming.api.operators.AbstractStreamOperator}
  * offers default implementation for the lifecycle and properties methods.
  *
- * <p> Methods of {@code StreamOperator} are guaranteed not to be called concurrently. Also, if using
+ * <p>Methods of {@code StreamOperator} are guaranteed not to be called concurrently. Also, if using
  * the timer service, timer callbacks are also guaranteed not to be called concurrently with
  * methods on {@code StreamOperator}.
  * 
@@ -68,11 +68,10 @@ public interface StreamOperator<OUT> extends Serializable {
 	 * {@link org.apache.flink.streaming.api.operators.OneInputStreamOperator#processElement(StreamRecord)}, or
 	 * {@link org.apache.flink.streaming.api.operators.TwoInputStreamOperator#processElement1(StreamRecord)} and
 	 * {@link org.apache.flink.streaming.api.operators.TwoInputStreamOperator#processElement2(StreamRecord)}.
-
-	 * <p>
-	 * The method is expected to flush all remaining buffered data. Exceptions during this flushing
-	 * of buffered should be propagated, in order to cause the operation to be recognized as failed,
-	 * because the last data items are not processed properly.
+	 *
+	 * <p>The method is expected to flush all remaining buffered data. Exceptions during this
+	 * flushing of buffered should be propagated, in order to cause the operation to be recognized
+	 * as failed, because the last data items are not processed properly.
 	 * 
 	 * @throws java.lang.Exception An exception in this method causes the operator to fail.
 	 */
@@ -82,7 +81,7 @@ public interface StreamOperator<OUT> extends Serializable {
 	 * This method is called at the very end of the operator's life, both in the case of a successful
 	 * completion of the operation, and in the case of a failure and canceling.
 	 * 
-	 * This method is expected to make a thorough effort to release all resources
+	 * <p>This method is expected to make a thorough effort to release all resources
 	 * that the operator has acquired.
 	 */
 	void dispose() throws Exception;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamProject.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamProject.java
index ef51d8e8ee05e..acc84d0d1515c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamProject.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamProject.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -22,6 +22,9 @@
 import org.apache.flink.api.java.tuple.Tuple;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A {@link StreamOperator} for executing projections on streams.
+ */
 @Internal
 public class StreamProject<IN, OUT extends Tuple>
 		extends AbstractStreamOperator<OUT>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSink.java
index e238566fb3354..d92d7891bc3e0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSink.java
@@ -22,6 +22,9 @@
 import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A {@link StreamOperator} for executing {@link SinkFunction SinkFunctions}.
+ */
 @Internal
 public class StreamSink<IN> extends AbstractUdfStreamOperator<Object, SinkFunction<IN>>
 		implements OneInputStreamOperator<IN, Object> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
index 36d10d3c98004..25537fa9d0042 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
@@ -200,7 +200,7 @@ protected boolean allowWatermark(Watermark mark) {
 			return mark.getTimestamp() == Long.MAX_VALUE && nextWatermarkTime != Long.MAX_VALUE;
 		}
 
-		/** This will only be called if allowWatermark returned {@code true} */
+		/** This will only be called if allowWatermark returned {@code true}. */
 		@Override
 		protected void processAndEmitWatermark(Watermark mark) {
 			nextWatermarkTime = Long.MAX_VALUE;
@@ -279,7 +279,7 @@ public void onProcessingTime(long timestamp) {
 	 * watermarks, but if records are emitted without timestamps, no timestamps are automatically
 	 * generated and attached. The records will simply have no timestamp in that case.
 	 *
-	 * Streaming topologies can use timestamp assigner functions to override the timestamps
+	 * <p>Streaming topologies can use timestamp assigner functions to override the timestamps
 	 * assigned here.
 	 */
 	private static class ManualWatermarkContext<T> extends WatermarkContext<T> {
@@ -325,12 +325,12 @@ protected boolean allowWatermark(Watermark mark) {
 	 * An abstract {@link SourceFunction.SourceContext} that should be used as the base for
 	 * stream source contexts that are relevant with {@link Watermark}s.
 	 *
-	 * Stream source contexts that are relevant with watermarks are responsible of manipulating
+	 * <p>Stream source contexts that are relevant with watermarks are responsible of manipulating
 	 * the current {@link StreamStatus}, so that stream status can be correctly propagated
 	 * downstream. Please refer to the class-level documentation of {@link StreamStatus} for
 	 * information on how stream status affects watermark advancement at downstream tasks.
 	 *
-	 * This class implements the logic of idleness detection. It fires idleness detection
+	 * <p>This class implements the logic of idleness detection. It fires idleness detection
 	 * tasks at a given interval; if no records or watermarks were collected by the source context
 	 * between 2 consecutive checks, it determines the source to be IDLE and correspondingly
 	 * toggles the status. ACTIVE status resumes as soon as some record or watermark is collected
@@ -349,8 +349,8 @@ private abstract static class WatermarkContext<T> implements SourceFunction.Sour
 		 * This flag will be reset to {@code true} every time the next check is scheduled.
 		 * Whenever a record or watermark is collected, the flag will be set to {@code false}.
 		 *
-		 * When the scheduled check is fired, if the flag remains to be {@code true}, the check will fail,
-		 * and our current status will determined to be IDLE.
+		 * <p>When the scheduled check is fired, if the flag remains to be {@code true}, the check
+		 * will fail, and our current status will determined to be IDLE.
 		 */
 		private volatile boolean failOnNextCheck;
 
@@ -491,10 +491,13 @@ protected void cancelNextIdleDetectionTask() {
 		/** Process and collect record with timestamp. */
 		protected abstract void processAndCollectWithTimestamp(T element, long timestamp);
 
-		/** Whether or not a watermark should be allowed */
+		/** Whether or not a watermark should be allowed. */
 		protected abstract boolean allowWatermark(Watermark mark);
 
-		/** Process and emit watermark. Only called if {@link WatermarkContext#allowWatermark(Watermark)} returns {@code true} */
+		/**
+		 * Process and emit watermark. Only called if
+		 * {@link WatermarkContext#allowWatermark(Watermark)} returns {@code true}.
+		 */
 		protected abstract void processAndEmitWatermark(Watermark mark);
 
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
index b666a2b7bc3f1..a7f69ac788436 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
@@ -51,10 +51,10 @@
 @PublicEvolving
 public class StreamingRuntimeContext extends AbstractRuntimeUDFContext {
 
-	/** The operator to which this function belongs */
+	/** The operator to which this function belongs. */
 	private final AbstractStreamOperator<?> operator;
 
-	/** The task environment running the operator */
+	/** The task environment running the operator. */
 	private final Environment taskEnvironment;
 
 	private final StreamConfig streamConfig;
@@ -157,6 +157,7 @@ private KeyedStateStore checkPreconditionsAndGetKeyedStateStore(StateDescriptor<
 
 	/**
 	 * Returns true if checkpointing is enabled for the running job.
+	 *
 	 * @return true if checkpointing is enabled.
 	 */
 	public boolean isCheckpointingEnabled() {
@@ -164,7 +165,8 @@ public boolean isCheckpointingEnabled() {
 	}
 
 	/**
-	 * Returns the checkpointing mode
+	 * Returns the checkpointing mode.
+	 *
 	 * @return checkpointing mode
 	 */
 	public CheckpointingMode getCheckpointMode() {
@@ -172,7 +174,8 @@ public CheckpointingMode getCheckpointMode() {
 	}
 
 	/**
-	 * Returns the buffer timeout of the job
+	 * Returns the buffer timeout of the job.
+	 *
 	 * @return buffer timeout (in milliseconds)
 	 */
 	public long getBufferTimeout() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
index ff507e340b315..7925be8a68ff2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
@@ -83,13 +83,13 @@ public class AsyncWaitOperator<IN, OUT>
 
 	private static final String STATE_NAME = "_async_wait_operator_state_";
 
-	/** Capacity of the stream element queue */
+	/** Capacity of the stream element queue. */
 	private final int capacity;
 
-	/** Output mode for this operator */
+	/** Output mode for this operator. */
 	private final AsyncDataStream.OutputMode outputMode;
 
-	/** Timeout for the async collectors */
+	/** Timeout for the async collectors. */
 	private final long timeout;
 
 	protected transient Object checkpointingLock;
@@ -97,21 +97,21 @@ public class AsyncWaitOperator<IN, OUT>
 	/** {@link TypeSerializer} for inputs while making snapshots. */
 	private transient StreamElementSerializer<IN> inStreamElementSerializer;
 
-	/** Recovered input stream elements */
+	/** Recovered input stream elements. */
 	private transient ListState<StreamElement> recoveredStreamElements;
 
-	/** Queue to store the currently in-flight stream elements into */
+	/** Queue to store the currently in-flight stream elements into. */
 	private transient StreamElementQueue queue;
 
-	/** Pending stream element which could not yet added to the queue */
+	/** Pending stream element which could not yet added to the queue. */
 	private transient StreamElementQueueEntry<?> pendingStreamElementQueueEntry;
 
 	private transient ExecutorService executor;
 
-	/** Emitter for the completed stream element queue entries */
+	/** Emitter for the completed stream element queue entries. */
 	private transient Emitter<OUT> emitter;
 
-	/** Thread running the emitter */
+	/** Thread running the emitter. */
 	private transient Thread emitterThread;
 
 
@@ -368,7 +368,7 @@ private void stopResources(boolean waitForShutdown) throws InterruptedException
 				Thread.currentThread().interrupt();
 			}
 
-			/**
+			/*
 			 * FLINK-5638: If we have the checkpoint lock we might have to free it for a while so
 			 * that the emitter thread can complete/react to the interrupt signal.
 			 */
@@ -387,8 +387,8 @@ private void stopResources(boolean waitForShutdown) throws InterruptedException
 	/**
 	 * Add the given stream element queue entry to the operator's stream element queue. This
 	 * operation blocks until the element has been added.
-	 * <p>
-	 * For that it tries to put the element into the queue and if not successful then it waits on
+	 *
+	 * <p>For that it tries to put the element into the queue and if not successful then it waits on
 	 * the checkpointing lock. The checkpointing lock is also used by the {@link Emitter} to output
 	 * elements. The emitter is also responsible for notifying this method if the queue has capacity
 	 * left again, by calling notifyAll on the checkpointing lock.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java
index 39be544672834..3073bc9829256 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java
@@ -43,18 +43,18 @@ public class Emitter<OUT> implements Runnable {
 
 	private static final Logger LOG = LoggerFactory.getLogger(Emitter.class);
 
-	/** Lock to hold before outputting */
+	/** Lock to hold before outputting. */
 	private final Object checkpointLock;
 
-	/** Output for the watermark elements */
+	/** Output for the watermark elements. */
 	private final Output<StreamRecord<OUT>> output;
 
-	/** Queue to consume the async results from */
+	/** Queue to consume the async results from. */
 	private final StreamElementQueue streamElementQueue;
 
 	private final OperatorActions operatorActions;
 
-	/** Output for stream records */
+	/** Output for stream records. */
 	private final TimestampedCollector<OUT> timestampedCollector;
 
 	private volatile boolean running;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java
index 414b3c07e55a7..5aee45b1ca9be 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java
@@ -43,21 +43,21 @@ public class OrderedStreamElementQueue implements StreamElementQueue {
 
 	private static final Logger LOG = LoggerFactory.getLogger(OrderedStreamElementQueue.class);
 
-	/** Capacity of this queue */
+	/** Capacity of this queue. */
 	private final int capacity;
 
-	/** Executor to run the onCompletion callback */
+	/** Executor to run the onCompletion callback. */
 	private final Executor executor;
 
-	/** Operator actions to signal a failure to the operator */
+	/** Operator actions to signal a failure to the operator. */
 	private final OperatorActions operatorActions;
 
-	/** Lock and conditions for the blocking queue */
+	/** Lock and conditions for the blocking queue. */
 	private final ReentrantLock lock;
 	private final Condition notFull;
 	private final Condition headIsCompleted;
 
-	/** Queue for the inserted StreamElementQueueEntries */
+	/** Queue for the inserted StreamElementQueueEntries. */
 	private final ArrayDeque<StreamElementQueueEntry<?>> queue;
 
 	public OrderedStreamElementQueue(
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
index 7db9d4fd61f09..253b5d71bf73b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
@@ -36,7 +36,6 @@
 @Internal
 public abstract class StreamElementQueueEntry<T> implements AsyncResult {
 
-	/** Stream element */
 	private final StreamElement streamElement;
 
 	public StreamElementQueueEntry(StreamElement streamElement) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java
index c654702218d65..708bf1703d02d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java
@@ -39,11 +39,11 @@
 public class StreamRecordQueueEntry<OUT> extends StreamElementQueueEntry<Collection<OUT>>
 	implements AsyncCollectionResult<OUT>, AsyncCollector<OUT> {
 
-	/** Timestamp information */
+	/** Timestamp information. */
 	private final boolean hasTimestamp;
 	private final long timestamp;
 
-	/** Future containing the collection result */
+	/** Future containing the collection result. */
 	private final CompletableFuture<Collection<OUT>> resultFuture;
 
 	public StreamRecordQueueEntry(StreamRecord<?> streamRecord) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
index f2c78f8da92fd..1ca36dbf5132a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
@@ -47,22 +47,22 @@ public class UnorderedStreamElementQueue implements StreamElementQueue {
 
 	private static final Logger LOG = LoggerFactory.getLogger(UnorderedStreamElementQueue.class);
 
-	/** Capacity of this queue */
+	/** Capacity of this queue. */
 	private final int capacity;
 
-	/** Executor to run the onComplete callbacks */
+	/** Executor to run the onComplete callbacks. */
 	private final Executor executor;
 
-	/** OperatorActions to signal the owning operator a failure */
+	/** OperatorActions to signal the owning operator a failure. */
 	private final OperatorActions operatorActions;
 
-	/** Queue of uncompleted stream element queue entries segmented by watermarks */
+	/** Queue of uncompleted stream element queue entries segmented by watermarks. */
 	private final ArrayDeque<Set<StreamElementQueueEntry<?>>> uncompletedQueue;
 
-	/** Queue of completed stream element queue entries */
+	/** Queue of completed stream element queue entries. */
 	private final ArrayDeque<StreamElementQueueEntry<?>> completedQueue;
 
-	/** First (chronologically oldest) uncompleted set of stream element queue entries */
+	/** First (chronologically oldest) uncompleted set of stream element queue entries. */
 	private Set<StreamElementQueueEntry<?>> firstSet;
 
 	// Last (chronologically youngest) uncompleted set of stream element queue entries. New
@@ -70,7 +70,7 @@ public class UnorderedStreamElementQueue implements StreamElementQueue {
 	private Set<StreamElementQueueEntry<?>> lastSet;
 	private volatile int numberEntries;
 
-	/** Locks and conditions for the blocking queue */
+	/** Locks and conditions for the blocking queue. */
 	private final ReentrantLock lock;
 	private final Condition notFull;
 	private final Condition hasCompletedEntries;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java
index 4133e7b1e911e..63ddb3f6f6ac0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java
@@ -31,6 +31,10 @@
 import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkState;
 
+/**
+ * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing
+ * {@link CoProcessFunction CoProcessFunctions}.
+ */
 @Internal
 public class CoProcessOperator<IN1, IN2, OUT>
 		extends AbstractUdfStreamOperator<OUT, CoProcessFunction<IN1, IN2, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java
index ee58a0a899b0f..eab479737416c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java
@@ -24,6 +24,10 @@
 import org.apache.flink.streaming.api.operators.TimestampedCollector;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * {@link org.apache.flink.streaming.api.operators.StreamOperator} for processing
+ * {@link CoFlatMapFunction CoFlatMapFunctions}.
+ */
 @Internal
 public class CoStreamFlatMap<IN1, IN2, OUT>
 		extends AbstractUdfStreamOperator<OUT, CoFlatMapFunction<IN1, IN2, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamMap.java
index 00d4722ba755a..0b3067900373e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamMap.java
@@ -23,6 +23,10 @@
 import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * {@link org.apache.flink.streaming.api.operators.StreamOperator} for processing
+ * {@link CoMapFunction CoMapFunctions}.
+ */
 @Internal
 public class CoStreamMap<IN1, IN2, OUT>
 		extends AbstractUdfStreamOperator<OUT, CoMapFunction<IN1, IN2, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java
index e721ab885bc1c..d125a79c3698b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java
@@ -35,6 +35,10 @@
 import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkState;
 
+/**
+ * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing keyed
+ * {@link CoProcessFunction CoProcessFunctions}.
+ */
 @Internal
 public class KeyedCoProcessOperator<K, IN1, IN2, OUT>
 		extends AbstractUdfStreamOperator<OUT, CoProcessFunction<IN1, IN2, OUT>>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java
index bdcbe357d195c..4073e74963033 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -34,19 +34,16 @@
  * The upstream {@code StreamTransformation} will be connected to the first input of the Co-Transform
  * while the feedback edges will be connected to the second input.
  *
- * <p>
- * Both the partitioning of the input and the feedback edges is preserved. They can also have
+ * <p>Both the partitioning of the input and the feedback edges is preserved. They can also have
  * differing partitioning strategies. This requires, however, that the parallelism of the feedback
  * {@code StreamTransformations} must match the parallelism of the input
  * {@code StreamTransformation}.
  *
- * <p>
- * The upstream {@code StreamTransformation} is not wired to this {@code CoFeedbackTransformation}.
+ * <p>The upstream {@code StreamTransformation} is not wired to this {@code CoFeedbackTransformation}.
  * It is instead directly wired to the {@code TwoInputTransformation} after this
  * {@code CoFeedbackTransformation}.
  *
- * <p>
- * This is different from Iterations in batch processing.
+ * <p>This is different from Iterations in batch processing.
  * @see org.apache.flink.streaming.api.transformations.FeedbackTransformation
  *
  * @param <F> The type of the feedback elements.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java
index 0e84a44c1bba3..330a2159ceb73 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -27,21 +27,18 @@
 /**
  * This represents a feedback point in a topology.
  *
- * <p>
- * This is different from how iterations work in batch processing. Once a feedback point is defined
- * you can connect one or several {@code StreamTransformations} as a feedback edges. Operations
- * downstream from the feedback point will receive elements from the input of this feedback point
- * and from the feedback edges.
+ * <p>This is different from how iterations work in batch processing. Once a feedback point is
+ * defined you can connect one or several {@code StreamTransformations} as a feedback edges.
+ * Operations downstream from the feedback point will receive elements from the input of this
+ * feedback point and from the feedback edges.
  *
- * <p>
- * Both the partitioning of the input and the feedback edges is preserved. They can also have
+ * <p>Both the partitioning of the input and the feedback edges is preserved. They can also have
  * differing partitioning strategies. This requires, however, that the parallelism of the feedback
  * {@code StreamTransformations} must match the parallelism of the input
  * {@code StreamTransformation}.
  *
- * <p>
- * The type of the input {@code StreamTransformation} and the feedback {@code StreamTransformation}
- * must match.
+ * <p>The type of the input {@code StreamTransformation} and the feedback
+ * {@code StreamTransformation} must match.
  *
  * @param <T> The type of the input elements and the feedback elements.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
index f026379b7e68e..b5d1343528ef9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
index d8ac2883a0903..f6a4e5aa37249 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -29,8 +29,7 @@
 /**
  * This transformation represents a change of partitioning of the input elements.
  *
- * <p>
- * This does not create a physical operation, it only affects how upstream operations are
+ * <p>This does not create a physical operation, it only affects how upstream operations are
  * connected to downstream operations.
  *
  * @param <T> The type of the elements that result from this {@code PartitionTransformation}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
index 5a01a87b2b359..5b388ac349810 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -29,8 +29,7 @@
  * follow a {@link org.apache.flink.streaming.api.transformations.SplitTransformation} that
  * splits elements into several logical streams with assigned names.
  *
- * <p>
- * This does not create a physical operation, it only affects how upstream operations are
+ * <p>This does not create a physical operation, it only affects how upstream operations are
  * connected to downstream operations.
  *
  * @param <T> The type of the elements that result from this {@code SelectTransformation}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
index 2cb090f9f0735..a2948142cb124 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java
index 772744e0971b3..f51f1313b0ebf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java
index 607d9e810ae42..317ada53b62a4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -30,8 +30,7 @@
  * {@link org.apache.flink.streaming.api.datastream.DataStream} into several {@code DataStreams}
  * using an {@link org.apache.flink.streaming.api.collector.selector.OutputSelector}.
  *
- * <p>
- * This does not create a physical operation, it only affects how upstream operations are
+ * <p>This does not create a physical operation, it only affects how upstream operations are
  * connected to downstream operations.
  *
  * @param <T> The type of the elements that result from this {@code SplitTransformation}
@@ -64,7 +63,7 @@ public StreamTransformation<T> getInput() {
 	}
 
 	/**
-	 * Returns the {@code OutputSelector}
+	 * Returns the {@code OutputSelector}.
 	 */
 	public OutputSelector<T> getOutputSelector() {
 		return outputSelector;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
index e86b3e8f04ff3..95f6d11133d24 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -38,20 +38,16 @@
  * {@link org.apache.flink.streaming.api.datastream.DataStream} has an underlying
  * {@code StreamTransformation} that is the origin of said DataStream.
  *
- * <p>
- * API operations such as {@link org.apache.flink.streaming.api.datastream.DataStream#map} create
- * a tree of {@code StreamTransformation}s underneath. When the stream program is to be executed this
- * graph is translated to a {@link StreamGraph} using
+ * <p>API operations such as {@link org.apache.flink.streaming.api.datastream.DataStream#map} create
+ * a tree of {@code StreamTransformation}s underneath. When the stream program is to be executed
+ * this graph is translated to a {@link StreamGraph} using
  * {@link org.apache.flink.streaming.api.graph.StreamGraphGenerator}.
  *
- * <p>
- * A {@code StreamTransformation} does not necessarily correspond to a physical operation
+ * <p>A {@code StreamTransformation} does not necessarily correspond to a physical operation
  * at runtime. Some operations are only logical concepts. Examples of this are union,
  * split/select data stream, partitioning.
  *
- * <p>
- * The following graph of {@code StreamTransformations}:
- *
+ * <p>The following graph of {@code StreamTransformations}:
  * <pre>{@code
  *   Source              Source        
  *      +                   +           
@@ -79,8 +75,7 @@
  *              Sink 
  * }</pre>
  *
- * Would result in this graph of operations at runtime:
- *
+ * <p>Would result in this graph of operations at runtime:
  * <pre>{@code
  *  Source              Source
  *    +                   +
@@ -93,7 +88,7 @@
  *             Sink
  * }</pre>
  *
- * The information about partitioning, union, split/select end up being encoded in the edges
+ * <p>The information about partitioning, union, split/select end up being encoded in the edges
  * that connect the sources to the map operation.
  *
  * @param <T> The type of the elements that result from this {@code StreamTransformation}
@@ -191,15 +186,16 @@ public String getName() {
 	}
 
 	/**
-	 * Returns the parallelism of this {@code StreamTransformation}
+	 * Returns the parallelism of this {@code StreamTransformation}.
 	 */
 	public int getParallelism() {
 		return parallelism;
 	}
 
 	/**
-	 * Sets the parallelism of this {@code StreamTransformation}
-	 * @param parallelism The new parallelism to set on this {@code StreamTransformation}
+	 * Sets the parallelism of this {@code StreamTransformation}.
+	 * 
+	 * @param parallelism The new parallelism to set on this {@code StreamTransformation}.
 	 */
 	public void setParallelism(int parallelism) {
 		Preconditions.checkArgument(parallelism > 0, "Parallelism must be bigger than zero.");
@@ -258,20 +254,22 @@ public ResourceSpec getPreferredResources() {
 	}
 
 	/**
-	 * Sets an user provided hash for this operator. This will be used AS IS the create the JobVertexID.
-	 * <p/>
-	 * <p>The user provided hash is an alternative to the generated hashes, that is considered when identifying an
-	 * operator through the default hash mechanics fails (e.g. because of changes between Flink versions).
-	 * <p/>
-	 * <p><strong>Important</strong>: this should be used as a workaround or for trouble shooting. The provided hash
-	 * needs to be unique per transformation and job. Otherwise, job submission will fail. Furthermore, you cannot
-	 * assign user-specified hash to intermediate nodes in an operator chain and trying so will let your job fail.
+	 * Sets an user provided hash for this operator. This will be used AS IS the create the
+	 * JobVertexID.
+	 *
+	 * <p>The user provided hash is an alternative to the generated hashes, that is considered when
+	 * identifying an operator through the default hash mechanics fails (e.g. because of changes
+	 * between Flink versions).
+	 *
+	 * <p><strong>Important</strong>: this should be used as a workaround or for trouble shooting.
+	 * The provided hash needs to be unique per transformation and job. Otherwise, job submission
+	 * will fail. Furthermore, you cannot assign user-specified hash to intermediate nodes in an
+	 * operator chain and trying so will let your job fail.
 	 *
-	 * <p>
-	 * A use case for this is in migration between Flink versions or changing the jobs in a way that changes the
-	 * automatically generated hashes. In this case, providing the previous hashes directly through this method (e.g.
-	 * obtained from old logs) can help to reestablish a lost mapping from states to their target operator.
-	 * <p/>
+	 * <p>A use case for this is in migration between Flink versions or changing the jobs in a way
+	 * that changes the automatically generated hashes. In this case, providing the previous hashes
+	 * directly through this method (e.g. obtained from old logs) can help to reestablish a lost
+	 * mapping from states to their target operator.
 	 *
 	 * @param uidHash The user provided hash for this operator. This will become the JobVertexID, which is shown in the
 	 *                 logs and web ui.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java
index e3699c81b609f..ef61284b9db4d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java
index 238531b1aa4ec..9e36784890f21 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -28,8 +28,7 @@
  * This transformation represents a union of several input
  * {@link StreamTransformation StreamTransformations}.
  *
- * <p>
- * This does not create a physical operation, it only affects how upstream operations are
+ * <p>This does not create a physical operation, it only affects how upstream operations are
  * connected to downstream operations.
  *
  * @param <T> The type of the elements that result from this {@code UnionTransformation}
@@ -41,8 +40,7 @@ public class UnionTransformation<T> extends StreamTransformation<T> {
 	/**
 	 * Creates a new {@code UnionTransformation} from the given input {@code StreamTransformations}.
 	 *
-	 * <p>
-	 * The input {@code StreamTransformations} must all have the same type.
+	 * <p>The input {@code StreamTransformations} must all have the same type.
 	 *
 	 * @param inputs The list of input {@code StreamTransformations}
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java
index dc12d93fdf6f6..4960512eff1fb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java
@@ -31,24 +31,22 @@
  * that buffer elements, such as window operators, must forward a watermark after emission of
  * elements that is triggered by the arriving watermark.
  *
- * <p>
- * In some cases a watermark is only a heuristic and operators should be able to deal with
+ * <p>In some cases a watermark is only a heuristic and operators should be able to deal with
  * late elements. They can either discard those or update the result and emit updates/retractions
  * to downstream operations.
  *
- * <p>
- * When a source closes it will emit a final watermark with timestamp {@code Long.MAX_VALUE}. When
- * an operator receives this it will know that no more input will be arriving in the future.
+ * <p>When a source closes it will emit a final watermark with timestamp {@code Long.MAX_VALUE}.
+ * When an operator receives this it will know that no more input will be arriving in the future.
  */
 @PublicEvolving
 public final class Watermark extends StreamElement {
 
-	/** The watermark that signifies end-of-event-time */
+	/** The watermark that signifies end-of-event-time. */
 	public static final Watermark MAX_WATERMARK = new Watermark(Long.MAX_VALUE);
 	
 	// ------------------------------------------------------------------------
 	
-	/** The timestamp of the watermark in milliseconds*/
+	/** The timestamp of the watermark in milliseconds. */
 	private final long timestamp;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java
index 6ea4b8e256873..2091bf251a127 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java
@@ -32,9 +32,8 @@
  * {@link org.apache.flink.streaming.runtime.operators.windowing.AggregatingProcessingTimeWindowOperator
  * AggregatingProcessingTimeWindowOperator}.
  *
- * <p>
- * For assigner that extend this one, the user can check the {@link TumblingAlignedProcessingTimeWindows}
- * and the {@link SlidingAlignedProcessingTimeWindows}.
+ * <p>For assigner that extend this one, the user can check the
+ * {@link TumblingAlignedProcessingTimeWindows} and the {@link SlidingAlignedProcessingTimeWindows}.
  * */
 public class BaseAlignedWindowAssigner extends WindowAssigner<Object, TimeWindow> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
index 1703f6c5c9e61..2602bc2aa5249 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -32,8 +32,7 @@
  * A {@link WindowAssigner} that windows elements into sessions based on the timestamp of the
  * elements. Windows cannot overlap.
  *
- * <p>
- * For example, in order to window into windows of 1 minute, every 10 seconds:
+ * <p>For example, in order to window into windows of 1 minute, every 10 seconds:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> in = ...;
  * KeyedStream<String, Tuple2<String, Integer>> keyed = in.keyBy(...);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
index 9e3846d7db2b7..47e47ef8e6d10 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -30,10 +30,9 @@
 import java.util.Collections;
 
 /**
- * A {@link WindowAssigner} that assigns all elements to the same global window.
+ * A {@link WindowAssigner} that assigns all elements to the same {@link GlobalWindow}.
  *
- * <p>
- * Use this if you want to use a {@link Trigger} and
+ * <p>Use this if you want to use a {@link Trigger} and
  * {@link org.apache.flink.streaming.api.windowing.evictors.Evictor} to do flexible, policy based
  * windows.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java
index 1450b1a380d17..ca77861a56585 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
index 02c680eb35039..8f727dcce8745 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -32,8 +32,7 @@
  * A {@link WindowAssigner} that windows elements into sessions based on the current processing
  * time. Windows cannot overlap.
  *
- * <p>
- * For example, in order to window into windows of 1 minute, every 10 seconds:
+ * <p>For example, in order to window into windows of 1 minute, every 10 seconds:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> in = ...;
  * KeyedStream<String, Tuple2<String, Integer>> keyed = in.keyBy(...);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
index ef6ed5649041b..f575c7db9454a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
@@ -35,8 +35,7 @@
  * A {@link WindowAssigner} that windows elements into sliding windows based on the timestamp of the
  * elements. Windows can possibly overlap.
  *
- * <p>
- * For example, in order to window into windows of 1 minute, every 10 seconds:
+ * <p>For example, in order to window into windows of 1 minute, every 10 seconds:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> in = ...;
  * KeyedStream<Tuple2<String, Integer>, String> keyed = in.keyBy(...);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
index c11045d187b5d..57f0c8c7dc3ac 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -33,8 +33,7 @@
  * A {@link WindowAssigner} that windows elements into sliding windows based on the current
  * system time of the machine the operation is running on. Windows can possibly overlap.
  *
- * <p>
- * For example, in order to window into windows of 1 minute, every 10 seconds:
+ * <p>For example, in order to window into windows of 1 minute, every 10 seconds:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> in = ...;
  * KeyedStream<String, Tuple2<String, Integer>> keyed = in.keyBy(...);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
index d695a0cadc8d8..d37f2cde5a53d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
@@ -34,8 +34,7 @@
  * A {@link WindowAssigner} that windows elements into windows based on the timestamp of the
  * elements. Windows cannot overlap.
  *
- * <p>
- * For example, in order to window into windows of 1 minute:
+ * <p>For example, in order to window into windows of 1 minute:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> in = ...;
  * KeyedStream<Tuple2<String, Integer>, String> keyed = in.keyBy(...);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
index 5b39fe0a00ee7..4f51998711d63 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -32,8 +32,7 @@
  * A {@link WindowAssigner} that windows elements into windows based on the current
  * system time of the machine the operation is running on. Windows cannot overlap.
  *
- * <p>
- * For example, in order to window into windows of 1 minute, every 10 seconds:
+ * <p>For example, in order to window into windows of 1 minute, every 10 seconds:
  * <pre> {@code
  * DataStream<Tuple2<String, Integer>> in = ...;
  * KeyedStream<String, Tuple2<String, Integer>> keyed = in.keyBy(...);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
index 7a27cc851bb2c..2ff25e4c671e6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
@@ -32,8 +32,7 @@
 /**
  * A {@code WindowAssigner} assigns zero or more {@link Window Windows} to an element.
  *
- * <p>
- * In a window operation, elements are grouped by their key (if available) and by the windows to
+ * <p>In a window operation, elements are grouped by their key (if available) and by the windows to
  * which it was assigned. The set of elements with the same key and window is called a pane.
  * When a {@link Trigger} decides that a certain pane should fire the
  * {@link org.apache.flink.streaming.api.functions.windowing.WindowFunction} is applied
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
index 8f5b2d99e4969..92a38775bbf60 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java
index 7ae33b8bbcf47..67c67b819b762 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -28,8 +28,7 @@
 /**
  * An {@link Evictor} that keeps elements based on a {@link DeltaFunction} and a threshold.
  *
- * <p>
- * Eviction starts from the first element of the buffer and removes all elements from the buffer
+ * <p>Eviction starts from the first element of the buffer and removes all elements from the buffer
  * which have a higher delta then the threshold.
  *
  * @param <W> The type of {@link Window Windows} on which this {@code Evictor} can operate.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java
index 470c736cc6827..7ba32aab83e54 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -26,11 +26,11 @@
 import java.io.Serializable;
 
 /**
- * An {@code Evictor} can remove elements from a pane before/after the evaluation of WindowFunction and
- * after the window evaluation gets triggered by a {@link org.apache.flink.streaming.api.windowing.triggers.Trigger}
+ * An {@code Evictor} can remove elements from a pane before/after the evaluation of WindowFunction
+ * and after the window evaluation gets triggered by a
+ * {@link org.apache.flink.streaming.api.windowing.triggers.Trigger}
  *
- * <p>
- * A pane is the bucket of elements that have the same key (assigned by the
+ * <p>A pane is the bucket of elements that have the same key (assigned by the
  * {@link org.apache.flink.api.java.functions.KeySelector}) and same {@link Window}. An element can
  * be in multiple panes of it was assigned to multiple windows by the
  * {@link org.apache.flink.streaming.api.windowing.assigners.WindowAssigner}. These panes all
@@ -64,7 +64,7 @@ public interface Evictor<T, W extends Window> extends Serializable {
 
 
 	/**
-	 * A context object that is given to {@link Evictor} methods
+	 * A context object that is given to {@link Evictor} methods.
 	 */
 	interface EvictorContext {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
index 33d1cb5be6d43..afd3ab796c0d3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java
index b97efa27cd055..c22f3a483ebf4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java
@@ -29,13 +29,13 @@
 @Public
 public final class Time {
 
-	/** The time unit for this policy's time interval */
+	/** The time unit for this policy's time interval. */
 	private final TimeUnit unit;
 
-	/** The size of the windows generated by this policy */
+	/** The size of the windows generated by this policy. */
 	private final long size;
 
-	/** Instantiation only via factory method */
+	/** Instantiation only via factory method. */
 	private Time(long size, TimeUnit unit) {
 		this.unit = checkNotNull(unit, "time unit may not be null");
 		this.size = size;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java
index 89cca2274a96c..3b7bef1af2a0f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/DeltaTrigger.java
@@ -28,8 +28,7 @@
 /**
  * A {@link Trigger} that fires based on a {@link DeltaFunction} and a threshold.
  *
- * <p>
- * This trigger calculates a delta between the data point which triggered last
+ * <p>This trigger calculates a delta between the data point which triggered last
  * and the currently arrived data point. It triggers if the delta is higher than
  * a specified threshold.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
index ae25e87685973..0589751206cb8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
@@ -80,8 +80,7 @@ public String toString() {
 	/**
 	 * Creates an event-time trigger that fires once the watermark passes the end of the window.
 	 *
-	 * <p>
-	 * Once the trigger fires all elements are discarded. Elements that arrive late immediately
+	 * <p>Once the trigger fires all elements are discarded. Elements that arrive late immediately
 	 * trigger window evaluation with just this one element.
 	 */
 	public static EventTimeTrigger create() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/PurgingTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/PurgingTrigger.java
index ed1d2fc5566eb..fb4568ceae7a6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/PurgingTrigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/PurgingTrigger.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -24,9 +24,8 @@
 /**
  * A trigger that can turn any {@link Trigger} into a purging {@code Trigger}.
  *
- * <p>
- * When the nested trigger fires, this will return a {@code FIRE_AND_PURGE}
- * {@link TriggerResult}
+ * <p>When the nested trigger fires, this will return a {@code FIRE_AND_PURGE}
+ * {@link TriggerResult}.
  *
  * @param <T> The type of elements on which this trigger can operate.
  * @param <W> The type of {@link Window Windows} on which this trigger can operate.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
index bcce54d2ea252..6c3921f665c66 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -24,6 +24,10 @@
 
 import java.io.IOException;
 
+/**
+ * The default window into which all data is placed (via
+ * {@link org.apache.flink.streaming.api.windowing.assigners.GlobalWindows}).
+ */
 @PublicEvolving
 public class GlobalWindow extends Window {
 
@@ -55,6 +59,9 @@ public String toString() {
 		return "GlobalWindow";
 	}
 
+	/**
+	 * A {@link TypeSerializer} for {@link GlobalWindow}.
+	 */
 	public static class Serializer extends TypeSerializer<GlobalWindow> {
 		private static final long serialVersionUID = 1L;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
index 781c32024095b..cd797653f382c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/Window.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/Window.java
index b17989cc6e83c..b2f17fe03bf01 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/Window.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/Window.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -24,8 +24,7 @@
  * A {@code Window} is a grouping of elements into finite buckets. Windows have a maximum timestamp
  * which means that, at some point, all elements that go into one window will have arrived.
  *
- * <p>
- * Subclasses should implement {@code equals()} and {@code hashCode()} so that logically
+ * <p>Subclasses should implement {@code equals()} and {@code hashCode()} so that logically
  * same windows are treated the same.
  */
 @PublicEvolving
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
index 2da838906e44b..de13a5e2a22d2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
@@ -54,52 +54,61 @@ public class BarrierBuffer implements CheckpointBarrierHandler {
 
 	private static final Logger LOG = LoggerFactory.getLogger(BarrierBuffer.class);
 
-	/** The gate that the buffer draws its input from */
+	/** The gate that the buffer draws its input from. */
 	private final InputGate inputGate;
 
-	/** Flags that indicate whether a channel is currently blocked/buffered */
+	/** Flags that indicate whether a channel is currently blocked/buffered. */
 	private final boolean[] blockedChannels;
 
-	/** The total number of channels that this buffer handles data from */
+	/** The total number of channels that this buffer handles data from. */
 	private final int totalNumberOfInputChannels;
 
-	/** To utility to write blocked data to a file channel */
+	/** To utility to write blocked data to a file channel. */
 	private final BufferSpiller bufferSpiller;
 
-	/** The pending blocked buffer/event sequences. Must be consumed before requesting
-	 * further data from the input gate. */
+	/**
+	 * The pending blocked buffer/event sequences. Must be consumed before requesting further data
+	 * from the input gate.
+	 */
 	private final ArrayDeque<BufferSpiller.SpilledBufferOrEventSequence> queuedBuffered;
 
-	/** The maximum number of bytes that may be buffered before an alignment is broken. -1 means unlimited */
+	/**
+	 * The maximum number of bytes that may be buffered before an alignment is broken. -1 means
+	 * unlimited.
+	 */
 	private final long maxBufferedBytes;
 
-	/** The sequence of buffers/events that has been unblocked and must now be consumed
-	 * before requesting further data from the input gate */
+	/**
+	 * The sequence of buffers/events that has been unblocked and must now be consumed before
+	 * requesting further data from the input gate.
+	 */
 	private BufferSpiller.SpilledBufferOrEventSequence currentBuffered;
 
-	/** Handler that receives the checkpoint notifications */
+	/** Handler that receives the checkpoint notifications. */
 	private StatefulTask toNotifyOnCheckpoint;
 
-	/** The ID of the checkpoint for which we expect barriers */
+	/** The ID of the checkpoint for which we expect barriers. */
 	private long currentCheckpointId = -1L;
 
-	/** The number of received barriers (= number of blocked/buffered channels)
-	 * IMPORTANT: A canceled checkpoint must always have 0 barriers */
+	/**
+	 * The number of received barriers (= number of blocked/buffered channels) IMPORTANT: A canceled
+	 * checkpoint must always have 0 barriers.
+	 */
 	private int numBarriersReceived;
 
-	/** The number of already closed channels */
+	/** The number of already closed channels. */
 	private int numClosedChannels;
 
-	/** The number of bytes in the queued spilled sequences */
+	/** The number of bytes in the queued spilled sequences. */
 	private long numQueuedBytes;
 
-	/** The timestamp as in {@link System#nanoTime()} at which the last alignment started */
+	/** The timestamp as in {@link System#nanoTime()} at which the last alignment started. */
 	private long startOfAlignmentTimestamp;
 
-	/** The time (in nanoseconds) that the latest alignment took */
+	/** The time (in nanoseconds) that the latest alignment took. */
 	private long latestAlignmentDurationNanos;
 
-	/** Flag to indicate whether we have drawn all available input */
+	/** Flag to indicate whether we have drawn all available input. */
 	private boolean endOfStream;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
index 8b1b65bae2d5a..c0f510e08df18 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
@@ -50,27 +50,33 @@ public class BarrierTracker implements CheckpointBarrierHandler {
 
 	private static final Logger LOG = LoggerFactory.getLogger(BarrierTracker.class);
 
-	/** The tracker tracks a maximum number of checkpoints, for which some, but not all
-	 * barriers have yet arrived. */
+	/**
+	 * The tracker tracks a maximum number of checkpoints, for which some, but not all barriers
+	 * have yet arrived.
+	 */
 	private static final int MAX_CHECKPOINTS_TO_TRACK = 50;
 
 	// ------------------------------------------------------------------------
 
-	/** The input gate, to draw the buffers and events from */
+	/** The input gate, to draw the buffers and events from. */
 	private final InputGate inputGate;
 	
-	/** The number of channels. Once that many barriers have been received for a checkpoint,
-	 * the checkpoint is considered complete. */
+	/**
+	 * The number of channels. Once that many barriers have been received for a checkpoint, the
+	 * checkpoint is considered complete.
+	 */
 	private final int totalNumberOfInputChannels;
 
-	/** All checkpoints for which some (but not all) barriers have been received,
-	 * and that are not yet known to be subsumed by newer checkpoints */
+	/**
+	 * All checkpoints for which some (but not all) barriers have been received, and that are not
+	 * yet known to be subsumed by newer checkpoints.
+	 */
 	private final ArrayDeque<CheckpointBarrierCount> pendingCheckpoints;
 	
-	/** The listener to be notified on complete checkpoints */
+	/** The listener to be notified on complete checkpoints. */
 	private StatefulTask toNotifyOnCheckpoint;
 	
-	/** The highest checkpoint ID encountered so far */
+	/** The highest checkpoint ID encountered so far. */
 	private long latestPendingCheckpointID = -1;
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java
index ec5e1eb24247e..db10de131452e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java
@@ -22,12 +22,18 @@
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.runtime.iterative.concurrent.Broker;
 
+/**
+ * {@link Broker} for the {@link BlockingQueue} used in
+ * {@link org.apache.flink.streaming.runtime.tasks.StreamIterationHead} and
+ * {@link org.apache.flink.streaming.runtime.tasks.StreamIterationTail} for implementing feedback
+ * streams.
+ */
 @Internal
 public class BlockingQueueBroker extends Broker<BlockingQueue<?>> {
 	
-	/** Singleton instance */
+	/** Singleton instance. */
 	public static final BlockingQueueBroker INSTANCE = new BlockingQueueBroker();
 
-	/** Cannot instantiate */
+	/** Cannot instantiate. */
 	private BlockingQueueBroker() {}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
index 0e3ee22b001d1..2de508352dc5f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
@@ -55,40 +55,40 @@ public class BufferSpiller {
 	/** Size of header in bytes (see add method). */
 	static final int HEADER_SIZE = 9;
 
-	/** The counter that selects the next directory to spill into */
+	/** The counter that selects the next directory to spill into. */
 	private static final AtomicInteger DIRECTORY_INDEX = new AtomicInteger(0);
 	
-	/** The size of the buffer with which data is read back in */
+	/** The size of the buffer with which data is read back in. */
 	private static final int READ_BUFFER_SIZE = 1024 * 1024;
 	
-	/** The directories to spill to */
+	/** The directories to spill to. */
 	private final File tempDir;
 	
-	/** The name prefix for spill files */
+	/** The name prefix for spill files. */
 	private final String spillFilePrefix;
 	
-	/** The buffer used for bulk reading data (used in the SpilledBufferOrEventSequence) */
+	/** The buffer used for bulk reading data (used in the SpilledBufferOrEventSequence). */
 	private final ByteBuffer readBuffer;
 	
-	/** The buffer that encodes the spilled header */
+	/** The buffer that encodes the spilled header. */
 	private final ByteBuffer headBuffer;
 	
-	/** The reusable array that holds header and contents buffers */
+	/** The reusable array that holds header and contents buffers. */
 	private final ByteBuffer[] sources;
 	
-	/** The file that we currently spill to */
+	/** The file that we currently spill to. */
 	private File currentSpillFile;
 	
-	/** The channel of the file we currently spill to */
+	/** The channel of the file we currently spill to. */
 	private FileChannel currentChannel;
 
-	/** The page size, to let this reader instantiate properly sized memory segments */
+	/** The page size, to let this reader instantiate properly sized memory segments. */
 	private final int pageSize;
 	
-	/** A counter, to created numbered spill files */
+	/** A counter, to created numbered spill files. */
 	private int fileCounter;
 	
-	/** The number of bytes written since the last roll over */
+	/** The number of bytes written since the last roll over. */
 	private long bytesWritten;
 	
 	/**
@@ -217,7 +217,7 @@ private SpilledBufferOrEventSequence rollOverInternal(boolean newBuffer) throws
 	/**
 	 * Cleans up the current spilling channel and file.
 	 * 
-	 * Does not clean up the SpilledBufferOrEventSequences generated by calls to 
+	 * <p>Does not clean up the SpilledBufferOrEventSequences generated by calls to
 	 * {@link #rollOver()}.
 	 * 
 	 * @throws IOException Thrown if channel closing or file deletion fail.
@@ -268,25 +268,25 @@ private void createSpillingChannel() throws IOException {
 	 */
 	public static class SpilledBufferOrEventSequence {
 
-		/** Header is "channel index" (4 bytes) + length (4 bytes) + buffer/event (1 byte) */
+		/** Header is "channel index" (4 bytes) + length (4 bytes) + buffer/event (1 byte). */
 		private static final int HEADER_LENGTH = 9;
 
-		/** The file containing the data */
+		/** The file containing the data. */
 		private final File file;
 
-		/** The file channel to draw the data from */
+		/** The file channel to draw the data from. */
 		private final FileChannel fileChannel;
 
-		/** The byte buffer for bulk reading */
+		/** The byte buffer for bulk reading. */
 		private final ByteBuffer buffer;
 
-		/** We store this size as a constant because it is crucial it never changes */
+		/** We store this size as a constant because it is crucial it never changes. */
 		private final long size;
 
-		/** The page size to instantiate properly sized memory segments */
+		/** The page size to instantiate properly sized memory segments. */
 		private final int pageSize;
 
-		/** Flag to track whether the sequence has been opened already */
+		/** Flag to track whether the sequence has been opened already. */
 		private boolean opened = false;
 
 		/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
index 3feaa52cd4aab..e0a83c9420576 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
@@ -56,15 +56,14 @@
 /**
  * Input reader for {@link org.apache.flink.streaming.runtime.tasks.OneInputStreamTask}.
  *
- * <p>
- * This internally uses a {@link StatusWatermarkValve} to keep track of {@link Watermark} and {@link StreamStatus} events,
- * and forwards them to event subscribers once the {@link StatusWatermarkValve} determines the {@link Watermark} from
- * all inputs has advanced, or that a {@link StreamStatus} needs to be propagated downstream to denote a status change.
+ * <p>This internally uses a {@link StatusWatermarkValve} to keep track of {@link Watermark} and
+ * {@link StreamStatus} events, and forwards them to event subscribers once the
+ * {@link StatusWatermarkValve} determines the {@link Watermark} from all inputs has advanced, or
+ * that a {@link StreamStatus} needs to be propagated downstream to denote a status change.
  *
- * <p>
- * Forwarding elements, watermarks, or status status elements must be protected by synchronizing on the given lock
- * object. This ensures that we don't call methods on a {@link OneInputStreamOperator} concurrently
- * with the timer callback or other things.
+ * <p>Forwarding elements, watermarks, or status status elements must be protected by synchronizing
+ * on the given lock object. This ensures that we don't call methods on a
+ * {@link OneInputStreamOperator} concurrently with the timer callback or other things.
  * 
  * @param <IN> The type of the record that can be read with this record reader.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
index 6d5e89b3d5a2a..4fb30c21857c5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
@@ -36,17 +36,17 @@
 @Internal
 public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWriter<T> {
 
-	/** Default name for teh output flush thread, if no name with a task reference is given */
+	/** Default name for teh output flush thread, if no name with a task reference is given. */
 	private static final String DEFAULT_OUTPUT_FLUSH_THREAD_NAME = "OutputFlusher";
 	
 	
-	/** The thread that periodically flushes the output, to give an upper latency bound */
+	/** The thread that periodically flushes the output, to give an upper latency bound. */
 	private final OutputFlusher outputFlusher;
 	
-	/** Flag indicating whether the output should be flushed after every element */
+	/** Flag indicating whether the output should be flushed after every element. */
 	private final boolean flushAlways;
 
-	/** The exception encountered in the flushing thread */
+	/** The exception encountered in the flushing thread. */
 	private Throwable flusherException;
 	
 	
@@ -145,7 +145,7 @@ private void checkErroneous() throws IOException {
 	/**
 	 * A dedicated thread that periodically flushes the output buffers, to set upper latency bounds.
 	 * 
-	 * The thread is daemonic, because it is only a utility thread.
+	 * <p>The thread is daemonic, because it is only a utility thread.
 	 */
 	private class OutputFlusher extends Thread {
 		
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
index a8ec9729d33f6..a1456015da6bf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
@@ -55,15 +55,14 @@
 /**
  * Input reader for {@link org.apache.flink.streaming.runtime.tasks.TwoInputStreamTask}.
  *
- * <p>
- * This internally uses a {@link StatusWatermarkValve} to keep track of {@link Watermark} and {@link StreamStatus} events,
- * and forwards watermarks to event subscribers once the {@link StatusWatermarkValve} determines the watermarks from
- * all inputs has advanced, or changes the task's {@link StreamStatus} once status change is toggled.
+ * <p>This internally uses a {@link StatusWatermarkValve} to keep track of {@link Watermark} and
+ * {@link StreamStatus} events, and forwards watermarks to event subscribers once the
+ * {@link StatusWatermarkValve} determines the watermarks from all inputs has advanced, or changes
+ * the task's {@link StreamStatus} once status change is toggled.
  *
- * <p>
- * Forwarding elements, watermarks, or status status elements must be protected by synchronizing on the given lock
- * object. This ensures that we don't call methods on a {@link TwoInputStreamOperator} concurrently
- * with the timer callback or other things.
+ * <p>Forwarding elements, watermarks, or status status elements must be protected by synchronizing
+ * on the given lock object. This ensures that we don't call methods on a
+ * {@link TwoInputStreamOperator} concurrently with the timer callback or other things.
  *
  * @param <IN1> The type of the records that arrive on the first input
  * @param <IN2> The type of the records that arrive on the second input
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamingReader.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamingReader.java
deleted file mode 100644
index bbce5877110fa..0000000000000
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamingReader.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.runtime.io;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.runtime.io.network.api.reader.ReaderBase;
-
-import java.io.IOException;
-
-@Internal
-public interface StreamingReader extends ReaderBase {
-
-	void cleanup() throws IOException;
-}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
index c2aa847bee891..cc0bd646c717c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
@@ -39,6 +39,9 @@
 import static java.util.Objects.requireNonNull;
 
 /**
+ * Base class for special window operator implementation for windows that fire at the same time for
+ * all keys.
+ *
  * @deprecated Deprecated in favour of the generic {@link WindowOperator}. This was an
  * optimized implementation used for aligned windows.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
index 026258c7d5431..037b4d7e930ed 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
@@ -40,10 +40,10 @@ public abstract class AbstractKeyedTimePanes<Type, Key, Aggregate, Result> {
 
 	private static final int BEGIN_OF_PANE_MAGIC_NUMBER = 0xBADF00D5;
 	
-	/** The latest time pane */
+	/** The latest time pane. */
 	protected KeyMap<Key, Aggregate> latestPane = new KeyMap<>();
 
-	/** The previous time panes, ordered by time (early to late) */
+	/** The previous time panes, ordered by time (early to late). */
 	protected final ArrayDeque<KeyMap<Key, Aggregate>> previousPanes = new ArrayDeque<>();
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
index d58b5cc429a07..954dc10a7f748 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
@@ -40,6 +40,9 @@
 
 import java.util.ArrayList;
 
+/**
+ * Key/value map organized in panes for accumulating windows (with a window function).
+ */
 @Internal
 public class AccumulatingKeyedTimePanes<Type, Key, Result> extends AbstractKeyedTimePanes<Type, Key, ArrayList<Type>, Result> {
 	
@@ -52,7 +55,9 @@ public class AccumulatingKeyedTimePanes<Type, Key, Result> extends AbstractKeyed
 	private final AccumulatingKeyedTimePanesContext context;
 
 	/**
-	 * IMPORTANT: This value needs to start at one, so it is fresher than the value that new entries have (zero) */
+	 * IMPORTANT: This value needs to start at one, so it is fresher than the value that new entries
+	 * have (zero).
+	 */
 	private long evaluationPass = 1L;   
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
index 57beb930b0987..507237455ca3f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
@@ -30,6 +30,9 @@
 import java.util.ArrayList;
 
 /**
+ * Special window operator implementation for windows that fire at the same time for all keys with
+ * accumulating windows.
+ *
  * @deprecated Deprecated in favour of the generic {@link WindowOperator}. This was an
  * optimized implementation used for aligned windows.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java
index 84686a71e8dcc..d26ab9d7ad646 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java
@@ -25,6 +25,9 @@
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.util.Collector;
 
+/**
+ * Key/value map organized in panes for aggregating windows (with a reduce function).
+ */
 @Internal
 public class AggregatingKeyedTimePanes<Type, Key> extends AbstractKeyedTimePanes<Type, Key, Type, Type> {
 	
@@ -33,7 +36,9 @@ public class AggregatingKeyedTimePanes<Type, Key> extends AbstractKeyedTimePanes
 	private final ReduceFunction<Type> reducer;
 
 	/**
-	 * IMPORTANT: This value needs to start at one, so it is fresher than the value that new entries have (zero) */
+	 * IMPORTANT: This value needs to start at one, so it is fresher than the value that new entries
+	 * have (zero).
+	 */
 	private long evaluationPass = 1L;
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
index 1be9650a91d28..51803e67c36bd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
@@ -25,6 +25,9 @@
 import org.apache.flink.api.java.functions.KeySelector;
 
 /**
+ * Special window operator implementation for windows that fire at the same time for all keys with
+ * aggregating windows.
+ *
  * @deprecated Deprecated in favour of the generic {@link WindowOperator}. This was an
  * optimized implementation used for aligned windows.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
index 85451a52f3d39..0d590c24becb6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
@@ -48,9 +48,9 @@
 /**
  * A {@link WindowOperator} that also allows an {@link Evictor} to be used.
  *
- * <p>
- * The {@code Evictor} is used to remove elements from a pane before/after the evaluation of WindowFunction and
- * after the window evaluation gets triggered by a {@link org.apache.flink.streaming.api.windowing.triggers.Trigger}.
+ * <p>The {@code Evictor} is used to remove elements from a pane before/after the evaluation of
+ * {@link InternalWindowFunction} and after the window evaluation gets triggered by a
+ * {@link org.apache.flink.streaming.api.windowing.triggers.Trigger}.
  *
  * @param <K> The type of key returned by the {@code KeySelector}.
  * @param <IN> The type of the incoming elements.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
index b0f4f46949bb2..4e50d8280ed3f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
@@ -30,46 +30,48 @@
 /**
  * A special Hash Map implementation that can be traversed efficiently in sync with other
  * hash maps.
- * <p>
- * The differences between this hash map and Java's "java.util.HashMap" are:
+ *
+ * <p>The differences between this hash map and Java's "java.util.HashMap" are:
  * <ul>
  *     <li>A different hashing scheme. This implementation uses extensible hashing, meaning that
  *         each hash table growth takes one more lower hash code bit into account, and values that where
- *         formerly in the same bucket will afterwards be in the two adjacent buckets.</li>
+ *         formerly in the same bucket will afterwards be in the two adjacent buckets.
  *     <li>This allows an efficient traversal of multiple hash maps together, even though the maps are
  *         of different sizes.</li>
- *     <li>The map offers functions such as "putIfAbsent()" and "putOrAggregate()"</li>
- *     <li>The map supports no removal/shrinking.</li>
+ *     <li>The map offers functions such as "putIfAbsent()" and "putOrAggregate()"
+ *     <li>The map supports no removal/shrinking.
  * </ul>
  */
 @Internal
 public class KeyMap<K, V> implements Iterable<KeyMap.Entry<K, V>> {
 	
-	/** The minimum table capacity, 64 entries */
+	/** The minimum table capacity, 64 entries. */
 	private static final int MIN_CAPACITY = 0x40;
 	
-	/** The maximum possible table capacity, the largest positive power of
-	 * two in the 32bit signed integer value range */
+	/**
+	 * The maximum possible table capacity, the largest positive power of two in the 32bit signed
+	 * integer value range.
+	 */
 	private static final int MAX_CAPACITY = 0x40000000;
 	
-	/** The number of bits used for table addressing when table is at max capacity */
+	/** The number of bits used for table addressing when table is at max capacity. */
 	private static final int FULL_BIT_RANGE = MathUtils.log2strict(MAX_CAPACITY);
 	
 	// ------------------------------------------------------------------------
 	
-	/** The hash index, as an array of entries */
+	/** The hash index, as an array of entries. */
 	private Entry<K, V>[] table;
 	
-	/** The number of bits by which the hash code is shifted right, to find the bucket */
+	/** The number of bits by which the hash code is shifted right, to find the bucket. */
 	private int shift;
 	
-	/** The number of elements in the hash table */
+	/** The number of elements in the hash table. */
 	private int numElements;
 	
-	/** The number of elements above which the hash table needs to grow */
+	/** The number of elements above which the hash table needs to grow. */
 	private int rehashThreshold;
 	
-	/** The base-2 logarithm of the table capacity */ 
+	/** The base-2 logarithm of the table capacity. */
 	private int log2size;
 
 	// ------------------------------------------------------------------------
@@ -141,8 +143,8 @@ public final V put(K key, V value) {
 	/**
 	 * Inserts a value for the given key, if no value is yet contained for that key. Otherwise,
 	 * returns the value currently contained for the key.
-	 * <p>
-	 * The value that is inserted in case that the key is not contained, yet, is lazily created
+	 *
+	 * <p>The value that is inserted in case that the key is not contained, yet, is lazily created
 	 * using the given factory.
 	 *
 	 * @param key The key to insert.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/TimestampedValue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/TimestampedValue.java
index 9c63a69c98e20..108162b68cbab 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/TimestampedValue.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/TimestampedValue.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -28,13 +28,13 @@
 @PublicEvolving
 public class TimestampedValue<T> {
 
-	/** The actual value held by this record */
+	/** The actual value held by this record. */
 	private T value;
 
-	/** The timestamp of the record */
+	/** The timestamp of the record. */
 	private long timestamp;
 
-	/** Flag whether the timestamp is actually set */
+	/** Flag whether the timestamp is actually set. */
 	private boolean hasTimestamp;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index 70692517960d9..e36d485a8e65e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -88,15 +88,13 @@
  * An operator that implements the logic for windowing based on a {@link WindowAssigner} and
  * {@link Trigger}.
  *
- * <p>
- * When an element arrives it gets assigned a key using a {@link KeySelector} and it gets
+ * <p>When an element arrives it gets assigned a key using a {@link KeySelector} and it gets
  * assigned to zero or more windows using a {@link WindowAssigner}. Based on this, the element
  * is put into panes. A pane is the bucket of elements that have the same key and same
  * {@code Window}. An element can be in multiple panes if it was assigned to multiple windows by the
  * {@code WindowAssigner}.
  *
- * <p>
- * Each pane gets its own instance of the provided {@code Trigger}. This trigger determines when
+ * <p>Each pane gets its own instance of the provided {@code Trigger}. This trigger determines when
  * the contents of the pane should be processed to emit results. When a trigger fires,
  * the given {@link InternalWindowFunction} is invoked to produce the results that are emitted for
  * the pane to which the {@code Trigger} belongs.
@@ -161,10 +159,13 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
 	 */
 	private transient InternalMergingState<W, IN, ACC> windowMergingState;
 
-	/** The state that holds the merging window metadata (the sets that describe what is merged) */
+	/** The state that holds the merging window metadata (the sets that describe what is merged). */
 	private transient InternalListState<VoidNamespace, Tuple2<W, W>> mergingSetsState;
 
-	/** This is given to the {@code InternalWindowFunction} for emitting elements with a given timestamp. */
+	/**
+	 * This is given to the {@code InternalWindowFunction} for emitting elements with a given
+	 * timestamp.
+	 */
 	protected transient TimestampedCollector<OUT> timestampedCollector;
 
 	protected transient Context triggerContext = new Context(null, null);
@@ -599,7 +600,7 @@ private void emitWindowContents(W window, ACC contents) throws Exception {
 	}
 
 	/**
-	 * Write skipped late arriving element to SideOutput
+	 * Write skipped late arriving element to SideOutput.
 	 *
 	 * @param element skipped late arriving element to side output
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java
index 98344801cd156..4055c4f6affcd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java
@@ -49,6 +49,11 @@ public interface InternalWindowFunction<IN, OUT, KEY, W extends Window> extends
 	 */
 	void clear(W window, InternalWindowContext context) throws Exception;
 
+	/**
+	 * A context for {@link InternalWindowFunction}, similar to
+	 * {@link org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction.Context} but
+	 * for internal use.
+	 */
 	interface InternalWindowContext extends java.io.Serializable {
 		KeyedStateStore windowState();
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ConfigurableStreamPartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ConfigurableStreamPartitioner.java
index c59c88ac9d7a0..d27246e937dfb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ConfigurableStreamPartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ConfigurableStreamPartitioner.java
@@ -23,7 +23,7 @@
  * of the stream transformation. The configure method is called by the StreamGraph when adding
  * internal edges.
  *
- * This interface is required since the stream partitioners are instantiated eagerly. Due to that
+ * <p>This interface is required since the stream partitioners are instantiated eagerly. Due to that
  * the maximum parallelism might not have been determined and needs to be set at a stage when the
  * maximum parallelism could have been determined.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/GlobalPartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/GlobalPartitioner.java
index bc77d5447b4fe..67eaa7300569f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/GlobalPartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/GlobalPartitioner.java
@@ -22,7 +22,7 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
 /**
- * Partitioner that sends all elements to the downstream operator with subtask ID=0;
+ * Partitioner that sends all elements to the downstream operator with subtask ID=0.
  *
  * @param <T> Type of the elements in the Stream being partitioned
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/StreamPartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/StreamPartitioner.java
index 6b4a810e0a48e..415a3c7f1f68f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/StreamPartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/StreamPartitioner.java
@@ -23,6 +23,9 @@
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+/**
+ * A special {@link ChannelSelector} for use in streaming programs.
+ */
 @Internal
 public abstract class StreamPartitioner<T> implements
 		ChannelSelector<SerializationDelegate<StreamRecord<T>>>, Serializable {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/LatencyMarker.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/LatencyMarker.java
index 714bdae35ec6e..131658db5ffc9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/LatencyMarker.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/LatencyMarker.java
@@ -24,7 +24,7 @@
  * Special record type carrying a timestamp of its creation time at a source operator
  * and the vertexId and subtask index of the operator.
  *
- * At sinks, the marker can be used to approximate the time a record needs to travel
+ * <p>At sinks, the marker can be used to approximate the time a record needs to travel
  * through the dataflow.
  */
 @PublicEvolving
@@ -32,7 +32,7 @@ public final class LatencyMarker extends StreamElement {
 
 	// ------------------------------------------------------------------------
 
-	/** The time the latency mark is denoting */
+	/** The time the latency mark is denoting. */
 	private final long markedTime;
 
 	private final int vertexID;
@@ -40,7 +40,7 @@ public final class LatencyMarker extends StreamElement {
 	private final int subtaskIndex;
 
 	/**
-	 * Creates a latency mark with the given timestamp
+	 * Creates a latency mark with the given timestamp.
 	 */
 	public LatencyMarker(long markedTime, int vertexID, int subtaskIndex) {
 		this.markedTime = markedTime;
@@ -49,7 +49,7 @@ public LatencyMarker(long markedTime, int vertexID, int subtaskIndex) {
 	}
 
 	/**
-	 * Returns the timestamp marked by the LatencyMarker
+	 * Returns the timestamp marked by the LatencyMarker.
 	 */
 	public long getMarkedTime() {
 		return markedTime;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index 3db649ae724d7..f2b8aec3687ec 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -30,10 +30,10 @@
 import static java.util.Objects.requireNonNull;
 
 /**
- * Serializer for {@link StreamRecord}, {@link Watermark}, {@link LatencyMarker}, and {@link StreamStatus}.
+ * Serializer for {@link StreamRecord}, {@link Watermark}, {@link LatencyMarker}, and
+ * {@link StreamStatus}.
  *
- * <p>
- * This does not behave like a normal {@link TypeSerializer}, instead, this is only used at the
+ * <p>This does not behave like a normal {@link TypeSerializer}, instead, this is only used at the
  * stream task/operator level for transmitting StreamRecords and Watermarks.
  *
  * @param <T> The type of value in the StreamRecord
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java
index da606a93ba0b7..03958976f191c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java
@@ -27,13 +27,13 @@
 @Internal
 public final class StreamRecord<T> extends StreamElement {
 	
-	/** The actual value held by this record */
+	/** The actual value held by this record. */
 	private T value;
 	
-	/** The timestamp of the record */
+	/** The timestamp of the record. */
 	private long timestamp;
 
-	/** Flag whether the timestamp is actually set */
+	/** Flag whether the timestamp is actually set. */
 	private boolean hasTimestamp;
 	
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java
index f17d240256779..1327c838bfaa3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java
@@ -49,13 +49,16 @@ public interface ValveOutputHandler {
 	//	Runtime state for watermark & stream status output determination
 	// ------------------------------------------------------------------------
 
-	/** Array of current status of all input channels. Changes as watermarks & stream statuses are fed into the valve */
+	/**
+	 * Array of current status of all input channels. Changes as watermarks & stream statuses are
+	 * fed into the valve.
+	 */
 	private final InputChannelStatus[] channelStatuses;
 
-	/** The last watermark emitted from the valve */
+	/** The last watermark emitted from the valve. */
 	private long lastOutputWatermark;
 
-	/** The last stream status emitted from the valve */
+	/** The last stream status emitted from the valve. */
 	private StreamStatus lastOutputStreamStatus;
 
 	/**
@@ -172,20 +175,25 @@ private void findAndOutputNewMinWatermarkAcrossAlignedChannels() {
 	}
 
 	/**
-	 * An {@code InputChannelStatus} keeps track of an input channel's last watermark, stream status, and whether or not
-	 * the channel's current watermark is aligned with the overall watermark output from the valve.
+	 * An {@code InputChannelStatus} keeps track of an input channel's last watermark, stream
+	 * status, and whether or not the channel's current watermark is aligned with the overall
+	 * watermark output from the valve.
 	 *
-	 * There are 2 situations where a channel's watermark is not considered aligned:
-	 *  - the current stream status of the channel is idle
-	 *  - the stream status has resumed to be active, but the watermark of the channel hasn't caught up to the
-	 *    last output watermark from the valve yet.
+	 * <p>There are 2 situations where a channel's watermark is not considered aligned:
+	 * <ul>
+	 *   <li>the current stream status of the channel is idle
+	 *   <li>the stream status has resumed to be active, but the watermark of the channel hasn't
+	 *   caught up to thelast output watermark from the valve yet.
+	 * </ul>
 	 */
 	private static class InputChannelStatus {
 		private long watermark;
 		private StreamStatus streamStatus;
 		private boolean isWatermarkAligned;
 
-		/** Utility to check if at least one channel in a given array of input channels is active */
+		/**
+		 * Utility to check if at least one channel in a given array of input channels is active.
+		 */
 		private static boolean hasActiveChannels(InputChannelStatus[] channelStatuses) {
 			for (InputChannelStatus status : channelStatuses) {
 				if (status.streamStatus.isActive()) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StreamStatus.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StreamStatus.java
index e82fad0420bf7..8e58340947c60 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StreamStatus.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StreamStatus.java
@@ -52,8 +52,7 @@
  *         {@link StreamStatus#ACTIVE}, the task is active.</li>
  * </ul>
  *
- * <p>
- * Stream Status elements received at downstream tasks also affect and control how their operators process and advance
+ * <p>Stream Status elements received at downstream tasks also affect and control how their operators process and advance
  * their watermarks. The below describes the effects (the logic is implemented as a {@link StatusWatermarkValve} which
  * downstream tasks should use for such purposes):
  *
@@ -64,16 +63,15 @@
  *         active. However, for watermarks, since there may be watermark generators that might produce watermarks
  *         anywhere in the middle of topologies regardless of whether there are input data at the operator, the current
  *         status of the task must be checked before forwarding watermarks emitted from
- *         an operator. It the status is actually idle, the watermark must be blocked.</li>
+ *         an operator. It the status is actually idle, the watermark must be blocked.
  *
  *     <li>For downstream tasks with multiple input streams, the watermarks of input streams that are temporarily idle,
  *         or has resumed to be active but its watermark is behind the overall min watermark of the operator, should not
  *         be accounted for when deciding whether or not to advance the watermark and propagated through the operator
- *         chain.</li>
+ *         chain.
  * </ul>
  *
- * <p>
- * Note that to notify downstream tasks that a source task is permanently closed and will no longer send any more
+ * <p>Note that to notify downstream tasks that a source task is permanently closed and will no longer send any more
  * elements, the source should still send a {@link Watermark#MAX_WATERMARK} instead of {@link StreamStatus#IDLE}.
  * Stream Status elements only serve as markers for temporary status.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java
index cda0511f120bc..39d3ed2fa545c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
index e04d316819302..0343b54cc213a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
@@ -25,6 +25,9 @@
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.runtime.io.StreamInputProcessor;
 
+/**
+ * A {@link StreamTask} for executing a {@link OneInputStreamOperator}.
+ */
 @Internal
 public class OneInputStreamTask<IN, OUT> extends StreamTask<OUT, OneInputStreamOperator<IN, OUT>> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index be4b456db9f21..aced2f416e514 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -203,7 +203,6 @@ public Output<StreamRecord<OUT>> getChainEntryPoint() {
 	}
 
 	/**
-	 *
 	 * This method should be called before finishing the record emission, to make sure any data
 	 * that is still buffered will be sent. It also ensures that all data sending related
 	 * exceptions are recognized.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
index 7588c3d54c9cb..1164ea02663e9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
@@ -29,10 +29,10 @@ public interface ProcessingTimeCallback {
 
 	/**
 	 * This method is invoked with the timestamp for which the trigger was scheduled.
-	 * <p>
-	 * If the triggering is delayed for whatever reason (trigger timer was blocked, JVM stalled due
-	 * to a garbage collection), the timestamp supplied to this function will still be the original
-	 * timestamp for which the trigger was scheduled.
+	 *
+	 * <p>If the triggering is delayed for whatever reason (trigger timer was blocked, JVM stalled
+	 * due to a garbage collection), the timestamp supplied to this function will still be the
+	 * original timestamp for which the trigger was scheduled.
 	 * 
 	 * @param timestamp The timestamp for which the trigger event was scheduled.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
index 18291408d0e35..a3913ce7a3450 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
@@ -23,9 +23,9 @@
 import org.apache.flink.streaming.api.operators.StreamSource;
 
 /**
- * Task for executing streaming sources.
+ * {@link StreamTask} for executing a {@link StreamSource}.
  *
- * One important aspect of this is that the checkpointing and the emission of elements must never
+ * <p>One important aspect of this is that the checkpointing and the emission of elements must never
  * occur at the same time. The execution must be serial. This is achieved by having the contract
  * with the StreamFunction that it must only modify its state or emit elements in
  * a synchronized block that locks on the lock Object. Also, the modification of the state
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
index 59edb8f192972..d05ec37ff1ca4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
@@ -30,6 +30,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * A special {@link StreamTask} that is used for executing feedback edges. This is used in
+ * combination with {@link StreamIterationTail}.
+ */
 @Internal
 public class StreamIterationHead<OUT> extends OneInputStreamTask<OUT, OUT> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
index e40f8344bc5eb..7326a8082dee4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
@@ -32,6 +32,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * A special {@link StreamTask} that is used for executing feedback edges. This is used in
+ * combination with {@link StreamIterationHead}.
+ */
 @Internal
 public class StreamIterationTail<IN> extends OneInputStreamTask<IN, IN> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 7de6c1a2705e3..e73c825aebaa7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -87,7 +87,7 @@
  * produced by the operators at the ends of the operator chain. Note that the chain may fork and
  * thus have multiple ends.
  *
- * The life cycle of the task is set up as follows:
+ * <p>The life cycle of the task is set up as follows:
  * <pre>{@code
  *  -- setInitialState -> provides state of all operators in the chain
  *
@@ -105,7 +105,7 @@
  *        +----> task specific cleanup()
  * }</pre>
  *
- * <p> The {@code StreamTask} has a lock object called {@code lock}. All calls to methods on a
+ * <p>The {@code StreamTask} has a lock object called {@code lock}. All calls to methods on a
  * {@code StreamOperator} must be synchronized on this lock object to ensure that no methods
  * are called concurrently.
  *
@@ -117,27 +117,27 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>>
 		extends AbstractInvokable
 		implements StatefulTask, AsyncExceptionHandler {
 
-	/** The thread group that holds all trigger timer threads */
+	/** The thread group that holds all trigger timer threads. */
 	public static final ThreadGroup TRIGGER_THREAD_GROUP = new ThreadGroup("Triggers");
 
-	/** The logger used by the StreamTask and its subclasses */
+	/** The logger used by the StreamTask and its subclasses. */
 	private static final Logger LOG = LoggerFactory.getLogger(StreamTask.class);
 
 	// ------------------------------------------------------------------------
 
 	/**
-	 * All interaction with the {@code StreamOperator} must be synchronized on this lock object to ensure that
-	 * we don't have concurrent method calls that void consistent checkpoints.
+	 * All interaction with the {@code StreamOperator} must be synchronized on this lock object to
+	 * ensure that we don't have concurrent method calls that void consistent checkpoints.
 	 */
 	private final Object lock = new Object();
 
-	/** the head operator that consumes the input streams of this task */
+	/** the head operator that consumes the input streams of this task. */
 	protected OP headOperator;
 
-	/** The chain of operators executed by this task */
+	/** The chain of operators executed by this task. */
 	protected OperatorChain<OUT, OP> operatorChain;
 
-	/** The configuration of this streaming task */
+	/** The configuration of this streaming task. */
 	private StreamConfig configuration;
 
 	/** Our state backend. We use this to create checkpoint streams and a keyed state backend. */
@@ -153,22 +153,24 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>>
 	 */
 	private ProcessingTimeService timerService;
 
-	/** The map of user-defined accumulators of this task */
+	/** The map of user-defined accumulators of this task. */
 	private Map<String, Accumulator<?, ?>> accumulatorMap;
 
 	private TaskStateHandles restoreStateHandles;
 
-	/** The currently active background materialization threads */
+	/** The currently active background materialization threads. */
 	private final CloseableRegistry cancelables = new CloseableRegistry();
 
-	/** Flag to mark the task "in operation", in which case check
-	 * needs to be initialized to true, so that early cancel() before invoke() behaves correctly */
+	/**
+	 * Flag to mark the task "in operation", in which case check needs to be initialized to true,
+	 * so that early cancel() before invoke() behaves correctly.
+	 */
 	private volatile boolean isRunning;
 
-	/** Flag to mark this task as canceled */
+	/** Flag to mark this task as canceled. */
 	private volatile boolean canceled;
 
-	/** Thread pool for async snapshot workers */
+	/** Thread pool for async snapshot workers. */
 	private ExecutorService asyncOperationsThreadPool;
 
 	// ------------------------------------------------------------------------
@@ -363,11 +365,10 @@ public final boolean isCanceled() {
 	}
 
 	/**
-	 * Execute the operator-specific {@link StreamOperator#open()} method in each
-	 * of the operators in the chain of this {@link StreamTask}. </b> Opening happens
-	 * from <b>tail to head</b> operator in the chain, contrary to
-	 * {@link StreamOperator#close()} which happens <b>head to tail</b>
-	 * operator (see {@link #closeAllOperators()}.
+	 * Execute {@link StreamOperator#open()} of each operator in the chain of this
+	 * {@link StreamTask}. Opening happens from <b>tail to head</b> operator in the chain, contrary
+	 * to {@link StreamOperator#close()} which happens <b>head to tail</b>
+	 * (see {@link #closeAllOperators()}.
 	 */
 	private void openAllOperators() throws Exception {
 		for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
@@ -378,10 +379,9 @@ private void openAllOperators() throws Exception {
 	}
 
 	/**
-	 * Execute the operator-specific {@link StreamOperator#close()} method in each
-	 * of the operators in the chain of this {@link StreamTask}. </b> Closing happens
-	 * from <b>head to tail</b> operator in the chain, contrary to
-	 * {@link StreamOperator#open()} which happens <b>tail to head</b> operator
+	 * Execute {@link StreamOperator#close()} of each operator in the chain of this
+	 * {@link StreamTask}. Closing happens from <b>head to tail</b> operator in the chain,
+	 * contrary to {@link StreamOperator#open()} which happens <b>tail to head</b>
 	 * (see {@link #openAllOperators()}.
 	 */
 	private void closeAllOperators() throws Exception {
@@ -397,9 +397,8 @@ private void closeAllOperators() throws Exception {
 	}
 
 	/**
-	 * Execute the operator-specific {@link StreamOperator#dispose()} method in each
-	 * of the operators in the chain of this {@link StreamTask}. </b> Disposing happens
-	 * from <b>tail to head</b> operator in the chain.
+	 * Execute {@link StreamOperator#dispose()} of each operator in the chain of this
+	 * {@link StreamTask}. Disposing happens from <b>tail to head</b> operator in the chain.
 	 */
 	private void tryDisposeAllOperators() throws Exception {
 		for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
@@ -416,11 +415,10 @@ private void shutdownAsyncThreads() throws Exception {
 	}
 
 	/**
-	 * Execute the operator-specific {@link StreamOperator#dispose()} method in each
-	 * of the operators in the chain of this {@link StreamTask}. </b> Disposing happens
-	 * from <b>tail to head</b> operator in the chain.
+	 * Execute @link StreamOperator#dispose()} of each operator in the chain of this
+	 * {@link StreamTask}. Disposing happens from <b>tail to head</b> operator in the chain.
 	 *
-	 * The difference with the {@link #tryDisposeAllOperators()} is that in case of an
+	 * <p>The difference with the {@link #tryDisposeAllOperators()} is that in case of an
 	 * exception, this method catches it and logs the message.
 	 */
 	private void disposeAllOperators() {
@@ -442,8 +440,7 @@ private void disposeAllOperators() {
 	 * The finalize method shuts down the timer. This is a fail-safe shutdown, in case the original
 	 * shutdown method was never called.
 	 *
-	 * <p>
-	 * This should not be relied upon! It will cause shutdown to happen much later than if manual
+	 * <p>This should not be relied upon! It will cause shutdown to happen much later than if manual
 	 * shutdown is attempted, and cause threads to linger for longer than needed.
 	 */
 	@Override
@@ -818,7 +815,7 @@ public ProcessingTimeService getProcessingTimeService() {
 	 * Handles an exception thrown by another thread (e.g. a TriggerTask),
 	 * other than the one executing the main task by failing the task entirely.
 	 *
-	 * In more detail, it marks task execution failed for an external reason
+	 * <p>In more detail, it marks task execution failed for an external reason
 	 * (a reason other than the task code itself throwing an exception). If the task
 	 * is already in a terminal state (such as FINISHED, CANCELED, FAILED), or if the
 	 * task is already canceling this does nothing. Otherwise it sets the state to
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
index abcb19bc9a6d2..4c2d731d2170a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
@@ -49,10 +49,10 @@ public class SystemProcessingTimeService extends ProcessingTimeService {
 	/** The containing task that owns this time service provider. */
 	private final AsyncExceptionHandler task;
 
-	/** The lock that timers acquire upon triggering */
+	/** The lock that timers acquire upon triggering. */
 	private final Object checkpointLock;
 
-	/** The executor service that schedules and calls the triggers of this task*/
+	/** The executor service that schedules and calls the triggers of this task. */
 	private final ScheduledThreadPoolExecutor timerService;
 
 	private final AtomicInteger status;
@@ -92,7 +92,8 @@ public long getCurrentProcessingTime() {
 	}
 
 	/**
-	 * Registers a task to be executed no sooner than time {@code timestamp}, but without strong guarantees of order
+	 * Registers a task to be executed no sooner than time {@code timestamp}, but without strong
+	 * guarantees of order.
 	 *
 	 * @param timestamp Time when the task is to be enabled (in processing time)
 	 * @param target    The task to be executed
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TimerException.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TimerException.java
index 2e39725a0a275..0bd78ac73f837 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TimerException.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TimerException.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
index 71346b83cd4fc..50aefc871831e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
@@ -28,6 +28,9 @@
 import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
 import org.apache.flink.streaming.runtime.io.StreamTwoInputProcessor;
 
+/**
+ * A {@link StreamTask} for executing a {@link TwoInputStreamOperator}.
+ */
 @Internal
 public class TwoInputStreamTask<IN1, IN2, OUT> extends StreamTask<OUT, TwoInputStreamOperator<IN1, IN2, OUT>> {
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java
index a40ae3a453156..e5b790868104c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java
@@ -20,8 +20,8 @@
  * executable stream consumers and producers that are scheduled by the distributed
  * dataflow runtime. Each task occupies one execution slot and is run with by an
  * executing thread.
- * <p>
- * The tasks merely set up the distributed stream coordination and the checkpointing.
+ *
+ * <p>The tasks merely set up the distributed stream coordination and the checkpointing.
  * Internally, the tasks create one or more operators, perform the stream transformations.
  */
 package org.apache.flink.streaming.runtime.tasks;
\ No newline at end of file
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
index ac9be395c91d4..e81a1c7194499 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
@@ -38,18 +38,20 @@ public class TypeInformationSerializationSchema<T> implements DeserializationSch
 	
 	private static final long serialVersionUID = -5359448468131559102L;
 	
-	/** The serializer for the actual de-/serialization */
+	/** The serializer for the actual de-/serialization. */
 	private final TypeSerializer<T> serializer;
 
-	/** The reusable output serialization buffer */
+	/** The reusable output serialization buffer. */
 	private transient DataOutputSerializer dos;
 	
-	/** The reusable input deserialization buffer */
+	/** The reusable input deserialization buffer. */
 	private transient DataInputDeserializer dis;
 
-	/** The type information, to be returned by {@link #getProducedType()}. It is
-	 * transient, because it is not serializable. Note that this means that the type information
-	 * is not available at runtime, but only prior to the first serialization / deserialization */
+	/**
+	 * The type information, to be returned by {@link #getProducedType()}. It is transient, because
+	 * it is not serializable. Note that this means that the type information is not available at
+	 * runtime, but only prior to the first serialization / deserialization.
+	 */
 	private transient TypeInformation<T> typeInfo;
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
index 21f477ae2ce13..aef87d9b9ba8c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
@@ -41,12 +41,13 @@
 
 /**
  * These classes encapsulate the logic of accessing a field specified by the user as either an index
- * or a field expression string. TypeInformation can also be requested for the field.
- * The position index might specify a field of a Tuple, an array, or a simple type (only "0th field").
+ * or a field expression string. TypeInformation can also be requested for the field. The position
+ * index might specify a field of a Tuple, an array, or a simple type(only "0th field").
  *
- * Field expressions that specify nested fields (e.g. "f1.a.foo") result in nested field accessors.
- * These penetrate one layer, and then delegate the rest of the work to an "innerAccesor".
- * (see PojoFieldAccessor, RecursiveTupleFieldAccessor, RecursiveProductFieldAccessor)
+ * <p>Field expressions that specify nested fields (e.g. "f1.a.foo") result in nested field
+ * accessors. These penetrate one layer, and then delegate the rest of the work to an
+ * "innerAccesor". (see PojoFieldAccessor, RecursiveTupleFieldAccessor,
+ * RecursiveProductFieldAccessor)
  */
 @Internal
 public abstract class FieldAccessor<T, F> implements Serializable {
@@ -75,7 +76,7 @@ public TypeInformation<F> getFieldType() {
 	/**
 	 * Sets the field (specified in the constructor) of the given record to the given value.
 	 *
-	 * Warning: This might modify the original object, or might return a new object instance.
+	 * <p>Warning: This might modify the original object, or might return a new object instance.
 	 * (This is necessary, because the record might be immutable.)
 	 *
 	 * @param record The record to modify
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java
index 51e995b37b908..28cdf9d6294be 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java
index e9d63de69bbea..e5d3ef02a5fe9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -27,7 +27,8 @@
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.TypeInfoParser;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.windowing.ReduceIterableWindowFunction;
+import org.apache.flink.streaming.api.functions.windowing.PassThroughWindowFunction;
+import org.apache.flink.streaming.api.functions.windowing.ReduceApplyWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.delta.DeltaFunction;
 import org.apache.flink.streaming.api.watermark.Watermark;
@@ -546,7 +547,11 @@ public void testCountTrigger() throws Exception {
 				new TupleKeySelector(),
 				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
 				stateDesc,
-				new InternalIterableWindowFunction<>(new ReduceIterableWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>(new SumReducer())),
+				new InternalIterableWindowFunction<>(
+						new ReduceApplyWindowFunction<>(
+								new SumReducer(),
+								// on some versions of Java we seem to need the explicit type
+								new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>())),
 				CountTrigger.of(WINDOW_SLIDE),
 				CountEvictor.of(WINDOW_SIZE),
 				0,
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTest.java
index 9633671376b04..341171dc31656 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java
index d86c809d3357f..a21e58bf92202 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java
index ab9e59b5e0116..3bd7adf1316ff 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
index 3676953a4b76e..9e26f9ec06c90 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
index ff07fa210eb04..a9d2ddfb1c5e5 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTaskTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTaskTestHarness.java
index 216765241a9bd..7ce4ab795b5ec 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTaskTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTaskTestHarness.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/EvenOddOutputSelector.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/EvenOddOutputSelector.java
index 9820ef81a2ef2..1745c469386bd 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/EvenOddOutputSelector.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/EvenOddOutputSelector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpIntMap.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpIntMap.java
index bcb569121d66a..05de3d0262ad3 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpIntMap.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpIntMap.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestHarnessUtil.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestHarnessUtil.java
index 4b6925d11ff72..b46ea6674fdcc 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestHarnessUtil.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestHarnessUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java
index 90bdcb2215b5b..7d791c0252930 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index ba2f4be6efe0c..3bf9863bae68b 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -251,30 +251,30 @@ This file is based on the checkstyle file of Apache Beam.
 
     <!-- Checks for Javadoc comments.                     -->
     <!-- See http://checkstyle.sf.net/config_javadoc.html -->
-    <!--<module name="JavadocMethod">-->
-      <!--<property name="scope" value="protected"/>-->
-      <!--<property name="severity" value="error"/>-->
-      <!--<property name="allowMissingJavadoc" value="true"/>-->
-      <!--<property name="allowMissingParamTags" value="true"/>-->
-      <!--<property name="allowMissingReturnTag" value="true"/>-->
-      <!--<property name="allowMissingThrowsTags" value="true"/>-->
-      <!--<property name="allowThrowsTagsForSubclasses" value="true"/>-->
-      <!--<property name="allowUndeclaredRTE" value="true"/>-->
-    <!--</module>-->
+    <module name="JavadocMethod">
+      <property name="scope" value="protected"/>
+      <property name="severity" value="error"/>
+      <property name="allowMissingJavadoc" value="true"/>
+      <property name="allowMissingParamTags" value="true"/>
+      <property name="allowMissingReturnTag" value="true"/>
+      <property name="allowMissingThrowsTags" value="true"/>
+      <property name="allowThrowsTagsForSubclasses" value="true"/>
+      <property name="allowUndeclaredRTE" value="true"/>
+    </module>
 
-    <!--&lt;!&ndash; Check that paragraph tags are used correctly in Javadoc. &ndash;&gt;-->
-    <!--<module name="JavadocParagraph"/>-->
+    <!-- Check that paragraph tags are used correctly in Javadoc. -->
+    <module name="JavadocParagraph"/>
 
-    <!--<module name="JavadocType">-->
-      <!--<property name="scope" value="protected"/>-->
-      <!--<property name="severity" value="error"/>-->
-      <!--<property name="allowMissingParamTags" value="true"/>-->
-    <!--</module>-->
+    <module name="JavadocType">
+      <property name="scope" value="protected"/>
+      <property name="severity" value="error"/>
+      <property name="allowMissingParamTags" value="true"/>
+    </module>
 
-    <!--<module name="JavadocStyle">-->
-      <!--<property name="severity" value="error"/>-->
-      <!--<property name="checkHtml" value="true"/>-->
-    <!--</module>-->
+    <module name="JavadocStyle">
+      <property name="severity" value="error"/>
+      <property name="checkHtml" value="true"/>
+    </module>
 
     <!--
 

From 1d9fa55b0333659812d50ce4b8f9a04df1e3a1b1 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 10:29:10 +0100
Subject: [PATCH 048/253] [FLINK-6107] Enable newline at EOF check in streaming
 checkstyle

---
 .../streaming/api/graph/StreamGraphUserHashHasher.java    | 2 +-
 .../flink/streaming/runtime/operators/package-info.java   | 2 +-
 .../streaming/runtime/tasks/OperatorStateHandles.java     | 2 +-
 .../flink/streaming/runtime/tasks/package-info.java       | 2 +-
 flink-streaming-java/src/test/resources/log4j.properties  | 2 +-
 tools/maven/strict-checkstyle.xml                         | 8 ++++----
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
index a5e77cc64b260..89ec45ad1a7b2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
@@ -54,4 +54,4 @@ public Map<Integer, byte[]> traverseStreamGraphAndGenerateHashes(StreamGraph str
 
 		return hashResult;
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java
index 5fe6873488419..fd1a71f16d9a2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java
@@ -19,4 +19,4 @@
  * This package contains the operators that perform the stream transformations.
  * One or more operators are bundled into a "chain" and executed in a stream task. 
  */
-package org.apache.flink.streaming.runtime.operators;
\ No newline at end of file
+package org.apache.flink.streaming.runtime.operators;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java
index 30d07b73bf7eb..1a79f5429c206 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java
@@ -106,4 +106,4 @@ public int getOperatorChainIndex() {
 	private static <T> T getSafeItemAtIndexOrNull(List<T> list, int idx) {
 		return CollectionUtil.isNullOrEmpty(list) ? null : list.get(idx);
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java
index e5b790868104c..a1179ff4452af 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/package-info.java
@@ -24,4 +24,4 @@
  * <p>The tasks merely set up the distributed stream coordination and the checkpointing.
  * Internally, the tasks create one or more operators, perform the stream transformations.
  */
-package org.apache.flink.streaming.runtime.tasks;
\ No newline at end of file
+package org.apache.flink.streaming.runtime.tasks;
diff --git a/flink-streaming-java/src/test/resources/log4j.properties b/flink-streaming-java/src/test/resources/log4j.properties
index ed2bbcbbd8167..4a30c6f18449c 100644
--- a/flink-streaming-java/src/test/resources/log4j.properties
+++ b/flink-streaming-java/src/test/resources/log4j.properties
@@ -24,4 +24,4 @@ log4j.rootLogger=OFF, console
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.target = System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index 3bf9863bae68b..7684a24e40262 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -35,10 +35,10 @@ This file is based on the checkstyle file of Apache Beam.
     <!--&lt;!&ndash; Checks that there are no tab characters in the file. &ndash;&gt;-->
   <!--</module>-->
 
-  <!--<module name="NewlineAtEndOfFile">-->
-    <!--&lt;!&ndash; windows can use \n\r vs \n, so enforce the most used one ie UNIx style &ndash;&gt;-->
-    <!--<property name="lineSeparator" value="lf" />-->
-  <!--</module>-->
+  <module name="NewlineAtEndOfFile">
+    <!-- windows can use \n\r vs \n, so enforce the most used one ie UNIx style -->
+    <property name="lineSeparator" value="lf" />
+  </module>
 
   <module name="RegexpSingleline">
     <!-- Checks that TODOs don't have stuff in parenthesis, e.g., username. -->

From d86809b12a78ba0eefab3a56a350b9de4696c929 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 11:17:47 +0100
Subject: [PATCH 049/253] [FLINK-6107] Enable trailing whitespace check in
 streaming checkstyle

---
 .../streamrecord/StreamRecordSerializer.java  |   9 +-
 .../streaming/api/CheckpointingMode.java      |  22 +--
 .../streaming/api/TimeCharacteristic.java     |   2 +-
 .../api/checkpoint/Checkpointed.java          |  14 +-
 .../CheckpointedAsynchronously.java           |  10 +-
 .../api/checkpoint/CheckpointedFunction.java  |  32 ++--
 .../api/checkpoint/ListCheckpointed.java      |  42 +++---
 .../collector/selector/DirectedOutput.java    |  20 +--
 .../collector/selector/OutputSelector.java    |   4 +-
 .../api/datastream/AllWindowedStream.java     |  12 +-
 .../api/datastream/CoGroupedStreams.java      |  42 +++---
 .../api/datastream/ConnectedStreams.java      |   8 +-
 .../streaming/api/datastream/DataStream.java  |  44 +++---
 .../api/datastream/DataStreamSource.java      |   2 +-
 .../api/datastream/IterativeStream.java       |  28 ++--
 .../api/datastream/JoinedStreams.java         |  26 ++--
 .../streaming/api/datastream/KeyedStream.java |  16 +-
 .../SingleOutputStreamOperator.java           |  24 +--
 .../streaming/api/datastream/SplitStream.java |   2 +-
 .../api/environment/CheckpointConfig.java     |  34 ++---
 .../Flip6LocalStreamEnvironment.java          |   2 +-
 .../environment/LocalStreamEnvironment.java   |  10 +-
 .../environment/RemoteStreamEnvironment.java  |  20 +--
 .../StreamExecutionEnvironment.java           |   6 +-
 .../AssignerWithPeriodicWatermarks.java       |   8 +-
 .../AssignerWithPunctuatedWatermarks.java     |  12 +-
 .../api/functions/IngestionTimeExtractor.java |   8 +-
 .../api/functions/ProcessFunction.java        |   2 +-
 .../api/functions/TimestampAssigner.java      |  10 +-
 .../api/functions/TimestampExtractor.java     |   2 +-
 .../aggregation/ComparableAggregator.java     |   2 +-
 .../api/functions/async/AsyncFunction.java    |  12 +-
 .../api/functions/co/CoFlatMapFunction.java   |  10 +-
 .../api/functions/co/CoProcessFunction.java   |   4 +-
 .../api/functions/sink/DiscardingSink.java    |   4 +-
 .../sink/OutputFormatSinkFunction.java        |   8 +-
 .../api/functions/sink/PrintSinkFunction.java |  24 +--
 .../api/functions/sink/SocketClientSink.java  |  54 +++----
 .../api/functions/sink/WriteFormatAsCsv.java  |   2 +-
 .../api/functions/sink/WriteSinkFunction.java |   6 +-
 .../sink/WriteSinkFunctionByMillis.java       |   2 +-
 .../source/FromElementsFunction.java          |  30 ++--
 .../source/InputFormatSourceFunction.java     |   2 +-
 .../source/RichParallelSourceFunction.java    |   2 +-
 .../source/SocketTextStreamFunction.java      |  24 +--
 .../api/functions/source/SourceFunction.java  |  28 ++--
 .../source/StatefulSequenceSource.java        |   2 +-
 .../AscendingTimestampExtractor.java          |  20 +--
 .../AggregateApplyWindowFunction.java         |   2 +-
 .../InternalProcessApplyWindowContext.java    |   2 +-
 .../functions/windowing/WindowFunction.java   |   4 +-
 .../windowing/delta/CosineDistance.java       |   4 +-
 .../windowing/delta/DeltaFunction.java        |   2 +-
 .../windowing/delta/EuclideanDistance.java    |   4 +-
 .../delta/ExtractionAwareDeltaFunction.java   |   8 +-
 .../delta/extractor/ArrayFromTuple.java       |   2 +-
 .../delta/extractor/ConcatenatedExtract.java  |   2 +-
 .../windowing/delta/extractor/Extractor.java  |   4 +-
 .../delta/extractor/FieldFromArray.java       |   6 +-
 .../delta/extractor/FieldFromTuple.java       |   4 +-
 .../delta/extractor/FieldsFromArray.java      |   4 +-
 .../delta/extractor/FieldsFromTuple.java      |   2 +-
 .../streaming/api/graph/StreamConfig.java     |  76 +++++-----
 .../flink/streaming/api/graph/StreamEdge.java |   2 +-
 .../streaming/api/graph/StreamGraph.java      |  16 +-
 .../api/graph/StreamGraphGenerator.java       |   2 +-
 .../api/graph/StreamingJobGraphGenerator.java |   4 +-
 .../api/operators/AbstractStreamOperator.java |  36 ++---
 .../operators/AbstractUdfStreamOperator.java  |  16 +-
 .../api/operators/ChainingStrategy.java       |   6 +-
 .../api/operators/OneInputStreamOperator.java |   2 +-
 .../api/operators/StreamGroupedFold.java      |  12 +-
 .../api/operators/StreamGroupedReduce.java    |   8 +-
 .../api/operators/StreamOperator.java         |  22 +--
 .../streaming/api/operators/StreamSource.java |  20 +--
 .../operators/StreamingRuntimeContext.java    |   2 +-
 .../api/operators/TimestampedCollector.java   |   4 +-
 .../api/operators/TwoInputStreamOperator.java |   2 +-
 .../operators/async/AsyncWaitOperator.java    |   4 +-
 .../OneInputTransformation.java               |   2 +-
 .../PartitionTransformation.java              |   2 +-
 .../transformations/SelectTransformation.java |   2 +-
 .../transformations/SinkTransformation.java   |   2 +-
 .../transformations/StreamTransformation.java |  50 +++----
 .../streaming/api/watermark/Watermark.java    |   6 +-
 .../SlidingAlignedProcessingTimeWindows.java  |  10 +-
 .../TumblingAlignedProcessingTimeWindows.java |   2 +-
 .../api/windowing/triggers/Trigger.java       |  20 +--
 .../api/windowing/triggers/TriggerResult.java |   4 +-
 .../api/windowing/windows/TimeWindow.java     |  12 +-
 .../streaming/runtime/io/BarrierBuffer.java   |  26 ++--
 .../streaming/runtime/io/BarrierTracker.java  |  14 +-
 .../runtime/io/BlockingQueueBroker.java       |   2 +-
 .../streaming/runtime/io/BufferSpiller.java   |  94 ++++++------
 .../runtime/io/CheckpointBarrierHandler.java  |  16 +-
 .../runtime/io/RecordWriterOutput.java        |  12 +-
 .../runtime/io/StreamInputProcessor.java      |  16 +-
 .../runtime/io/StreamRecordWriter.java        |  46 +++---
 .../runtime/io/StreamTwoInputProcessor.java   |  12 +-
 .../operators/ExtractTimestampsOperator.java  |   2 +-
 ...mestampsAndPeriodicWatermarksOperator.java |  12 +-
 ...stampsAndPunctuatedWatermarksOperator.java |   2 +-
 .../runtime/operators/package-info.java       |   2 +-
 ...ctAlignedProcessingTimeWindowOperator.java |  64 ++++----
 .../windowing/AbstractKeyedTimePanes.java     |  36 ++---
 .../windowing/AccumulatingKeyedTimePanes.java |  22 +--
 ...umulatingProcessingTimeWindowOperator.java |   4 +-
 .../windowing/AggregatingKeyedTimePanes.java  |  18 +--
 ...gregatingProcessingTimeWindowOperator.java |   6 +-
 .../runtime/operators/windowing/KeyMap.java   | 140 +++++++++---------
 .../operators/windowing/package-info.java     |   2 +-
 .../partitioner/ForwardPartitioner.java       |   6 +-
 .../partitioner/RebalancePartitioner.java     |   6 +-
 .../partitioner/RescalePartitioner.java       |   4 +-
 .../partitioner/ShufflePartitioner.java       |   2 +-
 .../runtime/streamrecord/StreamElement.java   |   2 +-
 .../streamrecord/StreamElementSerializer.java |  18 +--
 .../runtime/streamrecord/StreamRecord.java    |  22 +--
 .../runtime/tasks/AsynchronousException.java  |   2 +-
 .../ExceptionInChainedOperatorException.java  |   4 +-
 .../runtime/tasks/OneInputStreamTask.java     |   2 +-
 .../runtime/tasks/OperatorChain.java          |  54 +++----
 .../runtime/tasks/ProcessingTimeCallback.java |   2 +-
 .../runtime/tasks/ProcessingTimeService.java  |  10 +-
 .../runtime/tasks/SourceStreamTask.java       |   4 +-
 .../runtime/tasks/StreamIterationHead.java    |  14 +-
 .../runtime/tasks/StreamIterationTail.java    |   6 +-
 .../streaming/runtime/tasks/StreamTask.java   |   2 +-
 .../runtime/tasks/StreamTaskException.java    |   6 +-
 .../tasks/SystemProcessingTimeService.java    |   2 +-
 .../runtime/tasks/TwoInputStreamTask.java     |  12 +-
 .../streaming/util/keys/KeySelectorUtil.java  |  26 ++--
 .../AbstractDeserializationSchema.java        |   8 +-
 .../serialization/DeserializationSchema.java  |   8 +-
 .../serialization/SerializationSchema.java    |   4 +-
 .../serialization/SimpleStringSchema.java     |   4 +-
 .../TypeInformationSerializationSchema.java   |  18 +--
 .../api/AggregationFunctionTest.java          |  20 +--
 .../flink/streaming/api/DataStreamTest.java   |   6 +-
 .../api/StreamExecutionEnvironmentTest.java   |   4 +-
 .../flink/streaming/api/TypeFillTest.java     |  10 +-
 .../AscendingTimestampExtractorTest.java      |  26 ++--
 .../functions/FromElementsFunctionTest.java   |  56 +++----
 .../functions/IngestionTimeExtractorTest.java |  18 +--
 .../api/functions/ListSourceContext.java      |  12 +-
 .../api/functions/PrintSinkFunctionTest.java  |   2 +-
 .../source/SocketTextStreamFunctionTest.java  |  38 ++---
 .../api/graph/SlotAllocationTest.java         |   2 +-
 .../graph/StreamingJobGraphGeneratorTest.java |   2 +-
 .../operators/AbstractStreamOperatorTest.java |   2 +-
 .../operators/StateDescriptorPassingTest.java |  10 +-
 .../StateInitializationContextImplTest.java   |   2 +-
 .../api/operators/StreamGroupedFoldTest.java  |   6 +-
 .../operators/StreamGroupedReduceTest.java    |   4 +-
 .../api/operators/StreamMapTest.java          |   2 +-
 .../StreamingRuntimeContextTest.java          |  38 ++---
 .../api/streamtask/MockRecordWriter.java      |   2 +-
 .../deltafunction/CosineDistanceTest.java     |   8 +-
 .../deltafunction/EuclideanDistanceTest.java  |  10 +-
 .../streaming/graph/TranslationTest.java      |   8 +-
 .../io/BarrierBufferAlignmentLimitTest.java   |  14 +-
 .../io/BarrierBufferMassiveRandomTest.java    |  12 +-
 .../runtime/io/BarrierBufferTest.java         |  74 ++++-----
 .../runtime/io/BarrierTrackerTest.java        |  52 +++----
 .../runtime/io/BufferSpillerTest.java         |  70 ++++-----
 .../io/SpilledBufferOrEventSequenceTest.java  | 104 ++++++-------
 .../runtime/io/StreamRecordWriterTest.java    |  26 ++--
 .../flink/streaming/runtime/io/TestEvent.java |   2 +-
 .../operators/StreamSourceOperatorTest.java   |  28 ++--
 .../operators/StreamTaskTimerTest.java        |  16 +-
 ...ampsAndPeriodicWatermarksOperatorTest.java |  28 ++--
 ...psAndPunctuatedWatermarksOperatorTest.java |  16 +-
 ...ignedProcessingTimeWindowOperatorTest.java |  34 ++---
 .../windowing/AllWindowTranslationTest.java   |  14 +-
 .../windowing/KeyMapPutIfAbsentTest.java      |  22 +--
 .../operators/windowing/KeyMapPutTest.java    |  10 +-
 .../operators/windowing/KeyMapTest.java       |  78 +++++-----
 .../windowing/WindowOperatorTest.java         |  20 +--
 .../partitioner/BroadcastPartitionerTest.java |   2 +-
 .../partitioner/RebalancePartitionerTest.java |   8 +-
 .../partitioner/RescalePartitionerTest.java   |   8 +-
 .../StreamElementSerializerTest.java          |  22 +--
 .../streamrecord/StreamRecordTest.java        |  34 ++---
 .../tasks/OneInputStreamTaskTestHarness.java  |   2 +-
 .../tasks/SourceStreamTaskStoppingTest.java   |   4 +-
 .../StreamTaskCancellationBarrierTest.java    |   6 +-
 .../runtime/tasks/StreamTaskTest.java         |  20 +--
 .../SystemProcessingTimeServiceTest.java      |   6 +-
 .../AbstractDeserializationSchemaTest.java    |   8 +-
 .../util/CollectingSourceContext.java         |   2 +-
 .../flink/streaming/util/MockContext.java     |   6 +-
 .../streaming/util/SourceFunctionUtil.java    |   6 +-
 .../TwoInputStreamOperatorTestHarness.java    |   2 +-
 ...ypeInformationSerializationSchemaTest.java |  20 +--
 .../util/keys/ArrayKeySelectorTest.java       |  12 +-
 tools/maven/strict-checkstyle.xml             |  10 +-
 196 files changed, 1493 insertions(+), 1494 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
index 885ece58b1ae8..0e9e8092960b4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
@@ -45,7 +45,6 @@ public final class StreamRecordSerializer<T> extends TypeSerializer<StreamRecord
 	private static final long serialVersionUID = 1L;
 
 	private final TypeSerializer<T> typeSerializer;
-	
 
 	public StreamRecordSerializer(TypeSerializer<T> serializer) {
 		if (serializer instanceof StreamRecordSerializer) {
@@ -57,7 +56,7 @@ public StreamRecordSerializer(TypeSerializer<T> serializer) {
 	public TypeSerializer<T> getContainedTypeSerializer() {
 		return this.typeSerializer;
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  General serializer and type utils
 	// ------------------------------------------------------------------------
@@ -90,7 +89,7 @@ public StreamRecord<T> createInstance() {
 			throw new RuntimeException("Cannot instantiate StreamRecord.", e);
 		}
 	}
-	
+
 	@Override
 	public StreamRecord<T> copy(StreamRecord<T> from) {
 		return from.copy(typeSerializer.copy(from.getValue()));
@@ -106,7 +105,7 @@ public StreamRecord<T> copy(StreamRecord<T> from, StreamRecord<T> reuse) {
 	public void serialize(StreamRecord<T> value, DataOutputView target) throws IOException {
 		typeSerializer.serialize(value.getValue(), target);
 	}
-	
+
 	@Override
 	public StreamRecord<T> deserialize(DataInputView source) throws IOException {
 		return new StreamRecord<T>(typeSerializer.deserialize(source));
@@ -125,7 +124,7 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public boolean equals(Object obj) {
 		if (obj instanceof StreamRecordSerializer) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/CheckpointingMode.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/CheckpointingMode.java
index a879f17f55bfe..f4597b8378f81 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/CheckpointingMode.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/CheckpointingMode.java
@@ -23,13 +23,13 @@
 /**
  * The checkpointing mode defines what consistency guarantees the system gives in the presence of
  * failures.
- * 
+ *
  * <p>When checkpointing is activated, the data streams are replayed such that lost parts of the
  * processing are repeated. For stateful operations and functions, the checkpointing mode defines
  * whether the system draws checkpoints such that a recovery behaves as if the operators/functions
  * see each record "exactly once" ({@link #EXACTLY_ONCE}), or whether the checkpoints are drawn
  * in a simpler fashion that typically encounteres some duplicates upon recovery
- * ({@link #AT_LEAST_ONCE})</p> 
+ * ({@link #AT_LEAST_ONCE})</p>
  */
 @Public
 public enum CheckpointingMode {
@@ -38,21 +38,21 @@ public enum CheckpointingMode {
 	 * Sets the checkpointing mode to "exactly once". This mode means that the system will
 	 * checkpoint the operator and user function state in such a way that, upon recovery,
 	 * every record will be reflected exactly once in the operator state.
-	 * 
-	 * <p>For example, if a user function counts the number of elements in a stream, 
+	 *
+	 * <p>For example, if a user function counts the number of elements in a stream,
 	 * this number will consistently be equal to the number of actual elements in the stream,
 	 * regardless of failures and recovery.</p>
-	 * 
+	 *
 	 * <p>Note that this does not mean that each record flows through the streaming data flow
 	 * only once. It means that upon recovery, the state of operators/functions is restored such
-	 * that the resumed data streams pick up exactly at after the last modification to the state.</p> 
-	 *  
+	 * that the resumed data streams pick up exactly at after the last modification to the state.</p>
+	 *
 	 * <p>Note that this mode does not guarantee exactly-once behavior in the interaction with
 	 * external systems (only state in Flink's operators and user functions). The reason for that
 	 * is that a certain level of "collaboration" is required between two systems to achieve
 	 * exactly-once guarantees. However, for certain systems, connectors can be written that facilitate
 	 * this collaboration.</p>
-	 * 
+	 *
 	 * <p>This mode sustains high throughput. Depending on the data flow graph and operations,
 	 * this mode may increase the record latency, because operators need to align their input
 	 * streams, in order to create a consistent snapshot point. The latency increase for simple
@@ -65,11 +65,11 @@ public enum CheckpointingMode {
 	 * Sets the checkpointing mode to "at least once". This mode means that the system will
 	 * checkpoint the operator and user function state in a simpler way. Upon failure and recovery,
 	 * some records may be reflected multiple times in the operator state.
-	 * 
-	 * <p>For example, if a user function counts the number of elements in a stream, 
+	 *
+	 * <p>For example, if a user function counts the number of elements in a stream,
 	 * this number will equal to, or larger, than the actual number of elements in the stream,
 	 * in the presence of failure and recovery.</p>
-	 * 
+	 *
 	 * <p>This mode has minimal impact on latency and may be preferable in very-low latency
 	 * scenarios, where a sustained very-low latency (such as few milliseconds) is needed,
 	 * and where occasional duplicate messages (on recovery) do not matter.</p>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java
index 9ad5879272ac3..992560829f877 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/TimeCharacteristic.java
@@ -49,7 +49,7 @@ public enum TimeCharacteristic {
 	 *
 	 * <p>Ingestion time is often a good compromise between processing time and event time.
 	 * It does not need and special manual form of watermark generation, and events are typically
-	 * not too much out-or-order when they arrive at operators; in fact, out-of-orderness can 
+	 * not too much out-or-order when they arrive at operators; in fact, out-of-orderness can
 	 * only be introduced by streaming shuffles or split/join/union operations. The fact that
 	 * elements are not very much out-of-order means that the latency increase is moderate,
 	 * compared to event
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
index 2479e0a72b1da..7dedfbf8b01f9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
@@ -26,7 +26,7 @@
  * This method must be implemented by functions that have state that needs to be
  * checkpointed. The functions get a call whenever a checkpoint should take place
  * and return a snapshot of their state, which will be checkpointed.
- * 
+ *
  * <h1>Deprecation and Replacement</h1>
  * The short cut replacement for this interface is via {@link ListCheckpointed} and works
  * as shown in the example below. The {@code ListCheckpointed} interface returns a list of
@@ -51,9 +51,9 @@
  *     }
  * }
  * }</pre>
- * 
+ *
  * @param <T> The type of the operator state.
- * 
+ *
  * @deprecated Please use {@link ListCheckpointed} as illustrated above, or
  *             {@link CheckpointedFunction} for more control over the checkpointing process.
  */
@@ -63,14 +63,14 @@ public interface Checkpointed<T extends Serializable> extends CheckpointedRestor
 
 	/**
 	 * Gets the current state of the function of operator. The state must reflect the result of all
-	 * prior invocations to this function. 
-	 * 
+	 * prior invocations to this function.
+	 *
 	 * @param checkpointId The ID of the checkpoint.
 	 * @param checkpointTimestamp The timestamp of the checkpoint, as derived by
 	 *                            System.currentTimeMillis() on the JobManager.
-	 *                            
+	 *
 	 * @return A snapshot of the operator state.
-	 * 
+	 *
 	 * @throws Exception Thrown if the creation of the state object failed. This causes the
 	 *                   checkpoint to fail. The system may decide to fail the operation (and trigger
 	 *                   recovery), or to discard this checkpoint attempt and to continue running
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
index 4be522ca42ac3..2230f5962e720 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
@@ -25,17 +25,17 @@
 /**
  * This interface marks a function/operator as checkpointed similar to the
  * {@link Checkpointed} interface, but gives the Flink framework the option to
- * perform the checkpoint asynchronously. Note that asynchronous checkpointing for 
+ * perform the checkpoint asynchronously. Note that asynchronous checkpointing for
  * this interface has not been implemented.
- * 
+ *
  * <h1>Deprecation and Replacement</h1>
  * The shortcut replacement for this interface is via {@link ListCheckpointed} and works
  * as shown in the example below. Please refer to the JavaDocs of {@link ListCheckpointed} for
  * a more detailed description of how to use the new interface.
- * 
+ *
  * <p><pre>{@code
  * public class ExampleFunction<T> implements MapFunction<T, T>, ListCheckpointed<Integer> {
- * 
+ *
  *     private int count;
  *
  *     public List<Integer> snapshotState(long checkpointId, long timestamp) throws Exception {
@@ -45,7 +45,7 @@
  *     public void restoreState(List<Integer> state) throws Exception {
  *         this.value = state.count.isEmpty() ? 0 : state.get(0);
  *     }
- * 
+ *
  *     public T map(T value) {
  *         count++;
  *         return value;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java
index 556e95744152e..ecf38a53c12f3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedFunction.java
@@ -30,17 +30,17 @@
  * that maintain state across individual stream records.
  * While more lightweight interfaces exist as shortcuts for various types of state, this interface offer the
  * greatest flexibility in managing both <i>keyed state</i> and <i>operator state</i>.
- * 
+ *
  * <p>The section <a href="#shortcuts">Shortcuts</a> illustrates the common lightweight
  * ways to setup stateful functions typically used instead of the full fledged
  * abstraction represented by this interface.
- * 
+ *
  * <h1>Initialization</h1>
  * The {@link CheckpointedFunction#initializeState(FunctionInitializationContext)} is called when
  * the parallel instance of the transformation function is created during distributed execution.
  * The method gives access to the {@link FunctionInitializationContext} which in turn gives access
  * to the to the {@link OperatorStateStore} and {@link KeyedStateStore}.
- * 
+ *
  * <p>The {@code OperatorStateStore} and {@code KeyedStateStore} give access to the data structures
  * in which state should be stored for Flink to transparently manage and checkpoint it, such as
  * {@link org.apache.flink.api.common.state.ValueState} or
@@ -48,14 +48,14 @@
  *
  * <p><b>Note:</b> The {@code KeyedStateStore} can only be used when the transformation supports
  * <i>keyed state</i>, i.e., when it is applied on a keyed stream (after a {@code keyBy(...)}).
- * 
+ *
  * <h1>Snapshot</h1>
  * The {@link CheckpointedFunction#snapshotState(FunctionSnapshotContext)} is called whenever a
  * checkpoint takes a state snapshot of the transformation function. Inside this method, functions typically
  * make sure that the checkpointed data structures (obtained in the initialization phase) are up
  * to date for a snapshot to be taken. The given snapshot context gives access to the metadata
  * of the checkpoint.
- * 
+ *
  * <p>In addition, functions can use this method as a hook to flush/commit/synchronize with
  * external systems.
  *
@@ -66,7 +66,7 @@
  * The example also changes of parallelism, which affect the count-per-parallel-partition by
  * adding up the counters of partitions that get merged on scale-down. Note that this is a
  * toy example, but should illustrate the basic skeleton for a stateful function.
- * 
+ *
  * <p><pre>{@code
  * public class MyFunction<T> implements MapFunction<T, T>, CheckpointedFunction {
  *
@@ -106,38 +106,38 @@
  *     }
  * }
  * }</pre>
- * 
+ *
  * <hr>
- * 
+ *
  * <h1><a name="shortcuts">Shortcuts</a></h1>
  * There are various ways that transformation functions can use state without implementing the
  * full-fledged {@code CheckpointedFunction} interface:
- * 
+ *
  * <h4>Operator State</h4>
  * Checkpointing some state that is part of the function object itself is possible in a simpler way
- * by directly implementing the {@link ListCheckpointed} interface. 
+ * by directly implementing the {@link ListCheckpointed} interface.
  * That mechanism is similar to the previously used {@link Checkpointed} interface.
- * 
+ *
  * <h4>Keyed State</h4>
  * Access to keyed state is possible via the {@link RuntimeContext}'s methods:
  * <pre>{@code
  * public class CountPerKeyFunction<T> extends RichMapFunction<T, T> {
- * 
+ *
  *     private ValueState<Long> count;
- *     
+ *
  *     public void open(Configuration cfg) throws Exception {
  *         count = getRuntimeContext().getState(new ValueStateDescriptor<>("myCount", Long.class));
  *     }
- *     
+ *
  *     public T map(T value) throws Exception {
  *         Long current = count.get();
  *         count.update(current == null ? 1L : current + 1);
- *         
+ *
  *         return value;
  *     }
  * }
  * }</pre>
- * 
+ *
  * @see ListCheckpointed
  * @see RuntimeContext
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
index ab18c5891fc35..0b8b1b6af2e82 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
@@ -36,28 +36,28 @@
  * from the {@link OperatorStateStore}. Using the {@code OperatorStateStore} directly gives
  * more flexible options to use operator state, for example controlling the serialization
  * of the state objects, or have multiple named states.
- * 
+ *
  * <h2>State Redistribution</h2>
  * State redistribution happens when the parallelism of the operator is changed.
  * State redistribution of <i>operator state</i> (to which category the state handled by this
  * interface belongs) always goes through a checkpoint, so it appears
  * to the transformation functions like a failure/recovery combination, where recovery happens
  * with a different parallelism.
- * 
+ *
  * <p>Conceptually, the state in the checkpoint is the concatenated list of all lists
  * returned by the parallel transformation function instances. When restoring from a checkpoint,
- * the list is divided into sub-lists that are assigned to each parallel function instance. 
- * 
+ * the list is divided into sub-lists that are assigned to each parallel function instance.
+ *
  * <p>The following sketch illustrates the state redistribution.The function runs with parallelism
  * <i>3</i>. The first two parallel instance of the function return lists with two state elements,
- * the third one a list with one element.  
+ * the third one a list with one element.
  * <pre>
  *    func_1        func_2     func_3
  * +----+----+   +----+----+   +----+
  * | S1 | S2 |   | S3 | S4 |   | S5 |
  * +----+----+   +----+----+   +----+
  * </pre>
- * 
+ *
  * <p>Recovering the checkpoint with <i>parallelism = 5</i> yields the following state assignment:
  * <pre>
  * func_1   func_2   func_3   func_4   func_5
@@ -65,7 +65,7 @@
  * | S1 |   | S2 |   | S3 |   | S4 |   | S5 |
  * +----+   +----+   +----+   +----+   +----+
  * </pre>
- 
+
  * Recovering the checkpoint with <i>parallelism = 5</i> yields the following state assignment:
  * <pre>
  *      func_1          func_2
@@ -73,14 +73,14 @@
  * | S1 | S2 | S3 |   | S4 | S5 |
  * +----+----+----+   +----+----+
  * </pre>
- * 
+ *
  * <h2>Example</h2>
  * The following example illustrates how to implement a {@code MapFunction} that counts all elements
  * passing through it, keeping the total count accurate under re-scaling  (changes or parallelism):
  * <pre>{@code
  * public class CountingFunction<T> implements MapFunction<T, Tuple2<T, Long>>, ListCheckpointed<Long> {
- * 
- *     // this count is the number of elements in the parallel subtask 
+ *
+ *     // this count is the number of elements in the parallel subtask
  *     private long count;
  *
  *     {@literal @}Override
@@ -88,7 +88,7 @@
  *         // return a single element - our count
  *         return Collections.singletonList(count);
  *     }
- * 
+ *
  *     {@literal @}Override
  *     public void restoreState(List<Long> state) throws Exception {
  *         // in case of scale in, this adds up counters from different original subtasks
@@ -105,7 +105,7 @@
  *     }
  * }
  * }</pre>
- * 
+ *
  * @param <T> The type of the operator state.
  */
 @PublicEvolving
@@ -114,20 +114,20 @@ public interface ListCheckpointed<T extends Serializable> {
 	/**
 	 * Gets the current state of the function. The state must reflect the result of all prior
 	 * invocations to this function.
-	 * 
+	 *
 	 * <p>The returned list should contain one entry for redistributable unit of state. See
 	 * the {@link ListCheckpointed class docs} for an illustration how list-style state
 	 * redistribution works.
-	 * 
+	 *
 	 * <p>As special case, the returned list may be null or empty (if the operator has no state)
 	 * or it may contain a single element (if the operator state is indivisible).
 	 *
-	 * @param checkpointId The ID of the checkpoint - a unique and monotonously increasing value. 
+	 * @param checkpointId The ID of the checkpoint - a unique and monotonously increasing value.
 	 * @param timestamp The wall clock timestamp when the checkpoint was triggered by the master.
-	 *                     
+	 *
 	 * @return The operator state in a list of redistributable, atomic sub-states.
 	 *         Should not return null, but empty list instead.
-	 *         
+	 *
 	 * @throws Exception Thrown if the creation of the state object failed. This causes the
 	 *                   checkpoint to fail. The system may decide to fail the operation (and trigger
 	 *                   recovery), or to discard this checkpoint attempt and to continue running
@@ -140,16 +140,16 @@ public interface ListCheckpointed<T extends Serializable> {
 	 * This method is invoked when the function is executed after a failure recovery.
 	 * The state list may be empty if no state is to be recovered by the particular parallel instance
 	 * of the function.
-	 * 
+	 *
 	 * <p>The given state list will contain all the <i>sub states</i> that this parallel
 	 * instance of the function needs to handle. Refer to the  {@link ListCheckpointed class docs}
 	 * for an illustration how list-style state redistribution works.
-	 * 
+	 *
 	 * <p><b>Important:</b> When implementing this interface together with {@link RichFunction},
 	 * then the {@code restoreState()} method is called before {@link RichFunction#open(Configuration)}.
-	 * 
+	 *
 	 * @param state The state to be restored as a list of atomic sub-states.
-	 * 
+	 *
 	 * @throws Exception Throwing an exception in this method causes the recovery to fail.
 	 *                   The exact consequence depends on the configured failure handling strategy,
 	 *                   but typically the system will re-attempt the recovery, or try recovering
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
index a709d45e82640..d806e9dfb6dec 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
@@ -40,18 +40,18 @@
  * {@link OutputSelector OutputSelectors}.
  */
 public class DirectedOutput<OUT> implements Output<StreamRecord<OUT>> {
-	
+
 	protected final OutputSelector<OUT>[] outputSelectors;
 
 	protected final Output<StreamRecord<OUT>>[] selectAllOutputs;
-	
+
 	protected final HashMap<String, Output<StreamRecord<OUT>>[]> outputMap;
-	
+
 	protected final Output<StreamRecord<OUT>>[] allOutputs;
 
 	private final Random RNG = new XORShiftRandom();
 
-	
+
 	@SuppressWarnings({"unchecked", "rawtypes"})
 	public DirectedOutput(
 			List<OutputSelector<OUT>> outputSelectors,
@@ -63,15 +63,15 @@ public DirectedOutput(
 		for (int i = 0; i < outputs.size(); i++) {
 			allOutputs[i] = outputs.get(i).f0;
 		}
-		
-		
+
+
 		HashSet<Output<StreamRecord<OUT>>> selectAllOutputs = new HashSet<Output<StreamRecord<OUT>>>();
 		HashMap<String, ArrayList<Output<StreamRecord<OUT>>>> outputMap = new HashMap<String, ArrayList<Output<StreamRecord<OUT>>>>();
-		
+
 		for (Tuple2<Output<StreamRecord<OUT>>, StreamEdge> outputPair : outputs) {
 			final Output<StreamRecord<OUT>> output = outputPair.f0;
 			final StreamEdge edge = outputPair.f1;
-	
+
 			List<String> selectedNames = edge.getSelectedNames();
 
 			if (selectedNames.isEmpty()) {
@@ -91,9 +91,9 @@ public DirectedOutput(
 				}
 			}
 		}
-		
+
 		this.selectAllOutputs = selectAllOutputs.toArray(new Output[selectAllOutputs.size()]);
-		
+
 		this.outputMap = new HashMap<>();
 		for (Map.Entry<String, ArrayList<Output<StreamRecord<OUT>>>> entry : outputMap.entrySet()) {
 			Output<StreamRecord<OUT>>[] arr = entry.getValue().toArray(new Output[entry.getValue().size()]);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java
index 126effef275bf..b40f3f52c0bf4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/OutputSelector.java
@@ -27,7 +27,7 @@
  * Interface for defining an OutputSelector for a {@link SplitStream} using
  * the {@link SingleOutputStreamOperator#split} call. Every output object of a
  * {@link SplitStream} will run through this operator to select outputs.
- * 
+ *
  * @param <OUT>
  *            Type parameter of the split values.
  */
@@ -38,7 +38,7 @@ public interface OutputSelector<OUT> extends Serializable {
 	 * {@link SingleOutputStreamOperator#split} method. The values will be
 	 * emitted only to output names which are contained in the returned
 	 * iterable.
-	 * 
+	 *
 	 * @param value
 	 *            Output object for which the output selection should be made.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
index 7c0ebd22b8c8d..5aabf28805fd6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
@@ -416,7 +416,7 @@ public <R> SingleOutputStreamOperator<R> reduce(ReduceFunction<T> reduceFunction
 	// ------------------------------------------------------------------------
 
 	/**
-	 * Applies the given {@code AggregateFunction} to each window. The AggregateFunction 
+	 * Applies the given {@code AggregateFunction} to each window. The AggregateFunction
 	 * aggregates all elements of a window into a single result element. The stream of these
 	 * result elements (one per window) is interpreted as a regular non-windowed stream.
 	 *
@@ -425,7 +425,7 @@ public <R> SingleOutputStreamOperator<R> reduce(ReduceFunction<T> reduceFunction
 	 *
 	 * @param <ACC> The type of the AggregateFunction's accumulator
 	 * @param <R> The type of the elements in the resulting stream, equal to the
-	 *            AggregateFunction's result type   
+	 *            AggregateFunction's result type
 	 */
 	@PublicEvolving
 	public <ACC, R> SingleOutputStreamOperator<R> aggregate(AggregateFunction<T, ACC, R> function) {
@@ -445,7 +445,7 @@ public <ACC, R> SingleOutputStreamOperator<R> aggregate(AggregateFunction<T, ACC
 	}
 
 	/**
-	 * Applies the given {@code AggregateFunction} to each window. The AggregateFunction 
+	 * Applies the given {@code AggregateFunction} to each window. The AggregateFunction
 	 * aggregates all elements of a window into a single result element. The stream of these
 	 * result elements (one per window) is interpreted as a regular non-windowed stream.
 	 *
@@ -454,7 +454,7 @@ public <ACC, R> SingleOutputStreamOperator<R> aggregate(AggregateFunction<T, ACC
 	 *
 	 * @param <ACC> The type of the AggregateFunction's accumulator
 	 * @param <R> The type of the elements in the resulting stream, equal to the
-	 *            AggregateFunction's result type  
+	 *            AggregateFunction's result type
 	 */
 	@PublicEvolving
 	public <ACC, R> SingleOutputStreamOperator<R> aggregate(
@@ -488,7 +488,7 @@ public <ACC, R> SingleOutputStreamOperator<R> aggregate(
 	 * @return The data stream that is the result of applying the window function to the window.
 	 *
 	 * @param <ACC> The type of the AggregateFunction's accumulator
-	 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input  
+	 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 	 * @param <R> The type of the elements in the resulting stream, equal to the
 	 *            WindowFunction's result type
 	 */
@@ -528,7 +528,7 @@ public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
 	 * @return The data stream that is the result of applying the window function to the window.
 	 *
 	 * @param <ACC> The type of the AggregateFunction's accumulator
-	 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input  
+	 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 	 * @param <R> The type of the elements in the resulting stream, equal to the
 	 *            WindowFunction's result type
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
index 5c0dbf901e57f..98c57f1917a22 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
@@ -77,7 +77,7 @@ public class CoGroupedStreams<T1, T2> {
 	/**
 	 * Creates new CoGroped data streams, which are the first step towards building a streaming
 	 * co-group.
-	 * 
+	 *
 	 * @param input1 The first data stream.
 	 * @param input2 The second data stream.
 	 */
@@ -95,10 +95,10 @@ public <KEY> Where<KEY> where(KeySelector<T1, KEY> keySelector)  {
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * CoGrouped streams that have the key for one side defined.
-	 * 
+	 *
 	 * @param <KEY> The type of the key.
 	 */
 	@Public
@@ -111,22 +111,22 @@ public class Where<KEY> {
 			this.keySelector1 = keySelector1;
 			this.keyType = keyType;
 		}
-	
+
 		/**
 		 * Specifies a {@link KeySelector} for elements from the second input.
 		 */
 		public EqualTo equalTo(KeySelector<T2, KEY> keySelector)  {
 			TypeInformation<KEY> otherKey = TypeExtractor.getKeySelectorTypes(keySelector, input2.getType());
 			if (!otherKey.equals(this.keyType)) {
-				throw new IllegalArgumentException("The keys for the two inputs are not equal: " + 
+				throw new IllegalArgumentException("The keys for the two inputs are not equal: " +
 						"first key = " + this.keyType + " , second key = " + otherKey);
 			}
-			
+
 			return new EqualTo(input2.clean(keySelector));
 		}
 
 		// --------------------------------------------------------------------
-		
+
 		/**
 		 * A co-group operation that has {@link KeySelector KeySelectors} defined for both inputs.
 		 */
@@ -150,7 +150,7 @@ public <W extends Window> WithWindow<T1, T2, KEY, W> window(WindowAssigner<? sup
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * A co-group operation that has {@link KeySelector KeySelectors} defined for both inputs as
 	 * well as a {@link WindowAssigner}.
@@ -167,7 +167,7 @@ public static class WithWindow<T1, T2, KEY, W extends Window> {
 
 		private final KeySelector<T1, KEY> keySelector1;
 		private final KeySelector<T2, KEY> keySelector2;
-		
+
 		private final TypeInformation<KEY> keyType;
 
 		private final WindowAssigner<? super TaggedUnion<T1, T2>, W> windowAssigner;
@@ -190,7 +190,7 @@ protected WithWindow(DataStream<T1> input1,
 			this.keySelector1 = keySelector1;
 			this.keySelector2 = keySelector2;
 			this.keyType = keyType;
-			
+
 			this.windowAssigner = windowAssigner;
 			this.trigger = trigger;
 			this.evictor = evictor;
@@ -221,7 +221,7 @@ public WithWindow<T1, T2, KEY, W> evictor(Evictor<? super TaggedUnion<T1, T2>, ?
 		/**
 		 * Completes the co-group operation with the user function that is executed
 		 * for windowed groups.
-		 * 
+		 *
 		 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 		 * Due to binary backwards compatibility, this cannot be altered. Use the {@link #with(CoGroupFunction)}
 		 * method to set an operator-specific parallelism.
@@ -248,7 +248,7 @@ public <T> DataStream<T> apply(CoGroupFunction<T1, T2, T> function) {
 		 * <p><b>Note:</b> This is a temporary workaround while the {@link #apply(CoGroupFunction)}
 		 * method has the wrong return type and hence does not allow one to set an operator-specific
 		 * parallelism
-		 * 
+		 *
 		 * @deprecated This method will be removed once the {@link #apply(CoGroupFunction)} method is fixed
 		 *             in the next major version of Flink (2.0).
 		 */
@@ -261,7 +261,7 @@ public <T> SingleOutputStreamOperator<T> with(CoGroupFunction<T1, T2, T> functio
 		/**
 		 * Completes the co-group operation with the user function that is executed
 		 * for windowed groups.
-		 * 
+		 *
 		 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 		 * Due to binary backwards compatibility, this cannot be altered. Use the
 		 * {@link #with(CoGroupFunction, TypeInformation)} method to set an operator-specific parallelism.
@@ -272,7 +272,7 @@ public <T> DataStream<T> apply(CoGroupFunction<T1, T2, T> function, TypeInformat
 
 			UnionTypeInfo<T1, T2> unionType = new UnionTypeInfo<>(input1.getType(), input2.getType());
 			UnionKeySelector<T1, T2, KEY> unionKeySelector = new UnionKeySelector<>(keySelector1, keySelector2);
-			
+
 			DataStream<TaggedUnion<T1, T2>> taggedInput1 = input1
 					.map(new Input1Tagger<T1, T2>())
 					.setParallelism(input1.getParallelism())
@@ -283,9 +283,9 @@ public <T> DataStream<T> apply(CoGroupFunction<T1, T2, T> function, TypeInformat
 					.returns(unionType);
 
 			DataStream<TaggedUnion<T1, T2>> unionStream = taggedInput1.union(taggedInput2);
-			
+
 			// we explicitly create the keyed stream to manually pass the key type information in
-			WindowedStream<TaggedUnion<T1, T2>, KEY, W> windowOp = 
+			WindowedStream<TaggedUnion<T1, T2>, KEY, W> windowOp =
 					new KeyedStream<TaggedUnion<T1, T2>, KEY>(unionStream, unionKeySelector, keyType)
 					.window(windowAssigner);
 
@@ -320,7 +320,7 @@ public <T> SingleOutputStreamOperator<T> with(CoGroupFunction<T1, T2, T> functio
 	// ------------------------------------------------------------------------
 	//  Data type and type information for Tagged Union
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Internal class for implementing tagged union co-group.
 	 */
@@ -554,7 +554,7 @@ public boolean canEqual(Object obj) {
 	//  Utility functions that implement the CoGroup logic based on the tagged
 	//  untion window reduce
 	// ------------------------------------------------------------------------
-	
+
 	private static class Input1Tagger<T1, T2> implements MapFunction<T1, TaggedUnion<T1, T2>> {
 		private static final long serialVersionUID = 1L;
 
@@ -598,7 +598,7 @@ public KEY getKey(TaggedUnion<T1, T2> value) throws Exception{
 	private static class CoGroupWindowFunction<T1, T2, T, KEY, W extends Window>
 			extends WrappingFunction<CoGroupFunction<T1, T2, T>>
 			implements WindowFunction<TaggedUnion<T1, T2>, T, KEY, W> {
-		
+
 		private static final long serialVersionUID = 1L;
 
 		public CoGroupWindowFunction(CoGroupFunction<T1, T2, T> userFunction) {
@@ -610,10 +610,10 @@ public void apply(KEY key,
 				W window,
 				Iterable<TaggedUnion<T1, T2>> values,
 				Collector<T> out) throws Exception {
-			
+
 			List<T1> oneValues = new ArrayList<>();
 			List<T2> twoValues = new ArrayList<>();
-			
+
 			for (TaggedUnion<T1, T2> val: values) {
 				if (val.isOne()) {
 					oneValues.add(val.getOne());
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
index e884140383f64..b45881a13661a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
@@ -44,13 +44,13 @@
  *
  * <p>An example for the use of connected streams would be to apply rules that change over time
  * onto another stream. One of the connected streams has the rules, the other stream the
- * elements to apply the rules to. The operation on the connected stream maintains the 
+ * elements to apply the rules to. The operation on the connected stream maintains the
  * current set of rules in the state. It may receive either a rule update and update the state
  * or a data element and apply the rules in the state to the element.
  *
  * <p>The connected stream can be conceptually viewed as a union stream of an Either type, that
  * holds either the first stream's type or the second stream's type.
- * 
+ *
  * @param <IN1> Type of the first input data steam.
  * @param <IN2> Type of the second input data stream.
  */
@@ -197,7 +197,7 @@ public ConnectedStreams<IN1, IN2> keyBy(KeySelector<IN1, ?> keySelector1, KeySel
 	 * {@link CoMapFunction#map1} for each element of the first input and
 	 * {@link CoMapFunction#map2} for each element of the second input. Each
 	 * CoMapFunction call returns exactly one element.
-	 * 
+	 *
 	 * @param coMapper The CoMapFunction used to jointly transform the two input DataStreams
 	 * @return The transformed {@link DataStream}
 	 */
@@ -218,7 +218,7 @@ CoMapFunction.class, false, true, getType1(), getType2(),
 	 * and {@link CoFlatMapFunction#flatMap2} for each element of the second
 	 * input. Each CoFlatMapFunction call returns any number of elements
 	 * including none.
-	 * 
+	 *
 	 * @param coFlatMapper
 	 *            The CoFlatMapFunction used to jointly transform the two input
 	 *            DataStreams
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 43a1aead44ddd..e73e4e7652df8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -217,7 +217,7 @@ public final DataStream<T> union(DataStream<T>... streams) {
 				throw new IllegalArgumentException("Cannot union streams of different types: "
 						+ getType() + " and " + newStream.getType());
 			}
-			
+
 			unionedTransforms.add(newStream.getTransformation());
 		}
 		return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms));
@@ -254,7 +254,7 @@ public <R> ConnectedStreams<T, R> connect(DataStream<R> dataStream) {
 
 	/**
 	 * It creates a new {@link KeyedStream} that uses the provided key for partitioning
-	 * its operator states. 
+	 * its operator states.
 	 *
 	 * @param key
 	 *            The KeySelector to be used for extracting the key for partitioning
@@ -265,7 +265,7 @@ public <K> KeyedStream<T, K> keyBy(KeySelector<T, K> key) {
 	}
 
 	/**
-	 * Partitions the operator state of a {@link DataStream} by the given key positions. 
+	 * Partitions the operator state of a {@link DataStream} by the given key positions.
 	 *
 	 * @param fields
 	 *            The position of the fields on which the {@link DataStream}
@@ -765,7 +765,7 @@ public <W extends Window> AllWindowedStream<T, W> windowAll(WindowAssigner<? sup
 	// ------------------------------------------------------------------------
 	//  Timestamps and watermarks
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Extracts a timestamp from an element and assigns it as the internal timestamp of that element.
 	 * The internal timestamps are, for example, used to to event-time window operations.
@@ -776,7 +776,7 @@ public <W extends Window> AllWindowedStream<T, W> windowAll(WindowAssigner<? sup
 	 * {@link TimestampExtractor#getCurrentWatermark()} to keep track of watermarks.
 	 *
 	 * @param extractor The TimestampExtractor that is called for each element of the DataStream.
-	 * 
+	 *
 	 * @deprecated Please use {@link #assignTimestampsAndWatermarks(AssignerWithPeriodicWatermarks)}
 	 *             of {@link #assignTimestampsAndWatermarks(AssignerWithPunctuatedWatermarks)}
 	 *             instead.
@@ -797,13 +797,13 @@ public SingleOutputStreamOperator<T> assignTimestamps(TimestampExtractor<T> extr
 	/**
 	 * Assigns timestamps to the elements in the data stream and periodically creates
 	 * watermarks to signal event time progress.
-	 * 
+	 *
 	 * <p>This method creates watermarks periodically (for example every second), based
 	 * on the watermarks indicated by the given watermark generator. Even when no new elements
 	 * in the stream arrive, the given watermark generator will be periodically checked for
 	 * new watermarks. The interval in which watermarks are generated is defined in
 	 * {@link ExecutionConfig#setAutoWatermarkInterval(long)}.
-	 * 
+	 *
 	 * <p>Use this method for the common cases, where some characteristic over all elements
 	 * should generate the watermarks, or where watermarks are simply trailing behind the
 	 * wall clock time by a certain amount.
@@ -812,31 +812,31 @@ public SingleOutputStreamOperator<T> assignTimestamps(TimestampExtractor<T> extr
 	 * timestamp seen so far in the elements of the stream by a fixed amount of time, and this
 	 * amount is known in advance, use the
 	 * {@link BoundedOutOfOrdernessTimestampExtractor}.
-	 * 
+	 *
 	 * <p>For cases where watermarks should be created in an irregular fashion, for example
 	 * based on certain markers that some element carry, use the
 	 * {@link AssignerWithPunctuatedWatermarks}.
-	 * 
+	 *
 	 * @param timestampAndWatermarkAssigner The implementation of the timestamp assigner and
-	 *                                      watermark generator.   
+	 *                                      watermark generator.
 	 * @return The stream after the transformation, with assigned timestamps and watermarks.
-	 * 
+	 *
 	 * @see AssignerWithPeriodicWatermarks
 	 * @see AssignerWithPunctuatedWatermarks
-	 * @see #assignTimestampsAndWatermarks(AssignerWithPunctuatedWatermarks) 
+	 * @see #assignTimestampsAndWatermarks(AssignerWithPunctuatedWatermarks)
 	 */
 	public SingleOutputStreamOperator<T> assignTimestampsAndWatermarks(
 			AssignerWithPeriodicWatermarks<T> timestampAndWatermarkAssigner) {
-		
+
 		// match parallelism to input, otherwise dop=1 sources could lead to some strange
 		// behaviour: the watermark will creep along very slowly because the elements
 		// from the source go to each extraction operator round robin.
 		final int inputParallelism = getTransformation().getParallelism();
 		final AssignerWithPeriodicWatermarks<T> cleanedAssigner = clean(timestampAndWatermarkAssigner);
-		
-		TimestampsAndPeriodicWatermarksOperator<T> operator = 
+
+		TimestampsAndPeriodicWatermarksOperator<T> operator =
 				new TimestampsAndPeriodicWatermarksOperator<>(cleanedAssigner);
-		
+
 		return transform("Timestamps/Watermarks", getTransformation().getOutputType(), operator)
 				.setParallelism(inputParallelism);
 	}
@@ -852,7 +852,7 @@ public SingleOutputStreamOperator<T> assignTimestampsAndWatermarks(
 	 * non-negative and greater than the previous watermark.
 	 *
 	 * <p>This method is useful when the data stream embeds watermark elements, or certain elements
-	 * carry a marker that can be used to determine the current event time watermark. 
+	 * carry a marker that can be used to determine the current event time watermark.
 	 * This operation gives the programmer full control over the watermark generation. Users
 	 * should be aware that too aggressive watermark generation (i.e., generating hundreds of
 	 * watermarks every second) can cost some performance.
@@ -861,7 +861,7 @@ public SingleOutputStreamOperator<T> assignTimestampsAndWatermarks(
 	 * every x milliseconds, use the {@link AssignerWithPeriodicWatermarks}.
 	 *
 	 * @param timestampAndWatermarkAssigner The implementation of the timestamp assigner and
-	 *                                      watermark generator.   
+	 *                                      watermark generator.
 	 * @return The stream after the transformation, with assigned timestamps and watermarks.
 	 *
 	 * @see AssignerWithPunctuatedWatermarks
@@ -870,16 +870,16 @@ public SingleOutputStreamOperator<T> assignTimestampsAndWatermarks(
 	 */
 	public SingleOutputStreamOperator<T> assignTimestampsAndWatermarks(
 			AssignerWithPunctuatedWatermarks<T> timestampAndWatermarkAssigner) {
-		
+
 		// match parallelism to input, otherwise dop=1 sources could lead to some strange
 		// behaviour: the watermark will creep along very slowly because the elements
 		// from the source go to each extraction operator round robin.
 		final int inputParallelism = getTransformation().getParallelism();
 		final AssignerWithPunctuatedWatermarks<T> cleanedAssigner = clean(timestampAndWatermarkAssigner);
 
-		TimestampsAndPunctuatedWatermarksOperator<T> operator = 
+		TimestampsAndPunctuatedWatermarksOperator<T> operator =
 				new TimestampsAndPunctuatedWatermarksOperator<>(cleanedAssigner);
-		
+
 		return transform("Timestamps/Watermarks", getTransformation().getOutputType(), operator)
 				.setParallelism(inputParallelism);
 	}
@@ -887,7 +887,7 @@ public SingleOutputStreamOperator<T> assignTimestampsAndWatermarks(
 	// ------------------------------------------------------------------------
 	//  Data sinks
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Writes a DataStream to the standard output stream (stdout).
 	 *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
index e824758969aa8..b8e71bc90adf7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
@@ -25,7 +25,7 @@
 
 /**
  * The DataStreamSource represents the starting point of a DataStream.
- * 
+ *
  * @param <T> Type of the elements in the DataStream created from the this source.
  */
 @Public
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
index c6426a68def74..ed87b1b51f0ad 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
@@ -31,7 +31,7 @@
 
 /**
  * The iterative data stream represents the start of an iteration in a {@link DataStream}.
- * 
+ *
  * @param <T> Type of the elements in this Stream
  */
 @PublicEvolving
@@ -40,7 +40,7 @@ public class IterativeStream<T> extends SingleOutputStreamOperator<T> {
 	// We store these so that we can create a co-iteration if we need to
 	private DataStream<T> originalInput;
 	private long maxWaitTime;
-	
+
 	protected IterativeStream(DataStream<T> dataStream, long maxWaitTime) {
 		super(dataStream.getExecutionEnvironment(),
 				new FeedbackTransformation<>(dataStream.getTransformation(), maxWaitTime));
@@ -57,13 +57,13 @@ protected IterativeStream(DataStream<T> dataStream, long maxWaitTime) {
 	 * splitting to send a part of the closing data stream to the head. Refer to
 	 * {@link DataStream#split(org.apache.flink.streaming.api.collector.selector.OutputSelector)}
 	 * for more information.
-	 * 
+	 *
 	 * @param feedbackStream
 	 *            {@link DataStream} that will be used as input to the iteration
 	 *            head.
 	 *
 	 * @return The feedback stream.
-	 * 
+	 *
 	 */
 	@SuppressWarnings({ "unchecked", "rawtypes" })
 	public DataStream<T> closeWith(DataStream<T> feedbackStream) {
@@ -86,7 +86,7 @@ public DataStream<T> closeWith(DataStream<T> feedbackStream) {
 	 * {@link ConnectedStreams}.
 	 *
 	 * <p>For type safety the user needs to define the feedback type
-	 * 
+	 *
 	 * @param feedbackTypeString
 	 *            String describing the type information of the feedback stream.
 	 * @return A {@link ConnectedIterativeStreams}.
@@ -101,7 +101,7 @@ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(String feedbackTypeS
 	 * {@link ConnectedStreams}.
 	 *
 	 * <p>For type safety the user needs to define the feedback type
-	 * 
+	 *
 	 * @param feedbackTypeClass
 	 *            Class of the elements in the feedback stream.
 	 * @return A {@link ConnectedIterativeStreams}.
@@ -116,7 +116,7 @@ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(Class<F> feedbackTyp
 	 * {@link ConnectedStreams}.
 	 *
 	 * <p>For type safety the user needs to define the feedback type
-	 * 
+	 *
 	 * @param feedbackType
 	 *            The type information of the feedback stream.
 	 * @return A {@link ConnectedIterativeStreams}.
@@ -124,7 +124,7 @@ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(Class<F> feedbackTyp
 	public <F> ConnectedIterativeStreams<T, F> withFeedbackType(TypeInformation<F> feedbackType) {
 		return new ConnectedIterativeStreams<>(originalInput, feedbackType, maxWaitTime);
 	}
-	
+
 	/**
 	 * The {@link ConnectedIterativeStreams} represent a start of an
 	 * iterative part of a streaming program, where the original input of the
@@ -134,7 +134,7 @@ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(TypeInformation<F> f
 	 * <p>The user can distinguish between the two inputs using co-transformation,
 	 * thus eliminating the need for mapping the inputs and outputs to a common
 	 * type.
-	 * 
+	 *
 	 * @param <I>
 	 *            Type of the input of the iteration
 	 * @param <F>
@@ -161,12 +161,12 @@ public ConnectedIterativeStreams(DataStream<I> input,
 		 * Closes the iteration. This method defines the end of the iterative
 		 * program part that will be fed back to the start of the iteration as
 		 * the second input in the {@link ConnectedStreams}.
-		 * 
+		 *
 		 * @param feedbackStream
 		 *            {@link DataStream} that will be used as second input to
 		 *            the iteration head.
 		 * @return The feedback stream.
-		 * 
+		 *
 		 */
 		public DataStream<F> closeWith(DataStream<F> feedbackStream) {
 
@@ -181,12 +181,12 @@ public DataStream<F> closeWith(DataStream<F> feedbackStream) {
 
 			return feedbackStream;
 		}
-		
+
 		private UnsupportedOperationException groupingException =
 				new UnsupportedOperationException("Cannot change the input partitioning of an" +
 						"iteration head directly. Apply the partitioning on the input and" +
 						"feedback streams instead.");
-		
+
 		@Override
 		public ConnectedStreams<I, F> keyBy(int[] keyPositions1, int[] keyPositions2) {throw groupingException;}
 
@@ -198,6 +198,6 @@ public DataStream<F> closeWith(DataStream<F> feedbackStream) {
 
 		@Override
 		public ConnectedStreams<I, F> keyBy(KeySelector<I, ?> keySelector1,KeySelector<F, ?> keySelector2) {throw groupingException;}
-		
+
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
index 40059cf385611..d02a2e2dad7fb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
@@ -139,7 +139,7 @@ public <W extends Window> WithWindow<T1, T2, KEY, W> window(WindowAssigner<? sup
 			}
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	/**
@@ -153,7 +153,7 @@ public <W extends Window> WithWindow<T1, T2, KEY, W> window(WindowAssigner<? sup
 	 */
 	@Public
 	public static class WithWindow<T1, T2, KEY, W extends Window> {
-		
+
 		private final DataStream<T1> input1;
 		private final DataStream<T2> input2;
 
@@ -176,16 +176,16 @@ protected WithWindow(DataStream<T1> input1,
 				WindowAssigner<? super TaggedUnion<T1, T2>, W> windowAssigner,
 				Trigger<? super TaggedUnion<T1, T2>, ? super W> trigger,
 				Evictor<? super TaggedUnion<T1, T2>, ? super W> evictor) {
-			
+
 			this.input1 = requireNonNull(input1);
 			this.input2 = requireNonNull(input2);
 
 			this.keySelector1 = requireNonNull(keySelector1);
 			this.keySelector2 = requireNonNull(keySelector2);
 			this.keyType = requireNonNull(keyType);
-			
+
 			this.windowAssigner = requireNonNull(windowAssigner);
-			
+
 			this.trigger = trigger;
 			this.evictor = evictor;
 		}
@@ -214,7 +214,7 @@ public WithWindow<T1, T2, KEY, W> evictor(Evictor<? super TaggedUnion<T1, T2>, ?
 		/**
 		 * Completes the join operation with the user function that is executed
 		 * for each combination of elements with the same key in a window.
-		 * 
+		 *
 		 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 		 * Due to binary backwards compatibility, this cannot be altered. Use the {@link #with(JoinFunction)}
 		 * method to set an operator-specific parallelism.
@@ -240,7 +240,7 @@ public <T> DataStream<T> apply(JoinFunction<T1, T2, T> function) {
 		 * <p><b>Note:</b> This is a temporary workaround while the {@link #apply(JoinFunction)}
 		 * method has the wrong return type and hence does not allow one to set an operator-specific
 		 * parallelism
-		 * 
+		 *
 		 * @deprecated This method will be removed once the {@link #apply(JoinFunction)} method is fixed
 		 *             in the next major version of Flink (2.0).
 		 */
@@ -253,7 +253,7 @@ public <T> SingleOutputStreamOperator<T> with(JoinFunction<T1, T2, T> function)
 		/**
 		 * Completes the join operation with the user function that is executed
 		 * for each combination of elements with the same key in a window.
-		 * 
+		 *
 		 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 		 * Due to binary backwards compatibility, this cannot be altered. Use the
 		 * {@link #with(JoinFunction, TypeInformation)}, method to set an operator-specific parallelism.
@@ -293,7 +293,7 @@ public <T> SingleOutputStreamOperator<T> with(FlatJoinFunction<T1, T2, T> functi
 		/**
 		 * Completes the join operation with the user function that is executed
 		 * for each combination of elements with the same key in a window.
-		 * 
+		 *
 		 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 		 * Due to binary backwards compatibility, this cannot be altered. Use the
 		 * {@link #with(FlatJoinFunction)}, method to set an operator-specific parallelism.
@@ -319,7 +319,7 @@ public <T> DataStream<T> apply(FlatJoinFunction<T1, T2, T> function) {
 		 * <p><b>Note:</b> This is a temporary workaround while the {@link #apply(FlatJoinFunction)}
 		 * method has the wrong return type and hence does not allow one to set an operator-specific
 		 * parallelism.
-		 * 
+		 *
 		 * @deprecated This method will be removed once the {@link #apply(FlatJoinFunction)}
 		 *             method is fixed in the next major version of Flink (2.0).
 		 */
@@ -332,7 +332,7 @@ public <T> SingleOutputStreamOperator<T> with(FlatJoinFunction<T1, T2, T> functi
 		/**
 		 * Completes the join operation with the user function that is executed
 		 * for each combination of elements with the same key in a window.
-		 * 
+		 *
 		 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 		 * Due to binary backwards compatibility, this cannot be altered. Use the
 		 * {@link #with(JoinFunction, TypeInformation)}, method to set an operator-specific parallelism.
@@ -358,7 +358,7 @@ public <T> DataStream<T> apply(JoinFunction<T1, T2, T> function, TypeInformation
 		 * <p><b>Note:</b> This is a temporary workaround while the {@link #apply(FlatJoinFunction, TypeInformation)}
 		 * method has the wrong return type and hence does not allow one to set an operator-specific
 		 * parallelism
-		 * 
+		 *
 		 * @deprecated This method will be removed once the {@link #apply(JoinFunction, TypeInformation)}
 		 *             method is fixed in the next major version of Flink (2.0).
 		 */
@@ -368,7 +368,7 @@ public <T> SingleOutputStreamOperator<T> with(JoinFunction<T1, T2, T> function,
 			return (SingleOutputStreamOperator<T>) apply(function, resultType);
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Implementation of the functions
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
index 27545c220a9cf..6f8fd07181282 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
@@ -95,11 +95,11 @@ public class KeyedStream<T, KEY> extends DataStream<T> {
 
 	/** The type of the key by which the stream is partitioned. */
 	private final TypeInformation<KEY> keyType;
-	
+
 	/**
 	 * Creates a new {@link KeyedStream} using the given {@link KeySelector}
 	 * to partition operator state by key.
-	 * 
+	 *
 	 * @param dataStream
 	 *            Base stream of data
 	 * @param keySelector
@@ -148,10 +148,10 @@ private TypeInformation<KEY> validateKeyType(TypeInformation<KEY> keyType) {
 			if (!validateKeyTypeIsHashable(typeInfo)) {
 				unsupportedTypes.add(typeInfo);
 			}
-			
+
 			if (typeInfo instanceof TupleTypeInfoBase) {
 				for (int i = 0; i < typeInfo.getArity(); i++) {
-					stack.push(((TupleTypeInfoBase) typeInfo).getTypeAt(i));	
+					stack.push(((TupleTypeInfoBase) typeInfo).getTypeAt(i));
 				}
 			}
 		}
@@ -205,7 +205,7 @@ public KeySelector<T, KEY> getKeySelector() {
 	}
 
 	/**
-	 * Gets the type of the key by which the stream is partitioned. 
+	 * Gets the type of the key by which the stream is partitioned.
 	 * @return The type of the key by which the stream is partitioned.
 	 */
 	@Internal
@@ -221,7 +221,7 @@ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) {
 	// ------------------------------------------------------------------------
 	//  basic transformations
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	@PublicEvolving
 	public <R> SingleOutputStreamOperator<R> transform(String operatorName,
@@ -233,10 +233,10 @@ public <R> SingleOutputStreamOperator<R> transform(String operatorName,
 		OneInputTransformation<T, R> transform = (OneInputTransformation<T, R>) returnStream.getTransformation();
 		transform.setStateKeySelector(keySelector);
 		transform.setStateKeyType(keyType);
-		
+
 		return returnStream;
 	}
-	
+
 	@Override
 	public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {
 		DataStreamSink<T> result = super.addSink(sinkFunction);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 5f063ded12100..d540a9fcec0fa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -230,7 +230,7 @@ public SingleOutputStreamOperator<T> forceNonParallel() {
 	/**
 	 * Sets the maximum time frequency (ms) for the flushing of the output
 	 * buffer. By default the output buffers flush only when they are full.
-	 * 
+	 *
 	 * @param timeoutMillis
 	 *            The maximum time between two output flushes.
 	 * @return The operator with buffer timeout set.
@@ -244,7 +244,7 @@ public SingleOutputStreamOperator<T> setBufferTimeout(long timeoutMillis) {
 	 * Sets the {@link ChainingStrategy} for the given operator affecting the
 	 * way operators will possibly be co-located on the same thread for
 	 * increased performance.
-	 * 
+	 *
 	 * @param strategy
 	 *            The selected {@link ChainingStrategy}
 	 * @return The operator with the modified chaining strategy
@@ -262,7 +262,7 @@ private SingleOutputStreamOperator<T> setChainingStrategy(ChainingStrategy strat
 	 * <p>Chaining can be turned off for the whole job by
 	 * {@link StreamExecutionEnvironment#disableOperatorChaining()} however it is not advised for
 	 * performance considerations.
-	 * 
+	 *
 	 * @return The operator with chaining disabled
 	 */
 	@PublicEvolving
@@ -274,7 +274,7 @@ public SingleOutputStreamOperator<T> disableChaining() {
 	 * Starts a new task chain beginning at this operator. This operator will
 	 * not be chained (thread co-located for increased performance) to any
 	 * previous tasks even if possible.
-	 * 
+	 *
 	 * @return The operator with chaining set.
 	 */
 	@PublicEvolving
@@ -320,7 +320,7 @@ public SingleOutputStreamOperator<T> returns(Class<T> typeClass) {
 	 *
 	 * <p>Use this method the following way:
 	 * <pre>{@code
-	 *     DataStream<Tuple2<String, Double>> result = 
+	 *     DataStream<Tuple2<String, Double>> result =
 	 *         stream.flatMap(new FunctionWithNonInferrableReturnType())
 	 *               .returns(new TypeHint<Tuple2<String, Double>>(){});
 	 * }</pre>
@@ -354,17 +354,17 @@ public SingleOutputStreamOperator<T> returns(TypeHint<T> typeHint) {
 	 */
 	public SingleOutputStreamOperator<T> returns(TypeInformation<T> typeInfo) {
 		requireNonNull(typeInfo, "TypeInformation must not be null");
-		
+
 		transformation.setOutputType(typeInfo);
 		return this;
 	}
-	
+
 	/**
-	 * Adds a type information hint about the return type of this operator. 
-	 * 
+	 * Adds a type information hint about the return type of this operator.
+	 *
 	 * <p>Type hints are important in cases where the Java compiler throws away generic type
 	 * information necessary for efficient execution.
-	 * 
+	 *
 	 * <p>This method takes a type information string that will be parsed. A type information string
 	 * can contain the following types:
 	 *
@@ -391,7 +391,7 @@ public SingleOutputStreamOperator<T> returns(TypeInformation<T> typeInfo) {
 	 * @param typeInfoString
 	 *            type information string to be parsed
 	 * @return This operator with a given return type hint.
-	 * 
+	 *
 	 * @deprecated Please use {@link #returns(Class)} or {@link #returns(TypeHint)} instead.
 	 */
 	@Deprecated
@@ -402,7 +402,7 @@ public SingleOutputStreamOperator<T> returns(String typeInfoString) {
 		}
 		return returns(TypeInfoParser.<T>parse(typeInfoString));
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Miscellaneous
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java
index 50c3aa9ff4c9e..9fcd2fcc8685b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java
@@ -41,7 +41,7 @@ protected SplitStream(DataStream<OUT> dataStream, OutputSelector<OUT> outputSele
 
 	/**
 	 * Sets the output names for which the next operator will receive values.
-	 * 
+	 *
 	 * @param outputNames
 	 *            The output names for which the operator will receive the
 	 *            input.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
index 1cb48a26a1292..da65147c00b91 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
@@ -74,16 +74,16 @@ public class CheckpointConfig implements java.io.Serializable {
 
 	/**
 	 * Checks whether checkpointing is enabled.
-	 * 
+	 *
 	 * @return True if checkpointing is enables, false otherwise.
 	 */
 	public boolean isCheckpointingEnabled() {
 		return checkpointInterval > 0;
 	}
-	
+
 	/**
 	 * Gets the checkpointing mode (exactly-once vs. at-least-once).
-	 * 
+	 *
 	 * @return The checkpointing mode.
 	 */
 	public CheckpointingMode getCheckpointingMode() {
@@ -92,7 +92,7 @@ public CheckpointingMode getCheckpointingMode() {
 
 	/**
 	 * Sets the checkpointing mode (exactly-once vs. at-least-once).
-	 * 
+	 *
 	 * @param checkpointingMode The checkpointing mode.
 	 */
 	public void setCheckpointingMode(CheckpointingMode checkpointingMode) {
@@ -101,10 +101,10 @@ public void setCheckpointingMode(CheckpointingMode checkpointingMode) {
 
 	/**
 	 * Gets the interval in which checkpoints are periodically scheduled.
-	 * 
+	 *
 	 * <p>This setting defines the base interval. Checkpoint triggering may be delayed by the settings
 	 * {@link #getMaxConcurrentCheckpoints()} and {@link #getMinPauseBetweenCheckpoints()}.
-	 * 
+	 *
 	 * @return The checkpoint interval, in milliseconds.
 	 */
 	public long getCheckpointInterval() {
@@ -128,7 +128,7 @@ public void setCheckpointInterval(long checkpointInterval) {
 
 	/**
 	 * Gets the maximum time that a checkpoint may take before being discarded.
-	 * 
+	 *
 	 * @return The checkpoint timeout, in milliseconds.
 	 */
 	public long getCheckpointTimeout() {
@@ -137,7 +137,7 @@ public long getCheckpointTimeout() {
 
 	/**
 	 * Sets the maximum time that a checkpoint may take before being discarded.
-	 * 
+	 *
 	 * @param checkpointTimeout The checkpoint timeout, in milliseconds.
 	 */
 	public void setCheckpointTimeout(long checkpointTimeout) {
@@ -164,10 +164,10 @@ public long getMinPauseBetweenCheckpoints() {
 	 * checkpoint coordinator may trigger another checkpoint after it becomes possible to trigger
 	 * another checkpoint with respect to the maximum number of concurrent checkpoints
 	 * (see {@link #setMaxConcurrentCheckpoints(int)}).
-	 * 
+	 *
 	 * <p>If the maximum number of concurrent checkpoints is set to one, this setting makes effectively sure
 	 * that a minimum amount of time passes where no checkpoint is in progress at all.
-	 * 
+	 *
 	 * @param minPauseBetweenCheckpoints The minimal pause before the next checkpoint is triggered.
 	 */
 	public void setMinPauseBetweenCheckpoints(long minPauseBetweenCheckpoints) {
@@ -182,7 +182,7 @@ public void setMinPauseBetweenCheckpoints(long minPauseBetweenCheckpoints) {
 	 * value is <i>n</i>, then no checkpoints will be triggered while <i>n</i> checkpoint attempts are
 	 * currently in flight. For the next checkpoint to be triggered, one checkpoint attempt would need
 	 * to finish or expire.
-	 * 
+	 *
 	 * @return The maximum number of concurrent checkpoint attempts.
 	 */
 	public int getMaxConcurrentCheckpoints() {
@@ -194,7 +194,7 @@ public int getMaxConcurrentCheckpoints() {
 	 * value is <i>n</i>, then no checkpoints will be triggered while <i>n</i> checkpoint attempts are
 	 * currently in flight. For the next checkpoint to be triggered, one checkpoint attempt would need
 	 * to finish or expire.
-	 * 
+	 *
 	 * @param maxConcurrentCheckpoints The maximum number of concurrent checkpoint attempts.
 	 */
 	public void setMaxConcurrentCheckpoints(int maxConcurrentCheckpoints) {
@@ -206,9 +206,9 @@ public void setMaxConcurrentCheckpoints(int maxConcurrentCheckpoints) {
 
 	/**
 	 * Checks whether checkpointing is forced, despite currently non-checkpointable iteration feedback.
-	 * 
+	 *
 	 * @return True, if checkpointing is forced, false otherwise.
-	 * 
+	 *
 	 * @deprecated This will be removed once iterations properly participate in checkpointing.
 	 */
 	@Deprecated
@@ -219,9 +219,9 @@ public boolean isForceCheckpointing() {
 
 	/**
 	 * Checks whether checkpointing is forced, despite currently non-checkpointable iteration feedback.
-	 * 
-	 * @param forceCheckpointing The flag to force checkpointing. 
-	 * 
+	 *
+	 * @param forceCheckpointing The flag to force checkpointing.
+	 *
 	 * @deprecated This will be removed once iterations properly participate in checkpointing.
 	 */
 	@Deprecated
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
index 0a4209fdb4a58..cebd15ff0427a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
@@ -75,7 +75,7 @@ public Flip6LocalStreamEnvironment(Configuration config) {
 	/**
 	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
 	 * specified name.
-	 * 
+	 *
 	 * @param jobName
 	 *            name of the job
 	 * @return The result of the job execution, containing elapsed time and accumulators.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
index 584955997dd63..b4031fa4b4c9d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
@@ -47,7 +47,7 @@
 public class LocalStreamEnvironment extends StreamExecutionEnvironment {
 
 	private static final Logger LOG = LoggerFactory.getLogger(LocalStreamEnvironment.class);
-	
+
 	/** The configuration to use for the local cluster. */
 	private final Configuration conf;
 
@@ -69,14 +69,14 @@ public LocalStreamEnvironment(Configuration config) {
 					"The LocalStreamEnvironment cannot be used when submitting a program through a client, " +
 							"or running in a TestEnvironment context.");
 		}
-		
+
 		this.conf = config == null ? new Configuration() : config;
 	}
 
 	/**
 	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
 	 * specified name.
-	 * 
+	 *
 	 * @param jobName
 	 *            name of the job
 	 * @return The result of the job execution, containing elapsed time and accumulators.
@@ -94,10 +94,10 @@ public JobExecutionResult execute(String jobName) throws Exception {
 
 		configuration.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, -1L);
 		configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());
-		
+
 		// add (and override) the settings with what the user defined
 		configuration.addAll(this.conf);
-		
+
 		if (LOG.isInfoEnabled()) {
 			LOG.info("Running job on local embedded Flink mini cluster");
 		}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
index 28a06756f28ec..ff5af87b634cb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
@@ -46,7 +46,7 @@
  */
 @Public
 public class RemoteStreamEnvironment extends StreamExecutionEnvironment {
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(RemoteStreamEnvironment.class);
 
 	/** The hostname of the JobManager. */
@@ -60,14 +60,14 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment {
 
 	/** The jar files that need to be attached to each job. */
 	private final List<URL> jarFiles;
-	
+
 	/** The classpaths that need to be attached to each job. */
 	private final List<URL> globalClasspaths;
 
 	/**
 	 * Creates a new RemoteStreamEnvironment that points to the master
 	 * (JobManager) described by the given host name and port.
-	 * 
+	 *
 	 * @param host
 	 *            The host name or address of the master (JobManager), where the
 	 *            program should be executed.
@@ -125,9 +125,9 @@ public RemoteStreamEnvironment(String host, int port, Configuration clientConfig
 	 *            cluster. If the program uses user-defined functions,
 	 *            user-defined input formats, or any libraries, those must be
 	 *            provided in the JAR files.
-	 * @param globalClasspaths 
-	 *            The paths of directories and JAR files that are added to each user code 
-	 *            classloader on all nodes in the cluster. Note that the paths must specify a 
+	 * @param globalClasspaths
+	 *            The paths of directories and JAR files that are added to each user code
+	 *            classloader on all nodes in the cluster. Note that the paths must specify a
 	 *            protocol (e.g. file://) and be accessible on all nodes (e.g. by means of a NFS share).
 	 *            The protocol must be supported by the {@link java.net.URLClassLoader}.
 	 */
@@ -137,7 +137,7 @@ public RemoteStreamEnvironment(String host, int port, Configuration clientConfig
 					"The RemoteEnvironment cannot be used when submitting a program through a client, " +
 							"or running in a TestEnvironment context.");
 		}
-		
+
 		if (host == null) {
 			throw new NullPointerException("Host must not be null.");
 		}
@@ -178,7 +178,7 @@ public JobExecutionResult execute(String jobName) throws ProgramInvocationExcept
 
 	/**
 	 * Executes the remote job.
-	 * 
+	 *
 	 * @param streamGraph
 	 *            Stream Graph to execute
 	 * @param jarFiles
@@ -192,10 +192,10 @@ protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL>
 
 		ClassLoader usercodeClassLoader = JobWithJars.buildUserCodeClassLoader(jarFiles, globalClasspaths,
 			getClass().getClassLoader());
-		
+
 		Configuration configuration = new Configuration();
 		configuration.addAll(this.clientConfiguration);
-		
+
 		configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, host);
 		configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, port);
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 354eb4ab0f19c..3f73eb231183a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -1639,11 +1639,11 @@ public static LocalStreamEnvironment createLocalEnvironment(int parallelism, Con
 	/**
 	 * Creates a {@link LocalStreamEnvironment} for local program execution that also starts the
 	 * web monitoring UI.
-	 * 
+	 *
 	 * <p>The local execution environment will run the program in a multi-threaded fashion in
 	 * the same JVM as the environment was created in. It will use the parallelism specified in the
 	 * parameter.
-	 * 
+	 *
 	 * <p>If the configuration key 'jobmanager.web.port' was set in the configuration, that particular
 	 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 	 */
@@ -1773,7 +1773,7 @@ public static void setDefaultLocalParallelism(int parallelism) {
 	protected static void initializeContextEnvironment(StreamExecutionEnvironmentFactory ctx) {
 		contextEnvironmentFactory = ctx;
 	}
-	
+
 	protected static void resetContextEnvironment() {
 		contextEnvironmentFactory = null;
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java
index 4388c9d0e82db..823a5c3442be0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java
@@ -28,7 +28,7 @@
  * and generates low watermarks that signal event time progress within the stream.
  * These timestamps and watermarks are used by functions and operators that operate
  * on event time, for example event time windows.
- * 
+ *
  * <p>Use this class to generate watermarks in a periodical interval.
  * At most every {@code i} milliseconds (configured via
  * {@link ExecutionConfig#getAutoWatermarkInterval()}), the system will call the
@@ -45,9 +45,9 @@
  * milliseconds since the Epoch (midnight, January 1, 1970 UTC).
  * A watermark with a certain value {@code t} indicates that no elements with event
  * timestamps {@code x}, where {@code x} is lower or equal to {@code t}, will occur any more.
- * 
+ *
  * @param <T> The type of the elements to which this assigner assigns timestamps.
- * 
+ *
  * @see org.apache.flink.streaming.api.watermark.Watermark
  */
 public interface AssignerWithPeriodicWatermarks<T> extends TimestampAssigner<T> {
@@ -56,7 +56,7 @@ public interface AssignerWithPeriodicWatermarks<T> extends TimestampAssigner<T>
 	 * Returns the current watermark. This method is periodically called by the
 	 * system to retrieve the current watermark. The method may return {@code null} to
 	 * indicate that no new Watermark is available.
-	 * 
+	 *
 	 * <p>The returned watermark will be emitted only if it is non-null and its timestamp
 	 * is larger than that of the previously emitted watermark (to preserve the contract of
 	 * ascending watermarks). If the current watermark is still
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java
index 5b5694c489dff..a69a8394965e5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java
@@ -33,7 +33,7 @@
  * The system will generate a new watermark, if the probed value is non-null
  * and has a timestamp larger than that of the previous watermark (to preserve
  * the contract of ascending watermarks).
- * 
+ *
  * <p>For use cases that should periodically emit watermarks based on element timestamps,
  * use the {@link AssignerWithPeriodicWatermarks} instead.
  *
@@ -41,7 +41,7 @@
  * generator. It assumes elements carry a timestamp that describes when they were created,
  * and that some elements carry a flag, marking them as the end of a sequence such that no
  * elements with smaller timestamps can come anymore.
- * 
+ *
  * <pre>{@code
  * public class WatermarkOnFlagAssigner implements AssignerWithPunctuatedWatermarks<MyElement> {
  *
@@ -54,18 +54,18 @@
  *     }
  * }
  * }</pre>
- * 
+ *
  * <p>Timestamps and watermarks are defined as {@code longs} that represent the
  * milliseconds since the Epoch (midnight, January 1, 1970 UTC).
  * A watermark with a certain value {@code t} indicates that no elements with event
  * timestamps {@code x}, where {@code x} is lower or equal to {@code t}, will occur any more.
- * 
+ *
  * @param <T> The type of the elements to which this assigner assigns timestamps.
- * 
+ *
  * @see org.apache.flink.streaming.api.watermark.Watermark
  */
 public interface AssignerWithPunctuatedWatermarks<T> extends TimestampAssigner<T> {
-	
+
 	/**
 	 * Asks this implementation if it wants to emit a watermark. This method is called right after
 	 * the {@link #extractTimestamp(Object, long)} method.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractor.java
index c7c60d5b56cdb..0b18c1c41d703 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractor.java
@@ -22,16 +22,16 @@
 
 /**
  * A timestamp assigner that assigns timestamps based on the machine's wall clock.
- * 
+ *
  * <p>If this assigner is used after a stream source, it realizes "ingestion time" semantics.
- * 
+ *
  * @param <T> The elements that get timestamps assigned.
  */
 public class IngestionTimeExtractor<T> implements AssignerWithPeriodicWatermarks<T> {
 	private static final long serialVersionUID = -4072216356049069301L;
-	
+
 	private long maxTimestamp;
-	
+
 	@Override
 	public long extractTimestamp(T element, long previousElementTimestamp) {
 		// make sure timestamps are monotonously increasing, even when the system clock re-syncs
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/ProcessFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/ProcessFunction.java
index 43683eff9e224..c2c130ef58da4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/ProcessFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/ProcessFunction.java
@@ -105,7 +105,7 @@ public abstract class Context {
 
 		/**
 		 * Emits a record to the side output identified by the {@link OutputTag}.
-		 * 
+		 *
 		 * @param outputTag the {@code OutputTag} that identifies the side output to emit to.
 		 * @param value The record to emit.
 		 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java
index 5167fc898c031..60debb966d07e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java
@@ -24,25 +24,25 @@
  * A {@code TimestampAssigner} assigns event time timestamps to elements.
  * These timestamps are used by all functions that operate on event time,
  * for example event time windows.
- * 
+ *
  * <p>Timestamps are represented in milliseconds since the Epoch
  * (midnight, January 1, 1970 UTC).
- * 
+ *
  * @param <T> The type of the elements to which this assigner assigns timestamps.
  */
 public interface TimestampAssigner<T> extends Function {
 
 	/**
 	 * Assigns a timestamp to an element, in milliseconds since the Epoch.
-	 * 
+	 *
 	 * <p>The method is passed the previously assigned timestamp of the element.
 	 * That previous timestamp may have been assigned from a previous assigner,
 	 * by ingestion time. If the element did not carry a timestamp before, this value is
-	 * {@code Long.MIN_VALUE}. 
+	 * {@code Long.MIN_VALUE}.
 	 *
 	 * @param element The element that the timestamp is wil be assigned to.
 	 * @param previousElementTimestamp The previous internal timestamp of the element,
-	 *                                 or a negative value, if no timestamp has been assigned, yet.   
+	 *                                 or a negative value, if no timestamp has been assigned, yet.
 	 * @return The new timestamp.
 	 */
 	long extractTimestamp(T element, long previousElementTimestamp);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java
index 2dc909160ec0a..c71e10b471459 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampExtractor.java
@@ -35,7 +35,7 @@
  *
  * @deprecated This class has been replaced by {@link AssignerWithPeriodicWatermarks} and
  *             {@link AssignerWithPunctuatedWatermarks}
- *             
+ *
  * @see AssignerWithPeriodicWatermarks
  * @see AssignerWithPunctuatedWatermarks
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java
index 3216e3a711163..a2a84fe2a3ce2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/ComparableAggregator.java
@@ -36,7 +36,7 @@ public class ComparableAggregator<T> extends AggregationFunction<T> {
 	private boolean byAggregate;
 	private boolean first;
 	private final FieldAccessor<T, Object> fieldAccessor;
-	
+
 	private ComparableAggregator(AggregationType aggregationType, FieldAccessor<T, Object> fieldAccessor, boolean first) {
 		this.comparator = Comparator.getForAggregation(aggregationType);
 		this.byAggregate = (aggregationType == AggregationType.MAXBY) || (aggregationType == AggregationType.MINBY);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java
index 419c3ee170870..5bb4459ac11e8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java
@@ -26,21 +26,21 @@
 
 /**
  * A function to trigger Async I/O operation.
- * 
+ *
  * <p>For each #asyncInvoke, an async io operation can be triggered, and once it has been done,
  * the result can be collected by calling {@link AsyncCollector#collect}. For each async
  * operation, its context is stored in the operator immediately after invoking
  * #asyncInvoke, avoiding blocking for each stream input as long as the internal buffer is not full.
- * 
+ *
  * <p>{@link AsyncCollector} can be passed into callbacks or futures to collect the result data.
  * An error can also be propagate to the async IO operator by
  * {@link AsyncCollector#collect(Throwable)}.
  *
  * <p>Callback example usage:
- * 
+ *
  * <pre>{@code
  * public class HBaseAsyncFunc implements AsyncFunction<String, String> {
- * 
+ *
  *   public void asyncInvoke(String row, AsyncCollector<String> collector) throws Exception {
  *     HBaseCallback cb = new HBaseCallback(collector);
  *     Get get = new Get(Bytes.toBytes(row));
@@ -50,10 +50,10 @@
  * }</pre>
  *
  * <p>Future example usage:
- * 
+ *
  * <pre>{@code
  * public class HBaseAsyncFunc implements AsyncFunction<String, String> {
- * 
+ *
  *   public void asyncInvoke(String row, final AsyncCollector<String> collector) throws Exception {
  *     Get get = new Get(Bytes.toBytes(row));
  *     ListenableFuture<Result> future = hbase.asyncGet(get);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoFlatMapFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoFlatMapFunction.java
index 3f996f5cc6e83..03bfcbdedba83 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoFlatMapFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoFlatMapFunction.java
@@ -27,14 +27,14 @@
 /**
  * A CoFlatMapFunction implements a flat-map transformation over two
  * connected streams.
- * 
+ *
  * <p>The same instance of the transformation function is used to transform
  * both of the connected streams. That way, the stream transformations can
  * share state.
- * 
+ *
  * <p>An example for the use of connected streams would be to apply rules that change over time
  * onto elements of a stream. One of the connected streams has the rules, the other stream the
- * elements to apply the rules to. The operation on the connected stream maintains the 
+ * elements to apply the rules to. The operation on the connected stream maintains the
  * current set of rules in the state. It may receive either a rule update (from the first stream)
  * and update the state, or a data element (from the second stream) and apply the rules in the
  * state to the element. The result of applying the rules would be emitted.
@@ -48,7 +48,7 @@ public interface CoFlatMapFunction<IN1, IN2, OUT> extends Function, Serializable
 
 	/**
 	 * This method is called for each element in the first of the connected streams.
-	 * 
+	 *
 	 * @param value The stream element
 	 * @param out The collector to emit resulting elements to
 	 * @throws Exception The function may throw exceptions which cause the streaming program
@@ -58,7 +58,7 @@ public interface CoFlatMapFunction<IN1, IN2, OUT> extends Function, Serializable
 
 	/**
 	 * This method is called for each element in the second of the connected streams.
-	 * 
+	 *
 	 * @param value The stream element
 	 * @param out The collector to emit resulting elements to
 	 * @throws Exception The function may throw exceptions which cause the streaming program
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java
index 4285cc3ec0498..89a52d27f755d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/CoProcessFunction.java
@@ -53,7 +53,7 @@ public abstract class CoProcessFunction<IN1, IN2, OUT> extends AbstractRichFunct
 	 *
 	 * <p>This function can output zero or more elements using the {@link Collector} parameter
 	 * and also update internal state or set timers using the {@link Context} parameter.
-	 * 
+	 *
 	 * @param value The stream element
 	 * @param ctx A {@link Context} that allows querying the timestamp of the element,
 	 *            querying the {@link TimeDomain} of the firing timer and getting a
@@ -70,7 +70,7 @@ public abstract class CoProcessFunction<IN1, IN2, OUT> extends AbstractRichFunct
 	 *
 	 * <p>This function can output zero or more elements using the {@link Collector} parameter
 	 * and also update internal state or set timers using the {@link Context} parameter.
-	 * 
+	 *
 	 * @param value The stream element
 	 * @param ctx A {@link Context} that allows querying the timestamp of the element,
 	 *            querying the {@link TimeDomain} of the firing timer and getting a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/DiscardingSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/DiscardingSink.java
index c79b0dd17fd34..d116e74fe89a8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/DiscardingSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/DiscardingSink.java
@@ -22,12 +22,12 @@
 
 /**
  * A stream sink that ignores all elements.
- * 
+ *
  * @param <T> The type of elements received by the sink.
  */
 @Public
 public class DiscardingSink<T> implements SinkFunction<T> {
-	
+
 	private static final long serialVersionUID = 1L;
 
 	@Override
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/OutputFormatSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/OutputFormatSinkFunction.java
index 3f400952b15a7..ed308c5f04519 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/OutputFormatSinkFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/OutputFormatSinkFunction.java
@@ -35,16 +35,16 @@
 /**
  * Simple implementation of the SinkFunction writing tuples in the specified
  * OutputFormat format.
- * 
+ *
  * @param <IN> Input type
  */
 @PublicEvolving
 public class OutputFormatSinkFunction<IN> extends RichSinkFunction<IN> implements InputTypeConfigurable {
-	
+
 	private static final long serialVersionUID = 1L;
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(OutputFormatSinkFunction.class);
-	
+
 	private OutputFormat<IN> format;
 	private boolean cleanupCalled = false;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/PrintSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/PrintSinkFunction.java
index bbdd7a7fdf0f1..bc47e9f58aa88 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/PrintSinkFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/PrintSinkFunction.java
@@ -26,7 +26,7 @@
 /**
  * Implementation of the SinkFunction writing every tuple to the standard
  * output or standard error stream.
- * 
+ *
  * @param <IN>
  *            Input record type
  */
@@ -36,19 +36,19 @@ public class PrintSinkFunction<IN> extends RichSinkFunction<IN> {
 
 	private static final boolean STD_OUT = false;
 	private static final boolean STD_ERR = true;
-	
-	private boolean target; 
+
+	private boolean target;
 	private transient PrintStream stream;
 	private transient String prefix;
-	
+
 	/**
 	 * Instantiates a print sink function that prints to standard out.
 	 */
 	public PrintSinkFunction() {}
-	
+
 	/**
 	 * Instantiates a print sink function that prints to standard out.
-	 * 
+	 *
 	 * @param stdErr True, if the format should print to standard error instead of standard out.
 	 */
 	public PrintSinkFunction(boolean stdErr) {
@@ -58,20 +58,20 @@ public PrintSinkFunction(boolean stdErr) {
 	public void setTargetToStandardOut() {
 		target = STD_OUT;
 	}
-	
+
 	public void setTargetToStandardErr() {
 		target = STD_ERR;
 	}
-	
+
 	@Override
 	public void open(Configuration parameters) throws Exception {
 		super.open(parameters);
 		StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
 		// get the target stream
 		stream = target == STD_OUT ? System.out : System.err;
-		
+
 		// set the prefix if we have a >1 parallelism
-		prefix = (context.getNumberOfParallelSubtasks() > 1) ? 
+		prefix = (context.getNumberOfParallelSubtasks() > 1) ?
 				((context.getIndexOfThisSubtask() + 1) + "> ") : null;
 	}
 
@@ -84,13 +84,13 @@ public void invoke(IN record) {
 			stream.println(record.toString());
 		}
 	}
-	
+
 	@Override
 	public void close() {
 		this.stream = null;
 		this.prefix = null;
 	}
-	
+
 	@Override
 	public String toString() {
 		return "Print to " + (target == STD_OUT ? "System.out" : "System.err");
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
index 0d0669a778360..21c35a318ec28 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
@@ -46,26 +46,26 @@
 public class SocketClientSink<IN> extends RichSinkFunction<IN> {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(SocketClientSink.class);
 
 	private static final int CONNECTION_RETRY_DELAY = 500;
-	
-	
+
+
 	private final SerializableObject lock = new SerializableObject();
 	private final SerializationSchema<IN> schema;
 	private final String hostName;
 	private final int port;
 	private final int maxNumRetries;
 	private final boolean autoFlush;
-	
+
 	private transient Socket client;
 	private transient OutputStream outputStream;
-	
+
 	private int retries;
 
 	private volatile boolean isRunning = true;
-	
+
 	/**
 	 * Creates a new SocketClientSink. The sink will not attempt to retry connections upon failure
 	 * and will not auto-flush the stream.
@@ -118,7 +118,7 @@ public SocketClientSink(String hostName, int port, SerializationSchema<IN> schem
 	// ------------------------------------------------------------------------
 	//  Life cycle
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Initialize the connection with the Socket in the server.
 	 * @param parameters Configuration.
@@ -134,8 +134,8 @@ public void open(Configuration parameters) throws Exception {
 			throw new IOException("Cannot connect to socket server at " + hostName + ":" + port, e);
 		}
 	}
-	
-	
+
+
 	/**
 	 * Called when new data arrives to the sink, and forwards it to Socket.
 	 *
@@ -157,19 +157,19 @@ public void invoke(IN value) throws Exception {
 				throw new IOException("Failed to send message '" + value + "' to socket server at "
 						+ hostName + ":" + port + ". Connection re-tries are not enabled.", e);
 			}
-			
-			LOG.error("Failed to send message '" + value + "' to socket server at " + hostName + ":" + port + 
+
+			LOG.error("Failed to send message '" + value + "' to socket server at " + hostName + ":" + port +
 					". Trying to reconnect..." , e);
-			
+
 			// do the retries in locked scope, to guard against concurrent close() calls
 			// note that the first re-try comes immediately, without a wait!
-		
+
 			synchronized (lock) {
 				IOException lastException = null;
 				retries = 0;
-				
+
 				while (isRunning && (maxNumRetries < 0 || retries < maxNumRetries)) {
-					
+
 					// first, clean up the old resources
 					try {
 						if (outputStream != null) {
@@ -187,17 +187,17 @@ public void invoke(IN value) throws Exception {
 					catch (IOException ee) {
 						LOG.error("Could not close socket from failed write attempt", ee);
 					}
-					
+
 					// try again
 					retries++;
-					
+
 					try {
 						// initialize a new connection
 						createConnection();
-						
+
 						// re-try the write
 						outputStream.write(msg);
-						
+
 						// success!
 						return;
 					}
@@ -209,7 +209,7 @@ public void invoke(IN value) throws Exception {
 					// wait before re-attempting to connect
 					lock.wait(CONNECTION_RETRY_DELAY);
 				}
-				
+
 				// throw an exception if the task is still running, otherwise simply leave the method
 				if (isRunning) {
 					throw new IOException("Failed to send message '" + value + "' to socket server at "
@@ -218,7 +218,7 @@ public void invoke(IN value) throws Exception {
 			}
 		}
 	}
-	
+
 	/**
 	 * Closes the connection with the Socket server.
 	 */
@@ -226,13 +226,13 @@ public void invoke(IN value) throws Exception {
 	public void close() throws Exception {
 		// flag this as not running any more
 		isRunning = false;
-		
+
 		// clean up in locked scope, so there is no concurrent change to the stream and client
 		synchronized (lock) {
 			// we notify first (this statement cannot fail). The notified thread will not continue
 			// anyways before it can re-acquire the lock
 			lock.notifyAll();
-			
+
 			try {
 				if (outputStream != null) {
 					outputStream.close();
@@ -249,19 +249,19 @@ public void close() throws Exception {
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	private void createConnection() throws IOException {
 		client = new Socket(hostName, port);
 		client.setKeepAlive(true);
 		client.setTcpNoDelay(true);
-		
+
 		outputStream = client.getOutputStream();
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  For testing
 	// ------------------------------------------------------------------------
-	
+
 	int getCurrentNumberOfRetries() {
 		synchronized (lock) {
 			return retries;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java
index da038597f7552..234dd39323fff 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java
@@ -27,7 +27,7 @@
 
 /**
  * Writes tuples in csv format.
- * 
+ *
  * @param <IN>
  *            Input tuple type
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java
index a9c724fb0284b..fa2bd8d1c26f5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java
@@ -28,7 +28,7 @@
  * the file specified by path. Tuples are collected to a list and written to the
  * file periodically. The file specified by path is created if it does not
  * exist, cleared if it exists before the writing.
- * 
+ *
  * @param <IN>
  *            Input tuple type
  */
@@ -48,7 +48,7 @@ public WriteSinkFunction(String path, WriteFormat<IN> format) {
 
 	/**
 	 * Creates target file if it does not exist, cleans it if it exists.
-	 * 
+	 *
 	 * @param path
 	 *            is the path to the location where the tuples are written
 	 */
@@ -65,7 +65,7 @@ protected void cleanFile(String path) {
 
 	/**
 	 * Condition for writing the contents of tupleList and clearing it.
-	 * 
+	 *
 	 * @return value of the updating condition
 	 */
 	protected abstract boolean updateCondition();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunctionByMillis.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunctionByMillis.java
index 62ad8c390ff9c..4753c5c1fa324 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunctionByMillis.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunctionByMillis.java
@@ -22,7 +22,7 @@
 /**
  * Implementation of WriteSinkFunction. Writes tuples to file in every millis
  * milliseconds.
- * 
+ *
  * @param <IN>
  *            Input tuple type
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
index 68a17508cadd0..691d3d6d9ff93 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
@@ -40,7 +40,7 @@
 
 /**
  * A stream source function that returns a sequence of elements.
- * 
+ *
  * <p>Upon construction, this source function serializes the elements using Flink's type information.
  * That way, any object transport using Java serialization will not be affected by the serializability
  * of the elements.</p>
@@ -51,15 +51,15 @@
  */
 @PublicEvolving
 public class FromElementsFunction<T> implements SourceFunction<T>, CheckpointedFunction {
-	
+
 	private static final long serialVersionUID = 1L;
 
 	/** The (de)serializer to be used for the data elements. */
 	private final TypeSerializer<T> serializer;
-	
+
 	/** The actual data elements, in serialized form. */
 	private final byte[] elementsSerialized;
-	
+
 	/** The number of serialized elements. */
 	private final int numElements;
 
@@ -68,7 +68,7 @@ public class FromElementsFunction<T> implements SourceFunction<T>, CheckpointedF
 
 	/** The number of elements to skip initially. */
 	private volatile int numElementsToSkip;
-	
+
 	/** Flag to make the source cancelable. */
 	private volatile boolean isRunning = true;
 
@@ -77,7 +77,7 @@ public class FromElementsFunction<T> implements SourceFunction<T>, CheckpointedF
 	public FromElementsFunction(TypeSerializer<T> serializer, T... elements) throws IOException {
 		this(serializer, Arrays.asList(elements));
 	}
-	
+
 	public FromElementsFunction(TypeSerializer<T> serializer, Iterable<T> elements) throws IOException {
 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
 		DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);
@@ -128,7 +128,7 @@ public void initializeState(FunctionInitializationContext context) throws Except
 	public void run(SourceContext<T> ctx) throws Exception {
 		ByteArrayInputStream bais = new ByteArrayInputStream(elementsSerialized);
 		final DataInputView input = new DataInputViewStreamWrapper(bais);
-		
+
 		// if we are restored from a checkpoint and need to skip elements, skip them now.
 		int toSkip = numElementsToSkip;
 		if (toSkip > 0) {
@@ -143,12 +143,12 @@ public void run(SourceContext<T> ctx) throws Exception {
 						"If you are using user-defined serialization (Value and Writable types), check the " +
 						"serialization functions.\nSerializer is " + serializer);
 			}
-			
+
 			this.numElementsEmitted = this.numElementsToSkip;
 		}
-		
+
 		final Object lock = ctx.getCheckpointLock();
-		
+
 		while (isRunning && numElementsEmitted < numElements) {
 			T next;
 			try {
@@ -159,7 +159,7 @@ public void run(SourceContext<T> ctx) throws Exception {
 						"If you are using user-defined serialization (Value and Writable types), check the " +
 						"serialization functions.\nSerializer is " + serializer);
 			}
-			
+
 			synchronized (lock) {
 				ctx.collect(next);
 				numElementsEmitted++;
@@ -175,7 +175,7 @@ public void cancel() {
 
 	/**
 	 * Gets the number of elements produced in total by this function.
-	 * 
+	 *
 	 * @return The number of elements produced in total.
 	 */
 	public int getNumElements() {
@@ -184,7 +184,7 @@ public int getNumElements() {
 
 	/**
 	 * Gets the number of elements emitted so far.
-	 * 
+	 *
 	 * @return The number of elements emitted so far.
 	 */
 	public int getNumElementsEmitted() {
@@ -211,10 +211,10 @@ public void snapshotState(FunctionSnapshotContext context) throws Exception {
 	/**
 	 * Verifies that all elements in the collection are non-null, and are of the given class, or
 	 * a subclass thereof.
-	 * 
+	 *
 	 * @param elements The collection to check.
 	 * @param viewedAs The class to which the elements must be assignable to.
-	 * 
+	 *
 	 * @param <OUT> The generic type of the collection to be checked.
 	 */
 	public static <OUT> void checkCollection(Collection<OUT> elements, Class<OUT> viewedAs) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
index 9592c889f5e30..3fdedf118d225 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
@@ -86,7 +86,7 @@ public void run(SourceContext<OUT> ctx) throws Exception {
 
 				// for each element we also check if cancel
 				// was called by checking the isRunning flag
-				
+
 				while (isRunning && !format.reachedEnd()) {
 					nextElement = format.nextRecord(nextElement);
 					if (nextElement != null) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/RichParallelSourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/RichParallelSourceFunction.java
index 1a0437ec21d9a..94b85b69aba99 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/RichParallelSourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/RichParallelSourceFunction.java
@@ -24,7 +24,7 @@
  * Base class for implementing a parallel data source. Upon execution, the runtime will
  * execute as many parallel instances of this function function as configured parallelism
  * of the source.
- * 
+ *
  * <p>The data source has access to context information (such as the number of parallel
  * instances of the source, and which parallel instance the current instance is)
  * via {@link #getRuntimeContext()}. It also provides additional life-cycle methods
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
index 8d77b15fbac20..dd852a47191b3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
@@ -46,7 +46,7 @@
 public class SocketTextStreamFunction implements SourceFunction<String> {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(SocketTextStreamFunction.class);
 
 	/** Default delay between successive connection attempts. */
@@ -54,28 +54,28 @@ public class SocketTextStreamFunction implements SourceFunction<String> {
 
 	/** Default connection timeout when connecting to the server socket (infinite). */
 	private static final int CONNECTION_TIMEOUT_TIME = 0;
-	
-	
+
+
 	private final String hostname;
 	private final int port;
 	private final String delimiter;
 	private final long maxNumRetries;
 	private final long delayBetweenRetries;
-	
+
 	private transient Socket currentSocket;
-	
+
 	private volatile boolean isRunning = true;
 
-	
+
 	public SocketTextStreamFunction(String hostname, int port, String delimiter, long maxNumRetries) {
 		this(hostname, port, delimiter, maxNumRetries, DEFAULT_CONNECTION_RETRY_SLEEP);
 	}
-	
+
 	public SocketTextStreamFunction(String hostname, int port, String delimiter, long maxNumRetries, long delayBetweenRetries) {
 		checkArgument(port > 0 && port < 65536, "port is out of range");
 		checkArgument(maxNumRetries >= -1, "maxNumRetries must be zero or larger (num retries), or -1 (infinite retries)");
 		checkArgument(delayBetweenRetries >= 0, "delayBetweenRetries must be zero or positive");
-		
+
 		this.hostname = checkNotNull(hostname, "hostname must not be null");
 		this.port = port;
 		this.delimiter = delimiter;
@@ -87,12 +87,12 @@ public SocketTextStreamFunction(String hostname, int port, String delimiter, lon
 	public void run(SourceContext<String> ctx) throws Exception {
 		final StringBuilder buffer = new StringBuilder();
 		long attempt = 0;
-		
+
 		while (isRunning) {
-			
+
 			try (Socket socket = new Socket()) {
 				currentSocket = socket;
-				
+
 				LOG.info("Connecting to server socket " + hostname + ':' + port);
 				socket.connect(new InetSocketAddress(hostname, port), CONNECTION_TIMEOUT_TIME);
 				BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream()));
@@ -138,7 +138,7 @@ record = record.substring(0, record.length() - 1);
 	@Override
 	public void cancel() {
 		isRunning = false;
-		
+
 		// we need to close the socket as well, because the Thread.interrupt() function will
 		// not wake the thread in the socketStream.read() method when blocked.
 		Socket theSocket = this.currentSocket;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
index ba18df442c941..df5f9b6e02c31 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
@@ -70,8 +70,8 @@
  *
  * <h3>Timestamps and watermarks:</h3>
  * Sources may assign timestamps to elements and may manually emit watermarks.
- * However, these are only interpreted if the streaming program runs on 
- * {@link TimeCharacteristic#EventTime}. On other time characteristics 
+ * However, these are only interpreted if the streaming program runs on
+ * {@link TimeCharacteristic#EventTime}. On other time characteristics
  * ({@link TimeCharacteristic#IngestionTime} and {@link TimeCharacteristic#ProcessingTime}),
  * the watermarks from the source function are ignored.
  *
@@ -79,13 +79,13 @@
  * Functions may additionally implement the {@link org.apache.flink.api.common.functions.StoppableFunction}
  * interface. "Stopping" a function, in contrast to "canceling" means a graceful exit that leaves the
  * state and the emitted elements in a consistent state.
- * 
+ *
  * <p>When a source is stopped, the executing thread is not interrupted, but expected to leave the
  * {@link #run(SourceContext)} method in reasonable time on its own, preserving the atomicity
- * of state updates and element emission. 
- * 
+ * of state updates and element emission.
+ *
  * @param <T> The type of the elements produced by this source.
- * 
+ *
  * @see org.apache.flink.api.common.functions.StoppableFunction
  * @see org.apache.flink.streaming.api.TimeCharacteristic
  */
@@ -95,7 +95,7 @@ public interface SourceFunction<T> extends Function, Serializable {
 	/**
 	 * Starts the source. Implementations can use the {@link SourceContext} emit
 	 * elements.
-	 * 
+	 *
 	 * <p>Sources that implement {@link org.apache.flink.streaming.api.checkpoint.Checkpointed}
 	 * must lock on the checkpoint lock (using a synchronized block) before updating internal
 	 * state and emitting elements, to make both an atomic operation:
@@ -132,10 +132,10 @@ public interface SourceFunction<T> extends Function, Serializable {
 	 * Cancels the source. Most sources will have a while loop inside the
 	 * {@link #run(SourceContext)} method. The implementation needs to ensure that the
 	 * source will break out of that loop after this method is called.
-	 * 
+	 *
 	 * <p>A typical pattern is to have an {@code "volatile boolean isRunning"} flag that is set to
 	 * {@code false} in this method. That flag is checked in the loop condition.
-	 * 
+	 *
 	 * <p>When a source is canceled, the executing thread will also be interrupted
 	 * (via {@link Thread#interrupt()}). The interruption happens strictly after this
 	 * method has been called, so any interruption handler can rely on the fact that
@@ -148,7 +148,7 @@ public interface SourceFunction<T> extends Function, Serializable {
 	// ------------------------------------------------------------------------
 	//  source context
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Interface that source functions use to emit elements, and possibly watermarks.
 	 *
@@ -160,7 +160,7 @@ interface SourceContext<T> {
 		/**
 		 * Emits one element from the source, without attaching a timestamp. In most cases,
 		 * this is the default way of emitting elements.
-		 * 
+		 *
 		 * <p>The timestamp that the element will get assigned depends on the time characteristic of
 		 * the streaming program:
 		 * <ul>
@@ -203,7 +203,7 @@ interface SourceContext<T> {
 		 * Emits the given {@link Watermark}. A Watermark of value {@code t} declares that no
 		 * elements with a timestamp {@code t' <= t} will occur any more. If further such
 		 * elements will be emitted, those elements are considered <i>late</i>.
-		 * 
+		 *
 		 * <p>This method is only relevant when running on {@link TimeCharacteristic#EventTime}.
 		 * On {@link TimeCharacteristic#ProcessingTime},Watermarks will be ignored. On
 		 * {@link TimeCharacteristic#IngestionTime}, the Watermarks will be replaced by the
@@ -233,8 +233,8 @@ interface SourceContext<T> {
 		 * Returns the checkpoint lock. Please refer to the class-level comment in
 		 * {@link SourceFunction} for details about how to write a consistent checkpointed
 		 * source.
-		 * 
-		 * @return The object to use as the lock 
+		 *
+		 * @return The object to use as the lock
 		 */
 		Object getCheckpointLock();
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java
index fb1c94d05bc44..b6a0c3b9b1cc3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java
@@ -42,7 +42,7 @@
  */
 @PublicEvolving
 public class StatefulSequenceSource extends RichParallelSourceFunction<Long> implements CheckpointedFunction {
-	
+
 	private static final long serialVersionUID = 1L;
 
 	private final long start;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
index 253608f383e9c..ceed996badf2e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
@@ -36,16 +36,16 @@
  */
 @PublicEvolving
 public abstract class AscendingTimestampExtractor<T> implements AssignerWithPeriodicWatermarks<T> {
-	
+
 	private static final long serialVersionUID = 1L;
-	
+
 	/** The current timestamp. */
 	private long currentTimestamp = Long.MIN_VALUE;
 
 	/** Handler that is called when timestamp monotony is violated. */
 	private MonotonyViolationHandler violationHandler = new LoggingHandler();
-	
-	
+
+
 	/**
 	 * Extracts the timestamp from the given element. The timestamp must be monotonically increasing.
 	 *
@@ -56,7 +56,7 @@ public abstract class AscendingTimestampExtractor<T> implements AssignerWithPeri
 
 	/**
 	 * Sets the handler for violations to the ascending timestamp order.
-	 * 
+	 *
 	 * @param handler The violation handler to use.
 	 * @return This extractor.
 	 */
@@ -64,9 +64,9 @@ public AscendingTimestampExtractor<T> withViolationHandler(MonotonyViolationHand
 		this.violationHandler = requireNonNull(handler);
 		return this;
 	}
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public final long extractTimestamp(T element, long elementPrevTimestamp) {
 		final long newTimestamp = extractAscendingTimestamp(element);
@@ -97,7 +97,7 @@ public interface MonotonyViolationHandler extends java.io.Serializable {
 		/**
 		 * Called when the property of monotonously ascending timestamps is violated, i.e.,
 		 * when {@code elementTimestamp < lastTimestamp}.
-		 * 
+		 *
 		 * @param elementTimestamp The timestamp of the current element.
 		 * @param lastTimestamp The last timestamp.
 		 */
@@ -119,7 +119,7 @@ public void handleViolation(long elementTimestamp, long lastTimestamp) {}
 	 */
 	public static final class FailingHandler implements MonotonyViolationHandler {
 		private static final long serialVersionUID = 1L;
-		
+
 		@Override
 		public void handleViolation(long elementTimestamp, long lastTimestamp) {
 			throw new RuntimeException("Ascending timestamps condition violated. Element timestamp "
@@ -134,7 +134,7 @@ public static final class LoggingHandler implements MonotonyViolationHandler {
 		private static final long serialVersionUID = 1L;
 
 		private static final Logger LOG = LoggerFactory.getLogger(AscendingTimestampExtractor.class);
-		
+
 		@Override
 		public void handleViolation(long elementTimestamp, long lastTimestamp) {
 			LOG.warn("Timestamp monotony violated: {} < {}", elementTimestamp, lastTimestamp);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
index 912a5c03be53d..0341084b8dc52 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
@@ -29,7 +29,7 @@
  * A {@link WindowFunction} that composes an {@link AggregateFunction} and {@link WindowFunction}.
  * Upon invocation, this first applies {@code AggregateFunction} to the input, and then
  * finally the {@code WindowFunction} to the single result element.
- * 
+ *
  * @param <K> The key type
  * @param <W> The window type
  * @param <T> The type of the input to the AggregateFunction
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java
index f547adc59d82b..0cf0a8dce4543 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java
@@ -26,7 +26,7 @@
  *
  * @param <IN> The type of the input value.
  * @param <OUT> The type of the output value.
- * @param <KEY> The type of the window key. 
+ * @param <KEY> The type of the window key.
  * @param <W> The type of the window.
  */
 @Internal
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java
index 154fe88d13aea..030e9735cc76b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java
@@ -43,8 +43,8 @@ public interface WindowFunction<IN, OUT, KEY, W extends Window> extends Function
 	 * @param window The window that is being evaluated.
 	 * @param input The elements in the window being evaluated.
 	 * @param out A collector for emitting elements.
-	 * 
-	 * @throws Exception The function may throw exceptions to fail the program and trigger recovery. 
+	 *
+	 * @throws Exception The function may throw exceptions to fail the program and trigger recovery.
 	 */
 	void apply(KEY key, W window, Iterable<IN> input, Collector<OUT> out) throws Exception;
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java
index a8fc60077a89f..2bf15a0efda27 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/CosineDistance.java
@@ -23,9 +23,9 @@
 /**
  * This delta function calculates the cosine distance between two given vectors.
  * The cosine distance is defined as: cosineDistance=1-cosineSimilarity
- * 
+ *
  * <p>Cosine similarity: http://en.wikipedia.org/wiki/Cosine_similarity
- * 
+ *
  * @param <DATA>
  *            The input data type. This delta function works with a double[],
  *            but can extract/convert to it from any other given object in case
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
index d9295c42a72ad..ed1f830eb84e4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
@@ -35,7 +35,7 @@ public interface DeltaFunction<DATA> extends Serializable {
 
 	/**
 	 * Calculates the delta between two given data points.
-	 * 
+	 *
 	 * @param oldDataPoint
 	 *            the old data point.
 	 * @param newDataPoint
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java
index c4b416ce3083a..28f9c241da903 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/EuclideanDistance.java
@@ -23,9 +23,9 @@
 /**
  * This delta function calculates the euclidean distance between two given
  * points.
- * 
+ *
  * <p>Euclidean distance: http://en.wikipedia.org/wiki/Euclidean_distance
- * 
+ *
  * @param <DATA>
  *            The input data type. This delta function works with a double[],
  *            but can extract/convert to it from any other given object in case
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java
index 535d513f47b61..dc5b6be068d0f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/ExtractionAwareDeltaFunction.java
@@ -25,7 +25,7 @@
  * extracting the data on which the delta is calculated from a more complex data
  * structure. For example in case you want to be able to run a delta only on one
  * field of a Tuple type or only on some fields from an array.
- * 
+ *
  * @param <DATA>
  *            The input data type. The input of this type will be passed to the
  *            extractor which will transform into a TO-object. The delta
@@ -38,7 +38,7 @@
 public abstract class ExtractionAwareDeltaFunction<DATA, TO> implements DeltaFunction<DATA> {
 
 	private static final long serialVersionUID = 6927486219702689554L;
-	
+
 	private Extractor<DATA, TO> converter;
 
 	public ExtractionAwareDeltaFunction(Extractor<DATA, TO> converter) {
@@ -51,7 +51,7 @@ public ExtractionAwareDeltaFunction(Extractor<DATA, TO> converter) {
 	 * with the extracted data. In case no extractor is set the input data gets
 	 * passes to {@link #getNestedDelta} as-is. The return value is just
 	 * forwarded from {@link #getNestedDelta}.
-	 * 
+	 *
 	 * @param oldDataPoint
 	 *            the older data point as raw data (before extraction).
 	 * @param newDataPoint
@@ -78,7 +78,7 @@ public double getDelta(DATA oldDataPoint, DATA newDataPoint) {
 	 * result of the previously done extractions as input. Therefore, this
 	 * method only does the actual calculation of the delta but no data
 	 * extraction or conversion.
-	 * 
+	 *
 	 * @param oldDataPoint
 	 *            the older data point.
 	 * @param newDataPoint
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java
index 16d0935ac79fc..fd892b75dee61 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ArrayFromTuple.java
@@ -41,7 +41,7 @@ public ArrayFromTuple() {
 	/**
 	 * Using this constructor the extractor will combine the fields as specified
 	 * in the indexes parameter in an object array.
-	 * 
+	 *
 	 * @param indexes
 	 *            the field ids (enumerated from 0)
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java
index 6fd9a67dc1c9e..71233cf90aeda 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/ConcatenatedExtract.java
@@ -41,7 +41,7 @@ public class ConcatenatedExtract<FROM, OVER, TO> implements Extractor<FROM, TO>
 
 	/**
 	 * Combines two extractors which will be executed one after each other.
-	 * 
+	 *
 	 * @param e1
 	 *            First extractor: This extractor gets applied to the input data
 	 *            first. Its output as then passed as input to the second
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
index 76d95b9202ba1..426328575b09e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
@@ -25,7 +25,7 @@
  * Extractors allow to extract/convert one type to another. They are mostly used
  * to extract some fields out of a more complex structure (Tuple/Array) to run
  * further calculation on the extraction result.
- * 
+ *
  * @param <FROM>
  *            The input data type.
  * @param <TO>
@@ -36,7 +36,7 @@ public interface Extractor<FROM, TO> extends Serializable {
 
 	/**
 	 * Extracts/Converts the given input to an object of the output type.
-	 * 
+	 *
 	 * @param in
 	 *            the input data
 	 * @return the extracted/converted data
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
index c44f32ba12407..9fc63547a1e92 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
@@ -23,7 +23,7 @@
 
 /**
  * Extracts a single field out of an array.
- * 
+ *
  * @param <OUT>
  *            The type of the extracted field.
  */
@@ -31,7 +31,7 @@
 public class FieldFromArray<OUT> implements Extractor<Object, OUT> {
 
 	private static final long serialVersionUID = -5161386546695574359L;
-	
+
 	private int fieldId = 0;
 
 	/**
@@ -43,7 +43,7 @@ public FieldFromArray() {
 
 	/**
 	 * Extracts the field with the given id from the array.
-	 * 
+	 *
 	 * @param fieldId
 	 *            The id of the field which will be extracted from the array.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java
index 468b878ddb3ed..a456a227e2306 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromTuple.java
@@ -22,7 +22,7 @@
 
 /**
  * Extracts a single field out of a tuple.
- * 
+ *
  * @param <OUT>
  *            The type of the extracted field.
  */
@@ -35,7 +35,7 @@ public class FieldFromTuple<OUT> implements Extractor<Tuple, OUT> {
 
 	/**
 	 * Extracts the field with the given id from the tuple.
-	 * 
+	 *
 	 * @param fieldId
 	 *            The id of the field which will be extracted from the tuple.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
index afc2b4ddc59f1..1fa9eb8f98f79 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
@@ -34,14 +34,14 @@
 public class FieldsFromArray<OUT> implements Extractor<Object, OUT[]> {
 
 	private static final long serialVersionUID = 8075055384516397670L;
-	
+
 	private int[] order;
 	private Class<OUT> clazz;
 
 	/**
 	 * Extracts multiple fields from an array and puts them in the given order
 	 * into a new array of the specified type.
-	 * 
+	 *
 	 * @param clazz
 	 *            the Class object representing the component type of the new
 	 *            array
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java
index 2321878b8f9b1..e2ba284abc859 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromTuple.java
@@ -33,7 +33,7 @@ public class FieldsFromTuple implements Extractor<Tuple, double[]> {
 	/**
 	 * Extracts one or more fields of the the type Double from a tuple and puts
 	 * them into a new double[] (in the specified order).
-	 * 
+	 *
 	 * @param indexes
 	 *            The indexes of the fields to be extracted.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
index be591f756cc8b..b053bff55e24b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
@@ -49,11 +49,11 @@
 public class StreamConfig implements Serializable {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	// ------------------------------------------------------------------------
 	//  Config Keys
 	// ------------------------------------------------------------------------
-	
+
 	private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs";
 	private static final String NUMBER_OF_INPUTS = "numberOfInputs";
 	private static final String CHAINED_OUTPUTS = "chainedOutputs";
@@ -80,22 +80,22 @@ public class StreamConfig implements Serializable {
 
 	private static final String CHECKPOINTING_ENABLED = "checkpointing";
 	private static final String CHECKPOINT_MODE = "checkpointMode";
-	
+
 	private static final String STATE_BACKEND = "statebackend";
 	private static final String STATE_PARTITIONER = "statePartitioner";
 
 	private static final String STATE_KEY_SERIALIZER = "statekeyser";
-	
+
 	private static final String TIME_CHARACTERISTIC = "timechar";
-	
+
 	// ------------------------------------------------------------------------
 	//  Default Values
 	// ------------------------------------------------------------------------
-	
+
 	private static final long DEFAULT_TIMEOUT = 100;
 	private static final CheckpointingMode DEFAULT_CHECKPOINTING_MODE = CheckpointingMode.EXACTLY_ONCE;
-	
-	
+
+
 	// ------------------------------------------------------------------------
 	//  Config
 	// ------------------------------------------------------------------------
@@ -121,7 +121,7 @@ public void setVertexID(Integer vertexID) {
 	public Integer getVertexID() {
 		return config.getInteger(VERTEX_NAME, -1);
 	}
-	
+
 	public void setTimeCharacteristic(TimeCharacteristic characteristic) {
 		config.setInteger(TIME_CHARACTERISTIC, characteristic.ordinal());
 	}
@@ -134,7 +134,7 @@ public TimeCharacteristic getTimeCharacteristic() {
 			throw new CorruptConfigurationException("time characteristic is not set");
 		}
 	}
-	
+
 	public void setTypeSerializerIn1(TypeSerializer<?> serializer) {
 		setTypeSerializer(TYPE_SERIALIZER_IN_1, serializer);
 	}
@@ -150,7 +150,7 @@ public void setTypeSerializerOut(TypeSerializer<?> serializer) {
 	public void setTypeSerializerSideOut(OutputTag<?> outputTag, TypeSerializer<?> serializer) {
 		setTypeSerializer(TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), serializer);
 	}
-	
+
 	public <T> TypeSerializer<T> getTypeSerializerIn1(ClassLoader cl) {
 		try {
 			return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_IN_1, cl);
@@ -158,7 +158,7 @@ public <T> TypeSerializer<T> getTypeSerializerIn1(ClassLoader cl) {
 			throw new StreamTaskException("Could not instantiate serializer.", e);
 		}
 	}
-	
+
 	public <T> TypeSerializer<T> getTypeSerializerIn2(ClassLoader cl) {
 		try {
 			return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_IN_2, cl);
@@ -212,7 +212,7 @@ public void setStreamOperator(StreamOperator<?> operator) {
 			}
 		}
 	}
-	
+
 	public <T> T getStreamOperator(ClassLoader cl) {
 		try {
 			return InstantiationUtil.readObjectFromConfig(this.config, SERIALIZEDUDF, cl);
@@ -220,13 +220,13 @@ public <T> T getStreamOperator(ClassLoader cl) {
 		catch (ClassNotFoundException e) {
 			String classLoaderInfo = ClassLoaderUtil.getUserCodeClassLoaderInfo(cl);
 			boolean loadableDoubleCheck = ClassLoaderUtil.validateClassLoadable(e, cl);
-			
+
 			String exceptionMessage = "Cannot load user class: " + e.getMessage()
-					+ "\nClassLoader info: " + classLoaderInfo + 
-					(loadableDoubleCheck ? 
+					+ "\nClassLoader info: " + classLoaderInfo +
+					(loadableDoubleCheck ?
 							"\nClass was actually found in classloader - deserialization issue." :
 							"\nClass not resolvable through given classloader.");
-			
+
 			throw new StreamTaskException(exceptionMessage);
 		}
 		catch (Exception e) {
@@ -241,13 +241,13 @@ public void setOutputSelectors(List<OutputSelector<?>> outputSelectors) {
 			throw new StreamTaskException("Could not serialize output selectors", e);
 		}
 	}
-	
+
 	public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader userCodeClassloader) {
 		try {
-			List<OutputSelector<T>> selectors = 
+			List<OutputSelector<T>> selectors =
 					InstantiationUtil.readObjectFromConfig(this.config, OUTPUT_SELECTOR_WRAPPER, userCodeClassloader);
 			return selectors == null ? Collections.<OutputSelector<T>>emptyList() : selectors;
-			
+
 		} catch (Exception e) {
 			throw new StreamTaskException("Could not read output selectors", e);
 		}
@@ -292,7 +292,7 @@ public void setNonChainedOutputs(List<StreamEdge> outputvertexIDs) {
 			throw new StreamTaskException("Cannot serialize non chained outputs.", e);
 		}
 	}
-	
+
 	public List<StreamEdge> getNonChainedOutputs(ClassLoader cl) {
 		try {
 			List<StreamEdge> nonChainedOutputs = InstantiationUtil.readObjectFromConfig(this.config, NONCHAINED_OUTPUTS, cl);
@@ -309,7 +309,7 @@ public void setChainedOutputs(List<StreamEdge> chainedOutputs) {
 			throw new StreamTaskException("Cannot serialize chained outputs.", e);
 		}
 	}
-	
+
 	public List<StreamEdge> getChainedOutputs(ClassLoader cl) {
 		try {
 			List<StreamEdge> chainedOutputs = InstantiationUtil.readObjectFromConfig(this.config, CHAINED_OUTPUTS, cl);
@@ -326,7 +326,7 @@ public void setOutEdges(List<StreamEdge> outEdges) {
 			throw new StreamTaskException("Cannot serialize outward edges.", e);
 		}
 	}
-	
+
 	public List<StreamEdge> getOutEdges(ClassLoader cl) {
 		try {
 			List<StreamEdge> outEdges = InstantiationUtil.readObjectFromConfig(this.config, OUT_STREAM_EDGES, cl);
@@ -343,7 +343,7 @@ public void setInPhysicalEdges(List<StreamEdge> inEdges) {
 			throw new StreamTaskException("Cannot serialize inward edges.", e);
 		}
 	}
-	
+
 	public List<StreamEdge> getInPhysicalEdges(ClassLoader cl) {
 		try {
 			List<StreamEdge> inEdges = InstantiationUtil.readObjectFromConfig(this.config, IN_STREAM_EDGES, cl);
@@ -354,7 +354,7 @@ public List<StreamEdge> getInPhysicalEdges(ClassLoader cl) {
 	}
 
 	// --------------------- checkpointing -----------------------
-	
+
 	public void setCheckpointingEnabled(boolean enabled) {
 		config.setBoolean(CHECKPOINTING_ENABLED, enabled);
 	}
@@ -362,7 +362,7 @@ public void setCheckpointingEnabled(boolean enabled) {
 	public boolean isCheckpointingEnabled() {
 		return config.getBoolean(CHECKPOINTING_ENABLED, false);
 	}
-	
+
 	public void setCheckpointMode(CheckpointingMode mode) {
 		config.setInteger(CHECKPOINT_MODE, mode.ordinal());
 	}
@@ -372,7 +372,7 @@ public CheckpointingMode getCheckpointMode() {
 		if (ordinal >= 0) {
 			return CheckpointingMode.values()[ordinal];
 		} else {
-			return DEFAULT_CHECKPOINTING_MODE; 
+			return DEFAULT_CHECKPOINTING_MODE;
 		}
 	}
 
@@ -383,7 +383,7 @@ public void setOutEdgesInOrder(List<StreamEdge> outEdgeList) {
 			throw new StreamTaskException("Could not serialize outputs in order.", e);
 		}
 	}
-	
+
 	public List<StreamEdge> getOutEdgesInOrder(ClassLoader cl) {
 		try {
 			List<StreamEdge> outEdgesInOrder = InstantiationUtil.readObjectFromConfig(this.config, EDGES_IN_ORDER, cl);
@@ -401,7 +401,7 @@ public void setTransitiveChainedTaskConfigs(Map<Integer, StreamConfig> chainedTa
 			throw new StreamTaskException("Could not serialize configuration.", e);
 		}
 	}
-	
+
 	public Map<Integer, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl) {
 		try {
 			Map<Integer, StreamConfig> confs = InstantiationUtil.readObjectFromConfig(this.config, CHAINED_TASK_CONFIG, cl);
@@ -410,11 +410,11 @@ public Map<Integer, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl
 			throw new StreamTaskException("Could not instantiate configuration.", e);
 		}
 	}
-	
+
 	public void setOperatorName(String name) {
 		this.config.setString(OPERATOR_NAME,name);
 	}
-	
+
 	public String getOperatorName() {
 		return this.config.getString(OPERATOR_NAME, null);
 	}
@@ -426,11 +426,11 @@ public void setChainIndex(int index) {
 	public int getChainIndex() {
 		return this.config.getInteger(CHAIN_INDEX, 0);
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  State backend
 	// ------------------------------------------------------------------------
-	
+
 	public void setStateBackend(AbstractStateBackend backend) {
 		if (backend != null) {
 			try {
@@ -440,7 +440,7 @@ public void setStateBackend(AbstractStateBackend backend) {
 			}
 		}
 	}
-	
+
 	public AbstractStateBackend getStateBackend(ClassLoader cl) {
 		try {
 			return InstantiationUtil.readObjectFromConfig(this.config, STATE_BACKEND, cl);
@@ -452,7 +452,7 @@ public AbstractStateBackend getStateBackend(ClassLoader cl) {
 	public byte[] getSerializedStateBackend() {
 		return this.config.getBytes(STATE_BACKEND, null);
 	}
-	
+
 	public void setStatePartitioner(int input, KeySelector<?, ?> partitioner) {
 		try {
 			InstantiationUtil.writeObjectToConfig(partitioner, this.config, STATE_PARTITIONER + input);
@@ -460,7 +460,7 @@ public void setStatePartitioner(int input, KeySelector<?, ?> partitioner) {
 			throw new StreamTaskException("Could not serialize state partitioner.", e);
 		}
 	}
-	
+
 	public KeySelector<?, Serializable> getStatePartitioner(int input, ClassLoader cl) {
 		try {
 			return InstantiationUtil.readObjectFromConfig(this.config, STATE_PARTITIONER + input, cl);
@@ -486,11 +486,11 @@ public <K> TypeSerializer<K> getStateKeySerializer(ClassLoader cl) {
 	}
 
 
-	
+
 	// ------------------------------------------------------------------------
 	//  Miscellansous
 	// ------------------------------------------------------------------------
-	
+
 	public void setChainStart() {
 		config.setBoolean(IS_CHAINED_VERTEX, true);
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
index 94e624ff4e9b4..555d67b251392 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
@@ -102,7 +102,7 @@ public List<String> getSelectedNames() {
 	public StreamPartitioner<?> getPartitioner() {
 		return outputPartitioner;
 	}
-	
+
 	public void setPartitioner(StreamPartitioner<?> partitioner) {
 		this.outputPartitioner = partitioner;
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
index 157023ef8f598..b132416a54e8d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
@@ -67,11 +67,11 @@
 /**
  * Class representing the streaming topology. It contains all the information
  * necessary to build the jobgraph for the execution.
- * 
+ *
  */
 @Internal
 public class StreamGraph extends StreamingPlan {
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(StreamGraph.class);
 
 	private String jobName = StreamExecutionEnvironment.DEFAULT_JOB_NAME;
@@ -79,7 +79,7 @@ public class StreamGraph extends StreamingPlan {
 	private final StreamExecutionEnvironment environment;
 	private final ExecutionConfig executionConfig;
 	private final CheckpointConfig checkpointConfig;
-	
+
 	private boolean chaining;
 
 	private Map<Integer, StreamNode> streamNodes;
@@ -118,8 +118,8 @@ public void clear() {
 		sources = new HashSet<>();
 		sinks = new HashSet<>();
 	}
-	
-	
+
+
 	public StreamExecutionEnvironment getEnvironment() {
 		return environment;
 	}
@@ -127,7 +127,7 @@ public StreamExecutionEnvironment getEnvironment() {
 	public ExecutionConfig getExecutionConfig() {
 		return executionConfig;
 	}
-	
+
 	public CheckpointConfig getCheckpointConfig() {
 		return checkpointConfig;
 	}
@@ -153,11 +153,11 @@ public AbstractStateBackend getStateBackend() {
 	}
 
 	// Checkpointing
-	
+
 	public boolean isChainingEnabled() {
 		return chaining;
 	}
-	
+
 
 	public boolean isIterative() {
 		return!vertexIDtoLoopTimeout.isEmpty();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
index be80371239935..4a141ebc2478e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
@@ -273,7 +273,7 @@ private <T> Collection<Integer> transformSplit(SplitTransformation<T> split) {
 	 * Transforms a {@code SelectTransformation}.
 	 *
 	 * <p>For this we create a virtual node in the {@code StreamGraph} holds the selected names.
-	 * 
+	 *
 	 * @see org.apache.flink.streaming.api.graph.StreamGraphGenerator
 	 */
 	private <T> Collection<Integer> transformSelect(SelectTransformation<T> select) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 9eca9a89705e0..accb15d21d071 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -406,7 +406,7 @@ private void setVertexConfig(Integer vertexID, StreamConfig config,
 		config.setChainedOutputs(chainableOutputs);
 
 		config.setTimeCharacteristic(streamGraph.getEnvironment().getStreamTimeCharacteristic());
-		
+
 		final CheckpointConfig ceckpointCfg = streamGraph.getCheckpointConfig();
 
 		config.setStateBackend(streamGraph.getStateBackend());
@@ -526,7 +526,7 @@ private void setSlotSharing() {
 		}
 
 	}
-	
+
 	private void configureCheckpointing() {
 		CheckpointConfig cfg = streamGraph.getCheckpointConfig();
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index e5e5dfc605c1c..b260de30da621 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -79,9 +79,9 @@
 import static org.apache.flink.util.Preconditions.checkArgument;
 
 /**
- * Base class for all stream operators. Operators that contain a user function should extend the class 
- * {@link AbstractUdfStreamOperator} instead (which is a specialized subclass of this class). 
- * 
+ * Base class for all stream operators. Operators that contain a user function should extend the class
+ * {@link AbstractUdfStreamOperator} instead (which is a specialized subclass of this class).
+ *
  * <p>For concrete implementations, one of the following two interfaces must also be implemented, to
  * mark the operator as unary or binary:
  * {@link OneInputStreamOperator} or {@link TwoInputStreamOperator}.
@@ -97,7 +97,7 @@ public abstract class AbstractStreamOperator<OUT>
 		implements StreamOperator<OUT>, Serializable, KeyContext {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	/** The logger used by the operator class and its subclasses. */
 	protected static final Logger LOG = LoggerFactory.getLogger(AbstractStreamOperator.class);
 
@@ -110,7 +110,7 @@ public abstract class AbstractStreamOperator<OUT>
 
 	/** The task that contains this operator (and other operators in the same chain). */
 	private transient StreamTask<?, ?> container;
-	
+
 	protected transient StreamConfig config;
 
 	protected transient Output<StreamRecord<OUT>> output;
@@ -179,7 +179,7 @@ public abstract class AbstractStreamOperator<OUT>
 	public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
 		this.container = containingTask;
 		this.config = config;
-		
+
 		this.metrics = container.getEnvironment().getMetricGroup().addOperator(config.getOperatorName());
 		this.output = new CountingOutput(output, ((OperatorMetricGroup) this.metrics).getIOMetricGroup().getNumRecordsOutCounter());
 		if (config.isChainStart()) {
@@ -201,7 +201,7 @@ public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<S
 		stateKeySelector1 = config.getStatePartitioner(0, getUserCodeClassloader());
 		stateKeySelector2 = config.getStatePartitioner(1, getUserCodeClassloader());
 	}
-	
+
 	@Override
 	public MetricGroup getMetricGroup() {
 		return metrics;
@@ -294,7 +294,7 @@ private void restoreStreamCheckpointed(OperatorStateHandles stateHandles) throws
 	 * operator's initialization logic, e.g. state initialization.
 	 *
 	 * <p>The default implementation does nothing.
-	 * 
+	 *
 	 * @throws Exception An exception in this method causes the operator to fail.
 	 */
 	@Override
@@ -347,7 +347,7 @@ private void initOperatorState(Collection<OperatorStateHandle> operatorStateHand
 	 */
 	@Override
 	public void close() throws Exception {}
-	
+
 	/**
 	 * This method is called at the very end of the operator's life, both in the case of a successful
 	 * completion of the operation, and in the case of a failure and canceling.
@@ -539,21 +539,21 @@ CheckpointStreamFactory getCheckpointStreamFactory(CheckpointOptions checkpointO
 	/**
 	 * Gets the execution config defined on the execution environment of the job to which this
 	 * operator belongs.
-	 * 
+	 *
 	 * @return The job's execution config.
 	 */
 	public ExecutionConfig getExecutionConfig() {
 		return container.getExecutionConfig();
 	}
-	
+
 	public StreamConfig getOperatorConfig() {
 		return config;
 	}
-	
+
 	public StreamTask<?, ?> getContainingTask() {
 		return container;
 	}
-	
+
 	public ClassLoader getUserCodeClassloader() {
 		return container.getUserCodeClassLoader();
 	}
@@ -572,7 +572,7 @@ protected String getOperatorName() {
 			return getClass().getSimpleName();
 		}
 	}
-	
+
 	/**
 	 * Returns a context that allows the operator to query information about the execution and also
 	 * to interact with systems such as broadcast variables and managed state. This also allows
@@ -601,7 +601,7 @@ protected ProcessingTimeService getProcessingTimeService() {
 
 	/**
 	 * Creates a partitioned state handle, using the state backend configured for this task.
-	 * 
+	 *
 	 * @throws IllegalStateException Thrown, if the key/value state was already initialized.
 	 * @throws Exception Thrown, if the state backend cannot create the key/value state.
 	 */
@@ -641,7 +641,7 @@ protected <S extends State, N> S getPartitionedState(
 	    This method should be removed for the sake of namespaces being lazily fetched from the keyed
 	    state backend, or being set on the state directly.
 	    */
-		
+
 		if (keyedStateStore != null) {
 			return keyedStateBackend.getPartitionedState(namespace, namespaceSerializer, stateDescriptor);
 		} else {
@@ -701,12 +701,12 @@ public KeyedStateStore getKeyedStateStore() {
 	// ------------------------------------------------------------------------
 	//  Context and chaining properties
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public final void setChainingStrategy(ChainingStrategy strategy) {
 		this.chainingStrategy = strategy;
 	}
-	
+
 	@Override
 	public final ChainingStrategy getChainingStrategy() {
 		return chainingStrategy;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
index 550add18d5b18..0dc1d2b63c7ca 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
@@ -48,7 +48,7 @@
  * This is used as the base class for operators that have a user-defined
  * function. This class handles the opening and closing of the user-defined functions,
  * as part of the operator life cycle.
- * 
+ *
  * @param <OUT>
  *            The output type of the operator
  * @param <F>
@@ -61,14 +61,14 @@ public abstract class AbstractUdfStreamOperator<OUT, F extends Function>
 		StreamCheckpointedOperator {
 
 	private static final long serialVersionUID = 1L;
-	
-	
+
+
 	/** The user function. */
 	protected final F userFunction;
-	
+
 	/** Flag to prevent duplicate function.close() calls in close() and dispose(). */
 	private transient boolean functionsClosed = false;
-	
+
 	public AbstractUdfStreamOperator(F userFunction) {
 		this.userFunction = requireNonNull(userFunction);
 		checkUdfCheckpointingPreconditions();
@@ -81,7 +81,7 @@ public AbstractUdfStreamOperator(F userFunction) {
 	public F getUserFunction() {
 		return userFunction;
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  operator life cycle
 	// ------------------------------------------------------------------------
@@ -131,7 +131,7 @@ public void dispose() throws Exception {
 	// ------------------------------------------------------------------------
 	//  checkpointing and recovery
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public void snapshotState(FSDataOutputStream out, long checkpointId, long timestamp) throws Exception {
 		if (userFunction instanceof Checkpointed) {
@@ -207,7 +207,7 @@ public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig exec
 	/**
 	 * Since the streaming API does not implement any parametrization of functions via a
 	 * configuration, the config returned here is actually empty.
-	 * 
+	 *
 	 * @return The user function parameters (currently empty)
 	 */
 	public Configuration getUserFunctionParameters() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ChainingStrategy.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ChainingStrategy.java
index c2a7bb15bc048..cf8f4ee9be5d9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ChainingStrategy.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ChainingStrategy.java
@@ -24,7 +24,7 @@
  * Defines the chaining scheme for the operator. When an operator is chained to the
  * predecessor, it means that they run in the same thread. They become one operator
  * consisting of multiple steps.
- * 
+ *
  * <p>The default value used by the {@link StreamOperator} is {@link #HEAD}, which means that
  * the operator is not chained to its predecessor. Most operators override this with
  * {@link #ALWAYS}, meaning they will be chained to predecessors whenever possible.
@@ -32,9 +32,9 @@
 @PublicEvolving
 public enum ChainingStrategy {
 
-	/** 
+	/**
 	 * Operators will be eagerly chained whenever possible.
-	 * 
+	 *
 	 * <p>To optimize performance, it is generally a good practice to allow maximal
 	 * chaining and increase operator parallelism.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OneInputStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OneInputStreamOperator.java
index d9de2302d85a6..7c30cfa162f15 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OneInputStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OneInputStreamOperator.java
@@ -27,7 +27,7 @@
  * Interface for stream operators with one input. Use
  * {@link org.apache.flink.streaming.api.operators.AbstractStreamOperator} as a base class if
  * you want to implement a custom operator.
- * 
+ *
  * @param <IN> The input type of the operator
  * @param <OUT> The output type of the operator
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
index f984a330d8ceb..3e4493078527f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
@@ -42,19 +42,19 @@ public class StreamGroupedFold<IN, OUT, KEY>
 		implements OneInputStreamOperator<IN, OUT>, OutputTypeConfigurable<OUT> {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	private static final String STATE_NAME = "_op_state";
 
 	// Grouped values
 	private transient ValueState<OUT> values;
-	
+
 	private transient OUT initialValue;
-	
+
 	// Initial value serialization
 	private byte[] serializedInitialValue;
-	
+
 	private TypeSerializer<OUT> outTypeSerializer;
-	
+
 	public StreamGroupedFold(FoldFunction<IN, OUT> folder, OUT initialValue) {
 		super(folder);
 		this.initialValue = initialValue;
@@ -74,7 +74,7 @@ public void open() throws Exception {
 		{
 			initialValue = outTypeSerializer.deserialize(in);
 		}
-		
+
 		ValueStateDescriptor<OUT> stateId = new ValueStateDescriptor<>(STATE_NAME, outTypeSerializer);
 		values = getPartitionedState(stateId);
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java
index 2fd3f543eedfc..fb5a7891f994d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedReduce.java
@@ -36,12 +36,12 @@ public class StreamGroupedReduce<IN> extends AbstractUdfStreamOperator<IN, Reduc
 	private static final long serialVersionUID = 1L;
 
 	private static final String STATE_NAME = "_op_state";
-	
+
 	private transient ValueState<IN> values;
-	
+
 	private TypeSerializer<IN> serializer;
 
-	
+
 	public StreamGroupedReduce(ReduceFunction<IN> reducer, TypeSerializer<IN> serializer) {
 		super(reducer);
 		this.serializer = serializer;
@@ -58,7 +58,7 @@ public void open() throws Exception {
 	public void processElement(StreamRecord<IN> element) throws Exception {
 		IN value = element.getValue();
 		IN currentValue = values.value();
-		
+
 		if (currentValue != null) {
 			IN reduced = userFunction.reduce(currentValue, value);
 			values.update(reduced);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index b0b770322918e..61578b23a6d51 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -33,23 +33,23 @@
  * {@link org.apache.flink.streaming.api.operators.OneInputStreamOperator} or
  * {@link org.apache.flink.streaming.api.operators.TwoInputStreamOperator} to create operators
  * that process elements.
- * 
+ *
  * <p>The class {@link org.apache.flink.streaming.api.operators.AbstractStreamOperator}
  * offers default implementation for the lifecycle and properties methods.
  *
  * <p>Methods of {@code StreamOperator} are guaranteed not to be called concurrently. Also, if using
  * the timer service, timer callbacks are also guaranteed not to be called concurrently with
  * methods on {@code StreamOperator}.
- * 
+ *
  * @param <OUT> The output type of the operator
  */
 @PublicEvolving
 public interface StreamOperator<OUT> extends Serializable {
-	
+
 	// ------------------------------------------------------------------------
 	//  life cycle
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Initializes the operator. Sets access to the context and the output.
 	 */
@@ -58,7 +58,7 @@ public interface StreamOperator<OUT> extends Serializable {
 	/**
 	 * This method is called immediately before any elements are processed, it should contain the
 	 * operator's initialization logic.
-	 * 
+	 *
 	 * @throws java.lang.Exception An exception in this method causes the operator to fail.
 	 */
 	void open() throws Exception;
@@ -72,7 +72,7 @@ public interface StreamOperator<OUT> extends Serializable {
 	 * <p>The method is expected to flush all remaining buffered data. Exceptions during this
 	 * flushing of buffered should be propagated, in order to cause the operation to be recognized
 	 * as failed, because the last data items are not processed properly.
-	 * 
+	 *
 	 * @throws java.lang.Exception An exception in this method causes the operator to fail.
 	 */
 	void close() throws Exception;
@@ -80,7 +80,7 @@ public interface StreamOperator<OUT> extends Serializable {
 	/**
 	 * This method is called at the very end of the operator's life, both in the case of a successful
 	 * completion of the operation, and in the case of a failure and canceling.
-	 * 
+	 *
 	 * <p>This method is expected to make a thorough effort to release all resources
 	 * that the operator has acquired.
 	 */
@@ -95,7 +95,7 @@ public interface StreamOperator<OUT> extends Serializable {
 	 *
 	 * @return a runnable future to the state handle that points to the snapshotted state. For synchronous implementations,
 	 * the runnable might already be finished.
-	 * 
+	 *
 	 * @throws Exception exception that happened during snapshotting.
 	 */
 	OperatorSnapshotResult snapshotState(
@@ -105,10 +105,10 @@ OperatorSnapshotResult snapshotState(
 
 	/**
 	 * Takes a snapshot of the legacy operator state defined via {@link StreamCheckpointedOperator}.
-	 * 
+	 *
 	 * @return The handle to the legacy operator state, or null, if no state was snapshotted.
 	 * @throws Exception This method should forward any type of exception that happens during snapshotting.
-	 * 
+	 *
 	 * @deprecated This method will be removed as soon as no more operators use the legacy state code paths
 	 */
 	@SuppressWarnings("deprecation")
@@ -138,7 +138,7 @@ StreamStateHandle snapshotLegacyOperatorState(
 	// ------------------------------------------------------------------------
 	//  miscellaneous
 	// ------------------------------------------------------------------------
-	
+
 	void setKeyContextElement1(StreamRecord<?> record) throws Exception;
 
 	void setKeyContextElement2(StreamRecord<?> record) throws Exception;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
index 36f7c6a3d7207..cf59cdeb2527e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
@@ -36,16 +36,16 @@
  * @param <SRC> Type of the source function of this stream source operator
  */
 @Internal
-public class StreamSource<OUT, SRC extends SourceFunction<OUT>> 
+public class StreamSource<OUT, SRC extends SourceFunction<OUT>>
 		extends AbstractUdfStreamOperator<OUT, SRC> implements StreamOperator<OUT> {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	private transient SourceFunction.SourceContext<OUT> ctx;
 
 	private transient volatile boolean canceledOrStopped = false;
-	
-	
+
+
 	public StreamSource(SRC sourceFunction) {
 		super(sourceFunction);
 
@@ -56,7 +56,7 @@ public void run(final Object lockingObject, final StreamStatusMaintainer streamS
 		run(lockingObject, streamStatusMaintainer, output);
 	}
 
-	
+
 	public void run(final Object lockingObject,
 			final StreamStatusMaintainer streamStatusMaintainer,
 			final Output<StreamRecord<OUT>> collector) throws Exception {
@@ -108,7 +108,7 @@ public void cancel() {
 		// the happens-before relationship
 		markCanceledOrStopped();
 		userFunction.cancel();
-		
+
 		// the context may not be initialized if the source was never running.
 		if (ctx != null) {
 			ctx.close();
@@ -117,16 +117,16 @@ public void cancel() {
 
 	/**
 	 * Marks this source as canceled or stopped.
-	 * 
+	 *
 	 * <p>This indicates that any exit of the {@link #run(Object, StreamStatusMaintainer, Output)} method
-	 * cannot be interpreted as the result of a finite source.  
+	 * cannot be interpreted as the result of a finite source.
 	 */
 	protected void markCanceledOrStopped() {
 		this.canceledOrStopped = true;
 	}
-	
+
 	/**
-	 * Checks whether the source has been canceled or stopped. 
+	 * Checks whether the source has been canceled or stopped.
 	 * @return True, if the source is canceled or stopped, false is not.
 	 */
 	protected boolean isCanceledOrStopped() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
index a7f69ac788436..bd3d5006e81f7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
@@ -138,7 +138,7 @@ public <T, ACC> FoldingState<T, ACC> getFoldingState(FoldingStateDescriptor<T, A
 		stateProperties.initializeSerializerUnlessSet(getExecutionConfig());
 		return keyedStateStore.getFoldingState(stateProperties);
 	}
-	
+
 	@Override
 	public <UK, UV> MapState<UK, UV> getMapState(MapStateDescriptor<UK, UV> stateProperties) {
 		KeyedStateStore keyedStateStore = checkPreconditionsAndGetKeyedStateStore(stateProperties);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java
index dc80e81108121..e0e09694f17c2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java
@@ -33,7 +33,7 @@
  */
 @Internal
 public class TimestampedCollector<T> implements Collector<T> {
-	
+
 	private final Output<StreamRecord<T>> output;
 
 	private final StreamRecord<T> reuse;
@@ -50,7 +50,7 @@ public TimestampedCollector(Output<StreamRecord<T>> output) {
 	public void collect(T record) {
 		output.collect(reuse.replace(record));
 	}
-	
+
 	public void setTimestamp(StreamRecord<?> timestampBase) {
 		if (timestampBase.hasTimestamp()) {
 			reuse.setTimestamp(timestampBase.getTimestamp());
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TwoInputStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TwoInputStreamOperator.java
index e45fedf801b87..10909af4668fd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TwoInputStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TwoInputStreamOperator.java
@@ -27,7 +27,7 @@
  * Interface for stream operators with two inputs. Use
  * {@link org.apache.flink.streaming.api.operators.AbstractStreamOperator} as a base class if
  * you want to implement a custom operator.
- * 
+ *
  * @param <IN1> The input type of the operator
  * @param <IN2> The input type of the operator
  * @param <OUT> The output type of the operator
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
index 7925be8a68ff2..82e25aee099c4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
@@ -62,12 +62,12 @@
  * Within the async function, the user can complete the async collector arbitrarily. Once the async
  * collector has been completed, the result is emitted by the operator's emitter to downstream
  * operators.
- * 
+ *
  * <p>The operator offers different output modes depending on the chosen
  * {@link OutputMode}. In order to give exactly once processing guarantees, the
  * operator stores all currently in-flight {@link StreamElement} in it's operator state. Upon
  * recovery the recorded set of stream elements is replayed.
- * 
+ *
  * <p>In case of chaining of this operator, it has to be made sure that the operators in the chain are
  * opened tail to head. The reason for this is that an opened {@link AsyncWaitOperator} starts
  * already emitting recovered {@link StreamElement} to downstream operators.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
index b5d1343528ef9..471cb226e08be 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
@@ -43,7 +43,7 @@ public class OneInputTransformation<IN, OUT> extends StreamTransformation<OUT> {
 	private final OneInputStreamOperator<IN, OUT> operator;
 
 	private KeySelector<IN, ?> stateKeySelector;
-	
+
 	private TypeInformation<?> stateKeyType;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
index f6a4e5aa37249..588accd0c627f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
@@ -36,7 +36,7 @@
  */
 @Internal
 public class PartitionTransformation<T> extends StreamTransformation<T> {
-	
+
 	private final StreamTransformation<T> input;
 	private final StreamPartitioner<T> partitioner;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
index 5b388ac349810..cbc5b36d36a51 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
@@ -36,7 +36,7 @@
  */
 @Internal
 public class SelectTransformation<T> extends StreamTransformation<T> {
-	
+
 	private final StreamTransformation<T> input;
 	private final List<String> selectedNames;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
index a2948142cb124..0a803f538ddb1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
@@ -42,7 +42,7 @@ public class SinkTransformation<T> extends StreamTransformation<Object> {
 
 	// We need this because sinks can also have state that is partitioned by key
 	private KeySelector<T, ?> stateKeySelector;
-	
+
 	private TypeInformation<?> stateKeyType;
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
index 95f6d11133d24..f79697c6bbd2d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
@@ -49,30 +49,30 @@
  *
  * <p>The following graph of {@code StreamTransformations}:
  * <pre>{@code
- *   Source              Source        
- *      +                   +           
- *      |                   |           
- *      v                   v           
- *  Rebalance          HashPartition    
- *      +                   +           
- *      |                   |           
- *      |                   |           
- *      +------>Union<------+           
- *                +                     
- *                |                     
- *                v                     
- *              Split                   
- *                +                     
- *                |                     
- *                v                     
- *              Select                  
- *                +                     
- *                v                     
- *               Map                    
- *                +                     
- *                |                     
- *                v                     
- *              Sink 
+ *   Source              Source
+ *      +                   +
+ *      |                   |
+ *      v                   v
+ *  Rebalance          HashPartition
+ *      +                   +
+ *      |                   |
+ *      |                   |
+ *      +------>Union<------+
+ *                +
+ *                |
+ *                v
+ *              Split
+ *                +
+ *                |
+ *                v
+ *              Select
+ *                +
+ *                v
+ *               Map
+ *                +
+ *                |
+ *                v
+ *              Sink
  * }</pre>
  *
  * <p>Would result in this graph of operations at runtime:
@@ -194,7 +194,7 @@ public int getParallelism() {
 
 	/**
 	 * Sets the parallelism of this {@code StreamTransformation}.
-	 * 
+	 *
 	 * @param parallelism The new parallelism to set on this {@code StreamTransformation}.
 	 */
 	public void setParallelism(int parallelism) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java
index 4960512eff1fb..d157cc64f087a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/watermark/Watermark.java
@@ -43,9 +43,9 @@ public final class Watermark extends StreamElement {
 
 	/** The watermark that signifies end-of-event-time. */
 	public static final Watermark MAX_WATERMARK = new Watermark(Long.MAX_VALUE);
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	/** The timestamp of the watermark in milliseconds. */
 	private final long timestamp;
 
@@ -64,7 +64,7 @@ public long getTimestamp() {
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public boolean equals(Object o) {
 		return this == o ||
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingAlignedProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingAlignedProcessingTimeWindows.java
index 984c31ac984e9..dc2984bdc7cfa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingAlignedProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingAlignedProcessingTimeWindows.java
@@ -24,25 +24,25 @@
 /**
  * This is a special window assigner used to tell the system to use the
  * <i>"Fast Aligned Processing Time Window Operator"</i> for windowing.
- * 
+ *
  * <p>Prior Flink versions used that operator automatically for simple processing time
  * windows (tumbling and sliding) when no custom trigger and no evictor was specified.
- * In the current Flink version, that operator is only used when programs explicitly 
+ * In the current Flink version, that operator is only used when programs explicitly
  * specify this window assigner. This is only intended for special cases where programs relied on
  * the better performance of the fast aligned window operator, and are willing to accept the lack
  * of support for various features as indicated below:
- * 
+ *
  * <ul>
  *     <li>No custom state backend can be selected, the operator always stores data on the Java heap.</li>
  *     <li>The operator does not support key groups, meaning it cannot change the parallelism.</li>
  *     <li>Future versions of Flink may not be able to resume from checkpoints/savepoints taken by this
  *         operator.</li>
  * </ul>
- * 
+ *
  * <p>Future implementation plans: We plan to add some of the optimizations used by this operator to
  * the general window operator, so that future versions of Flink will not have the performance/functionality
  * trade-off any more.
- * 
+ *
  * <p>Note on implementation: The concrete operator instantiated by this assigner is either the
  * {@link org.apache.flink.streaming.runtime.operators.windowing.AggregatingProcessingTimeWindowOperator}
  * or {@link org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingAlignedProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingAlignedProcessingTimeWindows.java
index c00eb7d861613..252f997d0b30b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingAlignedProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingAlignedProcessingTimeWindows.java
@@ -27,7 +27,7 @@
  *
  * <p>Prior Flink versions used that operator automatically for simple processing time
  * windows (tumbling and sliding) when no custom trigger and no evictor was specified.
- * In the current Flink version, that operator is only used when programs explicitly 
+ * In the current Flink version, that operator is only used when programs explicitly
  * specify this window assigner. This is only intended for special cases where programs relied on
  * the better performance of the fast aligned window operator, and are willing to accept the lack
  * of support for various features as indicated below:
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
index 3d11a440c86ec..1c47de3a68338 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
@@ -53,7 +53,7 @@
  */
 @PublicEvolving
 public abstract class Trigger<T, W extends Window> implements Serializable {
-	
+
 	private static final long serialVersionUID = -4104633972991191369L;
 
 	/**
@@ -117,7 +117,7 @@ public void onMerge(W window, OnMergeContext ctx) throws Exception {
 	public abstract void clear(W window, TriggerContext ctx) throws Exception;
 
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * A context object that is given to {@link Trigger} methods to allow them to register timer
 	 * callbacks and deal with state.
@@ -144,7 +144,7 @@ public interface TriggerContext {
 		 * Returns the current watermark time.
 		 */
 		long getCurrentWatermark();
-	
+
 		/**
 		 * Register a system time callback. When the current system time passes the specified
 		 * time {@link Trigger#onProcessingTime(long, Window, TriggerContext)} is called with the time specified here.
@@ -152,7 +152,7 @@ public interface TriggerContext {
 		 * @param time The time at which to invoke {@link Trigger#onProcessingTime(long, Window, TriggerContext)}
 		 */
 		void registerProcessingTimeTimer(long time);
-	
+
 		/**
 		 * Register an event-time callback. When the current watermark passes the specified
 		 * time {@link Trigger#onEventTime(long, Window, TriggerContext)} is called with the time specified here.
@@ -161,17 +161,17 @@ public interface TriggerContext {
 		 * @see org.apache.flink.streaming.api.watermark.Watermark
 		 */
 		void registerEventTimeTimer(long time);
-	
+
 		/**
 		 * Delete the processing time trigger for the given time.
 		 */
 		void deleteProcessingTimeTimer(long time);
-	
+
 		/**
 		 * Delete the event-time trigger for the given time.
 		 */
 		void deleteEventTimeTimer(long time);
-	
+
 		/**
 		 * Retrieves a {@link State} object that can be used to interact with
 		 * fault-tolerant state that is scoped to the window and key of the current
@@ -185,7 +185,7 @@ public interface TriggerContext {
 		 *                                       function (function is not part os a KeyedStream).
 		 */
 		<S extends State> S getPartitionedState(StateDescriptor<S, ?> stateDescriptor);
-	
+
 		/**
 		 * Retrieves a {@link ValueState} object that can be used to interact with
 		 * fault-tolerant state that is scoped to the window and key of the current
@@ -205,8 +205,8 @@ public interface TriggerContext {
 		 */
 		@Deprecated
 		<S extends Serializable> ValueState<S> getKeyValueState(String name, Class<S> stateType, S defaultState);
-	
-	
+
+
 		/**
 		 * Retrieves a {@link ValueState} object that can be used to interact with
 		 * fault-tolerant state that is scoped to the window and key of the current
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/TriggerResult.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/TriggerResult.java
index 814993038a04f..a37777cc8a83c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/TriggerResult.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/TriggerResult.java
@@ -32,7 +32,7 @@ public enum TriggerResult {
 
 	/**
 	 * {@code FIRE_AND_PURGE} evaluates the window function and emits the window
-	 * result. 
+	 * result.
 	 */
 	FIRE_AND_PURGE(true, true),
 
@@ -49,7 +49,7 @@ public enum TriggerResult {
 	PURGE(false, true);
 
 	// ------------------------------------------------------------------------
-	
+
 	private final boolean fire;
 	private final boolean purge;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
index cd797653f382c..92fb8bb78c306 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
@@ -52,7 +52,7 @@ public TimeWindow(long start, long end) {
 	/**
 	 * Gets the starting timestamp of the window. This is the first timestamp that belongs
 	 * to this window.
-	 * 
+	 *
 	 * @return The starting timestamp of this window.
 	 */
 	public long getStart() {
@@ -62,7 +62,7 @@ public long getStart() {
 	/**
 	 * Gets the end timestamp of this window. The end timestamp is exclusive, meaning it
 	 * is the first timestamp that does not belong to this window any more.
-	 * 
+	 *
 	 * @return The exclusive end timestamp of this window.
 	 */
 	public long getEnd() {
@@ -71,12 +71,12 @@ public long getEnd() {
 
 	/**
 	 * Gets the largest timestamp that still belongs to this window.
-	 * 
+	 *
 	 * <p>This timestamp is identical to {@code getEnd() - 1}.
-	 * 
+	 *
 	 * @return The largest timestamp that still belongs to this window.
-	 * 
-	 * @see #getEnd() 
+	 *
+	 * @see #getEnd()
 	 */
 	@Override
 	public long maxTimestamp() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
index de13a5e2a22d2..c572023a21b2a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
@@ -44,9 +44,9 @@
 /**
  * The barrier buffer is {@link CheckpointBarrierHandler} that blocks inputs with barriers until
  * all inputs have received the barrier for a given checkpoint.
- * 
+ *
  * <p>To avoid back-pressuring the input streams (which may cause distributed deadlocks), the
- * BarrierBuffer continues receiving buffers from the blocked channels and stores them internally until 
+ * BarrierBuffer continues receiving buffers from the blocked channels and stores them internally until
  * the blocks are released.
  */
 @Internal
@@ -113,9 +113,9 @@ public class BarrierBuffer implements CheckpointBarrierHandler {
 
 	/**
 	 * Creates a new checkpoint stream aligner.
-	 * 
+	 *
 	 * <p>There is no limit to how much data may be buffered during an alignment.
-	 * 
+	 *
 	 * @param inputGate The input gate to draw the buffers and events from.
 	 * @param ioManager The I/O manager that gives access to the temp directories.
 	 *
@@ -127,15 +127,15 @@ public BarrierBuffer(InputGate inputGate, IOManager ioManager) throws IOExceptio
 
 	/**
 	 * Creates a new checkpoint stream aligner.
-	 * 
+	 *
 	 * <p>The aligner will allow only alignments that buffer up to the given number of bytes.
 	 * When that number is exceeded, it will stop the alignment and notify the task that the
 	 * checkpoint has been cancelled.
-	 * 
+	 *
 	 * @param inputGate The input gate to draw the buffers and events from.
 	 * @param ioManager The I/O manager that gives access to the temp directories.
 	 * @param maxBufferedBytes The maximum bytes to be buffered before the checkpoint aborts.
-	 * 
+	 *
 	 * @throws IOException Thrown, when the spilling to temp files cannot be initialized.
 	 */
 	public BarrierBuffer(InputGate inputGate, IOManager ioManager, long maxBufferedBytes) throws IOException {
@@ -446,7 +446,7 @@ private void beginNewAlignment(long checkpointId, int channelIndex) throws IOExc
 
 	/**
 	 * Checks whether the channel with the given index is blocked.
-	 * 
+	 *
 	 * @param channelIndex The channel index to check.
 	 * @return True if the channel is blocked, false if not.
 	 */
@@ -456,7 +456,7 @@ private boolean isBlocked(int channelIndex) {
 
 	/**
 	 * Blocks the given channel index, from which a barrier has been received.
-	 * 
+	 *
 	 * @param channelIndex The channel index to block.
 	 */
 	private void onBarrier(int channelIndex) throws IOException {
@@ -528,8 +528,8 @@ private void releaseBlocksAndResetBarriers() throws IOException {
 
 	/**
 	 * Gets the ID defining the current pending, or just completed, checkpoint.
-	 * 
-	 * @return The ID of the pending of completed checkpoint. 
+	 *
+	 * @return The ID of the pending of completed checkpoint.
 	 */
 	public long getCurrentCheckpointId() {
 		return this.currentCheckpointId;
@@ -546,9 +546,9 @@ public long getAlignmentDurationNanos() {
 	}
 
 	// ------------------------------------------------------------------------
-	// Utilities 
+	// Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public String toString() {
 		return String.format("last checkpoint: %d, current barriers: %d, closed channels: %d",
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
index c0f510e08df18..1479437c81df4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
@@ -38,11 +38,11 @@
  * The BarrierTracker keeps track of what checkpoint barriers have been received from
  * which input channels. Once it has observed all checkpoint barriers for a checkpoint ID,
  * it notifies its listener of a completed checkpoint.
- * 
+ *
  * <p>Unlike the {@link BarrierBuffer}, the BarrierTracker does not block the input
  * channels that have sent barriers, so it cannot be used to gain "exactly-once" processing
  * guarantees. It can, however, be used to gain "at least once" processing guarantees.
- * 
+ *
  * <p>NOTE: This implementation strictly assumes that newer checkpoints have higher checkpoint IDs.
  */
 @Internal
@@ -60,7 +60,7 @@ public class BarrierTracker implements CheckpointBarrierHandler {
 
 	/** The input gate, to draw the buffers and events from. */
 	private final InputGate inputGate;
-	
+
 	/**
 	 * The number of channels. Once that many barriers have been received for a checkpoint, the
 	 * checkpoint is considered complete.
@@ -72,15 +72,15 @@ public class BarrierTracker implements CheckpointBarrierHandler {
 	 * yet known to be subsumed by newer checkpoints.
 	 */
 	private final ArrayDeque<CheckpointBarrierCount> pendingCheckpoints;
-	
+
 	/** The listener to be notified on complete checkpoints. */
 	private StatefulTask toNotifyOnCheckpoint;
-	
+
 	/** The highest checkpoint ID encountered so far. */
 	private long latestPendingCheckpointID = -1;
 
 	// ------------------------------------------------------------------------
-	
+
 	public BarrierTracker(InputGate inputGate) {
 		this.inputGate = inputGate;
 		this.totalNumberOfInputChannels = inputGate.getNumberOfInputChannels();
@@ -189,7 +189,7 @@ private void processBarrier(CheckpointBarrier receivedBarrier, int channelIndex)
 			if (barrierId > latestPendingCheckpointID) {
 				latestPendingCheckpointID = barrierId;
 				pendingCheckpoints.addLast(new CheckpointBarrierCount(barrierId));
-				
+
 				// make sure we do not track too many checkpoints
 				if (pendingCheckpoints.size() > MAX_CHECKPOINTS_TO_TRACK) {
 					pendingCheckpoints.pollFirst();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java
index db10de131452e..2acf1416e9844 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BlockingQueueBroker.java
@@ -30,7 +30,7 @@
  */
 @Internal
 public class BlockingQueueBroker extends Broker<BlockingQueue<?>> {
-	
+
 	/** Singleton instance. */
 	public static final BlockingQueueBroker INSTANCE = new BlockingQueueBroker();
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
index 2de508352dc5f..fc2e376cc444c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
@@ -41,11 +41,11 @@
  * The buffer spiller takes the buffers and events from a data stream and adds them to a spill file.
  * After a number of elements have been spilled, the spiller can "roll over": It presents the spilled
  * elements as a readable sequence, and opens a new spill file.
- * 
+ *
  * <p>This implementation buffers data effectively in the OS cache, which gracefully extends to the
  * disk. Most data is written and re-read milliseconds later. The file is deleted after the read.
  * Consequently, in most cases, the data will never actually hit the physical disks.</p>
- * 
+ *
  * <p>IMPORTANT: The SpilledBufferOrEventSequences created by this spiller all reuse the same
  * reading memory (to reduce overhead) and can consequently not be read concurrently.</p>
  */
@@ -57,72 +57,72 @@ public class BufferSpiller {
 
 	/** The counter that selects the next directory to spill into. */
 	private static final AtomicInteger DIRECTORY_INDEX = new AtomicInteger(0);
-	
+
 	/** The size of the buffer with which data is read back in. */
 	private static final int READ_BUFFER_SIZE = 1024 * 1024;
-	
+
 	/** The directories to spill to. */
 	private final File tempDir;
-	
+
 	/** The name prefix for spill files. */
 	private final String spillFilePrefix;
-	
+
 	/** The buffer used for bulk reading data (used in the SpilledBufferOrEventSequence). */
 	private final ByteBuffer readBuffer;
-	
+
 	/** The buffer that encodes the spilled header. */
 	private final ByteBuffer headBuffer;
-	
+
 	/** The reusable array that holds header and contents buffers. */
 	private final ByteBuffer[] sources;
-	
+
 	/** The file that we currently spill to. */
 	private File currentSpillFile;
-	
+
 	/** The channel of the file we currently spill to. */
 	private FileChannel currentChannel;
 
 	/** The page size, to let this reader instantiate properly sized memory segments. */
 	private final int pageSize;
-	
+
 	/** A counter, to created numbered spill files. */
 	private int fileCounter;
-	
+
 	/** The number of bytes written since the last roll over. */
 	private long bytesWritten;
-	
+
 	/**
 	 * Creates a new buffer spiller, spilling to one of the I/O manager's temp directories.
-	 * 
+	 *
 	 * @param ioManager The I/O manager for access to teh temp directories.
 	 * @param pageSize The page size used to re-create spilled buffers.
 	 * @throws IOException Thrown if the temp files for spilling cannot be initialized.
 	 */
 	public BufferSpiller(IOManager ioManager, int pageSize) throws IOException {
 		this.pageSize = pageSize;
-		
+
 		this.readBuffer = ByteBuffer.allocateDirect(READ_BUFFER_SIZE);
 		this.readBuffer.order(ByteOrder.LITTLE_ENDIAN);
-		
+
 		this.headBuffer = ByteBuffer.allocateDirect(16);
 		this.headBuffer.order(ByteOrder.LITTLE_ENDIAN);
-		
+
 		this.sources = new ByteBuffer[] { this.headBuffer, null };
-		
+
 		File[] tempDirs = ioManager.getSpillingDirectories();
 		this.tempDir = tempDirs[DIRECTORY_INDEX.getAndIncrement() % tempDirs.length];
-		
+
 		byte[] rndBytes = new byte[32];
 		new Random().nextBytes(rndBytes);
 		this.spillFilePrefix = StringUtils.byteToHexString(rndBytes) + '.';
-		
+
 		// prepare for first contents
 		createSpillingChannel();
 	}
 
 	/**
 	 * Adds a buffer or event to the sequence of spilled buffers and events.
-	 * 
+	 *
 	 * @param boe The buffer or event to add and spill.
 	 * @throws IOException Thrown, if the buffer of event could not be spilled.
 	 */
@@ -159,12 +159,12 @@ public void add(BufferOrEvent boe) throws IOException {
 	 * Starts a new sequence of spilled buffers and event and returns the current sequence of spilled buffers
 	 * for reading. This method returns {@code null}, if nothing was added since the creation of the spiller, or the
 	 * last call to this method.
-	 * 
+	 *
 	 * <p>NOTE: The SpilledBufferOrEventSequences created by this method all reuse the same
 	 * reading memory (to reduce overhead) and can consequently not be read concurrently with each other.
 	 * To create a sequence that can be read concurrently with the previous SpilledBufferOrEventSequence, use the
 	 * {@link #rollOverWithNewBuffer()} method.</p>
-	 * 
+	 *
 	 * @return The readable sequence of spilled buffers and events, or 'null', if nothing was added.
 	 * @throws IOException Thrown, if the readable sequence could not be created, or no new spill
 	 *                     file could be created.
@@ -177,7 +177,7 @@ public SpilledBufferOrEventSequence rollOver() throws IOException {
 	 * Starts a new sequence of spilled buffers and event and returns the current sequence of spilled buffers
 	 * for reading. This method returns {@code null}, if nothing was added since the creation of the spiller, or the
 	 * last call to this method.
-	 * 
+	 *
 	 * <p>The SpilledBufferOrEventSequence returned by this method is safe for concurrent consumption with
 	 * any previously returned sequence.</p>
 	 *
@@ -188,7 +188,7 @@ public SpilledBufferOrEventSequence rollOver() throws IOException {
 	public SpilledBufferOrEventSequence rollOverWithNewBuffer() throws IOException {
 		return rollOverInternal(true);
 	}
-	
+
 	private SpilledBufferOrEventSequence rollOverInternal(boolean newBuffer) throws IOException {
 		if (bytesWritten == 0) {
 			return null;
@@ -204,7 +204,7 @@ private SpilledBufferOrEventSequence rollOverInternal(boolean newBuffer) throws
 
 		// create a reader for the spilled data
 		currentChannel.position(0L);
-		SpilledBufferOrEventSequence seq = 
+		SpilledBufferOrEventSequence seq =
 				new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize);
 
 		// create ourselves a new spill file
@@ -216,10 +216,10 @@ private SpilledBufferOrEventSequence rollOverInternal(boolean newBuffer) throws
 
 	/**
 	 * Cleans up the current spilling channel and file.
-	 * 
+	 *
 	 * <p>Does not clean up the SpilledBufferOrEventSequences generated by calls to
 	 * {@link #rollOver()}.
-	 * 
+	 *
 	 * @throws IOException Thrown if channel closing or file deletion fail.
 	 */
 	public void close() throws IOException {
@@ -244,15 +244,15 @@ public long getBytesWritten() {
 	File getCurrentSpillFile() {
 		return currentSpillFile;
 	}
-	
+
 	FileChannel getCurrentChannel() {
 		return currentChannel;
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@SuppressWarnings("resource")
 	private void createSpillingChannel() throws IOException {
 		currentSpillFile = new File(tempDir, spillFilePrefix + (fileCounter++) +".buffer");
@@ -291,7 +291,7 @@ public static class SpilledBufferOrEventSequence {
 
 		/**
 		 * Create a reader that reads a sequence of spilled buffers and events.
-		 * 
+		 *
 		 * @param file The file with the data.
 		 * @param fileChannel The file channel to read the data from.
 		 * @param buffer The buffer used for bulk reading.
@@ -322,14 +322,14 @@ public void open() {
 		/**
 		 * Gets the next BufferOrEvent from the spilled sequence, or {@code null}, if the
 		 * sequence is exhausted.
-		 *         
+		 *
 		 * @return The next BufferOrEvent from the spilled sequence, or {@code null} (end of sequence).
 		 * @throws IOException Thrown, if the reads failed, of if the byte stream is corrupt.
 		 */
 		public BufferOrEvent getNext() throws IOException {
 			if (buffer.remaining() < HEADER_LENGTH) {
 				buffer.compact();
-				
+
 				while (buffer.position() < HEADER_LENGTH) {
 					if (fileChannel.read(buffer) == -1) {
 						if (buffer.position() == 0) {
@@ -340,15 +340,15 @@ public BufferOrEvent getNext() throws IOException {
 						}
 					}
 				}
-				
+
 				buffer.flip();
 			}
-			
+
 			final int channel = buffer.getInt();
 			final int length = buffer.getInt();
 			final boolean isBuffer = buffer.get() == 0;
-			
-			
+
+
 			if (isBuffer) {
 				// deserialize buffer
 				if (length > pageSize) {
@@ -357,10 +357,10 @@ public BufferOrEvent getNext() throws IOException {
 				}
 
 				MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);
-				
+
 				int segPos = 0;
 				int bytesRemaining = length;
-				
+
 				while (true) {
 					int toCopy = Math.min(buffer.remaining(), bytesRemaining);
 					if (toCopy > 0) {
@@ -368,7 +368,7 @@ public BufferOrEvent getNext() throws IOException {
 						segPos += toCopy;
 						bytesRemaining -= toCopy;
 					}
-					
+
 					if (bytesRemaining == 0) {
 						break;
 					}
@@ -380,11 +380,11 @@ public BufferOrEvent getNext() throws IOException {
 						buffer.flip();
 					}
 				}
-				
-				
+
+
 				Buffer buf = new Buffer(seg, FreeingBufferRecycler.INSTANCE);
 				buf.setSize(length);
-				
+
 				return new BufferOrEvent(buf, channel);
 			}
 			else {
@@ -409,15 +409,15 @@ public BufferOrEvent getNext() throws IOException {
 				buffer.limit(buffer.position() + length);
 				AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader());
 				buffer.limit(oldLimit);
-				
+
 				return new BufferOrEvent(evt, channel);
 			}
 		}
 
 		/**
 		 * Cleans up all file resources held by this spilled sequence.
-		 * 
-		 * @throws IOException Thrown, if file channel closing or file deletion fail. 
+		 *
+		 * @throws IOException Thrown, if file channel closing or file deletion fail.
 		 */
 		public void cleanup() throws IOException {
 			fileChannel.close();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java
index 56859fbe47316..0c676e051ce31 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java
@@ -36,28 +36,28 @@ public interface CheckpointBarrierHandler {
 	 * Returns the next {@link BufferOrEvent} that the operator may consume.
 	 * This call blocks until the next BufferOrEvent is available, or until the stream
 	 * has been determined to be finished.
-	 * 
+	 *
 	 * @return The next BufferOrEvent, or {@code null}, if the stream is finished.
-	 * 
+	 *
 	 * @throws IOException Thrown if the network or local disk I/O fails.
-	 * 
+	 *
 	 * @throws InterruptedException Thrown if the thread is interrupted while blocking during
 	 *                              waiting for the next BufferOrEvent to become available.
-	 * @throws Exception Thrown in case that a checkpoint fails that is started as the result of receiving 
-	 *                   the last checkpoint barrier 
+	 * @throws Exception Thrown in case that a checkpoint fails that is started as the result of receiving
+	 *                   the last checkpoint barrier
 	 */
 	BufferOrEvent getNextNonBlocked() throws Exception;
 
 	/**
 	 * Registers the task be notified once all checkpoint barriers have been received for a checkpoint.
-	 * 
+	 *
 	 * @param task The task to notify
 	 */
 	void registerCheckpointEventHandler(StatefulTask task);
 
 	/**
 	 * Cleans up all internally held resources.
-	 * 
+	 *
 	 * @throws IOException Thrown if the cleanup of I/O resources failed.
 	 */
 	void cleanup() throws IOException;
@@ -72,7 +72,7 @@ public interface CheckpointBarrierHandler {
 	 * Gets the time that the latest alignment took, in nanoseconds.
 	 * If there is currently an alignment in progress, it will return the time spent in the
 	 * current alignment so far.
-	 * 
+	 *
 	 * @return The duration in nanoseconds
 	 */
 	long getAlignmentDurationNanos();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
index 365c78e9e209d..b8efb1c0e9b8a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
@@ -43,7 +43,7 @@
 public class RecordWriterOutput<OUT> implements Output<StreamRecord<OUT>> {
 
 	private StreamRecordWriter<SerializationDelegate<StreamElement>> recordWriter;
-	
+
 	private SerializationDelegate<StreamElement> serializationDelegate;
 
 	private final StreamStatusProvider streamStatusProvider;
@@ -59,9 +59,9 @@ public RecordWriterOutput(
 
 		checkNotNull(recordWriter);
 		this.outputTag = outputTag;
-		// generic hack: cast the writer to generic Object type so we can use it 
+		// generic hack: cast the writer to generic Object type so we can use it
 		// with multiplexed records and watermarks
-		this.recordWriter = (StreamRecordWriter<SerializationDelegate<StreamElement>>) 
+		this.recordWriter = (StreamRecordWriter<SerializationDelegate<StreamElement>>)
 				(StreamRecordWriter<?>) recordWriter;
 
 		TypeSerializer<StreamElement> outRecordSerializer =
@@ -145,12 +145,12 @@ public void emitLatencyMarker(LatencyMarker latencyMarker) {
 	public void broadcastEvent(AbstractEvent event) throws IOException, InterruptedException {
 		recordWriter.broadcastEvent(event);
 	}
-	
-	
+
+
 	public void flush() throws IOException {
 		recordWriter.flush();
 	}
-	
+
 	@Override
 	public void close() {
 		recordWriter.close();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
index e0a83c9420576..22fcf513d21fc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
@@ -64,7 +64,7 @@
  * <p>Forwarding elements, watermarks, or status status elements must be protected by synchronizing
  * on the given lock object. This ensures that we don't call methods on a
  * {@link OneInputStreamOperator} concurrently with the timer callback or other things.
- * 
+ *
  * @param <IN> The type of the record that can be read with this record reader.
  */
 @Internal
@@ -95,7 +95,7 @@ public class StreamInputProcessor<IN> {
 	private int currentChannel = -1;
 
 	private final StreamStatusMaintainer streamStatusMaintainer;
-	
+
 	private final OneInputStreamOperator<IN, ?> streamOperator;
 
 	// ---------------- Metrics ------------------
@@ -134,19 +134,19 @@ else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
 		else {
 			throw new IllegalArgumentException("Unrecognized Checkpointing Mode: " + checkpointMode);
 		}
-		
+
 		if (checkpointedTask != null) {
 			this.barrierHandler.registerCheckpointEventHandler(checkpointedTask);
 		}
 
 		this.lock = checkNotNull(lock);
-		
+
 		StreamElementSerializer<IN> ser = new StreamElementSerializer<>(inputSerializer);
 		this.deserializationDelegate = new NonReusingDeserializationDelegate<>(ser);
 
 		// Initialize one deserializer per input channel
 		this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];
-		
+
 		for (int i = 0; i < recordDeserializers.length; i++) {
 			recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
 					ioManager.getSpillingDirectoriesPaths());
@@ -238,7 +238,7 @@ public boolean processInput() throws Exception {
 
 	/**
 	 * Sets the metric group for this StreamInputProcessor.
-	 * 
+	 *
 	 * @param metrics metric group
 	 */
 	public void setMetricGroup(TaskIOMetricGroup metrics) {
@@ -256,7 +256,7 @@ public Long getValue() {
 			}
 		});
 	}
-	
+
 	public void cleanup() throws IOException {
 		// clear the buffers first. this part should not ever fail
 		for (RecordDeserializer<?> deserializer : recordDeserializers) {
@@ -265,7 +265,7 @@ public void cleanup() throws IOException {
 				buffer.recycle();
 			}
 		}
-		
+
 		// cleanup the barrier handler resources
 		barrierHandler.cleanup();
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
index 4fb30c21857c5..a67251d1926f7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
@@ -30,7 +30,7 @@
 /**
  * This record writer keeps data in buffers at most for a certain timeout. It spawns a separate thread
  * that flushes the outputs in a defined interval, to make sure data does not linger in the buffers for too long.
- * 
+ *
  * @param <T> The type of elements written.
  */
 @Internal
@@ -38,30 +38,30 @@ public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWrit
 
 	/** Default name for teh output flush thread, if no name with a task reference is given. */
 	private static final String DEFAULT_OUTPUT_FLUSH_THREAD_NAME = "OutputFlusher";
-	
-	
+
+
 	/** The thread that periodically flushes the output, to give an upper latency bound. */
 	private final OutputFlusher outputFlusher;
-	
+
 	/** Flag indicating whether the output should be flushed after every element. */
 	private final boolean flushAlways;
 
 	/** The exception encountered in the flushing thread. */
 	private Throwable flusherException;
-	
-	
-	
+
+
+
 	public StreamRecordWriter(ResultPartitionWriter writer, ChannelSelector<T> channelSelector, long timeout) {
 		this(writer, channelSelector, timeout, null);
 	}
-	
+
 	public StreamRecordWriter(ResultPartitionWriter writer, ChannelSelector<T> channelSelector,
 								long timeout, String taskName) {
-		
+
 		super(writer, channelSelector);
-		
+
 		checkArgument(timeout >= -1);
-		
+
 		if (timeout == -1) {
 			flushAlways = false;
 			outputFlusher = null;
@@ -74,12 +74,12 @@ else if (timeout == 0) {
 			flushAlways = false;
 			String threadName = taskName == null ?
 								DEFAULT_OUTPUT_FLUSH_THREAD_NAME : "Output Timeout Flusher - " + taskName;
-			
+
 			outputFlusher = new OutputFlusher(threadName, timeout);
 			outputFlusher.start();
 		}
 	}
-	
+
 	@Override
 	public void emit(T record) throws IOException, InterruptedException {
 		checkErroneous();
@@ -125,7 +125,7 @@ public void close() {
 
 	/**
 	 * Notifies the writer that the output flusher thread encountered an exception.
-	 * 
+	 *
 	 * @param t The exception to report.
 	 */
 	private void notifyFlusherException(Throwable t) {
@@ -133,7 +133,7 @@ private void notifyFlusherException(Throwable t) {
 			this.flusherException = t;
 		}
 	}
-	
+
 	private void checkErroneous() throws IOException {
 		if (flusherException != null) {
 			throw new IOException("An exception happened while flushing the outputs", flusherException);
@@ -141,25 +141,25 @@ private void checkErroneous() throws IOException {
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * A dedicated thread that periodically flushes the output buffers, to set upper latency bounds.
-	 * 
+	 *
 	 * <p>The thread is daemonic, because it is only a utility thread.
 	 */
 	private class OutputFlusher extends Thread {
-		
+
 		private final long timeout;
-		
+
 		private volatile boolean running = true;
 
-		
+
 		OutputFlusher(String name, long timeout) {
 			super(name);
 			setDaemon(true);
 			this.timeout = timeout;
 		}
-		
+
 		public void terminate() {
 			running = false;
 			interrupt();
@@ -179,9 +179,9 @@ public void run() {
 							throw new Exception(e);
 						}
 					}
-					
+
 					// any errors here should let the thread come to a halt and be
-					// recognized by the writer 
+					// recognized by the writer
 					flush();
 				}
 			}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
index a1456015da6bf..5774094a25220 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
@@ -148,13 +148,13 @@ else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
 		else {
 			throw new IllegalArgumentException("Unrecognized CheckpointingMode: " + checkpointMode);
 		}
-		
+
 		if (checkpointedTask != null) {
 			this.barrierHandler.registerCheckpointEventHandler(checkpointedTask);
 		}
 
 		this.lock = checkNotNull(lock);
-		
+
 		StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
 		this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);
 
@@ -163,7 +163,7 @@ else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
 
 		// Initialize one deserializer per input channel
 		this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];
-		
+
 		for (int i = 0; i < recordDeserializers.length; i++) {
 			recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
 					ioManager.getSpillingDirectoriesPaths());
@@ -174,7 +174,7 @@ else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
 		for (InputGate gate: inputGates1) {
 			numInputChannels1 += gate.getNumberOfInputChannels();
 		}
-		
+
 		this.numInputChannels1 = numInputChannels1;
 		this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;
 
@@ -273,7 +273,7 @@ else if (recordOrWatermark.isLatencyMarker()) {
 					currentChannel = bufferOrEvent.getChannelIndex();
 					currentRecordDeserializer = recordDeserializers[currentChannel];
 					currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
-	
+
 				} else {
 					// Event received
 					final AbstractEvent event = bufferOrEvent.getEvent();
@@ -312,7 +312,7 @@ public Long getValue() {
 			}
 		});
 	}
-	
+
 	public void cleanup() throws IOException {
 		// clear the buffers first. this part should not ever fail
 		for (RecordDeserializer<?> deserializer : recordDeserializers) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/ExtractTimestampsOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/ExtractTimestampsOperator.java
index a10e4579be8ae..cec1f060eff9b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/ExtractTimestampsOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/ExtractTimestampsOperator.java
@@ -30,7 +30,7 @@
  * from user elements and assigning them as the internal timestamp of the {@link StreamRecord}.
  *
  * @param <T> The type of the input elements
- * 
+ *
  * @deprecated Subsumed by {@link TimestampsAndPeriodicWatermarksOperator} and
  *             {@link TimestampsAndPunctuatedWatermarksOperator}.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperator.java
index 086358034582e..8fe65cf34f5f6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperator.java
@@ -36,12 +36,12 @@ public class TimestampsAndPeriodicWatermarksOperator<T>
 		implements OneInputStreamOperator<T, T>, ProcessingTimeCallback {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	private transient long watermarkInterval;
 
 	private transient long currentWatermark;
 
-	
+
 	public TimestampsAndPeriodicWatermarksOperator(AssignerWithPeriodicWatermarks<T> assigner) {
 		super(assigner);
 		this.chainingStrategy = ChainingStrategy.ALWAYS;
@@ -53,7 +53,7 @@ public void open() throws Exception {
 
 		currentWatermark = Long.MIN_VALUE;
 		watermarkInterval = getExecutionConfig().getAutoWatermarkInterval();
-		
+
 		if (watermarkInterval > 0) {
 			long now = getProcessingTimeService().getCurrentProcessingTime();
 			getProcessingTimeService().registerTimer(now + watermarkInterval, this);
@@ -62,9 +62,9 @@ public void open() throws Exception {
 
 	@Override
 	public void processElement(StreamRecord<T> element) throws Exception {
-		final long newTimestamp = userFunction.extractTimestamp(element.getValue(), 
+		final long newTimestamp = userFunction.extractTimestamp(element.getValue(),
 				element.hasTimestamp() ? element.getTimestamp() : Long.MIN_VALUE);
-		
+
 		output.collect(element.replace(element.getValue(), newTimestamp));
 	}
 
@@ -100,7 +100,7 @@ public void processWatermark(Watermark mark) throws Exception {
 	@Override
 	public void close() throws Exception {
 		super.close();
-		
+
 		// emit a final watermark
 		Watermark newWatermark = userFunction.getCurrentWatermark();
 		if (newWatermark != null && newWatermark.getTimestamp() > currentWatermark) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperator.java
index 3fc9f9dd18406..45a55defdfcdf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperator.java
@@ -47,7 +47,7 @@ public TimestampsAndPunctuatedWatermarksOperator(AssignerWithPunctuatedWatermark
 	@Override
 	public void processElement(StreamRecord<T> element) throws Exception {
 		final T value = element.getValue();
-		final long newTimestamp = userFunction.extractTimestamp(value, 
+		final long newTimestamp = userFunction.extractTimestamp(value,
 				element.hasTimestamp() ? element.getTimestamp() : Long.MIN_VALUE);
 
 		output.collect(element.replace(element.getValue(), newTimestamp));
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java
index fd1a71f16d9a2..ee5d7f19649b8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/package-info.java
@@ -17,6 +17,6 @@
 
 /**
  * This package contains the operators that perform the stream transformations.
- * One or more operators are bundled into a "chain" and executed in a stream task. 
+ * One or more operators are bundled into a "chain" and executed in a stream task.
  */
 package org.apache.flink.streaming.runtime.operators;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
index cc0bd646c717c..70d76d4ab54d4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
@@ -47,38 +47,38 @@
  */
 @Internal
 @Deprecated
-public abstract class AbstractAlignedProcessingTimeWindowOperator<KEY, IN, OUT, STATE, F extends Function> 
-		extends AbstractUdfStreamOperator<OUT, F> 
+public abstract class AbstractAlignedProcessingTimeWindowOperator<KEY, IN, OUT, STATE, F extends Function>
+		extends AbstractUdfStreamOperator<OUT, F>
 		implements OneInputStreamOperator<IN, OUT>, ProcessingTimeCallback {
-	
+
 	private static final long serialVersionUID = 3245500864882459867L;
-	
+
 	private static final long MIN_SLIDE_TIME = 50;
-	
+
 	// ----- fields for operator parametrization -----
-	
+
 	private final Function function;
 	private final KeySelector<IN, KEY> keySelector;
-	
+
 	private final TypeSerializer<KEY> keySerializer;
 	private final TypeSerializer<STATE> stateTypeSerializer;
-	
+
 	private final long windowSize;
 	private final long windowSlide;
 	private final long paneSize;
 	private final int numPanesPerWindow;
-	
+
 	// ----- fields for operator functionality -----
-	
+
 	private transient AbstractKeyedTimePanes<IN, KEY, STATE, OUT> panes;
-	
+
 	private transient TimestampedCollector<OUT> out;
-	
+
 	private transient RestoredState<IN, KEY, STATE, OUT> restoredState;
-	
+
 	private transient long nextEvaluationTime;
 	private transient long nextSlideTime;
-	
+
 	protected AbstractAlignedProcessingTimeWindowOperator(
 			F function,
 			KeySelector<IN, KEY> keySelector,
@@ -88,7 +88,7 @@ protected AbstractAlignedProcessingTimeWindowOperator(
 			long windowSlide)
 	{
 		super(function);
-		
+
 		if (windowLength < MIN_SLIDE_TIME) {
 			throw new IllegalArgumentException("Window length must be at least " + MIN_SLIDE_TIME + " msecs");
 		}
@@ -98,14 +98,14 @@ protected AbstractAlignedProcessingTimeWindowOperator(
 		if (windowLength < windowSlide) {
 			throw new IllegalArgumentException("The window size must be larger than the window slide");
 		}
-		
+
 		final long paneSlide = ArithmeticUtils.gcd(windowLength, windowSlide);
 		if (paneSlide < MIN_SLIDE_TIME) {
 			throw new IllegalArgumentException(String.format(
 					"Cannot compute window of size %d msecs sliding by %d msecs. " +
 							"The unit of grouping is too small: %d msecs", windowLength, windowSlide, paneSlide));
 		}
-		
+
 		this.function = requireNonNull(function);
 		this.keySelector = requireNonNull(keySelector);
 		this.keySerializer = requireNonNull(keySerializer);
@@ -115,8 +115,8 @@ protected AbstractAlignedProcessingTimeWindowOperator(
 		this.paneSize = paneSlide;
 		this.numPanesPerWindow = MathUtils.checkedDownCast(windowLength / paneSlide);
 	}
-	
-	
+
+
 	protected abstract AbstractKeyedTimePanes<IN, KEY, STATE, OUT> createPanes(
 			KeySelector<IN, KEY> keySelector, Function function);
 
@@ -129,7 +129,7 @@ public void open() throws Exception {
 		super.open();
 
 		out = new TimestampedCollector<>(output);
-		
+
 		// decide when to first compute the window and when to slide it
 		// the values should align with the start of time (that is, the UNIX epoch, not the big bang)
 		final long now = getProcessingTimeService().getCurrentProcessingTime();
@@ -137,7 +137,7 @@ public void open() throws Exception {
 		nextSlideTime = now + paneSize - (now % paneSize);
 
 		final long firstTriggerTime = Math.min(nextEvaluationTime, nextSlideTime);
-		
+
 		// check if we restored state and if we need to fire some windows based on that restored state
 		if (restoredState == null) {
 			// initial empty state: create empty panes that gather the elements per slide
@@ -146,12 +146,12 @@ public void open() throws Exception {
 		else {
 			// restored state
 			panes = restoredState.panes;
-			
+
 			long nextPastEvaluationTime = restoredState.nextEvaluationTime;
 			long nextPastSlideTime = restoredState.nextSlideTime;
 			long nextPastTriggerTime = Math.min(nextPastEvaluationTime, nextPastSlideTime);
 			int numPanesRestored = panes.getNumPanes();
-			
+
 			// fire windows from the past as long as there are more panes with data and as long
 			// as the missed trigger times have not caught up with the presence
 			while (numPanesRestored > 0 && nextPastTriggerTime < firstTriggerTime) {
@@ -160,7 +160,7 @@ public void open() throws Exception {
 					computeWindow(nextPastTriggerTime);
 					nextPastEvaluationTime += windowSlide;
 				}
-				
+
 				// evaluate slide from the past
 				if (nextPastTriggerTime == nextPastSlideTime) {
 					panes.slidePanes(numPanesPerWindow);
@@ -179,7 +179,7 @@ public void open() throws Exception {
 	@Override
 	public void close() throws Exception {
 		super.close();
-		
+
 		// early stop the triggering thread, so it does not attempt to return any more data
 		stopTriggers();
 	}
@@ -187,7 +187,7 @@ public void close() throws Exception {
 	@Override
 	public void dispose() throws Exception {
 		super.dispose();
-		
+
 		// acquire the lock during shutdown, to prevent trigger calls at the same time
 		// fail-safe stop of the triggering thread (in case of an error)
 		stopTriggers();
@@ -198,7 +198,7 @@ public void dispose() throws Exception {
 			panes.dispose();
 		}
 	}
-	
+
 	private void stopTriggers() {
 		// reset the action timestamps. this makes sure any pending triggers will not evaluate
 		nextEvaluationTime = -1L;
@@ -208,7 +208,7 @@ private void stopTriggers() {
 	// ------------------------------------------------------------------------
 	//  Receiving elements and triggers
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public void processElement(StreamRecord<IN> element) throws Exception {
 		panes.addElementToLatestPane(element.getValue());
@@ -234,7 +234,7 @@ public void onProcessingTime(long timestamp) throws Exception {
 		long nextTriggerTime = Math.min(nextEvaluationTime, nextSlideTime);
 		getProcessingTimeService().registerTimer(nextTriggerTime, this);
 	}
-	
+
 	private void computeWindow(long timestamp) throws Exception {
 		out.setAbsoluteTimestamp(timestamp);
 		panes.truncatePanes(numPanesPerWindow);
@@ -293,7 +293,7 @@ public long getWindowSlide() {
 	public long getPaneSize() {
 		return paneSize;
 	}
-	
+
 	public int getNumPanesPerWindow() {
 		return numPanesPerWindow;
 	}
@@ -309,7 +309,7 @@ public long getNextSlideTime() {
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public String toString() {
 		return "Window (processing time) (length=" + windowSize + ", slide=" + windowSlide + ')';
@@ -317,7 +317,7 @@ public String toString() {
 
 	// ------------------------------------------------------------------------
 	// ------------------------------------------------------------------------
-	
+
 	private static final class RestoredState<IN, KEY, STATE, OUT> {
 
 		final AbstractKeyedTimePanes<IN, KEY, STATE, OUT> panes;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
index 037b4d7e930ed..76f2a97ce7f4e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
@@ -35,11 +35,11 @@
  */
 @Internal
 public abstract class AbstractKeyedTimePanes<Type, Key, Aggregate, Result> {
-	
+
 	private static final int BEGIN_OF_STATE_MAGIC_NUMBER = 0x0FF1CE42;
 
 	private static final int BEGIN_OF_PANE_MAGIC_NUMBER = 0xBADF00D5;
-	
+
 	/** The latest time pane. */
 	protected KeyMap<Key, Aggregate> latestPane = new KeyMap<>();
 
@@ -51,19 +51,19 @@ public abstract class AbstractKeyedTimePanes<Type, Key, Aggregate, Result> {
 	public abstract void addElementToLatestPane(Type element) throws Exception;
 
 	public abstract void evaluateWindow(Collector<Result> out, TimeWindow window, AbstractStreamOperator<Result> operator) throws Exception;
-	
-	
+
+
 	public void dispose() {
 		// since all is heap data, there is no need to clean up anything
 		latestPane = null;
 		previousPanes.clear();
 	}
-	
+
 	public int getNumPanes() {
 		return previousPanes.size() + 1;
 	}
-	
-	
+
+
 	public void slidePanes(int panesToKeep) {
 		if (panesToKeep > 1) {
 			// the current pane becomes the latest previous pane
@@ -78,13 +78,13 @@ public void slidePanes(int panesToKeep) {
 		// we need a new latest pane
 		latestPane = new KeyMap<>();
 	}
-	
+
 	public void truncatePanes(int numToRetain) {
 		while (previousPanes.size() >= numToRetain) {
 			previousPanes.removeFirst();
 		}
 	}
-	
+
 	protected void traverseAllPanes(KeyMap.TraversalEvaluator<Key, Aggregate> traversal, long traversalPass) throws Exception{
 		// gather all panes in an array (faster iterations)
 		@SuppressWarnings({"unchecked", "rawtypes"})
@@ -94,7 +94,7 @@ protected void traverseAllPanes(KeyMap.TraversalEvaluator<Key, Aggregate> traver
 		// let the maps make a coordinated traversal and evaluate the window function per contained key
 		KeyMap.traverseMaps(panes, traversal, traversalPass);
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Serialization and de-serialization
 	// ------------------------------------------------------------------------
@@ -105,16 +105,16 @@ public void writeToOutput(
 			final TypeSerializer<Aggregate> aggSerializer) throws IOException
 	{
 		output.writeInt(BEGIN_OF_STATE_MAGIC_NUMBER);
-		
+
 		int numPanes = getNumPanes();
 		output.writeInt(numPanes);
-		
+
 		// write from the past
 		Iterator<KeyMap<Key, Aggregate>> previous = previousPanes.iterator();
 		for (int paneNum = 0; paneNum < numPanes; paneNum++) {
 			output.writeInt(BEGIN_OF_PANE_MAGIC_NUMBER);
 			KeyMap<Key, Aggregate> pane = (paneNum == numPanes - 1) ? latestPane : previous.next();
-			
+
 			output.writeInt(pane.size());
 			for (KeyMap.Entry<Key, Aggregate> entry : pane) {
 				keySerializer.serialize(entry.getKey(), output);
@@ -122,7 +122,7 @@ public void writeToOutput(
 			}
 		}
 	}
-	
+
 	public void readFromInput(
 			final DataInputView input,
 			final TypeSerializer<Key> keySerializer,
@@ -130,26 +130,26 @@ public void readFromInput(
 	{
 		validateMagicNumber(BEGIN_OF_STATE_MAGIC_NUMBER, input.readInt());
 		int numPanes = input.readInt();
-		
+
 		// read from the past towards the presence
 		while (numPanes > 0) {
 			validateMagicNumber(BEGIN_OF_PANE_MAGIC_NUMBER, input.readInt());
 			KeyMap<Key, Aggregate> pane = (numPanes == 1) ? latestPane : new KeyMap<Key, Aggregate>();
-			
+
 			final int numElementsInPane = input.readInt();
 			for (int i = numElementsInPane - 1; i >= 0; i--) {
 				Key k = keySerializer.deserialize(input);
 				Aggregate a = aggSerializer.deserialize(input);
 				pane.put(k, a);
 			}
-			
+
 			if (numPanes > 1) {
 				previousPanes.addLast(pane);
 			}
 			numPanes--;
 		}
 	}
-	
+
 	private static void validateMagicNumber(int expected, int found) throws IOException {
 		if (expected != found) {
 			throw new IOException("Corrupt state stream - wrong magic number. " +
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
index 954dc10a7f748..5d95cdc07d121 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
@@ -45,7 +45,7 @@
  */
 @Internal
 public class AccumulatingKeyedTimePanes<Type, Key, Result> extends AbstractKeyedTimePanes<Type, Key, ArrayList<Type>, Result> {
-	
+
 	private final KeySelector<Type, Key> keySelector;
 
 	private final KeyMap.LazyFactory<ArrayList<Type>> listFactory = getListFactory();
@@ -58,10 +58,10 @@ public class AccumulatingKeyedTimePanes<Type, Key, Result> extends AbstractKeyed
 	 * IMPORTANT: This value needs to start at one, so it is fresher than the value that new entries
 	 * have (zero).
 	 */
-	private long evaluationPass = 1L;   
+	private long evaluationPass = 1L;
 
 	// ------------------------------------------------------------------------
-	
+
 	public AccumulatingKeyedTimePanes(KeySelector<Type, Key> keySelector, InternalWindowFunction<Iterable<Type>, Result, Key, Window> function) {
 		this.keySelector = keySelector;
 		this.function = function;
@@ -97,28 +97,28 @@ public void evaluateWindow(Collector<Result> out, final TimeWindow window,
 					function, window, out, operator, context);
 			traverseAllPanes(evaluator, evaluationPass);
 		}
-		
+
 		evaluationPass++;
 	}
 
 	// ------------------------------------------------------------------------
 	//  Running a window function in a map traversal
 	// ------------------------------------------------------------------------
-	
+
 	static final class WindowFunctionTraversal<Key, Type, Result> implements KeyMap.TraversalEvaluator<Key, ArrayList<Type>> {
 
 		private final InternalWindowFunction<Iterable<Type>, Result, Key, Window> function;
-		
+
 		private final UnionIterator<Type> unionIterator;
-		
+
 		private final Collector<Result> out;
 
 		private final TimeWindow window;
-		
+
 		private final AbstractStreamOperator<Result> contextOperator;
 
 		private Key currentKey;
-		
+
 		private AccumulatingKeyedTimePanesContext context;
 
 		WindowFunctionTraversal(InternalWindowFunction<Iterable<Type>, Result, Key, Window> function, TimeWindow window,
@@ -150,11 +150,11 @@ public void keyDone() throws Exception {
 			function.process(currentKey, window, context, unionIterator, out);
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Lazy factory for lists (put if absent)
 	// ------------------------------------------------------------------------
-	
+
 	@SuppressWarnings("unchecked")
 	private static <V> KeyMap.LazyFactory<ArrayList<V>> getListFactory() {
 		return (KeyMap.LazyFactory<ArrayList<V>>) LIST_FACTORY;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
index 507237455ca3f..d71f80e5a5956 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
@@ -43,7 +43,7 @@ public class AccumulatingProcessingTimeWindowOperator<KEY, IN, OUT>
 
 	private static final long serialVersionUID = 7305948082830843475L;
 
-	
+
 	public AccumulatingProcessingTimeWindowOperator(
 			InternalWindowFunction<Iterable<IN>, OUT, KEY, TimeWindow> function,
 			KeySelector<IN, KEY> keySelector,
@@ -60,7 +60,7 @@ public AccumulatingProcessingTimeWindowOperator(
 	protected AccumulatingKeyedTimePanes<IN, KEY, OUT> createPanes(KeySelector<IN, KEY> keySelector, Function function) {
 		@SuppressWarnings("unchecked")
 		InternalWindowFunction<Iterable<IN>, OUT, KEY, Window> windowFunction = (InternalWindowFunction<Iterable<IN>, OUT, KEY, Window>) function;
-		
+
 		return new AccumulatingKeyedTimePanes<>(keySelector, windowFunction);
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java
index d26ab9d7ad646..66d41f1548206 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingKeyedTimePanes.java
@@ -30,9 +30,9 @@
  */
 @Internal
 public class AggregatingKeyedTimePanes<Type, Key> extends AbstractKeyedTimePanes<Type, Key, Type, Type> {
-	
+
 	private final KeySelector<Type, Key> keySelector;
-	
+
 	private final ReduceFunction<Type> reducer;
 
 	/**
@@ -42,7 +42,7 @@ public class AggregatingKeyedTimePanes<Type, Key> extends AbstractKeyedTimePanes
 	private long evaluationPass = 1L;
 
 	// ------------------------------------------------------------------------
-	
+
 	public AggregatingKeyedTimePanes(KeySelector<Type, Key> keySelector, ReduceFunction<Type> reducer) {
 		this.keySelector = keySelector;
 		this.reducer = reducer;
@@ -57,7 +57,7 @@ public void addElementToLatestPane(Type element) throws Exception {
 	}
 
 	@Override
-	public void evaluateWindow(Collector<Type> out, TimeWindow window, 
+	public void evaluateWindow(Collector<Type> out, TimeWindow window,
 								AbstractStreamOperator<Type> operator) throws Exception {
 		if (previousPanes.isEmpty()) {
 			// optimized path for single pane case
@@ -70,22 +70,22 @@ public void evaluateWindow(Collector<Type> out, TimeWindow window,
 			AggregatingTraversal<Key, Type> evaluator = new AggregatingTraversal<>(reducer, out, operator);
 			traverseAllPanes(evaluator, evaluationPass);
 		}
-		
+
 		evaluationPass++;
 	}
 
 	// ------------------------------------------------------------------------
 	//  The maps traversal that performs the final aggregation
 	// ------------------------------------------------------------------------
-	
+
 	static final class AggregatingTraversal<Key, Type> implements KeyMap.TraversalEvaluator<Key, Type> {
 
 		private final ReduceFunction<Type> function;
-		
+
 		private final Collector<Type> out;
-		
+
 		private final AbstractStreamOperator<Type> operator;
-		
+
 		private Type currentValue;
 
 		AggregatingTraversal(ReduceFunction<Type> function, Collector<Type> out,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
index 51803e67c36bd..b83815c5ba4d2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
@@ -33,12 +33,12 @@
  */
 @Internal
 @Deprecated
-public class AggregatingProcessingTimeWindowOperator<KEY, IN> 
+public class AggregatingProcessingTimeWindowOperator<KEY, IN>
 		extends AbstractAlignedProcessingTimeWindowOperator<KEY, IN, IN, IN, ReduceFunction<IN>> {
 
 	private static final long serialVersionUID = 7305948082830843475L;
 
-	
+
 	public AggregatingProcessingTimeWindowOperator(
 			ReduceFunction<IN> function,
 			KeySelector<IN, KEY> keySelector,
@@ -54,7 +54,7 @@ public AggregatingProcessingTimeWindowOperator(
 	protected AggregatingKeyedTimePanes<IN, KEY> createPanes(KeySelector<IN, KEY> keySelector, Function function) {
 		@SuppressWarnings("unchecked")
 		ReduceFunction<IN> windowFunction = (ReduceFunction<IN>) function;
-		
+
 		return new AggregatingKeyedTimePanes<IN, KEY>(keySelector, windowFunction);
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
index 4e50d8280ed3f..d251ddd417731 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
@@ -44,33 +44,33 @@
  */
 @Internal
 public class KeyMap<K, V> implements Iterable<KeyMap.Entry<K, V>> {
-	
+
 	/** The minimum table capacity, 64 entries. */
 	private static final int MIN_CAPACITY = 0x40;
-	
+
 	/**
 	 * The maximum possible table capacity, the largest positive power of two in the 32bit signed
 	 * integer value range.
 	 */
 	private static final int MAX_CAPACITY = 0x40000000;
-	
+
 	/** The number of bits used for table addressing when table is at max capacity. */
 	private static final int FULL_BIT_RANGE = MathUtils.log2strict(MAX_CAPACITY);
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	/** The hash index, as an array of entries. */
 	private Entry<K, V>[] table;
-	
+
 	/** The number of bits by which the hash code is shifted right, to find the bucket. */
 	private int shift;
-	
+
 	/** The number of elements in the hash table. */
 	private int numElements;
-	
+
 	/** The number of elements above which the hash table needs to grow. */
 	private int rehashThreshold;
-	
+
 	/** The base-2 logarithm of the table capacity. */
 	private int log2size;
 
@@ -85,14 +85,14 @@ public KeyMap() {
 
 	/**
 	 * Creates a new table with a capacity tailored to the given expected number of elements.
-	 * 
+	 *
 	 * @param expectedNumberOfElements The number of elements to tailor the capacity to.
 	 */
 	public KeyMap(int expectedNumberOfElements) {
 		if (expectedNumberOfElements < 0) {
 			throw new IllegalArgumentException("Invalid capacity: " + expectedNumberOfElements);
 		}
-		
+
 		// round up to the next power or two
 		// guard against too small capacity and integer overflows
 		int capacity = Integer.highestOneBit(expectedNumberOfElements) << 1;
@@ -113,17 +113,17 @@ public KeyMap(int expectedNumberOfElements) {
 	 * Inserts the given value, mapped under the given key. If the table already contains a value for
 	 * the key, the value is replaced and returned. If no value is contained, yet, the function
 	 * returns null.
-	 * 
+	 *
 	 * @param key The key to insert.
 	 * @param value The value to insert.
 	 * @return The previously mapped value for the key, or null, if no value was mapped for the key.
-	 * 
+	 *
 	 * @throws java.lang.NullPointerException Thrown, if the key is null.
 	 */
 	public final V put(K key, V value) {
 		final int hash = hash(key);
 		final int slot = indexOf (hash);
-		
+
 		// search the chain from the slot
 		for (Entry<K, V> e = table[slot]; e != null; e = e.next) {
 			Object k;
@@ -151,7 +151,7 @@ public final V put(K key, V value) {
 	 * @param factory The factory that produces the value, if no value is contained, yet, for the key.
 	 * @return The value in the map after this operation (either the previously contained value, or the
 	 *         newly created value).
-	 * 
+	 *
 	 * @throws java.lang.NullPointerException Thrown, if the key is null.
 	 */
 	public final V putIfAbsent(K key, LazyFactory<V> factory) {
@@ -178,14 +178,14 @@ public final V putIfAbsent(K key, LazyFactory<V> factory) {
 	 * Inserts or aggregates a value into the hash map. If the hash map does not yet contain the key,
 	 * this method inserts the value. If the table already contains the key (and a value) this
 	 * method will use the given ReduceFunction function to combine the existing value and the
-	 * given value to a new value, and store that value for the key. 
-	 * 
+	 * given value to a new value, and store that value for the key.
+	 *
 	 * @param key The key to map the value.
 	 * @param value The new value to insert, or aggregate with the existing value.
 	 * @param aggregator The aggregator to use if a value is already contained.
-	 * 
+	 *
 	 * @return The value in the map after this operation: Either the given value, or the aggregated value.
-	 * 
+	 *
 	 * @throws java.lang.NullPointerException Thrown, if the key is null.
 	 * @throws Exception The method forwards exceptions from the aggregation function.
 	 */
@@ -210,23 +210,23 @@ public final V putOrAggregate(K key, V value, ReduceFunction<V> aggregator) thro
 
 	/**
 	 * Looks up the value mapped under the given key. Returns null if no value is mapped under this key.
-	 * 
+	 *
 	 * @param key The key to look up.
 	 * @return The value associated with the key, or null, if no value is found for the key.
-	 * 
+	 *
 	 * @throws java.lang.NullPointerException Thrown, if the key is null.
 	 */
 	public V get(K key) {
 		final int hash = hash(key);
 		final int slot = indexOf(hash);
-		
+
 		// search the chain from the slot
 		for (Entry<K, V> entry = table[slot]; entry != null; entry = entry.next) {
 			if (entry.hashCode == hash && entry.key.equals(key)) {
 				return entry.value;
 			}
 		}
-		
+
 		// not found
 		return null;
 	}
@@ -241,26 +241,26 @@ private void insertNewEntry(int hashCode, K key, V value, int position) {
 			growTable();
 		}
 	}
-	
+
 	private int indexOf(int hashCode) {
 		return (hashCode >> shift) & (table.length - 1);
 	}
 
 	/**
 	 * Creates an iterator over the entries of this map.
-	 * 
+	 *
 	 * @return An iterator over the entries of this map.
 	 */
 	@Override
 	public Iterator<Entry<K, V>> iterator() {
 		return new Iterator<Entry<K, V>>() {
-			
+
 			private final Entry<K, V>[] tab = KeyMap.this.table;
-			
+
 			private Entry<K, V> nextEntry;
-			
+
 			private int nextPos = 0;
-			
+
 			@Override
 			public boolean hasNext() {
 				if (nextEntry != null) {
@@ -296,11 +296,11 @@ public void remove() {
 			}
 		};
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Properties
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Gets the number of elements currently in the map.
 	 * @return The number of elements currently in the map.
@@ -329,33 +329,33 @@ public int getCurrentTableCapacity() {
 	/**
 	 * Gets the base-2 logarithm of the hash table capacity, as returned by
 	 * {@link #getCurrentTableCapacity()}.
-	 * 
+	 *
 	 * @return The base-2 logarithm of the hash table capacity.
 	 */
 	public int getLog2TableCapacity() {
 		return log2size;
 	}
-	
+
 	public int getRehashThreshold() {
 		return rehashThreshold;
 	}
-	
+
 	public int getShift() {
 		return shift;
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@SuppressWarnings("unchecked")
 	private Entry<K, V>[] allocateTable(int numElements) {
 		return (Entry<K, V>[]) new Entry<?, ?>[numElements];
 	}
-	
+
 	private void growTable() {
 		final int newSize = table.length << 1;
-				
+
 		// only grow if there is still space to grow the table
 		if (newSize > 0) {
 			final Entry<K, V>[] oldTable = this.table;
@@ -363,7 +363,7 @@ private void growTable() {
 
 			final int newShift = shift - 1;
 			final int newMask = newSize - 1;
-			
+
 			// go over all slots from the table. since we hash to adjacent positions in
 			// the new hash table, this is actually cache efficient
 			for (Entry<K, V> entry : oldTable) {
@@ -376,17 +376,17 @@ private void growTable() {
 					entry = nextEntry;
 				}
 			}
-			
+
 			this.table = newTable;
 			this.shift = newShift;
 			this.rehashThreshold = getRehashThreshold(newSize);
 			this.log2size += 1;
 		}
 	}
-	
+
 	private static int hash(Object key) {
 		int code = key.hashCode();
-		
+
 		// we need a strong hash function that generates diverse upper bits
 		// this hash function is more expensive than the "scramble" used by "java.util.HashMap",
 		// but required for this sort of hash table
@@ -397,7 +397,7 @@ private static int hash(Object key) {
 		code = (code + 0xfd7046c5) + (code << 3);
 		return (code ^ 0xb55a4f09) ^ (code >>> 16);
 	}
-	
+
 	private static int getRehashThreshold(int capacity) {
 		// divide before multiply, to avoid overflow
 		return capacity / 4 * 3;
@@ -410,26 +410,26 @@ private static int getRehashThreshold(int capacity) {
 	/**
 	 * For testing only: Actively counts the number of entries, rather than using the
 	 * counter variable. This method has linear complexity, rather than constant.
-	 * 
+	 *
 	 * @return The counted number of entries.
 	 */
 	int traverseAndCountElements() {
 		int num = 0;
-		
+
 		for (Entry<?, ?> entry : table) {
 			while (entry != null) {
 				num++;
 				entry = entry.next;
 			}
 		}
-		
+
 		return num;
 	}
 
 	/**
 	 * For testing only: Gets the length of the longest overflow chain.
 	 * This method has linear complexity.
-	 * 
+	 *
 	 * @return The length of the longest overflow chain.
 	 */
 	int getLongestChainLength() {
@@ -451,15 +451,15 @@ int getLongestChainLength() {
 
 	/**
 	 * An entry in the hash table.
-	 * 
+	 *
 	 * @param <K> Type of the key.
 	 * @param <V> Type of the value.
 	 */
 	public static final class Entry<K, V> {
-		
+
 		final K key;
 		final int hashCode;
-		
+
 		V value;
 		Entry<K, V> next;
 		long touchedTag;
@@ -479,20 +479,20 @@ public V getValue() {
 			return value;
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	/**
 	 * Performs a traversal about logical the multi-map that results from the union of the
 	 * given maps. This method does not actually build a union of the map, but traverses the hash maps
 	 * together.
-	 * 
+	 *
 	 * @param maps The array uf maps whose union should be traversed.
 	 * @param visitor The visitor that is called for each key and all values.
 	 * @param touchedTag A tag that is used to mark elements that have been touched in this specific
 	 *                   traversal. Each successive traversal should supply a larger value for this
 	 *                   tag than the previous one.
-	 * 
+	 *
 	 * @param <K> The type of the map's key.
 	 * @param <V> The type of the map's value.
 	 */
@@ -504,21 +504,21 @@ public static <K, V> void traverseMaps(
 	{
 		// we need to work on the maps in descending size
 		Arrays.sort(maps, CapacityDescendingComparator.INSTANCE);
-		
+
 		final int[] shifts = new int[maps.length];
 		final int[] lowBitsMask = new int[maps.length];
 		final int numSlots = maps[0].table.length;
 		final int numTables = maps.length;
-		
+
 		// figure out how much each hash table collapses the entries
 		for (int i = 0; i < numTables; i++) {
 			shifts[i] = maps[0].log2size - maps[i].log2size;
 			lowBitsMask[i] = (1 << shifts[i]) - 1;
 		}
-		
+
 		// go over all slots (based on the largest hash table)
 		for (int pos = 0; pos < numSlots; pos++) {
-			
+
 			// for each slot, go over all tables, until the table does not have that slot any more
 			// for tables where multiple slots collapse into one, we visit that one when we process the
 			// latest of all slots that collapse to that one
@@ -534,14 +534,14 @@ public static <K, V> void traverseMaps(
 					// take only entries that have not been collected as part of other tables
 					if (entry.touchedTag < touchedTag) {
 						entry.touchedTag = touchedTag;
-						
+
 						final K key = entry.key;
 						final int hashCode = entry.hashCode;
 						visitor.startNewKey(key);
 						visitor.nextValue(entry.value);
-						
+
 						addEntriesFromChain(entry.next, visitor, key, touchedTag, hashCode);
-						
+
 						// go over the other hash tables and collect their entries for the key
 						for (int followupTable = rootTable + 1; followupTable < numTables; followupTable++) {
 							Entry<K, V> followupEntry = maps[followupTable].table[pos >> shifts[followupTable]];
@@ -552,13 +552,13 @@ public static <K, V> void traverseMaps(
 
 						visitor.keyDone();
 					}
-					
+
 					entry = entry.next;
 				}
 			}
 		}
 	}
-	
+
 	private static <K, V> void addEntriesFromChain(
 			Entry<K, V> entry,
 			TraversalEvaluator<K, V> visitor,
@@ -576,15 +576,15 @@ private static <K, V> void addEntriesFromChain(
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Comparator that defines a descending order on maps depending on their table capacity
 	 * and number of elements.
 	 */
 	static final class CapacityDescendingComparator implements Comparator<KeyMap<?, ?>> {
-		
+
 		static final CapacityDescendingComparator INSTANCE = new CapacityDescendingComparator();
-		
+
 		private CapacityDescendingComparator() {}
 
 
@@ -622,8 +622,8 @@ public interface LazyFactory<V> {
 	/**
 	 * A visitor for a traversal over the union of multiple hash maps. The visitor is
 	 * called for each key in the union of the maps and all values associated with that key
-	 * (one per map, but multiple across maps). 
-	 * 
+	 * (one per map, but multiple across maps).
+	 *
 	 * @param <K> The type of the key.
 	 * @param <V> The type of the value.
 	 */
@@ -631,7 +631,7 @@ public interface TraversalEvaluator<K, V> {
 
 		/**
 		 * Called whenever the traversal starts with a new key.
-		 * 
+		 *
 		 * @param key The key traversed.
 		 * @throws Exception Method forwards all exceptions.
 		 */
@@ -639,7 +639,7 @@ public interface TraversalEvaluator<K, V> {
 
 		/**
 		 * Called for each value found for the current key.
-		 * 
+		 *
 		 * @param value The next value.
 		 * @throws Exception Method forwards all exceptions.
 		 */
@@ -647,7 +647,7 @@ public interface TraversalEvaluator<K, V> {
 
 		/**
 		 * Called when the traversal for the current key is complete.
-		 * 
+		 *
 		 * @throws Exception Method forwards all exceptions.
 		 */
 		void keyDone() throws Exception;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/package-info.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/package-info.java
index 55749a112d56a..229812cd03e02 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/package-info.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/package-info.java
@@ -17,6 +17,6 @@
 
 /**
  * This package contains the operators that implement the various window operations
- * on data streams. 
+ * on data streams.
  */
 package org.apache.flink.streaming.runtime.operators.windowing;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ForwardPartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ForwardPartitioner.java
index ad313fdb530d6..0ae737c344d18 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ForwardPartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ForwardPartitioner.java
@@ -23,7 +23,7 @@
 
 /**
  * Partitioner that forwards elements only to the locally running downstream operation.
- * 
+ *
  * @param <T> Type of the elements in the Stream
  */
 @Internal
@@ -36,11 +36,11 @@ public class ForwardPartitioner<T> extends StreamPartitioner<T> {
 	public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record, int numberOfOutputChannels) {
 		return returnArray;
 	}
-	
+
 	public StreamPartitioner<T> copy() {
 		return this;
 	}
-	
+
 	@Override
 	public String toString() {
 		return "FORWARD";
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitioner.java
index dd2d4fff50bae..a81f9739d5b0f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitioner.java
@@ -24,7 +24,7 @@
 /**
  * Partitioner that distributes the data equally by cycling through the output
  * channels.
- * 
+ *
  * @param <T> Type of the elements in the Stream being rebalanced
  */
 @Internal
@@ -39,11 +39,11 @@ public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
 		this.returnArray[0] = (this.returnArray[0] + 1) % numberOfOutputChannels;
 		return this.returnArray;
 	}
-	
+
 	public StreamPartitioner<T> copy() {
 		return this;
 	}
-	
+
 	@Override
 	public String toString() {
 		return "REBALANCE";
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitioner.java
index a6d9d0d71746c..90615230048d3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitioner.java
@@ -55,11 +55,11 @@ public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record, int n
 		this.returnArray[0] = (this.returnArray[0] + 1) % numberOfOutputChannels;
 		return this.returnArray;
 	}
-	
+
 	public StreamPartitioner<T> copy() {
 		return this;
 	}
-	
+
 	@Override
 	public String toString() {
 		return "RESCALE";
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ShufflePartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ShufflePartitioner.java
index 36c123a724ce2..87243fc5275ba 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ShufflePartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/ShufflePartitioner.java
@@ -26,7 +26,7 @@
 /**
  * Partitioner that distributes the data equally by selecting one output channel
  * randomly.
- * 
+ *
  * @param <T>
  *            Type of the Tuple
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
index 643e240893d71..10e0a9393f5a2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElement.java
@@ -27,7 +27,7 @@
  */
 @Internal
 public abstract class StreamElement {
-	
+
 	/**
 	 * Checks whether this element is a watermark.
 	 * @return True, if this element is a watermark, false otherwise.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index f2b8aec3687ec..64396da0eec3e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -42,17 +42,17 @@
 public final class StreamElementSerializer<T> extends TypeSerializer<StreamElement> {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	private static final int TAG_REC_WITH_TIMESTAMP = 0;
 	private static final int TAG_REC_WITHOUT_TIMESTAMP = 1;
 	private static final int TAG_WATERMARK = 2;
 	private static final int TAG_LATENCY_MARKER = 3;
 	private static final int TAG_STREAM_STATUS = 4;
-	
-	
+
+
 	private final TypeSerializer<T> typeSerializer;
 
-	
+
 	public StreamElementSerializer(TypeSerializer<T> serializer) {
 		if (serializer instanceof StreamElementSerializer) {
 			throw new RuntimeException("StreamRecordSerializer given to StreamRecordSerializer as value TypeSerializer: " + serializer);
@@ -67,7 +67,7 @@ public TypeSerializer<T> getContainedTypeSerializer() {
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -82,7 +82,7 @@ public StreamElementSerializer<T> duplicate() {
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public StreamRecord<T> createInstance() {
 		return new StreamRecord<T>(typeSerializer.createInstance());
@@ -160,7 +160,7 @@ else if (tag == TAG_LATENCY_MARKER) {
 	public void serialize(StreamElement value, DataOutputView target) throws IOException {
 		if (value.isRecord()) {
 			StreamRecord<T> record = value.asRecord();
-			
+
 			if (record.hasTimestamp()) {
 				target.write(TAG_REC_WITH_TIMESTAMP);
 				target.writeLong(record.getTimestamp());
@@ -187,7 +187,7 @@ else if (value.isLatencyMarker()) {
 			throw new RuntimeException();
 		}
 	}
-	
+
 	@Override
 	public StreamElement deserialize(DataInputView source) throws IOException {
 		int tag = source.readByte();
@@ -242,7 +242,7 @@ else if (tag == TAG_LATENCY_MARKER) {
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public boolean equals(Object obj) {
 		if (obj instanceof StreamElementSerializer) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java
index 03958976f191c..a0610ab4cf1c5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecord.java
@@ -21,21 +21,21 @@
 
 /**
  * One value in a data stream. This stores the value and an optional associated timestamp.
- * 
+ *
  * @param <T> The type encapsulated with the stream record.
  */
 @Internal
 public final class StreamRecord<T> extends StreamElement {
-	
+
 	/** The actual value held by this record. */
 	private T value;
-	
+
 	/** The timestamp of the record. */
 	private long timestamp;
 
 	/** Flag whether the timestamp is actually set. */
 	private boolean hasTimestamp;
-	
+
 	/**
 	 * Creates a new StreamRecord. The record does not have a timestamp.
 	 */
@@ -59,7 +59,7 @@ public StreamRecord(T value, long timestamp) {
 	// ------------------------------------------------------------------------
 	//  Accessors
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Returns the value wrapped in this stream value.
 	 */
@@ -82,7 +82,7 @@ public long getTimestamp() {
 	}
 
 	/** Checks whether this record has a timestamp.
-	 * 
+	 *
  	 * @return True if the record has a timestamp, false if not.
 	 */
 	public boolean hasTimestamp() {
@@ -92,7 +92,7 @@ public boolean hasTimestamp() {
 	// ------------------------------------------------------------------------
 	//  Updating
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * Replace the currently stored value by the given new value. This returns a StreamElement
 	 * with the generic type parameter that matches the new value while keeping the old
@@ -121,10 +121,10 @@ public <X> StreamRecord<X> replace(X value, long timestamp) {
 		this.timestamp = timestamp;
 		this.value = (T) value;
 		this.hasTimestamp = true;
-		
+
 		return (StreamRecord<X>) this;
 	}
-	
+
 	public void setTimestamp(long timestamp) {
 		this.timestamp = timestamp;
 		this.hasTimestamp = true;
@@ -133,7 +133,7 @@ public void setTimestamp(long timestamp) {
 	public void eraseTimestamp() {
 		this.hasTimestamp = false;
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Copying
 	// ------------------------------------------------------------------------
@@ -162,7 +162,7 @@ public void copyTo(T valueCopy, StreamRecord<T> target) {
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public boolean equals(Object o) {
 		if (this == o) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java
index 39d3ed2fa545c..1af81d009fb63 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsynchronousException.java
@@ -22,7 +22,7 @@
 
 /**
  * An exception for wrapping exceptions that are thrown by an operator in threads other than the
- * main compute thread of that operator. 
+ * main compute thread of that operator.
  */
 @Internal
 public class AsynchronousException extends Exception {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
index c36d45d0b8eb0..9d7c3d70300fe 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
@@ -37,12 +37,12 @@ public ExceptionInChainedOperatorException(Throwable cause) {
 	public ExceptionInChainedOperatorException(String message, Throwable cause) {
 		super(message, requireNonNull(cause));
 	}
-	
+
 	public Throwable getOriginalCause() {
 		Throwable ex = this;
 		do {
 			ex = ex.getCause();
 		} while (ex instanceof ExceptionInChainedOperatorException);
-		return ex; 
+		return ex;
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
index 0343b54cc213a..f461e31540f0b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
@@ -32,7 +32,7 @@
 public class OneInputStreamTask<IN, OUT> extends StreamTask<OUT, OneInputStreamOperator<IN, OUT>> {
 
 	private StreamInputProcessor<IN> inputProcessor;
-	
+
 	private volatile boolean running = true;
 
 	@Override
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index aced2f416e514..93cf6463031d6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -61,19 +61,19 @@
 /**
  * The {@code OperatorChain} contains all operators that are executed as one chain within a single
  * {@link StreamTask}.
- * 
+ *
  * @param <OUT> The type of elements accepted by the chain, i.e., the input type of the chain's
  *              head operator.
  */
 @Internal
 public class OperatorChain<OUT, OP extends StreamOperator<OUT>> implements StreamStatusMaintainer {
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(OperatorChain.class);
-	
+
 	private final StreamOperator<?>[] allOperators;
 
 	private final RecordWriterOutput<?>[] streamOutputs;
-	
+
 	private final Output<StreamRecord<OUT>> chainEntryPoint;
 
 	private final OP headOperator;
@@ -87,7 +87,7 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> implements Strea
 	private StreamStatus streamStatus = StreamStatus.ACTIVE;
 
 	public OperatorChain(StreamTask<OUT, OP> containingTask) {
-		
+
 		final ClassLoader userCodeClassloader = containingTask.getUserCodeClassLoader();
 		final StreamConfig configuration = containingTask.getConfiguration();
 
@@ -102,7 +102,7 @@ public OperatorChain(StreamTask<OUT, OP> containingTask) {
 		List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(userCodeClassloader);
 		Map<StreamEdge, RecordWriterOutput<?>> streamOutputMap = new HashMap<>(outEdgesInOrder.size());
 		this.streamOutputs = new RecordWriterOutput<?>[outEdgesInOrder.size()];
-		
+
 		// from here on, we need to make sure that the output writers are shut down again on failure
 		boolean success = false;
 		try {
@@ -112,11 +112,11 @@ public OperatorChain(StreamTask<OUT, OP> containingTask) {
 				RecordWriterOutput<?> streamOutput = createStreamOutput(
 						outEdge, chainedConfigs.get(outEdge.getSourceId()), i,
 						containingTask.getEnvironment(), containingTask.getName());
-	
+
 				this.streamOutputs[i] = streamOutput;
 				streamOutputMap.put(outEdge, streamOutput);
 			}
-	
+
 			// we create the chain of operators and grab the collector that leads into the chain
 			List<StreamOperator<?>> allOps = new ArrayList<>(chainedConfigs.size());
 			this.chainEntryPoint = createOutputCollector(containingTask, configuration,
@@ -131,7 +131,7 @@ public OperatorChain(StreamTask<OUT, OP> containingTask) {
 			allOps.add(headOperator);
 
 			this.allOperators = allOps.toArray(new StreamOperator<?>[allOps.size()]);
-			
+
 			success = true;
 		}
 		finally {
@@ -146,7 +146,7 @@ public OperatorChain(StreamTask<OUT, OP> containingTask) {
 				}
 			}
 		}
-		
+
 	}
 
 	@Override
@@ -193,7 +193,7 @@ public void broadcastCheckpointCancelMarker(long id) throws IOException {
 	public RecordWriterOutput<?>[] getStreamOutputs() {
 		return streamOutputs;
 	}
-	
+
 	public StreamOperator<?>[] getAllOperators() {
 		return allOperators;
 	}
@@ -247,7 +247,7 @@ public int getChainLength() {
 	// ------------------------------------------------------------------------
 	//  initialization utilities
 	// ------------------------------------------------------------------------
-	
+
 	private <T> Output<StreamRecord<T>> createOutputCollector(
 			StreamTask<?, ?> containingTask,
 			StreamConfig operatorConfig,
@@ -257,7 +257,7 @@ private <T> Output<StreamRecord<T>> createOutputCollector(
 			List<StreamOperator<?>> allOperators)
 	{
 		List<Tuple2<Output<StreamRecord<T>>, StreamEdge>> allOutputs = new ArrayList<>(4);
-		
+
 		// create collectors for the network outputs
 		for (StreamEdge outputEdge : operatorConfig.getNonChainedOutputs(userCodeClassloader)) {
 			@SuppressWarnings("unchecked")
@@ -275,12 +275,12 @@ private <T> Output<StreamRecord<T>> createOutputCollector(
 					containingTask, chainedOpConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperators, outputEdge.getOutputTag());
 			allOutputs.add(new Tuple2<>(output, outputEdge));
 		}
-		
+
 		// if there are multiple outputs, or the outputs are directed, we need to
 		// wrap them as one output
-		
+
 		List<OutputSelector<T>> selectors = operatorConfig.getOutputSelectors(userCodeClassloader);
-		
+
 		if (selectors == null || selectors.isEmpty()) {
 			// simple path, no selector necessary
 			if (allOutputs.size() == 1) {
@@ -316,10 +316,10 @@ private <T> Output<StreamRecord<T>> createOutputCollector(
 			} else {
 				return new DirectedOutput<>(selectors, allOutputs);
 			}
-			
+
 		}
 	}
-	
+
 	private <IN, OUT> Output<StreamRecord<IN>> createChainedOperator(
 			StreamTask<?, ?> containingTask,
 			StreamConfig operatorConfig,
@@ -348,7 +348,7 @@ private <IN, OUT> Output<StreamRecord<IN>> createChainedOperator(
 			return new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
 		}
 	}
-	
+
 	private <T> RecordWriterOutput<T> createStreamOutput(
 			StreamEdge edge, StreamConfig upStreamConfig, int outputIndex,
 			Environment taskEnvironment,
@@ -370,7 +370,7 @@ private <T> RecordWriterOutput<T> createStreamOutput(
 		StreamPartitioner<T> outputPartitioner = (StreamPartitioner<T>) edge.getPartitioner();
 
 		LOG.debug("Using partitioner {} for output {} of task ", outputPartitioner, outputIndex, taskName);
-		
+
 		ResultPartitionWriter bufferWriter = taskEnvironment.getWriter(outputIndex);
 
 		// we initialize the partitioner here with the number of key groups (aka max. parallelism)
@@ -384,16 +384,16 @@ private <T> RecordWriterOutput<T> createStreamOutput(
 		StreamRecordWriter<SerializationDelegate<StreamRecord<T>>> output =
 				new StreamRecordWriter<>(bufferWriter, outputPartitioner, upStreamConfig.getBufferTimeout());
 		output.setMetricGroup(taskEnvironment.getMetricGroup().getIOMetricGroup());
-		
+
 		return new RecordWriterOutput<>(output, outSerializer, sideOutputTag, this);
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Collectors for output chaining
-	// ------------------------------------------------------------------------ 
+	// ------------------------------------------------------------------------
 
 	private static class ChainingOutput<T> implements Output<StreamRecord<T>> {
-		
+
 		protected final OneInputStreamOperator<T, ?> operator;
 		protected final Counter numRecordsIn;
 
@@ -482,7 +482,7 @@ public void close() {
 	}
 
 	private static final class CopyingChainingOutput<T> extends ChainingOutput<T> {
-		
+
 		private final TypeSerializer<T> serializer;
 
 		public CopyingChainingOutput(
@@ -533,9 +533,9 @@ protected <X> void pushToOperator(StreamRecord<X> record) {
 
 		}
 	}
-	
+
 	private static class BroadcastingOutputCollector<T> implements Output<StreamRecord<T>> {
-		
+
 		protected final Output<StreamRecord<T>>[] outputs;
 
 		private final Random RNG = new XORShiftRandom();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
index 1164ea02663e9..9bccae093d850 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeCallback.java
@@ -33,7 +33,7 @@ public interface ProcessingTimeCallback {
 	 * <p>If the triggering is delayed for whatever reason (trigger timer was blocked, JVM stalled
 	 * due to a garbage collection), the timestamp supplied to this function will still be the
 	 * original timestamp for which the trigger was scheduled.
-	 * 
+	 *
 	 * @param timestamp The timestamp for which the trigger event was scheduled.
 	 */
 	void onProcessingTime(long timestamp) throws Exception;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeService.java
index 240aba8026eb9..11074a2afc079 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ProcessingTimeService.java
@@ -22,10 +22,10 @@
 /**
  * Defines the current processing time and handles all related actions,
  * such as register timers for tasks to be executed in the future.
- * 
+ *
  * <p>The access to the time via {@link #getCurrentProcessingTime()} is always available, regardless of
  * whether the timer service has been shut down.
- * 
+ *
  * <p>The registration of timers follows a life cycle of three phases:
  * <ol>
  *     <li>In the initial state, it accepts timer registrations and triggers when the time is reached.</li>
@@ -46,10 +46,10 @@ public abstract class ProcessingTimeService {
 
 	/**
 	 * Registers a task to be executed when (processing) time is {@code timestamp}.
-	 * 
+	 *
 	 * @param timestamp   Time when the task is to be executed (in processing time)
 	 * @param target      The task to be executed
-	 * 
+	 *
 	 * @return The future that represents the scheduled task. This always returns some future,
 	 *         even if the timer was shut down
 	 */
@@ -75,7 +75,7 @@ public abstract class ProcessingTimeService {
 	 * returns for each call to {@link #registerTimer(long, ProcessingTimeCallback)} only a "mock" future.
 	 * Furthermore, the method clears all not yet started timers, and awaits the completion
 	 * of currently executing timers.
-	 * 
+	 *
 	 * <p>This method can be used to cleanly shut down the timer service. The using components
 	 * will not notice that the service is shut down (as for example via exceptions when registering
 	 * a new timer), but the service will simply not fire any timer any more.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
index a3913ce7a3450..66e92dff70179 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
@@ -49,13 +49,13 @@ protected void init() {
 	protected void cleanup() {
 		// does not hold any resources, so no cleanup needed
 	}
-	
+
 
 	@Override
 	protected void run() throws Exception {
 		headOperator.run(getCheckpointLock(), getStreamStatusMaintainer());
 	}
-	
+
 	@Override
 	protected void cancelTask() throws Exception {
 		if (headOperator != null) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
index d05ec37ff1ca4..0d2fe1fa409d4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
@@ -42,18 +42,18 @@ public class StreamIterationHead<OUT> extends OneInputStreamTask<OUT, OUT> {
 	private volatile boolean running = true;
 
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	protected void run() throws Exception {
-		
+
 		final String iterationId = getConfiguration().getIterationId();
 		if (iterationId == null || iterationId.length() == 0) {
 			throw new Exception("Missing iteration ID in the task configuration");
 		}
-		
+
 		final String brokerID = createBrokerIdString(getEnvironment().getJobID(), iterationId ,
 				getEnvironment().getTaskInfo().getIndexOfThisSubtask());
-		
+
 		final long iterationWaitTime = getConfiguration().getIterationWaitTime();
 		final boolean shouldWait = iterationWaitTime > 0;
 
@@ -63,7 +63,7 @@ protected void run() throws Exception {
 		BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
 		LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);
 
-		// do the work 
+		// do the work
 		try {
 			@SuppressWarnings("unchecked")
 			RecordWriterOutput<OUT>[] outputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();
@@ -114,7 +114,7 @@ public void init() {
 	protected void cleanup() throws Exception {
 		// does not hold any resources, no cleanup necessary
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
@@ -123,7 +123,7 @@ protected void cleanup() throws Exception {
 	 * Creates the identification string with which head and tail task find the shared blocking
 	 * queue for the back channel. The identification string is unique per parallel head/tail pair
 	 * per iteration per job.
-	 * 
+	 *
 	 * @param jid The job ID.
 	 * @param iterationID The id of the iteration in the job.
 	 * @param subtaskIndex The parallel subtask number
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
index 7326a8082dee4..df5edb1c78c6e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
@@ -70,7 +70,7 @@ public void init() throws Exception {
 	}
 
 	private static class RecordPusher<IN> extends AbstractStreamOperator<IN> implements OneInputStreamOperator<IN, IN> {
-		
+
 		private static final long serialVersionUID = 1L;
 
 		@Override
@@ -93,9 +93,9 @@ private static class IterationTailOutput<IN> implements Output<StreamRecord<IN>>
 
 		@SuppressWarnings("NonSerializableFieldInSerializableClass")
 		private final BlockingQueue<StreamRecord<IN>> dataChannel;
-		
+
 		private final long iterationWaitTime;
-		
+
 		private final boolean shouldWait;
 
 		IterationTailOutput(BlockingQueue<StreamRecord<IN>> dataChannel, long iterationWaitTime) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index e73c825aebaa7..da4f9b63ef903 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -543,7 +543,7 @@ public void triggerCheckpointOnBarrier(
 			performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics);
 		}
 		catch (CancelTaskException e) {
-			LOG.info("Operator {} was cancelled while performing checkpoint {}.", 
+			LOG.info("Operator {} was cancelled while performing checkpoint {}.",
 					getName(), checkpointMetaData.getCheckpointId());
 			throw e;
 		}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskException.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskException.java
index 94e2213e9bacc..8bfb9a0984ff5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskException.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTaskException.java
@@ -39,7 +39,7 @@ public StreamTaskException() {
 
 	/**
 	 * Creates a compiler exception with the given message and no cause.
-	 * 
+	 *
 	 * @param message
 	 *            The message for the exception.
 	 */
@@ -49,7 +49,7 @@ public StreamTaskException(String message) {
 
 	/**
 	 * Creates a compiler exception with the given cause and no message.
-	 * 
+	 *
 	 * @param cause
 	 *            The <tt>Throwable</tt> that caused this exception.
 	 */
@@ -59,7 +59,7 @@ public StreamTaskException(Throwable cause) {
 
 	/**
 	 * Creates a compiler exception with the given message and cause.
-	 * 
+	 *
 	 * @param message
 	 *            The message for the exception.
 	 * @param cause
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
index 4c2d731d2170a..34202fbc7446c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
@@ -169,7 +169,7 @@ public void quiesceAndAwaitPending() throws InterruptedException {
 
 	@Override
 	public void shutdownService() {
-		if (status.compareAndSet(STATUS_ALIVE, STATUS_SHUTDOWN) || 
+		if (status.compareAndSet(STATUS_ALIVE, STATUS_SHUTDOWN) ||
 				status.compareAndSet(STATUS_QUIESCED, STATUS_SHUTDOWN))
 		{
 			timerService.shutdownNow();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
index 50aefc871831e..7214201be8ca2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
@@ -35,24 +35,24 @@
 public class TwoInputStreamTask<IN1, IN2, OUT> extends StreamTask<OUT, TwoInputStreamOperator<IN1, IN2, OUT>> {
 
 	private StreamTwoInputProcessor<IN1, IN2> inputProcessor;
-	
+
 	private volatile boolean running = true;
 
 	@Override
 	public void init() throws Exception {
 		StreamConfig configuration = getConfiguration();
 		ClassLoader userClassLoader = getUserCodeClassLoader();
-		
+
 		TypeSerializer<IN1> inputDeserializer1 = configuration.getTypeSerializerIn1(userClassLoader);
 		TypeSerializer<IN2> inputDeserializer2 = configuration.getTypeSerializerIn2(userClassLoader);
-	
+
 		int numberOfInputs = configuration.getNumberOfInputs();
-	
+
 		ArrayList<InputGate> inputList1 = new ArrayList<InputGate>();
 		ArrayList<InputGate> inputList2 = new ArrayList<InputGate>();
-	
+
 		List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);
-	
+
 		for (int i = 0; i < numberOfInputs; i++) {
 			int inputType = inEdges.get(i).getTypeNumber();
 			InputGate reader = getEnvironment().getInputGate(i);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
index b377d701265c0..43440460a1b00 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
@@ -50,7 +50,7 @@ public static <X> KeySelector<X, Tuple> getSelectorForKeys(Keys<X> keys, TypeInf
 		}
 
 		CompositeType<X> compositeType = (CompositeType<X>) typeInfo;
-		
+
 		int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
 		int numKeyFields = logicalKeyPositions.length;
 
@@ -69,9 +69,9 @@ public static <X> ArrayKeySelector<X> getSelectorForArray(int[] positions, TypeI
 		if (positions == null || positions.length == 0 || positions.length > Tuple.MAX_ARITY) {
 			throw new IllegalArgumentException("Array keys must have between 1 and " + Tuple.MAX_ARITY + " fields.");
 		}
-		
+
 		TypeInformation<?> componentType;
-		
+
 		if (typeInfo instanceof BasicArrayTypeInfo) {
 			BasicArrayTypeInfo<X, ?>  arrayInfo = (BasicArrayTypeInfo<X, ?>) typeInfo;
 			componentType = arrayInfo.getComponentInfo();
@@ -83,14 +83,14 @@ else if (typeInfo instanceof PrimitiveArrayTypeInfo) {
 		else {
 			throw new IllegalArgumentException("This method only supports arrays of primitives and boxed primitives.");
 		}
-		
+
 		TypeInformation<?>[] primitiveInfos = new TypeInformation<?>[positions.length];
 		Arrays.fill(primitiveInfos, componentType);
 
 		return new ArrayKeySelector<>(positions, new TupleTypeInfo<>(primitiveInfos));
 	}
 
-	
+
 	public static <X, K> KeySelector<X, K> getSelectorForOneKey(
 			Keys<X> keys, Partitioner<K> partitioner, TypeInformation<X> typeInfo, ExecutionConfig executionConfig)
 	{
@@ -107,7 +107,7 @@ public static <X, K> KeySelector<X, K> getSelectorForOneKey(
 		if (logicalKeyPositions.length != 1) {
 			throw new IllegalArgumentException("There must be exactly 1 key specified");
 		}
-		
+
 		TypeComparator<X> comparator = compositeType.createComparator(
 				logicalKeyPositions, new boolean[] { true }, 0, executionConfig);
 		return new OneKeySelector<>(comparator);
@@ -125,8 +125,8 @@ private KeySelectorUtil() {
 	// ------------------------------------------------------------------------
 
 	/**
-	 * Key extractor that extracts a single field via a generic comparator. 
-	 * 
+	 * Key extractor that extracts a single field via a generic comparator.
+	 *
 	 * @param <IN> The type of the elements where the key is extracted from.
 	 * @param <K> The type of the key.
 	 */
@@ -140,7 +140,7 @@ public static final class OneKeySelector<IN, K> implements KeySelector<IN, K> {
 		 * are null), it does not have any serialization problems */
 		@SuppressWarnings("NonSerializableFieldInSerializableClass")
 		private final Object[] keyArray;
-		
+
 		OneKeySelector(TypeComparator<IN> comparator) {
 			this.comparator = comparator;
 			this.keyArray = new Object[1];
@@ -155,7 +155,7 @@ public K getKey(IN value) throws Exception {
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * A key selector for selecting key fields via a TypeComparator.
 	 *
@@ -201,16 +201,16 @@ public TypeInformation<Tuple> getProducedType() {
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	/**
 	 * A key selector for selecting individual array fields as keys and returns them as a Tuple.
-	 * 
+	 *
 	 * @param <IN> The type from which the key is extracted, i.e., the array type.
 	 */
 	public static final class ArrayKeySelector<IN> implements KeySelector<IN, Tuple>, ResultTypeQueryable<Tuple> {
 
 		private static final long serialVersionUID = 1L;
-		
+
 		private final int[] fields;
 		private final Class<? extends Tuple> tupleClass;
 		private transient TupleTypeInfo<Tuple> returnType;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java
index 77e76e5b5c96d..02ea0041f0d73 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java
@@ -27,10 +27,10 @@
  * The deserialization schema describes how to turn the byte messages delivered by certain
  * data sources (for example Apache Kafka) into data types (Java/Scala objects) that are
  * processed by Flink.
- * 
+ *
  * <p>This base variant of the deserialization schema produces the type information
  * automatically by extracting it from the generic class arguments.
- * 
+ *
  * @param <T> The type created by the deserialization schema.
  */
 public abstract class AbstractDeserializationSchema<T> implements DeserializationSchema<T> {
@@ -49,7 +49,7 @@ public abstract class AbstractDeserializationSchema<T> implements Deserializatio
 	/**
 	 * Method to decide whether the element signals the end of the stream. If
 	 * true is returned the element won't be emitted.
-	 * 
+	 *
 	 * <p>This default implementation returns always false, meaning the stream is interpreted
 	 * to be unbounded.
 	 *
@@ -60,7 +60,7 @@ public abstract class AbstractDeserializationSchema<T> implements Deserializatio
 	public boolean isEndOfStream(T nextElement) {
 		return false;
 	}
-	
+
 	@Override
 	public TypeInformation<T> getProducedType() {
 		return TypeExtractor.createTypeInfo(AbstractDeserializationSchema.class, getClass(), 0, null, null);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java
index 03cab209941b4..57cb27efc5ede 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java
@@ -27,10 +27,10 @@
  * The deserialization schema describes how to turn the byte messages delivered by certain
  * data sources (for example Apache Kafka) into data types (Java/Scala objects) that are
  * processed by Flink.
- * 
+ *
  * <p>Note: In most cases, one should start from {@link AbstractDeserializationSchema}, which
  * takes care of producing the return type information automatically.
- * 
+ *
  * @param <T> The type created by the deserialization schema.
  */
 @Public
@@ -38,7 +38,7 @@ public interface DeserializationSchema<T> extends Serializable, ResultTypeQuerya
 
 	/**
 	 * Deserializes the byte message.
-	 * 
+	 *
 	 * @param message The message, as a byte array.
 	 *
 	 * @return The deserialized message as an object (null if the message cannot be deserialized).
@@ -48,7 +48,7 @@ public interface DeserializationSchema<T> extends Serializable, ResultTypeQuerya
 	/**
 	 * Method to decide whether the element signals the end of the stream. If
 	 * true is returned the element won't be emitted.
-	 * 
+	 *
 	 * @param nextElement The element to test for the end-of-stream signal.
 	 * @return True, if the element signals end of stream, false otherwise.
 	 */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java
index 27c85e3d048f5..986cfb3039fda 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java
@@ -25,7 +25,7 @@
  * The serialization schema describes how to turn a data object into a different serialized
  * representation. Most data sinks (for example Apache Kafka) require the data to be handed
  * to them in a specific format (for example as byte strings).
- * 
+ *
  * @param <T> The type to be serialized.
  */
 @Public
@@ -33,7 +33,7 @@ public interface SerializationSchema<T> extends Serializable {
 
 	/**
 	 * Serializes the incoming element to a specified type.
-	 * 
+	 *
 	 * @param element
 	 *            The incoming element to be serialized
 	 * @return The serialized element.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
index ddc55a2bf1fb8..27ba9e913331f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
@@ -30,7 +30,7 @@
 
 /**
  * Very simple serialization schema for strings.
- * 
+ *
  * <p>By default, the serializer uses "UTF-8" for string/byte conversion.
  */
 @PublicEvolving
@@ -51,7 +51,7 @@ public SimpleStringSchema() {
 
 	/**
 	 * Creates a new SimpleStringSchema that uses the given charset to convert between strings and bytes.
-	 * 
+	 *
 	 * @param charset The charset to use to convert between strings and bytes.
 	 */
 	public SimpleStringSchema(Charset charset) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
index e81a1c7194499..b187b041f7250 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
@@ -30,20 +30,20 @@
 /**
  * A serialization and deserialization schema that uses Flink's serialization stack to
  * transform typed from and to byte arrays.
- * 
+ *
  * @param <T> The type to be serialized.
  */
 @Public
 public class TypeInformationSerializationSchema<T> implements DeserializationSchema<T>, SerializationSchema<T> {
-	
+
 	private static final long serialVersionUID = -5359448468131559102L;
-	
+
 	/** The serializer for the actual de-/serialization. */
 	private final TypeSerializer<T> serializer;
 
 	/** The reusable output serialization buffer. */
 	private transient DataOutputSerializer dos;
-	
+
 	/** The reusable input deserialization buffer. */
 	private transient DataInputDeserializer dis;
 
@@ -58,7 +58,7 @@ public class TypeInformationSerializationSchema<T> implements DeserializationSch
 
 	/**
 	 * Creates a new de-/serialization schema for the given type.
-	 * 
+	 *
 	 * @param typeInfo The type information for the type de-/serialized by this schema.
 	 * @param ec The execution config, which is used to parametrize the type serializers.
 	 */
@@ -68,7 +68,7 @@ public TypeInformationSerializationSchema(TypeInformation<T> typeInfo, Execution
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	@Override
 	public T deserialize(byte[] message) {
 		if (dis != null) {
@@ -76,7 +76,7 @@ public T deserialize(byte[] message) {
 		} else {
 			dis = new DataInputDeserializer(message, 0, message.length);
 		}
-		
+
 		try {
 			return serializer.deserialize(dis);
 		}
@@ -100,14 +100,14 @@ public byte[] serialize(T element) {
 		if (dos == null) {
 			dos = new DataOutputSerializer(16);
 		}
-		
+
 		try {
 			serializer.serialize(element, dos);
 		}
 		catch (IOException e) {
 			throw new RuntimeException("Unable to serialize record", e);
 		}
-		
+
 		byte[] ret = dos.getByteArray();
 		if (ret.length != dos.length()) {
 			byte[] n = new byte[dos.length()];
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
index 1183306576bb9..7dc889cdbbe53 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
@@ -166,7 +166,7 @@ public void pojoGroupSumIntegerTest() throws Exception {
 				new StreamGroupedReduce<>(sumFunction, typeInfo.createSerializer(config)),
 				getInputPojoList(),
 				keySelector, keyType);
-		
+
 		List<MyPojo> groupedMinList = MockContext.createAndExecuteForKeyedStream(
 				new StreamGroupedReduce<>(minFunction, typeInfo.createSerializer(config)),
 				getInputPojoList(),
@@ -181,11 +181,11 @@ public void pojoGroupSumIntegerTest() throws Exception {
 		assertEquals(expectedGroupMinList, groupedMinList);
 		assertEquals(expectedGroupMaxList, groupedMaxList);
 	}
-	
+
 	@Test
 	public void minMaxByTest() throws Exception {
 		// Tuples are grouped on field 0, aggregated on field 1
-		
+
 		// preparing expected outputs
 		List<Tuple3<Integer, Integer, Integer>> maxByFirstExpected = ImmutableList.of(
 				Tuple3.of(0,0,0), Tuple3.of(0,1,1), Tuple3.of(0,2,2),
@@ -217,7 +217,7 @@ public void minMaxByTest() throws Exception {
 				new Keys.ExpressionKeys<>(new int[]{0}, typeInfo),
 				typeInfo, config);
 		TypeInformation<Tuple> keyType = TypeExtractor.getKeySelectorTypes(keySelector, typeInfo);
-		
+
 		// aggregations tested
 		ReduceFunction<Tuple3<Integer, Integer, Integer>> maxByFunctionFirst =
 				new ComparableAggregator<>(1, typeInfo, AggregationType.MAXBY, true, config);
@@ -232,17 +232,17 @@ public void minMaxByTest() throws Exception {
 				new StreamGroupedReduce<>(maxByFunctionFirst, typeInfo.createSerializer(config)),
 				getInputByList(),
 				keySelector, keyType));
-		
+
 		assertEquals(maxByLastExpected, MockContext.createAndExecuteForKeyedStream(
 				new StreamGroupedReduce<>(maxByFunctionLast, typeInfo.createSerializer(config)),
 				getInputByList(),
 				keySelector, keyType));
-		
+
 		assertEquals(minByLastExpected, MockContext.createAndExecuteForKeyedStream(
 				new StreamGroupedReduce<>(minByFunctionLast, typeInfo.createSerializer(config)),
 				getInputByList(),
 				keySelector, keyType));
-		
+
 		assertEquals(minByFirstExpected, MockContext.createAndExecuteForKeyedStream(
 				new StreamGroupedReduce<>(minByFunctionFirst, typeInfo.createSerializer(config)),
 				getInputByList(),
@@ -298,12 +298,12 @@ public void pojoMinMaxByTest() throws Exception {
 						new StreamGroupedReduce<>(maxByFunctionFirst, typeInfo.createSerializer(config)),
 						getInputByPojoList(),
 						keySelector, keyType));
-		
+
 		assertEquals(maxByLastExpected, MockContext.createAndExecuteForKeyedStream(
 				new StreamGroupedReduce<>(maxByFunctionLast, typeInfo.createSerializer(config)),
 				getInputByPojoList(),
 				keySelector, keyType));
-		
+
 		assertEquals(minByLastExpected, MockContext.createAndExecuteForKeyedStream(
 				new StreamGroupedReduce<>(minByFunctionLast, typeInfo.createSerializer(config)),
 				getInputByPojoList(),
@@ -352,7 +352,7 @@ private List<MyPojo3> getInputByPojoList() {
 	}
 
 	public static class MyPojo implements Serializable {
-		
+
 		private static final long serialVersionUID = 1L;
 		public int f0;
 		public int f1;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java
index 566065594ab0a..e5fbfdac2f512 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java
@@ -252,9 +252,9 @@ public void flatMap1(Long value, Collector<Long> out) throws Exception {}
 
 					@Override
 					public void flatMap2(Long value, Collector<Long> out) throws Exception {}
-					
+
 				}).name("testCoFlatMap")
-				
+
 				.windowAll(GlobalWindows.create())
 				.trigger(PurgingTrigger.of(CountTrigger.of(10)))
 				.fold(0L, new FoldFunction<Long, Long>() {
@@ -848,7 +848,7 @@ public String map2(Integer value) {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void sinkKeyTest() {
 		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
index 3fc13447835d9..ff1eaaa6591dd 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
@@ -77,7 +77,7 @@ public void testFromCollectionParallelism() {
 			}
 
 			dataStream1.addSink(new DiscardingSink<Integer>());
-	
+
 			DataStreamSource<Integer> dataStream2 = env.fromParallelCollection(new DummySplittableIterator<Integer>(),
 					typeInfo).setParallelism(4);
 
@@ -87,7 +87,7 @@ public void testFromCollectionParallelism() {
 
 			assertEquals("Parallelism of collection source must be 1.", 1, env.getStreamGraph().getStreamNode(dataStream1.getId()).getParallelism());
 			assertEquals("Parallelism of parallel collection source must be 4.",
-					4, 
+					4,
 					env.getStreamGraph().getStreamNode(dataStream2.getId()).getParallelism());
 		}
 		catch (Exception e) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java
index a4abbcba442b1..83fb2f7adfd56 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java
@@ -46,7 +46,7 @@ public void test() {
 			env.addSource(new TestSource<Integer>()).print();
 			fail();
 		} catch (Exception ignored) {}
-		
+
 
 		DataStream<Long> source = env.generateSequence(1, 10);
 
@@ -54,17 +54,17 @@ public void test() {
 			source.map(new TestMap<Long, Long>()).print();
 			fail();
 		} catch (Exception ignored) {}
-		
+
 		try {
 			source.flatMap(new TestFlatMap<Long, Long>()).print();
 			fail();
 		} catch (Exception ignored) {}
-		
+
 		try {
 			source.connect(source).map(new TestCoMap<Long, Long, Integer>()).print();
 			fail();
 		} catch (Exception ignored) {}
-		
+
 		try {
 			source.connect(source).flatMap(new TestCoFlatMap<Long, Long, Integer>()).print();
 			fail();
@@ -76,7 +76,7 @@ public void test() {
 		source.connect(source).map(new TestCoMap<Long, Long, Integer>()).returns(BasicTypeInfo.INT_TYPE_INFO).print();
 		source.connect(source).flatMap(new TestCoFlatMap<Long, Long, Integer>())
 				.returns(BasicTypeInfo.INT_TYPE_INFO).print();
-		
+
 		assertEquals(BasicTypeInfo.LONG_TYPE_INFO,
 				source.map(new TestMap<Long, Long>()).returns(Long.class).getType());
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractorTest.java
index 5d036a954a339..2e92807e6a09d 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractorTest.java
@@ -25,12 +25,12 @@
 import static org.junit.Assert.fail;
 
 public class AscendingTimestampExtractorTest {
-	
+
 	@Test
 	public void testWithFailingHandler() {
 		AscendingTimestampExtractor<Long> extractor = new LongExtractor()
 				.withViolationHandler(new AscendingTimestampExtractor.FailingHandler());
-		
+
 		runValidTests(extractor);
 		try {
 			runInvalidTest(extractor);
@@ -63,20 +63,20 @@ public void testWithDefaultHandler() {
 		runValidTests(extractor);
 		runInvalidTest(extractor);
 	}
-	
+
 	@Test
 	public void testInitialAndFinalWatermark() {
 		AscendingTimestampExtractor<Long> extractor = new LongExtractor();
 		assertEquals(Long.MIN_VALUE, extractor.getCurrentWatermark().getTimestamp());
 
 		extractor.extractTimestamp(Long.MIN_VALUE, -1L);
-		
+
 		extractor.extractTimestamp(Long.MAX_VALUE, -1L);
 		assertEquals(Long.MAX_VALUE - 1, extractor.getCurrentWatermark().getTimestamp());
 	}
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	private void runValidTests(AscendingTimestampExtractor<Long> extractor) {
 		assertEquals(13L, extractor.extractTimestamp(13L, -1L));
 		assertEquals(13L, extractor.extractTimestamp(13L, 0L));
@@ -85,25 +85,25 @@ private void runValidTests(AscendingTimestampExtractor<Long> extractor) {
 		assertEquals(20L, extractor.extractTimestamp(20L, 0L));
 		assertEquals(20L, extractor.extractTimestamp(20L, 0L));
 		assertEquals(500L, extractor.extractTimestamp(500L, 0L));
-		
+
 		assertEquals(Long.MAX_VALUE - 1, extractor.extractTimestamp(Long.MAX_VALUE - 1, 99999L));
-		
-		
+
+
 	}
-	
+
 	private void runInvalidTest(AscendingTimestampExtractor<Long> extractor) {
 		assertEquals(1000L, extractor.extractTimestamp(1000L, 100));
 		assertEquals(1000L, extractor.extractTimestamp(1000L, 100));
-		
+
 		// violation
 		assertEquals(999L, extractor.extractTimestamp(999L, 100));
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	private static class LongExtractor extends AscendingTimestampExtractor<Long> {
 		private static final long serialVersionUID = 1L;
-		
+
 		@Override
 		public long extractAscendingTimestamp(Long element) {
 			return element;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/FromElementsFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/FromElementsFunctionTest.java
index 01540da0bcdc2..3744eb9fd3170 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/FromElementsFunctionTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/FromElementsFunctionTest.java
@@ -47,7 +47,7 @@
  * Tests for the {@link org.apache.flink.streaming.api.functions.source.FromElementsFunction}.
  */
 public class FromElementsFunctionTest {
-	
+
 	@Test
 	public void testStrings() {
 		try {
@@ -55,10 +55,10 @@ public void testStrings() {
 
 			FromElementsFunction<String> source = new FromElementsFunction<String>(
 					BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), data);
-			
+
 			List<String> result = new ArrayList<String>();
 			source.run(new ListSourceContext<String>(result));
-			
+
 			assertEquals(Arrays.asList(data), result);
 		}
 		catch (Exception e) {
@@ -66,7 +66,7 @@ public void testStrings() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testNonJavaSerializableType() {
 		try {
@@ -85,17 +85,17 @@ public void testNonJavaSerializableType() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testSerializationError() {
 		try {
-			TypeInformation<SerializationErrorType> info = 
+			TypeInformation<SerializationErrorType> info =
 					new ValueTypeInfo<SerializationErrorType>(SerializationErrorType.class);
-			
+
 			try {
 				new FromElementsFunction<SerializationErrorType>(
 					info.createSerializer(new ExecutionConfig()), new SerializationErrorType());
-				
+
 				fail("should fail with an exception");
 			}
 			catch (IOException e) {
@@ -116,7 +116,7 @@ public void testDeSerializationError() {
 
 			FromElementsFunction<DeserializeTooMuchType> source = new FromElementsFunction<DeserializeTooMuchType>(
 					info.createSerializer(new ExecutionConfig()), new DeserializeTooMuchType());
-			
+
 			try {
 				source.run(new ListSourceContext<DeserializeTooMuchType>(new ArrayList<DeserializeTooMuchType>()));
 				fail("should fail with an exception");
@@ -130,19 +130,19 @@ public void testDeSerializationError() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testCheckpointAndRestore() {
 		try {
 			final int NUM_ELEMENTS = 10000;
-			
+
 			List<Integer> data = new ArrayList<Integer>(NUM_ELEMENTS);
 			List<Integer> result = new ArrayList<Integer>(NUM_ELEMENTS);
-			
+
 			for (int i = 0; i < NUM_ELEMENTS; i++) {
 				data.add(i);
 			}
-			
+
 			final FromElementsFunction<Integer> source = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
 			StreamSource<Integer, FromElementsFunction<Integer>> src = new StreamSource<>(source);
 			AbstractStreamOperatorTestHarness<Integer> testHarness =
@@ -150,9 +150,9 @@ public void testCheckpointAndRestore() {
 			testHarness.open();
 
 			final SourceFunction.SourceContext<Integer> ctx = new ListSourceContext<Integer>(result, 2L);
-			
+
 			final Throwable[] error = new Throwable[1];
-			
+
 			// run the source asynchronously
 			Thread runner = new Thread() {
 				@Override
@@ -166,10 +166,10 @@ public void run() {
 				}
 			};
 			runner.start();
-			
-			// wait for a bit 
+
+			// wait for a bit
 			Thread.sleep(1000);
-			
+
 			// make a checkpoint
 			List<Integer> checkpointData = new ArrayList<>(NUM_ELEMENTS);
 			OperatorStateHandles handles = null;
@@ -177,11 +177,11 @@ public void run() {
 				handles = testHarness.snapshot(566, System.currentTimeMillis());
 				checkpointData.addAll(result);
 			}
-			
+
 			// cancel the source
 			source.cancel();
 			runner.join();
-			
+
 			// check for errors
 			if (error[0] != null) {
 				System.err.println("Error in asynchronous source runner");
@@ -201,7 +201,7 @@ public void run() {
 			SourceFunction.SourceContext<Integer> newCtx = new ListSourceContext<>(checkpointData);
 
 			sourceCopy.run(newCtx);
-			
+
 			assertEquals(data, checkpointData);
 		}
 		catch (Exception e) {
@@ -209,19 +209,19 @@ public void run() {
 			fail(e.getMessage());
 		}
 	}
-	
-	
+
+
 	// ------------------------------------------------------------------------
 	//  Test Types
 	// ------------------------------------------------------------------------
-	
+
 	public static class MyPojo {
-		
+
 		public long val1;
 		public int val2;
 
 		public MyPojo() {}
-		
+
 		public MyPojo(long val1, int val2) {
 			this.val1 = val1;
 			this.val2 = val2;
@@ -236,14 +236,14 @@ public int hashCode() {
 		public boolean equals(Object obj) {
 			if (obj instanceof MyPojo) {
 				MyPojo that = (MyPojo) obj;
-				return this.val1 == that.val1 && this.val2 == that.val2; 
+				return this.val1 == that.val1 && this.val2 == that.val2;
 			}
 			else {
 				return false;
 			}
 		}
 	}
-	
+
 	public static class SerializationErrorType implements Value {
 
 		private static final long serialVersionUID = -6037206294939421807L;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractorTest.java
index 95366b77572a5..a9764535c200b 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/IngestionTimeExtractorTest.java
@@ -24,37 +24,37 @@
 import static org.junit.Assert.*;
 
 public class IngestionTimeExtractorTest {
-	
+
 	@Test
 	public void testMonotonousTimestamps() {
 		AssignerWithPeriodicWatermarks<String> assigner = new IngestionTimeExtractor<>();
-		
+
 		long maxRecordSoFar = 0L;
 		long maxWatermarkSoFar = 0L;
-		
+
 		for (int i = 0; i < 1343; i++) {
 			if (i % 7 == 1) {
 				Watermark mark = assigner.getCurrentWatermark();
 				assertNotNull(mark);
-				
+
 				// increasing watermarks
 				assertTrue(mark.getTimestamp() >= maxWatermarkSoFar);
 				maxWatermarkSoFar = mark.getTimestamp();
-				
+
 				// tight watermarks
 				assertTrue(mark.getTimestamp() >= maxRecordSoFar - 1);
 			} else {
 				long next = assigner.extractTimestamp("a", Long.MIN_VALUE);
-				
+
 				// increasing timestamps
 				assertTrue(next >= maxRecordSoFar);
-				
+
 				// timestamps are never below or at the watermark
 				assertTrue(next > maxWatermarkSoFar);
-				
+
 				maxRecordSoFar = next;
 			}
-			
+
 			if (i % 9 == 0) {
 				try {
 					Thread.sleep(1);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/ListSourceContext.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/ListSourceContext.java
index a4c1bea83a04b..55a0e7f6c3df9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/ListSourceContext.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/ListSourceContext.java
@@ -25,18 +25,18 @@
 
 /**
  * Mock context that collects elements in a List.
- * 
+ *
  * @param <T> Type of the collected elements.
  */
 public class ListSourceContext<T> implements SourceFunction.SourceContext<T> {
-	
+
 	private final Object lock = new Object();
-	
+
 	private final List<T> target;
 
 	private final long delay;
-	
-	
+
+
 	public ListSourceContext(List<T> target) {
 		this(target, 0L);
 	}
@@ -49,7 +49,7 @@ public ListSourceContext(List<T> target, long delay) {
 	@Override
 	public void collect(T element) {
 		target.add(element);
-		
+
 		if (delay > 0) {
 			try {
 				Thread.sleep(delay);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/PrintSinkFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/PrintSinkFunctionTest.java
index f7c6e53b47f60..9e7ecddf720b8 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/PrintSinkFunctionTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/PrintSinkFunctionTest.java
@@ -121,4 +121,4 @@ public void restoreSystemOut() {
 		System.setOut(printStreamOriginal);
 	}
 
-}
\ No newline at end of file
+}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunctionTest.java
index 87376e7b63d15..85fa30b9c5ec7 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunctionTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunctionTest.java
@@ -43,16 +43,16 @@ public class SocketTextStreamFunctionTest {
 	public void testSocketSourceSimpleOutput() throws Exception {
 		ServerSocket server = new ServerSocket(0);
 		Socket channel = null;
-		
+
 		try {
 			SocketTextStreamFunction source = new SocketTextStreamFunction(LOCALHOST, server.getLocalPort(), "\n", 0);
-	
+
 			SocketSourceThread runner = new SocketSourceThread(source, "test1", "check");
 			runner.start();
-	
+
 			channel = server.accept();
 			OutputStreamWriter writer = new OutputStreamWriter(channel.getOutputStream());
-			
+
 			writer.write("test1\n");
 			writer.write("check\n");
 			writer.flush();
@@ -60,9 +60,9 @@ public void testSocketSourceSimpleOutput() throws Exception {
 
 			runner.cancel();
 			runner.interrupt();
-			
+
 			runner.waitUntilDone();
-			
+
 			channel.close();
 		}
 		finally {
@@ -86,7 +86,7 @@ public void testExitNoRetries() throws Exception {
 
 			channel = server.accept();
 			channel.close();
-			
+
 			try {
 				runner.waitUntilDone();
 			}
@@ -235,22 +235,22 @@ public void testSocketSourceOutputAcrossRetries() throws Exception {
 			IOUtils.closeQuietly(server);
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	private static class SocketSourceThread extends Thread {
-		
+
 		private final Object sync = new Object();
-		
+
 		private final SocketTextStreamFunction socketSource;
-		
+
 		private final String[] expectedData;
-		
+
 		private volatile Throwable error;
 		private volatile int numElementsReceived;
 		private volatile boolean canceled;
 		private volatile boolean done;
-		
+
 		public SocketSourceThread(SocketTextStreamFunction socketSource, String... expectedData) {
 			this.socketSource = socketSource;
 			this.expectedData = expectedData;
@@ -259,19 +259,19 @@ public SocketSourceThread(SocketTextStreamFunction socketSource, String... expec
 		public void run() {
 			try {
 				SourceFunction.SourceContext<String> ctx = new SourceFunction.SourceContext<String>() {
-					
+
 					private final Object lock = new Object();
-					
+
 					@Override
 					public void collect(String element) {
 						int pos = numElementsReceived;
-						
+
 						// make sure waiter know of us
 						synchronized (sync) {
 							numElementsReceived++;
 							sync.notifyAll();
 						}
-						
+
 						if (expectedData != null && expectedData.length > pos) {
 							assertEquals(expectedData[pos], element);
 						}
@@ -300,7 +300,7 @@ public Object getCheckpointLock() {
 					@Override
 					public void close() {}
 				};
-				
+
 				socketSource.run(ctx);
 			}
 			catch (Throwable t) {
@@ -318,7 +318,7 @@ public void close() {}
 				}
 			}
 		}
-		
+
 		public void cancel() {
 			synchronized (sync) {
 				canceled = true;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/SlotAllocationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/SlotAllocationTest.java
index d87377167d034..cac59aef6267f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/SlotAllocationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/SlotAllocationTest.java
@@ -39,7 +39,7 @@
  */
 @SuppressWarnings("serial")
 public class SlotAllocationTest extends TestLogger {
-	
+
 	@Test
 	public void testTwoPipelines() {
 		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
index 8b5aef59d03e8..6c5bacaf28ddb 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
@@ -90,7 +90,7 @@ public void invoke(Tuple2<String, String> value) {}
 		streamGraph.setJobName("test job");
 		JobGraph jobGraph = streamGraph.getJobGraph();
 		List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
-		
+
 		assertEquals(2, jobGraph.getNumberOfVertices());
 		assertEquals(1, verticesSorted.get(0).getParallelism());
 		assertEquals(1, verticesSorted.get(1).getParallelism());
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
index 8f42c1ae59cab..1f68af9009bf1 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
@@ -495,7 +495,7 @@ public void testSnapshotMethod() throws Exception {
 		StateSnapshotContextSynchronousImpl context = mock(StateSnapshotContextSynchronousImpl.class);
 
 		whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context);
-		
+
 		StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class);
 		when(containingTask.getCancelables()).thenReturn(closeableRegistry);
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java
index f3062319ce6f2..9b78b08fb5c80 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateDescriptorPassingTest.java
@@ -52,7 +52,7 @@
 /**
  * Various tests around the proper passing of state descriptors to the operators
  * and their serialization.
- * 
+ *
  * The tests use an arbitrary generic type to validate the behavior.
  */
 @SuppressWarnings("serial")
@@ -102,7 +102,7 @@ public String getKey(File value) {
 				})
 				.timeWindow(Time.milliseconds(1000))
 				.reduce(new ReduceFunction<File>() {
-					
+
 					@Override
 					public File reduce(File value1, File value2) {
 						return null;
@@ -117,7 +117,7 @@ public void testApplyWindowState() throws Exception {
 		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
 		env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
-		
+
 		DataStream<File> src = env.fromElements(new File("/"));
 
 		SingleOutputStreamOperator<?> result = src
@@ -130,7 +130,7 @@ public String getKey(File value) {
 				.timeWindow(Time.milliseconds(1000))
 				.apply(new WindowFunction<File, String, String, TimeWindow>() {
 					@Override
-					public void apply(String s, TimeWindow window, 
+					public void apply(String s, TimeWindow window,
 										Iterable<File> input, Collector<String> out) {}
 				});
 
@@ -255,7 +255,7 @@ private void validateStateDescriptorConfigured(SingleOutputStreamOperator<?> res
 
 		Kryo kryo = ((KryoSerializer<?>) serializer).getKryo();
 
-		assertTrue("serializer registration was not properly passed on", 
+		assertTrue("serializer registration was not properly passed on",
 				kryo.getSerializer(File.class) instanceof JavaSerializer);
 	}
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateInitializationContextImplTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateInitializationContextImplTest.java
index 8e0edfcab9d69..37450311f4358 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateInitializationContextImplTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StateInitializationContextImplTest.java
@@ -281,4 +281,4 @@ public void close() throws IOException {
 		}
 	}
 
-}
\ No newline at end of file
+}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedFoldTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedFoldTest.java
index 68a2bb25c04ce..05a910f94f526 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedFoldTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedFoldTest.java
@@ -60,13 +60,13 @@ public String fold(String accumulator, Integer value) throws Exception {
 	public void testGroupedFold() throws Exception {
 
 		KeySelector<Integer, String> keySelector = new KeySelector<Integer, String>() {
-			
+
 			@Override
 			public String getKey(Integer value) {
 				return value.toString();
 			}
 		};
-		
+
 		StreamGroupedFold<Integer, String, String> operator = new StreamGroupedFold<>(new MyFolder(), "100");
 		operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
 
@@ -103,7 +103,7 @@ public Integer getKey(Integer value) {
 				return value;
 			}
 		};
-		
+
 		StreamGroupedFold<Integer, String, Integer> operator = new StreamGroupedFold<>(
 				new TestOpenCloseFoldFunction(), "init");
 		operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedReduceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedReduceTest.java
index 0f304a0173f25..b61c76038c6f0 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedReduceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamGroupedReduceTest.java
@@ -50,7 +50,7 @@ public class StreamGroupedReduceTest {
 	public void testGroupedReduce() throws Exception {
 
 		KeySelector<Integer, Integer> keySelector = new IntegerKeySelector();
-		
+
 		StreamGroupedReduce<Integer> operator = new StreamGroupedReduce<>(new MyReducer(), IntSerializer.INSTANCE);
 
 		OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
@@ -82,7 +82,7 @@ public void testGroupedReduce() throws Exception {
 	public void testOpenClose() throws Exception {
 
 		KeySelector<Integer, Integer> keySelector = new IntegerKeySelector();
-		
+
 		StreamGroupedReduce<Integer> operator =
 				new StreamGroupedReduce<>(new TestOpenCloseReduceFunction(), IntSerializer.INSTANCE);
 		OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamMapTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamMapTest.java
index f0113d1453bcb..ce85b6a584413 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamMapTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamMapTest.java
@@ -48,7 +48,7 @@ public String map(Integer value) throws Exception {
 			return "+" + (value + 1);
 		}
 	}
-	
+
 	@Test
 	public void testMap() throws Exception {
 		StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map());
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContextTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContextTest.java
index 36496f2e0e41d..5ca9cb43eb4d4 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContextTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContextTest.java
@@ -67,15 +67,15 @@
 import static org.mockito.Mockito.when;
 
 public class StreamingRuntimeContextTest {
-	
+
 	@Test
 	public void testValueStateInstantiation() throws Exception {
-		
+
 		final ExecutionConfig config = new ExecutionConfig();
 		config.registerKryoType(Path.class);
-		
+
 		final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
-		
+
 		StreamingRuntimeContext context = new StreamingRuntimeContext(
 				createDescriptorCapturingMockOp(descriptorCapture, config),
 				createMockEnvironment(),
@@ -83,10 +83,10 @@ public void testValueStateInstantiation() throws Exception {
 
 		ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
 		context.getState(descr);
-		
+
 		StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
 		TypeSerializer<?> serializer = descrIntercepted.getSerializer();
-		
+
 		// check that the Path class is really registered, i.e., the execution config was applied
 		assertTrue(serializer instanceof KryoSerializer);
 		assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
@@ -107,10 +107,10 @@ public void testReducingStateInstantiation() throws Exception {
 
 		@SuppressWarnings("unchecked")
 		ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);
-		
-		ReducingStateDescriptor<TaskInfo> descr = 
+
+		ReducingStateDescriptor<TaskInfo> descr =
 				new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);
-		
+
 		context.getReducingState(descr);
 
 		StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
@@ -192,7 +192,7 @@ public void testListStateReturnsEmptyListByDefault() throws Exception {
 		assertNotNull(value);
 		assertFalse(value.iterator().hasNext());
 	}
-	
+
 	@Test
 	public void testMapStateInstantiation() throws Exception {
 
@@ -218,7 +218,7 @@ public void testMapStateInstantiation() throws Exception {
 		assertTrue(valueSerializer instanceof KryoSerializer);
 		assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0);
 	}
-	
+
 	@Test
 	public void testMapStateReturnsEmptyMapByDefault() throws Exception {
 
@@ -234,15 +234,15 @@ public void testMapStateReturnsEmptyMapByDefault() throws Exception {
 		assertNotNull(value);
 		assertFalse(value.iterator().hasNext());
 	}
-	
+
 	// ------------------------------------------------------------------------
-	//  
+	//
 	// ------------------------------------------------------------------------
-	
+
 	@SuppressWarnings("unchecked")
 	private static AbstractStreamOperator<?> createDescriptorCapturingMockOp(
 			final AtomicReference<Object> ref, final ExecutionConfig config) throws Exception {
-		
+
 		AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
 
 		KeyedStateBackend keyedStateBackend= mock(KeyedStateBackend.class);
@@ -300,7 +300,7 @@ public ListState<String> answer(InvocationOnMock invocationOnMock) throws Throwa
 		when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
 		return operatorMock;
 	}
-	
+
 	@SuppressWarnings("unchecked")
 	private static AbstractStreamOperator<?> createMapPlainMockOp() throws Exception {
 
@@ -312,7 +312,7 @@ private static AbstractStreamOperator<?> createMapPlainMockOp() throws Exception
 		DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);
 
 		when(operatorMock.getExecutionConfig()).thenReturn(config);
-		
+
 		doAnswer(new Answer<MapState<Integer, String>>() {
 
 			@Override
@@ -332,11 +332,11 @@ public MapState<Integer, String> answer(InvocationOnMock invocationOnMock) throw
 				return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
 			}
 		}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(MapStateDescriptor.class));
-		
+
 		when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
 		return operatorMock;
 	}
-	
+
 	private static Environment createMockEnvironment() {
 		Environment env = mock(Environment.class);
 		when(env.getUserClassLoader()).thenReturn(StreamingRuntimeContextTest.class.getClassLoader());
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/MockRecordWriter.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/MockRecordWriter.java
index 317a21c023247..6ee7d38d4255d 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/MockRecordWriter.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/MockRecordWriter.java
@@ -37,7 +37,7 @@ public boolean initList() {
 		emittedRecords = new ArrayList<Integer>();
 		return true;
 	}
-	
+
 	@Override
 	public void emit(SerializationDelegate<StreamRecord<Tuple1<Integer>>> record) {
 		emittedRecords.add(record.getInstance().getValue().f0);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
index bdc7e94a2f505..b640d6f1f513a 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
@@ -23,11 +23,11 @@
 import org.junit.Test;
 
 public class CosineDistanceTest {
-	
+
 	@SuppressWarnings({ "rawtypes", "unchecked" })
 	@Test
 	public void testCosineDistance() {
-		
+
 		//Reference calculated using wolfram alpha
 		double[][][] testdata={
 				{{0,0,0},{0,0,0}},
@@ -51,14 +51,14 @@ public void testCosineDistance() {
 				0.016130,
 				0
 		};
-		
+
 		for (int i = 0; i < testdata.length; i++) {
 			assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
 					+ arrayToString(testdata[i][0]), referenceSolutions[i],
 					new CosineDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
 		}
 	}
-	
+
 	private String arrayToString(double[] in){
 		if (in.length==0) return "{}";
 		String result="{";
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
index 85a08827a7084..1ba5f847df597 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
@@ -23,11 +23,11 @@
 import org.junit.Test;
 
 public class EuclideanDistanceTest {
-	
+
 	@SuppressWarnings({ "rawtypes", "unchecked" })
 	@Test
 	public void testEuclideanDistance() {
-		
+
 		//Reference calculated using wolfram alpha
 		double[][][] testdata={
 				{{0,0,0},{0,0,0}},
@@ -51,15 +51,15 @@ public void testEuclideanDistance() {
 				2.828427,
 				1
 		};
-		
+
 		for (int i = 0; i < testdata.length; i++) {
 			assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
 					+ arrayToString(testdata[i][0]), referenceSolutions[i],
 					new EuclideanDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
 		}
-		
+
 	}
-	
+
 	private String arrayToString(double[] in){
 		if (in.length==0) return "{}";
 		String result="{";
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java
index 66c6a40191fc7..4175f1889af8d 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java
@@ -30,13 +30,13 @@
 
 @SuppressWarnings("serial")
 public class TranslationTest {
-	
+
 	@Test
 	public void testCheckpointModeTranslation() {
 		try {
 			// with deactivated fault tolerance, the checkpoint mode should be at-least-once
 			StreamExecutionEnvironment deactivated = getSimpleJob();
-			
+
 			for (JobVertex vertex : deactivated.getStreamGraph().getJobGraph().getVertices()) {
 				assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode());
 			}
@@ -60,7 +60,7 @@ public void testCheckpointModeTranslation() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	private static StreamExecutionEnvironment getSimpleJob() {
 		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.generateSequence(1, 10000000)
@@ -69,7 +69,7 @@ private static StreamExecutionEnvironment getSimpleJob() {
 					public void invoke(Long value) {
 					}
 				});
-		
+
 		return env;
 	}
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferAlignmentLimitTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferAlignmentLimitTest.java
index e4074432aaf63..4322748404449 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferAlignmentLimitTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferAlignmentLimitTest.java
@@ -57,7 +57,7 @@
 import static org.mockito.Mockito.verify;
 
 /**
- * Tests for the barrier buffer's maximum limit of buffered/spilled bytes 
+ * Tests for the barrier buffer's maximum limit of buffered/spilled bytes
  */
 public class BarrierBufferAlignmentLimitTest {
 
@@ -96,7 +96,7 @@ public void testBreakCheckpointAtAlignmentLimit() throws Exception {
 				/*  2 */ createBuffer(0, 42), createBuffer(2, 111),
 
 				// starting a checkpoint
-				/*  4 */ createBarrier(7, 1), 
+				/*  4 */ createBarrier(7, 1),
 				/*  5 */ createBuffer(1, 100), createBuffer(2, 200), createBuffer(1, 300), createBuffer(0, 50),
 				/*  9 */ createBarrier(7, 0),
 				/* 10 */ createBuffer(2, 100), createBuffer(0, 100), createBuffer(1, 200), createBuffer(0, 200),
@@ -106,7 +106,7 @@ public void testBreakCheckpointAtAlignmentLimit() throws Exception {
 
 				// additional data
 				/* 15 */ createBuffer(0, 100), createBuffer(1, 100), createBuffer(2, 100),
-				
+
 				// checkpoint completes - this should not result in a "completion notification"
 				/* 18 */ createBarrier(7, 2),
 
@@ -173,7 +173,7 @@ public void testBreakCheckpointAtAlignmentLimit() throws Exception {
 	 *   - an alignment starts
 	 *   - barriers from a second checkpoint queue before the first completes
 	 *   - together they are larger than the threshold
-	 *   - after the first checkpoint (with second checkpoint data queued) aborts, the second completes 
+	 *   - after the first checkpoint (with second checkpoint data queued) aborts, the second completes
 	 */
 	@Test
 	public void testAlignmentLimitWithQueuedAlignments() throws Exception {
@@ -182,8 +182,8 @@ public void testAlignmentLimitWithQueuedAlignments() throws Exception {
 				/*  0 */ createBuffer(1, 100), createBuffer(2, 70),
 
 				// starting a checkpoint
-				/*  2 */ createBarrier(3, 2), 
-				/*  3 */ createBuffer(1, 100), createBuffer(2, 100), 
+				/*  2 */ createBarrier(3, 2),
+				/*  3 */ createBuffer(1, 100), createBuffer(2, 100),
 				/*  5 */ createBarrier(3, 0),
 				/*  6 */ createBuffer(0, 100), createBuffer(1, 100),
 
@@ -239,7 +239,7 @@ public void testAlignmentLimitWithQueuedAlignments() throws Exception {
 		startTs = System.nanoTime();
 		check(sequence[12], buffer.getNextNonBlocked());
 
-		// only checkpoint 4 is pending now - the last checkpoint 3 barrier will not trigger success 
+		// only checkpoint 4 is pending now - the last checkpoint 3 barrier will not trigger success
 		check(sequence[17], buffer.getNextNonBlocked());
 
 		// checkpoint 4 completed - check and validate buffered replay
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java
index 4344a468e1b69..3514f56da58d6 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java
@@ -44,22 +44,22 @@
 public class BarrierBufferMassiveRandomTest {
 
 	private static final int PAGE_SIZE = 1024;
-	
+
 	@Test
 	public void testWithTwoChannelsAndRandomBarriers() {
 		IOManager ioMan = null;
 		try {
 			ioMan = new IOManagerAsync();
-			
+
 			BufferPool pool1 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100, 100);
 			BufferPool pool2 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100, 100);
 
 			RandomGeneratingInputGate myIG = new RandomGeneratingInputGate(
 					new BufferPool[] { pool1, pool2 },
 					new BarrierGenerator[] { new CountBarrier(100000), new RandomBarrier(100000) });
-	
+
 			BarrierBuffer barrierBuffer = new BarrierBuffer(myIG, ioMan);
-			
+
 			for (int i = 0; i < 2000000; i++) {
 				BufferOrEvent boe = barrierBuffer.getNextNonBlocked();
 				if (boe.isBuffer()) {
@@ -81,13 +81,13 @@ public void testWithTwoChannelsAndRandomBarriers() {
 	// ------------------------------------------------------------------------
 	//  Mocks and Generators
 	// ------------------------------------------------------------------------
-	
+
 	protected interface BarrierGenerator {
 		public boolean isNextBarrier();
 	}
 
 	protected static class RandomBarrier implements BarrierGenerator {
-		
+
 		private static final Random rnd = new Random();
 
 		private final double threshold;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java
index d6056d22cc028..fedf4fc8a0cdb 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java
@@ -93,7 +93,7 @@ public static void shutdownIOManager() {
 	@Test
 	public void testSingleChannelNoBarriers() {
 		try {
-			BufferOrEvent[] sequence = { 
+			BufferOrEvent[] sequence = {
 					createBuffer(0), createBuffer(0), createBuffer(0),
 					createEndOfPartition(0)
 			};
@@ -109,7 +109,7 @@ public void testSingleChannelNoBarriers() {
 
 			assertNull(buffer.getNextNonBlocked());
 			assertNull(buffer.getNextNonBlocked());
-			
+
 			buffer.cleanup();
 
 			checkNoTempFilesRemain();
@@ -156,7 +156,7 @@ public void testMultiChannelNoBarriers() {
 	}
 
 	/**
-	 * Validates that the buffer preserved the order of elements for a 
+	 * Validates that the buffer preserved the order of elements for a
 	 * input with a single input channel, and checkpoint events.
 	 */
 	@Test
@@ -178,7 +178,7 @@ public void testSingleChannelWithBarriers() {
 			ValidatingCheckpointHandler handler = new ValidatingCheckpointHandler();
 			buffer.registerCheckpointEventHandler(handler);
 			handler.setNextExpectedCheckpointId(1L);
-			
+
 			for (BufferOrEvent boe : sequence) {
 				if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
 					assertEquals(boe, buffer.getNextNonBlocked());
@@ -211,17 +211,17 @@ public void testMultiChannelWithBarriers() {
 					createBarrier(1, 1), createBarrier(1, 2),
 					createBuffer(2), createBuffer(1), createBuffer(0),
 					createBarrier(1, 0),
-					
+
 					// checkpoint without blocked data
 					createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2),
 					createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2),
-					
+
 					// checkpoint with data only from one channel
 					createBuffer(2), createBuffer(2),
 					createBarrier(3, 2),
 					createBuffer(2), createBuffer(2),
 					createBarrier(3, 0), createBarrier(3, 1),
-					
+
 					// empty checkpoint
 					createBarrier(4, 1), createBarrier(4, 2), createBarrier(4, 0),
 
@@ -232,7 +232,7 @@ public void testMultiChannelWithBarriers() {
 					createBarrier(5, 2),
 					createBuffer(1), createBuffer(0), createBuffer(2), createBuffer(1),
 					createBarrier(5, 0),
-					
+
 					// some trailing data
 					createBuffer(0),
 					createEndOfPartition(0), createEndOfPartition(1), createEndOfPartition(2)
@@ -298,19 +298,19 @@ public void testMultiChannelWithBarriers() {
 
 			check(sequence[28], buffer.getNextNonBlocked());
 			check(sequence[29], buffer.getNextNonBlocked());
-			
+
 			// checkpoint 5 aligning
 			check(sequence[31], buffer.getNextNonBlocked());
 			check(sequence[32], buffer.getNextNonBlocked());
 			check(sequence[33], buffer.getNextNonBlocked());
 			check(sequence[37], buffer.getNextNonBlocked());
-			
+
 			// buffered data from checkpoint 5 alignment
 			check(sequence[34], buffer.getNextNonBlocked());
 			check(sequence[36], buffer.getNextNonBlocked());
 			check(sequence[38], buffer.getNextNonBlocked());
 			check(sequence[39], buffer.getNextNonBlocked());
-			
+
 			// remaining data
 			check(sequence[41], buffer.getNextNonBlocked());
 			check(sequence[42], buffer.getNextNonBlocked());
@@ -450,7 +450,7 @@ public void testMultiChannelWithQueuedFutureBarriers() {
 			check(sequence[1], buffer.getNextNonBlocked());
 			check(sequence[2], buffer.getNextNonBlocked());
 			check(sequence[7], buffer.getNextNonBlocked());
-			
+
 			check(sequence[5], buffer.getNextNonBlocked());
 			assertEquals(2L, handler.getNextExpectedCheckpointId());
 			check(sequence[6], buffer.getNextNonBlocked());
@@ -477,7 +477,7 @@ public void testMultiChannelWithQueuedFutureBarriers() {
 			check(sequence[18], buffer.getNextNonBlocked());
 			check(sequence[19], buffer.getNextNonBlocked());
 			check(sequence[28], buffer.getNextNonBlocked());
-			
+
 			// past checkpoint 3
 			check(sequence[36], buffer.getNextNonBlocked());
 			check(sequence[38], buffer.getNextNonBlocked());
@@ -488,23 +488,23 @@ public void testMultiChannelWithQueuedFutureBarriers() {
 			check(sequence[31], buffer.getNextNonBlocked());
 			check(sequence[33], buffer.getNextNonBlocked());
 			check(sequence[39], buffer.getNextNonBlocked());
-			
+
 			// past checkpoint 4, alignment for checkpoint 5
 			check(sequence[42], buffer.getNextNonBlocked());
 			check(sequence[45], buffer.getNextNonBlocked());
 			check(sequence[46], buffer.getNextNonBlocked());
-			
+
 			// abort checkpoint 5 (end of partition)
 			check(sequence[37], buffer.getNextNonBlocked());
-			
+
 			// start checkpoint 6 alignment
 			check(sequence[47], buffer.getNextNonBlocked());
 			check(sequence[48], buffer.getNextNonBlocked());
-			
+
 			// end of input, emit remainder
 			check(sequence[43], buffer.getNextNonBlocked());
 			check(sequence[44], buffer.getNextNonBlocked());
-			
+
 			assertNull(buffer.getNextNonBlocked());
 			assertNull(buffer.getNextNonBlocked());
 
@@ -539,7 +539,7 @@ public void testMultiChannelSkippingCheckpoints() {
 					createBarrier(2, 0),
 					createBuffer(2), createBuffer(0),
 					createBarrier(3, 2),
-					
+
 					createBuffer(2),
 					createBuffer(1), createEndOfPartition(1),
 					createBuffer(2), createEndOfPartition(2),
@@ -603,7 +603,7 @@ public void testMultiChannelSkippingCheckpoints() {
 
 			assertNull(buffer.getNextNonBlocked());
 			assertNull(buffer.getNextNonBlocked());
-			
+
 			buffer.cleanup();
 
 			checkNoTempFilesRemain();
@@ -730,7 +730,7 @@ public void testMultiChannelSkippingCheckpointsViaBlockedInputs() {
 
 					createBarrier(3, 0), // queued barrier on blocked input
 					createBuffer(0),
-					
+
 					createBarrier(4, 1), // pre-mature barrier on blocked input
 					createBuffer(1),
 					createBuffer(0),
@@ -739,13 +739,13 @@ public void testMultiChannelSkippingCheckpointsViaBlockedInputs() {
 					// complete checkpoint 2
 					createBarrier(2, 2),
 					createBuffer(0),
-					
+
 					createBarrier(3, 2), // should be ignored
 					createBuffer(2),
 					createBarrier(4, 0),
 					createBuffer(0), createBuffer(1), createBuffer(2),
 					createBarrier(4, 2),
-					
+
 					createBuffer(1), createEndOfPartition(1),
 					createBuffer(2), createEndOfPartition(2),
 					createBuffer(0), createEndOfPartition(0)
@@ -774,7 +774,7 @@ public void testMultiChannelSkippingCheckpointsViaBlockedInputs() {
 			check(sequence[12], buffer.getNextNonBlocked());
 			check(sequence[15], buffer.getNextNonBlocked());
 			check(sequence[16], buffer.getNextNonBlocked());
-			
+
 			// checkpoint 3 skipped, alignment for 4 started
 			check(sequence[18], buffer.getNextNonBlocked());
 			assertEquals(4L, buffer.getCurrentCheckpointId());
@@ -782,19 +782,19 @@ public void testMultiChannelSkippingCheckpointsViaBlockedInputs() {
 			check(sequence[24], buffer.getNextNonBlocked());
 			check(sequence[26], buffer.getNextNonBlocked());
 			check(sequence[30], buffer.getNextNonBlocked());
-			
+
 			// checkpoint 4 completed
 			check(sequence[20], buffer.getNextNonBlocked());
 			check(sequence[28], buffer.getNextNonBlocked());
 			check(sequence[29], buffer.getNextNonBlocked());
-			
+
 			check(sequence[32], buffer.getNextNonBlocked());
 			check(sequence[33], buffer.getNextNonBlocked());
 			check(sequence[34], buffer.getNextNonBlocked());
 			check(sequence[35], buffer.getNextNonBlocked());
 			check(sequence[36], buffer.getNextNonBlocked());
 			check(sequence[37], buffer.getNextNonBlocked());
-			
+
 			assertNull(buffer.getNextNonBlocked());
 			assertNull(buffer.getNextNonBlocked());
 
@@ -858,12 +858,12 @@ public void testEarlyCleanup() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testStartAlignmentWithClosedChannels() {
 		try {
 			BufferOrEvent[] sequence = {
-					// close some channels immediately 
+					// close some channels immediately
 					createEndOfPartition(2), createEndOfPartition(1),
 
 					// checkpoint without blocked data
@@ -878,11 +878,11 @@ public void testStartAlignmentWithClosedChannels() {
 
 					// empty checkpoint
 					createBarrier(4, 0), createBarrier(4, 3),
-					
+
 					// some data, one channel closes
 					createBuffer(0), createBuffer(0), createBuffer(3),
 					createEndOfPartition(0),
-					
+
 					// checkpoint on last remaining channel
 					createBuffer(3),
 					createBarrier(5, 3),
@@ -891,9 +891,9 @@ public void testStartAlignmentWithClosedChannels() {
 			};
 
 			MockInputGate gate = new MockInputGate(PAGE_SIZE, 4, Arrays.asList(sequence));
-			
+
 			BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
-			
+
 			// pre checkpoint 2
 			check(sequence[0], buffer.getNextNonBlocked());
 			check(sequence[1], buffer.getNextNonBlocked());
@@ -1023,7 +1023,7 @@ public void testSingleChannelAbortCheckpoint() throws Exception {
 		assertEquals(6L, buffer.getCurrentCheckpointId());
 		verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(6L), any(CheckpointDeclineOnCancellationBarrierException.class));
 		assertEquals(0L, buffer.getAlignmentDurationNanos());
-		
+
 		buffer.cleanup();
 		checkNoTempFilesRemain();
 	}
@@ -1417,12 +1417,12 @@ private static BufferOrEvent createBuffer(int channel) {
 	private static BufferOrEvent createEndOfPartition(int channel) {
 		return new BufferOrEvent(EndOfPartitionEvent.INSTANCE, channel);
 	}
-	
+
 	private static void check(BufferOrEvent expected, BufferOrEvent present) {
 		assertNotNull(expected);
 		assertNotNull(present);
 		assertEquals(expected.isBuffer(), present.isBuffer());
-		
+
 		if (expected.isBuffer()) {
 			assertEquals(expected.getBuffer().getSize(), present.getBuffer().getSize());
 			MemorySegment expectedMem = expected.getBuffer().getMemorySegment();
@@ -1495,7 +1495,7 @@ public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, Checkpoi
 		@Override
 		public void triggerCheckpointOnBarrier(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, CheckpointMetrics checkpointMetrics) throws Exception {
 			assertTrue("wrong checkpoint id",
-					nextExpectedCheckpointId == -1L || 
+					nextExpectedCheckpointId == -1L ||
 					nextExpectedCheckpointId == checkpointMetaData.getCheckpointId());
 
 			assertTrue(checkpointMetaData.getTimestamp() > 0);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java
index 05f7da6e3a8f3..cbcf45dbc06a7 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java
@@ -62,7 +62,7 @@ public void testSingleChannelNoBarriers() {
 			for (BufferOrEvent boe : sequence) {
 				assertEquals(boe, tracker.getNextNonBlocked());
 			}
-			
+
 			assertNull(tracker.getNextNonBlocked());
 			assertNull(tracker.getNextNonBlocked());
 		}
@@ -115,7 +115,7 @@ public void testSingleChannelWithBarriers() {
 			CheckpointSequenceValidator validator =
 					new CheckpointSequenceValidator(1, 2, 3, 4, 5, 6);
 			tracker.registerCheckpointEventHandler(validator);
-			
+
 			for (BufferOrEvent boe : sequence) {
 				if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
 					assertEquals(boe, tracker.getNextNonBlocked());
@@ -174,17 +174,17 @@ public void testMultiChannelWithBarriers() {
 					createBarrier(1, 1), createBarrier(1, 2),
 					createBuffer(2), createBuffer(1),
 					createBarrier(1, 0),
-					
+
 					createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2),
 					createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2),
-					
+
 					createBuffer(2), createBuffer(2),
 					createBarrier(3, 2),
 					createBuffer(2), createBuffer(2),
 					createBarrier(3, 0), createBarrier(3, 1),
-					
+
 					createBarrier(4, 1), createBarrier(4, 2), createBarrier(4, 0),
-					
+
 					createBuffer(0)
 			};
 
@@ -225,14 +225,14 @@ public void testMultiChannelSkippingCheckpoints() {
 					createBuffer(2), createBuffer(2),
 					createBarrier(3, 2),
 					createBuffer(2), createBuffer(2),
-					
+
 					// jump to checkpoint 4
 					createBarrier(4, 0),
 					createBuffer(0), createBuffer(1), createBuffer(2),
 					createBarrier(4, 1),
 					createBuffer(1),
 					createBarrier(4, 2),
-					
+
 					createBuffer(0)
 			};
 
@@ -248,7 +248,7 @@ public void testMultiChannelSkippingCheckpoints() {
 					assertEquals(boe, tracker.getNextNonBlocked());
 				}
 			}
-			
+
 			assertNull(tracker.getNextNonBlocked());
 			assertNull(tracker.getNextNonBlocked());
 		}
@@ -262,7 +262,7 @@ public void testMultiChannelSkippingCheckpoints() {
 	 * This test validates that the barrier tracker does not immediately
 	 * discard a pending checkpoint as soon as it sees a barrier from a
 	 * later checkpoint from some channel.
-	 * 
+	 *
 	 * This behavior is crucial, otherwise topologies where different inputs
 	 * have different latency (and that latency is close to or higher than the
 	 * checkpoint interval) may skip many checkpoints, or fail to complete a
@@ -275,47 +275,47 @@ public void testCompleteCheckpointsOnLateBarriers() {
 					// checkpoint 2
 					createBuffer(1), createBuffer(1), createBuffer(0), createBuffer(2),
 					createBarrier(2, 1), createBarrier(2, 0), createBarrier(2, 2),
-					
+
 					// incomplete checkpoint 3
 					createBuffer(1), createBuffer(0),
 					createBarrier(3, 1), createBarrier(3, 2),
-					
+
 					// some barriers from checkpoint 4
 					createBuffer(1), createBuffer(0),
 					createBarrier(4, 2), createBarrier(4, 1),
 					createBuffer(1), createBuffer(2),
-	
+
 					// last barrier from checkpoint 3
 					createBarrier(3, 0),
-					
+
 					// complete checkpoint 4
 					createBuffer(0), createBarrier(4, 0),
-					
+
 					// regular checkpoint 5
-					createBuffer(1), createBuffer(2), createBarrier(5, 1), 
+					createBuffer(1), createBuffer(2), createBarrier(5, 1),
 					createBuffer(0), createBarrier(5, 0),
 					createBuffer(1), createBarrier(5, 2),
-					
+
 					// checkpoint 6 (incomplete),
 					createBuffer(1), createBarrier(6, 1),
 					createBuffer(0), createBarrier(6, 0),
-					
+
 					// checkpoint 7, with early barriers for checkpoints 8 and 9
 					createBuffer(1), createBarrier(7, 1),
 					createBuffer(0), createBarrier(7, 2),
-					createBuffer(2), createBarrier(8, 2), 
+					createBuffer(2), createBarrier(8, 2),
 					createBuffer(0), createBarrier(8, 1),
 					createBuffer(1), createBarrier(9, 1),
-					
+
 					// complete checkpoint 7, first barriers from checkpoint 10
 					createBarrier(7, 0),
 					createBuffer(0), createBarrier(9, 2),
 					createBuffer(2), createBarrier(10, 2),
-					
+
 					// complete checkpoint 8 and 9
 					createBarrier(8, 0),
 					createBuffer(1), createBuffer(2), createBarrier(9, 0),
-					
+
 					// trailing data
 					createBuffer(1), createBuffer(0), createBuffer(2)
 			};
@@ -404,7 +404,7 @@ public void testMultiChannelAbortCheckpoint() throws Exception {
 				createBuffer(0), createBuffer(1), createBuffer(2),
 				createBarrier(5, 2), createBarrier(5, 1), createBarrier(5, 0),
 
-				// abort multiple cancellations and a barrier after the cancellations 
+				// abort multiple cancellations and a barrier after the cancellations
 				createBuffer(0), createBuffer(1),
 				createCancellationBarrier(6, 1), createCancellationBarrier(6, 2),
 				createBarrier(6, 0),
@@ -465,7 +465,7 @@ public void testInterleavedCancellationBarriers() throws Exception {
 		verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(1L), any(Throwable.class));
 		verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(2L), any(Throwable.class));
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Utils
 	// ------------------------------------------------------------------------
@@ -482,11 +482,11 @@ private static BufferOrEvent createBuffer(int channel) {
 		return new BufferOrEvent(
 				new Buffer(MemorySegmentFactory.wrap(new byte[]{1, 2}), FreeingBufferRecycler.INSTANCE), channel);
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Testing Mocks
 	// ------------------------------------------------------------------------
-	
+
 	private static class CheckpointSequenceValidator implements StatefulTask {
 
 		private final long[] checkpointIDs;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BufferSpillerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BufferSpillerTest.java
index 24aabcc54a40b..905bc591ac7e9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BufferSpillerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BufferSpillerTest.java
@@ -44,7 +44,7 @@
 import static org.junit.Assert.*;
 
 public class BufferSpillerTest {
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(BufferSpillerTest.class);
 
 	private static final int PAGE_SIZE = 4096;
@@ -57,7 +57,7 @@ public class BufferSpillerTest {
 	// ------------------------------------------------------------------------
 	//  Setup / Cleanup
 	// ------------------------------------------------------------------------
-	
+
 	@BeforeClass
 	public static void setupIOManager() {
 		IO_MANAGER = new IOManagerAsync();
@@ -67,7 +67,7 @@ public static void setupIOManager() {
 	public static void shutdownIOManager() {
 		IO_MANAGER.shutdown();
 	}
-	
+
 	@Before
 	public void createSpiller() {
 		try {
@@ -78,7 +78,7 @@ public void createSpiller() {
 			fail("Cannot create BufferSpiller: " + e.getMessage());
 		}
 	}
-	
+
 	@After
 	public void cleanupSpiller() {
 		if (spiller != null) {
@@ -89,18 +89,18 @@ public void cleanupSpiller() {
 				e.printStackTrace();
 				fail("Cannot properly close the BufferSpiller: " + e.getMessage());
 			}
-			
+
 			assertFalse(spiller.getCurrentChannel().isOpen());
 			assertFalse(spiller.getCurrentSpillFile().exists());
 		}
-		
+
 		checkNoTempFilesRemain();
 	}
 
 	// ------------------------------------------------------------------------
 	//  Tests
 	// ------------------------------------------------------------------------
-	
+
 	@Test
 	public void testRollOverEmptySequences() {
 		try {
@@ -125,13 +125,13 @@ public void testSpillAndRollOverSimple() {
 
 			// do multiple spilling / rolling over rounds
 			for (int round = 0; round < 5; round++) {
-				
+
 				final long bufferSeed = rnd.nextLong();
 				bufferRnd.setSeed(bufferSeed);
-				
+
 				final int numEventsAndBuffers = rnd.nextInt(maxNumEventsAndBuffers) + 1;
 				final int numChannels = rnd.nextInt(maxNumChannels) + 1;
-				
+
 				final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);
 
 				// generate sequence
@@ -150,7 +150,7 @@ public void testSpillAndRollOverSimple() {
 
 				// reset and create reader
 				bufferRnd.setSeed(bufferSeed);
-			
+
 				BufferSpiller.SpilledBufferOrEventSequence seq = spiller.rollOver();
 				seq.open();
 
@@ -175,7 +175,7 @@ public void testSpillAndRollOverSimple() {
 
 				// all events need to be consumed
 				assertEquals(events.size(), numEvent);
-				
+
 				seq.cleanup();
 			}
 		}
@@ -188,22 +188,22 @@ public void testSpillAndRollOverSimple() {
 	@Test
 	public void testSpillWhileReading() {
 		LOG.info("Starting SpillWhileReading test");
-		
+
 		try {
 			final int sequences = 10;
-			
+
 			final Random rnd = new Random();
-			
+
 			final int maxNumEventsAndBuffers = 30000;
 			final int maxNumChannels = 1656;
-			
+
 			int sequencesConsumed = 0;
-			
+
 			ArrayDeque<SequenceToConsume> pendingSequences = new ArrayDeque<SequenceToConsume>();
 			SequenceToConsume currentSequence = null;
 			int currentNumEvents = 0;
 			int currentNumRecordAndEvents = 0;
-			
+
 			// do multiple spilling / rolling over rounds
 			for (int round = 0; round < 2*sequences; round++) {
 
@@ -215,15 +215,15 @@ public void testSpillWhileReading() {
 					// proper spilled sequence
 					final long bufferSeed = rnd.nextLong();
 					final Random bufferRnd = new Random(bufferSeed);
-					
+
 					final int numEventsAndBuffers = rnd.nextInt(maxNumEventsAndBuffers) + 1;
 					final int numChannels = rnd.nextInt(maxNumChannels) + 1;
-	
+
 					final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);
-	
+
 					int generated = 0;
 					while (generated < numEventsAndBuffers) {
-						
+
 						if (currentSequence == null || rnd.nextDouble() < 0.5) {
 							// add a new record
 							boolean isEvent = rnd.nextDouble() < 0.05;
@@ -251,34 +251,34 @@ public void testSpillWhileReading() {
 								Random validationRnd = currentSequence.bufferRnd;
 								validateBuffer(next, validationRnd.nextInt(PAGE_SIZE) + 1, validationRnd.nextInt(currentSequence.numChannels));
 							}
-							
+
 							currentNumRecordAndEvents++;
 							if (currentNumRecordAndEvents == currentSequence.numBuffersAndEvents) {
 								// done with the sequence
 								currentSequence.sequence.cleanup();
 								sequencesConsumed++;
-								
+
 								// validate we had all events
 								assertEquals(currentSequence.events.size(), currentNumEvents);
-								
+
 								// reset
 								currentSequence = pendingSequences.pollFirst();
 								if (currentSequence != null) {
 									currentSequence.sequence.open();
 								}
-								
+
 								currentNumRecordAndEvents = 0;
 								currentNumEvents = 0;
 							}
 						}
 					}
-	
+
 					// done generating a sequence. queue it for consumption
 					bufferRnd.setSeed(bufferSeed);
 					BufferSpiller.SpilledBufferOrEventSequence seq = spiller.rollOver();
-					
+
 					SequenceToConsume stc = new SequenceToConsume(bufferRnd, events, seq, numEventsAndBuffers, numChannels);
-					
+
 					if (currentSequence == null) {
 						currentSequence = stc;
 						stc.sequence.open();
@@ -288,7 +288,7 @@ public void testSpillWhileReading() {
 					}
 				}
 			}
-			
+
 			// consume all the remainder
 			while (currentSequence != null) {
 				// consume a record
@@ -323,7 +323,7 @@ public void testSpillWhileReading() {
 					currentNumEvents = 0;
 				}
 			}
-			
+
 			assertEquals(sequences, sequencesConsumed);
 		}
 		catch (Exception e) {
@@ -350,7 +350,7 @@ public void testHeaderSizeStaticField() throws Exception {
 	// ------------------------------------------------------------------------
 	//  Utils
 	// ------------------------------------------------------------------------
-	
+
 	private static BufferOrEvent generateRandomEvent(Random rnd, int numChannels) {
 		long magicNumber = rnd.nextLong();
 		byte[] data = new byte[rnd.nextInt(1000)];
@@ -358,7 +358,7 @@ private static BufferOrEvent generateRandomEvent(Random rnd, int numChannels) {
 		TestEvent evt = new TestEvent(magicNumber, data);
 
 		int channelIndex = rnd.nextInt(numChannels);
-		
+
 		return new BufferOrEvent(evt, channelIndex);
 	}
 
@@ -367,7 +367,7 @@ private static BufferOrEvent generateRandomBuffer(int size, int channelIndex) {
 		for (int i = 0; i < size; i++) {
 			seg.put(i, (byte) i);
 		}
-		
+
 		Buffer buf = new Buffer(seg, FreeingBufferRecycler.INSTANCE);
 		buf.setSize(size);
 		return new BufferOrEvent(buf, channelIndex);
@@ -400,7 +400,7 @@ private static void checkNoTempFilesRemain() {
 			}
 		}
 	}
-	
+
 	private static class SequenceToConsume {
 
 		final BufferSpiller.SpilledBufferOrEventSequence sequence;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/SpilledBufferOrEventSequenceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/SpilledBufferOrEventSequenceTest.java
index 991b03320ae83..a6e834c67c016 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/SpilledBufferOrEventSequenceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/SpilledBufferOrEventSequenceTest.java
@@ -44,14 +44,14 @@
  * with respect to detecting corrupt sequences, trailing data, and interleaved buffers and events.
  */
 public class SpilledBufferOrEventSequenceTest {
-	
+
 	private final ByteBuffer buffer = ByteBuffer.allocateDirect(128 * 1024).order(ByteOrder.LITTLE_ENDIAN);
 	private final int pageSize = 32*1024;
-	
+
 	private File tempFile;
 	private FileChannel fileChannel;
-	
-	
+
+
 	@Before
 	public void initTempChannel() {
 		try {
@@ -62,7 +62,7 @@ public void initTempChannel() {
 			cleanup();
 		}
 	}
-	
+
 	@After
 	public void cleanup() {
 		if (fileChannel != null) {
@@ -78,18 +78,18 @@ public void cleanup() {
 			tempFile.delete();
 		}
 	}
-	
-	
+
+
 	// ------------------------------------------------------------------------
 	//  Tests
 	// ------------------------------------------------------------------------
-	
+
 	@Test
 	public void testEmptyChannel() {
 		try {
 			SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			seq.open();
-			
+
 			assertNull(seq.getNext());
 		}
 		catch (Exception e) {
@@ -97,19 +97,19 @@ public void testEmptyChannel() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testIncompleteHeaderOnFirstElement() {
 		try {
 			ByteBuffer buf = ByteBuffer.allocate(7);
 			buf.order(ByteOrder.LITTLE_ENDIAN);
-			
+
 			fileChannel.write(buf);
 			fileChannel.position(0);
-			
+
 			SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			seq.open();
-			
+
 			try {
 				seq.getNext();
 				fail("should fail with an exception");
@@ -129,12 +129,12 @@ public void testBufferSequence() {
 		try {
 			final Random rnd = new Random();
 			final long seed = rnd.nextLong();
-			
+
 			final int numBuffers = 325;
 			final int numChannels = 671;
-			
+
 			rnd.setSeed(seed);
-			
+
 			for (int i = 0; i < numBuffers; i++) {
 				writeBuffer(fileChannel, rnd.nextInt(pageSize) + 1, rnd.nextInt(numChannels));
 			}
@@ -144,11 +144,11 @@ public void testBufferSequence() {
 
 			SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			seq.open();
-			
+
 			for (int i = 0; i < numBuffers; i++) {
 				validateBuffer(seq.getNext(), rnd.nextInt(pageSize) + 1, rnd.nextInt(numChannels));
 			}
-			
+
 			// should have no more data
 			assertNull(seq.getNext());
 		}
@@ -157,16 +157,16 @@ public void testBufferSequence() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testBufferSequenceWithIncompleteBuffer() {
 		try {
 			writeBuffer(fileChannel, 1672, 7);
-			
+
 			// write an incomplete buffer
 			ByteBuffer data = ByteBuffer.allocate(615);
 			data.order(ByteOrder.LITTLE_ENDIAN);
-			
+
 			data.putInt(2);
 			data.putInt(999);
 			data.put((byte) 0);
@@ -177,10 +177,10 @@ public void testBufferSequenceWithIncompleteBuffer() {
 
 			SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			seq.open();
-			
+
 			// first one is valid
 			validateBuffer(seq.getNext(), 1672, 7);
-			
+
 			// next one should fail
 			try {
 				seq.getNext();
@@ -202,9 +202,9 @@ public void testEventSequence() {
 			final Random rnd = new Random();
 			final int numEvents = 3000;
 			final int numChannels = 1656;
-			
+
 			final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(numEvents);
-			
+
 			for (int i = 0; i < numEvents; i++) {
 				events.add(generateAndWriteEvent(fileChannel, rnd, numChannels));
 			}
@@ -212,7 +212,7 @@ public void testEventSequence() {
 			fileChannel.position(0L);
 			SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			seq.open();
-			
+
 			int i = 0;
 			BufferOrEvent boe;
 			while ((boe = seq.getNext()) != null) {
@@ -222,7 +222,7 @@ public void testEventSequence() {
 				assertEquals(expected.getChannelIndex(), boe.getChannelIndex());
 				i++;
 			}
-			
+
 			assertEquals(numEvents, i);
 		}
 		catch (Exception e) {
@@ -239,14 +239,14 @@ public void testMixedSequence() {
 
 			final long bufferSeed = rnd.nextLong();
 			bufferRnd.setSeed(bufferSeed);
-			
+
 			final int numEventsAndBuffers = 3000;
 			final int numChannels = 1656;
 
 			final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);
 
 			// generate sequence
-			
+
 			for (int i = 0; i < numEventsAndBuffers; i++) {
 				boolean isEvent = rnd.nextDouble() < 0.05d;
 				if (isEvent) {
@@ -256,16 +256,16 @@ public void testMixedSequence() {
 					writeBuffer(fileChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
 				}
 			}
-			
+
 			// reset and create reader
-			
+
 			fileChannel.position(0L);
 			bufferRnd.setSeed(bufferSeed);
 			SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			seq.open();
-			
+
 			// read and validate the sequence
-			
+
 			int numEvent = 0;
 			for (int i = 0; i < numEventsAndBuffers; i++) {
 				BufferOrEvent next = seq.getNext();
@@ -278,10 +278,10 @@ public void testMixedSequence() {
 					validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
 				}
 			}
-			
+
 			// no further data
 			assertNull(seq.getNext());
-			
+
 			// all events need to be consumed
 			assertEquals(events.size(), numEvent);
 		}
@@ -295,12 +295,12 @@ public void testMixedSequence() {
 	public void testMultipleSequences() {
 		File secondFile = null;
 		FileChannel secondChannel = null;
-		
+
 		try {
 			// create the second file channel
 			secondFile = File.createTempFile("testdata", "tmp");
 			secondChannel = new RandomAccessFile(secondFile, "rw").getChannel();
-			
+
 			final Random rnd = new Random();
 			final Random bufferRnd = new Random();
 
@@ -309,7 +309,7 @@ public void testMultipleSequences() {
 
 			final int numEventsAndBuffers1 = 272;
 			final int numEventsAndBuffers2 = 151;
-			
+
 			final int numChannels = 1656;
 
 			final ArrayList<BufferOrEvent> events1 = new ArrayList<BufferOrEvent>(128);
@@ -343,9 +343,9 @@ public void testMultipleSequences() {
 
 			fileChannel.position(0L);
 			secondChannel.position(0L);
-			
+
 			bufferRnd.setSeed(bufferSeed);
-			
+
 			SpilledBufferOrEventSequence seq1 = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			SpilledBufferOrEventSequence seq2 = new SpilledBufferOrEventSequence(secondFile, secondChannel, buffer, pageSize);
 
@@ -410,14 +410,14 @@ public void testCleanup() {
 		try {
 			ByteBuffer data = ByteBuffer.allocate(157);
 			data.order(ByteOrder.LITTLE_ENDIAN);
-			
+
 			fileChannel.write(data);
 			fileChannel.position(54);
-			
+
 			SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
 			seq.open();
 			seq.cleanup();
-			
+
 			assertFalse(fileChannel.isOpen());
 			assertFalse(tempFile.exists());
 		}
@@ -426,7 +426,7 @@ public void testCleanup() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Utils
 	// ------------------------------------------------------------------------
@@ -436,27 +436,27 @@ private static BufferOrEvent generateAndWriteEvent(FileChannel fileChannel, Rand
 		byte[] data = new byte[rnd.nextInt(1000)];
 		rnd.nextBytes(data);
 		TestEvent evt = new TestEvent(magicNumber, data);
-		
+
 		int channelIndex = rnd.nextInt(numChannels);
-		
+
 		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
 		ByteBuffer header = ByteBuffer.allocate(9);
 		header.order(ByteOrder.LITTLE_ENDIAN);
-		
+
 		header.putInt(channelIndex);
 		header.putInt(serializedEvent.remaining());
 		header.put((byte) 1);
 		header.flip();
-		
+
 		fileChannel.write(header);
 		fileChannel.write(serializedEvent);
 		return new BufferOrEvent(evt, channelIndex);
 	}
-	
+
 	private static void writeBuffer(FileChannel fileChannel, int size, int channelIndex) throws IOException {
 		ByteBuffer data = ByteBuffer.allocate(size + 9);
 		data.order(ByteOrder.LITTLE_ENDIAN);
-		
+
 		data.putInt(channelIndex);
 		data.putInt(size);
 		data.put((byte) 0);
@@ -470,10 +470,10 @@ private static void writeBuffer(FileChannel fileChannel, int size, int channelIn
 	private static void validateBuffer(BufferOrEvent boe, int expectedSize, int expectedChannelIndex) {
 		assertEquals("wrong channel index", expectedChannelIndex, boe.getChannelIndex());
 		assertTrue("is not buffer", boe.isBuffer());
-		
+
 		Buffer buf = boe.getBuffer();
 		assertEquals("wrong buffer size", expectedSize, buf.getSize());
-		
+
 		MemorySegment seg = buf.getMemorySegment();
 		for (int i = 0; i < expectedSize; i++) {
 			assertEquals("wrong buffer contents", (byte) i, seg.get(i));
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java
index 45bbbda4faf9f..38741baac2022 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java
@@ -42,7 +42,7 @@
 import static org.mockito.Mockito.*;
 
 /**
- * This test uses the PowerMockRunner runner to work around the fact that the 
+ * This test uses the PowerMockRunner runner to work around the fact that the
  * {@link ResultPartitionWriter} class is final.
  */
 @RunWith(PowerMockRunner.class)
@@ -58,19 +58,19 @@ public void testPropagateAsyncFlushError() {
 		FailingWriter<LongValue> testWriter = null;
 		try {
 			ResultPartitionWriter mockResultPartitionWriter = getMockWriter(5);
-			
+
 			// test writer that flushes every 5ms and fails after 3 flushes
 			testWriter = new FailingWriter<LongValue>(mockResultPartitionWriter,
 					new RoundRobinChannelSelector<LongValue>(), 5, 3);
-			
+
 			try {
 				long deadline = System.currentTimeMillis() + 20000; // in max 20 seconds (conservative)
 				long l = 0L;
-				
+
 				while (System.currentTimeMillis() < deadline) {
 					testWriter.emit(new LongValue(l++));
 				}
-				
+
 				fail("This should have failed with an exception");
 			}
 			catch (IOException e) {
@@ -88,7 +88,7 @@ public void testPropagateAsyncFlushError() {
 			}
 		}
 	}
-	
+
 	private static ResultPartitionWriter getMockWriter(int numPartitions) throws Exception {
 		BufferProvider mockProvider = mock(BufferProvider.class);
 		when(mockProvider.requestBufferBlocking()).thenAnswer(new Answer<Buffer>() {
@@ -99,21 +99,21 @@ public Buffer answer(InvocationOnMock invocation) {
 						FreeingBufferRecycler.INSTANCE);
 			}
 		});
-		
+
 		ResultPartitionWriter mockWriter = mock(ResultPartitionWriter.class);
 		when(mockWriter.getBufferProvider()).thenReturn(mockProvider);
 		when(mockWriter.getNumberOfOutputChannels()).thenReturn(numPartitions);
-		
-		
+
+
 		return mockWriter;
 	}
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	private static class FailingWriter<T extends IOReadableWritable> extends StreamRecordWriter<T> {
-		
+
 		private int flushesBeforeException;
-		
+
 		private FailingWriter(ResultPartitionWriter writer, ChannelSelector<T> channelSelector,
 								long timeout, int flushesBeforeException) {
 			super(writer, channelSelector, timeout);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/TestEvent.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/TestEvent.java
index 9fcb7fe5e35fa..85e7c6289c193 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/TestEvent.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/TestEvent.java
@@ -85,4 +85,4 @@ public boolean equals(Object obj) {
 	public String toString() {
 		return String.format("TestEvent %d (%s)", magicNumber, StringUtils.byteToHexString(payload));
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorTest.java
index 912b448fb2a10..2a6a723b6a3d2 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamSourceOperatorTest.java
@@ -60,19 +60,19 @@
 
 @SuppressWarnings("serial")
 public class StreamSourceOperatorTest {
-	
+
 	@Test
 	public void testEmitMaxWatermarkForFiniteSource() throws Exception {
 
 		// regular stream source operator
-		StreamSource<String, FiniteSource<String>> operator = 
+		StreamSource<String, FiniteSource<String>> operator =
 				new StreamSource<>(new FiniteSource<String>());
-		
+
 		final List<StreamElement> output = new ArrayList<>();
-		
+
 		setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, 0);
 		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));
-		
+
 		assertEquals(1, output.size());
 		assertEquals(Watermark.MAX_WATERMARK, output.get(0));
 	}
@@ -92,23 +92,23 @@ public void testNoMaxWatermarkOnImmediateCancel() throws Exception {
 
 		// run and exit
 		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));
-		
+
 		assertTrue(output.isEmpty());
 	}
-	
+
 	@Test
 	public void testNoMaxWatermarkOnAsyncCancel() throws Exception {
 
 		final List<StreamElement> output = new ArrayList<>();
 		final Thread runner = Thread.currentThread();
-		
+
 		// regular stream source operator
 		final StreamSource<String, InfiniteSource<String>> operator =
 				new StreamSource<>(new InfiniteSource<String>());
 
-		
+
 		setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, 0);
-		
+
 		// trigger an async cancel in a bit
 		new Thread("canceler") {
 			@Override
@@ -120,7 +120,7 @@ public void run() {
 				runner.interrupt();
 			}
 		}.start();
-		
+
 		// run and wait to be canceled
 		try {
 			operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));
@@ -289,7 +289,7 @@ private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
 
 		StreamConfig cfg = new StreamConfig(new Configuration());
 		cfg.setStateBackend(new MemoryStateBackend());
-		
+
 		cfg.setTimeCharacteristic(timeChar);
 
 		Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);
@@ -320,7 +320,7 @@ public ProcessingTimeService answer(InvocationOnMock invocation) throws Throwabl
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	private static final class FiniteSource<T> implements SourceFunction<T>, StoppableFunction {
 
 		@Override
@@ -336,7 +336,7 @@ public void stop() {}
 	private static final class InfiniteSource<T> implements SourceFunction<T>, StoppableFunction {
 
 		private volatile boolean running = true;
-		
+
 		@Override
 		public void run(SourceContext<T> ctx) throws Exception {
 			while (running) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java
index f41ec02efd039..6772db4551b07 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java
@@ -43,13 +43,13 @@ public class StreamTaskTimerTest {
 	@Test
 	public void testOpenCloseAndTimestamps() throws Exception {
 		final OneInputStreamTask<String, String> mapTask = new OneInputStreamTask<>();
-		
+
 		final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
 				mapTask, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
 		testHarness.setupOutputForSingletonOperatorChain();
 
 		StreamConfig streamConfig = testHarness.getStreamConfig();
-		
+
 		StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
 		streamConfig.setStreamOperator(mapOperator);
 
@@ -78,7 +78,7 @@ public void onProcessingTime(long timestamp) {
 		assertEquals("Trigger timer thread did not properly shut down",
 				0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
 	}
-	
+
 	@Test
 	public void checkScheduledTimestampe() {
 		try {
@@ -141,11 +141,11 @@ public void checkScheduledTimestampe() {
 	}
 
 	private static class ValidatingProcessingTimeCallback implements ProcessingTimeCallback {
-		
+
 		static int numInSequence;
-		
+
 		private final AtomicReference<Throwable> errorRef;
-		
+
 		private final long expectedTimestamp;
 		private final int expectedInSequence;
 
@@ -167,9 +167,9 @@ public void onProcessingTime(long timestamp) {
 			}
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	public static class DummyMapFunction<T> implements MapFunction<T, T> {
 		@Override
 		public T map(T value) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperatorTest.java
index 9ddea8cb8f181..f129c2053cd37 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPeriodicWatermarksOperatorTest.java
@@ -31,11 +31,11 @@
 import static org.junit.Assert.*;
 
 public class TimestampsAndPeriodicWatermarksOperatorTest {
-	
+
 	@Test
 	public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
-		
-		final TimestampsAndPeriodicWatermarksOperator<Long> operator = 
+
+		final TimestampsAndPeriodicWatermarksOperator<Long> operator =
 				new TimestampsAndPeriodicWatermarksOperator<Long>(new LongExtractor());
 
 		OneInputStreamOperatorTestHarness<Long, Long> testHarness =
@@ -46,20 +46,20 @@ public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
 		long currentTime = 0;
 
 		testHarness.open();
-		
+
 		testHarness.processElement(new StreamRecord<>(1L, 1));
 		testHarness.processElement(new StreamRecord<>(2L, 1));
 		testHarness.processWatermark(new Watermark(2)); // this watermark should be ignored
 		testHarness.processElement(new StreamRecord<>(3L, 3));
 		testHarness.processElement(new StreamRecord<>(4L, 3));
-		
+
 		// validate first part of the sequence. we poll elements until our
 		// watermark updates to "3", which must be the result of the "4" element.
 		{
 			ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
 			long nextElementValue = 1L;
 			long lastWatermark = -1L;
-			
+
 			while (lastWatermark < 3) {
 				if (output.size() > 0) {
 					Object next = output.poll();
@@ -67,7 +67,7 @@ public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
 					Tuple2<Long, Long> update = validateElement(next, nextElementValue, lastWatermark);
 					nextElementValue = update.f0;
 					lastWatermark = update.f1;
-					
+
 					// check the invariant
 					assertTrue(lastWatermark < nextElementValue);
 				} else {
@@ -75,7 +75,7 @@ public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
 					testHarness.setProcessingTime(currentTime);
 				}
 			}
-			
+
 			output.clear();
 		}
 
@@ -99,7 +99,7 @@ public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
 					Tuple2<Long, Long> update = validateElement(next, nextElementValue, lastWatermark);
 					nextElementValue = update.f0;
 					lastWatermark = update.f1;
-					
+
 					// check the invariant
 					assertTrue(lastWatermark < nextElementValue);
 				} else {
@@ -110,7 +110,7 @@ public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
 
 			output.clear();
 		}
-		
+
 		testHarness.processWatermark(new Watermark(Long.MAX_VALUE));
 		assertEquals(Long.MAX_VALUE, ((Watermark) testHarness.getOutput().poll()).getTimestamp());
 	}
@@ -131,20 +131,20 @@ public void testNegativeTimestamps() throws Exception {
 		testHarness.open();
 
 		long[] values = { Long.MIN_VALUE, -1L, 0L, 1L, 2L, 3L, Long.MAX_VALUE };
-		
+
 		for (long value : values) {
 			testHarness.processElement(new StreamRecord<>(value));
 		}
 
 		ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
-		
+
 		for (long value: values) {
 			assertEquals(value, ((StreamRecord<?>) output.poll()).getTimestamp());
 		}
 	}
 
 	// ------------------------------------------------------------------------
-	
+
 	private Tuple2<Long, Long> validateElement(Object element, long nextElementValue, long currentWatermark) {
 		if (element instanceof StreamRecord) {
 			@SuppressWarnings("unchecked")
@@ -162,7 +162,7 @@ else if (element instanceof Watermark) {
 			throw new IllegalArgumentException("unrecognized element: " + element);
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	private static class LongExtractor implements AssignerWithPeriodicWatermarks<Long> {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperatorTest.java
index 07199ac58617a..0333e9319dc17 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/TimestampsAndPunctuatedWatermarksOperatorTest.java
@@ -35,15 +35,15 @@ public class TimestampsAndPunctuatedWatermarksOperatorTest {
 	@Test
 	@SuppressWarnings("unchecked")
 	public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
-		
-		final TimestampsAndPunctuatedWatermarksOperator<Tuple2<Long, Boolean>> operator = 
+
+		final TimestampsAndPunctuatedWatermarksOperator<Tuple2<Long, Boolean>> operator =
 				new TimestampsAndPunctuatedWatermarksOperator<>(new PunctuatedExtractor());
-		
+
 		OneInputStreamOperatorTestHarness<Tuple2<Long, Boolean>, Tuple2<Long, Boolean>> testHarness =
 				new OneInputStreamOperatorTestHarness<>(operator);
 
 		testHarness.open();
-		
+
 		testHarness.processElement(new StreamRecord<>(new Tuple2<>(3L, true), 0L));
 		testHarness.processElement(new StreamRecord<>(new Tuple2<>(5L, false), 0L));
 		testHarness.processElement(new StreamRecord<>(new Tuple2<>(4L, false), 0L));
@@ -58,10 +58,10 @@ public void testTimestampsAndPeriodicWatermarksOperator() throws Exception {
 		testHarness.processWatermark(new Watermark(Long.MAX_VALUE));
 
 		ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
-		
+
 		assertEquals(3L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
 		assertEquals(3L, ((Watermark) output.poll()).getTimestamp());
-		
+
 		assertEquals(5L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
 		assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
 		assertEquals(4L, ((StreamRecord<Tuple2<Long, Boolean>>) output.poll()).getTimestamp());
@@ -102,12 +102,12 @@ public void testZeroOnNegativeTimestamps() throws Exception {
 			assertEquals(value, ((StreamRecord<?>) output.poll()).getTimestamp());
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	private static class PunctuatedExtractor implements AssignerWithPunctuatedWatermarks<Tuple2<Long, Boolean>> {
 		private static final long serialVersionUID = 1L;
-		
+
 		@Override
 		public long extractTimestamp(Tuple2<Long, Boolean> element, long previousTimestamp) {
 			return element.f0;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
index 6f0e881470aff..a8d31548aefdd 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
@@ -80,14 +80,14 @@ public class AccumulatingAlignedProcessingTimeWindowOperatorTest {
 
 	@SuppressWarnings("unchecked")
 	private final KeySelector<String, String> mockKeySelector = mock(KeySelector.class);
-	
+
 	private final KeySelector<Integer, Integer> identitySelector = new KeySelector<Integer, Integer>() {
 		@Override
 		public Integer getKey(Integer value) {
 			return value;
 		}
 	};
-	
+
 	private final InternalIterableWindowFunction<Integer, Integer, Integer, TimeWindow> validatingIdentityFunction =
 			new InternalIterableWindowFunction<>(new WindowFunction<Integer, Integer, Integer, TimeWindow>() {
 				@Override
@@ -117,7 +117,7 @@ public AccumulatingAlignedProcessingTimeWindowOperatorTest() {
 		ClosureCleaner.clean(validatingIdentityFunction, false);
 		ClosureCleaner.clean(validatingIdentityProcessFunction, false);
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	@After
@@ -134,9 +134,9 @@ public void checkNoTriggerThreadsRunning() {
 		assertTrue("Not all trigger threads where properly shut down",
 				StreamTask.TRIGGER_THREAD_GROUP.activeCount() == 0);
 	}
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	@Test
 	public void testInvalidParameters() {
 		try {
@@ -144,7 +144,7 @@ public void testInvalidParameters() {
 			assertInvalidParameter(10000L, -1L);
 			assertInvalidParameter(-1L, 1000L);
 			assertInvalidParameter(1000L, 2000L);
-			
+
 			// actual internal slide is too low here:
 			assertInvalidParameter(1000L, 999L);
 		}
@@ -153,12 +153,12 @@ public void testInvalidParameters() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testWindowSizeAndSlide() {
 		try {
 			AccumulatingProcessingTimeWindowOperator<String, String, String> op;
-			
+
 			op = new AccumulatingProcessingTimeWindowOperator<>(mockFunction, mockKeySelector,
 					StringSerializer.INSTANCE, StringSerializer.INSTANCE, 5000, 1000);
 			assertEquals(5000, op.getWindowSize());
@@ -568,7 +568,7 @@ public void testSlidingWindowSingleElements() throws Exception {
 			List<Integer> result = extractFromStreamRecords(testHarness.extractOutputStreamRecords());
 
 			assertEquals(6, result.size());
-			
+
 			Collections.sort(result);
 			assertEquals(Arrays.asList(1, 1, 1, 2, 2, 2), result);
 
@@ -898,7 +898,7 @@ public void checkpointRestoreWithPendingWindowSliding() {
 			// inject some elements
 			final int numElements = 1000;
 			final int numElementsFirst = 700;
-			
+
 			for (int i = 0; i < numElementsFirst; i++) {
 				testHarness.processElement(new StreamRecord<>(i));
 			}
@@ -964,13 +964,13 @@ public void checkpointRestoreWithPendingWindowSliding() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testKeyValueStateInWindowFunction() {
 		try {
 
 			StatefulFunction.globalCounts.clear();
-			
+
 			// tumbling window that triggers every 20 milliseconds
 			AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op =
 					new AccumulatingProcessingTimeWindowOperator<>(
@@ -1008,7 +1008,7 @@ public void testKeyValueStateInWindowFunction() {
 
 			assertEquals(4, StatefulFunction.globalCounts.get(1).intValue());
 			assertEquals(4, StatefulFunction.globalCounts.get(2).intValue());
-			
+
 			testHarness.close();
 			op.dispose();
 		}
@@ -1017,13 +1017,13 @@ public void testKeyValueStateInWindowFunction() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	private void assertInvalidParameter(long windowSize, long windowSlide) {
 		try {
 			new AccumulatingProcessingTimeWindowOperator<String, String, String>(
-					mockFunction, mockKeySelector, 
+					mockFunction, mockKeySelector,
 					StringSerializer.INSTANCE, StringSerializer.INSTANCE,
 					windowSize, windowSlide);
 			fail("This should fail with an IllegalArgumentException");
@@ -1064,7 +1064,7 @@ public void process(Integer key,
 				// the checks may fail
 				state.update(state.value() + 1);
 				globalCounts.put(key, state.value());
-				
+
 				out.collect(i);
 			}
 		}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
index 34eac9e384c84..468f14c2b0eb8 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
@@ -553,13 +553,13 @@ public void testAggregateEventTime() throws Exception {
 				.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
 				.aggregate(new DummyAggregationFunction());
 
-		OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = 
+		OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform =
 				(OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
 
 		OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
 
 		Assert.assertTrue(operator instanceof WindowOperator);
-		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = 
+		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator =
 				(WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
 
 		Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
@@ -581,13 +581,13 @@ public void testAggregateProcessingTime() throws Exception {
 				.windowAll(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
 				.aggregate(new DummyAggregationFunction());
 
-		OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = 
+		OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform =
 				(OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
 
 		OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
 
 		Assert.assertTrue(operator instanceof WindowOperator);
-		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = 
+		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator =
 				(WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
 
 		Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
@@ -615,7 +615,7 @@ public void testAggregateWithWindowFunctionEventTime() throws Exception {
 		OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
 
 		Assert.assertTrue(operator instanceof WindowOperator);
-		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = 
+		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator =
 				(WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
 
 		Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
@@ -643,7 +643,7 @@ public void testAggregateWithWindowFunctionProcessingTime() throws Exception {
 		OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
 
 		Assert.assertTrue(operator instanceof WindowOperator);
-		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = 
+		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator =
 				(WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
 
 		Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
@@ -1507,7 +1507,7 @@ private static class TestAllWindowFunction
 		@Override
 		public void apply(
 				TimeWindow window,
-				Iterable<Tuple2<String, Integer>> values, 
+				Iterable<Tuple2<String, Integer>> values,
 				Collector<Tuple3<String, String, Integer>> out) throws Exception {
 
 			for (Tuple2<String, Integer> in : values) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutIfAbsentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutIfAbsentTest.java
index c0b20a34780b6..8786c4ef56842 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutIfAbsentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutIfAbsentTest.java
@@ -25,15 +25,15 @@
 import static org.junit.Assert.fail;
 
 public class KeyMapPutIfAbsentTest {
-	
+
 	@Test
 	public void testPutIfAbsentUniqueKeysAndGrowth() {
 		try {
 			KeyMap<Integer, Integer> map = new KeyMap<>();
 			IntegerFactory factory = new IntegerFactory();
-			
+
 			final int numElements = 1000000;
-			
+
 			for (int i = 0; i < numElements; i++) {
 				factory.set(2 * i + 1);
 				map.putIfAbsent(i, factory);
@@ -43,7 +43,7 @@ public void testPutIfAbsentUniqueKeysAndGrowth() {
 				assertTrue(map.getCurrentTableCapacity() > map.getRehashThreshold());
 				assertTrue(map.size() <= map.getRehashThreshold());
 			}
-			
+
 			assertEquals(numElements, map.size());
 			assertEquals(numElements, map.traverseAndCountElements());
 			assertEquals(1 << 21, map.getCurrentTableCapacity());
@@ -51,7 +51,7 @@ public void testPutIfAbsentUniqueKeysAndGrowth() {
 			for (int i = 0; i < numElements; i++) {
 				assertEquals(2 * i + 1, map.get(i).intValue());
 			}
-			
+
 			for (int i = numElements - 1; i >= 0; i--) {
 				assertEquals(2 * i + 1, map.get(i).intValue());
 			}
@@ -66,13 +66,13 @@ public void testPutIfAbsentUniqueKeysAndGrowth() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testPutIfAbsentDuplicateKeysAndGrowth() {
 		try {
 			KeyMap<Integer, Integer> map = new KeyMap<>();
 			IntegerFactory factory = new IntegerFactory();
-			
+
 			final int numElements = 1000000;
 
 			for (int i = 0; i < numElements; i++) {
@@ -102,13 +102,13 @@ public void testPutIfAbsentDuplicateKeysAndGrowth() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
-	
+
 	private static class IntegerFactory implements KeyMap.LazyFactory<Integer> {
-		
+
 		private Integer toCreate;
-		
+
 		public void set(Integer toCreate) {
 			this.toCreate = toCreate;
 		}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutTest.java
index 09c44feb54fe8..5b59bea653640 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapPutTest.java
@@ -63,7 +63,7 @@ public void testPutUniqueKeysAndGrowth() {
 			int numContained = 0;
 			for (KeyMap.Entry<Integer, Integer> entry : map) {
 				numContained++;
-				
+
 				assertEquals(entry.getKey() * 2 + 1, entry.getValue().intValue());
 				assertFalse(bitset.get(entry.getKey()));
 				bitset.set(entry.getKey());
@@ -71,8 +71,8 @@ public void testPutUniqueKeysAndGrowth() {
 
 			assertEquals(numElements, numContained);
 			assertEquals(numElements, bitset.cardinality());
-			
-			
+
+
 			assertEquals(numElements, map.size());
 			assertEquals(numElements, map.traverseAndCountElements());
 			assertEquals(1 << 21, map.getCurrentTableCapacity());
@@ -105,13 +105,13 @@ public void testPutDuplicateKeysAndGrowth() {
 				int expected = (i % 3 == 0) ? (2*i) : (2*i+1);
 				assertEquals(expected, map.get(i).intValue());
 			}
-			
+
 			assertEquals(numElements, map.size());
 			assertEquals(numElements, map.traverseAndCountElements());
 			assertEquals(1 << 21, map.getCurrentTableCapacity());
 			assertTrue(map.getLongestChainLength() <= 7);
 
-			
+
 			BitSet bitset = new BitSet();
 			int numContained = 0;
 			for (KeyMap.Entry<Integer, Integer> entry : map) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapTest.java
index 49310df369a62..c7fb108d69b19 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMapTest.java
@@ -27,7 +27,7 @@
 import static org.junit.Assert.*;
 
 public class KeyMapTest {
-	
+
 	@Test
 	public void testInitialSizeComputation() {
 		try {
@@ -38,7 +38,7 @@ public void testInitialSizeComputation() {
 			assertEquals(6, map.getLog2TableCapacity());
 			assertEquals(24, map.getShift());
 			assertEquals(48, map.getRehashThreshold());
-			
+
 			map = new KeyMap<>(0);
 			assertEquals(64, map.getCurrentTableCapacity());
 			assertEquals(6, map.getLog2TableCapacity());
@@ -80,7 +80,7 @@ public void testInitialSizeComputation() {
 			assertEquals(7, map.getLog2TableCapacity());
 			assertEquals(23, map.getShift());
 			assertEquals(96, map.getRehashThreshold());
-			
+
 			// no negative number of elements
 			try {
 				new KeyMap<>(-1);
@@ -89,7 +89,7 @@ public void testInitialSizeComputation() {
 			catch (IllegalArgumentException e) {
 				// expected
 			}
-			
+
 			// check integer overflow
 			try {
 				map = new KeyMap<>(0x65715522);
@@ -115,17 +115,17 @@ public void testPutAndGetRandom() {
 		try {
 			final KeyMap<Integer, Integer> map = new KeyMap<>();
 			final Random rnd = new Random();
-			
+
 			final long seed = rnd.nextLong();
 			final int numElements = 10000;
-			
+
 			final HashMap<Integer, Integer> groundTruth = new HashMap<>();
-			
+
 			rnd.setSeed(seed);
 			for (int i = 0; i < numElements; i++) {
 				Integer key = rnd.nextInt();
 				Integer value = rnd.nextInt();
-				
+
 				if (rnd.nextBoolean()) {
 					groundTruth.put(key, value);
 					map.put(key, value);
@@ -139,7 +139,7 @@ public void testPutAndGetRandom() {
 				// skip these, evaluating it is tricky due to duplicates
 				rnd.nextInt();
 				rnd.nextBoolean();
-				
+
 				Integer expected = groundTruth.get(key);
 				if (expected == null) {
 					assertNull(map.get(key));
@@ -156,12 +156,12 @@ public void testPutAndGetRandom() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testConjunctTraversal() {
 		try {
 			final Random rootRnd = new Random(654685486325439L);
-			
+
 			final int numMaps = 7;
 			final int numKeys = 1000000;
 
@@ -171,7 +171,7 @@ public void testConjunctTraversal() {
 			for (int i = 0; i < numMaps; i++) {
 				maps[i] = new KeyMap<>();
 			}
-			
+
 			// ------ prepare probabilities for maps ------
 			final double[] probabilities = new double[numMaps];
 			final double[] probabilitiesTemp = new double[numMaps];
@@ -186,39 +186,39 @@ public void testConjunctTraversal() {
 				// compensate for rounding errors
 				probabilities[numMaps - 1] = remainingProb;
 			}
-			
+
 			// ------ generate random elements ------
 			final long probSeed = rootRnd.nextLong();
 			final long keySeed = rootRnd.nextLong();
-			
+
 			final Random probRnd = new Random(probSeed);
 			final Random keyRnd = new Random(keySeed);
-			
+
 			final int maxStride = Integer.MAX_VALUE / numKeys;
-			
+
 			int totalNumElements = 0;
 			int nextKeyValue = 1;
-			
+
 			for (int i = 0; i < numKeys; i++) {
 				int numCopies = (nextKeyValue % 3) + 1;
 				System.arraycopy(probabilities, 0, probabilitiesTemp, 0, numMaps);
-				
+
 				double totalProb = 1.0;
 				for (int copy = 0; copy < numCopies; copy++) {
 					int pos = drawPosProportionally(probabilitiesTemp, totalProb, probRnd);
 					totalProb -= probabilitiesTemp[pos];
 					probabilitiesTemp[pos] = 0.0;
-					
+
 					Integer boxed = nextKeyValue;
 					Integer previous = maps[pos].put(boxed, boxed);
 					assertNull("Test problem - test does not assign unique maps", previous);
 				}
-				
+
 				totalNumElements += numCopies;
 				nextKeyValue += keyRnd.nextInt(maxStride) + 1;
 			}
-			
-			
+
+
 			// check that all maps contain the total number of elements
 			int numContained = 0;
 			for (KeyMap<?, ?> map : maps) {
@@ -228,13 +228,13 @@ public void testConjunctTraversal() {
 
 			// ------ check that all elements can be found in the maps ------
 			keyRnd.setSeed(keySeed);
-			
+
 			numContained = 0;
 			nextKeyValue = 1;
 			for (int i = 0; i < numKeys; i++) {
 				int numCopiesExpected = (nextKeyValue % 3) + 1;
 				int numCopiesContained = 0;
-				
+
 				for (KeyMap<Integer, Integer> map : maps) {
 					Integer val = map.get(nextKeyValue);
 					if (val != null) {
@@ -242,10 +242,10 @@ public void testConjunctTraversal() {
 						numCopiesContained++;
 					}
 				}
-				
+
 				assertEquals(numCopiesExpected, numCopiesContained);
 				numContained += numCopiesContained;
-				
+
 				nextKeyValue += keyRnd.nextInt(maxStride) + 1;
 			}
 			assertEquals(totalNumElements, numContained);
@@ -256,12 +256,12 @@ public void testConjunctTraversal() {
 
 				private int key;
 				private int valueCount;
-				
+
 				@Override
 				public void startNewKey(Integer key) {
 					this.key = key;
 					this.valueCount = 0;
-					
+
 					keysStartedAndFinished[0]++;
 				}
 
@@ -277,13 +277,13 @@ public void keyDone() {
 					if (expected != valueCount) {
 						fail("Wrong count for key " + key + " ; expected=" + expected + " , count=" + valueCount);
 					}
-					
+
 					keysStartedAndFinished[1]++;
 				}
 			};
-			
+
 			KeyMap.traverseMaps(shuffleArray(maps, rootRnd), traversal, 17);
-			
+
 			assertEquals(numKeys, keysStartedAndFinished[0]);
 			assertEquals(numKeys, keysStartedAndFinished[1]);
 		}
@@ -298,9 +298,9 @@ public void testSizeComparator() {
 		try {
 			KeyMap<String, String> map1 = new KeyMap<>(5);
 			KeyMap<String, String> map2 = new KeyMap<>(80);
-			
+
 			assertTrue(map1.getCurrentTableCapacity() < map2.getCurrentTableCapacity());
-			
+
 			assertTrue(KeyMap.CapacityDescendingComparator.INSTANCE.compare(map1, map1) == 0);
 			assertTrue(KeyMap.CapacityDescendingComparator.INSTANCE.compare(map2, map2) == 0);
 			assertTrue(KeyMap.CapacityDescendingComparator.INSTANCE.compare(map1, map2) > 0);
@@ -311,12 +311,12 @@ public void testSizeComparator() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	private static int drawPosProportionally(double[] array, double totalProbability, Random rnd) {
 		double val = rnd.nextDouble() * totalProbability;
-		
+
 		double accum = 0;
 		for (int i = 0; i < array.length; i++) {
 			accum += array[i];
@@ -324,21 +324,21 @@ private static int drawPosProportionally(double[] array, double totalProbability
 				return i;
 			}
 		}
-		
+
 		// in case of rounding errors
 		return array.length - 1;
 	}
-	
+
 	private static <E> E[] shuffleArray(E[] array, Random rnd) {
 		E[] target = Arrays.copyOf(array, array.length);
-		
+
 		for (int i = target.length - 1; i > 0; i--) {
 			int swapPos = rnd.nextInt(i + 1);
 			E temp = target[i];
 			target[i] = target[swapPos];
 			target[swapPos] = temp;
 		}
-		
+
 		return target;
 	}
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
index 8f8667b3eadd1..1137c6a7da69e 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
@@ -686,7 +686,7 @@ public void testSessionWindowsWithCountTrigger() throws Exception {
 
 		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
 				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-		
+
 		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
 
 		testHarness.open();
@@ -862,7 +862,7 @@ public void testPointSessions() throws Exception {
 
 		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
 				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-		
+
 		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
 
 		testHarness.open();
@@ -1296,9 +1296,9 @@ public void testLateness() throws Exception {
 
 		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
 			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-		
+
 		testHarness.open();
-		
+
 		ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
 		ConcurrentLinkedQueue<Object> lateExpected = new ConcurrentLinkedQueue<>();
 
@@ -1366,7 +1366,7 @@ public void testCleanupTimeOverflow() throws Exception {
 
 		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
 			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-		
+
 		testHarness.open();
 
 		ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
@@ -1500,7 +1500,7 @@ public void testSideOutputDueToLatenessSliding() throws Exception {
 			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
 
 		testHarness.open();
-		
+
 		ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
 		ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
 
@@ -1579,7 +1579,7 @@ public void testSideOutputDueToLatenessSessionZeroLatenessPurgingTrigger() throw
 			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
 
 		testHarness.open();
-		
+
 		ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
 		ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
 
@@ -1671,7 +1671,7 @@ public void testSideOutputDueToLatenessSessionZeroLateness() throws Exception {
 			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
 
 		testHarness.open();
-		
+
 		ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
 		ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();
 
@@ -1848,7 +1848,7 @@ public void testNotSideOutputDueToLatenessSessionWithLateness() throws Exception
 			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
 
 		testHarness.open();
-		
+
 		ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
 
 		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
@@ -1949,7 +1949,7 @@ public void testNotSideOutputDueToLatenessSessionWithHugeLatenessPurgingTrigger(
 			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
 
 		testHarness.open();
-		
+
 		ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
 
 		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/BroadcastPartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/BroadcastPartitionerTest.java
index a1cea1303ec9a..ddfb9e74c67cc 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/BroadcastPartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/BroadcastPartitionerTest.java
@@ -30,7 +30,7 @@ public class BroadcastPartitionerTest {
 	private BroadcastPartitioner<Tuple> broadcastPartitioner1;
 	private BroadcastPartitioner<Tuple> broadcastPartitioner2;
 	private BroadcastPartitioner<Tuple> broadcastPartitioner3;
-	
+
 	private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>(null);
 	private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(null);
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitionerTest.java
index aa70e8ac6b6d5..06a1acd27d5e7 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RebalancePartitionerTest.java
@@ -26,17 +26,17 @@
 import org.junit.Test;
 
 public class RebalancePartitionerTest {
-	
+
 	private RebalancePartitioner<Tuple> distributePartitioner;
 	private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>(null);
 	private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
 			null);
-	
+
 	@Before
 	public void setPartitioner() {
 		distributePartitioner = new RebalancePartitioner<Tuple>();
 	}
-	
+
 	@Test
 	public void testSelectChannelsLength() {
 		sd.setInstance(streamRecord);
@@ -44,7 +44,7 @@ public void testSelectChannelsLength() {
 		assertEquals(1, distributePartitioner.selectChannels(sd, 2).length);
 		assertEquals(1, distributePartitioner.selectChannels(sd, 1024).length);
 	}
-	
+
 	@Test
 	public void testSelectChannelsInterval() {
 		sd.setInstance(streamRecord);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
index d72c37bdd8829..7102f27a941e9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
@@ -59,17 +59,17 @@
 
 @SuppressWarnings("serial")
 public class RescalePartitionerTest extends TestLogger {
-	
+
 	private RescalePartitioner<Tuple> distributePartitioner;
 	private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>(null);
 	private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
 			null);
-	
+
 	@Before
 	public void setPartitioner() {
 		distributePartitioner = new RescalePartitioner<Tuple>();
 	}
-	
+
 	@Test
 	public void testSelectChannelsLength() {
 		sd.setInstance(streamRecord);
@@ -77,7 +77,7 @@ public void testSelectChannelsLength() {
 		assertEquals(1, distributePartitioner.selectChannels(sd, 2).length);
 		assertEquals(1, distributePartitioner.selectChannels(sd, 1024).length);
 	}
-	
+
 	@Test
 	public void testSelectChannelsInterval() {
 		sd.setInstance(streamRecord);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
index 0f42a659bc552..2012c94563607 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializerTest.java
@@ -37,20 +37,20 @@
 import static org.mockito.Mockito.when;
 
 public class StreamElementSerializerTest {
-	
+
 	@Test
 	public void testDeepDuplication() {
 		@SuppressWarnings("unchecked")
 		TypeSerializer<Long> serializer1 = (TypeSerializer<Long>) mock(TypeSerializer.class);
-		
+
 		@SuppressWarnings("unchecked")
 		TypeSerializer<Long> serializer2 = (TypeSerializer<Long>) mock(TypeSerializer.class);
-		
+
 		when(serializer1.duplicate()).thenReturn(serializer2);
-		
+
 		StreamElementSerializer<Long> streamRecSer =
 				new StreamElementSerializer<Long>(serializer1);
-		
+
 		assertEquals(serializer1, streamRecSer.getContainedTypeSerializer());
 
 		StreamElementSerializer<Long> copy = streamRecSer.duplicate();
@@ -62,12 +62,12 @@ public void testDeepDuplication() {
 	public void testBasicProperties() {
 		StreamElementSerializer<Long> streamRecSer =
 				new StreamElementSerializer<Long>(LongSerializer.INSTANCE);
-		
+
 		assertFalse(streamRecSer.isImmutableType());
 		assertEquals(Long.class, streamRecSer.createInstance().getValue().getClass());
 		assertEquals(-1L, streamRecSer.getLength());
 	}
-	
+
 	@Test
 	public void testSerialization() throws Exception {
 		final StreamElementSerializer<String> serializer =
@@ -88,20 +88,20 @@ public void testSerialization() throws Exception {
 		Watermark negativeWatermark = new Watermark(-4647654567676555876L);
 		assertEquals(negativeWatermark, serializeAndDeserialize(negativeWatermark, serializer));
 	}
-	
+
 	@SuppressWarnings("unchecked")
 	private static <T, X extends StreamElement> X serializeAndDeserialize(
 			X record,
 			StreamElementSerializer<T> serializer) throws IOException {
-		
+
 		DataOutputSerializer output = new DataOutputSerializer(32);
 		serializer.serialize(record, output);
-		
+
 		// additional binary copy step
 		DataInputDeserializer copyInput = new DataInputDeserializer(output.getByteArray(), 0, output.length());
 		DataOutputSerializer copyOutput = new DataOutputSerializer(32);
 		serializer.copy(copyInput, copyOutput);
-		
+
 		DataInputDeserializer input = new DataInputDeserializer(copyOutput.getByteArray(), 0, copyOutput.length());
 		return (X) serializer.deserialize(input);
 	}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecordTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecordTest.java
index 5d34b74f08bff..08d9644a2ca57 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecordTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamrecord/StreamRecordTest.java
@@ -23,17 +23,17 @@
 import static org.junit.Assert.*;
 
 public class StreamRecordTest {
-	
+
 	@Test
 	public void testWithNoTimestamp() {
 		StreamRecord<String> record = new StreamRecord<>("test");
-		
+
 		assertTrue(record.isRecord());
 		assertFalse(record.isWatermark());
-		
+
 		assertFalse(record.hasTimestamp());
 		assertEquals("test", record.getValue());
-		
+
 //		try {
 //			record.getTimestamp();
 //			fail("should throw an exception");
@@ -42,13 +42,13 @@ public void testWithNoTimestamp() {
 //		}
 		// for now, the "no timestamp case" returns Long.MIN_VALUE
 		assertEquals(Long.MIN_VALUE, record.getTimestamp());
-		
+
 		assertNotNull(record.toString());
 		assertTrue(record.hashCode() == new StreamRecord<>("test").hashCode());
 		assertTrue(record.equals(new StreamRecord<>("test")));
-		
+
 		assertEquals(record, record.asRecord());
-		
+
 		try {
 			record.asWatermark();
 			fail("should throw an exception");
@@ -66,11 +66,11 @@ public void testWithTimestamp() {
 
 		assertTrue(record.hasTimestamp());
 		assertEquals(42L, record.getTimestamp());
-		
+
 		assertEquals("foo", record.getValue());
-		
+
 		assertNotNull(record.toString());
-		
+
 		assertTrue(record.hashCode() == new StreamRecord<>("foo", 42).hashCode());
 		assertTrue(record.hashCode() != new StreamRecord<>("foo").hashCode());
 
@@ -95,7 +95,7 @@ public void testAllowedTimestampRange() {
 		assertEquals(Long.MIN_VALUE, new StreamRecord<>("test", Long.MIN_VALUE).getTimestamp());
 		assertEquals(Long.MAX_VALUE, new StreamRecord<>("test", Long.MAX_VALUE).getTimestamp());
 	}
-	
+
 	@Test
 	public void testReplacePreservesTimestamp() {
 		StreamRecord<String> recNoTimestamp = new StreamRecord<>("o sole mio");
@@ -104,7 +104,7 @@ public void testReplacePreservesTimestamp() {
 
 		StreamRecord<String> recWithTimestamp = new StreamRecord<>("la dolce vita", 99);
 		StreamRecord<Integer> newRecWithTimestamp = recWithTimestamp.replace(17);
-		
+
 		assertTrue(newRecWithTimestamp.hasTimestamp());
 		assertEquals(99L, newRecWithTimestamp.getTimestamp());
 	}
@@ -113,12 +113,12 @@ public void testReplacePreservesTimestamp() {
 	public void testReplaceWithTimestampOverridesTimestamp() {
 		StreamRecord<String> record = new StreamRecord<>("la divina comedia");
 		assertFalse(record.hasTimestamp());
-		
+
 		StreamRecord<Double> newRecord = record.replace(3.14, 123);
 		assertTrue(newRecord.hasTimestamp());
 		assertEquals(123L, newRecord.getTimestamp());
 	}
-	
+
 	@Test
 	public void testCopy() {
 		StreamRecord<String> recNoTimestamp = new StreamRecord<String>("test");
@@ -142,16 +142,16 @@ public void testCopyTo() {
 		recWithTimestamp.copyTo("test", recWithTimestampCopy);
 		assertEquals(recWithTimestamp, recWithTimestampCopy);
 	}
-	
+
 	@Test
 	public void testSetAndEraseTimestamps() {
 		StreamRecord<String> rec = new StreamRecord<String>("hello");
 		assertFalse(rec.hasTimestamp());
-		
+
 		rec.setTimestamp(13456L);
 		assertTrue(rec.hasTimestamp());
 		assertEquals(13456L, rec.getTimestamp());
-		
+
 		rec.eraseTimestamp();
 		assertFalse(rec.hasTimestamp());
 	}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java
index a21e58bf92202..19e27f258a5bd 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTaskTestHarness.java
@@ -64,7 +64,7 @@ public OneInputStreamTaskTestHarness(
 		int numInputChannelsPerGate,
 		TypeInformation<IN> inputType,
 		TypeInformation<OUT> outputType) {
-		
+
 		super(task, outputType);
 
 		this.inputType = inputType;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java
index 3bd7adf1316ff..c5fd682e449f0 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskStoppingTest.java
@@ -52,11 +52,11 @@ public void testStopBeforeInitialization() throws Exception {
 
 		final StoppableSourceStreamTask<Object, StoppableFailingSource> sourceTask = new StoppableSourceStreamTask<>();
 		sourceTask.stop();
-		
+
 		sourceTask.headOperator = new StoppableStreamSource<>(new StoppableFailingSource());
 		sourceTask.run();
 	}
-	
+
 	// ------------------------------------------------------------------------
 
 	private static class StoppableSource extends RichSourceFunction<Object> implements StoppableFunction {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskCancellationBarrierTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskCancellationBarrierTest.java
index 53f77ca127902..f8d5393ea19ec 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskCancellationBarrierTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskCancellationBarrierTest.java
@@ -71,7 +71,7 @@ public void testEmitCancellationBarrierWhenNotReady() throws Exception {
 	/**
 	 * This test verifies (for onw input tasks) that the Stream tasks react the following way to
 	 * receiving a checkpoint cancellation barrier:
-	 * 
+	 *
 	 *   - send a "decline checkpoint" notification out (to the JobManager)
 	 *   - emit a cancellation barrier downstream
 	 */
@@ -167,7 +167,7 @@ private static class InitBlockingTask extends StreamTask<String, AbstractStreamO
 
 		private final Object lock = new Object();
 		private volatile boolean running = true;
-		
+
 		@Override
 		protected void init() throws Exception {
 			synchronized (lock) {
@@ -203,7 +203,7 @@ public String map(String value) throws Exception {
 
 	private static class UnionCoMap implements CoMapFunction<String, String, String> {
 		private static final long serialVersionUID = 1L;
-		
+
 		@Override
 		public String map1(String value) throws Exception {
 			return value;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index f34522b9831ac..546188e4c8d09 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -760,7 +760,7 @@ public static Task createTask(
 
 		LibraryCacheManager libCache = mock(LibraryCacheManager.class);
 		when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());
-		
+
 		ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
 		ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
 		PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
@@ -814,16 +814,16 @@ public static Task createTask(
 			partitionProducerStateChecker,
 			executor);
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Test operators
 	// ------------------------------------------------------------------------
-	
+
 	public static class SlowlyDeserializingOperator extends StreamSource<Long, SourceFunction<Long>> {
 		private static final long serialVersionUID = 1L;
 
 		private volatile boolean canceled = false;
-		
+
 		public SlowlyDeserializingOperator() {
 			super(new MockSourceFunction());
 		}
@@ -847,7 +847,7 @@ public void cancel() {
 		// slow deserialization
 		private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
 			in.defaultReadObject();
-			
+
 			long delay = 500;
 			long deadline = System.currentTimeMillis() + delay;
 			do {
@@ -857,7 +857,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
 			} while ((delay = deadline - System.currentTimeMillis()) > 0);
 		}
 	}
-	
+
 	private static class MockSourceFunction implements SourceFunction<Long> {
 		private static final long serialVersionUID = 1L;
 
@@ -888,7 +888,7 @@ public OperatorStateBackend answer(InvocationOnMock invocationOnMock) throws Thr
 							return Mockito.mock(OperatorStateBackend.class);
 						}
 					});
-	
+
 				Mockito.when(stateBackendMock.createKeyedStateBackend(
 						Mockito.any(Environment.class),
 						Mockito.any(JobID.class),
@@ -955,7 +955,7 @@ protected void cancelTask() throws Exception {}
 	}
 
 	/**
-	 * A task that locks if cancellation attempts to cleanly shut down 
+	 * A task that locks if cancellation attempts to cleanly shut down
 	 */
 	public static class CancelLockingTask extends StreamTask<String, AbstractStreamOperator<String>> {
 
@@ -995,11 +995,11 @@ protected void cancelTask() {
 			// do not interrupt the lock holder here, to simulate spawned threads that
 			// we cannot properly interrupt on cancellation
 		}
-		
+
 	}
 
 	/**
-	 * A task that locks if cancellation attempts to cleanly shut down 
+	 * A task that locks if cancellation attempts to cleanly shut down
 	 */
 	public static class CancelFailingTask extends StreamTask<String, AbstractStreamOperator<String>> {
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java
index fb4f0878cb8f7..1f8638ebb37d3 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java
@@ -312,7 +312,7 @@ public void onProcessingTime(long timestamp) throws Exception {
 			latch.await();
 			timer.quiesceAndAwaitPending();
 
-			// should be able to immediately acquire the lock, since the task must have exited by now 
+			// should be able to immediately acquire the lock, since the task must have exited by now
 			assertTrue(scopeLock.tryLock());
 
 			// should be able to schedule more tasks (that never get executed)
@@ -327,7 +327,7 @@ public void onProcessingTime(long timestamp) throws Exception {
 			// nothing should be scheduled right now
 			assertEquals(0, timer.getNumTasksScheduled());
 
-			// check that no asynchronous error was reported - that ensures that the newly scheduled 
+			// check that no asynchronous error was reported - that ensures that the newly scheduled
 			// triggerable did, in fact, not trigger
 			if (errorRef.get() != null) {
 				throw new Exception(errorRef.get());
@@ -397,7 +397,7 @@ public void handleAsyncException(String message, Throwable exception) {
 						latch.trigger();
 					}
 				}, lock);
-		
+
 		timeServiceProvider.registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() {
 			@Override
 			public void onProcessingTime(long timestamp) throws Exception {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java
index 220c1cd23391b..1ae144cfc73eb 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java
@@ -41,7 +41,7 @@ public void testTypeExtractionTuple() {
 		TypeInformation<Tuple2<byte[], byte[]>> expected = TypeInformation.of(new TypeHint<Tuple2<byte[], byte[]>>(){});
 		assertEquals(expected, type);
 	}
-	
+
 	@Test
 	public void testTypeExtractionTupleAnonymous() {
 		TypeInformation<Tuple2<byte[], byte[]>> type = new AbstractDeserializationSchema<Tuple2<byte[], byte[]>>() {
@@ -50,7 +50,7 @@ public Tuple2<byte[], byte[]> deserialize(byte[] message) throws IOException {
 				throw new UnsupportedOperationException();
 			}
 		}.getProducedType();
-		
+
 		TypeInformation<Tuple2<byte[], byte[]>> expected = TypeInformation.of(new TypeHint<Tuple2<byte[], byte[]>>(){});
 		assertEquals(expected, type);
 	}
@@ -84,7 +84,7 @@ public void testTypeExtractionRawException() {
 			// expected
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Test types
 	// ------------------------------------------------------------------------
@@ -96,7 +96,7 @@ public Tuple2<byte[], byte[]> deserialize(byte[] message) throws IOException {
 			throw new UnsupportedOperationException();
 		}
 	}
-	
+
 	private static class JsonSchema extends AbstractDeserializationSchema<JSONPObject> {
 
 		@Override
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectingSourceContext.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectingSourceContext.java
index d9ad24dcf4130..fa68082b89091 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectingSourceContext.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectingSourceContext.java
@@ -27,7 +27,7 @@
 import java.util.Collection;
 
 public class CollectingSourceContext<T extends Serializable> implements SourceFunction.SourceContext<T> {
-	
+
 	private final Object lock;
 	private final Collection<T> collection;
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java
index c2763d86d5a67..5d730157e16d2 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockContext.java
@@ -29,7 +29,7 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
 public class MockContext<IN, OUT> {
-	
+
 	private List<OUT> outputs;
 
 	private MockOutput<OUT> output;
@@ -54,7 +54,7 @@ public Output<StreamRecord<OUT>> getOutput() {
 	public static <IN, OUT> List<OUT> createAndExecute(OneInputStreamOperator<IN, OUT> operator, List<IN> inputs) throws Exception {
 		return createAndExecuteForKeyedStream(operator, inputs, null, null);
 	}
-	
+
 	public static <IN, OUT, KEY> List<OUT> createAndExecuteForKeyedStream(
 				OneInputStreamOperator<IN, OUT> operator, List<IN> inputs,
 				KeySelector<IN, KEY> keySelector, TypeInformation<KEY> keyType) throws Exception {
@@ -64,7 +64,7 @@ public static <IN, OUT, KEY> List<OUT> createAndExecuteForKeyedStream(
 
 		testHarness.setup();
 		testHarness.open();
-		
+
 		operator.open();
 
 		for (IN in: inputs) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SourceFunctionUtil.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SourceFunctionUtil.java
index ea036753f50bc..36ade8c38bfb6 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SourceFunctionUtil.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SourceFunctionUtil.java
@@ -39,17 +39,17 @@ public class SourceFunctionUtil {
 
 	public static <T extends Serializable> List<T> runSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
 		final List<T> outputs = new ArrayList<T>();
-		
+
 		if (sourceFunction instanceof RichFunction) {
 
 			AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
 			when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());
-			
+
 			RuntimeContext runtimeContext =  new StreamingRuntimeContext(
 					operator,
 					new MockEnvironment("MockTask", 3 * 1024 * 1024, new MockInputSplitProvider(), 1024),
 					new HashMap<String, Accumulator<?, ?>>());
-			
+
 			((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
 
 			((RichFunction) sourceFunction).open(new Configuration());
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java
index 7d791c0252930..5fbe371cb7f50 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java
@@ -37,7 +37,7 @@ public class TwoInputStreamOperatorTestHarness<IN1, IN2, OUT>extends AbstractStr
 	public TwoInputStreamOperatorTestHarness(TwoInputStreamOperator<IN1, IN2, OUT> operator) throws Exception {
 		this(operator, 1, 1, 0);
 	}
-		
+
 	public TwoInputStreamOperatorTestHarness(
 			TwoInputStreamOperator<IN1, IN2, OUT> operator,
 			int maxParallelism,
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TypeInformationSerializationSchemaTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TypeInformationSerializationSchemaTest.java
index e722f5363f0d0..a14d11352e473 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TypeInformationSerializationSchemaTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TypeInformationSerializationSchemaTest.java
@@ -34,22 +34,22 @@
 import static org.junit.Assert.*;
 
 public class TypeInformationSerializationSchemaTest {
-	
+
 	@Test
 	public void testDeSerialization() {
 		try {
 			TypeInformation<MyPOJO> info = TypeExtractor.getForClass(MyPOJO.class);
-			
+
 			TypeInformationSerializationSchema<MyPOJO> schema =
 					new TypeInformationSerializationSchema<MyPOJO>(info, new ExecutionConfig());
-			
+
 			MyPOJO[] types = {
 					new MyPOJO(72, new Date(763784523L), new Date(88234L)),
 					new MyPOJO(-1, new Date(11111111111111L)),
 					new MyPOJO(42),
 					new MyPOJO(17, new Date(222763784523L))
 			};
-			
+
 			for (MyPOJO val : types) {
 				byte[] serialized = schema.serialize(val);
 				MyPOJO deser = schema.deserialize(serialized);
@@ -61,7 +61,7 @@ public void testDeSerialization() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	@Test
 	public void testSerializability() {
 		try {
@@ -77,16 +77,16 @@ public void testSerializability() {
 			fail(e.getMessage());
 		}
 	}
-	
+
 	// ------------------------------------------------------------------------
 	//  Test data types
 	// ------------------------------------------------------------------------
-	
+
 	public static class MyPOJO {
-		
+
 		public int aField;
 		public List<Date> aList;
-		
+
 		public MyPOJO() {}
 
 		public MyPOJO(int iVal, Date... dates) {
@@ -103,7 +103,7 @@ public int hashCode() {
 		public boolean equals(Object obj) {
 			if (obj instanceof MyPOJO) {
 				MyPOJO that = (MyPOJO) obj;
-				return this.aField == that.aField && (this.aList == null ? 
+				return this.aField == that.aField && (this.aList == null ?
 						that.aList == null :
 						that.aList != null && this.aList.equals(that.aList));
 			}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/keys/ArrayKeySelectorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/keys/ArrayKeySelectorTest.java
index 63375a7fba594..5e363e9c53242 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/keys/ArrayKeySelectorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/keys/ArrayKeySelectorTest.java
@@ -33,19 +33,19 @@ public void testObjectArrays() {
 		try {
 			String[] array1 = { "a", "b", "c", "d", "e" };
 			String[] array2 = { "v", "w", "x", "y", "z" };
-			
+
 			KeySelectorUtil.ArrayKeySelector<String[]> singleFieldSelector =
 					KeySelectorUtil.getSelectorForArray(new int[] {1}, BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO);
-			
+
 			assertEquals(new Tuple1<>("b"), singleFieldSelector.getKey(array1));
 			assertEquals(new Tuple1<>("w"), singleFieldSelector.getKey(array2));
 
 			KeySelectorUtil.ArrayKeySelector<String[]> twoFieldsSelector =
 					KeySelectorUtil.getSelectorForArray(new int[] {3, 0}, BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO);
-			
+
 			assertEquals(new Tuple2<>("d", "a"), twoFieldsSelector.getKey(array1));
 			assertEquals(new Tuple2<>("y", "v"), twoFieldsSelector.getKey(array2));
-			
+
 		}
 		catch (Exception e) {
 			e.printStackTrace();
@@ -59,7 +59,7 @@ public void testPrimitiveArrays() {
 			int[] array1 = { 1, 2, 3, 4, 5 };
 			int[] array2 = { -5, -4, -3, -2, -1, 0 };
 
-			KeySelectorUtil.ArrayKeySelector<int[]> singleFieldSelector = 
+			KeySelectorUtil.ArrayKeySelector<int[]> singleFieldSelector =
 					KeySelectorUtil.getSelectorForArray(new int[] {1}, PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO);
 
 			assertEquals(new Tuple1<>(2), singleFieldSelector.getKey(array1));
@@ -67,7 +67,7 @@ public void testPrimitiveArrays() {
 
 			KeySelectorUtil.ArrayKeySelector<int[]> twoFieldsSelector =
 					KeySelectorUtil.getSelectorForArray(new int[] {3, 0}, PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO);
-			
+
 			assertEquals(new Tuple2<>(4, 1), twoFieldsSelector.getKey(array1));
 			assertEquals(new Tuple2<>(-2, -5), twoFieldsSelector.getKey(array2));
 
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index 7684a24e40262..3285e5cb5cb90 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -47,11 +47,11 @@ This file is based on the checkstyle file of Apache Beam.
     <property name="severity" value="error" />
   </module>
 
-  <!--<module name="RegexpSingleline">-->
-    <!--<property name="format" value="\s+$"/>-->
-    <!--<property name="message" value="Trailing whitespace"/>-->
-    <!--<property name="severity" value="error"/>-->
-  <!--</module>-->
+  <module name="RegexpSingleline">
+    <property name="format" value="\s+$"/>
+    <property name="message" value="Trailing whitespace"/>
+    <property name="severity" value="error"/>
+  </module>
 
   <module name="RegexpSingleline">
     <property name="format" value="Throwables.propagate\("/>

From 43ef1152f35edafe7167e1bd680e1e28d266266b Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 12:23:22 +0100
Subject: [PATCH 050/253] [FLINK-6107] Enable import order check in streaming
 checkstyle

---
 .../api/graph/StreamGraphHasherV1.java        | 25 +++++++++----------
 .../MultiplexingStreamRecordSerializer.java   |  7 +++---
 .../streamrecord/StreamRecordSerializer.java  |  3 +--
 .../api/checkpoint/Checkpointed.java          |  3 +--
 .../CheckpointedAsynchronously.java           |  3 +--
 .../api/checkpoint/CheckpointedRestoring.java |  3 +--
 .../api/checkpoint/ListCheckpointed.java      |  5 ++--
 .../selector/CopyingDirectedOutput.java       |  7 +++---
 .../api/datastream/AllWindowedStream.java     | 10 ++++----
 .../api/datastream/AsyncDataStream.java       |  3 +--
 .../api/datastream/CoGroupedStreams.java      | 13 +++++-----
 .../api/datastream/ConnectedStreams.java      |  6 ++---
 .../streaming/api/datastream/DataStream.java  | 17 ++++++-------
 .../api/datastream/DataStreamSink.java        |  2 +-
 .../api/datastream/IterativeStream.java       |  5 ++--
 .../api/datastream/JoinedStreams.java         |  6 ++---
 .../streaming/api/datastream/KeyedStream.java | 11 ++++----
 .../SingleOutputStreamOperator.java           | 11 ++++----
 .../api/datastream/WindowedStream.java        | 10 ++++----
 .../api/environment/CheckpointConfig.java     |  6 ++---
 .../StreamExecutionEnvironment.java           | 19 +++++++-------
 .../AssignerWithPeriodicWatermarks.java       |  3 +--
 .../AssignerWithPunctuatedWatermarks.java     |  3 +--
 .../functions/aggregation/SumAggregator.java  |  2 +-
 .../functions/aggregation/SumFunction.java    |  3 +--
 .../api/functions/async/AsyncFunction.java    |  3 +--
 .../functions/async/RichAsyncFunction.java    |  7 +++---
 .../async/collector/AsyncCollector.java       |  3 +--
 .../api/functions/sink/SocketClientSink.java  |  8 +++---
 .../api/functions/sink/WriteFormat.java       |  3 +--
 .../api/functions/sink/WriteFormatAsCsv.java  |  3 +--
 .../api/functions/sink/WriteFormatAsText.java |  3 +--
 .../api/functions/sink/WriteSinkFunction.java |  3 +--
 .../ContinuousFileMonitoringFunction.java     | 17 ++++++-------
 .../source/ContinuousFileReaderOperator.java  | 23 ++++++++---------
 .../source/FileMonitoringFunction.java        | 13 +++++-----
 .../source/FromElementsFunction.java          | 15 ++++++-----
 .../source/FromIteratorFunction.java          |  3 +--
 .../FromSplittableIteratorFunction.java       |  3 +--
 .../source/InputFormatSourceFunction.java     |  5 ++--
 ...ipleIdsMessageAcknowledgingSourceBase.java | 11 ++++----
 .../source/SocketTextStreamFunction.java      | 14 +++++------
 .../api/functions/source/SourceFunction.java  |  5 ++--
 .../source/StatefulSequenceSource.java        |  5 ++--
 .../source/TimestampedFileInputSplit.java     |  3 +--
 .../AscendingTimestampExtractor.java          |  5 ++--
 .../AggregateApplyAllWindowFunction.java      |  3 +--
 .../AggregateApplyWindowFunction.java         |  3 +--
 .../windowing/AllWindowFunction.java          |  3 +--
 .../windowing/FoldApplyAllWindowFunction.java |  9 +++----
 .../FoldApplyProcessAllWindowFunction.java    |  9 +++----
 .../FoldApplyProcessWindowFunction.java       |  9 +++----
 .../windowing/FoldApplyWindowFunction.java    |  9 +++----
 .../ReduceApplyAllWindowFunction.java         |  3 +--
 .../ReduceApplyProcessAllWindowFunction.java  |  3 +--
 .../ReduceApplyProcessWindowFunction.java     |  3 +--
 .../windowing/ReduceApplyWindowFunction.java  |  3 +--
 .../functions/windowing/WindowFunction.java   |  3 +--
 .../windowing/delta/DeltaFunction.java        |  3 +--
 .../windowing/delta/extractor/Extractor.java  |  3 +--
 .../delta/extractor/FieldFromArray.java       |  3 +--
 .../delta/extractor/FieldsFromArray.java      |  3 +--
 .../streaming/api/graph/StreamConfig.java     |  3 +--
 .../flink/streaming/api/graph/StreamEdge.java |  3 +--
 .../streaming/api/graph/StreamGraph.java      |  3 +--
 .../api/graph/StreamGraphGenerator.java       | 13 +++++-----
 .../api/graph/StreamGraphHasher.java          |  3 +--
 .../api/graph/StreamGraphHasherV2.java        | 21 ++++++++--------
 .../api/graph/StreamGraphUserHashHasher.java  |  3 +--
 .../flink/streaming/api/graph/StreamNode.java |  7 +++---
 .../api/graph/StreamingJobGraphGenerator.java | 19 +++++++-------
 .../api/operators/AbstractStreamOperator.java | 17 ++++++-------
 .../operators/AbstractUdfStreamOperator.java  |  9 +++----
 .../operators/HeapInternalTimerService.java   | 17 ++++++-------
 .../operators/InternalTimeServiceManager.java |  7 +++---
 .../api/operators/InternalTimer.java          |  3 +--
 .../InternalWatermarkCallbackService.java     | 13 +++++-----
 .../api/operators/KeyedProcessOperator.java   |  6 ++---
 .../api/operators/OnWatermarkCallback.java    |  3 +--
 .../api/operators/OperatorSnapshotResult.java |  3 +--
 .../api/operators/ProcessOperator.java        |  4 +--
 .../api/operators/StreamOperator.java         |  3 +--
 .../streaming/api/operators/StreamSource.java |  3 +--
 .../api/operators/StreamSourceContexts.java   |  5 ++--
 .../operators/StreamingRuntimeContext.java    |  5 ++--
 .../operators/async/AsyncWaitOperator.java    | 17 ++++++-------
 .../api/operators/async/Emitter.java          |  5 ++--
 .../async/queue/AsyncCollectionResult.java    |  3 +--
 .../queue/OrderedStreamElementQueue.java      | 13 +++++-----
 .../async/queue/StreamElementQueue.java       |  3 +--
 .../async/queue/StreamElementQueueEntry.java  |  3 +--
 .../async/queue/StreamRecordQueueEntry.java   |  3 +--
 .../queue/UnorderedStreamElementQueue.java    | 13 +++++-----
 .../api/operators/co/CoProcessOperator.java   |  6 ++---
 .../api/operators/co/CoStreamFlatMap.java     |  2 +-
 .../operators/co/KeyedCoProcessOperator.java  |  6 ++---
 .../CoFeedbackTransformation.java             |  7 +++---
 .../FeedbackTransformation.java               |  5 ++--
 .../OneInputTransformation.java               |  5 ++--
 .../PartitionTransformation.java              |  6 ++---
 .../transformations/SelectTransformation.java |  5 ++--
 .../SideOutputTransformation.java             |  5 ++--
 .../transformations/SinkTransformation.java   |  5 ++--
 .../transformations/SourceTransformation.java |  5 ++--
 .../transformations/SplitTransformation.java  |  5 ++--
 .../transformations/StreamTransformation.java |  7 +++---
 .../TwoInputTransformation.java               |  5 ++--
 .../transformations/UnionTransformation.java  |  5 ++--
 .../assigners/BaseAlignedWindowAssigner.java  |  3 +--
 .../assigners/EventTimeSessionWindows.java    |  5 ++--
 .../windowing/assigners/GlobalWindows.java    |  5 ++--
 .../assigners/MergingWindowAssigner.java      |  3 +--
 .../ProcessingTimeSessionWindows.java         |  5 ++--
 .../assigners/SlidingEventTimeWindows.java    |  9 +++----
 .../SlidingProcessingTimeWindows.java         |  7 +++---
 .../assigners/TumblingEventTimeWindows.java   |  7 +++---
 .../TumblingProcessingTimeWindows.java        |  5 ++--
 .../windowing/assigners/WindowAssigner.java   |  6 ++---
 .../api/windowing/evictors/CountEvictor.java  |  3 +--
 .../api/windowing/evictors/DeltaEvictor.java  |  3 +--
 .../api/windowing/evictors/Evictor.java       |  3 +--
 .../api/windowing/evictors/TimeEvictor.java   |  3 +--
 .../streaming/api/windowing/time/Time.java    |  3 ++-
 .../api/windowing/triggers/Trigger.java       |  3 +--
 .../api/windowing/windows/GlobalWindow.java   |  3 +--
 .../api/windowing/windows/TimeWindow.java     | 15 ++++++-----
 .../streaming/runtime/io/BarrierBuffer.java   | 12 ++++-----
 .../streaming/runtime/io/BarrierTracker.java  |  6 ++---
 .../runtime/io/CheckpointBarrierHandler.java  |  3 +--
 .../runtime/io/RecordWriterOutput.java        |  9 +++----
 .../runtime/io/StreamInputProcessor.java      | 17 ++++++-------
 .../runtime/io/StreamRecordWriter.java        |  5 ++--
 .../runtime/io/StreamTwoInputProcessor.java   | 19 +++++++-------
 .../operators/CheckpointCommitter.java        |  3 +--
 .../operators/GenericWriteAheadSink.java      | 13 +++++-----
 ...ctAlignedProcessingTimeWindowOperator.java | 11 ++++----
 .../windowing/AbstractKeyedTimePanes.java     |  7 +++---
 .../windowing/AccumulatingKeyedTimePanes.java |  7 +++---
 ...umulatingProcessingTimeWindowOperator.java |  3 +--
 .../windowing/EvictingWindowOperator.java     | 10 +++-----
 .../runtime/operators/windowing/KeyMap.java   |  7 +++---
 .../operators/windowing/MergingWindowSet.java | 11 ++++----
 .../operators/windowing/WindowOperator.java   | 24 ++++++++----------
 ...rnalAggregateProcessAllWindowFunction.java |  3 +--
 ...nternalAggregateProcessWindowFunction.java |  3 +--
 .../InternalSingleValueAllWindowFunction.java |  3 +--
 ...alSingleValueProcessAllWindowFunction.java |  3 +--
 ...ernalSingleValueProcessWindowFunction.java |  3 +--
 .../InternalSingleValueWindowFunction.java    |  3 +--
 .../streamrecord/StreamElementSerializer.java |  7 +++---
 .../streamstatus/StatusWatermarkValve.java    |  6 ++---
 .../ExceptionInChainedOperatorException.java  |  4 +--
 .../runtime/tasks/OperatorChain.java          | 21 ++++++++--------
 .../runtime/tasks/OperatorStateHandles.java   |  5 ++--
 .../runtime/tasks/StreamIterationHead.java    |  3 +--
 .../streaming/runtime/tasks/StreamTask.java   | 23 ++++++++---------
 .../tasks/SystemProcessingTimeService.java    |  9 +++----
 .../tasks/TestProcessingTimeService.java      |  5 ++--
 .../streaming/util/HDFSCopyFromLocal.java     |  5 ++--
 .../flink/streaming/util/HDFSCopyToLocal.java |  5 ++--
 .../functions/StreamingFunctionUtils.java     |  7 +++---
 .../streaming/util/keys/KeySelectorUtil.java  |  7 +++---
 .../AbstractDeserializationSchema.java        |  3 +--
 .../serialization/SerializationSchema.java    |  3 +--
 .../serialization/SimpleStringSchema.java     |  9 +++----
 .../TypeInformationSerializationSchema.java   |  3 +--
 .../util/typeutils/FieldAccessor.java         | 17 ++++++-------
 .../util/typeutils/FieldAccessorFactory.java  |  7 +++---
 tools/maven/strict-checkstyle.xml             | 20 +++++++--------
 169 files changed, 505 insertions(+), 665 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java
index a80ff7d8825fe..2fbfe1ca3b906 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java
@@ -18,20 +18,11 @@
 
 package org.apache.flink.migration.streaming.api.graph;
 
+import static org.apache.flink.util.StringUtils.byteToHexString;
+
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
-import org.apache.flink.streaming.api.graph.StreamEdge;
-import org.apache.flink.streaming.api.graph.StreamGraph;
-import org.apache.flink.streaming.api.graph.StreamGraphHasher;
-import org.apache.flink.streaming.api.graph.StreamNode;
-import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-import org.apache.flink.streaming.api.operators.StreamOperator;
-import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.nio.charset.Charset;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -43,8 +34,16 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
-
-import static org.apache.flink.util.StringUtils.byteToHexString;
+import org.apache.flink.streaming.api.graph.StreamEdge;
+import org.apache.flink.streaming.api.graph.StreamGraph;
+import org.apache.flink.streaming.api.graph.StreamGraphHasher;
+import org.apache.flink.streaming.api.graph.StreamNode;
+import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * StreamGraphHasher from Flink 1.1. This contains duplicated code to ensure that the algorithm does not change with
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
index db9977b5cdd84..fdcd5b89ae18c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.migration.streaming.runtime.streamrecord;
 
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
@@ -25,10 +28,6 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
-import java.io.IOException;
-
-import static java.util.Objects.requireNonNull;
-
 /**
  * Legacy multiplexing {@link TypeSerializer} for stream records, watermarks and other stream
  * elements.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
index 0e9e8092960b4..2c8dc4a328358 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.migration.streaming.runtime.streamrecord;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputView;
@@ -25,8 +26,6 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.util.Preconditions;
 
-import java.io.IOException;
-
 /**
  * Serializer for {@link StreamRecord}. This version ignores timestamps and only deals with
  * the element.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
index 7dedfbf8b01f9..6de7329c2cc7f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.streaming.api.checkpoint;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.Serializable;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * This method must be implemented by functions that have state that needs to be
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
index 2230f5962e720..b96c2421782a4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.streaming.api.checkpoint;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.Serializable;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * This interface marks a function/operator as checkpointed similar to the
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
index cfaa505f0847b..f695418069bc9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.streaming.api.checkpoint;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.Serializable;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * This deprecated interface contains the methods for restoring from the legacy checkpointing mechanism of state.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
index 0b8b1b6af2e82..49ce3e315a906 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ListCheckpointed.java
@@ -18,14 +18,13 @@
 
 package org.apache.flink.streaming.api.checkpoint;
 
+import java.io.Serializable;
+import java.util.List;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.RichFunction;
 import org.apache.flink.api.common.state.OperatorStateStore;
 import org.apache.flink.configuration.Configuration;
 
-import java.io.Serializable;
-import java.util.List;
-
 /**
  * This interface can be implemented by functions that want to store state in checkpoints.
  * It can be used in a similar way as the deprecated {@link Checkpointed} interface, but
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/CopyingDirectedOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/CopyingDirectedOutput.java
index 5f7e2787e3b8d..f74d652320704 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/CopyingDirectedOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/CopyingDirectedOutput.java
@@ -17,15 +17,14 @@
 
 package org.apache.flink.streaming.api.collector.selector;
 
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.api.graph.StreamEdge;
 import org.apache.flink.streaming.api.operators.Output;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
 
 /**
  * Special version of {@link DirectedOutput} that performs a shallow copy of the
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
index 5aabf28805fd6..24003301b33be 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
@@ -18,8 +18,11 @@
 
 package org.apache.flink.streaming.api.datastream;
 
-import org.apache.flink.annotation.PublicEvolving;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.AggregateFunction;
 import org.apache.flink.api.common.functions.FoldFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
@@ -40,9 +43,9 @@
 import org.apache.flink.streaming.api.functions.aggregation.SumAggregator;
 import org.apache.flink.streaming.api.functions.windowing.AggregateApplyAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction;
+import org.apache.flink.streaming.api.functions.windowing.FoldApplyAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.FoldApplyProcessAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.PassThroughAllWindowFunction;
-import org.apache.flink.streaming.api.functions.windowing.FoldApplyAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.ReduceApplyAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.ReduceApplyProcessAllWindowFunction;
@@ -66,9 +69,6 @@
 import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.Preconditions;
 
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * A {@code AllWindowedStream} represents a data stream where the stream of
  * elements is split into windows based on a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java
index 8461d2c859b01..759e32ab0ea91 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AsyncDataStream.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import java.util.concurrent.TimeUnit;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.Utils;
@@ -24,8 +25,6 @@
 import org.apache.flink.streaming.api.functions.async.AsyncFunction;
 import org.apache.flink.streaming.api.operators.async.AsyncWaitOperator;
 
-import java.util.concurrent.TimeUnit;
-
 /**
  * A helper class to apply {@link AsyncFunction} to a data stream.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
index 98c57f1917a22..13a8a248d7d7e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
@@ -17,9 +17,14 @@
 
 package org.apache.flink.streaming.api.datastream;
 
-import org.apache.flink.annotation.PublicEvolving;
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.CoGroupFunction;
 import org.apache.flink.api.common.functions.MapFunction;
@@ -37,12 +42,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static java.util.Objects.requireNonNull;
-
 /**
  *{@code CoGroupedStreams} represents two {@link DataStream DataStreams} that have been co-grouped.
  * A streaming co-group operation is evaluated over elements in a window.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
index b45881a13661a..0a2b4e1196dc3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedStreams.java
@@ -17,9 +17,11 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import static java.util.Objects.requireNonNull;
+
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.Utils;
 import org.apache.flink.api.java.functions.KeySelector;
@@ -35,8 +37,6 @@
 import org.apache.flink.streaming.api.operators.co.KeyedCoProcessOperator;
 import org.apache.flink.streaming.api.transformations.TwoInputTransformation;
 
-import static java.util.Objects.requireNonNull;
-
 /**
  * ConnectedStreams represent two connected streams of (possibly) different data types.
  * Connected streams are useful for cases where operations on one stream directly
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index e73e4e7652df8..c531987b3c883 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -19,10 +19,9 @@
 
 import java.util.ArrayList;
 import java.util.List;
-
-import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.FilterFunction;
 import org.apache.flink.api.common.functions.FlatMapFunction;
@@ -32,6 +31,7 @@
 import org.apache.flink.api.common.functions.RichFlatMapFunction;
 import org.apache.flink.api.common.functions.RichMapFunction;
 import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.api.common.operators.Keys;
 import org.apache.flink.api.common.operators.ResourceSpec;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
@@ -40,7 +40,6 @@
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.io.CsvOutputFormat;
 import org.apache.flink.api.java.io.TextOutputFormat;
-import org.apache.flink.api.common.operators.Keys;
 import org.apache.flink.api.java.tuple.Tuple;
 import org.apache.flink.api.java.typeutils.InputTypeConfigurable;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -52,13 +51,13 @@
 import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
 import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
 import org.apache.flink.streaming.api.functions.ProcessFunction;
-import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor;
-import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
 import org.apache.flink.streaming.api.functions.TimestampExtractor;
 import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction;
 import org.apache.flink.streaming.api.functions.sink.PrintSinkFunction;
 import org.apache.flink.streaming.api.functions.sink.SinkFunction;
 import org.apache.flink.streaming.api.functions.sink.SocketClientSink;
+import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor;
+import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.ProcessOperator;
 import org.apache.flink.streaming.api.operators.StreamFilter;
@@ -70,10 +69,10 @@
 import org.apache.flink.streaming.api.transformations.StreamTransformation;
 import org.apache.flink.streaming.api.transformations.UnionTransformation;
 import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows;
-import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeWindows;
 import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows;
-import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
+import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeWindows;
 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
 import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;
 import org.apache.flink.streaming.api.windowing.evictors.CountEvictor;
 import org.apache.flink.streaming.api.windowing.time.Time;
@@ -88,9 +87,9 @@
 import org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner;
 import org.apache.flink.streaming.runtime.partitioner.CustomPartitionerWrapper;
 import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
-import org.apache.flink.streaming.runtime.partitioner.RescalePartitioner;
-import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner;
 import org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner;
+import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner;
+import org.apache.flink.streaming.runtime.partitioner.RescalePartitioner;
 import org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner;
 import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
 import org.apache.flink.streaming.util.keys.KeySelectorUtil;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index d098c986e79ab..074d481da28a9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -17,9 +17,9 @@
 
 package org.apache.flink.streaming.api.datastream;
 
-import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.operators.ResourceSpec;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.StreamSink;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
index ed87b1b51f0ad..75699ae13a39c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
@@ -17,8 +17,9 @@
 
 package org.apache.flink.streaming.api.datastream;
 
-import org.apache.flink.annotation.PublicEvolving;
+import java.util.Collection;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -27,8 +28,6 @@
 import org.apache.flink.streaming.api.transformations.FeedbackTransformation;
 import org.apache.flink.streaming.api.transformations.StreamTransformation;
 
-import java.util.Collection;
-
 /**
  * The iterative data stream represents the start of an iteration in a {@link DataStream}.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
index d02a2e2dad7fb..77a252e82c3d4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/JoinedStreams.java
@@ -17,8 +17,10 @@
 
 package org.apache.flink.streaming.api.datastream;
 
-import org.apache.flink.annotation.PublicEvolving;
+import static java.util.Objects.requireNonNull;
+
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.CoGroupFunction;
 import org.apache.flink.api.common.functions.FlatJoinFunction;
 import org.apache.flink.api.common.functions.JoinFunction;
@@ -33,8 +35,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import static java.util.Objects.requireNonNull;
-
 /**
  *{@code JoinedStreams} represents two {@link DataStream DataStreams} that have been joined.
  * A streaming join operation is evaluated over elements in a window.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
index 6f8fd07181282..c042f2fdb64c7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
@@ -17,6 +17,10 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Stack;
+import java.util.UUID;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.Public;
@@ -46,8 +50,8 @@
 import org.apache.flink.streaming.api.functions.query.QueryableValueStateOperator;
 import org.apache.flink.streaming.api.functions.sink.SinkFunction;
 import org.apache.flink.streaming.api.graph.StreamGraphGenerator;
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.KeyedProcessOperator;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.StreamGroupedFold;
 import org.apache.flink.streaming.api.operators.StreamGroupedReduce;
 import org.apache.flink.streaming.api.transformations.OneInputTransformation;
@@ -68,11 +72,6 @@
 import org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner;
 import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Stack;
-import java.util.UUID;
-
 /**
  * A {@code KeyedStream} represents a {@link DataStream} on which operator state is
  * partitioned by key using a provided {@link KeySelector}. Typical operations supported by a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index d540a9fcec0fa..d033e98a6c7eb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -17,10 +17,13 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import static java.util.Objects.requireNonNull;
+
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.flink.annotation.Public;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.InvalidTypesException;
-import org.apache.flink.util.OutputTag;
 import org.apache.flink.api.common.operators.ResourceSpec;
 import org.apache.flink.api.common.typeinfo.TypeHint;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -31,13 +34,9 @@
 import org.apache.flink.streaming.api.transformations.SideOutputTransformation;
 import org.apache.flink.streaming.api.transformations.StreamTransformation;
 import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
+import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.Preconditions;
 
-import java.util.HashMap;
-import java.util.Map;
-
-import static java.util.Objects.requireNonNull;
-
 /**
  * {@code SingleOutputStreamOperator} represents a user defined transformation
  * applied on a {@link DataStream} with one predefined output type.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
index 6104b35df5b07..688e9b6df4cc0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
@@ -18,9 +18,12 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.AggregateFunction;
 import org.apache.flink.api.common.functions.FoldFunction;
 import org.apache.flink.api.common.functions.Function;
@@ -64,21 +67,18 @@
 import org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator;
 import org.apache.flink.streaming.runtime.operators.windowing.AggregatingProcessingTimeWindowOperator;
 import org.apache.flink.streaming.runtime.operators.windowing.EvictingWindowOperator;
+import org.apache.flink.streaming.runtime.operators.windowing.WindowOperator;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessWindowFunction;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessWindowFunction;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableWindowFunction;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueProcessWindowFunction;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueWindowFunction;
-import org.apache.flink.streaming.runtime.operators.windowing.WindowOperator;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalWindowFunction;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.Preconditions;
 
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * A {@code WindowedStream} represents a data stream where elements are grouped by
  * key, and for each key, the stream of elements is split into windows based on a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
index da65147c00b91..8d8abfb12c0c7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
@@ -18,15 +18,15 @@
 
 package org.apache.flink.streaming.api.environment;
 
+import static java.util.Objects.requireNonNull;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 import org.apache.flink.annotation.Public;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.streaming.api.CheckpointingMode;
 
-import static java.util.Objects.requireNonNull;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * Configuration that captures all checkpointing related settings.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 3f73eb231183a..7a6c7d6e94b20 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -17,7 +17,16 @@
 
 package org.apache.flink.streaming.api.environment;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 import com.esotericsoftware.kryo.Serializer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.Public;
 import org.apache.flink.annotation.PublicEvolving;
@@ -76,16 +85,6 @@
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.SplittableIterator;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * The StreamExecutionEnvironment is the context in which a streaming program is executed. A
  * {@link LocalStreamEnvironment} will cause execution in the current JVM, a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java
index 823a5c3442be0..462198857d66e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.java
@@ -18,11 +18,10 @@
 
 package org.apache.flink.streaming.api.functions;
 
+import javax.annotation.Nullable;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.streaming.api.watermark.Watermark;
 
-import javax.annotation.Nullable;
-
 /**
  * The {@code AssignerWithPeriodicWatermarks} assigns event time timestamps to elements,
  * and generates low watermarks that signal event time progress within the stream.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java
index a69a8394965e5..2040e77c9433f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.streaming.api.functions;
 
-import org.apache.flink.streaming.api.watermark.Watermark;
-
 import javax.annotation.Nullable;
+import org.apache.flink.streaming.api.watermark.Watermark;
 
 /**
  * The {@code AssignerWithPunctuatedWatermarks} assigns event time timestamps to elements,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
index 58b5981fc8967..86d91d0ce92e1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
@@ -22,9 +22,9 @@
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.streaming.util.typeutils.FieldAccessorFactory;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.streaming.util.typeutils.FieldAccessor;
+import org.apache.flink.streaming.util.typeutils.FieldAccessorFactory;
 
 /**
  * An {@link AggregationFunction} that sums up fields.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java
index a177c580c8159..9bbac3d4c078e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumFunction.java
@@ -17,9 +17,8 @@
 
 package org.apache.flink.streaming.api.functions.aggregation;
 
-import org.apache.flink.annotation.Internal;
-
 import java.io.Serializable;
+import org.apache.flink.annotation.Internal;
 
 /**
  * Internal function for summing up contents of fields. This is used with {@link SumAggregator}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java
index 5bb4459ac11e8..c0cde30eb3b22 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/AsyncFunction.java
@@ -18,12 +18,11 @@
 
 package org.apache.flink.streaming.api.functions.async;
 
+import java.io.Serializable;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.streaming.api.functions.async.collector.AsyncCollector;
 
-import java.io.Serializable;
-
 /**
  * A function to trigger Async I/O operation.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java
index 552f009b72b55..3885b1c7fdb9e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/RichAsyncFunction.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.streaming.api.functions.async;
 
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.accumulators.Accumulator;
@@ -47,10 +50,6 @@
 import org.apache.flink.types.Value;
 import org.apache.flink.util.Preconditions;
 
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
 /**
  * Rich variant of the {@link AsyncFunction}. As a {@link RichFunction}, it gives access to the
  * {@link RuntimeContext} and provides setup and teardown methods:
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java
index 964c13ab4e8b2..9c07d1affe1ed 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/async/collector/AsyncCollector.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.streaming.api.functions.async.collector;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.util.Collection;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * {@link AsyncCollector} collects data / error in user codes while processing async i/o.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
index 21c35a318ec28..44a96773f0e6b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
@@ -17,21 +17,19 @@
 
 package org.apache.flink.streaming.api.functions.sink;
 
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.Socket;
-
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.util.SerializableObject;
 import org.apache.flink.streaming.util.serialization.SerializationSchema;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * Socket client that acts as a streaming sink. The data is sent to a Socket as a byte array.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java
index 65915af19dae8..270b7b5f910fc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormat.java
@@ -17,10 +17,9 @@
 
 package org.apache.flink.streaming.api.functions.sink;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.Serializable;
 import java.util.ArrayList;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * Abstract class for formatting the output of the writeAsText and writeAsCsv
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java
index 234dd39323fff..ed1451bc649cf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsCsv.java
@@ -17,13 +17,12 @@
 
 package org.apache.flink.streaming.api.functions.sink;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.BufferedWriter;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * Writes tuples in csv format.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsText.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsText.java
index e331ed94d99bf..eb9ddcde4a1b6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsText.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteFormatAsText.java
@@ -17,13 +17,12 @@
 
 package org.apache.flink.streaming.api.functions.sink;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.BufferedWriter;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * Writes tuples in text format.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java
index fa2bd8d1c26f5..9c8f5bd5dc423 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/WriteSinkFunction.java
@@ -17,11 +17,10 @@
 
 package org.apache.flink.streaming.api.functions.sink;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.FileNotFoundException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * Simple implementation of the SinkFunction writing tuples as simple text to
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
index 869413592f908..1b7673394cf02 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
@@ -16,6 +16,14 @@
  */
 package org.apache.flink.streaming.api.functions.source;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.io.FileInputFormat;
@@ -36,15 +44,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
 /**
  * This is the single (non-parallel) monitoring task which takes a {@link FileInputFormat}
  * and, depending on the {@link FileProcessingMode} and the {@link FilePathFilter}, it is responsible for:
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java
index 6e8d01d321271..54b6a03a9cf38 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileReaderOperator.java
@@ -16,6 +16,17 @@
  */
 package org.apache.flink.streaming.api.functions.source;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.io.CheckpointableInputFormat;
@@ -42,18 +53,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Queue;
-
-import static org.apache.flink.util.Preconditions.checkState;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * The operator that reads the {@link TimestampedFileInputSplit splits} received from the preceding
  * {@link ContinuousFileMonitoringFunction}. Contrary to the {@link ContinuousFileMonitoringFunction}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
index 8cee0bcec0bb0..2f91227e911ea 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
@@ -17,6 +17,12 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.core.fs.FileStatus;
@@ -25,13 +31,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 /**
  * A {@link SourceFunction} that monitors a directory and sends events downstream when it detects
  * new files. Used together with {@link FileReadFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
index 691d3d6d9ff93..745a26b918af9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromElementsFunction.java
@@ -17,6 +17,13 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
@@ -30,14 +37,6 @@
 import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
 import org.apache.flink.util.Preconditions;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
 /**
  * A stream source function that returns a sequence of elements.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java
index a8b527f471e9d..b844a0af316b0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromIteratorFunction.java
@@ -17,9 +17,8 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.util.Iterator;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * A {@link SourceFunction} that reads elements from an {@link Iterator} and emits them.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java
index db6c8a22d37f3..68e18bb70da00 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FromSplittableIteratorFunction.java
@@ -17,12 +17,11 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
+import java.util.Iterator;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.util.SplittableIterator;
 
-import java.util.Iterator;
-
 /**
  * A {@link SourceFunction} that reads elements from an {@link SplittableIterator} and emits them.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
index 3fdedf118d225..e6e6ea3aea317 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
@@ -17,6 +17,8 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
+import java.util.Iterator;
+import java.util.NoSuchElementException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.io.InputFormat;
 import org.apache.flink.api.common.io.RichInputFormat;
@@ -29,9 +31,6 @@
 import org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException;
 import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
 
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
 /**
  * A {@link SourceFunction} that reads data using an {@link InputFormat}.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java
index 965dc02223c16..ceef4e01f3331 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MultipleIdsMessageAcknowledgingSourceBase.java
@@ -18,6 +18,11 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.List;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.tuple.Tuple2;
@@ -26,12 +31,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Deque;
-import java.util.Iterator;
-import java.util.List;
-
 /**
  * Abstract base class for data sources that receive elements from a message queue and
  * acknowledge them back by IDs. In contrast to {@link MessageAcknowledgingSourceBase}, this source
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
index dd852a47191b3..e7d167329a278 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
@@ -17,19 +17,17 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.util.IOUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
 
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
 import java.net.InetSocketAddress;
 import java.net.Socket;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.util.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A source function that reads strings from a socket. The source will read bytes from the socket
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
index df5f9b6e02c31..0db8984cdc164 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
@@ -18,15 +18,14 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
-import org.apache.flink.annotation.PublicEvolving;
+import java.io.Serializable;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.streaming.api.functions.TimestampAssigner;
 import org.apache.flink.streaming.api.watermark.Watermark;
 
-import java.io.Serializable;
-
 /**
  * Base interface for all stream data sources in Flink. The contract of a stream source
  * is the following: When the source should start emitting elements, the {@link #run} method
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java
index b6a0c3b9b1cc3..598ccb39b982a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/StatefulSequenceSource.java
@@ -17,6 +17,8 @@
  */
 package org.apache.flink.streaming.api.functions.source;
 
+import java.util.ArrayDeque;
+import java.util.Deque;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
@@ -26,9 +28,6 @@
 import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
 import org.apache.flink.util.Preconditions;
 
-import java.util.ArrayDeque;
-import java.util.Deque;
-
 /**
  * A stateful streaming source that emits each number from a given interval exactly once,
  * possibly in parallel.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
index 07eb1d9c49d16..9a679b9b7b98a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
@@ -17,12 +17,11 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
+import java.io.Serializable;
 import org.apache.flink.core.fs.FileInputSplit;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.util.Preconditions;
 
-import java.io.Serializable;
-
 /**
  * An extended {@link FileInputSplit} that also includes information about:
  * <ul>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
index ceed996badf2e..500348a436e15 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java
@@ -18,15 +18,14 @@
 
 package org.apache.flink.streaming.api.functions.timestamps;
 
-import org.apache.flink.annotation.PublicEvolving;
+import static java.util.Objects.requireNonNull;
 
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static java.util.Objects.requireNonNull;
-
 /**
  * A timestamp assigner and watermark generator for streams where timestamps are monotonously
  * ascending. In this case, the local watermarks for the streams are easy to generate, because
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java
index e20b8789c7109..3d6c1e1cf9b61 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyAllWindowFunction.java
@@ -18,14 +18,13 @@
 
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.AggregateFunction;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * A {@link AllWindowFunction} that composes an {@link AggregateFunction} and {@link AllWindowFunction}.
  * Upon invocation, this first applies {@code AggregateFunction} to the input, and then
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
index 0341084b8dc52..ce5363fb76556 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AggregateApplyWindowFunction.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.AggregateFunction;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * A {@link WindowFunction} that composes an {@link AggregateFunction} and {@link WindowFunction}.
  * Upon invocation, this first applies {@code AggregateFunction} to the input, and then
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java
index c497b4a8faf36..04817b48deade 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/AllWindowFunction.java
@@ -18,13 +18,12 @@
 
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.io.Serializable;
 import org.apache.flink.annotation.Public;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.io.Serializable;
-
 /**
  * Base interface for functions that are evaluated over non-keyed windows.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
index 2514355fe5231..30662f002d716 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
@@ -17,6 +17,10 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.FoldFunction;
@@ -30,11 +34,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Collections;
-
 /**
  * Internal {@link AllWindowFunction} that is used for implementing a fold on a window configuration
  * that only allows {@link AllWindowFunction} and cannot directly execute a {@link FoldFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
index 125da03f27357..38244dd0a25dc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
@@ -17,6 +17,10 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.FoldFunction;
@@ -31,11 +35,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Collections;
-
 /**
  * Internal {@link ProcessAllWindowFunction} that is used for implementing a fold on a window
  * configuration that only allows {@link ProcessAllWindowFunction} and cannot directly execute a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
index 9a72cab9d13ea..281afc75ff254 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
@@ -17,6 +17,10 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.FoldFunction;
@@ -31,11 +35,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Collections;
-
 /**
  * Internal {@link ProcessWindowFunction} that is used for implementing a fold on a window
  * configuration that only allows {@link ProcessWindowFunction} and cannot directly execute a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
index 21b9c3b6c6884..770deb0e49387 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
@@ -17,6 +17,10 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.FoldFunction;
@@ -30,11 +34,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Collections;
-
 /**
  * Internal {@link WindowFunction} that is used for implementing a fold on a window configuration
  * that only allows {@link WindowFunction} and cannot directly execute a {@link FoldFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java
index 5a635b77c7996..a8469f59a70f1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyAllWindowFunction.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal {@link AllWindowFunction} that is used for implementing a fold on a window configuration
  * that only allows {@link AllWindowFunction} and cannot directly execute a {@link ReduceFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
index 847a79352ed59..d1f9ccdf3a969 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RuntimeContext;
@@ -25,8 +26,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal {@link ProcessAllWindowFunction} that is used for implementing a fold on a window
  * configuration that only allows {@link AllWindowFunction} and cannot directly execute a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
index 0fdce83880290..f5694c3552465 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RuntimeContext;
@@ -25,8 +26,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal {@link ProcessWindowFunction} that is used for implementing a fold on a window
  * configuration that only allows {@link AllWindowFunction} and cannot directly execute a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java
index c8174641cc55e..5890ec768bc8c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyWindowFunction.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal {@link WindowFunction} that is used for implementing a fold on a window configuration
  * that only allows {@link WindowFunction} and cannot directly execute a {@link ReduceFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java
index 030e9735cc76b..96e665aea24f0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/WindowFunction.java
@@ -18,13 +18,12 @@
 
 package org.apache.flink.streaming.api.functions.windowing;
 
+import java.io.Serializable;
 import org.apache.flink.annotation.Public;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.io.Serializable;
-
 /**
  * Base interface for functions that are evaluated over keyed (grouped) windows.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
index ed1f830eb84e4..7420a5aafe6b8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/DeltaFunction.java
@@ -17,9 +17,8 @@
 
 package org.apache.flink.streaming.api.functions.windowing.delta;
 
-import org.apache.flink.annotation.PublicEvolving;
-
 import java.io.Serializable;
+import org.apache.flink.annotation.PublicEvolving;
 
 /**
  * This interface allows the implementation of a function which calculates the
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
index 426328575b09e..791479256bab0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/Extractor.java
@@ -17,9 +17,8 @@
 
 package org.apache.flink.streaming.api.functions.windowing.delta.extractor;
 
-import org.apache.flink.annotation.Internal;
-
 import java.io.Serializable;
+import org.apache.flink.annotation.Internal;
 
 /**
  * Extractors allow to extract/convert one type to another. They are mostly used
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
index 9fc63547a1e92..f8d1be328cc28 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldFromArray.java
@@ -17,9 +17,8 @@
 
 package org.apache.flink.streaming.api.functions.windowing.delta.extractor;
 
-import org.apache.flink.annotation.Internal;
-
 import java.lang.reflect.Array;
+import org.apache.flink.annotation.Internal;
 
 /**
  * Extracts a single field out of an array.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
index 1fa9eb8f98f79..0d4e0f9cf078b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/delta/extractor/FieldsFromArray.java
@@ -17,9 +17,8 @@
 
 package org.apache.flink.streaming.api.functions.windowing.delta.extractor;
 
-import org.apache.flink.annotation.Internal;
-
 import java.lang.reflect.Array;
+import org.apache.flink.annotation.Internal;
 
 /**
  * Extracts multiple fields from an array and puts them into a new array of the
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
index b053bff55e24b..f43b95ae6ca7d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
@@ -24,18 +24,17 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.operators.util.CorruptConfigurationException;
+import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.util.ClassLoaderUtil;
 import org.apache.flink.streaming.api.CheckpointingMode;
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.streaming.api.collector.selector.OutputSelector;
 import org.apache.flink.streaming.api.operators.StreamOperator;
-import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.streaming.runtime.tasks.StreamTaskException;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.OutputTag;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
index 555d67b251392..f7c1919879690 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
@@ -19,10 +19,9 @@
 
 import java.io.Serializable;
 import java.util.List;
-
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.util.OutputTag;
 import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
+import org.apache.flink.util.OutputTag;
 
 /**
  * An edge in the streaming topology. One edge like this does not necessarily
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
index b132416a54e8d..11e40d23dfc34 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
@@ -28,11 +28,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.util.OutputTag;
 import org.apache.flink.api.common.operators.ResourceSpec;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -61,6 +59,7 @@
 import org.apache.flink.streaming.runtime.tasks.StreamIterationHead;
 import org.apache.flink.streaming.runtime.tasks.StreamIterationTail;
 import org.apache.flink.streaming.runtime.tasks.TwoInputStreamTask;
+import org.apache.flink.util.OutputTag;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
index 4a141ebc2478e..b20f6d6aee41c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
@@ -17,6 +17,12 @@
  */
 package org.apache.flink.streaming.api.graph;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple2;
@@ -38,13 +44,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 /**
  * A generator that generates a {@link StreamGraph} from a graph of
  * {@link StreamTransformation StreamTransformations}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasher.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasher.java
index 866fd1f88e134..dd14d50e653e0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasher.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasher.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.streaming.api.graph;
 
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-
 import java.util.Map;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
 
 /**
  * Interface for different implementations of generating hashes over a stream graph.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
index c778b20da1d04..3772e58df83b8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
@@ -18,18 +18,11 @@
 
 package org.apache.flink.streaming.api.graph;
 
+import static org.apache.flink.util.StringUtils.byteToHexString;
+
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-import org.apache.flink.streaming.api.operators.StreamOperator;
-import org.apache.flink.streaming.api.transformations.StreamTransformation;
-import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.nio.charset.Charset;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -41,8 +34,14 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
-
-import static org.apache.flink.util.StringUtils.byteToHexString;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.transformations.StreamTransformation;
+import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * StreamGraphHasher from Flink 1.2. This contains duplicated code to ensure that the algorithm does not change with
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
index 89ec45ad1a7b2..c1750a1cabfa3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
@@ -18,10 +18,9 @@
 
 package org.apache.flink.streaming.api.graph;
 
-import org.apache.flink.util.StringUtils;
-
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.flink.util.StringUtils;
 
 /**
  * StreamGraphHasher that works with user provided hashes. This is useful in case we want to set (alternative) hashes
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
index 71f2a4a4e1bb5..4320e62879f5c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
@@ -17,6 +17,9 @@
 
 package org.apache.flink.streaming.api.graph;
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.io.InputFormat;
@@ -28,10 +31,6 @@
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.operators.StreamOperator;
 
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * Class representing the operators in the streaming programs, with all their properties.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index accb15d21d071..a1d33d88713ee 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -17,6 +17,15 @@
 
 package org.apache.flink.streaming.api.graph;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import org.apache.commons.lang3.StringUtils;
 
 import org.apache.flink.annotation.Internal;
@@ -54,16 +63,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
 /**
  * The StreamingJobGraphGenerator converts a {@link StreamGraph} into a {@link JobGraph}.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index b260de30da621..72ab6dd531a90 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -18,6 +18,14 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
 import org.apache.flink.annotation.PublicEvolving;
@@ -69,15 +77,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.ConcurrentModificationException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-
 /**
  * Base class for all stream operators. Operators that contain a user function should extend the class
  * {@link AbstractUdfStreamOperator} instead (which is a specialized subclass of this class).
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
index 0dc1d2b63c7ca..2fa1e382b97d1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import static java.util.Objects.requireNonNull;
+
+import java.io.Serializable;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.Function;
@@ -33,17 +36,13 @@
 import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
 import org.apache.flink.streaming.api.checkpoint.CheckpointedRestoring;
 import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
-import org.apache.flink.streaming.util.functions.StreamingFunctionUtils;
 import org.apache.flink.streaming.api.graph.StreamConfig;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.tasks.StreamTask;
+import org.apache.flink.streaming.util.functions.StreamingFunctionUtils;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.Migration;
 
-import java.io.Serializable;
-
-import static java.util.Objects.requireNonNull;
-
 /**
  * This is used as the base class for operators that have a user-defined
  * function. This class handles the opening and closing of the user-defined functions,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
index a6d66e82db668..de3a7d2550adb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
@@ -17,6 +17,14 @@
  */
 package org.apache.flink.streaming.api.operators;
 
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.concurrent.ScheduledFuture;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
@@ -28,15 +36,6 @@
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.Preconditions;
 
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ScheduledFuture;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * {@link InternalTimerService} that stores timers on the Java heap.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
index 0b602323d44af..5886468051ec9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -29,10 +32,6 @@
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
 import org.apache.flink.util.Preconditions;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
 /**
  * An entity keeping all the time-related services available to all operators extending the
  * {@link AbstractStreamOperator}. These are the different {@link HeapInternalTimerService timer services}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java
index 95630508ada9b..7b6ba8da66eaa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.operators;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.base.LongSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
-import java.io.IOException;
-
 /**
  * Internal class for keeping track of in-flight timers.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java
index 9a43853cac8ca..441835e51a67d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java
@@ -18,6 +18,12 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -28,13 +34,6 @@
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.util.InstantiationUtil;
 
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * The watermark callback service allows to register a {@link OnWatermarkCallback OnWatermarkCallback}
  * and multiple keys, for which the callback will be invoked every time a new {@link Watermark} is received
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
index a46897c974109..db4b18334521e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
@@ -17,6 +17,9 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.runtime.state.VoidNamespace;
 import org.apache.flink.runtime.state.VoidNamespaceSerializer;
@@ -27,9 +30,6 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.util.OutputTag;
 
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
 /**
  * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing keyed
  * {@link ProcessFunction ProcessFunctions}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OnWatermarkCallback.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OnWatermarkCallback.java
index bc317a954029d..e78288dc2d870 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OnWatermarkCallback.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OnWatermarkCallback.java
@@ -18,11 +18,10 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.watermark.Watermark;
 
-import java.io.IOException;
-
 /**
  * A callback registered with the {@link InternalWatermarkCallbackService} service. This callback will
  * be invoked for all keys registered with the service, upon reception of a watermark.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OperatorSnapshotResult.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OperatorSnapshotResult.java
index 8aa76a5f852f7..eb743dde0b686 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OperatorSnapshotResult.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/OperatorSnapshotResult.java
@@ -18,13 +18,12 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import java.util.concurrent.RunnableFuture;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
 import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.util.ExceptionUtils;
 
-import java.util.concurrent.RunnableFuture;
-
 /**
  * Result of {@link StreamOperator#snapshotState}.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java
index f08c1ee897888..bbb1cb725ce8e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/ProcessOperator.java
@@ -17,6 +17,8 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import static org.apache.flink.util.Preconditions.checkState;
+
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.TimerService;
 import org.apache.flink.streaming.api.functions.ProcessFunction;
@@ -25,8 +27,6 @@
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
 import org.apache.flink.util.OutputTag;
 
-import static org.apache.flink.util.Preconditions.checkState;
-
 /**
  * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing
  * {@link ProcessFunction ProcessFunctions}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index 61578b23a6d51..1894ca297ce54 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import java.io.Serializable;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
@@ -26,8 +27,6 @@
 import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
 import org.apache.flink.streaming.runtime.tasks.StreamTask;
 
-import java.io.Serializable;
-
 /**
  * Basic interface for stream operators. Implementers would implement one of
  * {@link org.apache.flink.streaming.api.operators.OneInputStreamOperator} or
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
index cf59cdeb2527e..37050689eff14 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import java.util.concurrent.ScheduledFuture;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
@@ -27,8 +28,6 @@
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
 
-import java.util.concurrent.ScheduledFuture;
-
 /**
  * {@link StreamOperator} for streaming sources.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
index 25537fa9d0042..3401eb99b7f49 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSourceContexts.java
@@ -17,18 +17,17 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import java.util.concurrent.ScheduledFuture;
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
 import org.apache.flink.util.Preconditions;
 
-import java.util.concurrent.ScheduledFuture;
-
 /**
  * Source contexts for various stream time characteristics.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
index bd3d5006e81f7..0db1c687e27a3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.streaming.api.operators;
 
+import java.util.List;
+import java.util.Map;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
@@ -41,9 +43,6 @@
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
 import org.apache.flink.util.Preconditions;
 
-import java.util.List;
-import java.util.Map;
-
 /**
  * Implementation of the {@link org.apache.flink.api.common.functions.RuntimeContext},
  * for streaming operators.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
index 82e25aee099c4..17dec2375cabf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java
@@ -18,6 +18,12 @@
 
 package org.apache.flink.streaming.api.operators.async;
 
+import java.util.Collection;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
@@ -34,12 +40,12 @@
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue;
 import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue;
 import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueueEntry;
 import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry;
-import org.apache.flink.streaming.api.operators.async.queue.WatermarkQueueEntry;
-import org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue;
 import org.apache.flink.streaming.api.operators.async.queue.UnorderedStreamElementQueue;
+import org.apache.flink.streaming.api.operators.async.queue.WatermarkQueueEntry;
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
@@ -49,13 +55,6 @@
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.Preconditions;
 
-import java.util.Collection;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
 /**
  * The {@link AsyncWaitOperator} allows to asynchronously process incoming stream records. For that
  * the operator creates an {@link AsyncCollector} which is passed to an {@link AsyncFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java
index 3073bc9829256..d2f7725ae53c5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java
@@ -18,20 +18,19 @@
 
 package org.apache.flink.streaming.api.operators.async;
 
+import java.util.Collection;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.operators.Output;
 import org.apache.flink.streaming.api.operators.TimestampedCollector;
 import org.apache.flink.streaming.api.operators.async.queue.AsyncCollectionResult;
-import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue;
 import org.apache.flink.streaming.api.operators.async.queue.AsyncResult;
 import org.apache.flink.streaming.api.operators.async.queue.AsyncWatermarkResult;
+import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collection;
-
 /**
  * Runnable responsible for consuming elements from the given queue and outputting them to the
  * given output/timestampedCollector.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncCollectionResult.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncCollectionResult.java
index 6226ae64d7ded..152b9fdc1052e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncCollectionResult.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncCollectionResult.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.streaming.api.operators.async.queue;
 
-import org.apache.flink.annotation.Internal;
-
 import java.util.Collection;
+import org.apache.flink.annotation.Internal;
 
 /**
  * {@link AsyncResult} sub class for asynchronous result collections.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java
index 5aee45b1ca9be..d3267631a6562 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java
@@ -18,19 +18,18 @@
 
 package org.apache.flink.streaming.api.operators.async.queue;
 
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.runtime.concurrent.AcceptFunction;
-import org.apache.flink.streaming.api.operators.async.OperatorActions;
-import org.apache.flink.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.util.ArrayDeque;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.concurrent.Executor;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.streaming.api.operators.async.OperatorActions;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Ordered {@link StreamElementQueue} implementation. The ordered stream element queue emits
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java
index e02b8b03ba659..cc637b591b34b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java
@@ -18,11 +18,10 @@
 
 package org.apache.flink.streaming.api.operators.async.queue;
 
+import java.util.Collection;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.operators.async.AsyncWaitOperator;
 
-import java.util.Collection;
-
 /**
  * Interface for blocking stream element queues for the {@link AsyncWaitOperator}.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
index 253b5d71bf73b..66872df317355 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
@@ -18,14 +18,13 @@
 
 package org.apache.flink.streaming.api.operators.async.queue;
 
+import java.util.concurrent.Executor;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.runtime.concurrent.AcceptFunction;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.util.Preconditions;
 
-import java.util.concurrent.Executor;
-
 /**
  * Entry class for the {@link StreamElementQueue}. The stream element queue entry stores the
  * {@link StreamElement} for which the stream element queue entry has been instantiated.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java
index 708bf1703d02d..e71459a545eb0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.streaming.api.operators.async.queue;
 
+import java.util.Collection;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.runtime.concurrent.CompletableFuture;
 import org.apache.flink.runtime.concurrent.Future;
@@ -26,8 +27,6 @@
 import org.apache.flink.streaming.api.functions.async.collector.AsyncCollector;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
-import java.util.Collection;
-
 /**
  * {@link StreamElementQueueEntry} implementation for {@link StreamRecord}. This class also acts
  * as the {@link AsyncCollector} implementation which is given to the {@link AsyncFunction}. The
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
index 1ca36dbf5132a..6334c49cefe62 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
@@ -18,13 +18,6 @@
 
 package org.apache.flink.streaming.api.operators.async.queue;
 
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.runtime.concurrent.AcceptFunction;
-import org.apache.flink.streaming.api.operators.async.OperatorActions;
-import org.apache.flink.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.util.ArrayDeque;
 import java.util.Arrays;
 import java.util.Collection;
@@ -34,6 +27,12 @@
 import java.util.concurrent.Executor;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.streaming.api.operators.async.OperatorActions;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Unordered implementation of the {@link StreamElementQueue}. The unordered stream element queue
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java
index 63ddb3f6f6ac0..4d81f1eaff726 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoProcessOperator.java
@@ -17,6 +17,9 @@
 
 package org.apache.flink.streaming.api.operators.co;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.TimerService;
 import org.apache.flink.streaming.api.functions.co.CoProcessFunction;
@@ -28,9 +31,6 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
 
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
 /**
  * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing
  * {@link CoProcessFunction CoProcessFunctions}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java
index eab479737416c..1091031303d46 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoStreamFlatMap.java
@@ -20,8 +20,8 @@
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction;
 import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
-import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
 import org.apache.flink.streaming.api.operators.TimestampedCollector;
+import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
 /**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java
index d125a79c3698b..290bb3ca85f43 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java
@@ -17,6 +17,9 @@
 
 package org.apache.flink.streaming.api.operators.co;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.runtime.state.VoidNamespace;
 import org.apache.flink.runtime.state.VoidNamespaceSerializer;
@@ -32,9 +35,6 @@
 import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
 /**
  * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing keyed
  * {@link CoProcessFunction CoProcessFunctions}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java
index 4073e74963033..1de015250f834 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/CoFeedbackTransformation.java
@@ -18,13 +18,12 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
 
 /**
  * This represents a feedback point in a topology. The type of the feedback elements need not match
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java
index 330a2159ceb73..be7a97ce7ec88 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/FeedbackTransformation.java
@@ -18,11 +18,10 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-
 import java.util.Collection;
 import java.util.List;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
 
 /**
  * This represents a feedback point in a topology.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
index 471cb226e08be..2df8e55912925 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/OneInputTransformation.java
@@ -18,15 +18,14 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 
-import java.util.Collection;
-import java.util.List;
-
 /**
  * This Transformation represents the application of a
  * {@link org.apache.flink.streaming.api.operators.OneInputStreamOperator} to one input
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
index 588accd0c627f..56f574ac60c02 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/PartitionTransformation.java
@@ -18,14 +18,12 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
-
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
 
-import java.util.Collection;
-import java.util.List;
-
 /**
  * This transformation represents a change of partitioning of the input elements.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
index cbc5b36d36a51..eeff8fa0a6d4c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SelectTransformation.java
@@ -18,11 +18,10 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-
 import java.util.Collection;
 import java.util.List;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
 
 /**
  * This transformation represents a selection of only certain upstream elements. This must
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SideOutputTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SideOutputTransformation.java
index a90b40da4f1a7..8d12ebff70bd2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SideOutputTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SideOutputTransformation.java
@@ -20,11 +20,10 @@
 import static java.util.Objects.requireNonNull;
 
 import com.google.common.collect.Lists;
-import org.apache.flink.util.OutputTag;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-
 import java.util.Collection;
 import java.util.List;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.util.OutputTag;
 
 
 /**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
index 0a803f538ddb1..a2a05dfac5fb2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SinkTransformation.java
@@ -18,6 +18,8 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
@@ -25,9 +27,6 @@
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.StreamSink;
 
-import java.util.Collection;
-import java.util.List;
-
 /**
  * This Transformation represents a Sink.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java
index f51f1313b0ebf..ca937e85321fa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SourceTransformation.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.transformations;
 
+import java.util.Collection;
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.StreamSource;
 
-import java.util.Collection;
-import java.util.Collections;
-
 /**
  * This represents a Source. This does not actually transform anything since it has no inputs but
  * it is the root {@code StreamTransformation} of any topology.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java
index 317ada53b62a4..1678e4593bba5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/SplitTransformation.java
@@ -18,13 +18,12 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.streaming.api.collector.selector.OutputSelector;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 
-import java.util.Collection;
-import java.util.List;
-
 /**
  * This transformation represents a split of one
  * {@link org.apache.flink.streaming.api.datastream.DataStream} into several {@code DataStreams}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
index f79697c6bbd2d..77f4f849ac6eb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/StreamTransformation.java
@@ -18,6 +18,9 @@
 package org.apache.flink.streaming.api.transformations;
 
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import java.util.Collection;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.InvalidTypesException;
 import org.apache.flink.api.common.operators.ResourceSpec;
@@ -28,10 +31,6 @@
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.util.Preconditions;
 
-import java.util.Collection;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * A {@code StreamTransformation} represents the operation that creates a
  * {@link org.apache.flink.streaming.api.datastream.DataStream}. Every
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java
index ef61284b9db4d..df4eaa1351048 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/TwoInputTransformation.java
@@ -18,15 +18,14 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
 
-import java.util.Collection;
-import java.util.List;
-
 /**
  * This Transformation represents the application of a
  * {@link org.apache.flink.streaming.api.operators.TwoInputStreamOperator} to two input
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java
index 9e36784890f21..c71ec533ae052 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/transformations/UnionTransformation.java
@@ -18,11 +18,10 @@
 package org.apache.flink.streaming.api.transformations;
 
 import com.google.common.collect.Lists;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-
 import java.util.Collection;
 import java.util.List;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
 
 /**
  * This transformation represents a union of several input
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java
index 2091bf251a127..4e74c23e6e3d8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/BaseAlignedWindowAssigner.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.Collection;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
-import java.util.Collection;
-
 /**
  * A base {@link WindowAssigner} used to instantiate one of the deprecated
  * {@link org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
index 2602bc2aa5249..a8247e62675ab 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
@@ -17,6 +17,8 @@
  */
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.Collection;
+import java.util.Collections;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -25,9 +27,6 @@
 import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
-import java.util.Collection;
-import java.util.Collections;
-
 /**
  * A {@link WindowAssigner} that windows elements into sessions based on the timestamp of the
  * elements. Windows cannot overlap.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
index 47e47ef8e6d10..5b35b27bee042 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
@@ -17,6 +17,8 @@
  */
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.Collection;
+import java.util.Collections;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
@@ -26,9 +28,6 @@
 import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
 import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
 
-import java.util.Collection;
-import java.util.Collections;
-
 /**
  * A {@link WindowAssigner} that assigns all elements to the same {@link GlobalWindow}.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java
index ca77861a56585..98223f459a9e3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/MergingWindowAssigner.java
@@ -17,11 +17,10 @@
  */
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.Collection;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 
-import java.util.Collection;
-
 /**
  * A {@code WindowAssigner} that can merge windows.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
index 8f727dcce8745..db31788a78cea 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
@@ -17,6 +17,8 @@
  */
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.Collection;
+import java.util.Collections;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -25,9 +27,6 @@
 import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
-import java.util.Collection;
-import java.util.Collections;
-
 /**
  * A {@link WindowAssigner} that windows elements into sessions based on the current processing
  * time. Windows cannot overlap.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
index f575c7db9454a..e45f2d9ba1791 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
@@ -18,19 +18,18 @@
 
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
 /**
  * A {@link WindowAssigner} that windows elements into sliding windows based on the timestamp of the
  * elements. Windows can possibly overlap.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
index 57f0c8c7dc3ac..53e9a0555edd6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
@@ -17,6 +17,9 @@
  */
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -25,10 +28,6 @@
 import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
 /**
  * A {@link WindowAssigner} that windows elements into sliding windows based on the current
  * system time of the machine the operation is running on. Windows can possibly overlap.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
index d37f2cde5a53d..93c1c37b7280a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
@@ -18,18 +18,17 @@
 
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.Collection;
+import java.util.Collections;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
-import java.util.Collection;
-import java.util.Collections;
-
 /**
  * A {@link WindowAssigner} that windows elements into windows based on the timestamp of the
  * elements. Windows cannot overlap.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
index 4f51998711d63..4e579b7259c10 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
@@ -17,6 +17,8 @@
  */
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.util.Collection;
+import java.util.Collections;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -25,9 +27,6 @@
 import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
-import java.util.Collection;
-import java.util.Collections;
-
 /**
  * A {@link WindowAssigner} that windows elements into windows based on the current
  * system time of the machine the operation is running on. Windows cannot overlap.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
index 2ff25e4c671e6..58c0b1da527b8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.streaming.api.windowing.assigners;
 
+import java.io.Serializable;
+import java.util.Collection;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -25,10 +27,6 @@
 import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 
-import java.io.Serializable;
-
-import java.util.Collection;
-
 /**
  * A {@code WindowAssigner} assigns zero or more {@link Window Windows} to an element.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
index 92a38775bbf60..5f102867ef49b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
@@ -17,12 +17,11 @@
  */
 package org.apache.flink.streaming.api.windowing.evictors;
 
+import java.util.Iterator;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.streaming.runtime.operators.windowing.TimestampedValue;
 
-import java.util.Iterator;
-
 /**
  * An {@link Evictor} that keeps up to a certain amount of elements.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java
index 67c67b819b762..737c9f5328625 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/DeltaEvictor.java
@@ -18,13 +18,12 @@
 package org.apache.flink.streaming.api.windowing.evictors;
 
 import com.google.common.collect.Iterables;
+import java.util.Iterator;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.streaming.api.functions.windowing.delta.DeltaFunction;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.streaming.runtime.operators.windowing.TimestampedValue;
 
-import java.util.Iterator;
-
 /**
  * An {@link Evictor} that keeps elements based on a {@link DeltaFunction} and a threshold.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java
index 7ba32aab83e54..67a862f4f42ab 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/Evictor.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.windowing.evictors;
 
+import java.io.Serializable;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.streaming.runtime.operators.windowing.TimestampedValue;
 
-import java.io.Serializable;
-
 /**
  * An {@code Evictor} can remove elements from a pane before/after the evaluation of WindowFunction
  * and after the window evaluation gets triggered by a
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
index afd3ab796c0d3..ba00403a05675 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
@@ -17,14 +17,13 @@
  */
 package org.apache.flink.streaming.api.windowing.evictors;
 
+import java.util.Iterator;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.streaming.runtime.operators.windowing.TimestampedValue;
 
-import java.util.Iterator;
-
 /**
  * An {@link Evictor} that keeps elements for a certain amount of time. Elements older
  * than {@code current_time - keep_time} are evicted. The current_time is time associated
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java
index c22f3a483ebf4..f1760d5f17d98 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/time/Time.java
@@ -18,9 +18,10 @@
 
 package org.apache.flink.streaming.api.windowing.time;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 import java.util.concurrent.TimeUnit;
 import org.apache.flink.annotation.Public;
-import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
  * The definition of a time interval for windowing. The time characteristic referred
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
index 1c47de3a68338..746e4ac4382f8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.streaming.api.windowing.triggers;
 
+import java.io.Serializable;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.api.common.state.MergingState;
@@ -28,8 +29,6 @@
 import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 
-import java.io.Serializable;
-
 /**
  * A {@code Trigger} determines when a pane of a window should be evaluated to emit the
  * results for that part of the window.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
index 6c3921f665c66..e1db976c74e86 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
@@ -17,13 +17,12 @@
  */
 package org.apache.flink.streaming.api.windowing.windows;
 
+import java.io.IOException;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
-import java.io.IOException;
-
 /**
  * The default window into which all data is placed (via
  * {@link org.apache.flink.streaming.api.windowing.assigners.GlobalWindows}).
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
index 92fb8bb78c306..6d896cb943dfe 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
@@ -17,14 +17,6 @@
  */
 package org.apache.flink.streaming.api.windowing.windows;
 
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
-import org.apache.flink.util.MathUtils;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -33,6 +25,13 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
+import org.apache.flink.util.MathUtils;
 
 /**
  * A {@link Window} that represents a time interval from {@code start} (inclusive) to
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
index c572023a21b2a..b9d37168a396d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierBuffer.java
@@ -17,13 +17,17 @@
 
 package org.apache.flink.streaming.runtime.io;
 
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
 import org.apache.flink.runtime.checkpoint.CheckpointMetrics;
 import org.apache.flink.runtime.checkpoint.decline.AlignmentLimitExceededException;
 import org.apache.flink.runtime.checkpoint.decline.CheckpointDeclineException;
 import org.apache.flink.runtime.checkpoint.decline.CheckpointDeclineOnCancellationBarrierException;
 import org.apache.flink.runtime.checkpoint.decline.CheckpointDeclineSubsumedException;
-import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
 import org.apache.flink.runtime.checkpoint.decline.InputEndOfStreamException;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
 import org.apache.flink.runtime.io.network.api.CancelCheckpointMarker;
@@ -32,15 +36,9 @@
 import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
 import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
 import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.ArrayDeque;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-
 /**
  * The barrier buffer is {@link CheckpointBarrierHandler} that blocks inputs with barriers until
  * all inputs have received the barrier for a given checkpoint.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
index 1479437c81df4..2e965dd289d3f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BarrierTracker.java
@@ -18,22 +18,20 @@
 
 package org.apache.flink.streaming.runtime.io;
 
+import java.util.ArrayDeque;
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
 import org.apache.flink.runtime.checkpoint.CheckpointMetrics;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
 import org.apache.flink.runtime.checkpoint.decline.CheckpointDeclineOnCancellationBarrierException;
-import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
 import org.apache.flink.runtime.io.network.api.CancelCheckpointMarker;
 import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
 import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
 import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
 import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayDeque;
-
 /**
  * The BarrierTracker keeps track of what checkpoint barriers have been received from
  * which input channels. Once it has observed all checkpoint barriers for a checkpoint ID,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java
index 0c676e051ce31..c39f7dfe34ec6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierHandler.java
@@ -18,12 +18,11 @@
 
 package org.apache.flink.streaming.runtime.io;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
 import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
 
-import java.io.IOException;
-
 /**
  * The CheckpointBarrierHandler reacts to checkpoint barrier arriving from the input channels.
  * Different implementations may either simply track barriers, or block certain inputs on
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
index b8efb1c0e9b8a..92f0217395fc2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
@@ -17,24 +17,23 @@
 
 package org.apache.flink.streaming.runtime.io;
 
-import java.io.IOException;
+import static org.apache.flink.util.Preconditions.checkNotNull;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.util.OutputTag;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.runtime.event.AbstractEvent;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
-import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatusProvider;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
+import org.apache.flink.util.OutputTag;
 
 /**
  * Implementation of {@link Output} that sends data using a {@link RecordWriter}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
index 22fcf513d21fc..33b16d835fda3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
@@ -18,8 +18,9 @@
 
 package org.apache.flink.streaming.runtime.io;
 
-import java.io.IOException;
+import static org.apache.flink.util.Preconditions.checkNotNull;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.configuration.Configuration;
@@ -27,9 +28,6 @@
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Gauge;
-import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
-import org.apache.flink.runtime.metrics.groups.OperatorMetricGroup;
-import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.event.AbstractEvent;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
@@ -39,20 +37,21 @@
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
 import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
+import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
+import org.apache.flink.runtime.metrics.groups.OperatorMetricGroup;
+import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.plugable.DeserializationDelegate;
 import org.apache.flink.runtime.plugable.NonReusingDeserializationDelegate;
 import org.apache.flink.streaming.api.CheckpointingMode;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve;
-import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
-import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve;
+import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
 
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * Input reader for {@link org.apache.flink.streaming.runtime.tasks.OneInputStreamTask}.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
index a67251d1926f7..1c11a28c21cde 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamRecordWriter.java
@@ -17,16 +17,15 @@
 
 package org.apache.flink.streaming.runtime.io;
 
-import java.io.IOException;
+import static org.apache.flink.util.Preconditions.checkArgument;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
 
-import static org.apache.flink.util.Preconditions.checkArgument;
-
 /**
  * This record writer keeps data in buffers at most for a certain timeout. It spawns a separate thread
  * that flushes the outputs in a defined interval, to make sure data does not linger in the buffers for too long.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
index 5774094a25220..d34686d4b26a0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
@@ -18,14 +18,16 @@
 
 package org.apache.flink.streaming.runtime.io;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import java.io.IOException;
+import java.util.Collection;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.metrics.Gauge;
-import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
-import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.event.AbstractEvent;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
@@ -35,23 +37,20 @@
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
 import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
+import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
+import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.plugable.DeserializationDelegate;
 import org.apache.flink.runtime.plugable.NonReusingDeserializationDelegate;
 import org.apache.flink.streaming.api.CheckpointingMode;
 import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
+import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
-import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
 
-import java.io.IOException;
-import java.util.Collection;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * Input reader for {@link org.apache.flink.streaming.runtime.tasks.TwoInputStreamTask}.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/CheckpointCommitter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/CheckpointCommitter.java
index 6e50dde851c85..455de3ebd2a8f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/CheckpointCommitter.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/CheckpointCommitter.java
@@ -17,11 +17,10 @@
  */
 package org.apache.flink.streaming.runtime.operators;
 
+import java.io.Serializable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.Serializable;
-
 /**
  * This class is used to save information about which sink operator instance has committed checkpoints to a backend.
  * <p/>
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
index 7a571ec52deae..35f420a7cc998 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
@@ -17,6 +17,12 @@
  */
 package org.apache.flink.streaming.runtime.operators;
 
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.fs.FSDataInputStream;
@@ -35,13 +41,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.UUID;
-
 /**
  * Generic Sink that emits its input elements into an arbitrary backend. This sink is integrated with Flink's checkpointing
  * mechanism and can provide exactly-once guarantees; depending on the storage backend and sink/committer implementation.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
index 70d76d4ab54d4..515f3922507e7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
@@ -18,8 +18,9 @@
 
 package org.apache.flink.streaming.runtime.operators.windowing;
 
-import org.apache.commons.math3.util.ArithmeticUtils;
+import static java.util.Objects.requireNonNull;
 
+import org.apache.commons.math3.util.ArithmeticUtils;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -28,15 +29,13 @@
 import org.apache.flink.core.fs.FSDataOutputStream;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-import org.apache.flink.util.MathUtils;
 import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
-import org.apache.flink.streaming.api.operators.TimestampedCollector;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.TimestampedCollector;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
-import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-
-import static java.util.Objects.requireNonNull;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
+import org.apache.flink.util.MathUtils;
 
 /**
  * Base class for special window operator implementation for windows that fire at the same time for
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
index 76f2a97ce7f4e..680fd5de4630a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.streaming.runtime.operators.windowing;
 
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Iterator;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputView;
@@ -26,10 +29,6 @@
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.util.Collector;
 
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Iterator;
-
 /**
  * Base class for a multiple key/value maps organized in panes.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
index 5d95cdc07d121..1dd5a818c1b94 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.streaming.runtime.operators.windowing;
 
+import java.util.ArrayList;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.state.FoldingState;
 import org.apache.flink.api.common.state.FoldingStateDescriptor;
@@ -31,14 +32,12 @@
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalWindowFunction;
-import org.apache.flink.util.UnionIterator;
 import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.streaming.api.windowing.windows.Window;
+import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalWindowFunction;
 import org.apache.flink.util.Collector;
-
-import java.util.ArrayList;
+import org.apache.flink.util.UnionIterator;
 
 /**
  * Key/value map organized in panes for accumulating windows (with a window function).
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
index d71f80e5a5956..830d480bd188c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.streaming.runtime.operators.windowing;
 
+import java.util.ArrayList;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -27,8 +28,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalWindowFunction;
 
-import java.util.ArrayList;
-
 /**
  * Special window operator implementation for windows that fire at the same time for all keys with
  * accumulating windows.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
index 0d590c24becb6..edf4f25d5682b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
@@ -17,14 +17,16 @@
 */
 package org.apache.flink.streaming.runtime.operators.windowing;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Iterables;
-
+import java.util.Collection;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.AppendingState;
+import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.StateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.functions.KeySelector;
@@ -41,10 +43,6 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.util.OutputTag;
 
-import java.util.Collection;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * A {@link WindowOperator} that also allows an {@link Evictor} to be used.
  *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
index d251ddd417731..c69b6dce9701e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
@@ -18,14 +18,13 @@
 
 package org.apache.flink.streaming.runtime.operators.windowing;
 
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.util.MathUtils;
-
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.util.MathUtils;
 
 /**
  * A special Hash Map implementation that can be traversed efficiently in sync with other
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/MergingWindowSet.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/MergingWindowSet.java
index de37cb6d2163d..540957135a76e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/MergingWindowSet.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/MergingWindowSet.java
@@ -17,6 +17,11 @@
  */
 package org.apache.flink.streaming.runtime.operators.windowing;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
@@ -24,12 +29,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 /**
  * Utility for keeping track of merging {@link Window Windows} when using a
  * {@link MergingWindowAssigner} in a {@link WindowOperator}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index e36d485a8e65e..317fc9f6f4537 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -18,8 +18,16 @@
 
 package org.apache.flink.streaming.runtime.operators.windowing;
 
-import org.apache.commons.math3.util.ArithmeticUtils;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
 
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+import org.apache.commons.math3.util.ArithmeticUtils;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.state.AppendingState;
@@ -37,7 +45,6 @@
 import org.apache.flink.api.common.state.StateDescriptor;
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.util.OutputTag;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.functions.KeySelector;
@@ -55,13 +62,13 @@
 import org.apache.flink.runtime.state.internal.InternalListState;
 import org.apache.flink.runtime.state.internal.InternalMergingState;
 import org.apache.flink.streaming.api.datastream.LegacyWindowOperatorType;
-import org.apache.flink.streaming.api.operators.Triggerable;
 import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.InternalTimer;
 import org.apache.flink.streaming.api.operators.InternalTimerService;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.TimestampedCollector;
+import org.apache.flink.streaming.api.operators.Triggerable;
 import org.apache.flink.streaming.api.windowing.assigners.BaseAlignedWindowAssigner;
 import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
 import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeWindows;
@@ -72,18 +79,9 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalWindowFunction;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.Preconditions;
 
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.List;
-import java.util.PriorityQueue;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 /**
  * An operator that implements the logic for windowing based on a {@link WindowAssigner} and
  * {@link Trigger}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java
index 83e896d21fdcf..2490d608fc476 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessAllWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.runtime.operators.windowing.functions;
 
+import java.util.Collections;
 import org.apache.flink.api.common.functions.AggregateFunction;
 import org.apache.flink.api.common.functions.IterationRuntimeContext;
 import org.apache.flink.api.common.functions.RuntimeContext;
@@ -26,8 +27,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal window function for wrapping a {@link ProcessAllWindowFunction} that takes an
  * {@code Iterable} and an {@link AggregateFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessWindowFunction.java
index e14c9bd4b161c..c69ec53aaa11d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalAggregateProcessWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.runtime.operators.windowing.functions;
 
+import java.util.Collections;
 import org.apache.flink.api.common.functions.AggregateFunction;
 import org.apache.flink.api.common.functions.IterationRuntimeContext;
 import org.apache.flink.api.common.functions.RuntimeContext;
@@ -25,8 +26,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal window function for wrapping a {@link ProcessWindowFunction} that takes an
  * {@code Iterable} and an {@link AggregateFunction}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueAllWindowFunction.java
index e90bcf4a9e072..03ad0bd1c8ebc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueAllWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.runtime.operators.windowing.functions;
 
+import java.util.Collections;
 import org.apache.flink.api.common.functions.IterationRuntimeContext;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
@@ -24,8 +25,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal window function for wrapping an {@link AllWindowFunction} that takes an {@code Iterable}
  * when the window state is a single value.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java
index f7c6a08254166..eeef8eb70f9db 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessAllWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.runtime.operators.windowing.functions;
 
+import java.util.Collections;
 import org.apache.flink.api.common.functions.IterationRuntimeContext;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
@@ -25,8 +26,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal window function for wrapping a {@link ProcessAllWindowFunction} that takes an {@code Iterable}
  * when the window state is a single value.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java
index 21d1639e838b5..490069c6349ce 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueProcessWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.runtime.operators.windowing.functions;
 
+import java.util.Collections;
 import org.apache.flink.api.common.functions.IterationRuntimeContext;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
@@ -24,8 +25,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal window function for wrapping a {@link ProcessWindowFunction} that takes an {@code Iterable}
  * when the window state is a single value.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueWindowFunction.java
index d5cc4a25ec6e1..a629c76a25dfa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalSingleValueWindowFunction.java
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.streaming.runtime.operators.windowing.functions;
 
+import java.util.Collections;
 import org.apache.flink.api.common.functions.IterationRuntimeContext;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
@@ -24,8 +25,6 @@
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
 
-import java.util.Collections;
-
 /**
  * Internal window function for wrapping a {@link WindowFunction} that takes an {@code Iterable}
  * when the window state is a single value.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index 64396da0eec3e..7fe088a599352 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.streaming.runtime.streamrecord;
 
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputView;
@@ -25,10 +28,6 @@
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
 
-import java.io.IOException;
-
-import static java.util.Objects.requireNonNull;
-
 /**
  * Serializer for {@link StreamRecord}, {@link Watermark}, {@link LatencyMarker}, and
  * {@link StreamStatus}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java
index 1327c838bfaa3..487fd8a134571 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java
@@ -18,12 +18,12 @@
 
 package org.apache.flink.streaming.runtime.streamstatus;
 
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.streaming.api.watermark.Watermark;
-
 import static org.apache.flink.util.Preconditions.checkArgument;
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.watermark.Watermark;
+
 /**
  * A {@code StatusWatermarkValve} embodies the logic of how {@link Watermark} and {@link StreamStatus} are propagated to
  * downstream outputs, given a set of one or multiple input channels that continuously receive them. Usages of this
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
index 9d7c3d70300fe..77c80c965d196 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
@@ -18,10 +18,10 @@
 
 package org.apache.flink.streaming.runtime.tasks;
 
-import org.apache.flink.annotation.Internal;
-
 import static java.util.Objects.requireNonNull;
 
+import org.apache.flink.annotation.Internal;
+
 /**
  * A special exception that signifies that the cause exception came from a chained operator.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index 93cf6463031d6..d96a3868e0d69 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -17,8 +17,13 @@
 
 package org.apache.flink.streaming.runtime.tasks;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.util.OutputTag;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.metrics.Counter;
@@ -32,32 +37,26 @@
 import org.apache.flink.streaming.api.collector.selector.CopyingDirectedOutput;
 import org.apache.flink.streaming.api.collector.selector.DirectedOutput;
 import org.apache.flink.streaming.api.collector.selector.OutputSelector;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
-import org.apache.flink.streaming.runtime.io.RecordWriterOutput;
 import org.apache.flink.streaming.api.graph.StreamConfig;
 import org.apache.flink.streaming.api.graph.StreamEdge;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.Output;
 import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.io.RecordWriterOutput;
 import org.apache.flink.streaming.runtime.io.StreamRecordWriter;
 import org.apache.flink.streaming.runtime.partitioner.ConfigurableStreamPartitioner;
 import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
 import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatusProvider;
+import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.XORShiftRandom;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
 /**
  * The {@code OperatorChain} contains all operators that are executed as one chain within a single
  * {@link StreamTask}.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java
index 1a79f5429c206..988ae7ffc8c29 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorStateHandles.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.streaming.runtime.tasks;
 
+import java.util.Collection;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.runtime.state.ChainedStateHandle;
@@ -28,9 +30,6 @@
 import org.apache.flink.util.CollectionUtil;
 import org.apache.flink.util.Preconditions;
 
-import java.util.Collection;
-import java.util.List;
-
 /**
  * This class holds all state handles for one operator.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
index 0d2fe1fa409d4..859e32edb6cc1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationHead.java
@@ -20,12 +20,11 @@
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.TimeUnit;
-
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.runtime.io.RecordWriterOutput;
 import org.apache.flink.streaming.runtime.io.BlockingQueueBroker;
+import org.apache.flink.streaming.runtime.io.RecordWriterOutput;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index da4f9b63ef903..57e43de7f508b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -17,6 +17,17 @@
 
 package org.apache.flink.streaming.runtime.tasks;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -60,18 +71,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.RunnableFuture;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicReference;
-
 /**
  * Base class for all streaming tasks. A task is the unit of local processing that is deployed
  * and executed by the TaskManagers. Each task runs one or more {@link StreamOperator}s which form
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
index 34202fbc7446c..2cb509af056e7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
@@ -17,10 +17,8 @@
 
 package org.apache.flink.streaming.runtime.tasks;
 
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.util.Preconditions;
+import static org.apache.flink.util.Preconditions.checkNotNull;
 
-import javax.annotation.Nonnull;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.Delayed;
@@ -31,8 +29,9 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
+import javax.annotation.Nonnull;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.util.Preconditions;
 
 /**
  * A {@link ProcessingTimeService} which assigns as current processing time the result of calling
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java
index e5fcc1a2e6d29..53b4dda565b64 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java
@@ -17,9 +17,6 @@
 
 package org.apache.flink.streaming.runtime.tasks;
 
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Preconditions;
-
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.PriorityQueue;
@@ -30,6 +27,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Preconditions;
 
 /**
  * This is a {@link ProcessingTimeService} used <b>strictly for testing</b> the
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyFromLocal.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyFromLocal.java
index 37dc2c703c3a3..1f44dc6e11a78 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyFromLocal.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyFromLocal.java
@@ -17,15 +17,14 @@
  */
 package org.apache.flink.streaming.util;
 
+import java.io.File;
+import java.net.URI;
 import org.apache.flink.api.java.tuple.Tuple1;
 import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import java.io.File;
-import java.net.URI;
-
 /**
  * Utility for copying from local file system to a HDFS {@link FileSystem}.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyToLocal.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyToLocal.java
index c50e22da1b348..0a006f1e027fe 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyToLocal.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/HDFSCopyToLocal.java
@@ -17,15 +17,14 @@
  */
 package org.apache.flink.streaming.util;
 
+import java.io.File;
+import java.net.URI;
 import org.apache.flink.api.java.tuple.Tuple1;
 import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import java.io.File;
-import java.net.URI;
-
 /**
  * Utility for copying from a HDFS {@link FileSystem} to the local file system.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/functions/StreamingFunctionUtils.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/functions/StreamingFunctionUtils.java
index 44824315429c0..36abd473306ff 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/functions/StreamingFunctionUtils.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/functions/StreamingFunctionUtils.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.streaming.util.functions;
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.Function;
@@ -33,10 +36,6 @@
 import org.apache.flink.streaming.api.operators.OutputTypeConfigurable;
 import org.apache.flink.util.Preconditions;
 
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * Utility class that contains helper methods to work with Flink Streaming
  * {@link Function Functions}. This is similar to
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
index 43440460a1b00..0e476346cbd34 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
@@ -17,26 +17,25 @@
 
 package org.apache.flink.streaming.util.keys;
 
+import static java.util.Objects.requireNonNull;
+
 import java.lang.reflect.Array;
 import java.util.Arrays;
-
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.InvalidTypesException;
 import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Keys;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.CompositeType;
 import org.apache.flink.api.common.typeutils.TypeComparator;
 import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.common.operators.Keys;
 import org.apache.flink.api.java.tuple.Tuple;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 
-import static java.util.Objects.requireNonNull;
-
 /**
  * Utility class that contains helper methods to manipulating {@link KeySelector} for streaming.
  */
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java
index 02ea0041f0d73..d394e9bbd6a46 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/AbstractDeserializationSchema.java
@@ -18,11 +18,10 @@
 
 package org.apache.flink.streaming.util.serialization;
 
+import java.io.IOException;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 
-import java.io.IOException;
-
 /**
  * The deserialization schema describes how to turn the byte messages delivered by certain
  * data sources (for example Apache Kafka) into data types (Java/Scala objects) that are
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java
index 986cfb3039fda..95428f64447aa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SerializationSchema.java
@@ -17,9 +17,8 @@
 
 package org.apache.flink.streaming.util.serialization;
 
-import org.apache.flink.annotation.Public;
-
 import java.io.Serializable;
+import org.apache.flink.annotation.Public;
 
 /**
  * The serialization schema describes how to turn a data object into a different serialized
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
index 27ba9e913331f..e25daebfef838 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
@@ -17,16 +17,15 @@
 
 package org.apache.flink.streaming.util.serialization;
 
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
+import static org.apache.flink.util.Preconditions.checkNotNull;
 
 import java.io.IOException;
 import java.io.ObjectOutputStream;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
 
 /**
  * Very simple serialization schema for strings.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
index b187b041f7250..49ad52ffc7a37 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/serialization/TypeInformationSerializationSchema.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.streaming.util.serialization;
 
+import java.io.IOException;
 import org.apache.flink.annotation.Public;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -25,8 +26,6 @@
 import org.apache.flink.runtime.util.DataInputDeserializer;
 import org.apache.flink.runtime.util.DataOutputSerializer;
 
-import java.io.IOException;
-
 /**
  * A serialization and deserialization schema that uses Flink's serialization stack to
  * transform typed from and to byte arrays.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
index aef87d9b9ba8c..c01379940235c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
@@ -17,6 +17,14 @@
 
 package org.apache.flink.streaming.util.typeutils;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
@@ -29,15 +37,6 @@
 import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase;
 import scala.Product;
 
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.lang.reflect.Array;
-import java.lang.reflect.Field;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
 
 /**
  * These classes encapsulate the logic of accessing a field specified by the user as either an index
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
index 1bfce5ba5468d..cca91216abb7d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
@@ -17,6 +17,9 @@
 
 package org.apache.flink.streaming.util.typeutils;
 
+import java.io.Serializable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.operators.Keys;
@@ -30,10 +33,6 @@
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TupleTypeInfoBase;
 
-import java.io.Serializable;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 
 /**
  * Static factories for the {@link FieldAccessor} utilities.
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index 3285e5cb5cb90..9473b8a561688 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -194,16 +194,16 @@ This file is based on the checkstyle file of Apache Beam.
                value="Redundant import {0}."/>
     </module>
 
-    <!--<module name="ImportOrder">-->
-      <!--&lt;!&ndash; Checks for out of order import statements. &ndash;&gt;-->
-      <!--<property name="severity" value="error"/>-->
-      <!--&lt;!&ndash; This ensures that static imports go first. &ndash;&gt;-->
-      <!--<property name="option" value="top"/>-->
-      <!--<property name="sortStaticImportsAlphabetically" value="true"/>-->
-      <!--<property name="tokens" value="STATIC_IMPORT, IMPORT"/>-->
-      <!--<message key="import.ordering"-->
-               <!--value="Import {0} appears after other imports that it should precede"/>-->
-    <!--</module>-->
+    <module name="ImportOrder">
+      <!-- Checks for out of order import statements. -->
+      <property name="severity" value="error"/>
+      <!-- This ensures that static imports go first. -->
+      <property name="option" value="top"/>
+      <property name="sortStaticImportsAlphabetically" value="true"/>
+      <property name="tokens" value="STATIC_IMPORT, IMPORT"/>
+      <message key="import.ordering"
+               value="Import {0} appears after other imports that it should precede"/>
+    </module>
 
     <module name="AvoidStarImport">
       <property name="severity" value="error"/>

From f3d69c1ba4d1927e2dd61b38fc730eff1cee413a Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 14:54:35 +0100
Subject: [PATCH 051/253] [FLINK-6107] Enable WhitespaceAfter check in
 streaming checkstyle

---
 .../api/datastream/IterativeStream.java       | 16 +++++++---
 .../streaming/api/datastream/KeyedStream.java |  2 +-
 .../functions/aggregation/SumAggregator.java  |  2 +-
 .../source/TimestampedFileInputSplit.java     |  2 +-
 .../streaming/api/graph/StreamConfig.java     |  2 +-
 .../flink/streaming/api/graph/StreamEdge.java |  2 +-
 .../flink/streaming/api/graph/StreamNode.java |  4 +--
 .../api/windowing/evictors/CountEvictor.java  |  6 ++--
 .../api/windowing/evictors/TimeEvictor.java   |  6 ++--
 .../windowing/EvictingWindowOperator.java     |  6 ++--
 .../runtime/operators/windowing/KeyMap.java   |  2 +-
 .../operators/windowing/WindowOperator.java   |  2 +-
 .../tasks/TestProcessingTimeService.java      |  2 +-
 .../util/typeutils/FieldAccessor.java         | 30 +++++++++----------
 .../util/typeutils/FieldAccessorFactory.java  |  2 +-
 tools/maven/strict-checkstyle.xml             | 12 ++++----
 16 files changed, 53 insertions(+), 45 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
index 75699ae13a39c..b0be71fb7ad75 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/IterativeStream.java
@@ -187,16 +187,24 @@ public DataStream<F> closeWith(DataStream<F> feedbackStream) {
 						"feedback streams instead.");
 
 		@Override
-		public ConnectedStreams<I, F> keyBy(int[] keyPositions1, int[] keyPositions2) {throw groupingException;}
+		public ConnectedStreams<I, F> keyBy(int[] keyPositions1, int[] keyPositions2) {
+			throw groupingException;
+		}
 
 		@Override
-		public ConnectedStreams<I, F> keyBy(String field1, String field2) {throw groupingException;}
+		public ConnectedStreams<I, F> keyBy(String field1, String field2) {
+			throw groupingException;
+		}
 
 		@Override
-		public ConnectedStreams<I, F> keyBy(String[] fields1, String[] fields2) {throw groupingException;}
+		public ConnectedStreams<I, F> keyBy(String[] fields1, String[] fields2) {
+			throw groupingException;
+		}
 
 		@Override
-		public ConnectedStreams<I, F> keyBy(KeySelector<I, ?> keySelector1,KeySelector<F, ?> keySelector2) {throw groupingException;}
+		public ConnectedStreams<I, F> keyBy(KeySelector<I, ?> keySelector1, KeySelector<F, ?> keySelector2) {
+			throw groupingException;
+		}
 
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
index c042f2fdb64c7..9334c660f6282 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
@@ -226,7 +226,7 @@ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) {
 	public <R> SingleOutputStreamOperator<R> transform(String operatorName,
 			TypeInformation<R> outTypeInfo, OneInputStreamOperator<T, R> operator) {
 
-		SingleOutputStreamOperator<R> returnStream = super.transform(operatorName, outTypeInfo,operator);
+		SingleOutputStreamOperator<R> returnStream = super.transform(operatorName, outTypeInfo, operator);
 
 		// inject the key selector and key type
 		OneInputTransformation<T, R> transform = (OneInputTransformation<T, R>) returnStream.getTransformation();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
index 86d91d0ce92e1..7fa5690ecb6e3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/aggregation/SumAggregator.java
@@ -67,7 +67,7 @@ public SumAggregator(String field, TypeInformation<T> typeInfo, ExecutionConfig
 	@SuppressWarnings("unchecked")
 	public T reduce(T value1, T value2) throws Exception {
 		if (isTuple) {
-			Tuple result = ((Tuple)value1).copy();
+			Tuple result = ((Tuple) value1).copy();
 			return fieldAccessor.set((T) result, adder.add(fieldAccessor.get(value1), fieldAccessor.get(value2)));
 		} else {
 			T result = serializer.copy(value1);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
index 9a679b9b7b98a..28432c5332e68 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
@@ -127,7 +127,7 @@ public boolean equals(Object o) {
 
 	@Override
 	public int hashCode() {
-		int res = 37 * (int)(this.modificationTime ^ (this.modificationTime >>> 32));
+		int res = 37 * (int) (this.modificationTime ^ (this.modificationTime >>> 32));
 		return 37 * res + super.hashCode();
 	}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
index f43b95ae6ca7d..d5ee13ac5ab06 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
@@ -411,7 +411,7 @@ public Map<Integer, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl
 	}
 
 	public void setOperatorName(String name) {
-		this.config.setString(OPERATOR_NAME,name);
+		this.config.setString(OPERATOR_NAME, name);
 	}
 
 	public String getOperatorName() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
index f7c1919879690..b82dc39c62ed3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
@@ -96,7 +96,7 @@ public List<String> getSelectedNames() {
 		return selectedNames;
 	}
 
-	public OutputTag getOutputTag() {return this.outputTag;}
+	public OutputTag getOutputTag() { return this.outputTag; }
 
 	public StreamPartitioner<?> getPartitioner() {
 		return outputPartitioner;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
index 4320e62879f5c..5c421aa5c9f11 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamNode.java
@@ -53,8 +53,8 @@ public class StreamNode implements Serializable {
 	private Long bufferTimeout = null;
 	private final String operatorName;
 	private String slotSharingGroup;
-	private KeySelector<?,?> statePartitioner1;
-	private KeySelector<?,?> statePartitioner2;
+	private KeySelector<?, ?> statePartitioner1;
+	private KeySelector<?, ?> statePartitioner2;
 	private TypeSerializer<?> stateKeySerializer;
 
 	private transient StreamOperator<?> operator;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
index 5f102867ef49b..53ef759767a2f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
@@ -34,7 +34,7 @@ public class CountEvictor<W extends Window> implements Evictor<Object, W> {
 	private final long maxCount;
 	private final boolean doEvictAfter;
 
-	private CountEvictor(long count,boolean doEvictAfter) {
+	private CountEvictor(long count, boolean doEvictAfter) {
 		this.maxCount = count;
 		this.doEvictAfter = doEvictAfter;
 	}
@@ -53,7 +53,7 @@ public void evictBefore(Iterable<TimestampedValue<Object>> elements, int size, W
 
 
 	@Override
-	public void evictAfter(Iterable<TimestampedValue<Object>> elements, int size,W window, EvictorContext ctx) {
+	public void evictAfter(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) {
 		if (doEvictAfter) {
 			evict(elements, size, ctx);
 		}
@@ -94,6 +94,6 @@ public static <W extends Window> CountEvictor<W> of(long maxCount) {
 	 * @param doEvictAfter Whether to do eviction after the window function.
      */
 	public static <W extends Window> CountEvictor<W> of(long maxCount, boolean doEvictAfter) {
-		return new CountEvictor<>(maxCount,doEvictAfter);
+		return new CountEvictor<>(maxCount, doEvictAfter);
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
index ba00403a05675..393a56b5c43d9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
@@ -52,14 +52,14 @@ public TimeEvictor(long windowSize, boolean doEvictAfter) {
 	@Override
 	public void evictBefore(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) {
 		if(!doEvictAfter) {
-			evict(elements,size,ctx);
+			evict(elements, size, ctx);
 		}
 	}
 
 	@Override
 	public void evictAfter(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) {
 		if(doEvictAfter) {
-			evict(elements,size,ctx);
+			evict(elements, size, ctx);
 		}
 	}
 
@@ -131,6 +131,6 @@ public static <W extends Window> TimeEvictor<W> of(Time windowSize) {
 	 * @param doEvictAfter Whether eviction is done after window function.
      */
 	public static <W extends Window> TimeEvictor<W> of(Time windowSize, boolean doEvictAfter) {
-		return new TimeEvictor<>(windowSize.toMilliseconds(),doEvictAfter);
+		return new TimeEvictor<>(windowSize.toMilliseconds(), doEvictAfter);
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
index edf4f25d5682b..3c98223f4bfaf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
@@ -410,11 +410,11 @@ public K getKey() {
 		}
 
 		void evictBefore(Iterable<TimestampedValue<IN>> elements, int size) {
-			evictor.evictBefore((Iterable)elements, size, window, this);
+			evictor.evictBefore((Iterable) elements, size, window, this);
 		}
 
 		void evictAfter(Iterable<TimestampedValue<IN>>  elements, int size) {
-			evictor.evictAfter((Iterable)elements, size, window, this);
+			evictor.evictAfter((Iterable) elements, size, window, this);
 		}
 	}
 
@@ -422,7 +422,7 @@ void evictAfter(Iterable<TimestampedValue<IN>>  elements, int size) {
 	public void open() throws Exception {
 		super.open();
 
-		evictorContext = new EvictorContext(null,null);
+		evictorContext = new EvictorContext(null, null);
 		evictingWindowState = (InternalListState<W, StreamRecord<IN>>)
 				getOrCreateKeyedState(windowSerializer, evictingWindowStateDescriptor);
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
index c69b6dce9701e..86cbee5e10d84 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
@@ -231,7 +231,7 @@ public V get(K key) {
 	}
 
 	private void insertNewEntry(int hashCode, K key, V value, int position) {
-		Entry<K,V> e = table[position];
+		Entry<K, V> e = table[position];
 		table[position] = new Entry<>(key, value, hashCode, e);
 		numElements++;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index 317fc9f6f4537..c87ac634d3df9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -274,7 +274,7 @@ public void open() throws Exception {
 				getInternalTimerService("window-timers", windowSerializer, this);
 
 		triggerContext = new Context(null, null);
-		processContext = new WindowContext( null);
+		processContext = new WindowContext(null);
 
 		windowAssignerContext = new WindowAssigner.WindowAssignerContext() {
 			@Override
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java
index 53b4dda565b64..7beba5cb2c7cf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TestProcessingTimeService.java
@@ -66,7 +66,7 @@ public void setCurrentTime(long timestamp) throws Exception {
 					callbackTask.onProcessingTime(entry.f0);
 
 					if (callbackTask instanceof PeriodicCallbackTask) {
-						priorityQueue.offer(Tuple2.of(((PeriodicCallbackTask)callbackTask).nextTimestamp(entry.f0), callbackTask));
+						priorityQueue.offer(Tuple2.of(((PeriodicCallbackTask) callbackTask).nextTimestamp(entry.f0), callbackTask));
 					}
 				}
 			}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
index c01379940235c..c087e7050c02a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
@@ -155,7 +155,7 @@ static final class SimpleTupleFieldAccessor<T extends Tuple, F> extends FieldAcc
 
 		SimpleTupleFieldAccessor(int pos, TypeInformation<T> typeInfo) {
 			checkNotNull(typeInfo, "typeInfo must not be null.");
-			int arity = ((TupleTypeInfo)typeInfo).getArity();
+			int arity = ((TupleTypeInfo) typeInfo).getArity();
 			if(pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
@@ -163,7 +163,7 @@ static final class SimpleTupleFieldAccessor<T extends Tuple, F> extends FieldAcc
 			}
 
 			this.pos = pos;
-			this.fieldType = ((TupleTypeInfo)typeInfo).getTypeAt(pos);
+			this.fieldType = ((TupleTypeInfo) typeInfo).getTypeAt(pos);
 		}
 
 		@SuppressWarnings("unchecked")
@@ -195,7 +195,7 @@ static final class RecursiveTupleFieldAccessor<T extends Tuple, R, F> extends Fi
 			checkNotNull(typeInfo, "typeInfo must not be null.");
 			checkNotNull(innerAccessor, "innerAccessor must not be null.");
 
-			int arity = ((TupleTypeInfo)typeInfo).getArity();
+			int arity = ((TupleTypeInfo) typeInfo).getArity();
 			if(pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
@@ -250,7 +250,7 @@ static final class PojoFieldAccessor<T, R, F> extends FieldAccessor<T, F> {
 		public F get(T pojo) {
 			try {
 				@SuppressWarnings("unchecked")
-				final R inner = (R)field.get(pojo);
+				final R inner = (R) field.get(pojo);
 				return innerAccessor.get(inner);
 			} catch (IllegalAccessException iaex) {
 				// The Field class is transient and when deserializing its value we also make it accessible
@@ -263,7 +263,7 @@ public F get(T pojo) {
 		public T set(T pojo, F valueToSet) {
 			try {
 				@SuppressWarnings("unchecked")
-				final R inner = (R)field.get(pojo);
+				final R inner = (R) field.get(pojo);
 				field.set(pojo, innerAccessor.set(inner, valueToSet));
 				return pojo;
 			} catch (IllegalAccessException iaex) {
@@ -301,7 +301,7 @@ static final class SimpleProductFieldAccessor<T, F> extends FieldAccessor<T, F>
 
 		SimpleProductFieldAccessor(int pos, TypeInformation<T> typeInfo, ExecutionConfig config) {
 			checkNotNull(typeInfo, "typeInfo must not be null.");
-			int arity = ((TupleTypeInfoBase)typeInfo).getArity();
+			int arity = ((TupleTypeInfoBase) typeInfo).getArity();
 			if(pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
@@ -309,8 +309,8 @@ static final class SimpleProductFieldAccessor<T, F> extends FieldAccessor<T, F>
 			}
 
 			this.pos = pos;
-			this.fieldType = ((TupleTypeInfoBase<T>)typeInfo).getTypeAt(pos);
-			this.serializer = (TupleSerializerBase<T>)typeInfo.createSerializer(config);
+			this.fieldType = ((TupleTypeInfoBase<T>) typeInfo).getTypeAt(pos);
+			this.serializer = (TupleSerializerBase<T>) typeInfo.createSerializer(config);
 			this.length = this.serializer.getArity();
 			this.fields = new Object[this.length];
 		}
@@ -318,13 +318,13 @@ static final class SimpleProductFieldAccessor<T, F> extends FieldAccessor<T, F>
 		@SuppressWarnings("unchecked")
 		@Override
 		public F get(T record) {
-			Product prod = (Product)record;
+			Product prod = (Product) record;
 			return (F) prod.productElement(pos);
 		}
 
 		@Override
 		public T set(T record, F fieldValue) {
-			Product prod = (Product)record;
+			Product prod = (Product) record;
 			for (int i = 0; i < length; i++) {
 				fields[i] = prod.productElement(i);
 			}
@@ -345,7 +345,7 @@ static final class RecursiveProductFieldAccessor<T, R, F> extends FieldAccessor<
 		private final FieldAccessor<R, F> innerAccessor;
 
 		RecursiveProductFieldAccessor(int pos, TypeInformation<T> typeInfo, FieldAccessor<R, F> innerAccessor, ExecutionConfig config) {
-			int arity = ((TupleTypeInfoBase)typeInfo).getArity();
+			int arity = ((TupleTypeInfoBase) typeInfo).getArity();
 			if(pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
@@ -355,8 +355,8 @@ static final class RecursiveProductFieldAccessor<T, R, F> extends FieldAccessor<
 			checkNotNull(innerAccessor, "innerAccessor must not be null.");
 
 			this.pos = pos;
-			this.fieldType = ((TupleTypeInfoBase<T>)typeInfo).getTypeAt(pos);
-			this.serializer = (TupleSerializerBase<T>)typeInfo.createSerializer(config);
+			this.fieldType = ((TupleTypeInfoBase<T>) typeInfo).getTypeAt(pos);
+			this.serializer = (TupleSerializerBase<T>) typeInfo.createSerializer(config);
 			this.length = this.serializer.getArity();
 			this.fields = new Object[this.length];
 			this.innerAccessor = innerAccessor;
@@ -365,7 +365,7 @@ static final class RecursiveProductFieldAccessor<T, R, F> extends FieldAccessor<
 		@SuppressWarnings("unchecked")
 		@Override
 		public F get(T record) {
-			return innerAccessor.get((R)((Product)record).productElement(pos));
+			return innerAccessor.get((R) ((Product) record).productElement(pos));
 		}
 
 		@SuppressWarnings("unchecked")
@@ -375,7 +375,7 @@ public T set(T record, F fieldValue) {
 			for (int i = 0; i < length; i++) {
 				fields[i] = prod.productElement(i);
 			}
-			fields[pos] = innerAccessor.set((R)fields[pos], fieldValue);
+			fields[pos] = innerAccessor.set((R) fields[pos], fieldValue);
 			return serializer.createInstance(fields);
 		}
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
index cca91216abb7d..893c640b2bf3e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
@@ -71,7 +71,7 @@ public static <T, F> FieldAccessor<T, F> getAccessor(TypeInformation<T> typeInfo
 		} else if (typeInfo.isTupleType() && ((TupleTypeInfoBase) typeInfo).isCaseClass()) {
 			TupleTypeInfoBase tupleTypeInfo = (TupleTypeInfoBase) typeInfo;
 			@SuppressWarnings("unchecked")
-			TypeInformation<F> fieldTypeInfo = (TypeInformation<F>)tupleTypeInfo.getTypeAt(pos);
+			TypeInformation<F> fieldTypeInfo = (TypeInformation<F>) tupleTypeInfo.getTypeAt(pos);
 			return new FieldAccessor.RecursiveProductFieldAccessor<>(
 				pos, typeInfo, new FieldAccessor.SimpleFieldAccessor<>(fieldTypeInfo), config);
 
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index 9473b8a561688..48cb5118402e5 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -493,12 +493,12 @@ This file is based on the checkstyle file of Apache Beam.
       <!--<property name="severity" value="error"/>-->
     <!--</module>-->
 
-    <!--<module name="WhitespaceAfter">-->
-      <!--&lt;!&ndash; Checks that commas, semicolons and typecasts are followed by-->
-           <!--whitespace.-->
-      <!--&ndash;&gt;-->
-      <!--<property name="tokens" value="COMMA, SEMI, TYPECAST"/>-->
-    <!--</module>-->
+    <module name="WhitespaceAfter">
+      <!-- Checks that commas, semicolons and typecasts are followed by
+           whitespace.
+      -->
+      <property name="tokens" value="COMMA, SEMI, TYPECAST"/>
+    </module>
 
     <module name="NoWhitespaceAfter">
       <!-- Checks that there is no whitespace after various unary operators.

From 999dabe16cf719f39e3530d4e1331e943f412cf2 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 15:02:00 +0100
Subject: [PATCH 052/253] [FLINK-6107] Enable WhitespaceAround check in
 streaming checkstyle

---
 .../api/datastream/StreamProjection.java      |  6 ++--
 .../StreamExecutionEnvironment.java           |  2 +-
 .../ContinuousFileMonitoringFunction.java     |  2 +-
 .../source/TimestampedFileInputSplit.java     |  4 +--
 ...undedOutOfOrdernessTimestampExtractor.java |  8 +++---
 .../streaming/api/graph/StreamGraph.java      |  2 +-
 .../api/operators/AbstractStreamOperator.java |  2 +-
 .../streaming/api/operators/StreamSource.java |  4 +--
 .../api/windowing/evictors/TimeEvictor.java   |  4 +--
 .../streaming/runtime/io/BufferSpiller.java   |  2 +-
 .../windowing/EvictingWindowOperator.java     |  2 +-
 .../operators/windowing/WindowOperator.java   |  2 +-
 .../runtime/tasks/OperatorChain.java          |  2 +-
 .../util/typeutils/FieldAccessor.java         | 12 ++++----
 .../util/typeutils/FieldAccessorFactory.java  | 10 +++----
 tools/maven/strict-checkstyle.xml             | 28 +++++++++----------
 16 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
index e1e56fda2422b..45e2db47a7763 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
@@ -63,15 +63,15 @@ protected StreamProjection(DataStream<IN> dataStream, int[] fieldIndexes) {
 		if (!dataStream.getType().isTupleType()) {
 			throw new RuntimeException("Only Tuple DataStreams can be projected");
 		}
-		if(fieldIndexes.length == 0) {
+		if (fieldIndexes.length == 0) {
 			throw new IllegalArgumentException("project() needs to select at least one (1) field.");
-		} else if(fieldIndexes.length > Tuple.MAX_ARITY - 1) {
+		} else if (fieldIndexes.length > Tuple.MAX_ARITY - 1) {
 			throw new IllegalArgumentException(
 					"project() may select only up to (" + (Tuple.MAX_ARITY - 1) + ") fields.");
 		}
 
 		int maxFieldIndex = (dataStream.getType()).getArity();
-		for(int i = 0; i < fieldIndexes.length; i++) {
+		for (int i = 0; i < fieldIndexes.length; i++) {
 			Preconditions.checkElementIndex(fieldIndexes[i], maxFieldIndex);
 		}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 7a6c7d6e94b20..8787e48f5102b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -1430,7 +1430,7 @@ public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, TypeI
 	@SuppressWarnings("unchecked")
 	public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, String sourceName, TypeInformation<OUT> typeInfo) {
 
-		if(typeInfo == null) {
+		if (typeInfo == null) {
 			if (function instanceof ResultTypeQueryable) {
 				typeInfo = ((ResultTypeQueryable<OUT>) function).getProducedType();
 			} else {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
index 1b7673394cf02..332d362717867 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/ContinuousFileMonitoringFunction.java
@@ -158,7 +158,7 @@ public void initializeState(FunctionInitializationContext context) throws Except
 				// The two should be mutually exclusive for the operator, thus we throw the exception.
 
 				throw new IllegalArgumentException(
-					"The " + getClass().getSimpleName() +" has already restored from a previous Flink version.");
+					"The " + getClass().getSimpleName() + " has already restored from a previous Flink version.");
 
 			} else if (retrievedStates.size() == 1) {
 				this.globalModificationTime = retrievedStates.get(0);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
index 28432c5332e68..ddc3559f97903 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/TimestampedFileInputSplit.java
@@ -59,7 +59,7 @@ public TimestampedFileInputSplit(long modificationTime, int num, Path file, long
 		super(num, file, start, length, hosts);
 
 		Preconditions.checkArgument(modificationTime >= 0 || modificationTime == Long.MIN_VALUE,
-			"Invalid File Split Modification Time: "+ modificationTime +".");
+			"Invalid File Split Modification Time: " + modificationTime + ".");
 
 		this.modificationTime = modificationTime;
 	}
@@ -133,7 +133,7 @@ public int hashCode() {
 
 	@Override
 	public String toString() {
-		return "[" + getSplitNumber() + "] " + getPath() +" mod@ "+
+		return "[" + getSplitNumber() + "] " + getPath() + " mod@ " +
 			modificationTime + " : " + getStart() + " + " + getLength();
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/BoundedOutOfOrdernessTimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/BoundedOutOfOrdernessTimestampExtractor.java
index cb1e2ca94e2ab..982c283c5dc06 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/BoundedOutOfOrdernessTimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/timestamps/BoundedOutOfOrdernessTimestampExtractor.java
@@ -45,9 +45,9 @@ public abstract class BoundedOutOfOrdernessTimestampExtractor<T> implements Assi
 	private final long maxOutOfOrderness;
 
 	public BoundedOutOfOrdernessTimestampExtractor(Time maxOutOfOrderness) {
-		if(maxOutOfOrderness.toMilliseconds() < 0) {
+		if (maxOutOfOrderness.toMilliseconds() < 0) {
 			throw new RuntimeException("Tried to set the maximum allowed " +
-				"lateness to "+ maxOutOfOrderness +". This parameter cannot be negative.");
+				"lateness to " + maxOutOfOrderness + ". This parameter cannot be negative.");
 		}
 		this.maxOutOfOrderness = maxOutOfOrderness.toMilliseconds();
 		this.currentMaxTimestamp = Long.MIN_VALUE + this.maxOutOfOrderness;
@@ -69,7 +69,7 @@ public long getMaxOutOfOrdernessInMillis() {
 	public final Watermark getCurrentWatermark() {
 		// this guarantees that the watermark never goes backwards.
 		long potentialWM = currentMaxTimestamp - maxOutOfOrderness;
-		if(potentialWM >= lastEmittedWatermark) {
+		if (potentialWM >= lastEmittedWatermark) {
 			lastEmittedWatermark = potentialWM;
 		}
 		return new Watermark(lastEmittedWatermark);
@@ -78,7 +78,7 @@ public final Watermark getCurrentWatermark() {
 	@Override
 	public final long extractTimestamp(T element, long previousElementTimestamp) {
 		long timestamp = extractTimestamp(element);
-		if(timestamp > currentMaxTimestamp) {
+		if (timestamp > currentMaxTimestamp) {
 			currentMaxTimestamp = timestamp;
 		}
 		return timestamp;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
index 11e40d23dfc34..5dd651c079381 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
@@ -159,7 +159,7 @@ public boolean isChainingEnabled() {
 
 
 	public boolean isIterative() {
-		return!vertexIDtoLoopTimeout.isEmpty();
+		return !vertexIDtoLoopTimeout.isEmpty();
 	}
 
 	public <IN, OUT> void addSource(Integer vertexID,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 72ab6dd531a90..1ef8614c5bf54 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -786,7 +786,7 @@ public Map<String, HashMap<String, Double>> getValue() {
 					// ConcurrentModificationExceptions. To avoid unnecessary blocking
 					// of the reportLatency() method, we retry this operation until
 					// it succeeds.
-				} catch(ConcurrentModificationException ignore) {
+				} catch (ConcurrentModificationException ignore) {
 					LOG.debug("Unable to report latency statistics", ignore);
 				}
 			}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
index 37050689eff14..b7f0d80514f73 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
@@ -63,7 +63,7 @@ public void run(final Object lockingObject,
 		final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic();
 
 		LatencyMarksEmitter latencyEmitter = null;
-		if(getExecutionConfig().isLatencyTrackingEnabled()) {
+		if (getExecutionConfig().isLatencyTrackingEnabled()) {
 			latencyEmitter = new LatencyMarksEmitter<>(
 				getProcessingTimeService(),
 				collector,
@@ -95,7 +95,7 @@ public void run(final Object lockingObject,
 		} finally {
 			// make sure that the context is closed in any case
 			ctx.close();
-			if(latencyEmitter != null) {
+			if (latencyEmitter != null) {
 				latencyEmitter.close();
 			}
 		}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
index 393a56b5c43d9..725b0bd614fad 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/TimeEvictor.java
@@ -51,14 +51,14 @@ public TimeEvictor(long windowSize, boolean doEvictAfter) {
 
 	@Override
 	public void evictBefore(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) {
-		if(!doEvictAfter) {
+		if (!doEvictAfter) {
 			evict(elements, size, ctx);
 		}
 	}
 
 	@Override
 	public void evictAfter(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) {
-		if(doEvictAfter) {
+		if (doEvictAfter) {
 			evict(elements, size, ctx);
 		}
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
index fc2e376cc444c..df60b757a21fa 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/BufferSpiller.java
@@ -255,7 +255,7 @@ FileChannel getCurrentChannel() {
 
 	@SuppressWarnings("resource")
 	private void createSpillingChannel() throws IOException {
-		currentSpillFile = new File(tempDir, spillFilePrefix + (fileCounter++) +".buffer");
+		currentSpillFile = new File(tempDir, spillFilePrefix + (fileCounter++) + ".buffer");
 		currentChannel = new RandomAccessFile(currentSpillFile, "rw").getChannel();
 	}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
index 3c98223f4bfaf..f9654e57abb22 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
@@ -354,7 +354,7 @@ public IN apply(TimestampedValue<IN> input) {
 		//work around to fix FLINK-4369, remove the evicted elements from the windowState.
 		//this is inefficient, but there is no other way to remove elements from ListState, which is an AppendingState.
 		windowState.clear();
-		for(TimestampedValue<IN> record : recordsWithTimestamp) {
+		for (TimestampedValue<IN> record : recordsWithTimestamp) {
 			windowState.add(record.getStreamRecord());
 		}
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index c87ac634d3df9..8d8f6b54bb7c9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -1202,7 +1202,7 @@ private void restoreFromLegacyWindowOperator(DataInputViewStreamWrapper in) thro
 
 		// just to read all the rest, although we do not really use this information.
 		int numProcessingTimeTimerTimestamp = in.readInt();
-		for (int i = 0; i< numProcessingTimeTimerTimestamp; i++) {
+		for (int i = 0; i < numProcessingTimeTimerTimestamp; i++) {
 			in.readLong();
 			in.readInt();
 		}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index d96a3868e0d69..98fe1d0f1ff04 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -561,7 +561,7 @@ public void emitWatermark(Watermark mark) {
 		public void emitLatencyMarker(LatencyMarker latencyMarker) {
 			if (outputs.length <= 0) {
 				// ignore
-			} else if(outputs.length == 1) {
+			} else if (outputs.length == 1) {
 				outputs[0].emitLatencyMarker(latencyMarker);
 			} else {
 				// randomly select an output
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
index c087e7050c02a..90c89acba9629 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessor.java
@@ -120,7 +120,7 @@ static final class ArrayFieldAccessor<T, F> extends FieldAccessor<T, F> {
 		private final int pos;
 
 		public ArrayFieldAccessor(int pos, TypeInformation typeInfo) {
-			if(pos < 0) {
+			if (pos < 0) {
 				throw new CompositeType.InvalidFieldReferenceException("The " + ((Integer) pos).toString() + ". field selected on" +
 					" an array, which is an invalid index.");
 			}
@@ -156,7 +156,7 @@ static final class SimpleTupleFieldAccessor<T extends Tuple, F> extends FieldAcc
 		SimpleTupleFieldAccessor(int pos, TypeInformation<T> typeInfo) {
 			checkNotNull(typeInfo, "typeInfo must not be null.");
 			int arity = ((TupleTypeInfo) typeInfo).getArity();
-			if(pos < 0 || pos >= arity) {
+			if (pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
 					typeInfo.toString() + "\", which is an invalid index.");
@@ -196,13 +196,13 @@ static final class RecursiveTupleFieldAccessor<T extends Tuple, R, F> extends Fi
 			checkNotNull(innerAccessor, "innerAccessor must not be null.");
 
 			int arity = ((TupleTypeInfo) typeInfo).getArity();
-			if(pos < 0 || pos >= arity) {
+			if (pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
 						typeInfo.toString() + "\", which is an invalid index.");
 			}
 
-			if(pos < 0) {
+			if (pos < 0) {
 				throw new CompositeType.InvalidFieldReferenceException("Tried to select " + ((Integer) pos).toString() + ". field.");
 			}
 
@@ -302,7 +302,7 @@ static final class SimpleProductFieldAccessor<T, F> extends FieldAccessor<T, F>
 		SimpleProductFieldAccessor(int pos, TypeInformation<T> typeInfo, ExecutionConfig config) {
 			checkNotNull(typeInfo, "typeInfo must not be null.");
 			int arity = ((TupleTypeInfoBase) typeInfo).getArity();
-			if(pos < 0 || pos >= arity) {
+			if (pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
 						typeInfo.toString() + "\", which is an invalid index.");
@@ -346,7 +346,7 @@ static final class RecursiveProductFieldAccessor<T, R, F> extends FieldAccessor<
 
 		RecursiveProductFieldAccessor(int pos, TypeInformation<T> typeInfo, FieldAccessor<R, F> innerAccessor, ExecutionConfig config) {
 			int arity = ((TupleTypeInfoBase) typeInfo).getArity();
-			if(pos < 0 || pos >= arity) {
+			if (pos < 0 || pos >= arity) {
 				throw new CompositeType.InvalidFieldReferenceException(
 					"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
 						typeInfo.toString() + "\", which is an invalid index.");
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
index 893c640b2bf3e..c46422f6adf41 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/typeutils/FieldAccessorFactory.java
@@ -198,21 +198,21 @@ public static <T, F> FieldAccessor<T, F> getAccessor(TypeInformation<T> typeInfo
 	// --------------------------------------------------------------------------------------------------
 
 	private  static final String REGEX_FIELD = "[\\p{L}\\p{Digit}_\\$]*"; // This can start with a digit (because of Tuples)
-	private static final String REGEX_NESTED_FIELDS = "("+REGEX_FIELD+")(\\.(.+))?";
+	private static final String REGEX_NESTED_FIELDS = "(" + REGEX_FIELD + ")(\\.(.+))?";
 	private static final String REGEX_NESTED_FIELDS_WILDCARD = REGEX_NESTED_FIELDS
-		+"|\\"+ Keys.ExpressionKeys.SELECT_ALL_CHAR
-		+"|\\"+ Keys.ExpressionKeys.SELECT_ALL_CHAR_SCALA;
+		+ "|\\" + Keys.ExpressionKeys.SELECT_ALL_CHAR
+		+ "|\\" + Keys.ExpressionKeys.SELECT_ALL_CHAR_SCALA;
 
 	private static final Pattern PATTERN_NESTED_FIELDS_WILDCARD = Pattern.compile(REGEX_NESTED_FIELDS_WILDCARD);
 
 	private static FieldExpression decomposeFieldExpression(String fieldExpression) {
 		Matcher matcher = PATTERN_NESTED_FIELDS_WILDCARD.matcher(fieldExpression);
 		if (!matcher.matches()) {
-			throw new CompositeType.InvalidFieldReferenceException("Invalid field expression \""+fieldExpression+"\".");
+			throw new CompositeType.InvalidFieldReferenceException("Invalid field expression \"" + fieldExpression + "\".");
 		}
 
 		String head = matcher.group(0);
-		if(head.equals(Keys.ExpressionKeys.SELECT_ALL_CHAR) || head.equals(Keys.ExpressionKeys.SELECT_ALL_CHAR_SCALA)) {
+		if (head.equals(Keys.ExpressionKeys.SELECT_ALL_CHAR) || head.equals(Keys.ExpressionKeys.SELECT_ALL_CHAR_SCALA)) {
 			throw new CompositeType.InvalidFieldReferenceException("No wildcards are allowed here.");
 		} else {
 			head = matcher.group(1);
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index 48cb5118402e5..e89311ddfd722 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -478,20 +478,20 @@ This file is based on the checkstyle file of Apache Beam.
 
     -->
 
-    <!--<module name="WhitespaceAround">-->
-      <!--&lt;!&ndash; Checks that various tokens are surrounded by whitespace.-->
-           <!--This includes most binary operators and keywords followed-->
-           <!--by regular or curly braces.-->
-      <!--&ndash;&gt;-->
-      <!--<property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR,-->
-        <!--BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN,-->
-        <!--EQUAL, GE, GT, LAND, LE, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,-->
-        <!--LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,-->
-        <!--LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS,-->
-        <!--MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION,-->
-        <!--SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN"/>-->
-      <!--<property name="severity" value="error"/>-->
-    <!--</module>-->
+    <module name="WhitespaceAround">
+      <!-- Checks that various tokens are surrounded by whitespace.
+           This includes most binary operators and keywords followed
+           by regular or curly braces.
+      -->
+      <property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR,
+        BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN,
+        EQUAL, GE, GT, LAND, LE, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+        LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+        LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS,
+        MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION,
+        SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN"/>
+      <property name="severity" value="error"/>
+    </module>
 
     <module name="WhitespaceAfter">
       <!-- Checks that commas, semicolons and typecasts are followed by

From b8956d96aac301c0193cb7c0de9b22de2e0e093f Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 15:29:02 +0100
Subject: [PATCH 053/253] [FLINK-6107] Enable MemberNameCheck in streaming
 checkstyle

---
 .../collector/selector/DirectedOutput.java    |  4 ++--
 .../runtime/tasks/OperatorChain.java          |  4 ++--
 tools/maven/strict-checkstyle.xml             | 20 +++++++++----------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
index d806e9dfb6dec..dcb4b59e13e09 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
@@ -49,7 +49,7 @@ public class DirectedOutput<OUT> implements Output<StreamRecord<OUT>> {
 
 	protected final Output<StreamRecord<OUT>>[] allOutputs;
 
-	private final Random RNG = new XORShiftRandom();
+	private final Random random = new XORShiftRandom();
 
 
 	@SuppressWarnings({"unchecked", "rawtypes"})
@@ -112,7 +112,7 @@ public void emitWatermark(Watermark mark) {
 	@Override
 	public void emitLatencyMarker(LatencyMarker latencyMarker) {
 		// randomly select an output
-		allOutputs[RNG.nextInt(allOutputs.length)].emitLatencyMarker(latencyMarker);
+		allOutputs[random.nextInt(allOutputs.length)].emitLatencyMarker(latencyMarker);
 	}
 
 	protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index 98fe1d0f1ff04..dd784309e773a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -537,7 +537,7 @@ private static class BroadcastingOutputCollector<T> implements Output<StreamReco
 
 		protected final Output<StreamRecord<T>>[] outputs;
 
-		private final Random RNG = new XORShiftRandom();
+		private final Random random = new XORShiftRandom();
 
 		private final StreamStatusProvider streamStatusProvider;
 
@@ -565,7 +565,7 @@ public void emitLatencyMarker(LatencyMarker latencyMarker) {
 				outputs[0].emitLatencyMarker(latencyMarker);
 			} else {
 				// randomly select an output
-				outputs[RNG.nextInt(outputs.length)].emitLatencyMarker(latencyMarker);
+				outputs[random.nextInt(outputs.length)].emitLatencyMarker(latencyMarker);
 			}
 		}
 
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index e89311ddfd722..2ce5644e1b823 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -327,16 +327,16 @@ This file is based on the checkstyle file of Apache Beam.
       <!--<property name="severity" value="error"/>-->
     <!--</module>-->
 
-    <!--<module name="MemberNameCheck">-->
-      <!--&lt;!&ndash; Validates non-static members against the supplied expression. &ndash;&gt;-->
-      <!--<metadata name="altname" value="MemberName"/>-->
-      <!--<property name="applyToPublic" value="true"/>-->
-      <!--<property name="applyToProtected" value="true"/>-->
-      <!--<property name="applyToPackage" value="true"/>-->
-      <!--<property name="applyToPrivate" value="true"/>-->
-      <!--<property name="format" value="^[a-z][a-zA-Z0-9]*$"/>-->
-      <!--<property name="severity" value="error"/>-->
-    <!--</module>-->
+    <module name="MemberNameCheck">
+      <!-- Validates non-static members against the supplied expression. -->
+      <metadata name="altname" value="MemberName"/>
+      <property name="applyToPublic" value="true"/>
+      <property name="applyToProtected" value="true"/>
+      <property name="applyToPackage" value="true"/>
+      <property name="applyToPrivate" value="true"/>
+      <property name="format" value="^[a-z][a-zA-Z0-9]*$"/>
+      <property name="severity" value="error"/>
+    </module>
 
     <module name="MethodNameCheck">
       <!-- Validates identifiers for method names. -->

From 03a2ce9dc9fa8220a4a57eaf94a0a704157059f9 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 15:30:23 +0100
Subject: [PATCH 054/253] [FLINK-6107] Enable StaticVariableNameCheck in
 streaming checkstyle

---
 .../api/windowing/windows/GlobalWindow.java   |  2 +-
 tools/maven/strict-checkstyle.xml             | 22 +++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
index e1db976c74e86..b1e06f5e8d105 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
@@ -30,7 +30,7 @@
 @PublicEvolving
 public class GlobalWindow extends Window {
 
-	private static GlobalWindow INSTANCE = new GlobalWindow();
+	private static final GlobalWindow INSTANCE = new GlobalWindow();
 
 	private GlobalWindow() { }
 
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index 2ce5644e1b823..459dc0b5b6841 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -315,17 +315,17 @@ This file is based on the checkstyle file of Apache Beam.
       <property name="severity" value="error"/>
     </module>
 
-    <!--<module name="StaticVariableNameCheck">-->
-      <!--&lt;!&ndash; Validates static, non-final fields against the supplied-->
-      <!--expression "^[a-z][a-zA-Z0-9]*_?$". &ndash;&gt;-->
-      <!--<metadata name="altname" value="StaticVariableName"/>-->
-      <!--<property name="applyToPublic" value="true"/>-->
-      <!--<property name="applyToProtected" value="true"/>-->
-      <!--<property name="applyToPackage" value="true"/>-->
-      <!--<property name="applyToPrivate" value="true"/>-->
-      <!--<property name="format" value="^[a-z][a-zA-Z0-9]*_?$"/>-->
-      <!--<property name="severity" value="error"/>-->
-    <!--</module>-->
+    <module name="StaticVariableNameCheck">
+      <!-- Validates static, non-final fields against the supplied
+      expression "^[a-z][a-zA-Z0-9]*_?$". -->
+      <metadata name="altname" value="StaticVariableName"/>
+      <property name="applyToPublic" value="true"/>
+      <property name="applyToProtected" value="true"/>
+      <property name="applyToPackage" value="true"/>
+      <property name="applyToPrivate" value="true"/>
+      <property name="format" value="^[a-z][a-zA-Z0-9]*_?$"/>
+      <property name="severity" value="error"/>
+    </module>
 
     <module name="MemberNameCheck">
       <!-- Validates non-static members against the supplied expression. -->

From b013ef7121ba733135a35421e42827b50aeeb576 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Sun, 19 Mar 2017 15:25:24 +0100
Subject: [PATCH 055/253] [FLINK-6107] Enable LeftCurly check in streaming
 checkstyle

---
 .../streaming/api/collector/selector/DirectedOutput.java | 3 +--
 .../api/environment/StreamExecutionEnvironment.java      | 6 ++----
 .../streaming/api/functions/sink/SocketClientSink.java   | 3 +--
 .../org/apache/flink/streaming/api/graph/StreamEdge.java | 4 +++-
 .../flink/streaming/api/operators/StreamGroupedFold.java | 3 +--
 .../AbstractAlignedProcessingTimeWindowOperator.java     | 3 +--
 .../operators/windowing/AbstractKeyedTimePanes.java      | 6 ++----
 .../operators/windowing/AccumulatingKeyedTimePanes.java  | 3 +--
 .../AccumulatingProcessingTimeWindowOperator.java        | 3 +--
 .../AggregatingProcessingTimeWindowOperator.java         | 3 +--
 .../streaming/runtime/operators/windowing/KeyMap.java    | 9 +++------
 .../flink/streaming/runtime/tasks/OperatorChain.java     | 6 ++----
 .../runtime/tasks/SystemProcessingTimeService.java       | 3 +--
 .../flink/streaming/util/keys/KeySelectorUtil.java       | 3 +--
 tools/maven/strict-checkstyle.xml                        | 8 ++++----
 15 files changed, 25 insertions(+), 41 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
index dcb4b59e13e09..660a0778945a2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
@@ -55,8 +55,7 @@ public class DirectedOutput<OUT> implements Output<StreamRecord<OUT>> {
 	@SuppressWarnings({"unchecked", "rawtypes"})
 	public DirectedOutput(
 			List<OutputSelector<OUT>> outputSelectors,
-			List<Tuple2<Output<StreamRecord<OUT>>, StreamEdge>> outputs)
-	{
+			List<Tuple2<Output<StreamRecord<OUT>>, StreamEdge>> outputs) {
 		this.outputSelectors = outputSelectors.toArray(new OutputSelector[outputSelectors.size()]);
 
 		this.allOutputs = new Output[outputs.size()];
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 8787e48f5102b..e8279454dfc7f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -1709,8 +1709,7 @@ public static StreamExecutionEnvironment createRemoteEnvironment(
 	 * @return A remote environment that executes the program on a cluster.
 	 */
 	public static StreamExecutionEnvironment createRemoteEnvironment(
-			String host, int port, int parallelism, String... jarFiles)
-	{
+			String host, int port, int parallelism, String... jarFiles) {
 		RemoteStreamEnvironment env = new RemoteStreamEnvironment(host, port, jarFiles);
 		env.setParallelism(parallelism);
 		return env;
@@ -1738,8 +1737,7 @@ public static StreamExecutionEnvironment createRemoteEnvironment(
 	 * @return A remote environment that executes the program on a cluster.
 	 */
 	public static StreamExecutionEnvironment createRemoteEnvironment(
-			String host, int port, Configuration clientConfig, String... jarFiles)
-	{
+			String host, int port, Configuration clientConfig, String... jarFiles) {
 		return new RemoteStreamEnvironment(host, port, clientConfig, jarFiles);
 	}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
index 44a96773f0e6b..88cd4df7f4d43 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/SocketClientSink.java
@@ -101,8 +101,7 @@ public SocketClientSink(String hostName, int port, SerializationSchema<IN> schem
 	 * @param autoflush Flag to indicate whether the socket stream should be flushed after each message.
 	 */
 	public SocketClientSink(String hostName, int port, SerializationSchema<IN> schema,
-							int maxNumRetries, boolean autoflush)
-	{
+							int maxNumRetries, boolean autoflush) {
 		checkArgument(port > 0 && port < 65536, "port is out of range");
 		checkArgument(maxNumRetries >= -1, "maxNumRetries must be zero or larger (num retries), or -1 (infinite retries)");
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
index b82dc39c62ed3..95c80a80b3650 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamEdge.java
@@ -96,7 +96,9 @@ public List<String> getSelectedNames() {
 		return selectedNames;
 	}
 
-	public OutputTag getOutputTag() { return this.outputTag; }
+	public OutputTag getOutputTag() {
+		return this.outputTag;
+	}
 
 	public StreamPartitioner<?> getPartitioner() {
 		return outputPartitioner;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
index 3e4493078527f..1ed717812a497 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
@@ -70,8 +70,7 @@ public void open() throws Exception {
 		}
 
 		try (ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
-			DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais))
-		{
+			DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {
 			initialValue = outTypeSerializer.deserialize(in);
 		}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
index 515f3922507e7..21e9e6961084f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
@@ -84,8 +84,7 @@ protected AbstractAlignedProcessingTimeWindowOperator(
 			TypeSerializer<KEY> keySerializer,
 			TypeSerializer<STATE> stateTypeSerializer,
 			long windowLength,
-			long windowSlide)
-	{
+			long windowSlide) {
 		super(function);
 
 		if (windowLength < MIN_SLIDE_TIME) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
index 680fd5de4630a..b6affa6e7bfff 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractKeyedTimePanes.java
@@ -101,8 +101,7 @@ protected void traverseAllPanes(KeyMap.TraversalEvaluator<Key, Aggregate> traver
 	public void writeToOutput(
 			final DataOutputView output,
 			final TypeSerializer<Key> keySerializer,
-			final TypeSerializer<Aggregate> aggSerializer) throws IOException
-	{
+			final TypeSerializer<Aggregate> aggSerializer) throws IOException {
 		output.writeInt(BEGIN_OF_STATE_MAGIC_NUMBER);
 
 		int numPanes = getNumPanes();
@@ -125,8 +124,7 @@ public void writeToOutput(
 	public void readFromInput(
 			final DataInputView input,
 			final TypeSerializer<Key> keySerializer,
-			final TypeSerializer<Aggregate> aggSerializer) throws IOException
-	{
+			final TypeSerializer<Aggregate> aggSerializer) throws IOException {
 		validateMagicNumber(BEGIN_OF_STATE_MAGIC_NUMBER, input.readInt());
 		int numPanes = input.readInt();
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
index 1dd5a818c1b94..55ba1e342c4f1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
@@ -78,8 +78,7 @@ public void addElementToLatestPane(Type element) throws Exception {
 
 	@Override
 	public void evaluateWindow(Collector<Result> out, final TimeWindow window,
-								AbstractStreamOperator<Result> operator) throws Exception
-	{
+								AbstractStreamOperator<Result> operator) throws Exception {
 		if (previousPanes.isEmpty()) {
 			// optimized path for single pane case (tumbling window)
 			for (KeyMap.Entry<Key, ArrayList<Type>> entry : latestPane) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
index 830d480bd188c..19474ecafbbbf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
@@ -49,8 +49,7 @@ public AccumulatingProcessingTimeWindowOperator(
 			TypeSerializer<KEY> keySerializer,
 			TypeSerializer<IN> valueSerializer,
 			long windowLength,
-			long windowSlide)
-	{
+			long windowSlide) {
 		super(function, keySelector, keySerializer,
 				new ArrayListSerializer<>(valueSerializer), windowLength, windowSlide);
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
index b83815c5ba4d2..724f3060d23f8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
@@ -45,8 +45,7 @@ public AggregatingProcessingTimeWindowOperator(
 			TypeSerializer<KEY> keySerializer,
 			TypeSerializer<IN> aggregateSerializer,
 			long windowLength,
-			long windowSlide)
-	{
+			long windowSlide) {
 		super(function, keySelector, keySerializer, aggregateSerializer, windowLength, windowSlide);
 	}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
index 86cbee5e10d84..2fd182571527c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
@@ -499,8 +499,7 @@ public static <K, V> void traverseMaps(
 					final KeyMap<K, V>[] maps,
 					final TraversalEvaluator<K, V> visitor,
 					final long touchedTag)
-		throws Exception
-	{
+		throws Exception {
 		// we need to work on the maps in descending size
 		Arrays.sort(maps, CapacityDescendingComparator.INSTANCE);
 
@@ -524,8 +523,7 @@ public static <K, V> void traverseMaps(
 			int mask;
 			for (int rootTable = 0;
 					rootTable < numTables && ((mask = lowBitsMask[rootTable]) & pos) == mask;
-					rootTable++)
-			{
+					rootTable++) {
 				// use that table to gather keys and start collecting keys from the following tables
 				// go over all entries of that slot in the table
 				Entry<K, V> entry = maps[rootTable].table[pos >> shifts[rootTable]];
@@ -563,8 +561,7 @@ private static <K, V> void addEntriesFromChain(
 			TraversalEvaluator<K, V> visitor,
 			K key,
 			long touchedTag,
-			int hashCode) throws Exception
-	{
+			int hashCode) throws Exception {
 		while (entry != null) {
 			if (entry.touchedTag < touchedTag && entry.hashCode == hashCode && entry.key.equals(key)) {
 				entry.touchedTag = touchedTag;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index dd784309e773a..b85461d956c6a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -253,8 +253,7 @@ private <T> Output<StreamRecord<T>> createOutputCollector(
 			Map<Integer, StreamConfig> chainedConfigs,
 			ClassLoader userCodeClassloader,
 			Map<StreamEdge, RecordWriterOutput<?>> streamOutputs,
-			List<StreamOperator<?>> allOperators)
-	{
+			List<StreamOperator<?>> allOperators) {
 		List<Tuple2<Output<StreamRecord<T>>, StreamEdge>> allOutputs = new ArrayList<>(4);
 
 		// create collectors for the network outputs
@@ -326,8 +325,7 @@ private <IN, OUT> Output<StreamRecord<IN>> createChainedOperator(
 			ClassLoader userCodeClassloader,
 			Map<StreamEdge, RecordWriterOutput<?>> streamOutputs,
 			List<StreamOperator<?>> allOperators,
-			OutputTag<IN> outputTag)
-	{
+			OutputTag<IN> outputTag) {
 		// create the output that the operator writes to first. this may recursively create more operators
 		Output<StreamRecord<OUT>> output = createOutputCollector(
 				containingTask, operatorConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperators);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
index 2cb509af056e7..139c61976ec65 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeService.java
@@ -169,8 +169,7 @@ public void quiesceAndAwaitPending() throws InterruptedException {
 	@Override
 	public void shutdownService() {
 		if (status.compareAndSet(STATUS_ALIVE, STATUS_SHUTDOWN) ||
-				status.compareAndSet(STATUS_QUIESCED, STATUS_SHUTDOWN))
-		{
+				status.compareAndSet(STATUS_QUIESCED, STATUS_SHUTDOWN)) {
 			timerService.shutdownNow();
 		}
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
index 0e476346cbd34..e148a0b1b3410 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
@@ -91,8 +91,7 @@ else if (typeInfo instanceof PrimitiveArrayTypeInfo) {
 
 
 	public static <X, K> KeySelector<X, K> getSelectorForOneKey(
-			Keys<X> keys, Partitioner<K> partitioner, TypeInformation<X> typeInfo, ExecutionConfig executionConfig)
-	{
+			Keys<X> keys, Partitioner<K> partitioner, TypeInformation<X> typeInfo, ExecutionConfig executionConfig) {
 		if (!(typeInfo instanceof CompositeType)) {
 			throw new InvalidTypesException(
 					"This key operation requires a composite type such as Tuples, POJOs, case classes, etc");
diff --git a/tools/maven/strict-checkstyle.xml b/tools/maven/strict-checkstyle.xml
index 459dc0b5b6841..9c589176b7542 100644
--- a/tools/maven/strict-checkstyle.xml
+++ b/tools/maven/strict-checkstyle.xml
@@ -402,10 +402,10 @@ This file is based on the checkstyle file of Apache Beam.
           <!--value="^(package .*;\s*)|(import .*;\s*)|( *\* .*https?://.*)$"/>-->
     <!--</module>-->
 
-    <!--<module name="LeftCurly">-->
-      <!--&lt;!&ndash; Checks for placement of the left curly brace ('{'). &ndash;&gt;-->
-      <!--<property name="severity" value="error"/>-->
-    <!--</module>-->
+    <module name="LeftCurly">
+      <!-- Checks for placement of the left curly brace ('{'). -->
+      <property name="severity" value="error"/>
+    </module>
 
     <!--<module name="RightCurly">-->
       <!--&lt;!&ndash; Checks right curlies on CATCH, ELSE, and TRY blocks are on-->

From a5bd82d5d55367fe959036d48629748274443b13 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Wed, 22 Mar 2017 13:58:01 +0100
Subject: [PATCH 056/253] [FLINK-6107] Add checkstyle section to IDE Setup
 Guide

---
 docs/internals/ide_setup.md | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/internals/ide_setup.md b/docs/internals/ide_setup.md
index 329f4cd476204..33abf6118e31c 100644
--- a/docs/internals/ide_setup.md
+++ b/docs/internals/ide_setup.md
@@ -81,6 +81,39 @@ to enable support for Scala projects and files:
    files for the IDE to work with but without installing libraries.
 8. Build the Project (Build -> Make Project)
 
+### Checkstyle
+IntelliJ supports checkstyle within the IDE using the Checkstyle-IDEA plugin.
+
+1. Install the "Checkstyle-IDEA" plugin from the IntelliJ plugin repository.
+1. Configure the plugin by going to Settings -> Other Settings -> Checkstyle.
+1. Set the "Scan Scope" to "Only Java sources (but not tests)".
+1. In the "Configuration File" pane, add a new configuration using the plus icon:
+    1. Set the "Description" to "Flink".
+    1. Select "Use a local Checkstyle file", and point it to
+      `"tools/maven/checkstyle.xml"` within
+      your repository.
+    1. Check the box for "Store relative to project location", and click
+      "Next".
+    1. Configure the "checkstyle.suppressions.file" property value to
+      `"suppressions.xml"`, and click "Next", then "Finish".
+1. Select "Flink" as the only active configuration file, and click "Apply" and
+   "OK".
+1. Checkstyle will now give warnings in the editor for any Checkstyle
+   violations.
+
+You can also scan an entire module by opening the Checkstyle tools window and
+clicking the "Check Module" button. The scan should report no errors.
+
+<span class="label label-info">Note</span> Selecting "Check Project" may report some errors from the archetype
+modules as they are not configured for Checkstyle validation.
+
+<span class="label label-info">Note</span> Some modules use a more strict `checkstyle.xml` file called
+`strict-checkstyle.xml`. You should setup an additional configuration by following the above
+steps for this file as well and activate it on a per-module basis. To set a per-module checkstyle
+configuration go to File -> Project Structure. Then select the module for which you want to change
+the checkstyle configuration in the modules list and change the checkstyle configuration in the
+Checkstyle tab.
+
 ## Eclipse
 
 **NOTE:** From our experience, this setup does not work with Flink

From 4026398312aaf95bc8e9991769c1023f3fc462ef Mon Sep 17 00:00:00 2001
From: kl0u <kkloudas@gmail.com>
Date: Tue, 4 Apr 2017 17:45:34 +0200
Subject: [PATCH 057/253] [FLINK-6248] [cep] Make the optional() available to
 all offered patterns.

Allow all patterns to be optional in a pattern sequence.
Singleton becomes simply optional, times(X) transforms from
"exactly X" to "0 or X", and "one or more" becomes "zero or more".
---
 .../flink/cep/scala/pattern/Pattern.scala     | 286 +++++-----
 .../flink/cep/nfa/compiler/NFACompiler.java   |  42 +-
 .../org/apache/flink/cep/pattern/Pattern.java | 271 ++++-----
 .../apache/flink/cep/pattern/Quantifier.java  |  90 ++-
 .../org/apache/flink/cep/nfa/NFAITCase.java   | 521 ++++++++++++++----
 .../cep/nfa/compiler/NFACompilerTest.java     |   2 +-
 .../apache/flink/cep/pattern/PatternTest.java |   8 +-
 7 files changed, 762 insertions(+), 458 deletions(-)

diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
index c636029ba8888..3214d09234f6d 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
@@ -20,7 +20,7 @@ package org.apache.flink.cep.scala.pattern
 import org.apache.flink.cep
 import org.apache.flink.cep.pattern.conditions.IterativeCondition
 import org.apache.flink.cep.pattern.conditions.IterativeCondition.Context
-import org.apache.flink.cep.pattern.{Quantifier, Pattern => JPattern}
+import org.apache.flink.cep.pattern.{MalformedPatternException, Quantifier, Pattern => JPattern}
 import org.apache.flink.streaming.api.windowing.time.Time
 
 /**
@@ -31,9 +31,8 @@ import org.apache.flink.streaming.api.windowing.time.Time
   *
   * {{{
   * Pattern<T, F> pattern = Pattern.<T>begin("start")
-  * .next("middle").subtype(F.class)
-  * .followedBy("end").where(new MyFilterFunction());
-  * }
+  *   .next("middle").subtype(F.class)
+  *   .followedBy("end").where(new MyCondition());
   * }}}
   *
   * @param jPattern Underlying Java API Pattern
@@ -44,6 +43,13 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
 
   private[flink] def wrappedPattern = jPattern
 
+  /**
+    * @return The previous pattern
+    */
+  def getPrevious(): Option[Pattern[T, _ <: T]] = {
+    wrapPattern(jPattern.getPrevious())
+  }
+
   /**
     *
     * @return Name of the pattern operator
@@ -64,170 +70,170 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
     */
   def getQuantifier: Quantifier = jPattern.getQuantifier
 
-  /**
-    *
-    * @return Filter condition for an event to be matched
-    */
   def getCondition(): Option[IterativeCondition[F]] = {
     Option(jPattern.getCondition())
   }
 
   /**
-    * Applies a subtype constraint on the current pattern operator. This means that an event has
-    * to be of the given subtype in order to be matched.
+    * Adds a condition that has to be satisfied by an event
+    * in order to be considered a match. If another condition has already been
+    * set, the new one is going to be combined with the previous with a
+    * logical {{{AND}}}. In other case, this is going to be the only
+    * condition.
     *
-    * @param clazz Class of the subtype
-    * @tparam S Type of the subtype
-    * @return The same pattern operator with the new subtype constraint
+    * @param condition The condition as an [[IterativeCondition]].
+    * @return The pattern with the new condition is set.
     */
-  def subtype[S <: F](clazz: Class[S]): Pattern[T, S] = {
-    jPattern.subtype(clazz)
-    this.asInstanceOf[Pattern[T, S]]
-  }
-
-  /**
-    * Defines the maximum time interval for a matching pattern. This means that the time gap
-    * between first and the last event must not be longer than the window time.
-    *
-    * @param windowTime Time of the matching window
-    * @return The same pattern operator with the new window length
-    */
-  def within(windowTime: Time): Pattern[T, F] = {
-    jPattern.within(windowTime)
+  def where(condition: IterativeCondition[F]): Pattern[T, F] = {
+    jPattern.where(condition)
     this
   }
 
   /**
-    * Appends a new pattern operator to the existing one. The new pattern operator enforces strict
-    * temporal contiguity. This means that the whole pattern only matches if an event which matches
-    * this operator directly follows the preceding matching event. Thus, there cannot be any
-    * events in between two matching events.
+    * Adds a condition that has to be satisfied by an event
+    * in order to be considered a match. If another condition has already been
+    * set, the new one is going to be combined with the previous with a
+    * logical {{{AND}}}. In other case, this is going to be the only
+    * condition.
     *
-    * @param name Name of the new pattern operator
-    * @return A new pattern operator which is appended to this pattern operator
+    * @param condition The condition to be set.
+    * @return The pattern with the new condition is set.
     */
-  def next(name: String): Pattern[T, T] = {
-    Pattern[T, T](jPattern.next(name))
-  }
+  def where(condition: (F, Context[F]) => Boolean): Pattern[T, F] = {
+    val condFun = new IterativeCondition[F] {
+      val cleanCond = cep.scala.cleanClosure(condition)
 
-  /**
-    * Appends a new pattern operator to the existing one. The new pattern operator enforces
-    * non-strict temporal contiguity. This means that a matching event of this operator and the
-    * preceding matching event might be interleaved with other events which are ignored.
-    *
-    * @param name Name of the new pattern operator
-    * @return A new pattern operator which is appended to this pattern operator
-    */
-  def followedBy(name: String): FollowedByPattern[T, T] = {
-    FollowedByPattern(jPattern.followedBy(name))
+      override def filter(value: F, ctx: Context[F]): Boolean = cleanCond(value, ctx)
+    }
+    where(condFun)
   }
 
   /**
-    * Specifies a filter condition which has to be fulfilled by an event in order to be matched.
+    * Adds a condition that has to be satisfied by an event
+    * in order to be considered a match. If another condition has already been
+    * set, the new one is going to be combined with the previous with a
+    * logical {{{AND}}}. In other case, this is going to be the only
+    * condition.
     *
-    * @param filter Filter condition
-    * @return The same pattern operator where the new filter condition is set
+    * @param condition The condition to be set.
+    * @return The pattern with the new condition is set.
     */
-  def where(filter: IterativeCondition[F]): Pattern[T, F] = {
-    jPattern.where(filter)
-    this
+  def where(condition: F => Boolean): Pattern[T, F] = {
+    val condFun = new IterativeCondition[F] {
+      val cleanCond = cep.scala.cleanClosure(condition)
+
+      override def filter(value: F, ctx: Context[F]): Boolean = cleanCond(value)
+    }
+    where(condFun)
   }
 
   /**
-    * Specifies a filter condition which is ORed with an existing filter function.
+    * Adds a condition that has to be satisfied by an event
+    * in order to be considered a match. If another condition has already been
+    * set, the new one is going to be combined with the previous with a
+    * logical {{{OR}}}. In other case, this is going to be the only
+    * condition.
     *
-    * @param filter Or filter function
-    * @return The same pattern operator where the new filter condition is set
+    * @param condition The condition as an [[IterativeCondition]].
+    * @return The pattern with the new condition is set.
     */
-  def or(filter: IterativeCondition[F]): Pattern[T, F] = {
-    jPattern.or(filter)
+  def or(condition: IterativeCondition[F]): Pattern[T, F] = {
+    jPattern.or(condition)
     this
   }
 
   /**
-    * Specifies a filter condition which is ORed with an existing filter function.
+    * Adds a condition that has to be satisfied by an event
+    * in order to be considered a match. If another condition has already been
+    * set, the new one is going to be combined with the previous with a
+    * logical {{{OR}}}. In other case, this is going to be the only
+    * condition.
     *
-    * @param filterFun Or filter function
-    * @return The same pattern operator where the new filter condition is set
+    * @param condition The {{{OR}}} condition.
+    * @return The pattern with the new condition is set.
     */
-  def or(filterFun: (F, Context[F]) => Boolean): Pattern[T, F] = {
-    val filter = new IterativeCondition[F] {
-      val cleanFilter = cep.scala.cleanClosure(filterFun)
+  def or(condition: (F, Context[F]) => Boolean): Pattern[T, F] = {
+    val condFun = new IterativeCondition[F] {
+      val cleanCond = cep.scala.cleanClosure(condition)
 
-      override def filter(value: F, ctx: Context[F]): Boolean = cleanFilter(value, ctx)
+      override def filter(value: F, ctx: Context[F]): Boolean = cleanCond(value, ctx)
     }
-    or(filter)
+    or(condFun)
   }
 
   /**
-    * Specifies a filter condition which has to be fulfilled by an event in order to be matched.
+    * Applies a subtype constraint on the current pattern. This means that an event has
+    * to be of the given subtype in order to be matched.
     *
-    * @param filterFun Filter condition
-    * @return The same pattern operator where the new filter condition is set
+    * @param clazz Class of the subtype
+    * @tparam S Type of the subtype
+    * @return The same pattern with the new subtype constraint
     */
-  def where(filterFun: (F, Context[F]) => Boolean): Pattern[T, F] = {
-    val filter = new IterativeCondition[F] {
-      val cleanFilter = cep.scala.cleanClosure(filterFun)
-
-      override def filter(value: F, ctx: Context[F]): Boolean = cleanFilter(value, ctx)
-    }
-    where(filter)
+  def subtype[S <: F](clazz: Class[S]): Pattern[T, S] = {
+    jPattern.subtype(clazz)
+    this.asInstanceOf[Pattern[T, S]]
   }
 
   /**
-    * Specifies a filter condition which has to be fulfilled by an event in order to be matched.
+    * Defines the maximum time interval in which a matching pattern has to be completed in
+    * order to be considered valid. This interval corresponds to the maximum time gap between first
+    * and the last event.
     *
-    * @param filterFun Filter condition
-    * @return The same pattern operator where the new filter condition is set
+    * @param windowTime Time of the matching window
+    * @return The same pattern operator with the new window length
     */
-  def where(filterFun: F => Boolean): Pattern[T, F] = {
-    val filter = new IterativeCondition[F] {
-      val cleanFilter = cep.scala.cleanClosure(filterFun)
-
-      override def filter(value: F, ctx: Context[F]): Boolean = cleanFilter(value)
-    }
-    where(filter)
+  def within(windowTime: Time): Pattern[T, F] = {
+    jPattern.within(windowTime)
+    this
   }
 
   /**
+    * Appends a new pattern to the existing one. The new pattern enforces strict
+    * temporal contiguity. This means that the whole pattern sequence matches only
+    * if an event which matches this pattern directly follows the preceding matching
+    * event. Thus, there cannot be any events in between two matching events.
     *
-    * @return The previous pattern operator
+    * @param name Name of the new pattern
+    * @return A new pattern which is appended to this one
     */
-  def getPrevious(): Option[Pattern[T, _ <: T]] = {
-    wrapPattern(jPattern.getPrevious())
+  def next(name: String): Pattern[T, T] = {
+    Pattern[T, T](jPattern.next(name))
   }
 
   /**
-    * Specifies that this pattern can occur zero or more times(kleene star).
-    * This means any number of events can be matched in this state.
+    * Appends a new pattern to the existing one. The new pattern enforces non-strict
+    * temporal contiguity. This means that a matching event of this pattern and the
+    * preceding matching event might be interleaved with other events which are ignored.
     *
-    * @return The same pattern with applied Kleene star operator
+    * @param name Name of the new pattern
+    * @return A new pattern which is appended to this one
     */
-  def zeroOrMore: Pattern[T, F] = {
-    jPattern.zeroOrMore()
-    this
-  }
+  def followedBy(name: String) = FollowedByPattern(jPattern.followedBy(name))
 
   /**
-    * Specifies that this pattern can occur zero or more times(kleene star).
-    * This means any number of events can be matched in this state.
-    *
-    * If eagerness is enabled for a pattern A*B and sequence A1 A2 B will generate patterns:
-    * B, A1 B and A1 A2 B. If disabled B, A1 B, A2 B and A1 A2 B.
+    * Specifies that this pattern is optional for a final match of the pattern
+    * sequence to happen.
     *
-    * @param eager if true the pattern always consumes earlier events
-    * @return The same pattern with applied Kleene star operator
+    * @return The same pattern as optional.
+    * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
     */
-  def zeroOrMore(eager: Boolean): Pattern[T, F] = {
-    jPattern.zeroOrMore(eager)
+  def optional: Pattern[T, F] = {
+    jPattern.optional()
     this
   }
 
   /**
-    * Specifies that this pattern can occur one or more times(kleene star).
-    * This means at least one and at most infinite number of events can be matched in this state.
+    * Specifies that this pattern can occur {{{one or more}}} times.
+    * This means at least one and at most infinite number of events can
+    * be matched to this pattern.
+    *
+    * If this quantifier is enabled for a
+    * pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of events
+    * {{{A1 A2 B}}} appears, this will generate patterns:
+    * {{{A1 B}}} and {{{A1 A2 B}}}. See also {{{allowCombinations()}}}.
     *
-    * @return The same pattern with applied Kleene plus operator
+    * @return The same pattern with a [[Quantifier.ONE_OR_MORE()]] quantifier applied.
+    * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
     */
   def oneOrMore: Pattern[T, F] = {
     jPattern.oneOrMore()
@@ -235,63 +241,51 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
   }
 
   /**
-    * Specifies that this pattern can occur one or more times(kleene star).
-    * This means at least one and at most infinite number of events can be matched in this state.
-    *
-    * If eagerness is enabled for a pattern A+B and sequence A1 A2 B will generate patterns:
-    * A1 B and A1 A2 B. If disabled A1 B, A2 B and A1 A2 B.
+    * Specifies exact number of times that this pattern should be matched.
     *
-    * @param eager if true the pattern always consumes earlier events
-    * @return The same pattern with applied Kleene plus operator
+    * @param times number of times matching event must appear
+    * @return The same pattern with number of times applied
+    * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
     */
-  def oneOrMore(eager: Boolean): Pattern[T, F] = {
-    jPattern.oneOrMore(eager)
+  def times(times: Int): Pattern[T, F] = {
+    jPattern.times(times)
     this
   }
 
   /**
-    * Specifies that this pattern can occur zero or once.
+    * Applicable only to [[Quantifier.ONE_OR_MORE()]] patterns, this option
+    * allows more flexibility to the matching events.
     *
-    * @return The same pattern with applied Kleene ? operator
-    */
-  def optional: Pattern[T, F] = {
-    jPattern.optional()
-    this
-  }
-
-  /**
-    * Specifies exact number of times that this pattern should be matched.
+    * If {{{allowCombinations()}}} is not applied for a
+    * pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of events
+    * {{{A1 A2 B}}} appears, this will generate patterns:
+    * {{{A1 B}}} and {{{A1 A2 B}}}. If this method is applied, we
+    * will have {{{A1 B}}}, {{{A2 B}}} and {{{A1 A2 B}}}.
     *
-    * @param times number of times matching event must appear
-    * @return The same pattern with number of times applied
+    * @return The same pattern with the updated quantifier.
+    * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
     */
-  def times(times: Int): Pattern[T, F] = {
-    jPattern.times(times)
+  def allowCombinations(): Pattern[T, F] = {
+    jPattern.allowCombinations()
     this
   }
 
-
   /**
-    * Works in conjunction with [[org.apache.flink.cep.scala.pattern.Pattern#zeroOrMore()]],
-    * [[org.apache.flink.cep.scala.pattern.Pattern#oneOrMore()]] or
-    * [[org.apache.flink.cep.scala.pattern.Pattern#times(int)]].
+    * Works in conjunction with [[Pattern#oneOrMore()]] or [[Pattern#times(int)]].
     * Specifies that any not matching element breaks the loop.
     *
-    * <p>E.g. a pattern like:
+    * E.g. a pattern like:
     * {{{
     * Pattern.begin("start").where(_.getName().equals("c"))
-    *        .followedBy("middle").where(_.getName().equals("a")).oneOrMore(true).consecutive()
+    *        .followedBy("middle").where(_.getName().equals("a")).oneOrMore().consecutive()
     *        .followedBy("end1").where(_.getName().equals("b"));
     * }}}
     *
-    * <p>for a sequence: C D A1 A2 A3 D A4 B
-    *
-    * <p>will generate matches: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}
+    * For a sequence: C D A1 A2 A3 D A4 B
     *
-    * <p><b>NOTICE:</b> This operator can be applied only when either zeroOrMore,
-    * oneOrMore or times was previously applied!
+    * will generate matches: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}
     *
-    * <p>By default a relaxed continuity is applied.
+    * By default a relaxed continuity is applied.
     * @return pattern with continuity changed to strict
     */
   def consecutive(): Pattern[T, F] = {
@@ -314,12 +308,12 @@ object Pattern {
   def apply[T, F <: T](jPattern: JPattern[T, F]) = new Pattern[T, F](jPattern)
 
   /**
-    * Starts a new pattern with the initial pattern operator whose name is provided. Furthermore,
-    * the base type of the event sequence is set.
+    * Starts a new pattern sequence. The provided name is the one of the initial pattern
+    * of the new sequence. Furthermore, the base type of the event sequence is set.
     *
-    * @param name Name of the new pattern operator
+    * @param name The name of starting pattern of the new pattern sequence
     * @tparam X Base type of the event pattern
-    * @return The first pattern operator of a pattern
+    * @return The first pattern of a pattern sequence
     */
   def begin[X](name: String): Pattern[X, X] = Pattern(JPattern.begin(name))
 
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
index 5ab44c0199ea8..add1583150132 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
@@ -31,7 +31,6 @@
 import org.apache.flink.cep.pattern.MalformedPatternException;
 import org.apache.flink.cep.pattern.conditions.NotCondition;
 import org.apache.flink.cep.pattern.Pattern;
-import org.apache.flink.cep.pattern.Quantifier;
 import org.apache.flink.cep.pattern.Quantifier.QuantifierProperty;
 import org.apache.flink.cep.pattern.conditions.IterativeCondition;
 import org.apache.flink.streaming.api.windowing.time.Time;
@@ -166,10 +165,10 @@ private State<T> createMiddleStates(final State<T> sinkState) {
 				if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.LOOPING)) {
 					final State<T> looping = createLooping(lastSink);
 
-					if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.AT_LEAST_ONE)) {
+					if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL)) {
 						lastSink = createFirstMandatoryStateOfLoop(looping);
 					} else if (currentPattern instanceof FollowedByPattern &&
-								currentPattern.getQuantifier().hasProperty(QuantifierProperty.STRICT)) {
+								currentPattern.getQuantifier().hasProperty(QuantifierProperty.CONSECUTIVE)) {
 						lastSink = createWaitingStateForZeroOrMore(looping, lastSink);
 					} else {
 						lastSink = looping;
@@ -237,7 +236,7 @@ private State<T> createStartState(State<T> sinkState) {
 			final State<T> beginningState;
 			if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.LOOPING)) {
 				final State<T> loopingState = createLooping(sinkState);
-				if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.AT_LEAST_ONE)) {
+				if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL)) {
 					beginningState = createFirstMandatoryStateOfLoop(loopingState);
 				} else {
 					beginningState = loopingState;
@@ -266,9 +265,29 @@ private State<T> createTimesState(final State<T> sinkState, int times) {
 			for (int i = 0; i < times - 1; i++) {
 				lastSink = createSingletonState(
 					lastSink,
-					!currentPattern.getQuantifier().hasProperty(QuantifierProperty.STRICT));
+					!currentPattern.getQuantifier().hasProperty(QuantifierProperty.CONSECUTIVE),
+					false);
 			}
-			return createSingletonState(lastSink, currentPattern instanceof FollowedByPattern);
+
+			// we created the intermediate states in the loop, now we create the start of the loop.
+			if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL)) {
+				return createSingletonState(lastSink, currentPattern instanceof FollowedByPattern, false);
+			}
+
+			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
+			final IterativeCondition<T> trueFunction = BooleanConditions.trueFunction();
+
+			final State<T> singletonState = createNormalState();
+			singletonState.addTake(lastSink, currentFilterFunction);
+			singletonState.addProceed(sinkState, trueFunction);
+
+			if (currentPattern instanceof FollowedByPattern) {
+				State<T> ignoreState = createNormalState();
+				ignoreState.addTake(lastSink, currentFilterFunction);
+				ignoreState.addIgnore(BooleanConditions.<T>trueFunction());
+				singletonState.addIgnore(ignoreState, trueFunction);
+			}
+			return singletonState;
 		}
 
 		/**
@@ -281,7 +300,8 @@ private State<T> createTimesState(final State<T> sinkState, int times) {
 		 */
 		@SuppressWarnings("unchecked")
 		private State<T> createSingletonState(final State<T> sinkState) {
-			return createSingletonState(sinkState, currentPattern instanceof FollowedByPattern);
+			return createSingletonState(sinkState, currentPattern instanceof FollowedByPattern,
+					currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL));
 		}
 
 		/**
@@ -294,20 +314,20 @@ private State<T> createSingletonState(final State<T> sinkState) {
 		 * @return the created state
 		 */
 		@SuppressWarnings("unchecked")
-		private State<T> createSingletonState(final State<T> sinkState, boolean addIgnore) {
+		private State<T> createSingletonState(final State<T> sinkState, boolean addIgnore, boolean isOptional) {
 			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
 			final IterativeCondition<T> trueFunction = BooleanConditions.trueFunction();
 
 			final State<T> singletonState = createNormalState();
 			singletonState.addTake(sinkState, currentFilterFunction);
 
-			if (currentPattern.getQuantifier() == Quantifier.OPTIONAL) {
+			if (isOptional) {
 				singletonState.addProceed(sinkState, trueFunction);
 			}
 
 			if (addIgnore) {
 				final State<T> ignoreState;
-				if (currentPattern.getQuantifier() == Quantifier.OPTIONAL) {
+				if (isOptional) {
 					ignoreState = createNormalState();
 					ignoreState.addTake(sinkState, currentFilterFunction);
 				} else {
@@ -356,7 +376,7 @@ private State<T> createLooping(final State<T> sinkState) {
 
 			loopingState.addProceed(sinkState, trueFunction);
 			loopingState.addTake(filterFunction);
-			if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.STRICT)) {
+			if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.CONSECUTIVE)) {
 				final State<T> ignoreState = createNormalState();
 
 				final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
index 92b655e090744..f1d0d631e46bd 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
@@ -30,13 +30,12 @@
 /**
  * Base class for a pattern definition.
  * <p>
- * A pattern definition is used by {@link org.apache.flink.cep.nfa.compiler.NFACompiler} to create
- * a {@link NFA}.
+ * A pattern definition is used by {@link org.apache.flink.cep.nfa.compiler.NFACompiler} to create a {@link NFA}.
  *
  * <pre>{@code
  * Pattern<T, F> pattern = Pattern.<T>begin("start")
  *   .next("middle").subtype(F.class)
- *   .followedBy("end").where(new MyFilterFunction());
+ *   .followedBy("end").where(new MyCondition());
  * }
  * </pre>
  *
@@ -45,20 +44,25 @@
  */
 public class Pattern<T, F extends T> {
 
-	// name of the pattern operator
+	/** Name of the pattern. */
 	private final String name;
 
-	// previous pattern operator
+	/** Previous pattern. */
 	private final Pattern<T, ? extends T> previous;
 
-	// filter condition for an event to be matched
+	/** The condition an event has to satisfy to be considered a matched. */
 	private IterativeCondition<F> condition;
 
-	// window length in which the pattern match has to occur
+	/** Window length in which the pattern match has to occur. */
 	private Time windowTime;
 
-	private Quantifier quantifier = Quantifier.ONE;
+	/** A quantifier for the pattern. By default set to {@link Quantifier#ONE()}. */
+	private Quantifier quantifier = Quantifier.ONE();
 
+	/**
+	 * Applicable to a {@code times} pattern, and holds
+	 * the number of times it has to appear.
+	 */
 	private int times;
 
 	protected Pattern(final String name, final Pattern<T, ? extends T> previous) {
@@ -66,16 +70,16 @@ protected Pattern(final String name, final Pattern<T, ? extends T> previous) {
 		this.previous = previous;
 	}
 
-	public String getName() {
-		return name;
-	}
-
 	public Pattern<T, ? extends T> getPrevious() {
 		return previous;
 	}
 
-	public IterativeCondition<F> getCondition() {
-		return condition;
+	public int getTimes() {
+		return times;
+	}
+
+	public String getName() {
+		return name;
 	}
 
 	public Time getWindowTime() {
@@ -86,15 +90,31 @@ public Quantifier getQuantifier() {
 		return quantifier;
 	}
 
-	public int getTimes() {
-		return times;
+	public IterativeCondition<F> getCondition() {
+		return condition;
 	}
 
 	/**
-	 * Specifies a filter condition which has to be fulfilled by an event in order to be matched.
+	 * Starts a new pattern sequence. The provided name is the one of the initial pattern
+	 * of the new sequence. Furthermore, the base type of the event sequence is set.
 	 *
-	 * @param condition Filter condition
-	 * @return The same pattern operator where the new filter condition is set
+	 * @param name The name of starting pattern of the new pattern sequence
+	 * @param <X> Base type of the event pattern
+	 * @return The first pattern of a pattern sequence
+	 */
+	public static <X> Pattern<X, X> begin(final String name) {
+		return new Pattern<X, X>(name, null);
+	}
+
+	/**
+	 * Adds a condition that has to be satisfied by an event
+	 * in order to be considered a match. If another condition has already been
+	 * set, the new one is going to be combined with the previous with a
+	 * logical {@code AND}. In other case, this is going to be the only
+	 * condition.
+	 *
+	 * @param condition The condition as an {@link IterativeCondition}.
+	 * @return The pattern with the new condition is set.
 	 */
 	public Pattern<T, F> where(IterativeCondition<F> condition) {
 		ClosureCleaner.clean(condition, true);
@@ -108,10 +128,14 @@ public Pattern<T, F> where(IterativeCondition<F> condition) {
 	}
 
 	/**
-	 * Specifies a filter condition which is OR'ed with an existing filter function.
+	 * Adds a condition that has to be satisfied by an event
+	 * in order to be considered a match. If another condition has already been
+	 * set, the new one is going to be combined with the previous with a
+	 * logical {@code OR}. In other case, this is going to be the only
+	 * condition.
 	 *
-	 * @param condition OR filter condition
-	 * @return The same pattern operator where the new filter condition is set
+	 * @param condition The condition as an {@link IterativeCondition}.
+	 * @return The pattern with the new condition is set.
 	 */
 	public Pattern<T, F> or(IterativeCondition<F> condition) {
 		ClosureCleaner.clean(condition, true);
@@ -125,19 +149,18 @@ public Pattern<T, F> or(IterativeCondition<F> condition) {
 	}
 
 	/**
-	 * Applies a subtype constraint on the current pattern operator. This means that an event has
+	 * Applies a subtype constraint on the current pattern. This means that an event has
 	 * to be of the given subtype in order to be matched.
 	 *
 	 * @param subtypeClass Class of the subtype
 	 * @param <S> Type of the subtype
-	 * @return The same pattern operator with the new subtype constraint
+	 * @return The same pattern with the new subtype constraint
 	 */
 	public <S extends F> Pattern<T, S> subtype(final Class<S> subtypeClass) {
 		if (condition == null) {
 			this.condition = new SubtypeCondition<F>(subtypeClass);
 		} else {
-			this.condition = new AndCondition<>(this.condition,
-					new SubtypeCondition<F>(subtypeClass));
+			this.condition = new AndCondition<>(condition, new SubtypeCondition<F>(subtypeClass));
 		}
 
 		@SuppressWarnings("unchecked")
@@ -147,8 +170,9 @@ public <S extends F> Pattern<T, S> subtype(final Class<S> subtypeClass) {
 	}
 
 	/**
-	 * Defines the maximum time interval for a matching pattern. This means that the time gap
-	 * between first and the last event must not be longer than the window time.
+	 * Defines the maximum time interval in which a matching pattern has to be completed in
+	 * order to be considered valid. This interval corresponds to the maximum time gap between first
+	 * and the last event.
 	 *
 	 * @param windowTime Time of the matching window
 	 * @return The same pattern operator with the new window length
@@ -162,130 +186,114 @@ public Pattern<T, F> within(Time windowTime) {
 	}
 
 	/**
-	 * Appends a new pattern operator to the existing one. The new pattern operator enforces strict
-	 * temporal contiguity. This means that the whole pattern only matches if an event which matches
-	 * this operator directly follows the preceding matching event. Thus, there cannot be any
-	 * events in between two matching events.
+	 * Appends a new pattern to the existing one. The new pattern enforces strict
+	 * temporal contiguity. This means that the whole pattern sequence matches only
+	 * if an event which matches this pattern directly follows the preceding matching
+	 * event. Thus, there cannot be any events in between two matching events.
 	 *
-	 * @param name Name of the new pattern operator
-	 * @return A new pattern operator which is appended to this pattern operator
+	 * @param name Name of the new pattern
+	 * @return A new pattern which is appended to this one
 	 */
 	public Pattern<T, T> next(final String name) {
 		return new Pattern<T, T>(name, this);
 	}
 
 	/**
-	 * Appends a new pattern operator to the existing one. The new pattern operator enforces
-	 * non-strict temporal contiguity. This means that a matching event of this operator and the
+	 * Appends a new pattern to the existing one. The new pattern enforces non-strict
+	 * temporal contiguity. This means that a matching event of this pattern and the
 	 * preceding matching event might be interleaved with other events which are ignored.
 	 *
-	 * @param name Name of the new pattern operator
-	 * @return A new pattern operator which is appended to this pattern operator
+	 * @param name Name of the new pattern
+	 * @return A new pattern which is appended to this one
 	 */
 	public FollowedByPattern<T, T> followedBy(final String name) {
 		return new FollowedByPattern<T, T>(name, this);
 	}
 
 	/**
-	 * Starts a new pattern with the initial pattern operator whose name is provided. Furthermore,
-	 * the base type of the event sequence is set.
-	 *
-	 * @param name Name of the new pattern operator
-	 * @param <X> Base type of the event pattern
-	 * @return The first pattern operator of a pattern
-	 */
-	public static <X> Pattern<X, X> begin(final String name) {
-		return new Pattern<X, X>(name, null);
-	}
-
-	/**
-	 * Specifies that this pattern can occur zero or more times(kleene star).
-	 * This means any number of events can be matched in this state.
+	 * Specifies that this pattern is optional for a final match of the pattern
+	 * sequence to happen.
 	 *
-	 * @return The same pattern with applied Kleene star operator
-	 *
-	 * @throws MalformedPatternException if quantifier already applied
+	 * @return The same pattern as optional.
+	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
-	public Pattern<T, F> zeroOrMore() {
-		return zeroOrMore(true);
+	public Pattern<T, F> optional() {
+		quantifier.makeOptional();
+		return this;
 	}
 
 	/**
-	 * Specifies that this pattern can occur zero or more times(kleene star).
-	 * This means any number of events can be matched in this state.
-	 *
-	 * If eagerness is enabled for a pattern A*B and sequence A1 A2 B will generate patterns:
-	 * B, A1 B and A1 A2 B. If disabled B, A1 B, A2 B and A1 A2 B.
+	 * Specifies that this pattern can occur {@code one or more} times.
+	 * This means at least one and at most infinite number of events can
+	 * be matched to this pattern.
 	 *
-	 * @param eager if true the pattern always consumes earlier events
-	 * @return The same pattern with applied Kleene star operator
+	 * <p>If this quantifier is enabled for a
+	 * pattern {@code A.oneOrMore().followedBy(B)} and a sequence of events
+	 * {@code A1 A2 B} appears, this will generate patterns:
+	 * {@code A1 B} and {@code A1 A2 B}. See also {@link #allowCombinations()}.
 	 *
-	 * @throws MalformedPatternException if quantifier already applied
+	 * @return The same pattern with a {@link Quantifier#ONE_OR_MORE()} quantifier applied.
+	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
-	public Pattern<T, F> zeroOrMore(final boolean eager) {
+	public Pattern<T, F> oneOrMore() {
 		checkIfQuantifierApplied();
-		if (eager) {
-			this.quantifier = Quantifier.ZERO_OR_MORE_EAGER;
-		} else {
-			this.quantifier = Quantifier.ZERO_OR_MORE_COMBINATIONS;
-		}
+		this.quantifier = Quantifier.ONE_OR_MORE();
 		return this;
 	}
 
 	/**
-	 * Specifies that this pattern can occur one or more times(kleene star).
-	 * This means at least one and at most infinite number of events can be matched in this state.
+	 * Specifies exact number of times that this pattern should be matched.
 	 *
-	 * @return The same pattern with applied Kleene plus operator
+	 * @param times number of times matching event must appear
+	 * @return The same pattern with number of times applied
 	 *
-	 * @throws MalformedPatternException if quantifier already applied
+	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
-	public Pattern<T, F> oneOrMore() {
-		return oneOrMore(true);
+	public Pattern<T, F> times(int times) {
+		checkIfQuantifierApplied();
+		Preconditions.checkArgument(times > 0, "You should give a positive number greater than 0.");
+		this.quantifier = Quantifier.TIMES();
+		this.times = times;
+		return this;
 	}
 
 	/**
-	 * Specifies that this pattern can occur one or more times(kleene star).
-	 * This means at least one and at most infinite number of events can be matched in this state.
+	 * Applicable only to {@link Quantifier#ONE_OR_MORE()} patterns, this option
+	 * allows more flexibility to the matching events.
 	 *
-	 * If eagerness is enabled for a pattern A+B and sequence A1 A2 B will generate patterns:
-	 * A1 B and A1 A2 B. If disabled A1 B, A2 B and A1 A2 B.
+	 * <p>If {@code allowCombinations()} is not applied for a
+	 * pattern {@code A.oneOrMore().followedBy(B)} and a sequence of events
+	 * {@code A1 A2 B} appears, this will generate patterns:
+	 * {@code A1 B} and {@code A1 A2 B}. If this method is applied, we
+	 * will have {@code A1 B}, {@code A2 B} and {@code A1 A2 B}.
 	 *
-	 * @param eager if true the pattern always consumes earlier events
-	 * @return The same pattern with applied Kleene plus operator
-	 *
-	 * @throws MalformedPatternException if quantifier already applied
+	 * @return The same pattern with the updated quantifier.	 *
+	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
-	public Pattern<T, F> oneOrMore(final boolean eager) {
-		checkIfQuantifierApplied();
-		if (eager) {
-			this.quantifier = Quantifier.ONE_OR_MORE_EAGER;
-		} else {
-			this.quantifier = Quantifier.ONE_OR_MORE_COMBINATIONS;
-		}
+	public Pattern<T, F> allowCombinations() {
+		quantifier.allowAllCombinations();
 		return this;
 	}
 
 	/**
-	 * Works in conjunction with {@link Pattern#zeroOrMore()}, {@link Pattern#oneOrMore()} or {@link Pattern#times(int)}.
+	 * Works in conjunction with {@link Pattern#oneOrMore()} or {@link Pattern#times(int)}.
 	 * Specifies that any not matching element breaks the loop.
 	 *
 	 * <p>E.g. a pattern like:
 	 * <pre>{@code
-	 * Pattern.<Event>begin("start").where(new FilterFunction<Event>() {
+	 * Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
 	 *      @Override
 	 *      public boolean filter(Event value) throws Exception {
 	 *          return value.getName().equals("c");
 	 *      }
 	 * })
-	 * .followedBy("middle").where(new FilterFunction<Event>() {
+	 * .followedBy("middle").where(new SimpleCondition<Event>() {
 	 *      @Override
 	 *      public boolean filter(Event value) throws Exception {
 	 *          return value.getName().equals("a");
 	 *      }
-	 * })
-	 * }<b>.oneOrMore(true).consecutive()</b>{@code
-	 * .followedBy("end1").where(new FilterFunction<Event>() {
+	 * }).oneOrMore().consecutive()
+	 * .followedBy("end1").where(new SimpleCondition<Event>() {
 	 *      @Override
 	 *      public boolean filter(Event value) throws Exception {
 	 *          return value.getName().equals("b");
@@ -297,78 +305,19 @@ public Pattern<T, F> oneOrMore(final boolean eager) {
 	 *
 	 * <p>will generate matches: {C A1 B}, {C A1 A2 B}, {C A1 A2 A3 B}
 	 *
-	 * <p><b>NOTICE:</b> This operator can be applied only when either zeroOrMore,
-	 * oneOrMore or times was previously applied!
-	 *
 	 * <p>By default a relaxed continuity is applied.
 	 *
 	 * @return pattern with continuity changed to strict
 	 */
 	public Pattern<T, F> consecutive() {
-		switch (this.quantifier) {
-
-			case ZERO_OR_MORE_EAGER:
-				this.quantifier = Quantifier.ZERO_OR_MORE_EAGER_STRICT;
-				break;
-			case ZERO_OR_MORE_COMBINATIONS:
-				this.quantifier = Quantifier.ZERO_OR_MORE_COMBINATIONS_STRICT;
-				break;
-			case ONE_OR_MORE_EAGER:
-				this.quantifier = Quantifier.ONE_OR_MORE_EAGER_STRICT;
-				break;
-			case ONE_OR_MORE_COMBINATIONS:
-				this.quantifier = Quantifier.ONE_OR_MORE_COMBINATIONS_STRICT;
-				break;
-			case TIMES:
-				this.quantifier = Quantifier.TIMES_STRICT;
-				break;
-			case ZERO_OR_MORE_COMBINATIONS_STRICT:
-			case ONE_OR_MORE_EAGER_STRICT:
-			case ONE_OR_MORE_COMBINATIONS_STRICT:
-			case ZERO_OR_MORE_EAGER_STRICT:
-			case TIMES_STRICT:
-				throw new MalformedPatternException("Strict continuity already applied! consecutive() called twice.");
-			case ONE:
-			case OPTIONAL:
-				throw new MalformedPatternException("Strict continuity cannot be applied to " + this.quantifier);
-		}
-
-		return this;
-	}
-
-	/**
-	 * Specifies that this pattern can occur zero or once.
-	 *
-	 * @return The same pattern with applied Kleene ? operator
-	 *
-	 * @throws MalformedPatternException if quantifier already applied
-	 */
-	public Pattern<T, F> optional() {
-		checkIfQuantifierApplied();
-		this.quantifier = Quantifier.OPTIONAL;
-		return this;
-	}
-
-	/**
-	 * Specifies exact number of times that this pattern should be matched.
-	 *
-	 * @param times number of times matching event must appear
-	 * @return The same pattern with number of times applied
-	 *
-	 * @throws MalformedPatternException if quantifier already applied
-	 */
-	public Pattern<T, F> times(int times) {
-		checkIfQuantifierApplied();
-		Preconditions.checkArgument(times > 0, "You should give a positive number greater than 0.");
-		this.quantifier = Quantifier.TIMES;
-		this.times = times;
+		quantifier.makeConsecutive();
 		return this;
 	}
 
 	private void checkIfQuantifierApplied() {
-		if (this.quantifier != Quantifier.ONE) {
-			throw new MalformedPatternException("Already applied quantifier to this Pattern. Current quantifier is: " + this.quantifier);
+		if (!quantifier.hasProperty(Quantifier.QuantifierProperty.SINGLE)) {
+			throw new MalformedPatternException("Already applied quantifier to this Pattern. " +
+					"Current quantifier is: " + quantifier);
 		}
 	}
-
 }
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
index 97890722781f6..9deacd4862dcd 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
@@ -18,51 +18,83 @@
 package org.apache.flink.cep.pattern;
 
 import java.util.EnumSet;
+import java.util.Objects;
 
-public enum Quantifier {
-	ONE,
-	ZERO_OR_MORE_EAGER(QuantifierProperty.LOOPING, QuantifierProperty.EAGER),
-	ZERO_OR_MORE_COMBINATIONS(QuantifierProperty.LOOPING),
-	ZERO_OR_MORE_EAGER_STRICT(QuantifierProperty.EAGER, QuantifierProperty.STRICT, QuantifierProperty.LOOPING),
-	ZERO_OR_MORE_COMBINATIONS_STRICT(QuantifierProperty.STRICT, QuantifierProperty.LOOPING),
-	ONE_OR_MORE_EAGER(
-		QuantifierProperty.LOOPING,
-		QuantifierProperty.EAGER,
-		QuantifierProperty.AT_LEAST_ONE),
-	ONE_OR_MORE_EAGER_STRICT(
-		QuantifierProperty.STRICT,
-		QuantifierProperty.LOOPING,
-		QuantifierProperty.EAGER,
-		QuantifierProperty.AT_LEAST_ONE),
-	ONE_OR_MORE_COMBINATIONS(QuantifierProperty.LOOPING, QuantifierProperty.AT_LEAST_ONE),
-	ONE_OR_MORE_COMBINATIONS_STRICT(
-		QuantifierProperty.STRICT,
-		QuantifierProperty.LOOPING,
-		QuantifierProperty.AT_LEAST_ONE),
-	TIMES(QuantifierProperty.TIMES),
-	TIMES_STRICT(QuantifierProperty.TIMES, QuantifierProperty.STRICT),
-	OPTIONAL;
+public class Quantifier {
 
 	private final EnumSet<QuantifierProperty> properties;
 
-	Quantifier(final QuantifierProperty first, final QuantifierProperty... rest) {
+	private Quantifier(final QuantifierProperty first, final QuantifierProperty... rest) {
 		this.properties = EnumSet.of(first, rest);
 	}
 
-	Quantifier() {
-		this.properties = EnumSet.noneOf(QuantifierProperty.class);
+	public static Quantifier ONE() {
+		return new Quantifier(QuantifierProperty.SINGLE);
+	}
+
+	public static Quantifier ONE_OR_MORE() {
+		return new Quantifier(QuantifierProperty.LOOPING, QuantifierProperty.EAGER);
+	}
+
+	public static Quantifier TIMES() {
+		return new Quantifier(QuantifierProperty.TIMES);
 	}
 
 	public boolean hasProperty(QuantifierProperty property) {
 		return properties.contains(property);
 	}
 
+	public void allowAllCombinations() {
+		if (!hasProperty(Quantifier.QuantifierProperty.EAGER)) {
+			throw new MalformedPatternException("Combinations already allowed!");
+		}
+
+		if (hasProperty(Quantifier.QuantifierProperty.LOOPING)) {
+			properties.remove(Quantifier.QuantifierProperty.EAGER);
+		} else {
+			throw new MalformedPatternException("Combinations not applicable to " + this + "!");
+		}
+	}
+
+	public void makeConsecutive() {
+		if (hasProperty(Quantifier.QuantifierProperty.CONSECUTIVE)) {
+			throw new MalformedPatternException("Strict continuity already applied!");
+		}
+
+		if (hasProperty(Quantifier.QuantifierProperty.LOOPING) || hasProperty(Quantifier.QuantifierProperty.TIMES)) {
+			properties.add(Quantifier.QuantifierProperty.CONSECUTIVE);
+		} else {
+			throw new MalformedPatternException("Strict continuity not applicable to " + this + "!");
+		}
+	}
+
+	public void makeOptional() {
+		if (hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
+			throw new MalformedPatternException("Optional already applied!");
+		}
+		properties.add(Quantifier.QuantifierProperty.OPTIONAL);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		return obj instanceof Quantifier && properties.equals(((Quantifier)obj).properties);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hashCode(properties);
+	}
+
+	/**
+	 * Properties that a {@link Quantifier} can have. Not all combinations are valid.
+	 */
 	public enum QuantifierProperty {
+		SINGLE,
 		LOOPING,
+		TIMES,
 		EAGER,
-		AT_LEAST_ONE,
-		STRICT,
-		TIMES
+		CONSECUTIVE,
+		OPTIONAL
 	}
 
 }
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index 7359ca8cb6818..cffb4e7724563 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -456,7 +456,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore(false).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -544,7 +544,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore(false).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -609,7 +609,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore(true).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().optional().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -664,7 +664,7 @@ public void testBeginWithZeroOrMore() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore().followedBy("end").where(new SimpleCondition<Event>() {
+		}).oneOrMore().optional().followedBy("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -731,14 +731,14 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore(false).followedBy("middle-second").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedBy("middle-second").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("d");
 			}
-		}).zeroOrMore(false).followedBy("end").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedBy("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -824,7 +824,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("d");
 			}
-		}).zeroOrMore(false).followedBy("end").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedBy("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -892,7 +892,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore()
+		}).oneOrMore().optional()
 			.next("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
@@ -947,7 +947,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore(false).next("end").where(new SimpleCondition<Event>() {
+		}).oneOrMore().optional().allowCombinations().next("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1007,7 +1007,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore(false).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1061,7 +1061,7 @@ public void testBeginWithAtLeastOne() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore(false).followedBy("end").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().followedBy("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1129,7 +1129,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("middle");
 			}
-		}).zeroOrMore().consecutive().followedBy("end").where(new SimpleCondition<Event>() {
+		}).oneOrMore().optional().consecutive().followedBy("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 7056763917392056548L;
 
 			@Override
@@ -1190,7 +1190,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore(true).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1602,7 +1602,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore();
+		}).oneOrMore().optional();
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
@@ -1656,7 +1656,7 @@ public void testStartAndEndWithZeroOrMore() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore();
+		}).oneOrMore().optional();
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
@@ -1788,6 +1788,228 @@ public boolean filter(Event value) throws Exception {
 		), resultingPatterns);
 	}
 
+	///////////////////////////////         Optional           ////////////////////////////////////////
+
+	@Test
+	public void testTimesNonStrictOptional1() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(3).optional().followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
+
+	@Test
+	public void testTimesNonStrictOptional2() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 3));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 5));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).optional().followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
+
+	@Test
+	public void testTimesStrictOptional() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 3));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 5));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).consecutive().optional().followedBy("end1").where(new SimpleCondition<Event>() { // TODO: 4/4/17 also check order consecutive() vs optional()
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
+
+	@Test
+	public void testTimesStrictOptional1() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 3));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 5));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).next("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).consecutive().optional().followedBy("end1").where(new SimpleCondition<Event>() { // TODO: 4/4/17 also check order consecutive() vs optional()
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
+
+	@Test
+	public void testOptionalTimesNonStrictWithNext() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 2));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 3));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 5));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).next("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).optional().followedBy("end1").where(new SimpleCondition<Event>() { // TODO: 4/4/17 also check order consecutive() vs optional()
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
 
 	///////////////////////////////         Consecutive           ////////////////////////////////////////
 
@@ -1841,29 +2063,56 @@ private List<List<Event>> testStrictOneOrMore(boolean eager) {
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent4, 7));
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 8));
 
-		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
+		Pattern<Event, ?> pattern;
+		if (eager) {
+			pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
 
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("c");
-			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("c");
+				}
+			}).followedBy("middle").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
 
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("a");
-			}
-		}).oneOrMore(eager).consecutive()
-			.followedBy("end1").where(new SimpleCondition<Event>() {
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("a");
+				}
+			}).oneOrMore().consecutive()
+					.followedBy("end1").where(new SimpleCondition<Event>() {
+						private static final long serialVersionUID = 5726188262756267490L;
+
+						@Override
+						public boolean filter(Event value) throws Exception {
+							return value.getName().equals("b");
+						}
+					});
+		} else {
+			pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("c");
+				}
+			}).followedBy("middle").where(new SimpleCondition<Event>() {
 				private static final long serialVersionUID = 5726188262756267490L;
 
 				@Override
 				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("b");
+					return value.getName().equals("a");
 				}
-			});
+			}).oneOrMore().allowCombinations().consecutive()
+					.followedBy("end1").where(new SimpleCondition<Event>() {
+						private static final long serialVersionUID = 5726188262756267490L;
+
+						@Override
+						public boolean filter(Event value) throws Exception {
+							return value.getName().equals("b");
+						}
+					});
+		}
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
@@ -1905,28 +2154,51 @@ private List<List<Event>> testStrictZeroOrMore(boolean eager) {
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
 
-		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
+		Pattern<Event, ?> pattern = eager
+				? Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
 
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("c");
-			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("c");
+					}
+				}).followedBy("middle").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
 
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("a");
-			}
-		}).zeroOrMore(eager).consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("a");
+					}
+				}).oneOrMore().optional().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
 
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("b");
-			}
-		});
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("b");
+					}
+				})
+				: Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
+
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("c");
+					}
+				}).followedBy("middle").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
+
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("a");
+					}
+				}).oneOrMore().allowCombinations().optional().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
+
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("b");
+					}
+				});
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
@@ -2033,7 +2305,7 @@ public void testStartWithZeroOrMoreStrict() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore().consecutive();
+		}).oneOrMore().optional().consecutive();
 
 		testStartWithOneOrZeroOrMoreStrict(pattern);
 	}
@@ -2183,7 +2455,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore(false).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2227,7 +2499,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).zeroOrMore(false).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2313,37 +2585,41 @@ private List<List<Event>> testIterativeWithBranchingPattern(boolean eager) {
 		inputEvents.add(new StreamRecord<>(nextOne, 6));
 		inputEvents.add(new StreamRecord<>(endEvent, 8));
 
-		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("start");
-			}
-		}).followedBy("middle").subtype(SubEvent.class).where(new IterativeCondition<SubEvent>() {
-			private static final long serialVersionUID = 6215754202506583964L;
-
-			@Override
-			public boolean filter(SubEvent value, Context<SubEvent> ctx) throws Exception {
-				if (!value.getName().startsWith("foo")) {
-					return false;
-				}
+		Pattern<Event, ?> pattern = eager
+				? Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
 
-				double sum = 0.0;
-				for (Event event : ctx.getEventsForPattern("middle")) {
-					sum += event.getPrice();
-				}
-				sum += value.getPrice();
-				return Double.compare(sum, 5.0) < 0;
-			}
-		}).oneOrMore(eager).followedBy("end").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 7056763917392056548L;
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("start");
+					}
+				})
+				.followedBy("middle").subtype(SubEvent.class).where(new MySubeventIterCondition()).oneOrMore()
+				.followedBy("end").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 7056763917392056548L;
+
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("end");
+					}
+				})
+				: Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 5726188262756267490L;
 
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("end");
-			}
-		});
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("start");
+					}
+				})
+				.followedBy("middle").subtype(SubEvent.class).where(new MySubeventIterCondition()).oneOrMore().allowCombinations()
+				.followedBy("end").where(new SimpleCondition<Event>() {
+					private static final long serialVersionUID = 7056763917392056548L;
+
+					@Override
+					public boolean filter(Event value) throws Exception {
+						return value.getName().equals("end");
+					}
+				});
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
@@ -2352,6 +2628,25 @@ public boolean filter(Event value) throws Exception {
 		return resultingPatterns;
 	}
 
+	private static class MySubeventIterCondition extends IterativeCondition<SubEvent> {
+
+		private static final long serialVersionUID = 6215754202506583964L;
+
+		@Override
+		public boolean filter (SubEvent value, Context < SubEvent > ctx) throws Exception {
+			if (!value.getName().startsWith("foo")) {
+				return false;
+			}
+
+			double sum = 0.0;
+			for (Event event : ctx.getEventsForPattern("middle")) {
+				sum += event.getPrice();
+			}
+			sum += value.getPrice();
+			return Double.compare(sum, 5.0) < 0;
+		}
+	}
+
 	@Test
 	public void testIterativeWithLoopingStartingEager() {
 		List<List<Event>> actual = testIterativeWithLoopingStarting(true);
@@ -2395,30 +2690,25 @@ private List<List<Event>> testIterativeWithLoopingStarting(boolean eager) {
 		// behavior (which is the one applied in the case that the pattern graph starts with such a pattern)
 		// of a looping pattern is with relaxed continuity (as in followedBy).
 
-		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new IterativeCondition<Event>() {
-			private static final long serialVersionUID = 6215754202506583964L;
-
-			@Override
-			public boolean filter(Event value, Context<Event> ctx) throws Exception {
-				if (!value.getName().equals("start")) {
-					return false;
-				}
-
-				double sum = 0.0;
-				for (Event event : ctx.getEventsForPattern("start")) {
-					sum += event.getPrice();
-				}
-				sum += value.getPrice();
-				return Double.compare(sum, 5.0) < 0;
-			}
-		}).zeroOrMore(eager).followedBy("end").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 7056763917392056548L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("end");
-			}
-		});
+		Pattern<Event, ?> pattern = eager
+				? Pattern.<Event>begin("start").where(new MyEventIterCondition()).oneOrMore().optional()
+					.followedBy("end").where(new SimpleCondition<Event>() {
+						private static final long serialVersionUID = 7056763917392056548L;
+
+						@Override
+						public boolean filter(Event value) throws Exception {
+							return value.getName().equals("end");
+						}
+					})
+				: Pattern.<Event>begin("start").where(new MyEventIterCondition()).oneOrMore().allowCombinations().optional()
+					.followedBy("end").where(new SimpleCondition<Event>() {
+						private static final long serialVersionUID = 7056763917392056548L;
+
+						@Override
+						public boolean filter(Event value) throws Exception {
+							return value.getName().equals("end");
+						}
+					});
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
@@ -2427,6 +2717,25 @@ public boolean filter(Event value) throws Exception {
 		return resultingPatterns;
 	}
 
+	private static class MyEventIterCondition extends IterativeCondition<Event> {
+
+		private static final long serialVersionUID = 6215754202506583964L;
+
+		@Override
+		public boolean filter(Event value, Context<Event> ctx) throws Exception {
+			if (!value.getName().equals("start")) {
+				return false;
+			}
+
+			double sum = 0.0;
+			for (Event event : ctx.getEventsForPattern("start")) {
+				sum += event.getPrice();
+			}
+			sum += value.getPrice();
+			return Double.compare(sum, 5.0) < 0;
+		}
+	}
+
 	@Test
 	public void testIterativeWithPrevPatternDependency() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
index 80b1bcb981583..da1f9c82fdee9 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
@@ -153,7 +153,7 @@ public void testNFACompilerWithSimplePattern() {
 	public void testNFACompilerWithKleeneStar() {
 
 		Pattern<Event, Event> pattern = Pattern.<Event>begin("start").where(startFilter)
-			.followedBy("middle").subtype(SubEvent.class).zeroOrMore()
+			.followedBy("middle").subtype(SubEvent.class).oneOrMore().optional()
 			.followedBy("end").where(endFilter);
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, serializer, false);
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
index e9aa7d2fc1ac2..8c4304aea0fc3 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
@@ -196,7 +196,7 @@ public void testPatternCanHaveQuantifierSpecifiedOnce1() throws Exception {
 			public boolean filter(Object value) throws Exception {
 				return true;
 			}
-		}).oneOrMore().zeroOrMore();
+		}).oneOrMore().oneOrMore().optional();
 	}
 
 	@Test(expected = MalformedPatternException.class)
@@ -209,7 +209,7 @@ public void testPatternCanHaveQuantifierSpecifiedOnce2() throws Exception {
 			public boolean filter(Object value) throws Exception {
 				return true;
 			}
-		}).zeroOrMore().times(1);
+		}).oneOrMore().optional().times(1);
 	}
 
 	@Test(expected = MalformedPatternException.class)
@@ -235,7 +235,7 @@ public void testPatternCanHaveQuantifierSpecifiedOnce4() throws Exception {
 			public boolean filter(Object value) throws Exception {
 				return true;
 			}
-		}).oneOrMore().oneOrMore(true);
+		}).oneOrMore().oneOrMore();
 	}
 
 	@Test(expected = MalformedPatternException.class)
@@ -248,6 +248,6 @@ public void testPatternCanHaveQuantifierSpecifiedOnce5() throws Exception {
 			public boolean filter(Object value) throws Exception {
 				return true;
 			}
-		}).oneOrMore(true).zeroOrMore(true);
+		}).oneOrMore().oneOrMore().optional();
 	}
 }

From 429be94dd4bf9041b738e4afd105efdb4740434d Mon Sep 17 00:00:00 2001
From: kl0u <kkloudas@gmail.com>
Date: Fri, 21 Apr 2017 19:59:43 +0200
Subject: [PATCH 058/253] [FLINK-6356] [cep] Make times() eager by default and
 allow allowCombinations().

By default, the times() quantifier operates in eager mode and the
user can disable eagerness by calling .allowCombinations().
---
 .../flink/cep/scala/pattern/Pattern.scala     |   4 +-
 .../flink/cep/nfa/compiler/NFACompiler.java   |   4 +-
 .../org/apache/flink/cep/pattern/Pattern.java |  10 +-
 .../apache/flink/cep/pattern/Quantifier.java  |  14 +-
 .../org/apache/flink/cep/nfa/NFAITCase.java   | 189 +++++++++++++++++-
 5 files changed, 198 insertions(+), 23 deletions(-)

diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
index 3214d09234f6d..3bdbfcf46f923 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
@@ -253,8 +253,8 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
   }
 
   /**
-    * Applicable only to [[Quantifier.ONE_OR_MORE()]] patterns, this option
-    * allows more flexibility to the matching events.
+    * Applicable only to [[Quantifier.ONE_OR_MORE()]] and [[Quantifier.TIMES()]] patterns,
+    * this option allows more flexibility to the matching events.
     *
     * If {{{allowCombinations()}}} is not applied for a
     * pattern {{{A.oneOrMore().followedBy(B)}}} and a sequence of events
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
index add1583150132..b4e05573b1913 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
@@ -284,7 +284,7 @@ private State<T> createTimesState(final State<T> sinkState, int times) {
 			if (currentPattern instanceof FollowedByPattern) {
 				State<T> ignoreState = createNormalState();
 				ignoreState.addTake(lastSink, currentFilterFunction);
-				ignoreState.addIgnore(BooleanConditions.<T>trueFunction());
+				ignoreState.addIgnore(trueFunction);
 				singletonState.addIgnore(ignoreState, trueFunction);
 			}
 			return singletonState;
@@ -333,7 +333,7 @@ private State<T> createSingletonState(final State<T> sinkState, boolean addIgnor
 				} else {
 					ignoreState = singletonState;
 				}
-				singletonState.addIgnore(ignoreState, trueFunction);
+				singletonState.addIgnore(ignoreState, getIgnoreCondition(currentPattern));
 			}
 			return singletonState;
 		}
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
index f1d0d631e46bd..3ce0c739e102d 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
@@ -218,7 +218,7 @@ public FollowedByPattern<T, T> followedBy(final String name) {
 	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
 	public Pattern<T, F> optional() {
-		quantifier.makeOptional();
+		quantifier.optional();
 		return this;
 	}
 
@@ -258,8 +258,8 @@ public Pattern<T, F> times(int times) {
 	}
 
 	/**
-	 * Applicable only to {@link Quantifier#ONE_OR_MORE()} patterns, this option
-	 * allows more flexibility to the matching events.
+	 * Applicable only to {@link Quantifier#ONE_OR_MORE()} and {@link Quantifier#TIMES()} patterns,
+	 * this option allows more flexibility to the matching events.
 	 *
 	 * <p>If {@code allowCombinations()} is not applied for a
 	 * pattern {@code A.oneOrMore().followedBy(B)} and a sequence of events
@@ -271,7 +271,7 @@ public Pattern<T, F> times(int times) {
 	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
 	public Pattern<T, F> allowCombinations() {
-		quantifier.allowAllCombinations();
+		quantifier.combinations();
 		return this;
 	}
 
@@ -310,7 +310,7 @@ public Pattern<T, F> allowCombinations() {
 	 * @return pattern with continuity changed to strict
 	 */
 	public Pattern<T, F> consecutive() {
-		quantifier.makeConsecutive();
+		quantifier.consecutive();
 		return this;
 	}
 
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
index 9deacd4862dcd..0332ed493ad22 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
@@ -37,27 +37,27 @@ public static Quantifier ONE_OR_MORE() {
 	}
 
 	public static Quantifier TIMES() {
-		return new Quantifier(QuantifierProperty.TIMES);
+		return new Quantifier(QuantifierProperty.TIMES, QuantifierProperty.EAGER);
 	}
 
 	public boolean hasProperty(QuantifierProperty property) {
 		return properties.contains(property);
 	}
 
-	public void allowAllCombinations() {
-		if (!hasProperty(Quantifier.QuantifierProperty.EAGER)) {
+	public void combinations() {
+		if (!hasProperty(QuantifierProperty.SINGLE) && !hasProperty(Quantifier.QuantifierProperty.EAGER)) {
 			throw new MalformedPatternException("Combinations already allowed!");
 		}
 
-		if (hasProperty(Quantifier.QuantifierProperty.LOOPING)) {
+		if (hasProperty(Quantifier.QuantifierProperty.LOOPING) || hasProperty(Quantifier.QuantifierProperty.TIMES)) {
 			properties.remove(Quantifier.QuantifierProperty.EAGER);
 		} else {
 			throw new MalformedPatternException("Combinations not applicable to " + this + "!");
 		}
 	}
 
-	public void makeConsecutive() {
-		if (hasProperty(Quantifier.QuantifierProperty.CONSECUTIVE)) {
+	public void consecutive() {
+		if (!hasProperty(QuantifierProperty.SINGLE) && hasProperty(Quantifier.QuantifierProperty.CONSECUTIVE)) {
 			throw new MalformedPatternException("Strict continuity already applied!");
 		}
 
@@ -68,7 +68,7 @@ public void makeConsecutive() {
 		}
 	}
 
-	public void makeOptional() {
+	public void optional() {
 		if (hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
 			throw new MalformedPatternException("Optional already applied!");
 		}
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index cffb4e7724563..f4bf647ff934a 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -1311,7 +1311,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).times(2).allowCombinations().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1422,7 +1422,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).times(2).allowCombinations().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1466,7 +1466,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
+		}).times(2).allowCombinations().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1484,6 +1484,135 @@ public boolean filter(Event value) throws Exception {
 		));
 	}
 
+	@Test
+	public void testTimesNotStrictWithFollowedByEager() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end)
+		));
+	}
+
+	@Test
+	public void testTimesNotStrictWithFollowedByNotEager() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).allowCombinations().followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent1, ConsecutiveData.end)
+		));
+	}
+
+	@Test
+	public void testTimesStrictWithFollowedByNotEager() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).allowCombinations().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end)
+		));
+	}
+
 	@Test
 	public void testTimesStrictWithNextAndConsecutive() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
@@ -1856,7 +1985,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).optional().followedBy("end1").where(new SimpleCondition<Event>() {
+		}).times(2).allowCombinations().optional().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1877,6 +2006,52 @@ public boolean filter(Event value) throws Exception {
 		));
 	}
 
+	@Test
+	public void testTimesNonStrictOptional3() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 3));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 5));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).times(2).optional().followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end), // this exists because of the optional()
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
+
 	@Test
 	public void testTimesStrictOptional() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
@@ -1991,7 +2166,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).optional().followedBy("end1").where(new SimpleCondition<Event>() { // TODO: 4/4/17 also check order consecutive() vs optional()
+		}).times(2).allowCombinations().optional().followedBy("end1").where(new SimpleCondition<Event>() { // TODO: 4/4/17 also check order consecutive() vs optional()
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2232,7 +2407,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
+		}).times(2).allowCombinations().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2276,7 +2451,7 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).followedBy("end1").where(new SimpleCondition<Event>() {
+		}).times(2).allowCombinations().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override

From 4458806adfaf2abb0c0136e812ad8dcfb5a2078d Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Tue, 25 Apr 2017 20:41:58 +0200
Subject: [PATCH 059/253] [FLINK-6384] [py] Remove python binary check via
 additional process

The PythonStreamer used to check for the existence of the python binary by
starting a python process. This process was not closed afterwards. This caused
the PythonPlanBinderTest to fail locally.

I think the check whether a python binary exists is not necessary since the
subsequent python command would fail anyway if there is no binary available on
the system. The system failure message is that there is no such file or directory.
This error message should be descriptive enough in order to debug such a problem.

This closes #3774.
---
 .../api/streaming/data/PythonStreamer.java    | 32 +++++++++++--------
 .../python/api/PythonPlanBinderTest.java      | 21 +++++++++---
 2 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
index 97d57800a3294..cc4ba43e7393e 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
@@ -19,6 +19,7 @@
 import org.apache.flink.python.api.streaming.util.SerializationUtils.IntSerializer;
 import org.apache.flink.python.api.streaming.util.SerializationUtils.StringSerializer;
 import org.apache.flink.python.api.streaming.util.StreamPrinter;
+import org.apache.flink.util.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -114,13 +115,7 @@ private void startPython() throws IOException {
 
 		String pythonBinaryPath = config.getString(PythonOptions.PYTHON_BINARY_PATH);
 
-		try {
-			Runtime.getRuntime().exec(pythonBinaryPath);
-		} catch (IOException ignored) {
-			throw new RuntimeException(pythonBinaryPath + " does not point to a valid python binary.");
-		}
-
-		process = Runtime.getRuntime().exec(pythonBinaryPath + " -O -B " + planPath + config.getString(PLAN_ARGUMENTS_KEY, ""));
+		process = Runtime.getRuntime().exec(new String[] {pythonBinaryPath, "-O", "-B", planPath, config.getString(PLAN_ARGUMENTS_KEY, "")});
 		outPrinter = new Thread(new StreamPrinter(process.getInputStream()));
 		outPrinter.start();
 		errorPrinter = new Thread(new StreamPrinter(process.getErrorStream(), msg));
@@ -130,8 +125,9 @@ private void startPython() throws IOException {
 			@Override
 			public void run() {
 				try {
-					destroyProcess();
-				} catch (IOException ignored) {
+					destroyProcess(process);
+				} catch (IOException ioException) {
+					LOG.warn("Could not destroy python process.", ioException);
 				}
 			}
 		};
@@ -192,20 +188,30 @@ private void checkPythonProcessHealth() {
 	 * @throws IOException
 	 */
 	public void close() throws IOException {
+		Throwable throwable = null;
+
 		try {
 			socket.close();
 			sender.close();
 			receiver.close();
-		} catch (Exception e) {
-			LOG.error("Exception occurred while closing Streamer. :{}", e.getMessage());
+		} catch (Throwable t) {
+			throwable = t;
 		}
-		destroyProcess();
+
+		try {
+			destroyProcess(process);
+		} catch (Throwable t) {
+			throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
+		}
+
 		if (shutdownThread != null) {
 			Runtime.getRuntime().removeShutdownHook(shutdownThread);
 		}
+
+		ExceptionUtils.tryRethrowIOException(throwable);
 	}
 
-	private void destroyProcess() throws IOException {
+	public static void destroyProcess(Process process) throws IOException {
 		try {
 			process.exitValue();
 		} catch (IllegalThreadStateException ignored) { //process still active
diff --git a/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
index ba8ea78932402..20f35038597b6 100644
--- a/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
+++ b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
@@ -16,6 +16,7 @@
 import org.apache.flink.core.fs.FileStatus;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
+import org.apache.flink.python.api.streaming.data.PythonStreamer;
 import org.apache.flink.test.util.JavaProgramTestBase;
 
 import java.io.IOException;
@@ -50,21 +51,33 @@ private static List<String> findTestFiles() throws Exception {
 		return files;
 	}
 
-	private static boolean isPython2Supported() {
+	private static boolean isPython2Supported() throws IOException {
+		Process process = null;
+
 		try {
-			Runtime.getRuntime().exec("python");
+			process = Runtime.getRuntime().exec("python");
 			return true;
 		} catch (IOException ex) {
 			return false;
+		} finally {
+			if (process != null) {
+				PythonStreamer.destroyProcess(process);
+			}
 		}
 	}
 
-	private static boolean isPython3Supported() {
+	private static boolean isPython3Supported() throws IOException {
+		Process process = null;
+
 		try {
-			Runtime.getRuntime().exec("python3");
+			process = Runtime.getRuntime().exec("python3");
 			return true;
 		} catch (IOException ex) {
 			return false;
+		} finally {
+			if (process != null) {
+				PythonStreamer.destroyProcess(process);
+			}
 		}
 	}
 

From 330a36d893981c5278c4a34d5061644348ad2f53 Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Mon, 3 Apr 2017 14:44:46 +0200
Subject: [PATCH 060/253] [FLINK-3871] [table] Add Kafka TableSource with Avro
 serialization

This closes #3663.
---
 docs/dev/table_api.md                         |  85 ++++++++--
 .../flink-connector-kafka-0.10/pom.xml        |   1 -
 .../kafka/Kafka010AvroTableSource.java        |  56 +++++++
 .../kafka/Kafka010JsonTableSource.java        |  28 +---
 .../connectors/kafka/Kafka010TableSource.java |  28 +---
 .../kafka/Kafka010AvroTableSourceTest.java    |  50 ++++++
 .../kafka/Kafka010JsonTableSourceTest.java    |  45 +++++
 .../kafka/Kafka08AvroTableSource.java         |  56 +++++++
 .../kafka/Kafka08JsonTableSource.java         |  26 +--
 .../connectors/kafka/Kafka08TableSource.java  |  28 +---
 .../kafka/Kafka08AvroTableSourceTest.java     |  49 ++++++
 .../kafka/Kafka08JsonTableSourceTest.java     |   4 +-
 .../kafka/Kafka09AvroTableSource.java         |  56 +++++++
 .../kafka/Kafka09JsonTableSource.java         |  26 +--
 .../connectors/kafka/Kafka09TableSource.java  |  28 +---
 .../kafka/Kafka09AvroTableSourceTest.java     |  49 ++++++
 .../kafka/Kafka09JsonTableSourceTest.java     |   4 +-
 .../flink-connector-kafka-base/pom.xml        |  27 ++-
 .../kafka/KafkaAvroTableSource.java           | 106 ++++++++++++
 .../kafka/KafkaJsonTableSource.java           |  40 +----
 .../connectors/kafka/KafkaTableSource.java    |  49 +-----
 .../connectors/kafka/internals/TypeUtil.java  |  38 -----
 .../AvroRowDeserializationSchema.java         | 155 ++++++++++++++++++
 .../AvroRowSerializationSchema.java           | 124 ++++++++++++++
 .../JsonRowDeserializationSchema.java         |  43 ++---
 .../AvroRowDeSerializationSchemaTest.java     | 120 ++++++++++++++
 .../JsonRowDeserializationSchemaTest.java     |  26 +--
 .../kafka/JsonRowSerializationSchemaTest.java |  16 +-
 .../kafka/KafkaTableSinkTestBase.java         |   4 +-
 .../kafka/KafkaTableSourceTestBase.java       |  38 ++++-
 .../kafka/testutils/AvroTestUtils.java        | 150 +++++++++++++++++
 .../flink/api/common/typeinfo/Types.java      |   2 +-
 .../flink/api/java/typeutils/RowTypeInfo.java |   7 +
 .../org/apache/flink/table/api/Types.scala    |  61 +++++--
 34 files changed, 1291 insertions(+), 334 deletions(-)
 create mode 100644 flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSource.java
 create mode 100644 flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSourceTest.java
 create mode 100644 flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceTest.java
 create mode 100644 flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSource.java
 create mode 100644 flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSourceTest.java
 create mode 100644 flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSource.java
 create mode 100644 flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSourceTest.java
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaAvroTableSource.java
 delete mode 100644 flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/TypeUtil.java
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowDeserializationSchema.java
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowSerializationSchema.java
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/AvroRowDeSerializationSchemaTest.java
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/AvroTestUtils.java

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 3d77b85c366f1..4b9e5653eef70 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -202,7 +202,7 @@ tableEnv.registerTableSource("Customers", custTS)
 
 A `TableSource` can provide access to data stored in various storage systems such as databases (MySQL, HBase, ...), file formats (CSV, Apache Parquet, Avro, ORC, ...), or messaging systems (Apache Kafka, RabbitMQ, ...).
 
-Currently, Flink provides the `CsvTableSource` to read CSV files and the `Kafka08JsonTableSource`/`Kafka09JsonTableSource` to read JSON objects from Kafka.
+Currently, Flink provides the `CsvTableSource` to read CSV files and various `TableSources` to read JSON or Avro objects from Kafka.
 A custom `TableSource` can be defined by implementing the `BatchTableSource` or `StreamTableSource` interface.
 
 ### Available Table Sources
@@ -210,7 +210,11 @@ A custom `TableSource` can be defined by implementing the `BatchTableSource` or
 | **Class name** | **Maven dependency** | **Batch?** | **Streaming?** | **Description**
 | `CsvTableSouce` | `flink-table` | Y | Y | A simple source for CSV files.
 | `Kafka08JsonTableSource` | `flink-connector-kafka-0.8` | N | Y | A Kafka 0.8 source for JSON data.
+| `Kafka08AvroTableSource` | `flink-connector-kafka-0.8` | N | Y | A Kafka 0.8 source for Avro data.
 | `Kafka09JsonTableSource` | `flink-connector-kafka-0.9` | N | Y | A Kafka 0.9 source for JSON data.
+| `Kafka09AvroTableSource` | `flink-connector-kafka-0.9` | N | Y | A Kafka 0.9 source for Avro data.
+| `Kafka010JsonTableSource` | `flink-connector-kafka-0.10` | N | Y | A Kafka 0.10 source for JSON data.
+| `Kafka010AvroTableSource` | `flink-connector-kafka-0.10` | N | Y | A Kafka 0.10 source for Avro data.
 
 All sources that come with the `flink-table` dependency can be directly used by your Table programs. For all other table sources, you have to add the respective dependency in addition to the `flink-table` dependency.
 
@@ -218,22 +222,42 @@ All sources that come with the `flink-table` dependency can be directly used by
 
 To use the Kafka JSON source, you have to add the Kafka connector dependency to your project:
 
-  - `flink-connector-kafka-0.8` for Kafka 0.8, and
-  - `flink-connector-kafka-0.9` for Kafka 0.9, respectively.
+  - `flink-connector-kafka-0.8` for Kafka 0.8,
+  - `flink-connector-kafka-0.9` for Kafka 0.9, or
+  - `flink-connector-kafka-0.10` for Kafka 0.10, respectively.
 
 You can then create the source as follows (example for Kafka 0.8):
-
-```java
-// The JSON field names and types
-String[] fieldNames =  new String[] { "id", "name", "score"};
-Class<?>[] fieldTypes = new Class<?>[] { Integer.class, String.class, Double.class };
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+// specify JSON field names and types
+TypeInformation<Row> typeInfo = Types.ROW(
+  new String[] { "id", "name", "score" },
+  new TypeInformation<?>[] { Types.INT(), Types.STRING(), Types.DOUBLE() }
+);
 
 KafkaJsonTableSource kafkaTableSource = new Kafka08JsonTableSource(
     kafkaTopic,
     kafkaProperties,
-    fieldNames,
-    fieldTypes);
-```
+    typeInfo);
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+// specify JSON field names and types
+val typeInfo = Types.ROW(
+  Array("id", "name", "score"),
+  Array(Types.INT, Types.STRING, Types.DOUBLE)
+)
+
+val kafkaTableSource = new Kafka08JsonTableSource(
+    kafkaTopic,
+    kafkaProperties,
+    typeInfo)
+{% endhighlight %}
+</div>
+</div>
 
 By default, a missing JSON field does not fail the source. You can configure this via:
 
@@ -249,6 +273,43 @@ tableEnvironment.registerTableSource("kafka-source", kafkaTableSource);
 Table result = tableEnvironment.ingest("kafka-source");
 ```
 
+#### KafkaAvroTableSource
+
+The `KafkaAvroTableSource` allows you to read Avro's `SpecificRecord` objects from Kafka.
+
+To use the Kafka Avro source, you have to add the Kafka connector dependency to your project:
+
+  - `flink-connector-kafka-0.8` for Kafka 0.8,
+  - `flink-connector-kafka-0.9` for Kafka 0.9, or
+  - `flink-connector-kafka-0.10` for Kafka 0.10, respectively.
+
+You can then create the source as follows (example for Kafka 0.8):
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+// pass the generated Avro class to the TableSource
+Class<? extends SpecificRecord> clazz = MyAvroType.class; 
+
+KafkaAvroTableSource kafkaTableSource = new Kafka08AvroTableSource(
+    kafkaTopic,
+    kafkaProperties,
+    clazz);
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+// pass the generated Avro class to the TableSource
+val clazz = classOf[MyAvroType]
+
+val kafkaTableSource = new Kafka08AvroTableSource(
+    kafkaTopic,
+    kafkaProperties,
+    clazz)
+{% endhighlight %}
+</div>
+</div>
+
 #### CsvTableSource
 
 The `CsvTableSource` is already included in `flink-table` without additional dependecies.
@@ -1684,6 +1745,8 @@ The Table API is built on top of Flink's DataSet and DataStream API. Internally,
 | `Types.TIMESTAMP`      | `TIMESTAMP(3)`              | `java.sql.Timestamp`   |
 | `Types.INTERVAL_MONTHS`| `INTERVAL YEAR TO MONTH`    | `java.lang.Integer`    |
 | `Types.INTERVAL_MILLIS`| `INTERVAL DAY TO SECOND(3)` | `java.lang.Long`       |
+| `Types.PRIMITIVE_ARRAY`| `ARRAY`                     | e.g. `int[]`           |
+| `Types.OBJECT_ARRAY`   | `ARRAY`                     | e.g. `java.lang.Byte[]`|
 
 
 Advanced types such as generic types, composite types (e.g. POJOs or Tuples), and array types (object or primitive arrays) can be fields of a row. 
diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml
index e061178b8488f..ab60946b9e758 100644
--- a/flink-connectors/flink-connector-kafka-0.10/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml
@@ -152,7 +152,6 @@ under the License.
 			<scope>test</scope>
 		</dependency>
 
-
 	</dependencies>
 
 	<build>
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSource.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSource.java
new file mode 100644
index 0000000000000..1b2abcc4bd224
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSource.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.specific.SpecificRecordBase;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.table.sources.StreamTableSource;
+import org.apache.flink.types.Row;
+
+/**
+ * Kafka {@link StreamTableSource} for Kafka 0.10.
+ */
+public class Kafka010AvroTableSource extends KafkaAvroTableSource {
+
+	/**
+	 * Creates a Kafka 0.10 Avro {@link StreamTableSource} using a given {@link SpecificRecord}.
+	 *
+	 * @param topic      Kafka topic to consume.
+	 * @param properties Properties for the Kafka consumer.
+	 * @param record     Avro specific record.
+	 */
+	public Kafka010AvroTableSource(
+		String topic,
+		Properties properties,
+		Class<? extends SpecificRecordBase> record) {
+
+		super(
+			topic,
+			properties,
+			record);
+	}
+
+	@Override
+	FlinkKafkaConsumerBase<Row> getKafkaConsumer(String topic, Properties properties, DeserializationSchema<Row> deserializationSchema) {
+		return new FlinkKafkaConsumer010<>(topic, deserializationSchema, properties);
+	}
+}
+
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSource.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSource.java
index bfae1a9ab99d0..78ef28e4a5288 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSource.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSource.java
@@ -28,40 +28,22 @@
 /**
  * Kafka {@link StreamTableSource} for Kafka 0.10.
  */
-public class Kafka010JsonTableSource extends Kafka09JsonTableSource {
+public class Kafka010JsonTableSource extends KafkaJsonTableSource {
 
 	/**
 	 * Creates a Kafka 0.10 JSON {@link StreamTableSource}.
 	 *
 	 * @param topic      Kafka topic to consume.
 	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
+	 * @param typeInfo   Type information describing the result type. The field names are used
+	 *                   to parse the JSON file and so are the types.
 	 */
 	public Kafka010JsonTableSource(
 			String topic,
 			Properties properties,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
-		super(topic, properties, fieldNames, fieldTypes);
-	}
-
-	/**
-	 * Creates a Kafka 0.10 JSON {@link StreamTableSource}.
-	 *
-	 * @param topic      Kafka topic to consume.
-	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
-	 */
-	public Kafka010JsonTableSource(
-			String topic,
-			Properties properties,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
-
-		super(topic, properties, fieldNames, fieldTypes);
+		super(topic, properties, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSource.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSource.java
index e5254c03b41ec..03e9125184492 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSource.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSource.java
@@ -36,36 +36,16 @@ public class Kafka010TableSource extends Kafka09TableSource {
 	 * @param topic                 Kafka topic to consume.
 	 * @param properties            Properties for the Kafka consumer.
 	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
+	 * @param typeInfo              Type information describing the result type. The field names are used
+	 *                              to parse the JSON file and so are the types.
 	 */
 	public Kafka010TableSource(
 			String topic,
 			Properties properties,
 			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
-		super(topic, properties, deserializationSchema, fieldNames, fieldTypes);
-	}
-
-	/**
-	 * Creates a Kafka 0.10 {@link StreamTableSource}.
-	 *
-	 * @param topic                 Kafka topic to consume.
-	 * @param properties            Properties for the Kafka consumer.
-	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
-	 */
-	public Kafka010TableSource(
-			String topic,
-			Properties properties,
-			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
-
-		super(topic, properties, deserializationSchema, fieldNames, fieldTypes);
+		super(topic, properties, deserializationSchema, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSourceTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSourceTest.java
new file mode 100644
index 0000000000000..ed93725fffbb6
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010AvroTableSourceTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.util.serialization.AvroRowDeserializationSchema;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.types.Row;
+
+public class Kafka010AvroTableSourceTest extends KafkaTableSourceTestBase {
+
+	@Override
+	protected KafkaTableSource createTableSource(String topic, Properties properties, TypeInformation<Row> typeInfo) {
+
+		return new Kafka010AvroTableSource(
+			topic,
+			properties,
+			AvroSpecificRecord.class);
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<DeserializationSchema<Row>> getDeserializationSchema() {
+		return (Class) AvroRowDeserializationSchema.class;
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<FlinkKafkaConsumerBase<Row>> getFlinkKafkaConsumer() {
+		return (Class) FlinkKafkaConsumer010.class;
+	}
+}
+
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceTest.java
new file mode 100644
index 0000000000000..55e8b9c0da188
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.streaming.util.serialization.JsonRowDeserializationSchema;
+import org.apache.flink.types.Row;
+
+public class Kafka010JsonTableSourceTest extends KafkaTableSourceTestBase {
+
+	@Override
+	protected KafkaTableSource createTableSource(String topic, Properties properties, TypeInformation<Row> typeInfo) {
+		return new Kafka010JsonTableSource(topic, properties, typeInfo);
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<DeserializationSchema<Row>> getDeserializationSchema() {
+		return (Class) JsonRowDeserializationSchema.class;
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<FlinkKafkaConsumerBase<Row>> getFlinkKafkaConsumer() {
+		return (Class) FlinkKafkaConsumer010.class;
+	}
+}
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSource.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSource.java
new file mode 100644
index 0000000000000..1a68c05cc9c01
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSource.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.specific.SpecificRecordBase;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.table.sources.StreamTableSource;
+import org.apache.flink.types.Row;
+
+/**
+ * Kafka {@link StreamTableSource} for Kafka 0.8.
+ */
+public class Kafka08AvroTableSource extends KafkaAvroTableSource {
+
+	/**
+	 * Creates a Kafka 0.8 Avro {@link StreamTableSource} using a given {@link SpecificRecord}.
+	 *
+	 * @param topic      Kafka topic to consume.
+	 * @param properties Properties for the Kafka consumer.
+	 * @param record     Avro specific record.
+	 */
+	public Kafka08AvroTableSource(
+		String topic,
+		Properties properties,
+		Class<? extends SpecificRecordBase> record) {
+
+		super(
+			topic,
+			properties,
+			record);
+	}
+
+	@Override
+	FlinkKafkaConsumerBase<Row> getKafkaConsumer(String topic, Properties properties, DeserializationSchema<Row> deserializationSchema) {
+		return new FlinkKafkaConsumer08<>(topic, deserializationSchema, properties);
+	}
+}
+
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSource.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSource.java
index 27e7e6ea4d234..1555a3bc72828 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSource.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSource.java
@@ -35,33 +35,15 @@ public class Kafka08JsonTableSource extends KafkaJsonTableSource {
 	 *
 	 * @param topic      Kafka topic to consume.
 	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
+	 * @param typeInfo   Type information describing the result type. The field names are used
+	 *                   to parse the JSON file and so are the types.
 	 */
 	public Kafka08JsonTableSource(
 			String topic,
 			Properties properties,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
-		super(topic, properties, fieldNames, fieldTypes);
-	}
-
-	/**
-	 * Creates a Kafka 0.8 JSON {@link StreamTableSource}.
-	 *
-	 * @param topic      Kafka topic to consume.
-	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
-	 */
-	public Kafka08JsonTableSource(
-			String topic,
-			Properties properties,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
-
-		super(topic, properties, fieldNames, fieldTypes);
+		super(topic, properties, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSource.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSource.java
index 19fd50ddcdb12..e1e481cddf04d 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSource.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSource.java
@@ -36,36 +36,16 @@ public class Kafka08TableSource extends KafkaTableSource {
 	 * @param topic                 Kafka topic to consume.
 	 * @param properties            Properties for the Kafka consumer.
 	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
+	 * @param typeInfo              Type information describing the result type. The field names are used
+	 *                              to parse the JSON file and so are the types.
 	 */
 	public Kafka08TableSource(
 			String topic,
 			Properties properties,
 			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
-		super(topic, properties, deserializationSchema, fieldNames, fieldTypes);
-	}
-
-	/**
-	 * Creates a Kafka 0.8 {@link StreamTableSource}.
-	 *
-	 * @param topic                 Kafka topic to consume.
-	 * @param properties            Properties for the Kafka consumer.
-	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
-	 */
-	public Kafka08TableSource(
-			String topic,
-			Properties properties,
-			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
-
-		super(topic, properties, deserializationSchema, fieldNames, fieldTypes);
+		super(topic, properties, deserializationSchema, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSourceTest.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSourceTest.java
new file mode 100644
index 0000000000000..2dedecbe31de9
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08AvroTableSourceTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.util.serialization.AvroRowDeserializationSchema;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.types.Row;
+
+public class Kafka08AvroTableSourceTest extends KafkaTableSourceTestBase {
+
+	@Override
+	protected KafkaTableSource createTableSource(String topic, Properties properties, TypeInformation<Row> typeInfo) {
+		return new Kafka08AvroTableSource(
+			topic,
+			properties,
+			AvroSpecificRecord.class);
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<DeserializationSchema<Row>> getDeserializationSchema() {
+		return (Class) AvroRowDeserializationSchema.class;
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<FlinkKafkaConsumerBase<Row>> getFlinkKafkaConsumer() {
+		return (Class) FlinkKafkaConsumer08.class;
+	}
+}
+
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceTest.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceTest.java
index f9ef2ce7efb56..27faff4ca78c7 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceTest.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceTest.java
@@ -27,8 +27,8 @@
 public class Kafka08JsonTableSourceTest extends KafkaTableSourceTestBase {
 
 	@Override
-	protected KafkaTableSource createTableSource(String topic, Properties properties, String[] fieldNames, TypeInformation<?>[] typeInfo) {
-		return new Kafka08JsonTableSource(topic, properties, fieldNames, typeInfo);
+	protected KafkaTableSource createTableSource(String topic, Properties properties, TypeInformation<Row> typeInfo) {
+		return new Kafka08JsonTableSource(topic, properties, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSource.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSource.java
new file mode 100644
index 0000000000000..9e1172be12c82
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSource.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.specific.SpecificRecordBase;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.table.sources.StreamTableSource;
+import org.apache.flink.types.Row;
+
+/**
+ * Kafka {@link StreamTableSource} for Kafka 0.9.
+ */
+public class Kafka09AvroTableSource extends KafkaAvroTableSource {
+
+	/**
+	 * Creates a Kafka 0.9 Avro {@link StreamTableSource} using a given {@link SpecificRecord}.
+	 *
+	 * @param topic      Kafka topic to consume.
+	 * @param properties Properties for the Kafka consumer.
+	 * @param record     Avro specific record.
+	 */
+	public Kafka09AvroTableSource(
+		String topic,
+		Properties properties,
+		Class<? extends SpecificRecordBase> record) {
+
+		super(
+			topic,
+			properties,
+			record);
+	}
+
+	@Override
+	FlinkKafkaConsumerBase<Row> getKafkaConsumer(String topic, Properties properties, DeserializationSchema<Row> deserializationSchema) {
+		return new FlinkKafkaConsumer09<>(topic, deserializationSchema, properties);
+	}
+}
+
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSource.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSource.java
index 5be09fbfbaad1..26fffa57428de 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSource.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSource.java
@@ -35,33 +35,15 @@ public class Kafka09JsonTableSource extends KafkaJsonTableSource {
 	 *
 	 * @param topic      Kafka topic to consume.
 	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
+	 * @param typeInfo   Type information describing the result type. The field names are used
+	 *                   to parse the JSON file and so are the types.
 	 */
 	public Kafka09JsonTableSource(
 			String topic,
 			Properties properties,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
-		super(topic, properties, fieldNames, fieldTypes);
-	}
-
-	/**
-	 * Creates a Kafka 0.9 JSON {@link StreamTableSource}.
-	 *
-	 * @param topic      Kafka topic to consume.
-	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
-	 */
-	public Kafka09JsonTableSource(
-			String topic,
-			Properties properties,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
-
-		super(topic, properties, fieldNames, fieldTypes);
+		super(topic, properties, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSource.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSource.java
index e226074d831e4..c581332afd6fb 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSource.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSource.java
@@ -36,36 +36,16 @@ public class Kafka09TableSource extends KafkaTableSource {
 	 * @param topic                 Kafka topic to consume.
 	 * @param properties            Properties for the Kafka consumer.
 	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
+	 * @param typeInfo              Type information describing the result type. The field names are used
+	 *                              to parse the JSON file and so are the types.
 	 */
 	public Kafka09TableSource(
 			String topic,
 			Properties properties,
 			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
-		super(topic, properties, deserializationSchema, fieldNames, fieldTypes);
-	}
-
-	/**
-	 * Creates a Kafka 0.9 {@link StreamTableSource}.
-	 *
-	 * @param topic                 Kafka topic to consume.
-	 * @param properties            Properties for the Kafka consumer.
-	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
-	 */
-	public Kafka09TableSource(
-			String topic,
-			Properties properties,
-			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
-
-		super(topic, properties, deserializationSchema, fieldNames, fieldTypes);
+		super(topic, properties, deserializationSchema, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSourceTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSourceTest.java
new file mode 100644
index 0000000000000..eff82648c0686
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09AvroTableSourceTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.util.serialization.AvroRowDeserializationSchema;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.types.Row;
+
+public class Kafka09AvroTableSourceTest extends KafkaTableSourceTestBase {
+
+	@Override
+	protected KafkaTableSource createTableSource(String topic, Properties properties, TypeInformation<Row> typeInfo) {
+
+		return new Kafka09AvroTableSource(
+			topic,
+			properties,
+			AvroSpecificRecord.class);
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<DeserializationSchema<Row>> getDeserializationSchema() {
+		return (Class) AvroRowDeserializationSchema.class;
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected Class<FlinkKafkaConsumerBase<Row>> getFlinkKafkaConsumer() {
+		return (Class) FlinkKafkaConsumer09.class;
+	}
+}
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceTest.java
index 10b9acc19dc2d..35cd9ce56d16b 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceTest.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceTest.java
@@ -27,8 +27,8 @@
 public class Kafka09JsonTableSourceTest extends KafkaTableSourceTestBase {
 
 	@Override
-	protected KafkaTableSource createTableSource(String topic, Properties properties, String[] fieldNames, TypeInformation<?>[] typeInfo) {
-		return new Kafka09JsonTableSource(topic, properties, fieldNames, typeInfo);
+	protected KafkaTableSource createTableSource(String topic, Properties properties, TypeInformation<Row> typeInfo) {
+		return new Kafka09JsonTableSource(topic, properties, typeInfo);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml
index fa401bd36d94a..263eb9aaed7e1 100644
--- a/flink-connectors/flink-connector-kafka-base/pom.xml
+++ b/flink-connectors/flink-connector-kafka-base/pom.xml
@@ -116,7 +116,6 @@ under the License.
 			<scope>test</scope>
 		</dependency>
 
-
 		<dependency>
 			<groupId>org.apache.curator</groupId>
 			<artifactId>curator-test</artifactId>
@@ -162,6 +161,14 @@ under the License.
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-avro_2.10</artifactId>
+			<version>${project.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-minikdc</artifactId>
@@ -206,6 +213,24 @@ under the License.
 				<inherited>true</inherited>
 				<extensions>true</extensions>
 			</plugin>
+			<!-- Add Avro generated classes for testing. -->
+			<plugin>
+				<groupId>org.codehaus.mojo</groupId>
+				<artifactId>build-helper-maven-plugin</artifactId>
+				<executions>
+					<execution>
+						<phase>generate-test-sources</phase>
+						<goals>
+							<goal>add-test-source</goal>
+						</goals>
+						<configuration>
+							<sources>
+								<source>${project.basedir}/../flink-avro/src/test/java/org/apache/flink/api/io/avro/generated</source>
+							</sources>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
 		</plugins>
 	</build>
 	
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaAvroTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaAvroTableSource.java
new file mode 100644
index 0000000000000..b88fb83a4fa84
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaAvroTableSource.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.List;
+import java.util.Properties;
+import org.apache.avro.Schema;
+import org.apache.avro.specific.SpecificData;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.specific.SpecificRecordBase;
+import org.apache.avro.util.Utf8;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.AvroTypeInfo;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
+import org.apache.flink.streaming.util.serialization.AvroRowDeserializationSchema;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.table.sources.StreamTableSource;
+import org.apache.flink.types.Row;
+
+/**
+ * A version-agnostic Kafka Avro {@link StreamTableSource}.
+ *
+ * <p>The version-specific Kafka consumers need to extend this class and
+ * override {@link #getKafkaConsumer(String, Properties, DeserializationSchema)}}.
+ */
+public abstract class KafkaAvroTableSource extends KafkaTableSource {
+
+	/**
+	 * Creates a generic Kafka Avro {@link StreamTableSource} using a given {@link SpecificRecord}.
+	 *
+	 * @param topic      Kafka topic to consume.
+	 * @param properties Properties for the Kafka consumer.
+	 * @param avroClass  Avro specific record.
+	 */
+	KafkaAvroTableSource(
+		String topic,
+		Properties properties,
+		Class<? extends SpecificRecordBase> avroClass) {
+
+		super(
+			topic,
+			properties,
+			createDeserializationSchema(avroClass),
+			convertToRowTypeInformation(avroClass));
+	}
+
+	private static AvroRowDeserializationSchema createDeserializationSchema(Class<? extends SpecificRecordBase> record) {
+		return new AvroRowDeserializationSchema(record);
+	}
+
+	/**
+	 * Converts the extracted AvroTypeInfo into a RowTypeInfo nested structure with deterministic field order.
+	 * Replaces generic Utf8 with basic String type information.
+	 */
+	@SuppressWarnings("unchecked")
+	private static <T extends SpecificRecordBase> TypeInformation<Row> convertToRowTypeInformation(Class<T> avroClass) {
+		final AvroTypeInfo<T> avroTypeInfo = new AvroTypeInfo<>(avroClass);
+		// determine schema to retrieve deterministic field order
+		final Schema schema = SpecificData.get().getSchema(avroClass);
+		return (TypeInformation<Row>) convertToTypeInformation(avroTypeInfo, schema);
+	}
+
+	/**
+	 * Recursively converts extracted AvroTypeInfo into a RowTypeInfo nested structure with deterministic field order.
+	 * Replaces generic Utf8 with basic String type information.
+	 */
+	private static TypeInformation<?> convertToTypeInformation(TypeInformation<?> extracted, Schema schema) {
+		if (schema.getType() == Schema.Type.RECORD) {
+			final List<Schema.Field> fields = schema.getFields();
+			final AvroTypeInfo<?> avroTypeInfo = (AvroTypeInfo<?>) extracted;
+
+			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
+			final String[] names = new String[fields.size()];
+			for (int i = 0; i < fields.size(); i++) {
+				final Schema.Field field = fields.get(i);
+				types[i] = convertToTypeInformation(avroTypeInfo.getTypeAt(field.name()), field.schema());
+				names[i] = field.name();
+			}
+			return new RowTypeInfo(types, names);
+		} else if (extracted instanceof GenericTypeInfo<?>) {
+			final GenericTypeInfo<?> genericTypeInfo = (GenericTypeInfo<?>) extracted;
+			if (genericTypeInfo.getTypeClass() == Utf8.class) {
+				return BasicTypeInfo.STRING_TYPE_INFO;
+			}
+		}
+		return extracted;
+	}
+}
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
index 3cdad0f3facd8..460f9484e9c89 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
@@ -24,6 +24,7 @@
 import org.apache.flink.streaming.util.serialization.JsonRowDeserializationSchema;
 
 import java.util.Properties;
+import org.apache.flink.types.Row;
 
 /**
  * A version-agnostic Kafka JSON {@link StreamTableSource}.
@@ -40,33 +41,15 @@ public abstract class KafkaJsonTableSource extends KafkaTableSource {
 	 *
 	 * @param topic      Kafka topic to consume.
 	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
+	 * @param typeInfo   Type information describing the result type. The field names are used
+	 *                   to parse the JSON file and so are the types.
 	 */
 	KafkaJsonTableSource(
 			String topic,
 			Properties properties,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
-		super(topic, properties, createDeserializationSchema(fieldNames, fieldTypes), fieldNames, fieldTypes);
-	}
-
-	/**
-	 * Creates a generic Kafka JSON {@link StreamTableSource}.
-	 *
-	 * @param topic      Kafka topic to consume.
-	 * @param properties Properties for the Kafka consumer.
-	 * @param fieldNames Row field names.
-	 * @param fieldTypes Row field types.
-	 */
-	KafkaJsonTableSource(
-			String topic,
-			Properties properties,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
-
-		super(topic, properties, createDeserializationSchema(fieldNames, fieldTypes), fieldNames, fieldTypes);
+		super(topic, properties, createDeserializationSchema(typeInfo), typeInfo);
 	}
 
 	/**
@@ -81,17 +64,8 @@ public void setFailOnMissingField(boolean failOnMissingField) {
 		deserializationSchema.setFailOnMissingField(failOnMissingField);
 	}
 
-	private static JsonRowDeserializationSchema createDeserializationSchema(
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
-
-		return new JsonRowDeserializationSchema(fieldNames, fieldTypes);
-	}
-
-	private static JsonRowDeserializationSchema createDeserializationSchema(
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
+	private static JsonRowDeserializationSchema createDeserializationSchema(TypeInformation<Row> typeInfo) {
 
-		return new JsonRowDeserializationSchema(fieldNames, fieldTypes);
+		return new JsonRowDeserializationSchema(typeInfo);
 	}
 }
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
index 506358d58d8cc..029aa453253b0 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
@@ -18,8 +18,8 @@
 
 package org.apache.flink.streaming.connectors.kafka;
 
+import java.util.Properties;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.RowTypeInfo;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.util.serialization.DeserializationSchema;
@@ -27,10 +27,6 @@
 import org.apache.flink.types.Row;
 import org.apache.flink.util.Preconditions;
 
-import java.util.Properties;
-
-import static org.apache.flink.streaming.connectors.kafka.internals.TypeUtil.toTypeInfo;
-
 /**
  * A version-agnostic Kafka {@link StreamTableSource}.
  *
@@ -48,30 +44,8 @@ public abstract class KafkaTableSource implements StreamTableSource<Row> {
 	/** Deserialization schema to use for Kafka records. */
 	private final DeserializationSchema<Row> deserializationSchema;
 
-	/** Row field names. */
-	private final String[] fieldNames;
-
-	/** Row field types. */
-	private final TypeInformation<?>[] fieldTypes;
-
-	/**
-	 * Creates a generic Kafka {@link StreamTableSource}.
-	 *
-	 * @param topic                 Kafka topic to consume.
-	 * @param properties            Properties for the Kafka consumer.
-	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
-	 */
-	KafkaTableSource(
-			String topic,
-			Properties properties,
-			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			Class<?>[] fieldTypes) {
-
-		this(topic, properties, deserializationSchema, fieldNames, toTypeInfo(fieldTypes));
-	}
+	/** Type information describing the result type. */
+	private final TypeInformation<Row> typeInfo;
 
 	/**
 	 * Creates a generic Kafka {@link StreamTableSource}.
@@ -79,24 +53,18 @@ public abstract class KafkaTableSource implements StreamTableSource<Row> {
 	 * @param topic                 Kafka topic to consume.
 	 * @param properties            Properties for the Kafka consumer.
 	 * @param deserializationSchema Deserialization schema to use for Kafka records.
-	 * @param fieldNames            Row field names.
-	 * @param fieldTypes            Row field types.
+	 * @param typeInfo              Type information describing the result type.
 	 */
 	KafkaTableSource(
 			String topic,
 			Properties properties,
 			DeserializationSchema<Row> deserializationSchema,
-			String[] fieldNames,
-			TypeInformation<?>[] fieldTypes) {
+			TypeInformation<Row> typeInfo) {
 
 		this.topic = Preconditions.checkNotNull(topic, "Topic");
 		this.properties = Preconditions.checkNotNull(properties, "Properties");
 		this.deserializationSchema = Preconditions.checkNotNull(deserializationSchema, "Deserialization schema");
-		this.fieldNames = Preconditions.checkNotNull(fieldNames, "Field names");
-		this.fieldTypes = Preconditions.checkNotNull(fieldTypes, "Field types");
-
-		Preconditions.checkArgument(fieldNames.length == fieldTypes.length,
-				"Number of provided field names and types does not match.");
+		this.typeInfo = Preconditions.checkNotNull(typeInfo, "Type information");
 	}
 
 	/**
@@ -107,13 +75,12 @@ public abstract class KafkaTableSource implements StreamTableSource<Row> {
 	public DataStream<Row> getDataStream(StreamExecutionEnvironment env) {
 		// Version-specific Kafka consumer
 		FlinkKafkaConsumerBase<Row> kafkaConsumer = getKafkaConsumer(topic, properties, deserializationSchema);
-		DataStream<Row> kafkaSource = env.addSource(kafkaConsumer);
-		return kafkaSource;
+		return env.addSource(kafkaConsumer);
 	}
 
 	@Override
 	public TypeInformation<Row> getReturnType() {
-		return new RowTypeInfo(fieldTypes, fieldNames);
+		return typeInfo;
 	}
 
 	/**
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/TypeUtil.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/TypeUtil.java
deleted file mode 100644
index 7a41ade5732bf..0000000000000
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/TypeUtil.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.connectors.kafka.internals;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-
-public class TypeUtil {
-	private TypeUtil() {}
-
-	/**
-	 * Creates TypeInformation array for an array of Classes.
-	 * @param fieldTypes classes to extract type information from
-	 * @return type information
-	 */
-	public static TypeInformation<?>[] toTypeInfo(Class<?>[] fieldTypes) {
-		TypeInformation<?>[] typeInfos = new TypeInformation[fieldTypes.length];
-		for (int i = 0; i < fieldTypes.length; i++) {
-			typeInfos[i] = TypeExtractor.getForClass(fieldTypes[i]);
-		}
-		return typeInfos;
-	}
-}
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowDeserializationSchema.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowDeserializationSchema.java
new file mode 100644
index 0000000000000..37241f56827e0
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowDeserializationSchema.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.util.serialization;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.specific.SpecificData;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.util.Utf8;
+import org.apache.flink.types.Row;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * Deserialization schema from Avro bytes over {@link SpecificRecord} to {@link Row}.
+ *
+ * Deserializes the <code>byte[]</code> messages into (nested) Flink Rows.
+ *
+ * {@link Utf8} is converted to regular Java Strings.
+ */
+public class AvroRowDeserializationSchema extends AbstractDeserializationSchema<Row> {
+
+	/**
+	 * Schema for deterministic field order.
+	 */
+	private final Schema schema;
+
+	/**
+	 * Reader that deserializes byte array into a record.
+	 */
+	private final DatumReader<SpecificRecord> datumReader;
+
+	/**
+	 * Input stream to read message from.
+	 */
+	private final MutableByteArrayInputStream inputStream;
+
+	/**
+	 * Avro decoder that decodes binary data
+	 */
+	private final Decoder decoder;
+
+	/**
+	 * Record to deserialize byte array to.
+	 */
+	private SpecificRecord record;
+
+	/**
+	 * Creates a Avro deserialization schema for the given record.
+	 *
+	 * @param recordClazz Avro record class used to deserialize Avro's record to Flink's row
+	 */
+	public AvroRowDeserializationSchema(Class<? extends SpecificRecord> recordClazz) {
+		Preconditions.checkNotNull(recordClazz, "Avro record class must not be null.");
+		this.schema = SpecificData.get().getSchema(recordClazz);
+		this.datumReader = new SpecificDatumReader<>(schema);
+		this.record = (SpecificRecord) SpecificData.newInstance(recordClazz, schema);
+		this.inputStream = new MutableByteArrayInputStream();
+		this.decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
+	}
+
+	@Override
+	public Row deserialize(byte[] message) throws IOException {
+		// read record
+		try {
+			inputStream.setBuffer(message);
+			this.record = datumReader.read(record, decoder);
+		} catch (IOException e) {
+			throw new RuntimeException("Failed to deserialize Row.", e);
+		}
+
+		// convert to row
+		final Object row = convertToRow(schema, record);
+		return (Row) row;
+	}
+
+	/**
+	 * Converts a (nested) Avro {@link SpecificRecord} into Flink's Row type.
+	 * Avro's {@link Utf8} fields are converted into regular Java strings.
+	 */
+	private static Object convertToRow(Schema schema, Object recordObj) {
+		if (recordObj instanceof GenericRecord) {
+			// records can be wrapped in a union
+			if (schema.getType() == Schema.Type.UNION) {
+				final List<Schema> types = schema.getTypes();
+				if (types.size() == 2 && types.get(0).getType() == Schema.Type.NULL && types.get(1).getType() == Schema.Type.RECORD) {
+					schema = types.get(1);
+				}
+				else {
+					throw new RuntimeException("Currently we only support schemas of the following form: UNION[null, RECORD]. Given: " + schema);
+				}
+			} else if (schema.getType() != Schema.Type.RECORD) {
+				throw new RuntimeException("Record type for row type expected. But is: " + schema);
+			}
+			final List<Schema.Field> fields = schema.getFields();
+			final Row row = new Row(fields.size());
+			final GenericRecord record = (GenericRecord) recordObj;
+			for (int i = 0; i < fields.size(); i++) {
+				final Schema.Field field = fields.get(i);
+				row.setField(i, convertToRow(field.schema(), record.get(field.pos())));
+			}
+			return row;
+		} else if (recordObj instanceof Utf8) {
+			return recordObj.toString();
+		} else {
+			return recordObj;
+		}
+	}
+
+	/**
+	 * An extension of the ByteArrayInputStream that allows to change a buffer that should be
+	 * read without creating a new ByteArrayInputStream instance. This allows to re-use the same
+	 * InputStream instance, copying message to process, and creation of Decoder on every new message.
+	 */
+	private static final class MutableByteArrayInputStream extends ByteArrayInputStream {
+		/**
+		 * Create MutableByteArrayInputStream
+		 */
+		public MutableByteArrayInputStream() {
+			super(new byte[0]);
+		}
+
+		/**
+		 * Set buffer that can be read via the InputStream interface and reset the input stream.
+		 * This has the same effect as creating a new ByteArrayInputStream with a new buffer.
+		 *
+		 * @param buf the new buffer to read.
+		 */
+		public void setBuffer(byte[] buf) {
+			this.buf = buf;
+			this.pos = 0;
+			this.count = buf.length;
+		}
+	}
+}
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowSerializationSchema.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowSerializationSchema.java
new file mode 100644
index 0000000000000..8388ab5ab1418
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/AvroRowSerializationSchema.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.util.serialization;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.specific.SpecificData;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.util.Utf8;
+import org.apache.flink.types.Row;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * Serialization schema that serializes {@link Row} over {@link SpecificRecord} into a Avro bytes.
+ */
+public class AvroRowSerializationSchema implements SerializationSchema<Row> {
+
+	/**
+	 * Avro serialization schema.
+	 */
+	private final Schema schema;
+
+	/**
+	 * Writer to serialize Avro record into a byte array.
+	 */
+	private final DatumWriter<GenericRecord> datumWriter;
+
+	/**
+	 * Output stream to serialize records into byte array.
+	 */
+	private final ByteArrayOutputStream arrayOutputStream =  new ByteArrayOutputStream();
+
+	/**
+	 * Low-level class for serialization of Avro values.
+	 */
+	private final Encoder encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
+
+	/**
+	 * Creates a Avro serialization schema for the given schema.
+	 *
+	 * @param recordClazz Avro record class used to deserialize Avro's record to Flink's row
+	 */
+	public AvroRowSerializationSchema(Class<? extends SpecificRecord> recordClazz) {
+		Preconditions.checkNotNull(recordClazz, "Avro record class must not be null.");
+		this.schema = SpecificData.get().getSchema(recordClazz);
+		this.datumWriter = new SpecificDatumWriter<>(schema);
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	public byte[] serialize(Row row) {
+		// convert to record
+		final Object record = convertToRecord(schema, row);
+
+		// write
+		try {
+			arrayOutputStream.reset();
+			datumWriter.write((GenericRecord) record, encoder);
+			encoder.flush();
+			return arrayOutputStream.toByteArray();
+		} catch (IOException e) {
+			throw new RuntimeException("Failed to serialize Row.", e);
+		}
+	}
+
+	/**
+	 * Converts a (nested) Flink Row into Avro's {@link GenericRecord}.
+	 * Strings are converted into Avro's {@link Utf8} fields.
+	 */
+	private static Object convertToRecord(Schema schema, Object rowObj) {
+		if (rowObj instanceof Row) {
+			// records can be wrapped in a union
+			if (schema.getType() == Schema.Type.UNION) {
+				final List<Schema> types = schema.getTypes();
+				if (types.size() == 2 && types.get(0).getType() == Schema.Type.NULL && types.get(1).getType() == Schema.Type.RECORD) {
+					schema = types.get(1);
+				}
+				else if (types.size() == 2 && types.get(0).getType() == Schema.Type.RECORD && types.get(1).getType() == Schema.Type.NULL) {
+					schema = types.get(0);
+				}
+				else {
+					throw new RuntimeException("Currently we only support schemas of the following form: UNION[null, RECORD] or UNION[RECORD, NULL] Given: " + schema);
+				}
+			} else if (schema.getType() != Schema.Type.RECORD) {
+				throw new RuntimeException("Record type for row type expected. But is: " + schema);
+			}
+			final List<Schema.Field> fields = schema.getFields();
+			final GenericRecord record = new GenericData.Record(schema);
+			final Row row = (Row) rowObj;
+			for (int i = 0; i < fields.size(); i++) {
+				final Schema.Field field = fields.get(i);
+				record.put(field.pos(), convertToRecord(field.schema(), row.getField(i)));
+			}
+			return record;
+		} else if (rowObj instanceof String) {
+			return new Utf8((String) rowObj);
+		} else {
+			return rowObj;
+		}
+	}
+}
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/JsonRowDeserializationSchema.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/JsonRowDeserializationSchema.java
index b4b3341fe75f6..be201fa3f9585 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/JsonRowDeserializationSchema.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/util/serialization/JsonRowDeserializationSchema.java
@@ -20,14 +20,12 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.types.Row;
 import org.apache.flink.api.java.typeutils.RowTypeInfo;
+import org.apache.flink.types.Row;
 import org.apache.flink.util.Preconditions;
 
-import java.io.IOException;
-
 /**
  * Deserialization schema from JSON to {@link Row}.
  *
@@ -38,6 +36,9 @@
  */
 public class JsonRowDeserializationSchema implements DeserializationSchema<Row> {
 
+	/** Type information describing the result type. */
+	private final TypeInformation<Row> typeInfo;
+
 	/** Field names to parse. Indices match fieldTypes indices. */
 	private final String[] fieldNames;
 
@@ -50,36 +51,18 @@ public class JsonRowDeserializationSchema implements DeserializationSchema<Row>
 	/** Flag indicating whether to fail on a missing field. */
 	private boolean failOnMissingField;
 
-	/**
-	 * Creates a JSON deserialization schema for the given fields and type classes.
-	 *
-	 * @param fieldNames Names of JSON fields to parse.
-	 * @param fieldTypes Type classes to parse JSON fields as.
-	 */
-	public JsonRowDeserializationSchema(String[] fieldNames, Class<?>[] fieldTypes) {
-		this.fieldNames = Preconditions.checkNotNull(fieldNames, "Field names");
-
-		this.fieldTypes = new TypeInformation[fieldTypes.length];
-		for (int i = 0; i < fieldTypes.length; i++) {
-			this.fieldTypes[i] = TypeExtractor.getForClass(fieldTypes[i]);
-		}
-
-		Preconditions.checkArgument(fieldNames.length == fieldTypes.length,
-				"Number of provided field names and types does not match.");
-	}
-
 	/**
 	 * Creates a JSON deserialization schema for the given fields and types.
 	 *
-	 * @param fieldNames Names of JSON fields to parse.
-	 * @param fieldTypes Types to parse JSON fields as.
+	 * @param typeInfo   Type information describing the result type. The field names are used
+	 *                   to parse the JSON file and so are the types.
 	 */
-	public JsonRowDeserializationSchema(String[] fieldNames, TypeInformation<?>[] fieldTypes) {
-		this.fieldNames = Preconditions.checkNotNull(fieldNames, "Field names");
-		this.fieldTypes = Preconditions.checkNotNull(fieldTypes, "Field types");
+	public JsonRowDeserializationSchema(TypeInformation<Row> typeInfo) {
+		Preconditions.checkNotNull(typeInfo, "Type information");
+		this.typeInfo = typeInfo;
 
-		Preconditions.checkArgument(fieldNames.length == fieldTypes.length,
-				"Number of provided field names and types does not match.");
+		this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames();
+		this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes();
 	}
 
 	@Override
@@ -118,7 +101,7 @@ public boolean isEndOfStream(Row nextElement) {
 
 	@Override
 	public TypeInformation<Row> getProducedType() {
-		return new RowTypeInfo(fieldTypes);
+		return typeInfo;
 	}
 
 	/**
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/AvroRowDeSerializationSchemaTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/AvroRowDeSerializationSchemaTest.java
new file mode 100644
index 0000000000000..e13968e6e2447
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/AvroRowDeSerializationSchemaTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.io.IOException;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.streaming.connectors.kafka.testutils.AvroTestUtils;
+import org.apache.flink.streaming.util.serialization.AvroRowDeserializationSchema;
+import org.apache.flink.streaming.util.serialization.AvroRowSerializationSchema;
+import org.apache.flink.types.Row;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+/**
+ * Test for the Avro serialization and deserialization schema.
+ */
+public class AvroRowDeSerializationSchemaTest {
+
+	@Test
+	public void testSerializeDeserializeSimpleRow() throws IOException {
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> testData = AvroTestUtils.getSimpleTestData();
+
+		final AvroRowSerializationSchema serializationSchema = new AvroRowSerializationSchema(testData.f0);
+		final AvroRowDeserializationSchema deserializationSchema = new AvroRowDeserializationSchema(testData.f0);
+
+		final byte[] bytes = serializationSchema.serialize(testData.f2);
+		final Row actual = deserializationSchema.deserialize(bytes);
+
+		assertEquals(testData.f2, actual);
+	}
+
+	@Test
+	public void testSerializeSimpleRowSeveralTimes() throws IOException {
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> testData = AvroTestUtils.getSimpleTestData();
+
+		final AvroRowSerializationSchema serializationSchema = new AvroRowSerializationSchema(testData.f0);
+		final AvroRowDeserializationSchema deserializationSchema = new AvroRowDeserializationSchema(testData.f0);
+
+		serializationSchema.serialize(testData.f2);
+		serializationSchema.serialize(testData.f2);
+		final byte[] bytes = serializationSchema.serialize(testData.f2);
+		final Row actual = deserializationSchema.deserialize(bytes);
+
+		assertEquals(testData.f2, actual);
+	}
+
+	@Test
+	public void testDeserializeRowSeveralTimes() throws IOException {
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> testData = AvroTestUtils.getSimpleTestData();
+
+		final AvroRowSerializationSchema serializationSchema = new AvroRowSerializationSchema(testData.f0);
+		final AvroRowDeserializationSchema deserializationSchema = new AvroRowDeserializationSchema(testData.f0);
+
+		final byte[] bytes = serializationSchema.serialize(testData.f2);
+		deserializationSchema.deserialize(bytes);
+		deserializationSchema.deserialize(bytes);
+		final Row actual = deserializationSchema.deserialize(bytes);
+
+		assertEquals(testData.f2, actual);
+	}
+
+	@Test
+	public void testSerializeDeserializeComplexRow() throws IOException {
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> testData = AvroTestUtils.getComplexTestData();
+
+		final AvroRowSerializationSchema serializationSchema = new AvroRowSerializationSchema(testData.f0);
+		final AvroRowDeserializationSchema deserializationSchema = new AvroRowDeserializationSchema(testData.f0);
+
+		final byte[] bytes = serializationSchema.serialize(testData.f2);
+		final Row actual = deserializationSchema.deserialize(bytes);
+
+		assertEquals(testData.f2, actual);
+	}
+
+	@Test
+	public void testSerializeComplexRowSeveralTimes() throws IOException {
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> testData = AvroTestUtils.getComplexTestData();
+
+		final AvroRowSerializationSchema serializationSchema = new AvroRowSerializationSchema(testData.f0);
+		final AvroRowDeserializationSchema deserializationSchema = new AvroRowDeserializationSchema(testData.f0);
+
+		serializationSchema.serialize(testData.f2);
+		serializationSchema.serialize(testData.f2);
+		final byte[] bytes = serializationSchema.serialize(testData.f2);
+		final Row actual = deserializationSchema.deserialize(bytes);
+
+		assertEquals(testData.f2, actual);
+	}
+
+	@Test
+	public void testDeserializeComplexRowSeveralTimes() throws IOException {
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> testData = AvroTestUtils.getComplexTestData();
+
+		final AvroRowSerializationSchema serializationSchema = new AvroRowSerializationSchema(testData.f0);
+		final AvroRowDeserializationSchema deserializationSchema = new AvroRowDeserializationSchema(testData.f0);
+
+		final byte[] bytes = serializationSchema.serialize(testData.f2);
+		deserializationSchema.deserialize(bytes);
+		deserializationSchema.deserialize(bytes);
+		final Row actual = deserializationSchema.deserialize(bytes);
+
+		assertEquals(testData.f2, actual);
+	}
+}
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowDeserializationSchemaTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowDeserializationSchemaTest.java
index 88f62f031ec2d..f03feeb26d9d1 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowDeserializationSchemaTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowDeserializationSchemaTest.java
@@ -20,6 +20,8 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.api.Types;
 import org.apache.flink.types.Row;
 import org.apache.flink.streaming.util.serialization.JsonRowDeserializationSchema;
 import org.junit.Test;
@@ -56,8 +58,11 @@ public void testDeserialization() throws Exception {
 		byte[] serializedJson = objectMapper.writeValueAsBytes(root);
 
 		JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(
+			Types.ROW(
 				new String[] { "id", "name", "bytes" },
-				new Class<?>[] { Long.class, String.class, byte[].class });
+				new TypeInformation<?>[] { Types.LONG(), Types.STRING(), Types.PRIMITIVE_ARRAY(Types.BYTE()) }
+			)
+		);
 
 		Row deserialized = deserializationSchema.deserialize(serializedJson);
 
@@ -80,8 +85,11 @@ public void testMissingNode() throws Exception {
 		byte[] serializedJson = objectMapper.writeValueAsBytes(root);
 
 		JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(
+			Types.ROW(
 				new String[] { "name" },
-				new Class<?>[] { String.class });
+				new TypeInformation<?>[] { Types.STRING() }
+			)
+		);
 
 		Row row = deserializationSchema.deserialize(serializedJson);
 
@@ -105,17 +113,11 @@ public void testMissingNode() throws Exception {
 	public void testNumberOfFieldNamesAndTypesMismatch() throws Exception {
 		try {
 			new JsonRowDeserializationSchema(
+				Types.ROW(
 					new String[] { "one", "two", "three" },
-					new Class<?>[] { Long.class });
-			fail("Did not throw expected Exception");
-		} catch (IllegalArgumentException ignored) {
-			// Expected
-		}
-
-		try {
-			new JsonRowDeserializationSchema(
-					new String[] { "one" },
-					new Class<?>[] { Long.class, String.class });
+					new TypeInformation<?>[] { Types.LONG() }
+				)
+			);
 			fail("Did not throw expected Exception");
 		} catch (IllegalArgumentException ignored) {
 			// Expected
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowSerializationSchemaTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowSerializationSchemaTest.java
index 78dedf4f9ed78..523eafef16c00 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowSerializationSchemaTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/JsonRowSerializationSchemaTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.flink.streaming.connectors.kafka;
 
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.api.Types;
 import org.apache.flink.types.Row;
 import org.apache.flink.streaming.util.serialization.JsonRowDeserializationSchema;
 import org.apache.flink.streaming.util.serialization.JsonRowSerializationSchema;
@@ -26,10 +28,11 @@
 import static org.junit.Assert.assertEquals;
 
 public class JsonRowSerializationSchemaTest {
+
 	@Test
 	public void testRowSerialization() throws IOException {
 		String[] fieldNames = new String[] {"f1", "f2", "f3"};
-		Class[] fieldTypes = new Class[] {Integer.class, Boolean.class, String.class};
+		TypeInformation<?>[] fieldTypes = new TypeInformation<?>[] { Types.INT(), Types.BOOLEAN(), Types.STRING() };
 		Row row = new Row(3);
 		row.setField(0, 1);
 		row.setField(1, true);
@@ -42,14 +45,17 @@ public void testRowSerialization() throws IOException {
 	@Test
 	public void testSerializationOfTwoRows() throws IOException {
 		String[] fieldNames = new String[] {"f1", "f2", "f3"};
-		Class[] fieldTypes = new Class[] {Integer.class, Boolean.class, String.class};
+		TypeInformation<Row> row = Types.ROW(
+			fieldNames,
+			new TypeInformation<?>[] { Types.INT(), Types.BOOLEAN(), Types.STRING() }
+		);
 		Row row1 = new Row(3);
 		row1.setField(0, 1);
 		row1.setField(1, true);
 		row1.setField(2, "str");
 
 		JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema(fieldNames);
-		JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(fieldNames, fieldTypes);
+		JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(row);
 
 		byte[] bytes = serializationSchema.serialize(row1);
 		assertEqualRows(row1, deserializationSchema.deserialize(bytes));
@@ -78,9 +84,9 @@ public void testSerializeRowWithInvalidNumberOfFields() {
 		serializationSchema.serialize(row);
 	}
 
-	private Row serializeAndDeserialize(String[] fieldNames, Class[] fieldTypes, Row row) throws IOException {
+	private Row serializeAndDeserialize(String[] fieldNames, TypeInformation<?>[] fieldTypes, Row row) throws IOException {
 		JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema(fieldNames);
-		JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(fieldNames, fieldTypes);
+		JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(Types.ROW(fieldNames, fieldTypes));
 
 		byte[] bytes = serializationSchema.serialize(row);
 		return deserializationSchema.deserialize(bytes);
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkTestBase.java
index cc1c16601633a..e0e8f842d536e 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkTestBase.java
@@ -18,10 +18,10 @@
 package org.apache.flink.streaming.connectors.kafka;
 
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.api.Types;
 import org.apache.flink.types.Row;
 import org.apache.flink.api.java.typeutils.RowTypeInfo;
 import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.connectors.kafka.internals.TypeUtil;
 import org.apache.flink.streaming.connectors.kafka.partitioner.KafkaPartitioner;
 import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper;
 import org.apache.flink.streaming.util.serialization.SerializationSchema;
@@ -43,7 +43,7 @@ public abstract class KafkaTableSinkTestBase {
 
 	private static final String TOPIC = "testTopic";
 	protected static final String[] FIELD_NAMES = new String[] {"field1", "field2"};
-	private static final TypeInformation[] FIELD_TYPES = TypeUtil.toTypeInfo(new Class[] {Integer.class, String.class});
+	private static final TypeInformation[] FIELD_TYPES = new TypeInformation[] { Types.INT(), Types.STRING() };
 	private static final KafkaPartitioner<Row> PARTITIONER = new CustomPartitioner();
 	private static final Properties PROPERTIES = createSinkProperties();
 	@SuppressWarnings("unchecked")
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceTestBase.java
index ad5199362dc6e..341df45c54248 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceTestBase.java
@@ -19,8 +19,12 @@
 package org.apache.flink.streaming.connectors.kafka;
 
 import java.util.Properties;
+import org.apache.avro.Schema;
+import org.apache.avro.specific.SpecificRecordBase;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.connectors.kafka.testutils.AvroTestUtils;
+import org.apache.flink.table.api.Types;
 import org.apache.flink.types.Row;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.util.serialization.DeserializationSchema;
@@ -34,7 +38,7 @@
 public abstract class KafkaTableSourceTestBase {
 
 	private static final String TOPIC = "testTopic";
-	private static final String[] FIELD_NAMES = new String[] { "long", "string", "boolean", "double", "missing-field" };
+	private static final String[] FIELD_NAMES = new String[] { "mylong", "mystring", "myboolean", "mydouble", "missingField" };
 	private static final TypeInformation<?>[] FIELD_TYPES = new TypeInformation<?>[] {
 		BasicTypeInfo.LONG_TYPE_INFO,
 		BasicTypeInfo.STRING_TYPE_INFO,
@@ -43,6 +47,33 @@ public abstract class KafkaTableSourceTestBase {
 		BasicTypeInfo.LONG_TYPE_INFO };
 	private static final Properties PROPERTIES = createSourceProperties();
 
+	// Avro record that matches above schema
+	public static class AvroSpecificRecord extends SpecificRecordBase {
+
+		public static Schema SCHEMA$ = AvroTestUtils.createFlatAvroSchema(FIELD_NAMES, FIELD_TYPES);
+
+		public Long mylong;
+		public String mystring;
+		public Boolean myboolean;
+		public Double mydouble;
+		public Long missingField;
+
+		@Override
+		public Schema getSchema() {
+			return null;
+		}
+
+		@Override
+		public Object get(int field) {
+			return null;
+		}
+
+		@Override
+		public void put(int field, Object value) {
+
+		}
+	}
+
 	@Test
 	public void testKafkaTableSource() {
 		KafkaTableSource kafkaTableSource = spy(createTableSource());
@@ -57,15 +88,14 @@ public void testKafkaTableSource() {
 			any(getDeserializationSchema()));
 	}
 
-	protected abstract KafkaTableSource createTableSource(String topic, Properties properties,
-			String[] fieldNames, TypeInformation<?>[] typeInfo);
+	protected abstract KafkaTableSource createTableSource(String topic, Properties properties, TypeInformation<Row> typeInfo);
 
 	protected abstract Class<DeserializationSchema<Row>> getDeserializationSchema();
 
 	protected abstract Class<FlinkKafkaConsumerBase<Row>> getFlinkKafkaConsumer();
 
 	private KafkaTableSource createTableSource() {
-		return createTableSource(TOPIC, PROPERTIES, FIELD_NAMES, FIELD_TYPES);
+		return createTableSource(TOPIC, PROPERTIES, Types.ROW(FIELD_NAMES, FIELD_TYPES));
 	}
 
 	private static Properties createSourceProperties() {
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/AvroTestUtils.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/AvroTestUtils.java
new file mode 100644
index 0000000000000..075b79b8e93a6
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/AvroTestUtils.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka.testutils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.io.avro.generated.Address;
+import org.apache.flink.api.io.avro.generated.Colors;
+import org.apache.flink.api.io.avro.generated.User;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.types.Row;
+
+/**
+ * Utilities for creating Avro Schemas.
+ */
+public final class AvroTestUtils {
+
+	private static String NAMESPACE = "org.apache.flink.streaming.connectors.kafka";
+
+	/**
+	 * Creates a flat Avro Schema for testing.
+	 */
+	public static Schema createFlatAvroSchema(String[] fieldNames, TypeInformation[] fieldTypes) {
+		final SchemaBuilder.FieldAssembler<Schema> fieldAssembler = SchemaBuilder
+			.record("BasicAvroRecord")
+			.namespace(NAMESPACE)
+			.fields();
+
+		final Schema nullSchema = Schema.create(Schema.Type.NULL);
+
+		for (int i = 0; i < fieldNames.length; i++) {
+			Schema schema = ReflectData.get().getSchema(fieldTypes[i].getTypeClass());
+			Schema unionSchema = Schema.createUnion(Arrays.asList(nullSchema, schema));
+			fieldAssembler.name(fieldNames[i]).type(unionSchema).noDefault();
+		}
+
+		return fieldAssembler.endRecord();
+	}
+
+	/**
+	 * Tests a simple Avro data types without nesting.
+	 */
+	public static Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> getSimpleTestData() {
+		final Address addr = Address.newBuilder()
+			.setNum(42)
+			.setStreet("Main Street 42")
+			.setCity("Test City")
+			.setState("Test State")
+			.setZip("12345")
+			.build();
+
+		final Row rowAddr = new Row(5);
+		rowAddr.setField(0, 42);
+		rowAddr.setField(1, "Main Street 42");
+		rowAddr.setField(2, "Test City");
+		rowAddr.setField(3, "Test State");
+		rowAddr.setField(4, "12345");
+
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> t = new Tuple3<>();
+		t.f0 = Address.class;
+		t.f1 = addr;
+		t.f2 = rowAddr;
+
+		return t;
+	}
+
+	/**
+	 * Tests all Avro data types as well as nested types.
+	 */
+	public static Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> getComplexTestData() {
+		final Address addr = Address.newBuilder()
+			.setNum(42)
+			.setStreet("Main Street 42")
+			.setCity("Test City")
+			.setState("Test State")
+			.setZip("12345")
+			.build();
+
+		final Row rowAddr = new Row(5);
+		rowAddr.setField(0, 42);
+		rowAddr.setField(1, "Main Street 42");
+		rowAddr.setField(2, "Test City");
+		rowAddr.setField(3, "Test State");
+		rowAddr.setField(4, "12345");
+
+		final User user = User.newBuilder()
+			.setName("Charlie")
+			.setFavoriteNumber(null)
+			.setFavoriteColor("blue")
+			.setTypeLongTest(1337L)
+			.setTypeDoubleTest(1.337d)
+			.setTypeNullTest(null)
+			.setTypeBoolTest(false)
+			.setTypeArrayString(new ArrayList<CharSequence>())
+			.setTypeArrayBoolean(new ArrayList<Boolean>())
+			.setTypeNullableArray(null)
+			.setTypeEnum(Colors.RED)
+			.setTypeMap(new HashMap<CharSequence, Long>())
+			.setTypeFixed(null)
+			.setTypeUnion(null)
+			.setTypeNested(addr)
+			.build();
+
+		final Row rowUser = new Row(15);
+		rowUser.setField(0, "Charlie");
+		rowUser.setField(1, null);
+		rowUser.setField(2, "blue");
+		rowUser.setField(3, 1337L);
+		rowUser.setField(4, 1.337d);
+		rowUser.setField(5, null);
+		rowUser.setField(6, false);
+		rowUser.setField(7, new ArrayList<CharSequence>());
+		rowUser.setField(8, new ArrayList<Boolean>());
+		rowUser.setField(9, null);
+		rowUser.setField(10, Colors.RED);
+		rowUser.setField(11, new HashMap<CharSequence, Long>());
+		rowUser.setField(12, null);
+		rowUser.setField(13, null);
+		rowUser.setField(14, rowAddr);
+
+		final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> t = new Tuple3<>();
+		t.f0 = User.class;
+		t.f1 = user;
+		t.f2 = rowUser;
+
+		return t;
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java
index dcbb645b6f1cd..e19cdd88d0b59 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java
@@ -48,7 +48,7 @@ public class Types {
 
 	/**
 	 * Generates a RowTypeInfo with fields of the given types.
-	 * The fields have the default names (f1, f2 ..).
+	 * The fields have the default names (f0, f1, f2 ..).
 	 * 
 	 * <p>This method is a shortcut to {@code new RowTypeInfo(types)}.
 	 *
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java
index a1b348a5ee8a1..1d4b8a151e63c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java
@@ -267,6 +267,13 @@ public String toString() {
 		return bld.toString();
 	}
 
+	/**
+	 * Returns the field types of the row. The order matches the order of the field names.
+	 */
+	public TypeInformation<?>[] getFieldTypes() {
+		return types;
+	}
+
 	private boolean hasDuplicateFieldNames(String[] fieldNames) {
 		HashSet<String> names = new HashSet<>();
 		for (String field : fieldNames) {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
index d82b9907efccb..f22fa3242dbc0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
@@ -17,13 +17,17 @@
  */
 package org.apache.flink.table.api
 
-import org.apache.flink.api.common.typeinfo.{Types => JTypes, TypeInformation}
+import org.apache.flink.api.common.typeinfo.{PrimitiveArrayTypeInfo, TypeInformation, Types => JTypes}
+import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
+import org.apache.flink.types.Row
+
+import _root_.scala.annotation.varargs
 
 /**
   * This class enumerates all supported types of the Table API.
   */
-object Types extends JTypes {
+object Types {
 
   val STRING = JTypes.STRING
   val BOOLEAN = JTypes.BOOLEAN
@@ -43,24 +47,57 @@ object Types extends JTypes {
   val INTERVAL_MILLIS = TimeIntervalTypeInfo.INTERVAL_MILLIS
 
   /**
-    * Generates RowTypeInfo with default names (f1, f2 ..).
-    * same as ``new RowTypeInfo(types)``
+    * Generates row type information.
+    *
+    * A row type consists of zero or more fields with a field name and a corresponding type.
     *
-    * @param types of Row fields. e.g. ROW(Types.STRING, Types.INT)
+    * The fields have the default names (f0, f1, f2 ..).
+    *
+    * @param types types of row fields; e.g. Types.STRING, Types.INT
     */
-  def ROW[T: Manifest](types: TypeInformation[_]*) = {
+  @varargs
+  def ROW(types: TypeInformation[_]*): TypeInformation[Row] = {
     JTypes.ROW(types: _*)
   }
 
   /**
-    * Generates RowTypeInfo.
-    * same as ``new RowTypeInfo(types, names)``
+    * Generates row type information.
+    *
+    * A row type consists of zero or more fields with a field name and a corresponding type.
     *
-    * @param fields of Row. e.g. ROW(("name", Types.STRING), ("number", Types.INT))
+    * @param names names of row fields, e.g. "userid", "name"
+    * @param types types of row fields; e.g. Types.STRING, Types.INT
     */
-  def ROW_NAMED(fields: (String, TypeInformation[_])*) = {
-    val names = fields.toList.map(_._1).toArray
-    val types = fields.toList.map(_._2)
+  def ROW(names: Array[String], types: Array[TypeInformation[_]]): TypeInformation[Row] = {
     JTypes.ROW_NAMED(names, types: _*)
   }
+
+  /**
+    * Generates type information for an array consisting of Java primitive elements.
+    *
+    * @param elementType type of the array elements; e.g. Types.INT
+    */
+  def PRIMITIVE_ARRAY(elementType: TypeInformation[_]): TypeInformation[_] = {
+    elementType match {
+      case BOOLEAN =>  PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO
+      case BYTE =>  PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO
+      case SHORT =>  PrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO
+      case INT =>  PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO
+      case LONG =>  PrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO
+      case FLOAT =>  PrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO
+      case DOUBLE =>  PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO
+      case _ =>
+        throw new TableException(s"$elementType cannot be an element of a primitive array." +
+            s"Only Java primitive types are supported.")
+    }
+  }
+
+  /**
+    * Generates type information for an array consisting of Java object elements.
+    *
+    * @param elementType type of the array elements; e.g. Types.STRING or Types.INT
+    */
+  def OBJECT_ARRAY(elementType: TypeInformation[_]): TypeInformation[_] = {
+    ObjectArrayTypeInfo.getInfoFor(elementType)
+  }
 }

From cdf7d26afef423898e2ba0409ce23d9bae1a1337 Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Wed, 5 Oct 2016 16:13:02 -0400
Subject: [PATCH 061/253] [FLINK-3722] [runtime] Don't / and % when sorting

Replace division and modulus with addition and subtraction.

This closes #2628
---
 .../sort/FixedLengthRecordSorter.java         |  59 ++++---
 .../operators/sort/IndexedSortable.java       |  38 +++++
 .../operators/sort/NormalizedKeySorter.java   |  66 +++++---
 .../runtime/operators/sort/QuickSort.java     | 147 ++++++++++++++----
 4 files changed, 234 insertions(+), 76 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java
index 3a44ab5b12bc1..22dfd29fee3cc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java
@@ -19,11 +19,6 @@
 
 package org.apache.flink.runtime.operators.sort;
 
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
 import org.apache.flink.api.common.typeutils.TypeComparator;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.MemorySegment;
@@ -32,6 +27,11 @@
 import org.apache.flink.runtime.memory.AbstractPagedOutputView;
 import org.apache.flink.util.MutableObjectIterator;
 
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * 
  */
@@ -134,6 +134,17 @@ public FixedLengthRecordSorter(TypeSerializer<T> serializer, TypeComparator<T> c
 		this.recordInstance = this.serializer.createInstance();
 	}
 
+	@Override
+	public int recordSize() {
+		return recordSize;
+	}
+
+	@Override
+	public int recordsPerSegment() {
+		return recordsPerSegment;
+	}
+
+
 	// -------------------------------------------------------------------------
 	// Memory Segment
 	// -------------------------------------------------------------------------
@@ -254,30 +265,40 @@ private MemorySegment nextMemorySegment() {
 
 	@Override
 	public int compare(int i, int j) {
-		final int bufferNumI = i / this.recordsPerSegment;
+		final int segmentNumberI = i / this.recordsPerSegment;
 		final int segmentOffsetI = (i % this.recordsPerSegment) * this.recordSize;
-		
-		final int bufferNumJ = j / this.recordsPerSegment;
+
+		final int segmentNumberJ = j / this.recordsPerSegment;
 		final int segmentOffsetJ = (j % this.recordsPerSegment) * this.recordSize;
-		
-		final MemorySegment segI = this.sortBuffer.get(bufferNumI);
-		final MemorySegment segJ = this.sortBuffer.get(bufferNumJ);
-		
+
+		return compare(segmentNumberI, segmentOffsetI, segmentNumberJ, segmentOffsetJ);
+	}
+
+	@Override
+	public int compare(int segmentNumberI, int segmentOffsetI, int segmentNumberJ, int segmentOffsetJ) {
+		final MemorySegment segI = this.sortBuffer.get(segmentNumberI);
+		final MemorySegment segJ = this.sortBuffer.get(segmentNumberJ);
+
 		int val = segI.compare(segJ, segmentOffsetI, segmentOffsetJ, this.numKeyBytes);
 		return this.useNormKeyUninverted ? val : -val;
 	}
 
 	@Override
 	public void swap(int i, int j) {
-		final int bufferNumI = i / this.recordsPerSegment;
+		final int segmentNumberI = i / this.recordsPerSegment;
 		final int segmentOffsetI = (i % this.recordsPerSegment) * this.recordSize;
-		
-		final int bufferNumJ = j / this.recordsPerSegment;
+
+		final int segmentNumberJ = j / this.recordsPerSegment;
 		final int segmentOffsetJ = (j % this.recordsPerSegment) * this.recordSize;
-		
-		final MemorySegment segI = this.sortBuffer.get(bufferNumI);
-		final MemorySegment segJ = this.sortBuffer.get(bufferNumJ);
-		
+
+		swap(segmentNumberI, segmentOffsetI, segmentNumberJ, segmentOffsetJ);
+	}
+
+	@Override
+	public void swap(int segmentNumberI, int segmentOffsetI, int segmentNumberJ, int segmentOffsetJ) {
+		final MemorySegment segI = this.sortBuffer.get(segmentNumberI);
+		final MemorySegment segJ = this.sortBuffer.get(segmentNumberJ);
+
 		segI.swapBytes(this.swapBuffer, segJ, segmentOffsetI, segmentOffsetJ, this.recordSize);
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/IndexedSortable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/IndexedSortable.java
index 6d7d49979ad71..c09cf033163ce 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/IndexedSortable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/IndexedSortable.java
@@ -29,11 +29,35 @@ public interface IndexedSortable {
 	 */
 	int compare(int i, int j);
 
+	/**
+	 * Compare records at the given addresses consistent with the semantics of
+	 * {@link java.util.Comparator#compare(Object, Object)}.
+
+	 * @param segmentNumberI index of memory segment containing first record
+	 * @param segmentOffsetI offset into memory segment containing first record
+	 * @param segmentNumberJ index of memory segment containing second record
+	 * @param segmentOffsetJ offset into memory segment containing second record
+	 * @return a negative integer, zero, or a positive integer as the
+	 *         first argument is less than, equal to, or greater than the
+	 *         second.
+	 */
+	int compare(int segmentNumberI, int segmentOffsetI, int segmentNumberJ, int segmentOffsetJ);
+
 	/**
 	 * Swap items at the given addresses.
 	 */
 	void swap(int i, int j);
 
+	/**
+	 * Swap records at the given addresses.
+	 *
+	 * @param segmentNumberI index of memory segment containing first record
+	 * @param segmentOffsetI offset into memory segment containing first record
+	 * @param segmentNumberJ index of memory segment containing second record
+	 * @param segmentOffsetJ offset into memory segment containing second record
+	 */
+	void swap(int segmentNumberI, int segmentOffsetI, int segmentNumberJ, int segmentOffsetJ);
+
 	/**
 	 * Gets the number of elements in the sortable.
 	 * 
@@ -41,4 +65,18 @@ public interface IndexedSortable {
 	 */
 	int size();
 
+	/**
+	 * Gets the size of each record, the number of bytes separating the head
+	 * of successive records.
+	 *
+	 * @return The record size
+	 */
+	int recordSize();
+
+	/**
+	 * Gets the number of elements in each memory segment.
+	 *
+	 * @return The number of records per segment
+	 */
+	int recordsPerSegment();
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/NormalizedKeySorter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/NormalizedKeySorter.java
index 2cade8d514316..0fd6f388f4cf5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/NormalizedKeySorter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/NormalizedKeySorter.java
@@ -18,11 +18,6 @@
 
 package org.apache.flink.runtime.operators.sort;
 
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
 import org.apache.flink.api.common.typeutils.TypeComparator;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.MemorySegment;
@@ -34,6 +29,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * 
  */
@@ -167,7 +167,7 @@ public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> compa
 		
 		// compute the index entry size and limits
 		this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
-		this.indexEntriesPerSegment = segmentSize / this.indexEntrySize;
+		this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
 		this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
 		this.swapBuffer = new byte[this.indexEntrySize];
 		
@@ -176,6 +176,16 @@ public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> compa
 		this.sortIndex.add(this.currentSortIndexSegment);
 	}
 
+	@Override
+	public int recordSize() {
+		return indexEntrySize;
+	}
+
+	@Override
+	public int recordsPerSegment() {
+		return indexEntriesPerSegment;
+	}
+
 	// -------------------------------------------------------------------------
 	// Memory Segment
 	// -------------------------------------------------------------------------
@@ -345,38 +355,48 @@ private MemorySegment nextMemorySegment() {
 
 	@Override
 	public int compare(int i, int j) {
-		final int bufferNumI = i / this.indexEntriesPerSegment;
+		final int segmentNumberI = i / this.indexEntriesPerSegment;
 		final int segmentOffsetI = (i % this.indexEntriesPerSegment) * this.indexEntrySize;
-		
-		final int bufferNumJ = j / this.indexEntriesPerSegment;
+
+		final int segmentNumberJ = j / this.indexEntriesPerSegment;
 		final int segmentOffsetJ = (j % this.indexEntriesPerSegment) * this.indexEntrySize;
-		
-		final MemorySegment segI = this.sortIndex.get(bufferNumI);
-		final MemorySegment segJ = this.sortIndex.get(bufferNumJ);
-		
+
+		return compare(segmentNumberI, segmentOffsetI, segmentNumberJ, segmentOffsetJ);
+	}
+
+	@Override
+	public int compare(int segmentNumberI, int segmentOffsetI, int segmentNumberJ, int segmentOffsetJ) {
+		final MemorySegment segI = this.sortIndex.get(segmentNumberI);
+		final MemorySegment segJ = this.sortIndex.get(segmentNumberJ);
+
 		int val = segI.compare(segJ, segmentOffsetI + OFFSET_LEN, segmentOffsetJ + OFFSET_LEN, this.numKeyBytes);
-		
+
 		if (val != 0 || this.normalizedKeyFullyDetermines) {
 			return this.useNormKeyUninverted ? val : -val;
 		}
-		
+
 		final long pointerI = segI.getLong(segmentOffsetI) & POINTER_MASK;
 		final long pointerJ = segJ.getLong(segmentOffsetJ) & POINTER_MASK;
-		
+
 		return compareRecords(pointerI, pointerJ);
 	}
 
 	@Override
 	public void swap(int i, int j) {
-		final int bufferNumI = i / this.indexEntriesPerSegment;
+		final int segmentNumberI = i / this.indexEntriesPerSegment;
 		final int segmentOffsetI = (i % this.indexEntriesPerSegment) * this.indexEntrySize;
-		
-		final int bufferNumJ = j / this.indexEntriesPerSegment;
+
+		final int segmentNumberJ = j / this.indexEntriesPerSegment;
 		final int segmentOffsetJ = (j % this.indexEntriesPerSegment) * this.indexEntrySize;
-		
-		final MemorySegment segI = this.sortIndex.get(bufferNumI);
-		final MemorySegment segJ = this.sortIndex.get(bufferNumJ);
-		
+
+		swap(segmentNumberI, segmentOffsetI, segmentNumberJ, segmentOffsetJ);
+	}
+
+	@Override
+	public void swap(int segmentNumberI, int segmentOffsetI, int segmentNumberJ, int segmentOffsetJ) {
+		final MemorySegment segI = this.sortIndex.get(segmentNumberI);
+		final MemorySegment segJ = this.sortIndex.get(segmentNumberJ);
+
 		segI.swapBytes(this.swapBuffer, segJ, segmentOffsetI, segmentOffsetJ, this.indexEntrySize);
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/QuickSort.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/QuickSort.java
index 474644e21967f..80ba6d348e6e1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/QuickSort.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/QuickSort.java
@@ -26,9 +26,19 @@ public final class QuickSort implements IndexedSorter {
 	public QuickSort() {
 	}
 
-	private static void fix(IndexedSortable s, int p, int r) {
-		if (s.compare(p, r) > 0) {
-			s.swap(p, r);
+	/**
+	 * Fix the records into sorted order, swapping when the first record is
+	 * greater than the second record.
+	 *
+	 * @param s paged sortable
+	 * @param pN page number of first record
+	 * @param pO page offset of first record
+	 * @param rN page number of second record
+	 * @param rO page offset of second record
+	 */
+	private static void fix(IndexedSortable s, int pN, int pO, int rN, int rO) {
+		if (s.compare(pN, pO, rN, rO) > 0) {
+			s.swap(pN, pO, rN, rO);
 		}
 	}
 
@@ -45,85 +55,154 @@ protected static int getMaxDepth(int x) {
 
 	/**
 	 * Sort the given range of items using quick sort. {@inheritDoc} If the recursion depth falls below
-	 * {@link #getMaxDepth},
-	 * then switch to {@link HeapSort}.
+	 * {@link #getMaxDepth}, then switch to {@link HeapSort}.
 	 */
 	public void sort(final IndexedSortable s, int p, int r) {
-		sortInternal(s, p, r, getMaxDepth(r - p));
+		int recordsPerSegment = s.recordsPerSegment();
+		int recordSize = s.recordSize();
+		int maxOffset = recordSize * (recordsPerSegment - 1);
+
+		int pN = p / recordsPerSegment;
+		int pO = (p % recordsPerSegment) * recordSize;
+
+		int rN = r / recordsPerSegment;
+		int rO = (r % recordsPerSegment) * recordSize;
+
+		sortInternal(s, recordsPerSegment, recordSize, maxOffset, p, pN, pO, r, rN, rO, getMaxDepth(r - p));
 	}
 
 	public void sort(IndexedSortable s) {
 		sort(s, 0, s.size());
 	}
 
-	private static void sortInternal(final IndexedSortable s, int p, int r, int depth) {
+	/**
+	 * Sort the given range of items using quick sort. If the recursion depth falls below
+	 * {@link #getMaxDepth}, then switch to {@link HeapSort}.
+	 *
+	 * @param s paged sortable
+	 * @param recordsPerSegment number of records per memory segment
+	 * @param recordSize number of bytes per record
+	 * @param maxOffset offset of a last record in a memory segment
+	 * @param p index of first record in range
+	 * @param pN page number of first record in range
+	 * @param pO page offset of first record in range
+	 * @param r index of last-plus-one'th record in range
+	 * @param rN page number of last-plus-one'th record in range
+	 * @param rO page offset of last-plus-one'th record in range
+	 * @param depth recursion depth
+	 *
+	 * @see #sort(IndexedSortable, int, int)
+	 */
+	private static void sortInternal(final IndexedSortable s, int recordsPerSegment, int recordSize, int maxOffset,
+			int p, int pN, int pO, int r, int rN, int rO, int depth) {
 		while (true) {
 			if (r - p < 13) {
-				for (int i = p; i < r; ++i) {
-					for (int j = i; j > p && s.compare(j - 1, j) > 0; --j) {
-						s.swap(j, j - 1);
+				// switch to insertion sort
+				int i = p+1, iN, iO; if (pO == maxOffset) { iN = pN+1; iO = 0; } else { iN = pN; iO = pO+recordSize; }
+
+				while (i < r) {
+					int j = i, jN = iN, jO = iO;
+					int jd = j-1, jdN, jdO; if (jO == 0) { jdN = jN-1; jdO = maxOffset; } else { jdN = jN; jdO = jO-recordSize; }
+
+					while (j > p && s.compare(jdN, jdO, jN, jO) > 0) {
+						s.swap(jN, jO, jdN, jdO);
+
+						j = jd; jN = jdN; jO = jdO;
+						jd--; if (jdO == 0) { jdN--; jdO = maxOffset; } else { jdO -= recordSize; }
 					}
+
+					i++; if (iO == maxOffset) { iN++; iO = 0; } else { iO += recordSize; }
 				}
 				return;
 			}
+
 			if (--depth < 0) {
-				// give up
+				// switch to heap sort
 				alt.sort(s, p, r);
 				return;
 			}
 
+			int rdN, rdO; if (rO == 0) { rdN = rN-1; rdO = maxOffset; } else { rdN = rN; rdO = rO-recordSize; }
+			int m = (p+r)>>>1, mN = m / recordsPerSegment, mO = (m % recordsPerSegment) * recordSize;
+
 			// select, move pivot into first position
-			fix(s, (p + r) >>> 1, p);
-			fix(s, (p + r) >>> 1, r - 1);
-			fix(s, p, r - 1);
+			fix(s, mN, mO, pN, pO);
+			fix(s, mN, mO, rdN, rdO);
+			fix(s, pN, pO, rdN, rdO);
 
 			// Divide
-			int i = p;
-			int j = r;
-			int ll = p;
-			int rr = r;
+			int i = p, iN = pN, iO = pO;
+			int j = r, jN = rN, jO = rO;
+			int ll = p, llN = pN, llO = pO;
+			int rr = r, rrN = rN, rrO = rO;
 			int cr;
 			while (true) {
-				while (++i < j) {
-					if ((cr = s.compare(i, p)) > 0) {
+				i++; if (iO == maxOffset) { iN++; iO = 0; } else { iO += recordSize; }
+
+				while (i < j) {
+					if ((cr = s.compare(iN, iO, pN, pO)) > 0) {
 						break;
 					}
-					if (0 == cr && ++ll != i) {
-						s.swap(ll, i);
+
+					if (0 == cr) {
+						ll++; if (llO == maxOffset) { llN++; llO = 0; } else { llO += recordSize; }
+
+						if (ll != i) {
+							s.swap(llN, llO, iN, iO);
+						}
 					}
+
+					i++; if (iO == maxOffset) { iN++; iO = 0; } else { iO += recordSize; }
 				}
-				while (--j > i) {
-					if ((cr = s.compare(p, j)) > 0) {
+
+				j--; if (jO == 0) { jN--; jO = maxOffset; } else { jO -= recordSize; }
+
+				while (j > i) {
+					if ((cr = s.compare(pN, pO, jN, jO)) > 0) {
 						break;
 					}
-					if (0 == cr && --rr != j) {
-						s.swap(rr, j);
+
+					if (0 == cr) {
+						rr--; if (rrO == 0) { rrN--; rrO = maxOffset; } else { rrO -= recordSize; }
+
+						if (rr != j) {
+							s.swap(rrN, rrO, jN, jO);
+						}
 					}
+
+					j--; if (jO == 0) { jN--; jO = maxOffset; } else { jO -= recordSize; }
 				}
 				if (i < j) {
-					s.swap(i, j);
+					s.swap(iN, iO, jN, jO);
 				} else {
 					break;
 				}
 			}
-			j = i;
+			j = i; jN = iN; jO = iO;
 			// swap pivot- and all eq values- into position
 			while (ll >= p) {
-				s.swap(ll--, --i);
+				i--; if (iO == 0) { iN--; iO = maxOffset; } else { iO -= recordSize; }
+
+				s.swap(llN, llO, iN, iO);
+
+				ll--; if (llO == 0) { llN--; llO = maxOffset; } else { llO -= recordSize; }
 			}
 			while (rr < r) {
-				s.swap(rr++, j++);
+				s.swap(rrN, rrO, jN, jO);
+
+				rr++; if (rrO == maxOffset) { rrN++; rrO = 0; } else { rrO += recordSize; }
+				j++; if (jO == maxOffset) { jN++; jO = 0; } else { jO += recordSize; }
 			}
 
 			// Conquer
 			// Recurse on smaller interval first to keep stack shallow
 			assert i != j;
 			if (i - p < r - j) {
-				sortInternal(s, p, i, depth);
-				p = j;
+				sortInternal(s, recordsPerSegment, recordSize, maxOffset, p, pN, pO, i, iN, iO, depth);
+				p = j; pN = jN; pO = jO;
 			} else {
-				sortInternal(s, j, r, depth);
-				r = i;
+				sortInternal(s, recordsPerSegment, recordSize, maxOffset, j, jN, jO, r, rN, rO, depth);
+				r = i; rN = iN; rO = iO;
 			}
 		}
 	}

From 9878f21c4f2da95a36eb85829c14f13805b65db6 Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Tue, 14 Mar 2017 14:25:57 +0100
Subject: [PATCH 062/253] [FLINK-6048] [checkpoint] Implement asynchronous
 snapshots for DefaultOperatorStateBackend

---
 .../state/RocksDBKeyedStateBackend.java       |   2 +-
 .../streaming/state/RocksDBStateBackend.java  |  17 +-
 .../state/RocksDBStateBackendTest.java        |   2 +-
 .../AbstractAsyncSnapshotIOCallable.java      | 109 +++++++
 .../runtime/state/AbstractStateBackend.java   |   7 +-
 .../state/DefaultOperatorStateBackend.java    | 220 ++++++++++---
 .../runtime/state/OperatorStateHandle.java    |   6 +-
 .../state/filesystem/FsStateBackend.java      |  13 +
 .../state/heap/HeapKeyedStateBackend.java     | 107 +++----
 .../state/memory/MemoryStateBackend.java      |  13 +
 .../BlockerCheckpointStreamFactory.java       | 112 +++++++
 .../state/OperatorStateBackendTest.java       | 301 ++++++++++++++++--
 .../tasks/BlockingCheckpointsTest.java        |  10 +
 .../streaming/runtime/StateBackendITCase.java |   9 +
 14 files changed, 787 insertions(+), 141 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractAsyncSnapshotIOCallable.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 91d947ecc481a..199a5a4a0ef2d 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -421,7 +421,6 @@ public void writeDBSnapshot() throws IOException, InterruptedException {
 		 */
 		public void closeCheckpointStream() throws IOException {
 			if (outStream != null) {
-				stateBackend.cancelStreamRegistry.unregisterClosable(outStream);
 				snapshotResultStateHandle = closeSnapshotStreamAndGetHandle();
 			} else {
 				snapshotResultStateHandle = null;
@@ -592,6 +591,7 @@ private void writeKVStateData() throws IOException, InterruptedException {
 		}
 
 		private KeyGroupsStateHandle closeSnapshotStreamAndGetHandle() throws IOException {
+			stateBackend.cancelStreamRegistry.unregisterClosable(outStream);
 			StreamStateHandle stateHandle = outStream.closeAndGetHandle();
 			outStream = null;
 			return stateHandle != null ? new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle) : null;
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
index 16b08693177c1..80c9a291ddef5 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
@@ -26,15 +26,15 @@
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
+import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
 import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.runtime.state.filesystem.FsStateBackend;
 import org.apache.flink.util.AbstractID;
-
 import org.rocksdb.ColumnFamilyOptions;
 import org.rocksdb.DBOptions;
 import org.rocksdb.NativeLibraryLoader;
 import org.rocksdb.RocksDB;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -426,6 +426,19 @@ public ColumnFamilyOptions getColumnOptions() {
 		return opt;
 	}
 
+	@Override
+	public OperatorStateBackend createOperatorStateBackend(
+		Environment env,
+		String operatorIdentifier) throws Exception {
+
+		//the default for RocksDB; eventually there can be a operator state backend based on RocksDB, too.
+		final boolean asyncSnapshots = true;
+		return new DefaultOperatorStateBackend(
+			env.getUserClassLoader(),
+			env.getExecutionConfig(),
+			asyncSnapshots);
+	}
+
 	@Override
 	public String toString() {
 		return "RocksDB State Backend {" +
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
index d95a9b41d9857..b5f18a4f30c45 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
@@ -26,6 +26,7 @@
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeutils.base.IntSerializer;
 import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.runtime.checkpoint.BlockerCheckpointStreamFactory;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
@@ -38,7 +39,6 @@
 import org.apache.flink.runtime.state.VoidNamespace;
 import org.apache.flink.runtime.state.VoidNamespaceSerializer;
 import org.apache.flink.runtime.state.filesystem.FsStateBackend;
-import org.apache.flink.runtime.util.BlockerCheckpointStreamFactory;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractAsyncSnapshotIOCallable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractAsyncSnapshotIOCallable.java
new file mode 100644
index 0000000000000..1aaa473c3a359
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractAsyncSnapshotIOCallable.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.runtime.io.async.AbstractAsyncIOCallable;
+import org.apache.flink.runtime.state.CheckpointStreamFactory;
+import org.apache.flink.runtime.state.StateObject;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Abstract base class for async IO operations of snapshots against a
+ * {@link java.util.zip.CheckedOutputStream}. This includes participating in lifecycle management
+ * through a {@link CloseableRegistry}.
+ */
+public abstract class AbstractAsyncSnapshotIOCallable<H extends StateObject>
+	extends AbstractAsyncIOCallable<H, CheckpointStreamFactory.CheckpointStateOutputStream> {
+
+	protected final  long checkpointId;
+	protected final  long timestamp;
+
+	protected final CheckpointStreamFactory streamFactory;
+	protected final CloseableRegistry closeStreamOnCancelRegistry;
+	protected final AtomicBoolean open;
+
+	public AbstractAsyncSnapshotIOCallable(
+		long checkpointId,
+		long timestamp,
+		CheckpointStreamFactory streamFactory,
+		CloseableRegistry closeStreamOnCancelRegistry) {
+
+		this.streamFactory = Preconditions.checkNotNull(streamFactory);
+		this.closeStreamOnCancelRegistry = Preconditions.checkNotNull(closeStreamOnCancelRegistry);
+		this.checkpointId = checkpointId;
+		this.timestamp = timestamp;
+		this.open = new AtomicBoolean(false);
+	}
+
+	@Override
+	public CheckpointStreamFactory.CheckpointStateOutputStream openIOHandle() throws Exception {
+		if (checkStreamClosedAndDoTransitionToOpen()) {
+			CheckpointStreamFactory.CheckpointStateOutputStream stream =
+				streamFactory.createCheckpointStateOutputStream(checkpointId, timestamp);
+			try {
+				closeStreamOnCancelRegistry.registerClosable(stream);
+				return stream;
+			} catch (Exception ex) {
+				open.set(false);
+				throw ex;
+			}
+		} else {
+			throw new IOException("Async snapshot: a checkpoint stream was already opened.");
+		}
+	}
+
+	@Override
+	public void done(boolean canceled) {
+		if (checkStreamOpenAndDoTransitionToClose()) {
+			CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle();
+			if (stream != null) {
+				closeStreamOnCancelRegistry.unregisterClosable(stream);
+				IOUtils.closeQuietly(stream);
+			}
+		}
+	}
+
+	protected boolean checkStreamClosedAndDoTransitionToOpen() {
+		return open.compareAndSet(false, true);
+	}
+
+	protected boolean checkStreamOpenAndDoTransitionToClose() {
+		return open.compareAndSet(true, false);
+	}
+
+	protected StreamStateHandle closeStreamAndGetStateHandle() throws IOException {
+		if (checkStreamOpenAndDoTransitionToClose()) {
+			final CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle();
+			try {
+				return stream.closeAndGetHandle();
+			} finally {
+				closeStreamOnCancelRegistry.unregisterClosable(stream);
+			}
+		} else {
+			throw new IOException("Checkpoint stream already closed.");
+		}
+	}
+
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java
index 74025bf2f8409..1594e2e73edc6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractStateBackend.java
@@ -91,12 +91,9 @@ public abstract <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
 			TaskKvStateRegistry kvStateRegistry) throws IOException;
 
 	@Override
-	public OperatorStateBackend createOperatorStateBackend(
+	public abstract OperatorStateBackend createOperatorStateBackend(
 			Environment env,
-			String operatorIdentifier) throws Exception {
-
-		return new DefaultOperatorStateBackend(env.getUserClassLoader(), env.getExecutionConfig());
-	}
+			String operatorIdentifier) throws Exception;
 
 	// ------------------------------------------------------------------------
 	//  Loading the state backend from a configuration 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
index eb3ba01e6b1f0..e7ed26f67fd11 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
@@ -20,10 +20,10 @@
 
 import org.apache.commons.io.IOUtils;
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.core.fs.CloseableRegistry;
 import org.apache.flink.core.fs.FSDataInputStream;
 import org.apache.flink.core.fs.FSDataOutputStream;
@@ -31,8 +31,12 @@
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.runtime.checkpoint.AbstractAsyncSnapshotIOCallable;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
+import org.apache.flink.runtime.io.async.AsyncStoppableTaskWithCallback;
 import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.Serializable;
@@ -50,21 +54,54 @@
 @Internal
 public class DefaultOperatorStateBackend implements OperatorStateBackend {
 
-	/** The default namespace for state in cases where no state name is provided */
+	private static final Logger LOG = LoggerFactory.getLogger(DefaultOperatorStateBackend.class);
+
+	/**
+	 * The default namespace for state in cases where no state name is provided
+	 */
 	public static final String DEFAULT_OPERATOR_STATE_NAME = "_default_";
-	
+
+	/**
+	 * Map for all registered operator states. Maps state name -> state
+	 */
 	private final Map<String, PartitionableListState<?>> registeredStates;
+
+	/**
+	 * CloseableRegistry to participate in the tasks lifecycle.
+	 */
 	private final CloseableRegistry closeStreamOnCancelRegistry;
+
+	/**
+	 * Default serializer. Only used for the default operator state.
+	 */
 	private final JavaSerializer<Serializable> javaSerializer;
+
+	/**
+	 * The user code classloader.
+	 */
 	private final ClassLoader userClassloader;
+
+	/**
+	 * The execution configuration.
+	 */
 	private final ExecutionConfig executionConfig;
 
-	public DefaultOperatorStateBackend(ClassLoader userClassLoader, ExecutionConfig executionConfig) throws IOException {
+	/**
+	 * Flag to de/activate asynchronous snapshots.
+	 */
+	private final boolean asynchronousSnapshots;
+
+	public DefaultOperatorStateBackend(
+		ClassLoader userClassLoader,
+		ExecutionConfig executionConfig,
+		boolean asynchronousSnapshots) throws IOException {
+
 		this.closeStreamOnCancelRegistry = new CloseableRegistry();
 		this.userClassloader = Preconditions.checkNotNull(userClassLoader);
 		this.executionConfig = executionConfig;
 		this.javaSerializer = new JavaSerializer<>();
 		this.registeredStates = new HashMap<>();
+		this.asynchronousSnapshots = asynchronousSnapshots;
 	}
 
 	public ExecutionConfig getExecutionConfig() {
@@ -131,59 +168,109 @@ public <T extends Serializable> ListState<T> getSerializableListState(String sta
 
 	@Override
 	public RunnableFuture<OperatorStateHandle> snapshot(
-			long checkpointId,
-			long timestamp,
-			CheckpointStreamFactory streamFactory,
-			CheckpointOptions checkpointOptions) throws Exception {
+			final long checkpointId,
+			final long timestamp,
+			final CheckpointStreamFactory streamFactory,
+			final CheckpointOptions checkpointOptions) throws Exception {
+
+		final long syncStartTime = System.currentTimeMillis();
 
 		if (registeredStates.isEmpty()) {
 			return DoneFuture.nullValue();
 		}
 
-		List<OperatorBackendSerializationProxy.StateMetaInfo<?>> metaInfoList =
-				new ArrayList<>(registeredStates.size());
-
-		for (Map.Entry<String, PartitionableListState<?>> entry : registeredStates.entrySet()) {
-			PartitionableListState<?> state = entry.getValue();
-			OperatorBackendSerializationProxy.StateMetaInfo<?> metaInfo =
-					new OperatorBackendSerializationProxy.StateMetaInfo<>(
-							state.getName(),
-							state.getPartitionStateSerializer(),
-							state.getAssignmentMode());
-			metaInfoList.add(metaInfo);
+		final Map<String, PartitionableListState<?>> registeredStatesDeepCopies =
+				new HashMap<>(registeredStates.size());
+
+		// eagerly create deep copies of the list states in the sync phase, so that we can use them in the async writing
+		for (Map.Entry<String, PartitionableListState<?>> entry : this.registeredStates.entrySet()) {
+
+			PartitionableListState<?> listState = entry.getValue();
+			if (null != listState) {
+				listState = listState.deepCopy();
+			}
+			registeredStatesDeepCopies.put(entry.getKey(), listState);
 		}
 
-		Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData = new HashMap<>(registeredStates.size());
+		// implementation of the async IO operation, based on FutureTask
+		final AbstractAsyncSnapshotIOCallable<OperatorStateHandle> ioCallable =
+			new AbstractAsyncSnapshotIOCallable<OperatorStateHandle>(
+				checkpointId,
+				timestamp,
+				streamFactory,
+				closeStreamOnCancelRegistry) {
+
+				@Override
+				public OperatorStateHandle performOperation() throws Exception {
+					long asyncStartTime = System.currentTimeMillis();
+
+					final Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData =
+						new HashMap<>(registeredStatesDeepCopies.size());
+
+					List<OperatorBackendSerializationProxy.StateMetaInfo<?>> metaInfoList =
+						new ArrayList<>(registeredStatesDeepCopies.size());
+
+					for (Map.Entry<String, PartitionableListState<?>> entry :
+						registeredStatesDeepCopies.entrySet()) {
+
+						PartitionableListState<?> state = entry.getValue();
+						OperatorBackendSerializationProxy.StateMetaInfo<?> metaInfo =
+							new OperatorBackendSerializationProxy.StateMetaInfo<>(
+								state.getName(),
+								state.getPartitionStateSerializer(),
+								state.getAssignmentMode());
+						metaInfoList.add(metaInfo);
+					}
 
-		CheckpointStreamFactory.CheckpointStateOutputStream out = streamFactory.
-				createCheckpointStateOutputStream(checkpointId, timestamp);
+					CheckpointStreamFactory.CheckpointStateOutputStream out = getIoHandle();
+					DataOutputView dov = new DataOutputViewStreamWrapper(out);
 
-		try {
-			closeStreamOnCancelRegistry.registerClosable(out);
+					OperatorBackendSerializationProxy backendSerializationProxy =
+						new OperatorBackendSerializationProxy(metaInfoList);
 
-			DataOutputView dov = new DataOutputViewStreamWrapper(out);
+					backendSerializationProxy.write(dov);
+
+					dov.writeInt(registeredStatesDeepCopies.size());
 
-			OperatorBackendSerializationProxy backendSerializationProxy =
-					new OperatorBackendSerializationProxy(metaInfoList);
+					for (Map.Entry<String, PartitionableListState<?>> entry :
+						registeredStatesDeepCopies.entrySet()) {
 
-			backendSerializationProxy.write(dov);
+						PartitionableListState<?> value = entry.getValue();
+						long[] partitionOffsets = value.write(out);
+						OperatorStateHandle.Mode mode = value.getAssignmentMode();
+						writtenStatesMetaData.put(
+							entry.getKey(),
+							new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
+					}
 
-			dov.writeInt(registeredStates.size());
-			for (Map.Entry<String, PartitionableListState<?>> entry : registeredStates.entrySet()) {
+					StreamStateHandle stateHandle = closeStreamAndGetStateHandle();
 
-				PartitionableListState<?> value = entry.getValue();
-				long[] partitionOffsets = value.write(out);
-				OperatorStateHandle.Mode mode = value.getAssignmentMode();
-				writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
-			}
+					if (asynchronousSnapshots) {
+						LOG.info("DefaultOperatorStateBackend snapshot ({}, asynchronous part) in thread {} took {} ms.",
+							streamFactory, Thread.currentThread(), (System.currentTimeMillis() - asyncStartTime));
+					}
+
+					if (stateHandle == null) {
+						return null;
+					}
 
-			OperatorStateHandle handle = new OperatorStateHandle(writtenStatesMetaData, out.closeAndGetHandle());
+					OperatorStateHandle operatorStateHandle =
+						new OperatorStateHandle(writtenStatesMetaData, stateHandle);
 
-			return new DoneFuture<>(handle);
-		} finally {
-			closeStreamOnCancelRegistry.unregisterClosable(out);
-			out.close();
+					return operatorStateHandle;
+				}
+			};
+
+		AsyncStoppableTaskWithCallback<OperatorStateHandle> task = AsyncStoppableTaskWithCallback.from(ioCallable);
+
+		if (!asynchronousSnapshots) {
+			task.run();
 		}
+
+		LOG.info("DefaultOperatorStateBackend snapshot (" + streamFactory + ", synchronous part) in thread " +
+				Thread.currentThread() + " took " + (System.currentTimeMillis() - syncStartTime) + " ms.");
+
+		return task;
 	}
 
 	@Override
@@ -253,22 +340,67 @@ public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exc
 		}
 	}
 
+	/**
+	 *
+	 * Implementation of operator list state.
+	 *
+	 * @param <S> the type of an operator state partition.
+	 */
 	static final class PartitionableListState<S> implements ListState<S> {
 
+		/**
+		 * The name of the state, as registered by the user
+		 */
 		private final String name;
+
+		/**
+		 * The type serializer for the elements in the state list
+		 */
 		private final TypeSerializer<S> partitionStateSerializer;
+
+		/**
+		 * The mode how elements in this state are assigned to tasks during restore
+		 */
 		private final OperatorStateHandle.Mode assignmentMode;
-		private final List<S> internalList;
+
+		/**
+		 * The internal list the holds the elements of the state
+		 */
+		private final ArrayList<S> internalList;
+
+		/**
+		 * A serializer that allows to perfom deep copies of internalList
+		 */
+		private final ArrayListSerializer<S> internalListCopySerializer;
 
 		public PartitionableListState(
 				String name,
 				TypeSerializer<S> partitionStateSerializer,
 				OperatorStateHandle.Mode assignmentMode) {
 
+			this(name, partitionStateSerializer, assignmentMode, new ArrayList<S>());
+		}
+
+		private PartitionableListState(
+				String name,
+				TypeSerializer<S> partitionStateSerializer,
+				OperatorStateHandle.Mode assignmentMode,
+				ArrayList<S> internalList) {
+
 			this.name = Preconditions.checkNotNull(name);
 			this.partitionStateSerializer = Preconditions.checkNotNull(partitionStateSerializer);
 			this.assignmentMode = Preconditions.checkNotNull(assignmentMode);
-			this.internalList = new ArrayList<>();
+			this.internalList = Preconditions.checkNotNull(internalList);
+			this.internalListCopySerializer = new ArrayListSerializer<>(partitionStateSerializer);
+		}
+
+		private PartitionableListState(PartitionableListState<S> toCopy) {
+
+			this(
+					toCopy.name,
+					toCopy.partitionStateSerializer.duplicate(),
+					toCopy.assignmentMode,
+					toCopy.internalListCopySerializer.copy(toCopy.internalList));
 		}
 
 		public String getName() {
@@ -287,6 +419,10 @@ public List<S> getInternalList() {
 			return internalList;
 		}
 
+		public PartitionableListState<S> deepCopy() {
+			return new PartitionableListState<>(this);
+		}
+
 		@Override
 		public void clear() {
 			internalList.clear();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateHandle.java
index c59fbadc1f9fd..7c493383ec2b0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateHandle.java
@@ -32,8 +32,12 @@
  */
 public class OperatorStateHandle implements StreamStateHandle {
 
+	/**
+	 * The modes that determine how an {@link OperatorStateHandle} is assigned to tasks during restore.
+	 */
 	public enum Mode {
-		SPLIT_DISTRIBUTE, BROADCAST
+		SPLIT_DISTRIBUTE, // The operator state partitions in the state handle are split and distributed to one task each.
+		BROADCAST // The operator state partitions are broadcasted to all task.
 	}
 
 	private static final long serialVersionUID = 35876522969227335L;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java
index e27712caa67a7..e320bf3140de7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackend.java
@@ -27,7 +27,9 @@
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
+import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
 import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.runtime.state.heap.HeapKeyedStateBackend;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -309,6 +311,17 @@ public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
 				env.getExecutionConfig());
 	}
 
+	@Override
+	public OperatorStateBackend createOperatorStateBackend(
+		Environment env,
+		String operatorIdentifier) throws Exception {
+
+		return new DefaultOperatorStateBackend(
+			env.getUserClassLoader(),
+			env.getExecutionConfig(),
+			asynchronousSnapshots);
+	}
+
 	@Override
 	public String toString() {
 		return "File State Backend @ " + basePath;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index 9247ffac5327c..38817cdb4bd57 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -37,10 +37,10 @@
 import org.apache.flink.migration.MigrationUtil;
 import org.apache.flink.migration.runtime.state.KvStateSnapshot;
 import org.apache.flink.migration.runtime.state.memory.MigrationRestoreSnapshot;
-import org.apache.flink.runtime.io.async.AbstractAsyncIOCallable;
-import org.apache.flink.runtime.io.async.AsyncStoppableTaskWithCallback;
 import org.apache.flink.migration.state.MigrationKeyGroupStateHandle;
+import org.apache.flink.runtime.checkpoint.AbstractAsyncSnapshotIOCallable;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
+import org.apache.flink.runtime.io.async.AsyncStoppableTaskWithCallback;
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.ArrayListSerializer;
@@ -72,7 +72,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.RunnableFuture;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * A {@link AbstractKeyedStateBackend} that keeps state on the Java Heap and will serialize state to
@@ -269,78 +268,50 @@ public  RunnableFuture<KeyedStateHandle> snapshot(
 		//--------------------------------------------------- this becomes the end of sync part
 
 		// implementation of the async IO operation, based on FutureTask
-		final AbstractAsyncIOCallable<KeyedStateHandle, CheckpointStreamFactory.CheckpointStateOutputStream> ioCallable =
-				new AbstractAsyncIOCallable<KeyedStateHandle, CheckpointStreamFactory.CheckpointStateOutputStream>() {
-
-					AtomicBoolean open = new AtomicBoolean(false);
-
-					@Override
-					public CheckpointStreamFactory.CheckpointStateOutputStream openIOHandle() throws Exception {
-						if (open.compareAndSet(false, true)) {
-							CheckpointStreamFactory.CheckpointStateOutputStream stream =
-									streamFactory.createCheckpointStateOutputStream(checkpointId, timestamp);
-							try {
-								cancelStreamRegistry.registerClosable(stream);
-								return stream;
-							} catch (Exception ex) {
-								open.set(false);
-								throw ex;
-							}
-						} else {
-							throw new IOException("Operation already opened.");
+		final AbstractAsyncSnapshotIOCallable<KeyedStateHandle> ioCallable =
+			new AbstractAsyncSnapshotIOCallable<KeyedStateHandle>(
+				checkpointId,
+				timestamp,
+				streamFactory,
+				cancelStreamRegistry) {
+
+				@Override
+				public KeyGroupsStateHandle performOperation() throws Exception {
+					long asyncStartTime = System.currentTimeMillis();
+					CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle();
+					DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(stream);
+					serializationProxy.write(outView);
+
+					long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()];
+
+					for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) {
+						int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos);
+						keyGroupRangeOffsets[keyGroupPos] = stream.getPos();
+						outView.writeInt(keyGroupId);
+
+						for (Map.Entry<String, StateTable<K, ?, ?>> kvState : stateTables.entrySet()) {
+							outView.writeShort(kVStateToId.get(kvState.getKey()));
+							cowStateStableSnapshots.get(kvState.getValue()).writeMappingsInKeyGroup(outView, keyGroupId);
 						}
 					}
 
-					@Override
-					public KeyGroupsStateHandle performOperation() throws Exception {
-						long asyncStartTime = System.currentTimeMillis();
-						CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle();
-						DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(stream);
-						serializationProxy.write(outView);
-
-						long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()];
-
-						for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) {
-							int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos);
-							keyGroupRangeOffsets[keyGroupPos] = stream.getPos();
-							outView.writeInt(keyGroupId);
-
-							for (Map.Entry<String, StateTable<K, ?, ?>> kvState : stateTables.entrySet()) {
-								outView.writeShort(kVStateToId.get(kvState.getKey()));
-								cowStateStableSnapshots.get(kvState.getValue()).writeMappingsInKeyGroup(outView, keyGroupId);
-							}
-						}
-
-						if (open.compareAndSet(true, false)) {
-							StreamStateHandle streamStateHandle = stream.closeAndGetHandle();
-							KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets);
-							final KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(offsets, streamStateHandle);
+					final StreamStateHandle streamStateHandle = closeStreamAndGetStateHandle();
 
-							if (asynchronousSnapshots) {
-								LOG.info("Heap backend snapshot ({}, asynchronous part) in thread {} took {} ms.",
-										streamFactory, Thread.currentThread(), (System.currentTimeMillis() - asyncStartTime));
-							}
-
-							return keyGroupsStateHandle;
-						} else {
-							throw new IOException("Checkpoint stream already closed.");
-						}
+					if (asynchronousSnapshots) {
+						LOG.info("Heap backend snapshot ({}, asynchronous part) in thread {} took {} ms.",
+							streamFactory, Thread.currentThread(), (System.currentTimeMillis() - asyncStartTime));
 					}
 
-					@Override
-					public void done(boolean canceled) {
-						if (open.compareAndSet(true, false)) {
-							CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle();
-							if (null != stream) {
-								cancelStreamRegistry.unregisterClosable(stream);
-								IOUtils.closeQuietly(stream);
-							}
-						}
-						for (StateTableSnapshot snapshot : cowStateStableSnapshots.values()) {
-							snapshot.release();
-						}
+					if (streamStateHandle == null) {
+						return null;
 					}
-				};
+
+					KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets);
+					final KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(offsets, streamStateHandle);
+
+					return keyGroupsStateHandle;
+				}
+			};
 
 		AsyncStoppableTaskWithCallback<KeyedStateHandle> task = AsyncStoppableTaskWithCallback.from(ioCallable);
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java
index f0bac1b17c56c..7ed1dea17a13f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemoryStateBackend.java
@@ -25,7 +25,9 @@
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
+import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
 import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.runtime.state.heap.HeapKeyedStateBackend;
 
 import java.io.IOException;
@@ -88,6 +90,17 @@ public MemoryStateBackend(int maxStateSize, boolean asynchronousSnapshots) {
 		this.asynchronousSnapshots = asynchronousSnapshots;
 	}
 
+	@Override
+	public OperatorStateBackend createOperatorStateBackend(
+		Environment env,
+		String operatorIdentifier) throws Exception {
+
+		return new DefaultOperatorStateBackend(
+			env.getUserClassLoader(),
+			env.getExecutionConfig(),
+			asynchronousSnapshots);
+	}
+
 	@Override
 	public String toString() {
 		return "MemoryStateBackend (data in heap memory / checkpoints to JobManager)";
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java
new file mode 100644
index 0000000000000..6f892e2053980
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.runtime.state.CheckpointStreamFactory;
+import org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory;
+
+import java.io.IOException;
+
+/**
+ * A {@link CheckpointStreamFactory} for tests that creates streams that block on a latch to test concurrency in
+ * checkpointing.
+ */
+public class BlockerCheckpointStreamFactory implements CheckpointStreamFactory {
+
+	private final int maxSize;
+	private int afterNumberInvocations;
+	private OneShotLatch blocker;
+	private OneShotLatch waiter;
+
+	MemCheckpointStreamFactory.MemoryCheckpointOutputStream lastCreatedStream;
+
+	public MemCheckpointStreamFactory.MemoryCheckpointOutputStream getLastCreatedStream() {
+		return lastCreatedStream;
+	}
+
+	public BlockerCheckpointStreamFactory(int maxSize) {
+		this.maxSize = maxSize;
+	}
+
+	public void setAfterNumberInvocations(int afterNumberInvocations) {
+		this.afterNumberInvocations = afterNumberInvocations;
+	}
+
+	public void setBlockerLatch(OneShotLatch latch) {
+		this.blocker = latch;
+	}
+
+	public void setWaiterLatch(OneShotLatch latch) {
+		this.waiter = latch;
+	}
+
+	@Override
+	public MemCheckpointStreamFactory.MemoryCheckpointOutputStream createCheckpointStateOutputStream(long checkpointID, long timestamp) throws Exception {
+		waiter.trigger();
+		this.lastCreatedStream = new MemCheckpointStreamFactory.MemoryCheckpointOutputStream(maxSize) {
+
+			private int afterNInvocations = afterNumberInvocations;
+			private final OneShotLatch streamBlocker = blocker;
+			private final OneShotLatch streamWaiter = waiter;
+
+			@Override
+			public void write(int b) throws IOException {
+
+				if (afterNInvocations > 0) {
+					--afterNInvocations;
+				}
+
+				if (0 == afterNInvocations && null != streamBlocker) {
+					try {
+						streamBlocker.await();
+					} catch (InterruptedException ignored) {
+					}
+				}
+				try {
+					super.write(b);
+				} catch (IOException ex) {
+					if (null != streamWaiter) {
+						streamWaiter.trigger();
+					}
+					throw ex;
+				}
+
+				if (0 == afterNInvocations && null != streamWaiter) {
+					streamWaiter.trigger();
+				}
+			}
+
+			@Override
+			public void close() {
+				super.close();
+				if (null != streamWaiter) {
+					streamWaiter.trigger();
+				}
+			}
+		};
+
+		return lastCreatedStream;
+	}
+
+	@Override
+	public void close() throws Exception {
+
+	}
+}
\ No newline at end of file
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
index 13a6307921588..c04ed8cf39562 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
@@ -21,22 +21,30 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.runtime.checkpoint.BlockerCheckpointStreamFactory;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.state.DefaultOperatorStateBackend.PartitionableListState;
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-
 import org.apache.flink.util.FutureUtil;
+import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.io.File;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -75,7 +83,7 @@ public void testRegisterStatesWithoutTypeSerializer() throws Exception {
 		final ExecutionConfig cfg = new ExecutionConfig();
 		cfg.registerTypeWithKryoSerializer(registeredType, com.esotericsoftware.kryo.serializers.JavaSerializer.class);
 
-		final OperatorStateBackend operatorStateBackend = new DefaultOperatorStateBackend(classLoader, cfg);
+		final OperatorStateBackend operatorStateBackend = new DefaultOperatorStateBackend(classLoader, cfg, false);
 
 		ListStateDescriptor<File> stateDescriptor = new ListStateDescriptor<>("test", File.class);
 		ListStateDescriptor<String> stateDescriptor2 = new ListStateDescriptor<>("test2", String.class);
@@ -108,7 +116,7 @@ public void testRegisterStatesWithoutTypeSerializer() throws Exception {
 	@Test
 	public void testRegisterStates() throws Exception {
 		final OperatorStateBackend operatorStateBackend =
-				new DefaultOperatorStateBackend(classLoader, new ExecutionConfig());
+				new DefaultOperatorStateBackend(classLoader, new ExecutionConfig(), false);
 
 		ListStateDescriptor<Serializable> stateDescriptor1 = new ListStateDescriptor<>("test1", new JavaSerializer<>());
 		ListStateDescriptor<Serializable> stateDescriptor2 = new ListStateDescriptor<>("test2", new JavaSerializer<>());
@@ -117,19 +125,19 @@ public void testRegisterStates() throws Exception {
 		assertNotNull(listState1);
 		assertEquals(1, operatorStateBackend.getRegisteredStateNames().size());
 		Iterator<Serializable> it = listState1.get().iterator();
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 		listState1.add(42);
 		listState1.add(4711);
 
 		it = listState1.get().iterator();
 		assertEquals(42, it.next());
 		assertEquals(4711, it.next());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 
 		ListState<Serializable> listState2 = operatorStateBackend.getListState(stateDescriptor2);
 		assertNotNull(listState2);
 		assertEquals(2, operatorStateBackend.getRegisteredStateNames().size());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 		listState2.add(7);
 		listState2.add(13);
 		listState2.add(23);
@@ -138,12 +146,12 @@ public void testRegisterStates() throws Exception {
 		assertEquals(7, it.next());
 		assertEquals(13, it.next());
 		assertEquals(23, it.next());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 
 		ListState<Serializable> listState3 = operatorStateBackend.getUnionListState(stateDescriptor3);
 		assertNotNull(listState3);
 		assertEquals(3, operatorStateBackend.getRegisteredStateNames().size());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 		listState3.add(17);
 		listState3.add(3);
 		listState3.add(123);
@@ -152,7 +160,7 @@ public void testRegisterStates() throws Exception {
 		assertEquals(17, it.next());
 		assertEquals(3, it.next());
 		assertEquals(123, it.next());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 
 		ListState<Serializable> listState1b = operatorStateBackend.getListState(stateDescriptor1);
 		assertNotNull(listState1b);
@@ -161,19 +169,19 @@ public void testRegisterStates() throws Exception {
 		assertEquals(42, it.next());
 		assertEquals(4711, it.next());
 		assertEquals(123, it.next());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 
 		it = listState1.get().iterator();
 		assertEquals(42, it.next());
 		assertEquals(4711, it.next());
 		assertEquals(123, it.next());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 
 		it = listState1b.get().iterator();
 		assertEquals(42, it.next());
 		assertEquals(4711, it.next());
 		assertEquals(123, it.next());
-		assertTrue(!it.hasNext());
+		assertFalse(it.hasNext());
 
 		try {
 			operatorStateBackend.getUnionListState(stateDescriptor2);
@@ -208,12 +216,10 @@ public void testSnapshotEmpty() throws Exception {
 	}
 
 	@Test
-	public void testSnapshotRestore() throws Exception {
+	public void testSnapshotRestoreSync() throws Exception {
 		AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
 
-		OperatorStateBackend operatorStateBackend =
-				abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "test-op-name");
-
+		OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "test-op-name");
 		ListStateDescriptor<Serializable> stateDescriptor1 = new ListStateDescriptor<>("test1", new JavaSerializer<>());
 		ListStateDescriptor<Serializable> stateDescriptor2 = new ListStateDescriptor<>("test2", new JavaSerializer<>());
 		ListStateDescriptor<Serializable> stateDescriptor3 = new ListStateDescriptor<>("test3", new JavaSerializer<>());
@@ -234,8 +240,9 @@ public void testSnapshotRestore() throws Exception {
 		listState3.add(20);
 
 		CheckpointStreamFactory streamFactory = abstractStateBackend.createStreamFactory(new JobID(), "testOperator");
-		OperatorStateHandle stateHandle = FutureUtil.runIfNotDoneAndGet(
-				operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint()));
+		RunnableFuture<OperatorStateHandle> runnableFuture =
+				operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint());
+		OperatorStateHandle stateHandle = FutureUtil.runIfNotDoneAndGet(runnableFuture);
 
 		try {
 
@@ -259,26 +266,278 @@ public void testSnapshotRestore() throws Exception {
 			Iterator<Serializable> it = listState1.get().iterator();
 			assertEquals(42, it.next());
 			assertEquals(4711, it.next());
-			assertTrue(!it.hasNext());
+			assertFalse(it.hasNext());
 
 			it = listState2.get().iterator();
 			assertEquals(7, it.next());
 			assertEquals(13, it.next());
 			assertEquals(23, it.next());
-			assertTrue(!it.hasNext());
+			assertFalse(it.hasNext());
 
 			it = listState3.get().iterator();
 			assertEquals(17, it.next());
 			assertEquals(18, it.next());
 			assertEquals(19, it.next());
 			assertEquals(20, it.next());
-			assertTrue(!it.hasNext());
+			assertFalse(it.hasNext());
+
+			operatorStateBackend.close();
+			operatorStateBackend.dispose();
+		} finally {
+			stateHandle.discardState();
+		}
+	}
+
+	@Test
+	public void testSnapshotRestoreAsync() throws Exception {
+		DefaultOperatorStateBackend operatorStateBackend =
+				new DefaultOperatorStateBackend(OperatorStateBackendTest.class.getClassLoader(), new ExecutionConfig(), true);
+
+		ListStateDescriptor<MutableType> stateDescriptor1 =
+				new ListStateDescriptor<>("test1", new JavaSerializer<MutableType>());
+		ListStateDescriptor<MutableType> stateDescriptor2 =
+				new ListStateDescriptor<>("test2", new JavaSerializer<MutableType>());
+		ListStateDescriptor<MutableType> stateDescriptor3 =
+				new ListStateDescriptor<>("test3", new JavaSerializer<MutableType>());
+		ListState<MutableType> listState1 = operatorStateBackend.getListState(stateDescriptor1);
+		ListState<MutableType> listState2 = operatorStateBackend.getListState(stateDescriptor2);
+		ListState<MutableType> listState3 = operatorStateBackend.getUnionListState(stateDescriptor3);
+
+		listState1.add(MutableType.of(42));
+		listState1.add(MutableType.of(4711));
+
+		listState2.add(MutableType.of(7));
+		listState2.add(MutableType.of(13));
+		listState2.add(MutableType.of(23));
+
+		listState3.add(MutableType.of(17));
+		listState3.add(MutableType.of(18));
+		listState3.add(MutableType.of(19));
+		listState3.add(MutableType.of(20));
+
+		BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
+
+		OneShotLatch waiterLatch = new OneShotLatch();
+		OneShotLatch blockerLatch = new OneShotLatch();
+
+		streamFactory.setWaiterLatch(waiterLatch);
+		streamFactory.setBlockerLatch(blockerLatch);
+
+		RunnableFuture<OperatorStateHandle> runnableFuture =
+				operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint());
+
+		ExecutorService executorService = Executors.newFixedThreadPool(1);
+
+		executorService.submit(runnableFuture);
+
+		// wait until the async checkpoint is in the write code, then continue
+		waiterLatch.await();
+
+		// do some mutations to the state, to test if our snapshot will NOT reflect them
+
+		listState1.add(MutableType.of(77));
+
+		int n = 0;
+
+		for (MutableType mutableType : listState2.get()) {
+			if (++n == 2) {
+				// allow the write code to continue, so that we could do changes while state is written in parallel.
+				blockerLatch.trigger();
+			}
+			mutableType.setValue(mutableType.getValue() + 10);
+		}
+
+		listState3.clear();
+
+		operatorStateBackend.getListState(
+				new ListStateDescriptor<>("test4", new JavaSerializer<MutableType>()));
+
+		// run the snapshot
+		OperatorStateHandle stateHandle = runnableFuture.get();
+
+		try {
+
+			operatorStateBackend.close();
+			operatorStateBackend.dispose();
+
+			AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
+
+			//TODO this is temporarily casted to test already functionality that we do not yet expose through public API
+			operatorStateBackend = (DefaultOperatorStateBackend) abstractStateBackend.createOperatorStateBackend(
+					createMockEnvironment(),
+					"testOperator");
+
+			operatorStateBackend.restore(Collections.singletonList(stateHandle));
+
+			assertEquals(3, operatorStateBackend.getRegisteredStateNames().size());
+
+			listState1 = operatorStateBackend.getListState(stateDescriptor1);
+			listState2 = operatorStateBackend.getListState(stateDescriptor2);
+			listState3 = operatorStateBackend.getUnionListState(stateDescriptor3);
+
+			assertEquals(3, operatorStateBackend.getRegisteredStateNames().size());
+
+			Iterator<MutableType> it = listState1.get().iterator();
+			assertEquals(42, it.next().value);
+			assertEquals(4711, it.next().value);
+			assertFalse(it.hasNext());
+
+			it = listState2.get().iterator();
+			assertEquals(7, it.next().value);
+			assertEquals(13, it.next().value);
+			assertEquals(23, it.next().value);
+			assertFalse(it.hasNext());
+
+			it = listState3.get().iterator();
+			assertEquals(17, it.next().value);
+			assertEquals(18, it.next().value);
+			assertEquals(19, it.next().value);
+			assertEquals(20, it.next().value);
+			assertFalse(it.hasNext());
 
 			operatorStateBackend.close();
 			operatorStateBackend.dispose();
 		} finally {
 			stateHandle.discardState();
 		}
+
+		executorService.shutdown();
+	}
+
+	@Test
+	public void testSnapshotAsyncClose() throws Exception {
+		DefaultOperatorStateBackend operatorStateBackend =
+				new DefaultOperatorStateBackend(OperatorStateBackendTest.class.getClassLoader(), new ExecutionConfig(), true);
+
+		ListStateDescriptor<MutableType> stateDescriptor1 =
+				new ListStateDescriptor<>("test1", new JavaSerializer<MutableType>());
+
+		ListState<MutableType> listState1 = operatorStateBackend.getOperatorState(stateDescriptor1);
+
+
+		listState1.add(MutableType.of(42));
+		listState1.add(MutableType.of(4711));
+
+		BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
+
+		OneShotLatch waiterLatch = new OneShotLatch();
+		OneShotLatch blockerLatch = new OneShotLatch();
+
+		streamFactory.setWaiterLatch(waiterLatch);
+		streamFactory.setBlockerLatch(blockerLatch);
+
+		RunnableFuture<OperatorStateHandle> runnableFuture =
+				operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint());
+
+		ExecutorService executorService = Executors.newFixedThreadPool(1);
+
+		executorService.submit(runnableFuture);
+
+		// wait until the async checkpoint is in the write code, then continue
+		waiterLatch.await();
+
+		operatorStateBackend.close();
+
+		blockerLatch.trigger();
+
+		try {
+			runnableFuture.get(60, TimeUnit.SECONDS);
+			Assert.fail();
+		} catch (ExecutionException eex) {
+			Assert.assertTrue(eex.getCause() instanceof IOException);
+		}
+	}
+
+	@Test
+	public void testSnapshotAsyncCancel() throws Exception {
+		DefaultOperatorStateBackend operatorStateBackend =
+				new DefaultOperatorStateBackend(OperatorStateBackendTest.class.getClassLoader(), new ExecutionConfig(), true);
+
+		ListStateDescriptor<MutableType> stateDescriptor1 =
+				new ListStateDescriptor<>("test1", new JavaSerializer<MutableType>());
+
+		ListState<MutableType> listState1 = operatorStateBackend.getOperatorState(stateDescriptor1);
+
+
+		listState1.add(MutableType.of(42));
+		listState1.add(MutableType.of(4711));
+
+		BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
+
+		OneShotLatch waiterLatch = new OneShotLatch();
+		OneShotLatch blockerLatch = new OneShotLatch();
+
+		streamFactory.setWaiterLatch(waiterLatch);
+		streamFactory.setBlockerLatch(blockerLatch);
+
+		RunnableFuture<OperatorStateHandle> runnableFuture =
+				operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint());
+
+		ExecutorService executorService = Executors.newFixedThreadPool(1);
+
+		executorService.submit(runnableFuture);
+
+		// wait until the async checkpoint is in the write code, then continue
+		waiterLatch.await();
+
+		runnableFuture.cancel(true);
+
+		blockerLatch.trigger();
+
+		try {
+			runnableFuture.get(60, TimeUnit.SECONDS);
+			Assert.fail();
+		} catch (CancellationException ignore) {
+
+		}
+	}
+
+	static final class MutableType implements Serializable {
+
+		private static final long serialVersionUID = 1L;
+
+		private int value;
+
+		public MutableType() {
+			this(0);
+		}
+
+		public MutableType(int value) {
+			this.value = value;
+		}
+
+		public int getValue() {
+			return value;
+		}
+
+		public void setValue(int value) {
+			this.value = value;
+		}
+
+		@Override
+		public boolean equals(Object o) {
+
+			if (this == o) {
+				return true;
+			}
+
+			if (o == null || getClass() != o.getClass()) {
+				return false;
+			}
+
+			MutableType that = (MutableType) o;
+
+			return value == that.value;
+		}
+
+		@Override
+		public int hashCode() {
+			return value;
+		}
+
+		static MutableType of(int value) {
+			return new MutableType(value);
+		}
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/BlockingCheckpointsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/BlockingCheckpointsTest.java
index e266ea1d053f6..bf5be7925c47a 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/BlockingCheckpointsTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/BlockingCheckpointsTest.java
@@ -53,7 +53,9 @@
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
 import org.apache.flink.runtime.state.CheckpointStreamFactory.CheckpointStateOutputStream;
+import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
 import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.runtime.state.OperatorStateCheckpointOutputStream;
 import org.apache.flink.runtime.state.StateSnapshotContext;
 import org.apache.flink.runtime.state.StreamStateHandle;
@@ -187,6 +189,14 @@ public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
 
 			throw new UnsupportedOperationException();
 		}
+
+		@Override
+		public OperatorStateBackend createOperatorStateBackend(Environment env, String operatorIdentifier) throws Exception {
+			return new DefaultOperatorStateBackend(
+				getClass().getClassLoader(),
+				new ExecutionConfig(),
+				true);
+		}
 	}
 
 	private static final class LockingOutputStreamFactory implements CheckpointStreamFactory {
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
index 4677242bb67c8..ce342c0b62f7d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
@@ -32,6 +32,7 @@
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
 import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
 import org.junit.Test;
@@ -112,6 +113,14 @@ public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
 				TaskKvStateRegistry kvStateRegistry) throws IOException {
 			throw new SuccessException();
 		}
+
+		@Override
+		public OperatorStateBackend createOperatorStateBackend(
+			Environment env,
+			String operatorIdentifier) throws Exception {
+
+			throw new UnsupportedOperationException();
+		}
 	}
 
 	static final class SuccessException extends IOException {

From a6c27717e16ffee787fbca7c6402a0f39fd547a4 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Mon, 27 Mar 2017 17:20:47 +0200
Subject: [PATCH 063/253] [FLINK-6390] [checkpoints] Add API for checkpoints
 that are triggered via external systems

This includes
  - A interface for hooks that are called by the checkpoint coordinator to trigger/restore a checkpoint
  - A source extension that triggers the operator checkpoints and barrier injection on certain events

Because this changes the checkpoint metadata format, the commit introduces a new metadata format version.

This closes #3782
---
 .../core/io/SimpleVersionedSerializer.java    |  80 +++
 .../org/apache/flink/util/StringUtils.java    |   7 +
 .../checkpoint/savepoint/SavepointV0.java     |  10 +
 .../savepoint/SavepointV0Serializer.java      |  12 +-
 .../checkpoint/CheckpointCoordinator.java     |  80 ++-
 .../checkpoint/CheckpointDeclineReason.java   |   4 +-
 .../checkpoint/CompletedCheckpoint.java       |  31 +-
 .../flink/runtime/checkpoint/MasterState.java |  62 +++
 .../checkpoint/MasterTriggerRestoreHook.java  | 140 ++++++
 .../runtime/checkpoint/PendingCheckpoint.java |  23 +-
 .../runtime/checkpoint/hooks/MasterHooks.java | 273 ++++++++++
 .../checkpoint/savepoint/Savepoint.java       |   6 +
 .../checkpoint/savepoint/SavepointLoader.java |   3 +-
 .../savepoint/SavepointSerializers.java       |   7 +-
 .../checkpoint/savepoint/SavepointV1.java     |  36 +-
 .../savepoint/SavepointV1Serializer.java      |  76 +--
 .../checkpoint/savepoint/SavepointV2.java     |  91 ++++
 .../savepoint/SavepointV2Serializer.java      | 468 ++++++++++++++++++
 .../executiongraph/ExecutionGraph.java        |   9 +
 .../executiongraph/ExecutionGraphBuilder.java |  18 +
 .../tasks/JobCheckpointingSettings.java       |  48 +-
 .../CheckpointCoordinatorMasterHooksTest.java | 421 ++++++++++++++++
 .../CompletedCheckpointStoreTest.java         |   2 +-
 .../checkpoint/CompletedCheckpointTest.java   |  24 +-
 ...ecutionGraphCheckpointCoordinatorTest.java |   1 +
 ...ntV1Test.java => CheckpointTestUtils.java} |  89 ++--
 .../savepoint/SavepointLoaderTest.java        |   2 +-
 .../savepoint/SavepointStoreTest.java         |  27 +-
 .../savepoint/SavepointV1SerializerTest.java  |  17 +-
 .../savepoint/SavepointV2SerializerTest.java  | 148 ++++++
 .../checkpoint/savepoint/SavepointV2Test.java |  68 +++
 .../ArchivedExecutionGraphTest.java           |   2 +
 .../checkpoint/ExternallyInducedSource.java   |  75 +++
 .../checkpoint/WithMasterCheckpointHook.java  |  38 ++
 .../FunctionMasterCheckpointHookFactory.java  |  45 ++
 .../api/graph/StreamingJobGraphGenerator.java |  28 +-
 .../runtime/tasks/SourceStreamTask.java       |  56 +++
 .../WithMasterCheckpointHookConfigTest.java   | 189 +++++++
 .../runtime/io/StreamRecordWriterTest.java    |   5 -
 .../SourceExternalCheckpointTriggerTest.java  | 171 +++++++
 .../runtime/tasks/StreamTaskTestHarness.java  |   7 +-
 .../test/checkpointing/SavepointITCase.java   |   4 +-
 42 files changed, 2748 insertions(+), 155 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/core/io/SimpleVersionedSerializer.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterState.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterTriggerRestoreHook.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/hooks/MasterHooks.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
 rename flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/{SavepointV1Test.java => CheckpointTestUtils.java} (69%)
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java
 create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ExternallyInducedSource.java
 create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/WithMasterCheckpointHook.java
 create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/FunctionMasterCheckpointHookFactory.java
 create mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java
 create mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceExternalCheckpointTriggerTest.java

diff --git a/flink-core/src/main/java/org/apache/flink/core/io/SimpleVersionedSerializer.java b/flink-core/src/main/java/org/apache/flink/core/io/SimpleVersionedSerializer.java
new file mode 100644
index 0000000000000..6c061a54ac127
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/io/SimpleVersionedSerializer.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.io;
+
+import java.io.IOException;
+
+/**
+ * A simple serializer interface for versioned serialization.
+ * 
+ * <p>The serializer has a version (returned by {@link #getVersion()}) which can be attached
+ * to the serialized data. When the serializer evolves, the version can be used to identify
+ * with which prior version the data was serialized.
+ * 
+ * <pre>{@code
+ * MyType someObject = ...;
+ * SimpleVersionedSerializer<MyType> serializer = ...;
+ *
+ * byte[] serializedData = serializer.serialize(someObject);
+ * int version = serializer.getVersion();
+ *
+ * MyType deserialized = serializer.deserialize(version, serializedData);
+ * 
+ * byte[] someOldData = ...;
+ * int oldVersion = ...;
+ * MyType deserializedOldObject = serializer.deserialize(oldVersion, someOldData);
+ * 
+ * }</pre>
+ * 
+ * @param <E> The data type serialized / deserialized by this serializer.
+ */
+public interface SimpleVersionedSerializer<E> extends Versioned {
+
+	/**
+	 * Gets the version with which this serializer serializes.
+	 * 
+	 * @return The version of the serialization schema.
+	 */
+	@Override
+	int getVersion();
+
+	/**
+	 * Serializes the given object. The serialization is assumed to correspond to the
+	 * current serialization version (as returned by {@link #getVersion()}.
+	 *
+	 * 
+	 * @param obj The object to serialize.
+	 * @return The serialized data (bytes).
+	 * 
+	 * @throws IOException Thrown, if the serialization fails.
+	 */
+	byte[] serialize(E obj) throws IOException;
+
+	/**
+	 * De-serializes the given data (bytes) which was serialized with the scheme of the
+	 * indicated version.
+	 * 
+	 * @param version The version in which the data was serialized
+	 * @param serialized The serialized data
+	 * @return The deserialized object
+	 * 
+	 * @throws IOException Thrown, if the deserialization fails.
+	 */
+	E deserialize(int version, byte[] serialized) throws IOException;
+}
diff --git a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
index b84f602d26bac..abd6ba6a4fae5 100644
--- a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
@@ -309,6 +309,13 @@ public static void writeNullableString(@Nullable String str, DataOutputView out)
 		}
 	}
 
+	/**
+	 * Checks if the string is null, empty, or contains only whitespace characters.
+	 * A whitespace character is defined via {@link Character#isWhitespace(char)}.
+	 * 
+	 * @param str The string to check
+	 * @return True, if the string is null or blank, false otherwise.
+	 */
 	public static boolean isNullOrWhitespaceOnly(String str) {
 		if (str == null || str.length() == 0) {
 			return true;
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java
index 1c51a695c3dfa..f3ec1cf2f4b5f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java
@@ -19,6 +19,7 @@
 package org.apache.flink.migration.runtime.checkpoint.savepoint;
 
 import org.apache.flink.migration.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.savepoint.Savepoint;
 import org.apache.flink.util.Preconditions;
 
@@ -58,6 +59,15 @@ public long getCheckpointId() {
 
 	@Override
 	public Collection<org.apache.flink.runtime.checkpoint.TaskState> getTaskStates() {
+		// since checkpoints are never deserialized into this format,
+		// this method should never be called
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public Collection<MasterState> getMasterStates() {
+		// since checkpoints are never deserialized into this format,
+		// this method should never be called
 		throw new UnsupportedOperationException();
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0Serializer.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0Serializer.java
index 4739033484194..d285906262565 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0Serializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0Serializer.java
@@ -34,7 +34,7 @@
 import org.apache.flink.migration.streaming.runtime.tasks.StreamTaskStateList;
 import org.apache.flink.migration.util.SerializedValue;
 import org.apache.flink.runtime.checkpoint.savepoint.SavepointSerializer;
-import org.apache.flink.runtime.checkpoint.savepoint.SavepointV1;
+import org.apache.flink.runtime.checkpoint.savepoint.SavepointV2;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.state.ChainedStateHandle;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
@@ -68,7 +68,7 @@
  * don't rely on any involved Java classes to stay the same.
  */
 @SuppressWarnings("deprecation")
-public class SavepointV0Serializer implements SavepointSerializer<SavepointV1> {
+public class SavepointV0Serializer implements SavepointSerializer<SavepointV2> {
 
 	public static final SavepointV0Serializer INSTANCE = new SavepointV0Serializer();
 	private static final StreamStateHandle SIGNAL_0 = new ByteStreamStateHandle("SIGNAL_0", new byte[]{0});
@@ -81,12 +81,12 @@ private SavepointV0Serializer() {
 
 
 	@Override
-	public void serialize(SavepointV1 savepoint, DataOutputStream dos) throws IOException {
+	public void serialize(SavepointV2 savepoint, DataOutputStream dos) throws IOException {
 		throw new UnsupportedOperationException("This serializer is read-only and only exists for backwards compatibility");
 	}
 
 	@Override
-	public SavepointV1 deserialize(DataInputStream dis, ClassLoader userClassLoader) throws IOException {
+	public SavepointV2 deserialize(DataInputStream dis, ClassLoader userClassLoader) throws IOException {
 
 		long checkpointId = dis.readLong();
 
@@ -165,7 +165,7 @@ private static SerializedValue<StateHandle<?>> readSerializedValueStateHandle(Da
 		return serializedValue;
 	}
 
-	private SavepointV1 convertSavepoint(
+	private SavepointV2 convertSavepoint(
 			List<TaskState> taskStates,
 			ClassLoader userClassLoader,
 			long checkpointID) throws Exception {
@@ -176,7 +176,7 @@ private SavepointV1 convertSavepoint(
 			newTaskStates.add(convertTaskState(taskState, userClassLoader, checkpointID));
 		}
 
-		return new SavepointV1(checkpointID, newTaskStates);
+		return new SavepointV2(checkpointID, newTaskStates);
 	}
 
 	private org.apache.flink.runtime.checkpoint.TaskState convertTaskState(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 256321eb854b1..23a38d4847f2c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -20,7 +20,9 @@
 
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.runtime.checkpoint.hooks.MasterHooks;
 import org.apache.flink.runtime.checkpoint.savepoint.SavepointLoader;
 import org.apache.flink.runtime.checkpoint.savepoint.SavepointStore;
 import org.apache.flink.runtime.concurrent.ApplyFunction;
@@ -39,7 +41,10 @@
 import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
 import org.apache.flink.runtime.state.TaskStateHandles;
 import org.apache.flink.runtime.taskmanager.DispatcherThreadFactory;
+import org.apache.flink.util.FlinkException;
 import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -138,6 +143,9 @@ public class CheckpointCoordinator {
 	/** The timer that handles the checkpoint timeouts and triggers periodic checkpoints */
 	private final ScheduledThreadPoolExecutor timer;
 
+	/** The master checkpoint hooks executed by this checkpoint coordinator */
+	private final HashMap<String, MasterTriggerRestoreHook<?>> masterHooks;
+
 	/** Actor that receives status updates from the execution graph this coordinator works for */
 	private JobStatusListener jobStatusListener;
 
@@ -220,6 +228,7 @@ public CheckpointCoordinator(
 		this.executor = checkNotNull(executor);
 
 		this.recentPendingCheckpoints = new ArrayDeque<>(NUM_GHOST_CHECKPOINT_IDS);
+		this.masterHooks = new HashMap<>();
 
 		this.timer = new ScheduledThreadPoolExecutor(1,
 				new DispatcherThreadFactory(Thread.currentThread().getThreadGroup(), "Checkpoint Timer"));
@@ -245,6 +254,45 @@ public CheckpointCoordinator(
 		}
 	}
 
+	// --------------------------------------------------------------------------------------------
+	//  Configuration
+	// --------------------------------------------------------------------------------------------
+
+	/**
+	 * Adds the given master hook to the checkpoint coordinator. This method does nothing, if
+	 * the checkpoint coordinator already contained a hook with the same ID (as defined via
+	 * {@link MasterTriggerRestoreHook#getIdentifier()}).
+	 * 
+	 * @param hook The hook to add.
+	 * @return True, if the hook was added, false if the checkpoint coordinator already
+	 *         contained a hook with the same ID.
+	 */
+	public boolean addMasterHook(MasterTriggerRestoreHook<?> hook) {
+		checkNotNull(hook);
+
+		final String id = hook.getIdentifier();
+		checkArgument(!StringUtils.isNullOrWhitespaceOnly(id), "The hook has a null or empty id");
+
+		synchronized (lock) {
+			if (!masterHooks.containsKey(id)) {
+				masterHooks.put(id, hook);
+				return true;
+			}
+			else {
+				return false;
+			}
+		}
+	}
+
+	/**
+	 * Gets the number of currently register master hooks.
+	 */
+	public int getNumberOfRegisteredMasterHooks() {
+		synchronized (lock) {
+			return masterHooks.size();
+		}
+	}
+
 	/**
 	 * Sets the checkpoint stats tracker.
 	 *
@@ -492,6 +540,20 @@ CheckpointTriggerResult triggerCheckpoint(
 				checkpoint.setStatsCallback(callback);
 			}
 
+			// trigger the master hooks for the checkpoint
+			try {
+				List<MasterState> masterStates = MasterHooks.triggerMasterHooks(masterHooks.values(),
+						checkpointID, timestamp, executor, Time.milliseconds(checkpointTimeout));
+
+				for (MasterState s : masterStates) {
+					checkpoint.addMasterState(s);
+				}
+			}
+			catch (FlinkException e) {
+				checkpoint.abortError(e);
+				return new CheckpointTriggerResult(CheckpointDeclineReason.EXCEPTION);
+			}
+
 			// schedule the timer that will clean up the expired checkpoints
 			final Runnable canceller = new Runnable() {
 				@Override
@@ -962,13 +1024,25 @@ public boolean restoreLatestCheckpointedState(
 
 			LOG.info("Restoring from latest valid checkpoint: {}.", latest);
 
+			// re-assign the task states
+
 			final Map<JobVertexID, TaskState> taskStates = latest.getTaskStates();
 
 			StateAssignmentOperation stateAssignmentOperation =
 					new StateAssignmentOperation(LOG, tasks, taskStates, allowNonRestoredState);
-
 			stateAssignmentOperation.assignStates();
 
+			// call master hooks for restore
+
+			MasterHooks.restoreMasterHooks(
+					masterHooks,
+					latest.getMasterHookStates(),
+					latest.getCheckpointID(),
+					allowNonRestoredState,
+					LOG);
+
+			// update metrics
+
 			if (statsTracker != null) {
 				long restoreTimestamp = System.currentTimeMillis();
 				RestoredCheckpointStats restored = new RestoredCheckpointStats(
@@ -1022,9 +1096,9 @@ public boolean restoreSavepoint(
 		return restoreLatestCheckpointedState(tasks, true, allowNonRestored);
 	}
 
-	// --------------------------------------------------------------------------------------------
+	// ------------------------------------------------------------------------
 	//  Accessors
-	// --------------------------------------------------------------------------------------------
+	// ------------------------------------------------------------------------
 
 	public int getNumberOfPendingCheckpoints() {
 		return this.pendingCheckpoints.size();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointDeclineReason.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointDeclineReason.java
index 60fe657c851b1..41c50cc08e45e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointDeclineReason.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointDeclineReason.java
@@ -36,7 +36,9 @@ public enum CheckpointDeclineReason {
 
 	NOT_ALL_REQUIRED_TASKS_RUNNING("Not all required tasks are currently running."),
 
-	EXCEPTION("An Exception occurred while triggering the checkpoint.");
+	EXCEPTION("An Exception occurred while triggering the checkpoint."),
+
+	EXPIRED("The checkpoint expired before triggering was complete");
 
 	// ------------------------------------------------------------------------
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
index 79fc31f862344..bb49b45637b8b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
@@ -27,11 +27,16 @@
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.Preconditions;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nullable;
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 
 import static org.apache.flink.util.Preconditions.checkArgument;
@@ -90,11 +95,14 @@ public class CompletedCheckpoint implements Serializable {
 	private final long duration;
 
 	/** States of the different task groups belonging to this checkpoint */
-	private final Map<JobVertexID, TaskState> taskStates;
+	private final HashMap<JobVertexID, TaskState> taskStates;
 
 	/** Properties for this checkpoint. */
 	private final CheckpointProperties props;
 
+	/** States that were created by a hook on the master (in the checkpoint coordinator) */
+	private final Collection<MasterState> masterHookStates;
+
 	/** The state handle to the externalized meta data, if the metadata has been externalized */
 	@Nullable
 	private final StreamStateHandle externalizedMetadata;
@@ -118,6 +126,7 @@ public class CompletedCheckpoint implements Serializable {
 			Map<JobVertexID, TaskState> taskStates) {
 
 		this(job, checkpointID, timestamp, completionTimestamp, taskStates,
+				Collections.<MasterState>emptyList(),
 				CheckpointProperties.forStandardCheckpoint());
 	}
 
@@ -127,9 +136,11 @@ public CompletedCheckpoint(
 			long timestamp,
 			long completionTimestamp,
 			Map<JobVertexID, TaskState> taskStates,
+			@Nullable Collection<MasterState> masterHookStates,
 			CheckpointProperties props) {
 
-		this(job, checkpointID, timestamp, completionTimestamp, taskStates, props, null, null);
+		this(job, checkpointID, timestamp, completionTimestamp, taskStates, 
+				masterHookStates, props, null, null);
 	}
 
 	public CompletedCheckpoint(
@@ -138,6 +149,7 @@ public CompletedCheckpoint(
 			long timestamp,
 			long completionTimestamp,
 			Map<JobVertexID, TaskState> taskStates,
+			@Nullable Collection<MasterState> masterHookStates,
 			CheckpointProperties props,
 			@Nullable StreamStateHandle externalizedMetadata,
 			@Nullable String externalPointer) {
@@ -156,7 +168,14 @@ public CompletedCheckpoint(
 		this.checkpointID = checkpointID;
 		this.timestamp = timestamp;
 		this.duration = completionTimestamp - timestamp;
-		this.taskStates = checkNotNull(taskStates);
+
+		// we create copies here, to make sure we have no shared mutable
+		// data structure with the "outside world"
+		this.taskStates = new HashMap<>(checkNotNull(taskStates));
+		this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
+				Collections.<MasterState>emptyList() :
+				new ArrayList<>(masterHookStates);
+
 		this.props = checkNotNull(props);
 		this.externalizedMetadata = externalizedMetadata;
 		this.externalPointer = externalPointer;
@@ -228,13 +247,17 @@ public long getStateSize() {
 	}
 
 	public Map<JobVertexID, TaskState> getTaskStates() {
-		return taskStates;
+		return Collections.unmodifiableMap(taskStates);
 	}
 
 	public TaskState getTaskState(JobVertexID jobVertexID) {
 		return taskStates.get(jobVertexID);
 	}
 
+	public Collection<MasterState> getMasterHookStates() {
+		return Collections.unmodifiableCollection(masterHookStates);
+	}
+
 	public boolean isExternalized() {
 		return externalizedMetadata != null;
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterState.java
new file mode 100644
index 0000000000000..2d09fdb62430a
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterState.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import java.util.Arrays;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Simple encapsulation of state generated by checkpoint coordinator.
+ */
+public class MasterState implements java.io.Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final String name;
+	private final byte[] bytes;
+	private final int version;
+
+	public MasterState(String name, byte[] bytes, int version) {
+		this.name = checkNotNull(name);
+		this.bytes = checkNotNull(bytes);
+		this.version = version;
+	}
+
+	// ------------------------------------------------------------------------
+
+	public String name() {
+		return name;
+	}
+
+	public byte[] bytes() {
+		return bytes;
+	}
+
+	public int version() {
+		return version;
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public String toString() {
+		return "name: " + name + " ; version: " + version + " ; bytes: " + Arrays.toString(bytes);
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterTriggerRestoreHook.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterTriggerRestoreHook.java
new file mode 100644
index 0000000000000..e77ed57653143
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/MasterTriggerRestoreHook.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.concurrent.Future;
+
+import javax.annotation.Nullable;
+import java.util.concurrent.Executor;
+
+/**
+ * The interface for hooks that can be called by the checkpoint coordinator when triggering or
+ * restoring a checkpoint. Such a hook is useful for example when preparing external systems for
+ * taking or restoring checkpoints.
+ * 
+ * <p>The {@link #triggerCheckpoint(long, long, Executor)} method (called when triggering a checkpoint)
+ * can return a result (via a future) that will be stored as part of the checkpoint metadata.
+ * When restoring a checkpoint, that stored result will be given to the {@link #restoreCheckpoint(long, Object)}
+ * method. The hook's {@link #getIdentifier() identifier} is used to map data to hook in the presence
+ * of multiple hooks, and when resuming a savepoint that was potentially created by a different job.
+ * The identifier has a similar role as for example the operator UID in the streaming API.
+ * 
+ * <p>The MasterTriggerRestoreHook is defined when creating the streaming dataflow graph. It is attached
+ * to the job graph, which gets sent to the cluster for execution. To avoid having to make the hook
+ * itself serializable, these hooks are attached to the job graph via a {@link MasterTriggerRestoreHook.Factory}.
+ * 
+ * @param <T> The type of the data produced by the hook and stored as part of the checkpoint metadata.
+ *            If the hook never stores any data, this can be typed to {@code Void}.
+ */
+public interface MasterTriggerRestoreHook<T> {
+
+	/**
+	 * Gets the identifier of this hook. The identifier is used to identify a specific hook in the
+	 * presence of multiple hooks and to give it the correct checkpointed data upon checkpoint restoration.
+	 * 
+	 * <p>The identifier should be unique between different hooks of a job, but deterministic/constant
+	 * so that upon resuming a savepoint, the hook will get the correct data.
+	 * For example, if the hook calls into another storage system and persists namespace/schema specific
+	 * information, then the name of the storage system, together with the namespace/schema name could
+	 * be an appropriate identifier.
+	 * 
+	 * <p>When multiple hooks of the same name are created and attached to a job graph, only the first
+	 * one is actually used. This can be exploited to deduplicate hooks that would do the same thing.
+	 * 
+	 * @return The identifier of the hook. 
+	 */
+	String getIdentifier();
+
+	/**
+	 * This method is called by the checkpoint coordinator prior when triggering a checkpoint, prior
+	 * to sending the "trigger checkpoint" messages to the source tasks.
+	 * 
+	 * <p>If the hook implementation wants to store data as part of the checkpoint, it may return
+	 * that data via a future, otherwise it should return null. The data is stored as part of
+	 * the checkpoint metadata under the hooks identifier (see {@link #getIdentifier()}).
+	 * 
+	 * <p>If the action by this hook needs to be executed synchronously, then this method should
+	 * directly execute the action synchronously and block until it is complete. The returned future
+	 * (if any) would typically be a completed future.
+	 * 
+	 * <p>If the action should be executed asynchronously and only needs to complete before the
+	 * checkpoint is considered completed, then the method may use the given executor to execute the
+	 * actual action and would signal its completion by completing the future. For hooks that do not
+	 * need to store data, the future would be completed with null.
+	 * 
+	 * @param checkpointId The ID (logical timestamp, monotonously increasing) of the checkpoint
+	 * @param timestamp The wall clock timestamp when the checkpoint was triggered, for
+	 *                  info/logging purposes. 
+	 * @param executor The executor for asynchronous actions
+	 * 
+	 * @return Optionally, a future that signals when the hook has completed and that contains
+	 *         data to be stored with the checkpoint.
+	 * 
+	 * @throws Exception Exceptions encountered when calling the hook will cause the checkpoint to abort.
+	 */
+	@Nullable
+	Future<T> triggerCheckpoint(long checkpointId, long timestamp, Executor executor) throws Exception;
+
+	/**
+	 * This method is called by the checkpoint coordinator prior to restoring the state of a checkpoint.
+	 * If the checkpoint did store data from this hook, that data will be passed to this method. 
+	 * 
+	 * @param checkpointId The The ID (logical timestamp) of the restored checkpoint
+	 * @param checkpointData The data originally stored in the checkpoint by this hook, possibly null. 
+	 * 
+	 * @throws Exception Exceptions thrown while restoring the checkpoint will cause the restore
+	 *                   operation to fail and to possibly fall back to another checkpoint. 
+	 */
+	void restoreCheckpoint(long checkpointId, @Nullable T checkpointData) throws Exception;
+
+	/**
+	 * Creates a the serializer to (de)serializes the data stored by this hook. The serializer
+	 * serializes the result of the Future returned by the {@link #triggerCheckpoint(long, long, Executor)}
+	 * method, and deserializes the data stored in the checkpoint into the object passed to the
+	 * {@link #restoreCheckpoint(long, Object)} method. 
+	 * 
+	 * <p>If the hook never returns any data to be stored, then this method may return null as the
+	 * serializer.
+	 * 
+	 * @return The serializer to (de)serializes the data stored by this hook
+	 */
+	@Nullable
+	SimpleVersionedSerializer<T> createCheckpointDataSerializer();
+
+	// ------------------------------------------------------------------------
+	//  factory
+	// ------------------------------------------------------------------------
+
+	/**
+	 * A factory to instantiate a {@code MasterTriggerRestoreHook}.
+	 * 
+	 * The hooks are defined when creating the streaming dataflow graph and are attached
+	 * to the job graph, which gets sent to the cluster for execution. To avoid having to make
+	 * the hook implementation serializable, a serializable hook factory is actually attached to the
+	 * job graph instead of the hook implementation itself.
+	 */
+	interface Factory extends java.io.Serializable {
+
+		/**
+		 * Instantiates the {@code MasterTriggerRestoreHook}.
+		 */
+		<V> MasterTriggerRestoreHook<V> create();
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
index 900331bb6c5eb..ce97edc676ca1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
@@ -21,7 +21,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.checkpoint.savepoint.Savepoint;
 import org.apache.flink.runtime.checkpoint.savepoint.SavepointStore;
-import org.apache.flink.runtime.checkpoint.savepoint.SavepointV1;
+import org.apache.flink.runtime.checkpoint.savepoint.SavepointV2;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -41,8 +41,10 @@
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.GuardedBy;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Executor;
@@ -89,6 +91,8 @@ public enum TaskAcknowledgeResult {
 
 	private final Map<ExecutionAttemptID, ExecutionVertex> notYetAcknowledgedTasks;
 
+	private final List<MasterState> masterState;
+
 	/** Set of acknowledged tasks */
 	private final Set<ExecutionAttemptID> acknowledgedTasks;
 
@@ -143,6 +147,7 @@ public PendingCheckpoint(
 		this.executor = Preconditions.checkNotNull(executor);
 
 		this.taskStates = new HashMap<>();
+		this.masterState = new ArrayList<>();
 		this.acknowledgedTasks = new HashSet<>(verticesToConfirm.size());
 		this.onCompletionPromise = new FlinkCompletableFuture<>();
 	}
@@ -256,7 +261,7 @@ public CompletedCheckpoint finalizeCheckpointExternalized() throws IOException {
 			// make sure we fulfill the promise with an exception if something fails
 			try {
 				// externalize the metadata
-				final Savepoint savepoint = new SavepointV1(checkpointId, taskStates.values());
+				final Savepoint savepoint = new SavepointV2(checkpointId, taskStates.values());
 
 				// TEMP FIX - The savepoint store is strictly typed to file systems currently
 				//            but the checkpoints think more generic. we need to work with file handles
@@ -321,7 +326,8 @@ private CompletedCheckpoint finalizeInternal(
 				checkpointId,
 				checkpointTimestamp,
 				System.currentTimeMillis(),
-				new HashMap<>(taskStates),
+				taskStates,
+				masterState,
 				props,
 				externalMetadata,
 				externalPointer);
@@ -344,6 +350,17 @@ private CompletedCheckpoint finalizeInternal(
 		return completed;
 	}
 
+	/**
+	 * Adds a master state (state generated on the checkpoint coordinator) to
+	 * the pending checkpoint.
+	 * 
+	 * @param state The state to add
+	 */
+	public void addMasterState(MasterState state) {
+		checkNotNull(state);
+		masterState.add(state);
+	}
+
 	/**
 	 * Acknowledges the task with the given execution attempt id and the given subtask state.
 	 *
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/hooks/MasterHooks.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/hooks/MasterHooks.java
new file mode 100644
index 0000000000000..409019e8327ca
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/hooks/MasterHooks.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint.hooks;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FlinkException;
+
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Collection of methods to deal with checkpoint master hooks.
+ */
+public class MasterHooks {
+
+	// ------------------------------------------------------------------------
+	//  checkpoint triggering
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Triggers all given master hooks and returns state objects for each hook that
+	 * produced a state.
+	 * 
+	 * @param hooks The hooks to trigger
+	 * @param checkpointId The checkpoint ID of the triggering checkpoint
+	 * @param timestamp The (informational) timestamp for the triggering checkpoint 
+	 * @param executor An executor that can be used for asynchronous I/O calls
+	 * @param timeout The maximum time that a hook may take to complete
+	 * 
+	 * @return A list containing all states produced by the hooks
+	 * 
+	 * @throws FlinkException Thrown, if the hooks throw an exception, or the state+
+	 *                        deserialization fails.
+	 */
+	public static List<MasterState> triggerMasterHooks(
+			Collection<MasterTriggerRestoreHook<?>> hooks,
+			long checkpointId,
+			long timestamp,
+			Executor executor,
+			Time timeout) throws FlinkException {
+
+		final ArrayList<MasterState> states = new ArrayList<>(hooks.size());
+
+		for (MasterTriggerRestoreHook<?> hook : hooks) {
+			MasterState state = triggerHook(hook, checkpointId, timestamp, executor, timeout);
+			if (state != null) {
+				states.add(state);
+			}
+		}
+
+		states.trimToSize();
+		return states;
+	}
+
+	private static <T> MasterState triggerHook(
+			MasterTriggerRestoreHook<?> hook,
+			long checkpointId,
+			long timestamp,
+			Executor executor,
+			Time timeout) throws FlinkException {
+
+		@SuppressWarnings("unchecked")
+		final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
+
+		final String id = typedHook.getIdentifier();
+		final SimpleVersionedSerializer<T> serializer = typedHook.createCheckpointDataSerializer();
+
+		// call the hook!
+		final Future<T> resultFuture;
+		try {
+			resultFuture = typedHook.triggerCheckpoint(checkpointId, timestamp, executor);
+		}
+		catch (Throwable t) {
+			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
+			throw new FlinkException("Error while triggering checkpoint master hook '" + id + '\'', t);
+		}
+
+		// is there is a result future, wait for its completion
+		// in the future we want to make this asynchronous with futures (no pun intended)
+		if (resultFuture == null) {
+			return null;
+		}
+		else {
+			final T result;
+			try {
+				result = resultFuture.get(timeout.getSize(), timeout.getUnit());
+			}
+			catch (InterruptedException e) {
+				// cannot continue here - restore interrupt status and leave
+				Thread.currentThread().interrupt();
+				throw new FlinkException("Checkpoint master hook was interrupted");
+			}
+			catch (ExecutionException e) {
+				throw new FlinkException("Checkpoint master hook '" + id + "' produced an exception", e.getCause());
+			}
+			catch (TimeoutException e) {
+				throw new FlinkException("Checkpoint master hook '" + id +
+						"' did not complete in time (" + timeout + ')');
+			}
+
+			// if the result of the future is not null, return it as state
+			if (result == null) {
+				return null;
+			}
+			else if (serializer != null) {
+				try {
+					final int version = serializer.getVersion();
+					final byte[] bytes = serializer.serialize(result);
+
+					return new MasterState(id, bytes, version);
+				}
+				catch (Throwable t) {
+					ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
+					throw new FlinkException("Failed to serialize state of master hook '" + id + '\'', t);
+				}
+			}
+			else {
+				throw new FlinkException("Checkpoint hook '" + id + " is stateful but creates no serializer");
+			}
+		}
+	}
+
+	// ------------------------------------------------------------------------
+	//  checkpoint restoring
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Calls the restore method given checkpoint master hooks and passes the given master
+	 * state to them where state with a matching name is found. 
+	 * 
+	 * <p>If state is found and no hook with the same name is found, the method throws an
+	 * exception, unless the {@code allowUnmatchedState} flag is set.
+	 *     
+	 * @param masterHooks The hooks to call restore on
+	 * @param states The state to pass to the hooks
+	 * @param checkpointId The checkpoint ID of the restored checkpoint
+	 * @param allowUnmatchedState True, 
+	 * @param log The logger for log messages
+	 * 
+	 * @throws FlinkException Thrown, if the hooks throw an exception, or the state+
+	 *                        deserialization fails.
+	 */
+	public static void restoreMasterHooks(
+			final Map<String, MasterTriggerRestoreHook<?>> masterHooks,
+			final Collection<MasterState> states,
+			final long checkpointId,
+			final boolean allowUnmatchedState,
+			final Logger log) throws FlinkException {
+
+		// early out
+		if (states == null || states.isEmpty() || masterHooks == null || masterHooks.isEmpty()) {
+			log.info("No master state to restore");
+			return;
+		}
+
+		log.info("Calling master restore hooks");
+
+		// collect the hooks
+		final LinkedHashMap<String, MasterTriggerRestoreHook<?>> allHooks = new LinkedHashMap<>(masterHooks);
+
+		// first, deserialize all hook state
+		final ArrayList<Tuple2<MasterTriggerRestoreHook<?>, Object>> hooksAndStates = new ArrayList<>();
+
+		for (MasterState state : states) {
+			if (state != null) {
+				final String name = state.name();
+				final MasterTriggerRestoreHook<?> hook = allHooks.remove(name);
+
+				if (hook != null) {
+					log.debug("Found state to restore for hook '{}'", name);
+
+					Object deserializedState = deserializeState(state, hook);
+					hooksAndStates.add(new Tuple2<MasterTriggerRestoreHook<?>, Object>(hook, deserializedState));
+				}
+				else if (!allowUnmatchedState) {
+					throw new IllegalStateException("Found state '" + state.name() +
+							"' which is not resumed by any hook.");
+				}
+				else {
+					log.info("Dropping unmatched state from '{}'", name);
+				}
+			}
+		}
+
+		// now that all is deserialized, call the hooks 
+		for (Tuple2<MasterTriggerRestoreHook<?>, Object> hookAndState : hooksAndStates) {
+			restoreHook(hookAndState.f1, hookAndState.f0, checkpointId);
+		}
+
+		// trigger the remaining hooks without checkpointed state
+		for (MasterTriggerRestoreHook<?> hook : allHooks.values()) {
+			restoreHook(null, hook, checkpointId);
+		}
+	}
+
+	private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
+		@SuppressWarnings("unchecked")
+		final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
+		final String id = hook.getIdentifier();
+
+		try {
+			final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
+			if (deserializer == null) {
+				throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
+			}
+
+			return deserializer.deserialize(state.version(), state.bytes());
+		}
+		catch (Throwable t) {
+			throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
+		}
+	}
+
+	private static <T> void restoreHook(
+			final Object state,
+			final MasterTriggerRestoreHook<?> hook,
+			final long checkpointId) throws FlinkException {
+
+		@SuppressWarnings("unchecked")
+		final T typedState = (T) state;
+
+		@SuppressWarnings("unchecked")
+		final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
+
+		try {
+			typedHook.restoreCheckpoint(checkpointId, typedState);
+		}
+		catch (FlinkException e) {
+			throw e;
+		}
+		catch (Throwable t) {
+			// catch all here, including Errors that may come from dependency and classpath issues
+			ExceptionUtils.rethrowIfFatalError(t);
+			throw new FlinkException("Error while calling restoreCheckpoint on checkpoint hook '"
+					+ hook.getIdentifier() + '\'', t);
+		}
+	}
+
+	// ------------------------------------------------------------------------
+
+	/** This class is not meant to be instantiated */
+	private MasterHooks() {}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java
index baad05f79b1e5..79ec59630bf73 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.core.io.Versioned;
 import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 
 import java.util.Collection;
@@ -57,6 +58,11 @@ public interface Savepoint extends Versioned {
 	 */
 	Collection<TaskState> getTaskStates();
 
+	/**
+	 * Gets the checkpointed states generated by the master.
+	 */
+	Collection<MasterState> getMasterStates();
+
 	/**
 	 * Disposes the savepoint.
 	 */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java
index 60f02878532b3..8ee38daaafd2f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java
@@ -26,6 +26,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.state.StreamStateHandle;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -122,7 +123,7 @@ public static CompletedCheckpoint loadAndValidateSavepoint(
 		// (3) convert to checkpoint so the system can fall back to it
 		CheckpointProperties props = CheckpointProperties.forStandardSavepoint();
 		return new CompletedCheckpoint(jobId, savepoint.getCheckpointId(), 0L, 0L,
-				taskStates, props, metadataHandle, savepointPath);
+				taskStates, savepoint.getMasterStates(), props, metadataHandle, savepointPath);
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointSerializers.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointSerializers.java
index 3155d609c8429..c1fcf4f0d4547 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointSerializers.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointSerializers.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.checkpoint.savepoint;
 
+import org.apache.flink.migration.runtime.checkpoint.savepoint.SavepointV0;
 import org.apache.flink.migration.runtime.checkpoint.savepoint.SavepointV0Serializer;
 import org.apache.flink.util.Preconditions;
 
@@ -30,14 +31,16 @@
 public class SavepointSerializers {
 
 
-	private static final int SAVEPOINT_VERSION_0 = 0;
 	private static final Map<Integer, SavepointSerializer<?>> SERIALIZERS = new HashMap<>(2);
 
 	static {
-		SERIALIZERS.put(SAVEPOINT_VERSION_0, SavepointV0Serializer.INSTANCE);
+		SERIALIZERS.put(SavepointV0.VERSION, SavepointV0Serializer.INSTANCE);
 		SERIALIZERS.put(SavepointV1.VERSION, SavepointV1Serializer.INSTANCE);
+		SERIALIZERS.put(SavepointV2.VERSION, SavepointV2Serializer.INSTANCE);
 	}
 
+	// ------------------------------------------------------------------------
+
 	/**
 	 * Returns the {@link SavepointSerializer} for the given savepoint.
 	 *
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java
index 5976bbfe782ba..196c8704d5a1a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.checkpoint.savepoint;
 
+import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.util.Preconditions;
 
@@ -60,36 +61,21 @@ public Collection<TaskState> getTaskStates() {
 	}
 
 	@Override
-	public void dispose() throws Exception {
-		for (TaskState taskState : taskStates) {
-			taskState.discardState();
-		}
-		taskStates.clear();
-	}
-
-	@Override
-	public String toString() {
-		return "Savepoint(version=" + VERSION + ")";
+	public Collection<MasterState> getMasterStates() {
+		// since checkpoints are never deserialized into this format,
+		// this method should never be called
+		throw new UnsupportedOperationException();
 	}
 
 	@Override
-	public boolean equals(Object o) {
-		if (this == o) {
-			return true;
-		}
-
-		if (o == null || getClass() != o.getClass()) {
-			return false;
-		}
-
-		SavepointV1 that = (SavepointV1) o;
-		return checkpointId == that.checkpointId && getTaskStates().equals(that.getTaskStates());
+	public void dispose() throws Exception {
+		// since checkpoints are never deserialized into this format,
+		// this method should never be called
+		throw new UnsupportedOperationException();
 	}
 
 	@Override
-	public int hashCode() {
-		int result = (int) (checkpointId ^ (checkpointId >>> 32));
-		result = 31 * result + taskStates.hashCode();
-		return result;
+	public String toString() {
+		return "Savepoint(version=" + VERSION + ")";
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
index 44461d8b8e5a8..ae9f4a9b1ea2d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.checkpoint.savepoint;
 
 import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.SubtaskState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -37,18 +38,19 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 /**
- * Serializer for {@link SavepointV1} instances.
- * <p>
- * <p>In contrast to previous savepoint versions, this serializer makes sure
- * that no default Java serialization is used for serialization. Therefore, we
- * don't rely on any involved Java classes to stay the same.
+ * Deserializer for checkpoints written in format {@code 1} (Flink 1.2.x format)
+ * 
+ * <p>In contrast to the previous versions, this serializer makes sure that no Java
+ * serialization is used for serialization. Therefore, we don't rely on any involved 
+ * classes to stay the same.
  */
-class SavepointV1Serializer implements SavepointSerializer<SavepointV1> {
+class SavepointV1Serializer implements SavepointSerializer<SavepointV2> {
 
 	private static final byte NULL_HANDLE = 0;
 	private static final byte BYTE_STREAM_STATE_HANDLE = 1;
@@ -63,39 +65,12 @@ private SavepointV1Serializer() {
 	}
 
 	@Override
-	public void serialize(SavepointV1 savepoint, DataOutputStream dos) throws IOException {
-		try {
-			dos.writeLong(savepoint.getCheckpointId());
-
-			Collection<TaskState> taskStates = savepoint.getTaskStates();
-			dos.writeInt(taskStates.size());
-
-			for (TaskState taskState : savepoint.getTaskStates()) {
-				// Vertex ID
-				dos.writeLong(taskState.getJobVertexID().getLowerPart());
-				dos.writeLong(taskState.getJobVertexID().getUpperPart());
-
-				// Parallelism
-				int parallelism = taskState.getParallelism();
-				dos.writeInt(parallelism);
-				dos.writeInt(taskState.getMaxParallelism());
-				dos.writeInt(taskState.getChainLength());
-
-				// Sub task states
-				Map<Integer, SubtaskState> subtaskStateMap = taskState.getSubtaskStates();
-				dos.writeInt(subtaskStateMap.size());
-				for (Map.Entry<Integer, SubtaskState> entry : subtaskStateMap.entrySet()) {
-					dos.writeInt(entry.getKey());
-					serializeSubtaskState(entry.getValue(), dos);
-				}
-			}
-		} catch (Exception e) {
-			throw new IOException(e);
-		}
+	public void serialize(SavepointV2 savepoint, DataOutputStream dos) throws IOException {
+		throw new UnsupportedOperationException("This serializer is read-only and only exists for backwards compatibility");
 	}
 
 	@Override
-	public SavepointV1 deserialize(DataInputStream dis, ClassLoader cl) throws IOException {
+	public SavepointV2 deserialize(DataInputStream dis, ClassLoader cl) throws IOException {
 		long checkpointId = dis.readLong();
 
 		// Task states
@@ -122,7 +97,34 @@ public SavepointV1 deserialize(DataInputStream dis, ClassLoader cl) throws IOExc
 			}
 		}
 
-		return new SavepointV1(checkpointId, taskStates);
+		return new SavepointV2(checkpointId, taskStates, Collections.<MasterState>emptyList());
+	}
+
+	public void serializeOld(SavepointV1 savepoint, DataOutputStream dos) throws IOException {
+		dos.writeLong(savepoint.getCheckpointId());
+
+		Collection<TaskState> taskStates = savepoint.getTaskStates();
+		dos.writeInt(taskStates.size());
+
+		for (TaskState taskState : savepoint.getTaskStates()) {
+			// Vertex ID
+			dos.writeLong(taskState.getJobVertexID().getLowerPart());
+			dos.writeLong(taskState.getJobVertexID().getUpperPart());
+
+			// Parallelism
+			int parallelism = taskState.getParallelism();
+			dos.writeInt(parallelism);
+			dos.writeInt(taskState.getMaxParallelism());
+			dos.writeInt(taskState.getChainLength());
+
+			// Sub task states
+			Map<Integer, SubtaskState> subtaskStateMap = taskState.getSubtaskStates();
+			dos.writeInt(subtaskStateMap.size());
+			for (Map.Entry<Integer, SubtaskState> entry : subtaskStateMap.entrySet()) {
+				dos.writeInt(entry.getKey());
+				serializeSubtaskState(entry.getValue(), dos);
+			}
+		}
 	}
 
 	private static void serializeSubtaskState(SubtaskState subtaskState, DataOutputStream dos) throws IOException {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java
new file mode 100644
index 0000000000000..100982d80a3c8
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint.savepoint;
+
+import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.TaskState;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * The persistent checkpoint metadata, format version 2.
+ * his format was introduced with Flink 1.3.0.
+ */
+public class SavepointV2 implements Savepoint {
+
+	/** The savepoint version. */
+	public static final int VERSION = 2;
+
+	/** The checkpoint ID */
+	private final long checkpointId;
+
+	/** The task states */
+	private final Collection<TaskState> taskStates;
+
+	/** The states generated by the CheckpointCoordinator */
+	private final Collection<MasterState> masterStates;
+
+
+	public SavepointV2(long checkpointId, Collection<TaskState> taskStates) {
+		this(checkpointId, taskStates, Collections.<MasterState>emptyList());
+	}
+
+	public SavepointV2(long checkpointId, Collection<TaskState> taskStates, Collection<MasterState> masterStates) {
+		this.checkpointId = checkpointId;
+		this.taskStates = checkNotNull(taskStates, "taskStates");
+		this.masterStates = checkNotNull(masterStates, "masterStates");
+	}
+
+	@Override
+	public int getVersion() {
+		return VERSION;
+	}
+
+	@Override
+	public long getCheckpointId() {
+		return checkpointId;
+	}
+
+	@Override
+	public Collection<TaskState> getTaskStates() {
+		return taskStates;
+	}
+
+	@Override
+	public Collection<MasterState> getMasterStates() {
+		return masterStates;
+	}
+
+	@Override
+	public void dispose() throws Exception {
+		for (TaskState taskState : taskStates) {
+			taskState.discardState();
+		}
+		taskStates.clear();
+		masterStates.clear();
+	}
+
+	@Override
+	public String toString() {
+		return "Checkpoint Metadata (version=" + VERSION + ')';
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
new file mode 100644
index 0000000000000..307ea1641fece
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint.savepoint;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.SubtaskState;
+import org.apache.flink.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyGroupRangeOffsets;
+import org.apache.flink.runtime.state.KeyGroupsStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.runtime.state.filesystem.FileStateHandle;
+import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * (De)serializer for checkpoint metadata format version 2.
+ * 
+ * <p>This format version adds
+ * 
+ * <p>Basic checkpoint metadata layout:
+ * <pre>
+ *  +--------------+---------------+-----------------+
+ *  | checkpointID | master states | operator states |
+ *  +--------------+---------------+-----------------+
+ *  
+ *  Master state:
+ *  +--------------+---------------------+---------+------+---------------+
+ *  | magic number | num remaining bytes | version | name | payload bytes |
+ *  +--------------+---------------------+---------+------+---------------+
+ * </pre>
+ */
+class SavepointV2Serializer implements SavepointSerializer<SavepointV2> {
+
+	/** Random magic number for consistency checks */
+	private static final int MASTER_STATE_MAGIC_NUMBER = 0xc96b1696;
+
+	private static final byte NULL_HANDLE = 0;
+	private static final byte BYTE_STREAM_STATE_HANDLE = 1;
+	private static final byte FILE_STREAM_STATE_HANDLE = 2;
+	private static final byte KEY_GROUPS_HANDLE = 3;
+	private static final byte PARTITIONABLE_OPERATOR_STATE_HANDLE = 4;
+
+	/** The singleton instance of the serializer */
+	public static final SavepointV2Serializer INSTANCE = new SavepointV2Serializer();
+
+	// ------------------------------------------------------------------------
+
+	/** Singleton, not meant to be instantiated */
+	private SavepointV2Serializer() {}
+
+	// ------------------------------------------------------------------------
+	//  (De)serialization entry points
+	// ------------------------------------------------------------------------
+
+	@Override
+	public void serialize(SavepointV2 checkpointMetadata, DataOutputStream dos) throws IOException {
+		// first: checkpoint ID
+		dos.writeLong(checkpointMetadata.getCheckpointId());
+
+		// second: master state
+		final Collection<MasterState> masterStates = checkpointMetadata.getMasterStates();
+		dos.writeInt(masterStates.size());
+		for (MasterState ms : masterStates) {
+			serializeMasterState(ms, dos);
+		}
+
+		// third: task states
+		final Collection<TaskState> taskStates = checkpointMetadata.getTaskStates();
+		dos.writeInt(taskStates.size());
+
+		for (TaskState taskState : checkpointMetadata.getTaskStates()) {
+			// Vertex ID
+			dos.writeLong(taskState.getJobVertexID().getLowerPart());
+			dos.writeLong(taskState.getJobVertexID().getUpperPart());
+
+			// Parallelism
+			int parallelism = taskState.getParallelism();
+			dos.writeInt(parallelism);
+			dos.writeInt(taskState.getMaxParallelism());
+			dos.writeInt(taskState.getChainLength());
+
+			// Sub task states
+			Map<Integer, SubtaskState> subtaskStateMap = taskState.getSubtaskStates();
+			dos.writeInt(subtaskStateMap.size());
+			for (Map.Entry<Integer, SubtaskState> entry : subtaskStateMap.entrySet()) {
+				dos.writeInt(entry.getKey());
+				serializeSubtaskState(entry.getValue(), dos);
+			}
+		}
+	}
+
+	@Override
+	public SavepointV2 deserialize(DataInputStream dis, ClassLoader cl) throws IOException {
+		// first: checkpoint ID
+		final long checkpointId = dis.readLong();
+		if (checkpointId < 0) {
+			throw new IOException("invalid checkpoint ID: " + checkpointId);
+		}
+
+		// second: master state
+		final List<MasterState> masterStates;
+		final int numMasterStates = dis.readInt();
+
+		if (numMasterStates == 0) {
+			masterStates = Collections.emptyList();
+		}
+		else if (numMasterStates > 0) {
+			masterStates = new ArrayList<>(numMasterStates);
+			for (int i = 0; i < numMasterStates; i++) {
+				masterStates.add(deserializeMasterState(dis));
+			}
+		}
+		else {
+			throw new IOException("invalid number of master states: " + numMasterStates);
+		}
+
+		// third: task states
+		final int numTaskStates = dis.readInt();
+		final ArrayList<TaskState> taskStates = new ArrayList<>(numTaskStates);
+
+		for (int i = 0; i < numTaskStates; i++) {
+			JobVertexID jobVertexId = new JobVertexID(dis.readLong(), dis.readLong());
+			int parallelism = dis.readInt();
+			int maxParallelism = dis.readInt();
+			int chainLength = dis.readInt();
+
+			// Add task state
+			TaskState taskState = new TaskState(jobVertexId, parallelism, maxParallelism, chainLength);
+			taskStates.add(taskState);
+
+			// Sub task states
+			int numSubTaskStates = dis.readInt();
+
+			for (int j = 0; j < numSubTaskStates; j++) {
+				int subtaskIndex = dis.readInt();
+				SubtaskState subtaskState = deserializeSubtaskState(dis);
+				taskState.putState(subtaskIndex, subtaskState);
+			}
+		}
+
+		return new SavepointV2(checkpointId, taskStates, masterStates);
+	}
+
+	// ------------------------------------------------------------------------
+	//  master state (de)serialization methods
+	// ------------------------------------------------------------------------
+
+	private void serializeMasterState(MasterState state, DataOutputStream dos) throws IOException {
+		// magic number for error detection
+		dos.writeInt(MASTER_STATE_MAGIC_NUMBER);
+
+		// for safety, we serialize first into an array and then write the array and its
+		// length into the checkpoint
+		final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+		final DataOutputStream out = new DataOutputStream(baos);
+
+		out.writeInt(state.version());
+		out.writeUTF(state.name());
+
+		final byte[] bytes = state.bytes();
+		out.writeInt(bytes.length);
+		out.write(bytes, 0, bytes.length);
+
+		out.close();
+		byte[] data = baos.toByteArray();
+
+		dos.writeInt(data.length);
+		dos.write(data, 0, data.length);
+	}
+
+	private MasterState deserializeMasterState(DataInputStream dis) throws IOException {
+		final int magicNumber = dis.readInt();
+		if (magicNumber != MASTER_STATE_MAGIC_NUMBER) {
+			throw new IOException("incorrect magic number in master styte byte sequence");
+		}
+
+		final int numBytes = dis.readInt();
+		if (numBytes <= 0) {
+			throw new IOException("found zero or negative length for master state bytes");
+		}
+
+		final byte[] data = new byte[numBytes];
+		dis.readFully(data);
+
+		final DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
+
+		final int version = in.readInt();
+		final String name = in.readUTF();
+
+		final byte[] bytes = new byte[in.readInt()];
+		in.readFully(bytes);
+
+		// check that the data is not corrupt
+		if (in.read() != -1) {
+			throw new IOException("found trailing bytes in master state");
+		}
+
+		return new MasterState(name, bytes, version);
+	}
+
+	// ------------------------------------------------------------------------
+	//  task state (de)serialization methods
+	// ------------------------------------------------------------------------
+
+	private static void serializeSubtaskState(SubtaskState subtaskState, DataOutputStream dos) throws IOException {
+
+		dos.writeLong(-1);
+
+		ChainedStateHandle<StreamStateHandle> nonPartitionableState = subtaskState.getLegacyOperatorState();
+
+		int len = nonPartitionableState != null ? nonPartitionableState.getLength() : 0;
+		dos.writeInt(len);
+		for (int i = 0; i < len; ++i) {
+			StreamStateHandle stateHandle = nonPartitionableState.get(i);
+			serializeStreamStateHandle(stateHandle, dos);
+		}
+
+		ChainedStateHandle<OperatorStateHandle> operatorStateBackend = subtaskState.getManagedOperatorState();
+
+		len = operatorStateBackend != null ? operatorStateBackend.getLength() : 0;
+		dos.writeInt(len);
+		for (int i = 0; i < len; ++i) {
+			OperatorStateHandle stateHandle = operatorStateBackend.get(i);
+			serializeOperatorStateHandle(stateHandle, dos);
+		}
+
+		ChainedStateHandle<OperatorStateHandle> operatorStateFromStream = subtaskState.getRawOperatorState();
+
+		len = operatorStateFromStream != null ? operatorStateFromStream.getLength() : 0;
+		dos.writeInt(len);
+		for (int i = 0; i < len; ++i) {
+			OperatorStateHandle stateHandle = operatorStateFromStream.get(i);
+			serializeOperatorStateHandle(stateHandle, dos);
+		}
+
+		KeyedStateHandle keyedStateBackend = subtaskState.getManagedKeyedState();
+		serializeKeyedStateHandle(keyedStateBackend, dos);
+
+		KeyedStateHandle keyedStateStream = subtaskState.getRawKeyedState();
+		serializeKeyedStateHandle(keyedStateStream, dos);
+	}
+
+	private static SubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException {
+		// Duration field has been removed from SubtaskState
+		long ignoredDuration = dis.readLong();
+
+		int len = dis.readInt();
+		List<StreamStateHandle> nonPartitionableState = new ArrayList<>(len);
+		for (int i = 0; i < len; ++i) {
+			StreamStateHandle streamStateHandle = deserializeStreamStateHandle(dis);
+			nonPartitionableState.add(streamStateHandle);
+		}
+
+
+		len = dis.readInt();
+		List<OperatorStateHandle> operatorStateBackend = new ArrayList<>(len);
+		for (int i = 0; i < len; ++i) {
+			OperatorStateHandle streamStateHandle = deserializeOperatorStateHandle(dis);
+			operatorStateBackend.add(streamStateHandle);
+		}
+
+		len = dis.readInt();
+		List<OperatorStateHandle> operatorStateStream = new ArrayList<>(len);
+		for (int i = 0; i < len; ++i) {
+			OperatorStateHandle streamStateHandle = deserializeOperatorStateHandle(dis);
+			operatorStateStream.add(streamStateHandle);
+		}
+
+		KeyedStateHandle keyedStateBackend = deserializeKeyedStateHandle(dis);
+
+		KeyedStateHandle keyedStateStream = deserializeKeyedStateHandle(dis);
+
+		ChainedStateHandle<StreamStateHandle> nonPartitionableStateChain =
+				new ChainedStateHandle<>(nonPartitionableState);
+
+		ChainedStateHandle<OperatorStateHandle> operatorStateBackendChain =
+				new ChainedStateHandle<>(operatorStateBackend);
+
+		ChainedStateHandle<OperatorStateHandle> operatorStateStreamChain =
+				new ChainedStateHandle<>(operatorStateStream);
+
+		return new SubtaskState(
+				nonPartitionableStateChain,
+				operatorStateBackendChain,
+				operatorStateStreamChain,
+				keyedStateBackend,
+				keyedStateStream);
+	}
+
+	private static void serializeKeyedStateHandle(
+			KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
+
+		if (stateHandle == null) {
+			dos.writeByte(NULL_HANDLE);
+		} else if (stateHandle instanceof KeyGroupsStateHandle) {
+			KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;
+
+			dos.writeByte(KEY_GROUPS_HANDLE);
+			dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
+			dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
+			for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
+				dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
+			}
+			serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
+		} else {
+			throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
+		}
+	}
+
+	private static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
+		final int type = dis.readByte();
+		if (NULL_HANDLE == type) {
+			return null;
+		} else if (KEY_GROUPS_HANDLE == type) {
+			int startKeyGroup = dis.readInt();
+			int numKeyGroups = dis.readInt();
+			KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
+			long[] offsets = new long[numKeyGroups];
+			for (int i = 0; i < numKeyGroups; ++i) {
+				offsets[i] = dis.readLong();
+			}
+			KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(
+				keyGroupRange, offsets);
+			StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
+			return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
+		} else {
+			throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
+		}
+	}
+
+	private static void serializeOperatorStateHandle(
+			OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {
+
+		if (stateHandle != null) {
+			dos.writeByte(PARTITIONABLE_OPERATOR_STATE_HANDLE);
+			Map<String, OperatorStateHandle.StateMetaInfo> partitionOffsetsMap =
+					stateHandle.getStateNameToPartitionOffsets();
+			dos.writeInt(partitionOffsetsMap.size());
+			for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : partitionOffsetsMap.entrySet()) {
+				dos.writeUTF(entry.getKey());
+
+				OperatorStateHandle.StateMetaInfo stateMetaInfo = entry.getValue();
+
+				int mode = stateMetaInfo.getDistributionMode().ordinal();
+				dos.writeByte(mode);
+
+				long[] offsets = stateMetaInfo.getOffsets();
+				dos.writeInt(offsets.length);
+				for (long offset : offsets) {
+					dos.writeLong(offset);
+				}
+			}
+			serializeStreamStateHandle(stateHandle.getDelegateStateHandle(), dos);
+		} else {
+			dos.writeByte(NULL_HANDLE);
+		}
+	}
+
+	private static OperatorStateHandle deserializeOperatorStateHandle(
+			DataInputStream dis) throws IOException {
+
+		final int type = dis.readByte();
+		if (NULL_HANDLE == type) {
+			return null;
+		} else if (PARTITIONABLE_OPERATOR_STATE_HANDLE == type) {
+			int mapSize = dis.readInt();
+			Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(mapSize);
+			for (int i = 0; i < mapSize; ++i) {
+				String key = dis.readUTF();
+
+				int modeOrdinal = dis.readByte();
+				OperatorStateHandle.Mode mode = OperatorStateHandle.Mode.values()[modeOrdinal];
+
+				long[] offsets = new long[dis.readInt()];
+				for (int j = 0; j < offsets.length; ++j) {
+					offsets[j] = dis.readLong();
+				}
+
+				OperatorStateHandle.StateMetaInfo metaInfo =
+						new OperatorStateHandle.StateMetaInfo(offsets, mode);
+				offsetsMap.put(key, metaInfo);
+			}
+			StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
+			return new OperatorStateHandle(offsetsMap, stateHandle);
+		} else {
+			throw new IllegalStateException("Reading invalid OperatorStateHandle, type: " + type);
+		}
+	}
+
+	private static void serializeStreamStateHandle(
+			StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {
+
+		if (stateHandle == null) {
+			dos.writeByte(NULL_HANDLE);
+
+		} else if (stateHandle instanceof FileStateHandle) {
+			dos.writeByte(FILE_STREAM_STATE_HANDLE);
+			FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
+			dos.writeLong(stateHandle.getStateSize());
+			dos.writeUTF(fileStateHandle.getFilePath().toString());
+
+		} else if (stateHandle instanceof ByteStreamStateHandle) {
+			dos.writeByte(BYTE_STREAM_STATE_HANDLE);
+			ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
+			dos.writeUTF(byteStreamStateHandle.getHandleName());
+			byte[] internalData = byteStreamStateHandle.getData();
+			dos.writeInt(internalData.length);
+			dos.write(byteStreamStateHandle.getData());
+
+		} else {
+			throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
+		}
+
+		dos.flush();
+	}
+
+	private static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
+		final int type = dis.read();
+		if (NULL_HANDLE == type) {
+			return null;
+		} else if (FILE_STREAM_STATE_HANDLE == type) {
+			long size = dis.readLong();
+			String pathString = dis.readUTF();
+			return new FileStateHandle(new Path(pathString), size);
+		} else if (BYTE_STREAM_STATE_HANDLE == type) {
+			String handleName = dis.readUTF();
+			int numBytes = dis.readInt();
+			byte[] data = new byte[numBytes];
+			dis.readFully(data);
+			return new ByteStreamStateHandle(handleName, data);
+		} else {
+			throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 29b980692b7ac..23ed99d26ee72 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -39,6 +39,7 @@
 import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
 import org.apache.flink.runtime.checkpoint.CheckpointStatsTracker;
 import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.concurrent.BiFunction;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.concurrent.FutureUtils;
@@ -360,6 +361,7 @@ public void enableCheckpointing(
 			List<ExecutionJobVertex> verticesToTrigger,
 			List<ExecutionJobVertex> verticesToWaitFor,
 			List<ExecutionJobVertex> verticesToCommitTo,
+			List<MasterTriggerRestoreHook<?>> masterHooks,
 			CheckpointIDCounter checkpointIDCounter,
 			CompletedCheckpointStore checkpointStore,
 			String checkpointDir,
@@ -395,6 +397,13 @@ public void enableCheckpointing(
 			checkpointDir,
 			ioExecutor);
 
+		// register the master hooks on the checkpoint coordinator
+		for (MasterTriggerRestoreHook<?> hook : masterHooks) {
+			if (!checkpointCoordinator.addMasterHook(hook)) {
+				LOG.warn("Trying to register multiple checkpoint hooks with the name: {}", hook.getIdentifier());
+			}
+		}
+
 		checkpointCoordinator.setCheckpointStatsTracker(checkpointStatsTracker);
 
 		// interval of max long value indicates disable periodic checkpoint,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
index a10c62e31882c..b40817ff8adc1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
@@ -31,6 +31,7 @@
 import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
 import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
 import org.apache.flink.runtime.checkpoint.CheckpointStatsTracker;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.client.JobSubmissionException;
 import org.apache.flink.runtime.executiongraph.metrics.DownTimeGauge;
@@ -51,6 +52,7 @@
 import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
@@ -230,6 +232,21 @@ public static ExecutionGraph buildGraph(
 				}
 			}
 
+			// instantiate the user-defined checkpoint hooks
+
+			final MasterTriggerRestoreHook.Factory[] hookFactories = snapshotSettings.getMasterHooks();
+			final List<MasterTriggerRestoreHook<?>> hooks;
+
+			if (hookFactories == null || hookFactories.length == 0) {
+				hooks = Collections.emptyList();
+			}
+			else {
+				hooks = new ArrayList<>(hookFactories.length);
+				for (MasterTriggerRestoreHook.Factory factory : hookFactories) {
+					hooks.add(factory.create());
+				}
+			}
+
 			executionGraph.enableCheckpointing(
 					snapshotSettings.getCheckpointInterval(),
 					snapshotSettings.getCheckpointTimeout(),
@@ -239,6 +256,7 @@ public static ExecutionGraph buildGraph(
 					triggerVertices,
 					ackVertices,
 					confirmVertices,
+					hooks,
 					checkpointIdCounter,
 					completedCheckpoints,
 					externalizedCheckpointsDir,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
index 38130d4c9978a..3dd037ed71505 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.jobgraph.tasks;
 
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.state.StateBackend;
 
@@ -32,21 +33,21 @@
  * need to participate.
  */
 public class JobCheckpointingSettings implements java.io.Serializable {
-	
+
 	private static final long serialVersionUID = -2593319571078198180L;
-	
+
 	private final List<JobVertexID> verticesToTrigger;
 
 	private final List<JobVertexID> verticesToAcknowledge;
 
 	private final List<JobVertexID> verticesToConfirm;
-	
+
 	private final long checkpointInterval;
-	
+
 	private final long checkpointTimeout;
-	
+
 	private final long minPauseBetweenCheckpoints;
-	
+
 	private final int maxConcurrentCheckpoints;
 
 	/** Settings for externalized checkpoints. */
@@ -56,6 +57,9 @@ public class JobCheckpointingSettings implements java.io.Serializable {
 	@Nullable
 	private final StateBackend defaultStateBackend;
 
+	/** (Factories for) hooks that are executed on the checkpoint coordinator */
+	private final MasterTriggerRestoreHook.Factory[] masterHooks;
+
 	/**
 	 * Flag indicating whether exactly once checkpoint mode has been configured.
 	 * If <code>false</code>, at least once mode has been configured. This is
@@ -77,12 +81,30 @@ public JobCheckpointingSettings(
 			@Nullable StateBackend defaultStateBackend,
 			boolean isExactlyOnce) {
 
+		this(verticesToTrigger, verticesToAcknowledge, verticesToConfirm,
+				checkpointInterval, checkpointTimeout, minPauseBetweenCheckpoints, maxConcurrentCheckpoints,
+				externalizedCheckpointSettings, defaultStateBackend, null, isExactlyOnce);
+	}
+
+	public JobCheckpointingSettings(
+			List<JobVertexID> verticesToTrigger,
+			List<JobVertexID> verticesToAcknowledge,
+			List<JobVertexID> verticesToConfirm,
+			long checkpointInterval,
+			long checkpointTimeout,
+			long minPauseBetweenCheckpoints,
+			int maxConcurrentCheckpoints,
+			ExternalizedCheckpointSettings externalizedCheckpointSettings,
+			@Nullable StateBackend defaultStateBackend,
+			@Nullable MasterTriggerRestoreHook.Factory[] masterHooks,
+			boolean isExactlyOnce) {
+
 		// sanity checks
 		if (checkpointInterval < 1 || checkpointTimeout < 1 ||
 				minPauseBetweenCheckpoints < 0 || maxConcurrentCheckpoints < 1) {
 			throw new IllegalArgumentException();
 		}
-		
+
 		this.verticesToTrigger = requireNonNull(verticesToTrigger);
 		this.verticesToAcknowledge = requireNonNull(verticesToAcknowledge);
 		this.verticesToConfirm = requireNonNull(verticesToConfirm);
@@ -93,14 +115,16 @@ public JobCheckpointingSettings(
 		this.externalizedCheckpointSettings = requireNonNull(externalizedCheckpointSettings);
 		this.defaultStateBackend = defaultStateBackend;
 		this.isExactlyOnce = isExactlyOnce;
+
+		this.masterHooks = masterHooks != null ? masterHooks : new MasterTriggerRestoreHook.Factory[0];
 	}
-	
+
 	// --------------------------------------------------------------------------------------------
 
 	public List<JobVertexID> getVerticesToTrigger() {
 		return verticesToTrigger;
 	}
-	
+
 	public List<JobVertexID> getVerticesToAcknowledge() {
 		return verticesToAcknowledge;
 	}
@@ -134,12 +158,16 @@ public StateBackend getDefaultStateBackend() {
 		return defaultStateBackend;
 	}
 
+	public MasterTriggerRestoreHook.Factory[] getMasterHooks() {
+		return masterHooks;
+	}
+
 	public boolean isExactlyOnce() {
 		return isExactlyOnce;
 	}
 
 	// --------------------------------------------------------------------------------------------
-	
+
 	@Override
 	public String toString() {
 		return String.format("SnapshotSettings: interval=%d, timeout=%d, pause-between=%d, " +
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
new file mode 100644
index 0000000000000..0ec46065b3606
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
+import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTest.mockExecutionVertex;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isNull;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for the user-defined hooks that the checkpoint coordinator can call.
+ */
+public class CheckpointCoordinatorMasterHooksTest {
+
+	// ------------------------------------------------------------------------
+	//  hook registration
+	// ------------------------------------------------------------------------
+
+	/**
+	 * This method tests that hooks with the same identifier are not registered
+	 * multiple times.
+	 */
+	@Test
+	public void testDeduplicateOnRegister() {
+		final CheckpointCoordinator cc = instantiateCheckpointCoordinator(new JobID());
+
+		MasterTriggerRestoreHook<?> hook1 = mock(MasterTriggerRestoreHook.class);
+		when(hook1.getIdentifier()).thenReturn("test id");
+
+		MasterTriggerRestoreHook<?> hook2 = mock(MasterTriggerRestoreHook.class);
+		when(hook2.getIdentifier()).thenReturn("test id");
+
+		MasterTriggerRestoreHook<?> hook3 = mock(MasterTriggerRestoreHook.class);
+		when(hook3.getIdentifier()).thenReturn("anotherId");
+
+		assertTrue(cc.addMasterHook(hook1));
+		assertFalse(cc.addMasterHook(hook2));
+		assertTrue(cc.addMasterHook(hook3));
+	}
+
+	/**
+	 * Test that validates correct exceptions when supplying hooks with invalid IDs.
+	 */
+	@Test
+	public void testNullOrInvalidId() {
+		final CheckpointCoordinator cc = instantiateCheckpointCoordinator(new JobID());
+
+		try {
+			cc.addMasterHook(null);
+			fail("expected an exception");
+		} catch (NullPointerException ignored) {}
+
+		try {
+			cc.addMasterHook(mock(MasterTriggerRestoreHook.class));
+			fail("expected an exception");
+		} catch (IllegalArgumentException ignored) {}
+
+		try {
+			MasterTriggerRestoreHook<?> hook = mock(MasterTriggerRestoreHook.class);
+			when(hook.getIdentifier()).thenReturn("        ");
+
+			cc.addMasterHook(hook);
+			fail("expected an exception");
+		} catch (IllegalArgumentException ignored) {}
+	}
+
+	// ------------------------------------------------------------------------
+	//  trigger / restore behavior
+	// ------------------------------------------------------------------------
+
+	@Test
+	public void testHooksAreCalledOnTrigger() throws Exception {
+		final String id1 = "id1";
+		final String id2 = "id2";
+
+		final String state1 = "the-test-string-state";
+		final byte[] state1serialized = new StringSerializer().serialize(state1);
+
+		final long state2 = 987654321L;
+		final byte[] state2serialized = new LongSerializer().serialize(state2);
+
+		final MasterTriggerRestoreHook<String> statefulHook1 = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statefulHook1.getIdentifier()).thenReturn(id1);
+		when(statefulHook1.createCheckpointDataSerializer()).thenReturn(new StringSerializer());
+		when(statefulHook1.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class)))
+				.thenReturn(FlinkCompletableFuture.completed(state1));
+
+		final MasterTriggerRestoreHook<Long> statefulHook2 = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statefulHook2.getIdentifier()).thenReturn(id2);
+		when(statefulHook2.createCheckpointDataSerializer()).thenReturn(new LongSerializer());
+		when(statefulHook2.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class)))
+				.thenReturn(FlinkCompletableFuture.completed(state2));
+
+		final MasterTriggerRestoreHook<Void> statelessHook = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statelessHook.getIdentifier()).thenReturn("some-id");
+
+		// create the checkpoint coordinator
+		final JobID jid = new JobID();
+		final ExecutionAttemptID execId = new ExecutionAttemptID();
+		final ExecutionVertex ackVertex = mockExecutionVertex(execId);
+		final CheckpointCoordinator cc = instantiateCheckpointCoordinator(jid, ackVertex);
+
+		cc.addMasterHook(statefulHook1);
+		cc.addMasterHook(statelessHook);
+		cc.addMasterHook(statefulHook2);
+
+		// trigger a checkpoint
+		assertTrue(cc.triggerCheckpoint(System.currentTimeMillis(), false));
+		assertEquals(1, cc.getNumberOfPendingCheckpoints());
+
+		verify(statefulHook1, times(1)).triggerCheckpoint(anyLong(), anyLong(), any(Executor.class));
+		verify(statefulHook2, times(1)).triggerCheckpoint(anyLong(), anyLong(), any(Executor.class));
+		verify(statelessHook, times(1)).triggerCheckpoint(anyLong(), anyLong(), any(Executor.class));
+
+		final long checkpointId = cc.getPendingCheckpoints().values().iterator().next().getCheckpointId();
+		cc.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, execId, checkpointId));
+		assertEquals(0, cc.getNumberOfPendingCheckpoints());
+
+		assertEquals(1, cc.getNumberOfRetainedSuccessfulCheckpoints());
+		final CompletedCheckpoint chk = cc.getCheckpointStore().getLatestCheckpoint();
+
+		final Collection<MasterState> masterStates = chk.getMasterHookStates();
+		assertEquals(2, masterStates.size());
+
+		for (MasterState ms : masterStates) {
+			if (ms.name().equals(id1)) {
+				assertArrayEquals(state1serialized, ms.bytes());
+				assertEquals(StringSerializer.VERSION, ms.version());
+			}
+			else if (ms.name().equals(id2)) {
+				assertArrayEquals(state2serialized, ms.bytes());
+				assertEquals(LongSerializer.VERSION, ms.version());
+			}
+			else {
+				fail("unrecognized state name: " + ms.name());
+			}
+		}
+	}
+
+	@Test
+	public void testHooksAreCalledOnRestore() throws Exception {
+		final String id1 = "id1";
+		final String id2 = "id2";
+
+		final String state1 = "the-test-string-state";
+		final byte[] state1serialized = new StringSerializer().serialize(state1);
+
+		final long state2 = 987654321L;
+		final byte[] state2serialized = new LongSerializer().serialize(state2);
+
+		final List<MasterState> masterHookStates = Arrays.asList(
+				new MasterState(id1, state1serialized, StringSerializer.VERSION),
+				new MasterState(id2, state2serialized, LongSerializer.VERSION));
+
+		final MasterTriggerRestoreHook<String> statefulHook1 = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statefulHook1.getIdentifier()).thenReturn(id1);
+		when(statefulHook1.createCheckpointDataSerializer()).thenReturn(new StringSerializer());
+		when(statefulHook1.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class)))
+				.thenThrow(new Exception("not expected"));
+
+		final MasterTriggerRestoreHook<Long> statefulHook2 = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statefulHook2.getIdentifier()).thenReturn(id2);
+		when(statefulHook2.createCheckpointDataSerializer()).thenReturn(new LongSerializer());
+		when(statefulHook2.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class)))
+				.thenThrow(new Exception("not expected"));
+
+		final MasterTriggerRestoreHook<Void> statelessHook = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statelessHook.getIdentifier()).thenReturn("some-id");
+
+		final JobID jid = new JobID();
+		final long checkpointId = 13L;
+
+		final CompletedCheckpoint checkpoint = new CompletedCheckpoint(
+				jid, checkpointId, 123L, 125L,
+				Collections.<JobVertexID, TaskState>emptyMap(),
+				masterHookStates,
+				CheckpointProperties.forStandardCheckpoint(),
+				null,
+				null);
+
+		final ExecutionAttemptID execId = new ExecutionAttemptID();
+		final ExecutionVertex ackVertex = mockExecutionVertex(execId);
+		final CheckpointCoordinator cc = instantiateCheckpointCoordinator(jid, ackVertex);
+
+		cc.addMasterHook(statefulHook1);
+		cc.addMasterHook(statelessHook);
+		cc.addMasterHook(statefulHook2);
+
+		cc.getCheckpointStore().addCheckpoint(checkpoint);
+		cc.restoreLatestCheckpointedState(
+				Collections.<JobVertexID, ExecutionJobVertex>emptyMap(),
+				true,
+				false);
+
+		verify(statefulHook1, times(1)).restoreCheckpoint(eq(checkpointId), eq(state1));
+		verify(statefulHook2, times(1)).restoreCheckpoint(eq(checkpointId), eq(state2));
+		verify(statelessHook, times(1)).restoreCheckpoint(eq(checkpointId), isNull(Void.class));
+	}
+
+	@Test
+	public void checkUnMatchedStateOnRestore() throws Exception {
+		final String id1 = "id1";
+		final String id2 = "id2";
+
+		final String state1 = "the-test-string-state";
+		final byte[] state1serialized = new StringSerializer().serialize(state1);
+
+		final long state2 = 987654321L;
+		final byte[] state2serialized = new LongSerializer().serialize(state2);
+
+		final List<MasterState> masterHookStates = Arrays.asList(
+				new MasterState(id1, state1serialized, StringSerializer.VERSION),
+				new MasterState(id2, state2serialized, LongSerializer.VERSION));
+
+		final MasterTriggerRestoreHook<String> statefulHook = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statefulHook.getIdentifier()).thenReturn(id1);
+		when(statefulHook.createCheckpointDataSerializer()).thenReturn(new StringSerializer());
+		when(statefulHook.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class)))
+				.thenThrow(new Exception("not expected"));
+
+		final MasterTriggerRestoreHook<Void> statelessHook = mockGeneric(MasterTriggerRestoreHook.class);
+		when(statelessHook.getIdentifier()).thenReturn("some-id");
+
+		final JobID jid = new JobID();
+		final long checkpointId = 44L;
+
+		final CompletedCheckpoint checkpoint = new CompletedCheckpoint(
+				jid, checkpointId, 123L, 125L,
+				Collections.<JobVertexID, TaskState>emptyMap(),
+				masterHookStates,
+				CheckpointProperties.forStandardCheckpoint(),
+				null,
+				null);
+
+		final ExecutionAttemptID execId = new ExecutionAttemptID();
+		final ExecutionVertex ackVertex = mockExecutionVertex(execId);
+		final CheckpointCoordinator cc = instantiateCheckpointCoordinator(jid, ackVertex);
+
+		cc.addMasterHook(statefulHook);
+		cc.addMasterHook(statelessHook);
+
+		cc.getCheckpointStore().addCheckpoint(checkpoint);
+
+		// since we have unmatched state, this should fail
+		try {
+			cc.restoreLatestCheckpointedState(
+					Collections.<JobVertexID, ExecutionJobVertex>emptyMap(),
+					true,
+					false);
+			fail("exception expected");
+		}
+		catch (IllegalStateException ignored) {}
+
+		// permitting unmatched state should succeed
+		cc.restoreLatestCheckpointedState(
+				Collections.<JobVertexID, ExecutionJobVertex>emptyMap(),
+				true,
+				true);
+
+		verify(statefulHook, times(1)).restoreCheckpoint(eq(checkpointId), eq(state1));
+		verify(statelessHook, times(1)).restoreCheckpoint(eq(checkpointId), isNull(Void.class));
+	}
+
+	// ------------------------------------------------------------------------
+	//  failure scenarios
+	// ------------------------------------------------------------------------
+
+	@Test
+	public void testSerializationFailsOnTrigger() {
+	}
+
+	@Test
+	public void testHookCallFailsOnTrigger() {
+	}
+
+	@Test
+	public void testDeserializationFailsOnRestore() {
+	}
+
+	@Test
+	public void testHookCallFailsOnRestore() {
+	}
+
+	@Test
+	public void testTypeIncompatibleWithSerializerOnStore() {
+	}
+
+	@Test
+	public void testTypeIncompatibleWithHookOnRestore() {
+	}
+
+	// ------------------------------------------------------------------------
+	//  utilities
+	// ------------------------------------------------------------------------
+
+	private static CheckpointCoordinator instantiateCheckpointCoordinator(JobID jid, ExecutionVertex... ackVertices) {
+		return new CheckpointCoordinator(
+				jid,
+				10000000L,
+				600000L,
+				0L,
+				1,
+				ExternalizedCheckpointSettings.none(),
+				new ExecutionVertex[0],
+				ackVertices,
+				new ExecutionVertex[0],
+				new StandaloneCheckpointIDCounter(),
+				new StandaloneCompletedCheckpointStore(10),
+				null,
+				Executors.directExecutor());
+	}
+
+	private static <T> T mockGeneric(Class<?> clazz) {
+		@SuppressWarnings("unchecked")
+		Class<T> typedClass = (Class<T>) clazz;
+		return mock(typedClass);
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static final class StringSerializer implements SimpleVersionedSerializer<String> {
+
+		static final int VERSION = 77;
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+
+		@Override
+		public byte[] serialize(String checkpointData) throws IOException {
+			return checkpointData.getBytes(StandardCharsets.UTF_8);
+		}
+
+		@Override
+		public String deserialize(int version, byte[] serialized) throws IOException {
+			if (version != VERSION) {
+				throw new IOException("version mismatch");
+			}
+			return new String(serialized, StandardCharsets.UTF_8);
+		}
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static final class LongSerializer implements SimpleVersionedSerializer<Long> {
+
+		static final int VERSION = 5;
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+
+		@Override
+		public byte[] serialize(Long checkpointData) throws IOException {
+			final byte[] bytes = new byte[8];
+			ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).putLong(0, checkpointData);
+			return bytes;
+		}
+
+		@Override
+		public Long deserialize(int version, byte[] serialized) throws IOException {
+			assertEquals(VERSION, version);
+			assertEquals(8, serialized.length);
+
+			return ByteBuffer.wrap(serialized).order(ByteOrder.LITTLE_ENDIAN).getLong(0);
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
index 4a36dd282c0a6..fc6e51664306e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
@@ -273,7 +273,7 @@ public TestCompletedCheckpoint(
 			Map<JobVertexID, TaskState> taskGroupStates,
 			CheckpointProperties props) {
 
-			super(jobId, checkpointId, timestamp, Long.MAX_VALUE, taskGroupStates, props);
+			super(jobId, checkpointId, timestamp, Long.MAX_VALUE, taskGroupStates, null, props);
 		}
 
 		@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
index 0b759d4056273..652cc767af366 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
@@ -45,7 +45,7 @@
 public class CompletedCheckpointTest {
 
 	@Rule
-	public TemporaryFolder tmpFolder = new TemporaryFolder();
+	public final TemporaryFolder tmpFolder = new TemporaryFolder();
 
 	/**
 	 * Tests that persistent checkpoints discard their header file.
@@ -61,7 +61,10 @@ public void testDiscard() throws Exception {
 
 		// Verify discard call is forwarded to state
 		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
-				new JobID(), 0, 0, 1, taskStates, CheckpointProperties.forStandardCheckpoint(),
+				new JobID(), 0, 0, 1,
+				taskStates,
+				Collections.<MasterState>emptyList(),
+				CheckpointProperties.forStandardCheckpoint(),
 				new FileStateHandle(new Path(file.toURI()), file.length()),
 				file.getAbsolutePath());
 
@@ -81,8 +84,12 @@ public void testCleanUpOnSubsume() throws Exception {
 
 		boolean discardSubsumed = true;
 		CheckpointProperties props = new CheckpointProperties(false, false, discardSubsumed, true, true, true, true);
+		
 		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
-				new JobID(), 0, 0, 1, taskStates, props);
+				new JobID(), 0, 0, 1,
+				taskStates,
+				Collections.<MasterState>emptyList(),
+				props);
 
 		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		checkpoint.registerSharedStates(sharedStateRegistry);
@@ -117,7 +124,10 @@ public void testCleanUpOnShutdown() throws Exception {
 			// Keep
 			CheckpointProperties props = new CheckpointProperties(false, true, false, false, false, false, false);
 			CompletedCheckpoint checkpoint = new CompletedCheckpoint(
-					new JobID(), 0, 0, 1, new HashMap<>(taskStates), props,
+					new JobID(), 0, 0, 1,
+					new HashMap<>(taskStates),
+					Collections.<MasterState>emptyList(),
+					props,
 					new FileStateHandle(new Path(file.toURI()), file.length()),
 					externalPath);
 
@@ -132,7 +142,10 @@ public void testCleanUpOnShutdown() throws Exception {
 			// Discard
 			props = new CheckpointProperties(false, false, true, true, true, true, true);
 			checkpoint = new CompletedCheckpoint(
-					new JobID(), 0, 0, 1, new HashMap<>(taskStates), props);
+					new JobID(), 0, 0, 1,
+					new HashMap<>(taskStates),
+					Collections.<MasterState>emptyList(),
+					props);
 
 			checkpoint.discardOnShutdown(status, sharedStateRegistry);
 			verify(state, times(1)).discardState();
@@ -155,6 +168,7 @@ public void testCompletedCheckpointStatsCallbacks() throws Exception {
 			0,
 			1,
 			new HashMap<>(taskStates),
+			Collections.<MasterState>emptyList(),
 			CheckpointProperties.forStandardCheckpoint());
 
 		CompletedCheckpointStats.DiscardCallback callback = mock(CompletedCheckpointStats.DiscardCallback.class);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
index 5fce62ebc2528..1f038bd274743 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
@@ -105,6 +105,7 @@ private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
 				Collections.<ExecutionJobVertex>emptyList(),
 				Collections.<ExecutionJobVertex>emptyList(),
 				Collections.<ExecutionJobVertex>emptyList(),
+				Collections.<MasterTriggerRestoreHook<?>>emptyList(),
 				counter,
 				store,
 				null,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
similarity index 69%
rename from flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java
rename to flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
index 08ec35e31994c..7d9874e96e2c2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.checkpoint.savepoint;
 
 import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.SubtaskState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -26,50 +27,41 @@
 import org.apache.flink.runtime.state.KeyGroupRangeOffsets;
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle.StateMetaInfo;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.util.TestByteStreamStateHandleDeepCompare;
-import org.junit.Test;
+import org.apache.flink.util.StringUtils;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
-public class SavepointV1Test {
+/**
+ * A collection of utility methods for testing the (de)serialization of
+ * checkpoint metadata for persistence.
+ */
+public class CheckpointTestUtils {
 
 	/**
-	 * Simple test of savepoint methods.
+	 * Creates a random collection of TaskState objects containing various types of state handles.
 	 */
-	@Test
-	public void testSavepointV1() throws Exception {
-		long checkpointId = ThreadLocalRandom.current().nextLong(Integer.MAX_VALUE);
-		int numTaskStates = 4;
-		int numSubtaskStates = 16;
-
-		Collection<TaskState> expected = createTaskStates(numTaskStates, numSubtaskStates);
-
-		SavepointV1 savepoint = new SavepointV1(checkpointId, expected);
-
-		assertEquals(SavepointV1.VERSION, savepoint.getVersion());
-		assertEquals(checkpointId, savepoint.getCheckpointId());
-		assertEquals(expected, savepoint.getTaskStates());
-
-		assertFalse(savepoint.getTaskStates().isEmpty());
-		savepoint.dispose();
-		assertTrue(savepoint.getTaskStates().isEmpty());
+	public static Collection<TaskState> createTaskStates(int numTaskStates, int numSubtasksPerTask) {
+		return createTaskStates(new Random(), numTaskStates, numSubtasksPerTask);
 	}
 
-	static Collection<TaskState> createTaskStates(int numTaskStates, int numSubtasksPerTask) throws IOException {
-
-		Random random = new Random(numTaskStates * 31 + numSubtasksPerTask);
+	/**
+	 * Creates a random collection of TaskState objects containing various types of state handles.
+	 */
+	public static Collection<TaskState> createTaskStates(
+			Random random,
+			int numTaskStates,
+			int numSubtasksPerTask) {
 
 		List<TaskState> taskStates = new ArrayList<>(numTaskStates);
 
@@ -96,12 +88,12 @@ static Collection<TaskState> createTaskStates(int numTaskStates, int numSubtasks
 
 					StreamStateHandle nonPartitionableState =
 							new TestByteStreamStateHandleDeepCompare("a-" + chainIdx, ("Hi-" + chainIdx).getBytes(
-								ConfigConstants.DEFAULT_CHARSET));
+									ConfigConstants.DEFAULT_CHARSET));
 					StreamStateHandle operatorStateBackend =
 							new TestByteStreamStateHandleDeepCompare("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
 					StreamStateHandle operatorStateStream =
 							new TestByteStreamStateHandleDeepCompare("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
-					Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
+					Map<String, StateMetaInfo> offsetsMap = new HashMap<>();
 					offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
 					offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
 					offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.BROADCAST));
@@ -130,14 +122,14 @@ static Collection<TaskState> createTaskStates(int numTaskStates, int numSubtasks
 					keyedStateBackend = new KeyGroupsStateHandle(
 							new KeyGroupRangeOffsets(1, 1, new long[]{42}),
 							new TestByteStreamStateHandleDeepCompare("c", "Hello"
-								.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
 				}
 
 				if (hasKeyedStream) {
 					keyedStateStream = new KeyGroupsStateHandle(
 							new KeyGroupRangeOffsets(1, 1, new long[]{23}),
 							new TestByteStreamStateHandleDeepCompare("d", "World"
-								.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
 				}
 
 				taskState.putState(subtaskIdx, new SubtaskState(
@@ -154,4 +146,39 @@ static Collection<TaskState> createTaskStates(int numTaskStates, int numSubtasks
 		return taskStates;
 	}
 
+	/**
+	 * Creates a bunch of random master states.
+	 */
+	public static Collection<MasterState> createRandomMasterStates(Random random, int num) {
+		final ArrayList<MasterState> states = new ArrayList<>(num);
+
+		for (int i = 0; i < num; i++) {
+			int version = random.nextInt(10);
+			String name = StringUtils.getRandomString(random, 5, 500);
+			byte[] bytes = new byte[random.nextInt(5000) + 1];
+			random.nextBytes(bytes);
+
+			states.add(new MasterState(name, bytes, version));
+		}
+
+		return states;
+	}
+
+	/**
+	 * Asserts that two MasterStates are equal.
+	 * 
+	 * <p>The MasterState avoids overriding {@code equals()} on purpose, because equality is not well
+	 * defined in the raw contents.
+	 */
+	public static void assertMasterStateEquality(MasterState a, MasterState b) {
+		assertEquals(a.version(), b.version());
+		assertEquals(a.name(), b.name());
+		assertArrayEquals(a.bytes(), b.bytes());
+
+	}
+
+	// ------------------------------------------------------------------------
+
+	/** utility class, not meant to be instantiated */
+	private CheckpointTestUtils() {}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java
index c66b29d9a82dd..20b1e5713bfa4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java
@@ -67,7 +67,7 @@ public void testLoadAndValidateSavepoint() throws Exception {
 		JobID jobId = new JobID();
 
 		// Store savepoint
-		SavepointV1 savepoint = new SavepointV1(checkpointId, taskStates.values());
+		SavepointV2 savepoint = new SavepointV2(checkpointId, taskStates.values());
 		String path = SavepointStore.storeSavepoint(tmp.getAbsolutePath(), savepoint);
 
 		ExecutionJobVertex vertex = mock(ExecutionJobVertex.class);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java
index 1eb805599967c..cf79282797479 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java
@@ -23,6 +23,7 @@
 import org.apache.flink.core.fs.FSDataOutputStream;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
 import org.junit.Rule;
@@ -68,7 +69,7 @@ public void testStoreLoadDispose() throws Exception {
 
 		// Store
 		String savepointDirectory = SavepointStore.createSavepointDirectory(root, new JobID());
-		SavepointV1 stored = new SavepointV1(1929292, SavepointV1Test.createTaskStates(4, 24));
+		SavepointV2 stored = new SavepointV2(1929292, CheckpointTestUtils.createTaskStates(4, 24));
 		String path = SavepointStore.storeSavepoint(savepointDirectory, stored);
 
 		list = rootFile.listFiles();
@@ -77,7 +78,10 @@ public void testStoreLoadDispose() throws Exception {
 
 		// Load
 		Savepoint loaded = SavepointStore.loadSavepoint(path, Thread.currentThread().getContextClassLoader());
-		assertEquals(stored, loaded);
+
+		assertEquals(stored.getCheckpointId(), loaded.getCheckpointId());
+		assertEquals(stored.getTaskStates(), loaded.getTaskStates());
+		assertEquals(stored.getMasterStates(), loaded.getMasterStates());
 
 		loaded.dispose();
 
@@ -126,8 +130,8 @@ public void testMultipleSavepointVersions() throws Exception {
 		File rootFile = new File(root);
 
 		// New savepoint type for test
-		int version = ThreadLocalRandom.current().nextInt();
-		long checkpointId = ThreadLocalRandom.current().nextLong();
+		int version = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE); // make this a positive number
+		long checkpointId = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); // make this a positive number
 
 		// Add serializer
 		serializers.put(version, NewSavepointSerializer.INSTANCE);
@@ -143,7 +147,7 @@ public void testMultipleSavepointVersions() throws Exception {
 
 		// Savepoint v0
 		String savepointDirectory2 = SavepointStore.createSavepointDirectory(root, new JobID());
-		Savepoint savepoint = new SavepointV1(checkpointId, SavepointV1Test.createTaskStates(4, 32));
+		SavepointV2 savepoint = new SavepointV2(checkpointId, CheckpointTestUtils.createTaskStates(4, 32));
 		String pathSavepoint = SavepointStore.storeSavepoint(savepointDirectory2, savepoint);
 
 		list = rootFile.listFiles();
@@ -156,7 +160,9 @@ public void testMultipleSavepointVersions() throws Exception {
 		assertEquals(newSavepoint, loaded);
 
 		loaded = SavepointStore.loadSavepoint(pathSavepoint, Thread.currentThread().getContextClassLoader());
-		assertEquals(savepoint, loaded);
+		assertEquals(savepoint.getCheckpointId(), loaded.getCheckpointId());
+		assertEquals(savepoint.getTaskStates(), loaded.getTaskStates());
+		assertEquals(savepoint.getMasterStates(), loaded.getMasterStates());
 	}
 
 	/**
@@ -199,7 +205,7 @@ public void testStoreExternalizedCheckpointsToSameDirectory() throws Exception {
 		FileSystem fs = FileSystem.get(new Path(root).toUri());
 
 		// Store
-		SavepointV1 savepoint = new SavepointV1(1929292, SavepointV1Test.createTaskStates(4, 24));
+		SavepointV2 savepoint = new SavepointV2(1929292, CheckpointTestUtils.createTaskStates(4, 24));
 
 		FileStateHandle store1 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
 		fs.exists(store1.getFilePath());
@@ -251,7 +257,12 @@ public long getCheckpointId() {
 
 		@Override
 		public Collection<TaskState> getTaskStates() {
-			return Collections.EMPTY_LIST;
+			return Collections.emptyList();
+		}
+
+		@Override
+		public Collection<MasterState> getMasterStates() {
+			return Collections.emptyList();
 		}
 
 		@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1SerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1SerializerTest.java
index 58cf1aa9b79e0..0eff7bc398f8e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1SerializerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1SerializerTest.java
@@ -27,6 +27,7 @@
 import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class SavepointV1SerializerTest {
 
@@ -35,25 +36,29 @@ public class SavepointV1SerializerTest {
 	 */
 	@Test
 	public void testSerializeDeserializeV1() throws Exception {
-		Random r = new Random(42);
-		for (int i = 0; i < 100; ++i) {
+		final Random r = new Random(42);
+
+		for (int i = 0; i < 50; ++i) {
 			SavepointV1 expected =
-					new SavepointV1(i+ 123123, SavepointV1Test.createTaskStates(1 + r.nextInt(64), 1 + r.nextInt(64)));
+					new SavepointV1(i+ 123123, CheckpointTestUtils.createTaskStates(r, 1 + r.nextInt(64), 1 + r.nextInt(64)));
 
 			SavepointV1Serializer serializer = SavepointV1Serializer.INSTANCE;
 
 			// Serialize
 			ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
-			serializer.serialize(expected, new DataOutputViewStreamWrapper(baos));
+			serializer.serializeOld(expected, new DataOutputViewStreamWrapper(baos));
 			byte[] bytes = baos.toByteArray();
 
 			// Deserialize
 			ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
-			Savepoint actual = serializer.deserialize(
+			SavepointV2 actual = serializer.deserialize(
 					new DataInputViewStreamWrapper(bais),
 					Thread.currentThread().getContextClassLoader());
 
-			assertEquals(expected, actual);
+
+			assertEquals(expected.getCheckpointId(), actual.getCheckpointId());
+			assertEquals(expected.getTaskStates(), actual.getTaskStates());
+			assertTrue(actual.getMasterStates().isEmpty());
 		}
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
new file mode 100644
index 0000000000000..deb14ddc85a8c
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint.savepoint;
+
+import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.TaskState;
+
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Various tests for the version 2 format serializer of a checkpoint. 
+ */
+public class SavepointV2SerializerTest {
+
+	@Test
+	public void testCheckpointWithNoState() throws Exception {
+		final Random rnd = new Random();
+
+		for (int i = 0; i < 100; ++i) {
+			final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
+			final Collection<TaskState> taskStates = Collections.emptyList();
+			final Collection<MasterState> masterStates = Collections.emptyList();
+
+			testCheckpointSerialization(checkpointId, taskStates, masterStates);
+		}
+	}
+
+	@Test
+	public void testCheckpointWithOnlyMasterState() throws Exception {
+		final Random rnd = new Random();
+		final int maxNumMasterStates = 5;
+
+		for (int i = 0; i < 100; ++i) {
+			final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
+
+			final Collection<TaskState> taskStates = Collections.emptyList();
+
+			final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
+			final Collection<MasterState> masterStates = 
+					CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
+
+			testCheckpointSerialization(checkpointId, taskStates, masterStates);
+		}
+	}
+
+	@Test
+	public void testCheckpointWithOnlyTaskState() throws Exception {
+		final Random rnd = new Random();
+		final int maxTaskStates = 20;
+		final int maxNumSubtasks = 20;
+
+		for (int i = 0; i < 100; ++i) {
+			final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
+
+			final int numTasks = rnd.nextInt(maxTaskStates) + 1;
+			final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
+			final Collection<TaskState> taskStates = 
+					CheckpointTestUtils.createTaskStates(rnd, numTasks, numSubtasks);
+
+			final Collection<MasterState> masterStates = Collections.emptyList();
+
+			testCheckpointSerialization(checkpointId, taskStates, masterStates);
+		}
+	}
+
+	@Test
+	public void testCheckpointWithMasterAndTaskState() throws Exception {
+		final Random rnd = new Random();
+
+		final int maxNumMasterStates = 5;
+		final int maxTaskStates = 20;
+		final int maxNumSubtasks = 20;
+
+		for (int i = 0; i < 100; ++i) {
+			final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
+
+			final int numTasks = rnd.nextInt(maxTaskStates) + 1;
+			final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
+			final Collection<TaskState> taskStates =
+					CheckpointTestUtils.createTaskStates(rnd, numTasks, numSubtasks);
+
+			final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
+			final Collection<MasterState> masterStates =
+					CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
+
+			testCheckpointSerialization(checkpointId, taskStates, masterStates);
+		}
+	}
+
+	private void testCheckpointSerialization(
+			long checkpointId,
+			Collection<TaskState> taskStates,
+			Collection<MasterState> masterStates) throws IOException {
+
+		SavepointV2Serializer serializer = SavepointV2Serializer.INSTANCE;
+
+		ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
+		DataOutputStream out = new DataOutputViewStreamWrapper(baos);
+
+		serializer.serialize(new SavepointV2(checkpointId, taskStates, masterStates), out);
+		out.close();
+
+		byte[] bytes = baos.toByteArray();
+
+		DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes));
+		SavepointV2 deserialized = serializer.deserialize(in, getClass().getClassLoader());
+
+		assertEquals(checkpointId, deserialized.getCheckpointId());
+		assertEquals(taskStates, deserialized.getTaskStates());
+
+		assertEquals(masterStates.size(), deserialized.getMasterStates().size());
+		for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator();
+				a.hasNext();)
+		{
+			CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next());
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java
new file mode 100644
index 0000000000000..428a62a675c7c
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint.savepoint;
+
+import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.TaskState;
+
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class SavepointV2Test {
+
+	/**
+	 * Simple test of savepoint methods.
+	 */
+	@Test
+	public void testSavepointV1() throws Exception {
+		final Random rnd = new Random();
+
+		final long checkpointId = rnd.nextInt(Integer.MAX_VALUE) + 1;
+		final int numTaskStates = 4;
+		final int numSubtaskStates = 16;
+		final int numMasterStates = 7;
+
+		Collection<TaskState> taskStates =
+				CheckpointTestUtils.createTaskStates(rnd, numTaskStates, numSubtaskStates);
+
+		Collection<MasterState> masterStates =
+				CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
+
+		SavepointV2 checkpoint = new SavepointV2(checkpointId, taskStates, masterStates);
+
+		assertEquals(2, checkpoint.getVersion());
+		assertEquals(checkpointId, checkpoint.getCheckpointId());
+		assertEquals(taskStates, checkpoint.getTaskStates());
+		assertEquals(masterStates, checkpoint.getMasterStates());
+
+		assertFalse(checkpoint.getTaskStates().isEmpty());
+		assertFalse(checkpoint.getMasterStates().isEmpty());
+
+		checkpoint.dispose();
+
+		assertTrue(checkpoint.getTaskStates().isEmpty());
+		assertTrue(checkpoint.getMasterStates().isEmpty());
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
index f96b62456dcf8..4e1d0f7175da7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
@@ -31,6 +31,7 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
 import org.apache.flink.runtime.checkpoint.CheckpointStatsTracker;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.checkpoint.StandaloneCheckpointIDCounter;
 import org.apache.flink.runtime.checkpoint.StandaloneCompletedCheckpointStore;
 import org.apache.flink.runtime.execution.ExecutionState;
@@ -128,6 +129,7 @@ public static void setupExecutionGraph() throws Exception {
 			Collections.<ExecutionJobVertex>emptyList(),
 			Collections.<ExecutionJobVertex>emptyList(),
 			Collections.<ExecutionJobVertex>emptyList(),
+			Collections.<MasterTriggerRestoreHook<?>>emptyList(),
 			new StandaloneCheckpointIDCounter(),
 			new StandaloneCompletedCheckpointStore(1),
 			null,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ExternallyInducedSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ExternallyInducedSource.java
new file mode 100644
index 0000000000000..b26cf4f1505e8
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/ExternallyInducedSource.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.checkpoint;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.util.FlinkException;
+
+/**
+ * Sources that implement this interface do not trigger checkpoints when receiving a
+ * trigger message from the checkpoint coordinator, but when their input data/events
+ * indicate that a checkpoint should be triggered.
+ *
+ * <p>Since sources cannot simply create a new checkpoint on their own, this mechanism
+ * always goes together with a {@link WithMasterCheckpointHook hook on the master side}.
+ * In a typical setup, the hook on the master tells the source system (for example
+ * the message queue) to prepare a checkpoint. The exact point when the checkpoint is
+ * taken is then controlled by the event stream received from the source, and triggered
+ * by the source function (implementing this interface) in Flink when seeing the relevant
+ * events.
+ *
+ * @param <T> Type of the elements produced by the source function
+ * @param <CD> The type of the data stored in the checkpoint by the master that triggers
+ */
+@PublicEvolving
+public interface ExternallyInducedSource<T, CD> extends SourceFunction<T>, WithMasterCheckpointHook<CD> {
+
+	/**
+	 * Sets the checkpoint trigger through which the source can trigger the checkpoint.
+	 *
+	 * @param checkpointTrigger The checkpoint trigger to set
+	 */
+	void setCheckpointTrigger(CheckpointTrigger checkpointTrigger);
+
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Through the {@code CheckpointTrigger}, the source function notifies the Flink
+	 * source operator when to trigger the checkpoint.
+	 */
+	interface CheckpointTrigger {
+
+		/**
+		 * Triggers a checkpoint. This method should be called by the source
+		 * when it sees the event that indicates that a checkpoint should be triggered.
+		 *
+		 * <p>When this method is called, the parallel operator instance in which the
+		 * calling source function runs will perform its checkpoint and insert the
+		 * checkpoint barrier into the data stream.
+		 *
+		 * @param checkpointId The ID that identifies the checkpoint.
+		 *
+		 * @throws FlinkException Thrown when the checkpoint could not be triggered, for example
+		 *                        because of an invalid state or errors when storing the
+		 *                        checkpoint state.
+		 */
+		void triggerCheckpoint(long checkpointId) throws FlinkException;
+	}
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/WithMasterCheckpointHook.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/WithMasterCheckpointHook.java
new file mode 100644
index 0000000000000..ef872de0e34d6
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/WithMasterCheckpointHook.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.checkpoint;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
+
+/**
+ * This interface can be implemented by streaming functions that need to trigger a
+ * "global action" on the master (in the checkpoint coordinator) as part of every
+ * checkpoint and restore operation.
+ *
+ * @param <E> The type of the data stored by the hook in the checkpoint, or {@code Void}, if none.
+ */
+@PublicEvolving
+public interface WithMasterCheckpointHook<E> extends java.io.Serializable {
+
+	/**
+	 * Creates the hook that should be called by the checkpoint coordinator.
+	 */
+	MasterTriggerRestoreHook<E> createMasterTriggerRestoreHook();
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/FunctionMasterCheckpointHookFactory.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/FunctionMasterCheckpointHookFactory.java
new file mode 100644
index 0000000000000..c2566980052c2
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/FunctionMasterCheckpointHookFactory.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.graph;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
+import org.apache.flink.streaming.api.checkpoint.WithMasterCheckpointHook;
+
+/**
+ * Utility class that turns a {@link WithMasterCheckpointHook} into a
+ * {@link org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook.Factory}.
+ */
+class FunctionMasterCheckpointHookFactory implements MasterTriggerRestoreHook.Factory {
+
+	private static final long serialVersionUID = 2L;
+
+	private final WithMasterCheckpointHook<?> creator;
+
+	FunctionMasterCheckpointHookFactory(WithMasterCheckpointHook<?> creator) {
+		this.creator = checkNotNull(creator);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public <V> MasterTriggerRestoreHook<V> create() {
+		return (MasterTriggerRestoreHook<V>) creator.createMasterTriggerRestoreHook();
+	}
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index a1d33d88713ee..7f24cd358e353 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -29,6 +29,7 @@
 import org.apache.commons.lang3.StringUtils;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.operators.ResourceSpec;
 import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
 import org.apache.flink.api.common.restartstrategy.RestartStrategies;
@@ -36,6 +37,7 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.migration.streaming.api.graph.StreamGraphHasherV1;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
 import org.apache.flink.runtime.jobgraph.InputFormatVertex;
@@ -51,7 +53,9 @@
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
 import org.apache.flink.runtime.operators.util.TaskConfig;
 import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.checkpoint.WithMasterCheckpointHook;
 import org.apache.flink.streaming.api.environment.CheckpointConfig;
+import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.operators.StreamOperator;
 import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
@@ -542,6 +546,8 @@ private void configureCheckpointing() {
 			interval = Long.MAX_VALUE;
 		}
 
+		//  --- configure the participating vertices ---
+
 		// collect the vertices that receive "trigger checkpoint" messages.
 		// currently, these are all the sources
 		List<JobVertexID> triggerVertices = new ArrayList<>();
@@ -552,7 +558,7 @@ private void configureCheckpointing() {
 
 		// collect the vertices that receive "commit checkpoint" messages
 		// currently, these are all vertices
-		List<JobVertexID> commitVertices = new ArrayList<>();
+		List<JobVertexID> commitVertices = new ArrayList<>(jobVertices.size());
 
 		for (JobVertex vertex : jobVertices.values()) {
 			if (vertex.isInputVertex()) {
@@ -562,6 +568,8 @@ private void configureCheckpointing() {
 			ackVertices.add(vertex.getID());
 		}
 
+		//  --- configure options ---
+
 		ExternalizedCheckpointSettings externalizedCheckpointSettings;
 		if (cfg.isExternalizedCheckpointsEnabled()) {
 			CheckpointConfig.ExternalizedCheckpointCleanup cleanup = cfg.getExternalizedCheckpointCleanup();
@@ -587,12 +595,30 @@ private void configureCheckpointing() {
 				"exactly-once or at-least-once.");
 		}
 
+		//  --- configure the master-side checkpoint hooks ---
+
+		final ArrayList<MasterTriggerRestoreHook.Factory> hooks = new ArrayList<>();
+
+		for (StreamNode node : streamGraph.getStreamNodes()) {
+			StreamOperator<?> op = node.getOperator();
+			if (op instanceof AbstractUdfStreamOperator) {
+				Function f = ((AbstractUdfStreamOperator<?, ?>) op).getUserFunction();
+
+				if (f instanceof WithMasterCheckpointHook) {
+					hooks.add(new FunctionMasterCheckpointHookFactory((WithMasterCheckpointHook<?>) f));
+				}
+			}
+		}
+
+		//  --- done, put it all together ---
+
 		JobCheckpointingSettings settings = new JobCheckpointingSettings(
 				triggerVertices, ackVertices, commitVertices, interval,
 				cfg.getCheckpointTimeout(), cfg.getMinPauseBetweenCheckpoints(),
 				cfg.getMaxConcurrentCheckpoints(),
 				externalizedCheckpointSettings,
 				streamGraph.getStateBackend(),
+				hooks.toArray(new MasterTriggerRestoreHook.Factory[hooks.size()]),
 				isExactlyOnce);
 
 		jobGraph.setSnapshotSettings(settings);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
index 66e92dff70179..31cd7c18d24d5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java
@@ -19,8 +19,12 @@
 package org.apache.flink.streaming.runtime.tasks;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
+import org.apache.flink.runtime.checkpoint.CheckpointOptions;
+import org.apache.flink.streaming.api.checkpoint.ExternallyInducedSource;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.streaming.api.operators.StreamSource;
+import org.apache.flink.util.FlinkException;
 
 /**
  * {@link StreamTask} for executing a {@link StreamSource}.
@@ -40,9 +44,44 @@
 public class SourceStreamTask<OUT, SRC extends SourceFunction<OUT>, OP extends StreamSource<OUT, SRC>>
 	extends StreamTask<OUT, OP> {
 
+	private volatile boolean externallyInducedCheckpoints;
+
 	@Override
 	protected void init() {
 		// does not hold any resources, so no initialization needed
+
+		// we check if the source is actually inducing the checkpoints, rather
+		// than the trigger ch
+		SourceFunction<?> source = headOperator.getUserFunction();
+		if (source instanceof ExternallyInducedSource) {
+			externallyInducedCheckpoints = true;
+
+			ExternallyInducedSource.CheckpointTrigger triggerHook = new ExternallyInducedSource.CheckpointTrigger() {
+
+				@Override
+				public void triggerCheckpoint(long checkpointId) throws FlinkException {
+					// TODO - we need to see how to derive those. We should probably not encode this in the
+					// TODO -   source's trigger message, but do a handshake in this task between the trigger
+					// TODO -   message from the master, and the source's trigger notification
+					final CheckpointOptions checkpointOptions = CheckpointOptions.forFullCheckpoint();
+					final long timestamp = System.currentTimeMillis();
+
+					final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
+
+					try {
+						SourceStreamTask.super.triggerCheckpoint(checkpointMetaData, checkpointOptions);
+					}
+					catch (RuntimeException | FlinkException e) {
+						throw e;
+					}
+					catch (Exception e) {
+						throw new FlinkException(e.getMessage(), e);
+					}
+				}
+			};
+
+			((ExternallyInducedSource<?, ?>) source).setCheckpointTrigger(triggerHook);
+		}
 	}
 
 	@Override
@@ -62,4 +101,21 @@ protected void cancelTask() throws Exception {
 			headOperator.cancel();
 		}
 	}
+
+	// ------------------------------------------------------------------------
+	//  Checkpointing
+	// ------------------------------------------------------------------------
+
+	@Override
+	public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) throws Exception {
+		if (!externallyInducedCheckpoints) {
+			return super.triggerCheckpoint(checkpointMetaData, checkpointOptions);
+		}
+		else {
+			// we do not trigger checkpoints here, we simply state whether we can trigger them
+			synchronized (getCheckpointLock()) {
+				return isRunning();
+			}
+		}
+	}
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java
new file mode 100644
index 0000000000000..b5a95eb3c0c18
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.graph;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook.Factory;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.streaming.api.checkpoint.WithMasterCheckpointHook;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.Executor;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.*;
+
+/**
+ * Tests that when sources implement {@link WithMasterCheckpointHook} the hooks are properly
+ * configured in the job's checkpoint settings.
+ */
+@SuppressWarnings("serial")
+public class WithMasterCheckpointHookConfigTest {
+
+	/**
+	 * This test creates a program with 4 sources (2 with master hooks, 2 without).
+	 * The resulting job graph must have 2 configured master hooks.
+	 */
+	@Test
+	public void testHookConfiguration() throws Exception {
+		// create some sources some of which configure master hooks
+		final TestSource source1 = new TestSource();
+		final TestSourceWithHook source2 = new TestSourceWithHook("foo");
+		final TestSource source3 = new TestSource();
+		final TestSourceWithHook source4 = new TestSourceWithHook("bar");
+
+		final MapFunction<String, String> identity = new Identity<>();
+		final IdentityWithHook<String> identityWithHook1 = new IdentityWithHook<>("apple");
+		final IdentityWithHook<String> identityWithHook2 = new IdentityWithHook<>("orange");
+
+		final Set<MasterTriggerRestoreHook<?>> hooks = new HashSet<MasterTriggerRestoreHook<?>>(asList(
+				source2.createMasterTriggerRestoreHook(),
+				source4.createMasterTriggerRestoreHook(),
+				identityWithHook1.createMasterTriggerRestoreHook(),
+				identityWithHook2.createMasterTriggerRestoreHook()));
+
+		// we can instantiate a local environment here, because we never actually execute something
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
+		env.enableCheckpointing(500);
+
+		env
+			.addSource(source1).map(identity)
+			.union(env.addSource(source2).map(identity))
+			.union(env.addSource(source3).map(identityWithHook1))
+			.union(env.addSource(source4).map(identityWithHook2))
+			.addSink(new DiscardingSink<String>());
+
+		final JobGraph jg = env.getStreamGraph().getJobGraph();
+		assertEquals(hooks.size(), jg.getCheckpointingSettings().getMasterHooks().length);
+
+		// check that all hooks are contained and exist exactly once
+		for (Factory f : jg.getCheckpointingSettings().getMasterHooks()) {
+			MasterTriggerRestoreHook<?> hook = f.create();
+			assertTrue(hooks.remove(hook));
+		}
+		assertTrue(hooks.isEmpty());
+	}
+
+	// -----------------------------------------------------------------------
+
+	private static class TestHook implements MasterTriggerRestoreHook<String> {
+
+		private final String id;
+
+		TestHook(String id) {
+			this.id = id;
+		}
+
+		@Override
+		public String getIdentifier() {
+			return id;
+		}
+
+		@Override
+		public Future<String> triggerCheckpoint(long checkpointId, long timestamp, Executor executor) {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public void restoreCheckpoint(long checkpointId, @Nullable String checkpointData) throws Exception {
+			throw new UnsupportedOperationException();
+		}
+
+		@Nullable
+		@Override
+		public SimpleVersionedSerializer<String> createCheckpointDataSerializer() {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return obj == this || (obj != null && obj.getClass() == getClass() && ((TestHook) obj).id.equals(id));
+		}
+
+		@Override
+		public int hashCode() {
+			return id.hashCode();
+		}
+	}
+
+	// -----------------------------------------------------------------------
+
+	private static class TestSource implements SourceFunction<String> {
+
+		@Override
+		public void run(SourceContext<String> ctx) {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public void cancel() {}
+	}
+
+	// -----------------------------------------------------------------------
+
+	private static class TestSourceWithHook extends TestSource implements WithMasterCheckpointHook<String> {
+
+		private final String id;
+
+		TestSourceWithHook(String id) {
+			this.id = id;
+		}
+
+		@Override
+		public TestHook createMasterTriggerRestoreHook() {
+			return new TestHook(id);
+		}
+	}
+
+	// -----------------------------------------------------------------------
+
+	private static class Identity<T> implements MapFunction<T, T> {
+
+		@Override
+		public T map(T value) {
+			return value;
+		}
+	}
+
+	// -----------------------------------------------------------------------
+
+	private static class IdentityWithHook<T> extends Identity<T> implements WithMasterCheckpointHook<String> {
+
+		private final String id;
+
+		IdentityWithHook(String id) {
+			this.id = id;
+		}
+
+		@Override
+		public TestHook createMasterTriggerRestoreHook() {
+			return new TestHook(id);
+		}
+	}
+}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java
index 38741baac2022..54cd186773c94 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamRecordWriterTest.java
@@ -29,12 +29,9 @@
 import org.apache.flink.types.LongValue;
 
 import org.junit.Test;
-import org.junit.runner.RunWith;
 
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
 
 import java.io.IOException;
 
@@ -45,8 +42,6 @@
  * This test uses the PowerMockRunner runner to work around the fact that the
  * {@link ResultPartitionWriter} class is final.
  */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(ResultPartitionWriter.class)
 public class StreamRecordWriterTest {
 
 	/**
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceExternalCheckpointTriggerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceExternalCheckpointTriggerTest.java
new file mode 100644
index 0000000000000..e5caff35777d8
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceExternalCheckpointTriggerTest.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.tasks;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.core.testutils.MultiShotLatch;
+import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
+import org.apache.flink.runtime.checkpoint.CheckpointOptions;
+import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
+import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
+import org.apache.flink.streaming.api.checkpoint.ExternallyInducedSource;
+import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.api.operators.StreamSource;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import org.junit.Test;
+
+import java.util.concurrent.BlockingQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * These tests verify the behavior of a source function that triggers checkpoints
+ * in response to received events.
+ */
+@SuppressWarnings("serial")
+public class SourceExternalCheckpointTriggerTest {
+
+	static final OneShotLatch ready = new OneShotLatch();
+	static final MultiShotLatch sync = new MultiShotLatch();
+
+	@Test
+	public void testCheckpointsTriggeredBySource() throws Exception {
+		// set up the basic test harness
+		final SourceStreamTask<Long, ?, ?> sourceTask = new SourceStreamTask<Long, ExternalCheckpointsSource, StreamSource<Long, ExternalCheckpointsSource>>();
+		final StreamTaskTestHarness<Long> testHarness = new StreamTaskTestHarness<>(sourceTask, BasicTypeInfo.LONG_TYPE_INFO);
+		testHarness.setupOutputForSingletonOperatorChain();
+		testHarness.getExecutionConfig().setLatencyTrackingInterval(-1);
+
+		final long numElements = 10;
+		final long checkpointEvery = 3;
+
+		// set up the source function
+		ExternalCheckpointsSource source = new ExternalCheckpointsSource(numElements, checkpointEvery);
+		StreamConfig streamConfig = testHarness.getStreamConfig();
+		StreamSource<Long, ?> sourceOperator = new StreamSource<>(source);
+		streamConfig.setStreamOperator(sourceOperator);
+
+		// this starts the source thread
+		testHarness.invoke();
+		ready.await();
+
+		// now send an external trigger that should be ignored
+		assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(32, 829), CheckpointOptions.forFullCheckpoint()));
+
+		// step by step let the source thread emit elements
+		sync.trigger();
+		verifyNextElement(testHarness.getOutput(), 1L);
+		sync.trigger();
+		verifyNextElement(testHarness.getOutput(), 2L);
+		sync.trigger();
+		verifyNextElement(testHarness.getOutput(), 3L);
+
+		verifyCheckpointBarrier(testHarness.getOutput(), 1L);
+
+		sync.trigger();
+		verifyNextElement(testHarness.getOutput(), 4L);
+
+		// now send an regular trigger command that should be ignored
+		assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(34, 900), CheckpointOptions.forFullCheckpoint()));
+
+		sync.trigger();
+		verifyNextElement(testHarness.getOutput(), 5L);
+		sync.trigger();
+		verifyNextElement(testHarness.getOutput(), 6L);
+
+		verifyCheckpointBarrier(testHarness.getOutput(), 2L);
+
+		// let the remainder run
+
+		for (long l = 7L, checkpoint = 3L; l <= numElements; l++) {
+			sync.trigger();
+			verifyNextElement(testHarness.getOutput(), l);
+
+			if (l % checkpointEvery == 0) {
+				verifyCheckpointBarrier(testHarness.getOutput(), checkpoint++);
+			}
+		}
+
+		// done!
+	}
+
+	@SuppressWarnings("unchecked")
+	private void verifyNextElement(BlockingQueue<Object> output, long expectedElement) throws InterruptedException {
+		Object next = output.take();
+		assertTrue("next element is not an event", next instanceof StreamRecord);
+		assertEquals("wrong event", expectedElement, ((StreamRecord<Long>) next).getValue().longValue());
+	}
+
+	private void verifyCheckpointBarrier(BlockingQueue<Object> output, long checkpointId) throws InterruptedException {
+		Object next = output.take();
+		assertTrue("next element is not a checkpoint barrier", next instanceof CheckpointBarrier);
+		assertEquals("wrong checkpoint id", checkpointId, ((CheckpointBarrier) next).getId());
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static class ExternalCheckpointsSource
+			implements ParallelSourceFunction<Long>, ExternallyInducedSource<Long, Object> {
+
+		private final long numEvents;
+		private final long checkpointFrequency;
+		
+		private CheckpointTrigger trigger;
+
+		ExternalCheckpointsSource(long numEvents, long checkpointFrequency) {
+			this.numEvents = numEvents;
+			this.checkpointFrequency = checkpointFrequency;
+		}
+
+		@Override
+		public void run(SourceContext<Long> ctx) throws Exception {
+			ready.trigger();
+
+			// for simplicity in this test, we just trigger checkpoints in ascending order
+			long checkpoint = 1;
+
+			for (long num = 1; num <= numEvents; num++) {
+				sync.await();
+				ctx.collect(num);
+				if (num % checkpointFrequency == 0) {
+					trigger.triggerCheckpoint(checkpoint++);
+				}
+			}
+		}
+
+		@Override
+		public void cancel() {}
+
+		@Override
+		public void setCheckpointTrigger(CheckpointTrigger checkpointTrigger) {
+			this.trigger = checkpointTrigger;
+		}
+
+		@Override
+		public MasterTriggerRestoreHook<Object> createMasterTriggerRestoreHook() {
+			// not relevant in this test
+			throw new UnsupportedOperationException("not implemented");
+		}
+	}
+}
+
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java
index c51af4e1b7610..0be85b1968b13 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java
@@ -45,6 +45,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.LinkedBlockingQueue;
 
 /**
  * Test harness for testing a {@link StreamTask}.
@@ -83,7 +84,7 @@ public class StreamTaskTestHarness<OUT> {
 	private TypeSerializer<OUT> outputSerializer;
 	private TypeSerializer<StreamElement> outputStreamRecordSerializer;
 
-	private ConcurrentLinkedQueue<Object> outputList;
+	private LinkedBlockingQueue<Object> outputList;
 
 	protected TaskThread taskThread;
 
@@ -125,7 +126,7 @@ protected void initializeInputs() throws IOException, InterruptedException {}
 
 	@SuppressWarnings("unchecked")
 	private void initializeOutput() {
-		outputList = new ConcurrentLinkedQueue<Object>();
+		outputList = new LinkedBlockingQueue<Object>();
 		mockEnv.addOutput(outputList, outputStreamRecordSerializer);
 	}
 
@@ -265,7 +266,7 @@ public void waitForTaskRunning(long timeout) throws Exception {
 	 * {@link org.apache.flink.streaming.util.TestHarnessUtil#getRawElementsFromOutput(java.util.Queue)}}
 	 * to extract only the StreamRecords.
 	 */
-	public ConcurrentLinkedQueue<Object> getOutput() {
+	public LinkedBlockingQueue<Object> getOutput() {
 		return outputList;
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
index 3718a947f9221..e0de7d2f39bfd 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
@@ -40,7 +40,7 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.checkpoint.SubtaskState;
 import org.apache.flink.runtime.checkpoint.TaskState;
-import org.apache.flink.runtime.checkpoint.savepoint.SavepointV1;
+import org.apache.flink.runtime.checkpoint.savepoint.SavepointV2;
 import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
 import org.apache.flink.runtime.executiongraph.TaskInformation;
@@ -200,7 +200,7 @@ public void testTriggerSavepointAndResumeWithFileBasedCheckpoints() throws Excep
 			LOG.info("Requesting the savepoint.");
 			Future<Object> savepointFuture = jobManager.ask(new RequestSavepoint(savepointPath), deadline.timeLeft());
 
-			SavepointV1 savepoint = (SavepointV1) ((ResponseSavepoint) Await.result(savepointFuture, deadline.timeLeft())).savepoint();
+			SavepointV2 savepoint = (SavepointV2) ((ResponseSavepoint) Await.result(savepointFuture, deadline.timeLeft())).savepoint();
 			LOG.info("Retrieved savepoint: " + savepointPath + ".");
 
 			// Shut down the Flink cluster (thereby canceling the job)

From e2744e345b5438278479f9b97a7e4d97a97afeb7 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Thu, 27 Apr 2017 12:32:25 +0200
Subject: [PATCH 064/253] [hotfix] [tests] Harden JobManagerRegistrationTest

The problem is that we don't wait until the JobManager becomes the leader. Due to this,
the sent RegisterTaskManager messages might get dropped.

This PR fixes the problem by waiting on the completion of the NotifyWhenLeader message.
---
 .../flink/runtime/jobmanager/JobManager.scala |  2 +-
 .../JobManagerRegistrationTest.scala          | 91 +++++++++++--------
 2 files changed, 52 insertions(+), 41 deletions(-)

diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 40e2c2a1f25f8..2fc3ef4bfbf93 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -216,7 +216,7 @@ class JobManager(
       case Some(group) =>
         instantiateMetrics(group)
       case None =>
-        log.warn("Could not instantiate JobManager metrics.")
+        log.warn("Could not instantiate JobManager metric group.")
     }
   }
 
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
index 6be8bcc049ad8..b2e8005f8b010 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
@@ -33,7 +33,8 @@ import org.apache.flink.runtime.jobmanager.JobManagerRegistrationTest.PlainForwa
 import org.apache.flink.runtime.messages.JobManagerMessages.LeaderSessionMessage
 import org.apache.flink.runtime.messages.RegistrationMessages.{AcknowledgeRegistration, AlreadyRegistered, RegisterTaskManager}
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation
-import org.apache.flink.runtime.testingUtils.TestingUtils
+import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenLeader
+import org.apache.flink.runtime.testingUtils.{TestingJobManager, TestingUtils}
 import org.apache.flink.runtime.testutils.TestingResourceManager
 import org.apache.flink.runtime.util.LeaderRetrievalUtils
 import org.junit.Assert.{assertNotEquals, assertNotNull}
@@ -41,6 +42,7 @@ import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner
 import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
 
+import scala.concurrent.Await
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
@@ -94,39 +96,44 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
         var id1: InstanceID = null
         var id2: InstanceID = null
 
+        // wait until the JobManager becomes the leader, otherwise the RegisterTaskManager messages
+        // are dropped
+        val leaderFuture = jm.ask(NotifyWhenLeader, TestingUtils.TESTING_TIMEOUT)
+        Await.ready(leaderFuture, TestingUtils.TESTING_TIMEOUT)
+
         // task manager 1
         within(10 seconds) {
-         jm.tell(
-           RegisterTaskManager(
-             resourceId1,
-             connectionInfo1,
-             hardwareDescription,
-             1),
-           new AkkaActorGateway(tm1, HighAvailabilityServices.DEFAULT_LEADER_ID))
-
-         val response = probe.expectMsgType[LeaderSessionMessage]
-         response match {
-           case LeaderSessionMessage(_, AcknowledgeRegistration(id, _)) => id1 = id
-           case _ => fail("Wrong response message: " + response)
-         }
-       }
+          jm.tell(
+            RegisterTaskManager(
+              resourceId1,
+              connectionInfo1,
+              hardwareDescription,
+              1),
+            new AkkaActorGateway(tm1, HighAvailabilityServices.DEFAULT_LEADER_ID))
+
+          val response = probe.expectMsgType[LeaderSessionMessage]
+          response match {
+            case LeaderSessionMessage(_, AcknowledgeRegistration(id, _)) => id1 = id
+            case _ => fail("Wrong response message: " + response)
+          }
+        }
 
         // task manager 2
         within(10 seconds) {
-         jm.tell(
-           RegisterTaskManager(
-             resourceId2,
-             connectionInfo2,
-             hardwareDescription,
-             1),
-           new AkkaActorGateway(tm2, HighAvailabilityServices.DEFAULT_LEADER_ID))
-
-         val response = probe.expectMsgType[LeaderSessionMessage]
-         response match {
-           case LeaderSessionMessage(leaderSessionID, AcknowledgeRegistration(id, _)) => id2 = id
-           case _ => fail("Wrong response message: " + response)
-         }
-       }
+          jm.tell(
+            RegisterTaskManager(
+              resourceId2,
+              connectionInfo2,
+              hardwareDescription,
+              1),
+            new AkkaActorGateway(tm2, HighAvailabilityServices.DEFAULT_LEADER_ID))
+
+          val response = probe.expectMsgType[LeaderSessionMessage]
+          response match {
+            case LeaderSessionMessage(leaderSessionID, AcknowledgeRegistration(id, _)) => id2 = id
+            case _ => fail("Wrong response message: " + response)
+          }
+        }
 
         assertNotNull(id1)
         assertNotNull(id2)
@@ -160,6 +167,11 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
         val connectionInfo = new TaskManagerLocation(resourceID, InetAddress.getLocalHost, 1)
         val hardwareDescription = HardwareDescription.extractFromSystem(10)
 
+        // wait until the JobManager becomes the leader, otherwise the RegisterTaskManager messages
+        // are dropped
+        val leaderFuture = jm.ask(NotifyWhenLeader, TestingUtils.TESTING_TIMEOUT)
+        Await.ready(leaderFuture, TestingUtils.TESTING_TIMEOUT)
+
         within(20 seconds) {
           jm.tell(
             RegisterTaskManager(
@@ -167,7 +179,7 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
               connectionInfo,
               hardwareDescription,
               1),
-              selfGateway)
+            selfGateway)
 
           jm.tell(
             RegisterTaskManager(
@@ -175,7 +187,7 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
               connectionInfo,
               hardwareDescription,
               1),
-              selfGateway)
+            selfGateway)
 
           jm.tell(
             RegisterTaskManager(
@@ -183,26 +195,26 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
               connectionInfo,
               hardwareDescription,
               1),
-              selfGateway)
+            selfGateway)
 
           probe.expectMsgType[LeaderSessionMessage] match {
             case LeaderSessionMessage(
-              HighAvailabilityServices.DEFAULT_LEADER_ID,
-              AcknowledgeRegistration(_, _)) =>
+            HighAvailabilityServices.DEFAULT_LEADER_ID,
+            AcknowledgeRegistration(_, _)) =>
             case m => fail("Wrong message type: " + m)
           }
 
           probe.expectMsgType[LeaderSessionMessage] match {
             case LeaderSessionMessage(
-              HighAvailabilityServices.DEFAULT_LEADER_ID,
-              AlreadyRegistered(_, _)) =>
+            HighAvailabilityServices.DEFAULT_LEADER_ID,
+            AlreadyRegistered(_, _)) =>
             case m => fail("Wrong message type: " + m)
           }
 
           probe.expectMsgType[LeaderSessionMessage] match {
             case LeaderSessionMessage(
-              HighAvailabilityServices.DEFAULT_LEADER_ID,
-              AlreadyRegistered(_, _)) =>
+            HighAvailabilityServices.DEFAULT_LEADER_ID,
+            AlreadyRegistered(_, _)) =>
             case m => fail("Wrong message type: " + m)
           }
         }
@@ -227,7 +239,7 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
     // if there exists already one of these actors (e.g. JobManager has not been properly shutdown),
     // then this will fail the JobManager creation
     val props = JobManager.getJobManagerProps(
-      classOf[JobManager],
+      classOf[TestingJobManager],
       config,
       executor,
       executor,
@@ -249,7 +261,6 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
   }
 
   private def startTestingResourceManager(system: ActorSystem, jm: ActorRef): ActorGateway = {
-    val jobManagerURL = AkkaUtils.getAkkaURL(system, jm)
     val config = new Configuration()
     val rm: ActorRef = FlinkResourceManager.startResourceManagerActors(
       config,

From b6d0ca060865d33b80b2b02684cd54a6a0426fe0 Mon Sep 17 00:00:00 2001
From: manuzhang <owenzhang1990@gmail.com>
Date: Thu, 27 Apr 2017 12:19:37 +0200
Subject: [PATCH 065/253] [hotfix] Fix wrong comparator in WindowOperatorTest

And also the same bug in WindowOperatorMigrationTest
---
 .../operators/windowing/WindowOperatorMigrationTest.java        | 2 +-
 .../runtime/operators/windowing/WindowOperatorTest.java         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
index 19fa04f51c444..1168eb0343110 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
@@ -826,7 +826,7 @@ public int compare(Object o1, Object o2) {
 					if (comparison != 0) {
 						return comparison;
 					}
-					return (int) (sr0.getValue().f1 - sr1.getValue().f1);
+					return (int) (sr0.getValue().f2 - sr1.getValue().f2);
 				}
 			}
 		}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
index 1137c6a7da69e..474548cb1b641 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
@@ -2560,7 +2560,7 @@ public int compare(Object o1, Object o2) {
 					if (comparison != 0) {
 						return comparison;
 					}
-					return (int) (sr0.getValue().f1 - sr1.getValue().f1);
+					return (int) (sr0.getValue().f2 - sr1.getValue().f2);
 				}
 			}
 		}

From 2e5351aceb17b01e5b33e88323c2312bd53225f8 Mon Sep 17 00:00:00 2001
From: manuzhang <owenzhang1990@gmail.com>
Date: Sun, 2 Apr 2017 19:27:14 +0800
Subject: [PATCH 066/253] [FLINK-4953] Allow access to "time" in
 ProcessWindowFunction.Context

---
 .../FoldApplyProcessWindowFunction.java       |  6 +-
 .../InternalProcessApplyWindowContext.java    | 16 +++-
 .../windowing/ProcessWindowFunction.java      |  8 +-
 .../ReduceApplyProcessWindowFunction.java     |  6 +-
 .../windowing/AccumulatingKeyedTimePanes.java | 10 +++
 .../operators/windowing/WindowOperator.java   | 10 +++
 .../InternalProcessWindowContext.java         | 10 +++
 .../functions/InternalWindowFunction.java     |  4 +
 .../FoldApplyProcessWindowFunctionTest.java   | 10 +++
 .../functions/InternalWindowFunctionTest.java | 64 ++++++++++++++--
 .../windowing/WindowOperatorContractTest.java | 75 +++++++++++++++++++
 .../OneInputStreamOperatorTestHarness.java    |  9 ++-
 .../function/ProcessWindowFunction.scala      | 12 ++-
 .../ScalaProcessWindowFunctionWrapper.scala   |  8 ++
 14 files changed, 225 insertions(+), 23 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
index 281afc75ff254..1b2c2e252fb6a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
@@ -100,16 +100,14 @@ public void process(K key, Context context, Iterable<T> values, Collector<R> out
 		}
 
 		this.ctx.window = context.window();
-		this.ctx.windowState = context.windowState();
-		this.ctx.globalState = context.globalState();
+		this.ctx.context = context;
 		windowFunction.process(key, ctx, Collections.singletonList(result), out);
 	}
 
 	@Override
 	public void clear(final Context context) throws Exception{
 		this.ctx.window = context.window();
-		this.ctx.windowState = context.windowState();
-		this.ctx.globalState = context.globalState();
+		this.ctx.context = context;
 		windowFunction.clear(ctx);
 	}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java
index 0cf0a8dce4543..fc31b0731a239 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/InternalProcessApplyWindowContext.java
@@ -34,8 +34,7 @@ public class InternalProcessApplyWindowContext<IN, OUT, KEY, W extends Window>
 	extends ProcessWindowFunction<IN, OUT, KEY, W>.Context {
 
 	W window;
-	KeyedStateStore windowState;
-	KeyedStateStore globalState;
+	ProcessWindowFunction.Context context;
 
 	InternalProcessApplyWindowContext(ProcessWindowFunction<IN, OUT, KEY, W> function) {
 		function.super();
@@ -47,12 +46,21 @@ public W window() {
 	}
 
 	@Override
+	public long currentProcessingTime() {
+		return context.currentProcessingTime();
+	}
+
+	@Override
+	public long currentWatermark() {
+		return context.currentWatermark();
+	}
+
 	public KeyedStateStore windowState() {
-		return windowState;
+		return context.windowState();
 	}
 
 	@Override
 	public KeyedStateStore globalState() {
-		return globalState;
+		return context.globalState();
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
index 980a94ac97ec7..2c80e9e7d9d4e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
@@ -63,10 +63,16 @@ public void clear(Context context) throws Exception {}
 	 */
 	public abstract class Context implements java.io.Serializable {
 		/**
-		 * @return The window that is being evaluated.
+		 * Returns the window that is being evaluated.
 		 */
 		public abstract W window();
 
+		/** Returns the current processing time. */
+		public abstract long currentProcessingTime();
+
+		/** Returns the current event-time watermark. */
+		public abstract long currentWatermark();
+
 		/**
 		 * State accessor for per-key and per-window state.
 		 *
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
index f5694c3552465..836726d850874 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
@@ -59,16 +59,14 @@ public void process(K k, final Context context, Iterable<T> input, Collector<R>
 		}
 
 		this.ctx.window = context.window();
-		this.ctx.windowState = context.windowState();
-		this.ctx.globalState = context.globalState();
+		this.ctx.context = context;
 		windowFunction.process(k, ctx, Collections.singletonList(curr), out);
 	}
 
 	@Override
 	public void clear(final Context context) throws Exception {
 		this.ctx.window = context.window();
-		this.ctx.windowState = context.windowState();
-		this.ctx.globalState = context.globalState();
+		this.ctx.context = context;
 		windowFunction.clear(ctx);
 	}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
index 55ba1e342c4f1..4511b6b7d5894 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
@@ -193,6 +193,16 @@ public AccumulatingKeyedTimePanesContext() {
 			this.throwingStore = new ThrowingKeyedStateStore();
 		}
 
+		@Override
+		public long currentProcessingTime() {
+			throw new UnsupportedOperationException("current processing time is not supported in this context");
+		}
+
+		@Override
+		public long currentWatermark() {
+			throw new UnsupportedOperationException("current watermark is not supported in this context");
+		}
+
 		@Override
 		public KeyedStateStore windowState() {
 			return throwingStore;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index 8d8f6b54bb7c9..a62c9ff020553 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -816,6 +816,16 @@ public void clear() throws Exception {
 			userFunction.clear(window, this);
 		}
 
+		@Override
+		public long currentProcessingTime() {
+			return internalTimerService.currentProcessingTime();
+		}
+
+		@Override
+		public long currentWatermark() {
+			return internalTimerService.currentWatermark();
+		}
+
 		@Override
 		public KeyedStateStore windowState() {
 			this.windowState.window = this.window;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalProcessWindowContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalProcessWindowContext.java
index 0f1c0ee02ac66..d575f01831e73 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalProcessWindowContext.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalProcessWindowContext.java
@@ -46,6 +46,16 @@ public W window() {
 		return window;
 	}
 
+	@Override
+	public long currentProcessingTime() {
+		return internalContext.currentProcessingTime();
+	}
+
+	@Override
+	public long currentWatermark() {
+		return internalContext.currentWatermark();
+	}
+
 	@Override
 	public KeyedStateStore windowState() {
 		return internalContext.windowState();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java
index 4055c4f6affcd..ea64ddafcfa3a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/functions/InternalWindowFunction.java
@@ -55,6 +55,10 @@ public interface InternalWindowFunction<IN, OUT, KEY, W extends Window> extends
 	 * for internal use.
 	 */
 	interface InternalWindowContext extends java.io.Serializable {
+		long currentProcessingTime();
+
+		long currentWatermark();
+
 		KeyedStateStore windowState();
 
 		KeyedStateStore globalState();
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java
index 7cac1e1b514e0..90470acdaba27 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/FoldApplyProcessWindowFunctionTest.java
@@ -153,6 +153,16 @@ public TimeWindow window() {
 				return new TimeWindow(0, 1);
 			}
 
+			@Override
+			public long currentProcessingTime() {
+				return 0;
+			}
+
+			@Override
+			public long currentWatermark() {
+				return 0;
+			}
+
 			@Override
 			public KeyedStateStore windowState() {
 				return new DummyKeyedStateStore();
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java
index 6f34607160e89..4b8057f5a6bcb 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.AggregateFunction;
 import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.state.KeyedStateStore;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.configuration.Configuration;
@@ -33,7 +32,6 @@
 import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
 import org.apache.flink.streaming.api.operators.OutputTypeConfigurable;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
-import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessAllWindowFunction;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessWindowFunction;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableAllWindowFunction;
@@ -48,6 +46,8 @@
 import org.apache.flink.util.Collector;
 import org.hamcrest.collection.IsIterableContainingInOrder;
 import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -62,6 +62,7 @@
 import static org.hamcrest.core.AllOf.allOf;
 import static org.mockito.Mockito.anyObject;
 import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -229,8 +230,24 @@ public void testInternalIterableProcessWindowFunction() throws Exception {
 		Collector<String> c = (Collector<String>) mock(Collector.class);
 		InternalWindowFunction.InternalWindowContext ctx = mock(InternalWindowFunction.InternalWindowContext.class);
 
+		doAnswer(new Answer() {
+			@Override
+			public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+				ProcessWindowFunctionMock.Context c =
+				(ProcessWindowFunction.Context) invocationOnMock.getArguments()[1];
+				c.currentProcessingTime();
+				c.currentWatermark();
+				c.windowState();
+				c.globalState();
+				return null;
+			}
+		}).when(mock).process(eq(42L), (ProcessWindowFunctionMock.Context) anyObject(), eq(i), eq(c));
+
 		windowFunction.process(42L, w, ctx, i, c);
-		verify(mock).process(eq(42L), (ProcessWindowFunctionMock.Context) anyObject(), eq(i), eq(c));
+		verify(ctx).currentProcessingTime();
+		verify(ctx).currentWatermark();
+		verify(ctx).windowState();
+		verify(ctx).globalState();
 
 		// check close
 		windowFunction.close();
@@ -396,8 +413,25 @@ public void testInternalSingleValueProcessWindowFunction() throws Exception {
 		Collector<String> c = (Collector<String>) mock(Collector.class);
 		InternalWindowFunction.InternalWindowContext ctx = mock(InternalWindowFunction.InternalWindowContext.class);
 
+		doAnswer(new Answer() {
+			@Override
+			public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+				ProcessWindowFunctionMock.Context c =
+					(ProcessWindowFunction.Context) invocationOnMock.getArguments()[1];
+				c.currentProcessingTime();
+				c.currentWatermark();
+				c.windowState();
+				c.globalState();
+				return null;
+			}
+		}).when(mock).process(eq(42L), (ProcessWindowFunctionMock.Context) anyObject(),
+		(Iterable<Long>)argThat(IsIterableContainingInOrder.contains(23L)), eq(c));
+
 		windowFunction.process(42L, w, ctx,23L, c);
-		verify(mock).process(eq(42L), (ProcessWindowFunctionMock.Context) anyObject(), (Iterable<Long>)argThat(IsIterableContainingInOrder.contains(23L)), eq(c));
+		verify(ctx).currentProcessingTime();
+		verify(ctx).currentWatermark();
+		verify(ctx).windowState();
+		verify(ctx).globalState();
 
 		// check close
 		windowFunction.close();
@@ -469,8 +503,18 @@ public Set<Long> merge(Set<Long> a, Set<Long> b) {
 		args.add(24L);
 		InternalWindowFunction.InternalWindowContext ctx = mock(InternalWindowFunction.InternalWindowContext.class);
 
-		windowFunction.process(42L, w, ctx, args, c);
-		verify(mock).process(
+		doAnswer(new Answer() {
+			@Override
+			public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+				ProcessWindowFunctionMock.Context c =
+					(ProcessWindowFunction.Context) invocationOnMock.getArguments()[1];
+				c.currentProcessingTime();
+				c.currentWatermark();
+				c.windowState();
+				c.globalState();
+				return null;
+			}
+		}).when(mock).process(
 				eq(42L),
 				(AggregateProcessWindowFunctionMock.Context) anyObject(),
 				(Iterable) argThat(containsInAnyOrder(allOf(
@@ -478,6 +522,12 @@ public Set<Long> merge(Set<Long> a, Set<Long> b) {
 						hasEntry(is(24L), is(24L))))),
 				eq(c));
 
+		windowFunction.process(42L, w, ctx, args, c);
+		verify(ctx).currentProcessingTime();
+		verify(ctx).currentWatermark();
+		verify(ctx).windowState();
+		verify(ctx).globalState();
+
 		// check close
 		windowFunction.close();
 		verify(mock).close();
@@ -572,7 +622,6 @@ public void setOutputType(TypeInformation<String> outTypeInfo, ExecutionConfig e
 
 		@Override
 		public void process(Long aLong, ProcessWindowFunction<Long, String, Long, TimeWindow>.Context context, Iterable<Long> elements, Collector<String> out) throws Exception {
-
 		}
 	}
 
@@ -587,7 +636,6 @@ public void setOutputType(TypeInformation<String> outTypeInfo, ExecutionConfig e
 
 		@Override
 		public void process(Long aLong, ProcessWindowFunction<Map<Long, Long>, String, Long, TimeWindow>.Context context, Iterable<Map<Long, Long>> elements, Collector<String> out) throws Exception {
-
 		}
 	}
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorContractTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorContractTest.java
index 3ae8f375fb0fe..0d80605b13728 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorContractTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorContractTest.java
@@ -45,6 +45,8 @@
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeutils.base.StringSerializer;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
 import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;
@@ -2450,6 +2452,61 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
 		testHarness.processElement(new StreamRecord<>(0, 0L));
 	}
 
+	@Test
+	public void testEventTimeQuerying() throws Exception {
+		testCurrentTimeQuerying(new EventTimeAdaptor());
+	}
+
+	@Test
+	public void testProcessingTimeQuerying() throws Exception {
+		testCurrentTimeQuerying(new ProcessingTimeAdaptor());
+	}
+
+	public void testCurrentTimeQuerying(final TimeDomainAdaptor timeAdaptor) throws Exception {
+		WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner();
+		timeAdaptor.setIsEventTime(mockAssigner);
+		Trigger<Integer, TimeWindow> mockTrigger = mockTrigger();
+		InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction();
+
+		final KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness =
+				createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction);
+
+		testHarness.open();
+
+		shouldFireOnElement(mockTrigger);
+
+		when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext()))
+				.thenReturn(Arrays.asList(new TimeWindow(0, 20)));
+
+		doAnswer(new Answer<Object>() {
+			@Override
+			public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+				InternalWindowFunction.InternalWindowContext context = (InternalWindowFunction.InternalWindowContext)invocationOnMock.getArguments()[2];
+				timeAdaptor.verifyCorrectTime(testHarness, context);
+				return null;
+			}
+		}).when(mockWindowFunction).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector());
+
+		doAnswer(new Answer<Object>() {
+			@Override
+			public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+				InternalWindowFunction.InternalWindowContext context = (InternalWindowFunction.InternalWindowContext)invocationOnMock.getArguments()[1];
+				timeAdaptor.verifyCorrectTime(testHarness, context);
+				return null;
+			}
+		}).when(mockWindowFunction).clear(anyTimeWindow(), anyInternalWindowContext());
+
+		timeAdaptor.advanceTime(testHarness, 10);
+
+		testHarness.processElement(new StreamRecord<>(0, 0L));
+
+		verify(mockWindowFunction, times(1)).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector());
+
+		timeAdaptor.advanceTime(testHarness, 100);
+
+		verify(mockWindowFunction, times(1)).clear(anyTimeWindow(), anyInternalWindowContext());
+	}
+
 	protected abstract <W extends Window, OUT> KeyedOneInputStreamOperatorTestHarness<Integer, Integer, OUT> createWindowOperator(
 			WindowAssigner<Integer, W> assigner,
 			Trigger<Integer, W> trigger,
@@ -2495,6 +2552,10 @@ void verifyTriggerCallback(
 				VerificationMode verificationMode,
 				Long time,
 				TimeWindow window) throws Exception;
+
+		void verifyCorrectTime(
+				OneInputStreamOperatorTestHarness testHarness,
+				InternalWindowFunction.InternalWindowContext context);
 	}
 
 	private static class EventTimeAdaptor implements TimeDomainAdaptor {
@@ -2588,6 +2649,13 @@ public void verifyTriggerCallback(
 						anyTriggerContext());
 			}
 		}
+
+		@Override
+		public void verifyCorrectTime(
+				OneInputStreamOperatorTestHarness testHarness,
+				InternalWindowFunction.InternalWindowContext context) {
+			assertEquals(testHarness.getCurrentWatermark(), context.currentWatermark());
+		}
 	}
 
 	private static class ProcessingTimeAdaptor implements TimeDomainAdaptor {
@@ -2681,5 +2749,12 @@ public void verifyTriggerCallback(
 						anyTriggerContext());
 			}
 		}
+
+		@Override
+		public void verifyCorrectTime(
+				OneInputStreamOperatorTestHarness testHarness,
+				InternalWindowFunction.InternalWindowContext context) {
+			assertEquals(testHarness.getProcessingTime(), context.currentProcessingTime());
+		}
 	}
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java
index 9b02378f90ef1..ced8cca0346bc 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OneInputStreamOperatorTestHarness.java
@@ -38,6 +38,8 @@ public class OneInputStreamOperatorTestHarness<IN, OUT>
 
 	private final OneInputStreamOperator<IN, OUT> oneInputOperator;
 
+	private long currentWatermark;
+
 	public OneInputStreamOperatorTestHarness(
 			OneInputStreamOperator<IN, OUT> operator,
 			TypeSerializer<IN> typeSerializerIn) throws Exception {
@@ -98,10 +100,15 @@ public void processElements(Collection<StreamRecord<IN>> elements) throws Except
 	}
 
 	public void processWatermark(long watermark) throws Exception {
-		oneInputOperator.processWatermark(new Watermark(watermark));
+		processWatermark(new Watermark(watermark));
 	}
 
 	public void processWatermark(Watermark mark) throws Exception {
+		currentWatermark = mark.getTimestamp();
 		oneInputOperator.processWatermark(mark);
 	}
+
+	public long getCurrentWatermark() {
+		return currentWatermark;
+	}
 }
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala
index bdf6ae6606e7c..bc79a267d6eb2 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala
@@ -63,10 +63,20 @@ abstract class ProcessWindowFunction[IN, OUT, KEY, W <: Window] extends Function
     */
   abstract class Context {
     /**
-      * @return The window that is being evaluated.
+      * Returns the window that is being evaluated.
       */
     def window: W
 
+    /**
+      * Returns the current processing time.
+      */
+    def currentProcessingTime: Long
+
+    /**
+      * Returns the current event-time watermark.
+      */
+    def currentWatermark: Long
+
     /**
       * State accessor for per-key and per-window state.
       */
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala
index fac59583da908..263373eeac7f3 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala
@@ -53,6 +53,10 @@ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window](
     val ctx = new func.Context {
       override def window = context.window
 
+      override def currentProcessingTime = context.currentProcessingTime
+
+      override def currentWatermark = context.currentWatermark
+
       override def windowState = context.windowState()
 
       override def globalState = context.globalState()
@@ -64,6 +68,10 @@ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window](
     val ctx = new func.Context {
       override def window = context.window
 
+      override def currentProcessingTime = context.currentProcessingTime
+
+      override def currentWatermark = context.currentWatermark
+
       override def windowState = context.windowState()
 
       override def globalState = context.globalState()

From a3879c36dc0351a545f00f32da744809f5da526f Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 17 Nov 2016 19:36:56 +0100
Subject: [PATCH 067/253] [FLINK-5090] [network] Add metrics for details about
 inbound/outbound network queues

These metrics are optimised go go through the channels only once in order to
gather all metrics, i.e. min, max, avg and sum. Whenever a request to either
of those is made, all metrics are refreshed and cached. Requests to the other
metrics will be served from the cache. However, each value will be served only
once from the cache and a second call to retrieve the minimum, for example,
will refresh the cache for all values.

This setup may at first be a bit strange but ensures that the statistics belong
together logically and originate from a common point in time. This is not
necessarily the point in time the metric was requested though.
---
 .../flink/configuration/ConfigConstants.java  |   6 +
 .../partition/PipelinedSubpartition.java      |   5 +-
 .../io/network/partition/ResultPartition.java |   7 +-
 .../partition/ResultPartitionMetrics.java     | 136 ++++++++++++++
 .../network/partition/ResultSubpartition.java |   7 +-
 .../partition/SpillableSubpartition.java      |   5 +-
 .../partition/consumer/InputGateMetrics.java  | 167 ++++++++++++++++++
 .../consumer/RemoteInputChannel.java          |   7 +-
 .../partition/consumer/SingleInputGate.java   |   6 +-
 .../flink/runtime/taskmanager/Task.java       |  17 ++
 10 files changed, 351 insertions(+), 12 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 61c1b27ecbeed..6863d14bad838 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -250,6 +250,12 @@ public final class ConfigConstants {
 	@Deprecated
 	public static final String TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY = "taskmanager.network.numberOfBuffers";
 
+	/**
+	 * Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths
+	 */
+	@PublicEvolving
+	public static final String NETWORK_DETAILED_METRICS_KEY = "taskmanager.net.detailed-metrics";
+
 	/**
 	 * Config parameter defining the size of memory buffers used by the network stack and the memory manager.
 	 *
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java
index 9e2f5ba1bf4cd..4fd74e2ea6ef8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java
@@ -211,7 +211,8 @@ public String toString() {
 	}
 
 	@Override
-	public int getNumberOfQueuedBuffers() {
-			return buffers.size();
+	public int unsynchronizedGetNumberOfQueuedBuffers() {
+		// since we do not synchronize, the size may actually be lower than 0!
+		return Math.max(buffers.size(), 0);
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
index eb1418b383219..29a7cafbdf39a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
@@ -30,6 +30,7 @@
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
 import org.apache.flink.runtime.taskmanager.TaskActions;
 import org.apache.flink.runtime.taskmanager.TaskManager;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -232,7 +233,7 @@ public int getNumberOfQueuedBuffers() {
 		int totalBuffers = 0;
 
 		for (ResultSubpartition subpartition : subpartitions) {
-			totalBuffers += subpartition.getNumberOfQueuedBuffers();
+			totalBuffers += subpartition.unsynchronizedGetNumberOfQueuedBuffers();
 		}
 
 		return totalBuffers;
@@ -446,6 +447,10 @@ else if (refCnt < 0) {
 				this, subpartitionIndex, pendingReferences);
 	}
 
+	ResultSubpartition[] getAllPartitions() {
+		return subpartitions;
+	}
+
 	// ------------------------------------------------------------------------
 
 	private void checkInProduceState() {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java
new file mode 100644
index 0000000000000..0fd32107339a8
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.partition;
+
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.MetricGroup;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+public class ResultPartitionMetrics {
+
+	private final ResultPartition partition;
+
+	private int lastMin = -1;
+
+	private int lastMax = -1;
+
+	private float lastAvg = -1.0f;
+
+	// ------------------------------------------------------------------------
+
+	private ResultPartitionMetrics(ResultPartition partition) {
+		this.partition = checkNotNull(partition);
+	}
+
+	// ------------------------------------------------------------------------
+
+	// these methods are package private to make access from the nested classes faster 
+
+	int refreshAndGetMin() {
+		int min;
+		if ((min = lastMin) == -1) {
+			refresh();
+			min = lastMin;
+		}
+
+		lastMin = -1;
+		return min;
+	}
+
+	int refreshAndGetMax() {
+		int max;
+		if ((max = lastMax) == -1) {
+			refresh();
+			max = lastMax;
+		}
+
+		lastMax = -1;
+		return max;
+	}
+
+	float refreshAndGetAvg() {
+		float avg;
+		if ((avg = lastAvg) < 0.0f) {
+			refresh();
+			avg = lastAvg;
+		}
+
+		lastAvg = -1.0f;
+		return avg;
+	}
+
+	private void refresh() {
+		int min = Integer.MAX_VALUE;
+		int max = 0;
+
+		for (ResultSubpartition part : partition.getAllPartitions()) {
+			int size = part.unsynchronizedGetNumberOfQueuedBuffers();
+			min = Math.min(min, size);
+			max = Math.max(max, size);
+		}
+
+		this.lastMin = min;
+		this.lastMax = max;
+		this.lastAvg = partition.getTotalNumberOfBuffers() / (float) partition.getNumberOfSubpartitions();
+	}
+
+	// ------------------------------------------------------------------------
+	//  Gauges to access the stats
+	// ------------------------------------------------------------------------
+
+	private Gauge<Integer> getMinQueueLenGauge() {
+		return new Gauge<Integer>() {
+			@Override
+			public Integer getValue() {
+				return refreshAndGetMin();
+			}
+		};
+	}
+
+	private Gauge<Integer> getMaxQueueLenGauge() {
+		return new Gauge<Integer>() {
+			@Override
+			public Integer getValue() {
+				return refreshAndGetMax();
+			}
+		};
+	}
+
+	private Gauge<Float> getAvgQueueLenGauge() {
+		return new Gauge<Float>() {
+			@Override
+			public Float getValue() {
+				return refreshAndGetAvg();
+			}
+		};
+	}
+
+	// ------------------------------------------------------------------------
+	//  Static access
+	// ------------------------------------------------------------------------
+
+	public static void registerQueueLengthMetrics(MetricGroup group, ResultPartition partition) {
+		ResultPartitionMetrics metrics = new ResultPartitionMetrics(partition);
+
+		group.gauge("min-queue-len", metrics.getMinQueueLenGauge());
+		group.gauge("max-queue-len", metrics.getMaxQueueLenGauge());
+		group.gauge("avg-queue-len", metrics.getAvgQueueLenGauge());
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java
index dd0e1526cd968..d3cd887ca559b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java
@@ -83,6 +83,11 @@ protected Throwable getFailureCause() {
 
 	abstract public boolean isReleased();
 
-	abstract public int getNumberOfQueuedBuffers();
+	/**
+	 * Makes a best effort to get the current size of the queue.
+	 * This method must not acquire locks or interfere with the task and network threads in
+	 * any way.
+	 */
+	abstract public int unsynchronizedGetNumberOfQueuedBuffers();
 
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
index ae97c4259711c..a5781880b51ef 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
@@ -237,8 +237,9 @@ public boolean isReleased() {
 	}
 
 	@Override
-	public int getNumberOfQueuedBuffers() {
-		return buffers.size();
+	public int unsynchronizedGetNumberOfQueuedBuffers() {
+		// since we do not synchronize, the size may actually be lower than 0!
+		return Math.max(buffers.size(), 0);
 	}
 
 	@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java
new file mode 100644
index 0000000000000..b5eb8601029a8
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.partition.consumer;
+
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.MetricGroup;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+public class InputGateMetrics {
+
+	private final SingleInputGate inputGate;
+
+	private long lastTotal = -1;
+
+	private int lastMin = -1;
+
+	private int lastMax = -1;
+
+	private float lastAvg = -1.0f;
+
+	// ------------------------------------------------------------------------
+
+	private InputGateMetrics(SingleInputGate inputGate) {
+		this.inputGate = checkNotNull(inputGate);
+	}
+
+	// ------------------------------------------------------------------------
+
+	// these methods are package private to make access from the nested classes faster 
+
+	long refreshAndGetTotal() {
+		long total;
+		if ((total = lastTotal) == -1) {
+			refresh();
+			total = lastTotal;
+		}
+
+		lastTotal = -1;
+		return total;
+	}
+
+	int refreshAndGetMin() {
+		int min;
+		if ((min = lastMin) == -1) {
+			refresh();
+			min = lastMin;
+		}
+
+		lastMin = -1;
+		return min;
+	}
+
+	int refreshAndGetMax() {
+		int max;
+		if ((max = lastMax) == -1) {
+			refresh();
+			max = lastMax;
+		}
+
+		lastMax = -1;
+		return max;
+	}
+
+	float refreshAndGetAvg() {
+		float avg;
+		if ((avg = lastAvg) < 0.0f) {
+			refresh();
+			avg = lastAvg;
+		}
+
+		lastAvg = -1.0f;
+		return avg;
+	}
+
+	private void refresh() {
+		long total = 0;
+		int min = Integer.MAX_VALUE;
+		int max = 0;
+		int count = 0;
+
+		for (InputChannel channel : inputGate.getInputChannels().values()) {
+			if (channel.getClass() == RemoteInputChannel.class) {
+				RemoteInputChannel rc = (RemoteInputChannel) channel;
+
+				int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
+				total += size;
+				min = Math.min(min, size);
+				max = Math.max(max, size);
+				count++;
+			}
+		}
+
+		this.lastMin = min;
+		this.lastMax = max;
+		this.lastAvg = total / (float) count;
+	}
+
+	// ------------------------------------------------------------------------
+	//  Gauges to access the stats
+	// ------------------------------------------------------------------------
+
+	private Gauge<Long> getTotalQueueLenGauge() {
+		return new Gauge<Long>() {
+			@Override
+			public Long getValue() {
+				return refreshAndGetTotal();
+			}
+		};
+	}
+
+	private Gauge<Integer> getMinQueueLenGauge() {
+		return new Gauge<Integer>() {
+			@Override
+			public Integer getValue() {
+				return refreshAndGetMin();
+			}
+		};
+	}
+
+	private Gauge<Integer> getMaxQueueLenGauge() {
+		return new Gauge<Integer>() {
+			@Override
+			public Integer getValue() {
+				return refreshAndGetMax();
+			}
+		};
+	}
+
+	private Gauge<Float> getAvgQueueLenGauge() {
+		return new Gauge<Float>() {
+			@Override
+			public Float getValue() {
+				return refreshAndGetAvg();
+			}
+		};
+	}
+
+	// ------------------------------------------------------------------------
+	//  Static access
+	// ------------------------------------------------------------------------
+
+	public static void registerQueueLengthMetrics(MetricGroup group, SingleInputGate gate) {
+		InputGateMetrics metrics = new InputGateMetrics(gate);
+
+		group.gauge("total-queue-len", metrics.getTotalQueueLenGauge());
+		group.gauge("min-queue-len", metrics.getMinQueueLenGauge());
+		group.gauge("max-queue-len", metrics.getMaxQueueLenGauge());
+		group.gauge("avg-queue-len", metrics.getAvgQueueLenGauge());
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
index ed3122ef2d3e6..719f340669b3d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
@@ -30,7 +30,6 @@
 
 import java.io.IOException;
 import java.util.ArrayDeque;
-import java.util.Queue;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -54,7 +53,7 @@ public class RemoteInputChannel extends InputChannel {
 	 * The received buffers. Received buffers are enqueued by the network I/O thread and the queue
 	 * is consumed by the receiving task thread.
 	 */
-	private final Queue<Buffer> receivedBuffers = new ArrayDeque<>();
+	private final ArrayDeque<Buffer> receivedBuffers = new ArrayDeque<>();
 
 	/**
 	 * Flag indicating whether this channel has been released. Either called by the receiving task
@@ -219,6 +218,10 @@ public int getNumberOfQueuedBuffers() {
 		}
 	}
 
+	public int unsynchronizedGetNumberOfQueuedBuffers() {
+		return Math.max(0, receivedBuffers.size());
+	}
+
 	public InputChannelID getInputChannelId() {
 		return id;
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
index afe872264a968..ebfb300a8e9d5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
@@ -18,8 +18,6 @@
 
 package org.apache.flink.runtime.io.network.partition.consumer;
 
-import com.google.common.collect.Maps;
-import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor;
 import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
@@ -48,6 +46,7 @@
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Timer;
@@ -188,7 +187,7 @@ public SingleInputGate(
 		checkArgument(numberOfInputChannels > 0);
 		this.numberOfInputChannels = numberOfInputChannels;
 
-		this.inputChannels = Maps.newHashMapWithExpectedSize(numberOfInputChannels);
+		this.inputChannels = new HashMap<>(numberOfInputChannels);
 		this.channelsWithEndOfPartitionEvents = new BitSet(numberOfInputChannels);
 
 		this.taskActions = checkNotNull(taskActions);
@@ -566,7 +565,6 @@ private void queueChannel(InputChannel channel) {
 
 	// ------------------------------------------------------------------------
 
-	@VisibleForTesting
 	Map<IntermediateResultPartitionID, InputChannel> getInputChannels() {
 		return inputChannels;
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index ef934de16eedc..f3dacc36271b9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -23,6 +23,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.TaskInfo;
 import org.apache.flink.api.common.cache.DistributedCache;
+import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.fs.FileSystemSafetyNet;
@@ -53,6 +54,8 @@
 import org.apache.flink.runtime.io.network.partition.ResultPartition;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionMetrics;
+import org.apache.flink.runtime.io.network.partition.consumer.InputGateMetrics;
 import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
 import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -386,6 +389,20 @@ public Task(
 			++counter;
 		}
 
+		// register detailed network metrics, if configured
+		if (tmConfig.getBoolean(ConfigConstants.NETWORK_DETAILED_METRICS_KEY, false)) {
+			// output metrics
+			for (int i = 0; i < producedPartitions.length; i++) {
+				ResultPartitionMetrics.registerQueueLengthMetrics(
+						metricGroup.addGroup("netout_" + i), producedPartitions[i]);
+			}
+
+			for (int i = 0; i < inputGates.length; i++) {
+				InputGateMetrics.registerQueueLengthMetrics(
+						metricGroup.addGroup("netin_" + i), inputGates[i]);
+			}
+		}
+
 		invokableHasBeenCanceled = new AtomicBoolean(false);
 
 		// finally, create the executing thread, but do not start it

From 0065371f6642d68f1a9168aa5b5dcb3443c05ca0 Mon Sep 17 00:00:00 2001
From: Nico Kruber <nico@data-artisans.com>
Date: Mon, 27 Feb 2017 11:14:35 +0100
Subject: [PATCH 068/253] [hotfix][metrics] remove an unnecessary check for
 non-null

The metrics group given to a Task must always be non-null otherwise the code
would have crashed anyway. Similarly, the group returned by getIOMetricGroup()
is always present.
---
 .../java/org/apache/flink/runtime/taskmanager/Task.java  | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index f3dacc36271b9..5a402fd666933 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -74,6 +74,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.IOException;
 import java.net.URL;
@@ -280,7 +281,7 @@ public Task(
 		LibraryCacheManager libraryCache,
 		FileCache fileCache,
 		TaskManagerRuntimeInfo taskManagerConfig,
-		TaskMetricGroup metricGroup,
+		@Nonnull TaskMetricGroup metricGroup,
 		ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
 		PartitionProducerStateChecker partitionProducerStateChecker,
 		Executor executor) {
@@ -408,10 +409,8 @@ public Task(
 		// finally, create the executing thread, but do not start it
 		executingThread = new Thread(TASK_THREADS_GROUP, this, taskNameWithSubtask);
 
-		if (this.metrics != null && this.metrics.getIOMetricGroup() != null) {
-			// add metrics for buffers
-			this.metrics.getIOMetricGroup().initializeBufferMetrics(this);
-		}
+		// add metrics for buffers
+		this.metrics.getIOMetricGroup().initializeBufferMetrics(this);
 	}
 
 	// ------------------------------------------------------------------------

From 7e79d60c1cbdd9b38caf87763fb0b224ee80c724 Mon Sep 17 00:00:00 2001
From: Nico Kruber <nico@data-artisans.com>
Date: Mon, 27 Feb 2017 17:30:59 +0100
Subject: [PATCH 069/253] [hotfix][metrics] keep the non-null assumption and
 implement tests properly

Previous commit in e908b62ab3 caused a regression leading to unit test failures.
---
 .../runtime/taskexecutor/TaskExecutorTest.java  | 17 +++++++++++++++--
 .../runtime/taskmanager/TaskAsyncCallTest.java  |  6 +++++-
 .../flink/runtime/taskmanager/TaskStopTest.java |  6 +++++-
 .../flink/runtime/taskmanager/TaskTest.java     |  7 +++++--
 4 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index bc6fe6801de1f..1d1840e8b5ca8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -59,6 +59,7 @@
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.memory.MemoryManager;
 import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
 import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
@@ -547,10 +548,13 @@ public void testTaskSubmission() throws Exception {
 
 		final TaskManagerMetricGroup taskManagerMetricGroup = mock(TaskManagerMetricGroup.class);
 
+		TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
+		when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));
+
 		when(taskManagerMetricGroup.addTaskForJob(
 				any(JobID.class), anyString(), any(JobVertexID.class), any(ExecutionAttemptID.class),
 				anyString(), anyInt(), anyInt())
-			).thenReturn(mock(TaskMetricGroup.class));
+			).thenReturn(taskMetricGroup);
 
 		final HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
 		when(haServices.getResourceManagerLeaderRetriever()).thenReturn(mock(LeaderRetrievalService.class));
@@ -1010,6 +1014,15 @@ public void testSubmitTaskBeforeAcceptSlot() throws Exception {
 		jobManagerTable.put(jobId, jobManagerConnection);
 
 		try {
+			final TaskManagerMetricGroup taskManagerMetricGroup = mock(TaskManagerMetricGroup.class);
+			TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
+			when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));
+
+			when(taskManagerMetricGroup.addTaskForJob(
+				any(JobID.class), anyString(), any(JobVertexID.class), any(ExecutionAttemptID.class),
+				anyString(), anyInt(), anyInt())
+			).thenReturn(taskMetricGroup);
+
 			final TaskExecutor taskManager = new TaskExecutor(
 				rpc,
 				taskManagerConfiguration,
@@ -1020,7 +1033,7 @@ public void testSubmitTaskBeforeAcceptSlot() throws Exception {
 				haServices,
 				mock(HeartbeatServices.class, RETURNS_MOCKS),
 				mock(MetricRegistry.class),
-				mock(TaskManagerMetricGroup.class),
+				taskManagerMetricGroup,
 				mock(BroadcastVariableManager.class),
 				mock(FileCache.class),
 				taskSlotTable,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
index 89ae5da72dd68..c6d2fec2f0daf 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
@@ -46,6 +46,7 @@
 import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
 import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
 import org.apache.flink.runtime.memory.MemoryManager;
+import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
 import org.apache.flink.runtime.state.TaskStateHandles;
@@ -157,6 +158,9 @@ private static Task createTask() throws Exception {
 		when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
 				.thenReturn(mock(TaskKvStateRegistry.class));
 
+		TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
+		when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));
+
 		JobInformation jobInformation = new JobInformation(
 			new JobID(),
 			"Job Name",
@@ -194,7 +198,7 @@ private static Task createTask() throws Exception {
 			libCache,
 			mock(FileCache.class),
 			new TestingTaskManagerRuntimeInfo(),
-			mock(TaskMetricGroup.class),
+			taskMetricGroup,
 			consumableNotifier,
 			partitionProducerStateChecker,
 			executor);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
index f3ac809f9ec23..40678de125424 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
@@ -29,6 +29,7 @@
 import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
 import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -68,6 +69,9 @@ public void doMocking(AbstractInvokable taskMock) throws Exception {
 		TaskManagerRuntimeInfo tmRuntimeInfo = mock(TaskManagerRuntimeInfo.class);
 		when(tmRuntimeInfo.getConfiguration()).thenReturn(new Configuration());
 
+		TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
+		when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));
+
 		task = new Task(
 			mock(JobInformation.class),
 			new TaskInformation(
@@ -95,7 +99,7 @@ public void doMocking(AbstractInvokable taskMock) throws Exception {
 			mock(LibraryCacheManager.class),
 			mock(FileCache.class),
 			tmRuntimeInfo,
-			mock(TaskMetricGroup.class),
+			taskMetricGroup,
 			mock(ResultPartitionConsumableNotifier.class),
 			mock(PartitionProducerStateChecker.class),
 			mock(Executor.class));
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index eb59b1bb7df5d..252228778895f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -53,9 +53,9 @@
 import org.apache.flink.runtime.memory.MemoryManager;
 import org.apache.flink.runtime.messages.TaskManagerMessages;
 import org.apache.flink.runtime.messages.TaskMessages;
+import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
 import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
-import org.apache.flink.runtime.util.EnvironmentInformation;
 import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
 import org.apache.flink.util.SerializedValue;
 import org.apache.flink.util.TestLogger;
@@ -941,6 +941,9 @@ private Task createTask(
 			1,
 			invokable.getName(),
 			new Configuration());
+
+		TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
+		when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));
 		
 		return new Task(
 			jobInformation,
@@ -963,7 +966,7 @@ private Task createTask(
 			libCache,
 			mock(FileCache.class),
 			new TestingTaskManagerRuntimeInfo(taskManagerConfig),
-			mock(TaskMetricGroup.class),
+			taskMetricGroup,
 			consumableNotifier,
 			partitionProducerStateChecker,
 			executor);

From 0f2c37eac3d81152c3a5a9852d1bcd38681bb5f8 Mon Sep 17 00:00:00 2001
From: Nico Kruber <nico@data-artisans.com>
Date: Tue, 21 Feb 2017 14:35:05 +0100
Subject: [PATCH 070/253] [FLINK-5090] Fixups for "Add metrics for details
 about inbound/outbound network queues"

This closes #3348.
---
 docs/monitoring/metrics.md                    |  20 ++-
 .../flink/configuration/ConfigConstants.java  |   6 -
 .../configuration/TaskManagerOptions.java     |   8 ++
 .../io/network/partition/ResultPartition.java |  10 ++
 .../partition/ResultPartitionMetrics.java     | 111 +++++++++++------
 .../partition/consumer/InputGateMetrics.java  | 115 +++++++++++-------
 .../flink/runtime/taskmanager/Task.java       |  41 ++++---
 7 files changed, 200 insertions(+), 111 deletions(-)

diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md
index e51f82ee9c9f3..290a4528c6989 100644
--- a/docs/monitoring/metrics.md
+++ b/docs/monitoring/metrics.md
@@ -640,7 +640,7 @@ Thus, in order to infer the metric identifier:
       <td>The number of allocated memory segments.</td>
     </tr>
     <tr>
-      <th rowspan="4">Task</th>
+      <th rowspan="8">Task</th>
       <td rowspan="4">buffers</td>
       <td>inputQueueLength</td>
       <td>The number of queued input buffers.</td>
@@ -657,6 +657,24 @@ Thus, in order to infer the metric identifier:
       <td>outPoolUsage</td>
       <td>An estimate of the output buffers usage.</td>
     </tr>
+    <tr>
+      <td rowspan="4">Network.&lt;Input|Output&gt;.&lt;gate&gt;<br />
+        <strong>(only available if <tt>taskmanager.net.detailed-metrics</tt> config option is set)</strong></td>
+      <td>totalQueueLen</td>
+      <td>Total number of queued buffers in all input/output channels.</td>
+    </tr>
+    <tr>
+      <td>minQueueLen</td>
+      <td>Minimum number of queued buffers in all input/output channels.</td>
+    </tr>
+    <tr>
+      <td>maxQueueLen</td>
+      <td>Maximum number of queued buffers in all input/output channels.</td>
+    </tr>
+    <tr>
+      <td>avgQueueLen</td>
+      <td>Average number of queued buffers in all input/output channels.</td>
+    </tr>
   </tbody>
 </table>
 
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 6863d14bad838..61c1b27ecbeed 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -250,12 +250,6 @@ public final class ConfigConstants {
 	@Deprecated
 	public static final String TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY = "taskmanager.network.numberOfBuffers";
 
-	/**
-	 * Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths
-	 */
-	@PublicEvolving
-	public static final String NETWORK_DETAILED_METRICS_KEY = "taskmanager.net.detailed-metrics";
-
 	/**
 	 * Config parameter defining the size of memory buffers used by the network stack and the memory manager.
 	 *
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
index 9a00a0ff5b60a..777ee215957f6 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
@@ -115,6 +115,14 @@ public class TaskManagerOptions {
 		key("taskmanager.net.memory.extra-buffers-per-gate")
 			.defaultValue(8);
 
+	/**
+	 * Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue
+	 * lengths.
+	 */
+	public static final ConfigOption<Boolean> NETWORK_DETAILED_METRICS =
+			key("taskmanager.net.detailed-metrics")
+			.defaultValue(false);
+
 	// ------------------------------------------------------------------------
 	//  Task Options
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
index 29a7cafbdf39a..d207f60137a61 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
@@ -221,10 +221,20 @@ public BufferPool getBufferPool() {
 		return bufferPool;
 	}
 
+	/**
+	 * Returns the total number of processed network buffers since initialization.
+	 *
+	 * @return overall number of processed network buffers
+	 */
 	public int getTotalNumberOfBuffers() {
 		return totalNumberOfBuffers;
 	}
 
+	/**
+	 * Returns the total size of processed network buffers since initialization.
+	 *
+	 * @return overall size of processed network buffers
+	 */
 	public long getTotalNumberOfBytes() {
 		return totalNumberOfBytes;
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java
index 0fd32107339a8..fde2ebd56e573 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionMetrics.java
@@ -23,16 +23,13 @@
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
+/**
+ * Collects metrics of a result partition.
+ */
 public class ResultPartitionMetrics {
 
 	private final ResultPartition partition;
 
-	private int lastMin = -1;
-
-	private int lastMax = -1;
-
-	private float lastAvg = -1.0f;
-
 	// ------------------------------------------------------------------------
 
 	private ResultPartitionMetrics(ResultPartition partition) {
@@ -41,60 +38,95 @@ private ResultPartitionMetrics(ResultPartition partition) {
 
 	// ------------------------------------------------------------------------
 
-	// these methods are package private to make access from the nested classes faster 
+	// these methods are package private to make access from the nested classes faster
 
-	int refreshAndGetMin() {
-		int min;
-		if ((min = lastMin) == -1) {
-			refresh();
-			min = lastMin;
+	/**
+	 * Iterates over all sub-partitions and collects the total number of queued buffers in a
+	 * best-effort way.
+	 *
+	 * @return total number of queued buffers
+	 */
+	long refreshAndGetTotal() {
+		long total = 0;
+
+		for (ResultSubpartition part : partition.getAllPartitions()) {
+			total += part.unsynchronizedGetNumberOfQueuedBuffers();
 		}
 
-		lastMin = -1;
-		return min;
+		return total;
 	}
 
-	int refreshAndGetMax() {
-		int max;
-		if ((max = lastMax) == -1) {
-			refresh();
-			max = lastMax;
-		}
+	/**
+	 * Iterates over all sub-partitions and collects the minimum number of queued buffers in a
+	 * sub-partition in a best-effort way.
+	 *
+	 * @return minimum number of queued buffers per sub-partition (<tt>0</tt> if sub-partitions exist)
+	 */
+	int refreshAndGetMin() {
+		int min = Integer.MAX_VALUE;
 
-		lastMax = -1;
-		return max;
-	}
+		ResultSubpartition[] allPartitions = partition.getAllPartitions();
+		if (allPartitions.length == 0) {
+			// meaningful value when no channels exist:
+			return 0;
+		}
 
-	float refreshAndGetAvg() {
-		float avg;
-		if ((avg = lastAvg) < 0.0f) {
-			refresh();
-			avg = lastAvg;
+		for (ResultSubpartition part : allPartitions) {
+			int size = part.unsynchronizedGetNumberOfQueuedBuffers();
+			min = Math.min(min, size);
 		}
 
-		lastAvg = -1.0f;
-		return avg;
+		return min;
 	}
 
-	private void refresh() {
-		int min = Integer.MAX_VALUE;
+	/**
+	 * Iterates over all sub-partitions and collects the maximum number of queued buffers in a
+	 * sub-partition in a best-effort way.
+	 *
+	 * @return maximum number of queued buffers per sub-partition
+	 */
+	int refreshAndGetMax() {
 		int max = 0;
 
 		for (ResultSubpartition part : partition.getAllPartitions()) {
 			int size = part.unsynchronizedGetNumberOfQueuedBuffers();
-			min = Math.min(min, size);
 			max = Math.max(max, size);
 		}
 
-		this.lastMin = min;
-		this.lastMax = max;
-		this.lastAvg = partition.getTotalNumberOfBuffers() / (float) partition.getNumberOfSubpartitions();
+		return max;
+	}
+
+	/**
+	 * Iterates over all sub-partitions and collects the average number of queued buffers in a
+	 * sub-partition in a best-effort way.
+	 *
+	 * @return average number of queued buffers per sub-partition
+	 */
+	float refreshAndGetAvg() {
+		long total = 0;
+
+		ResultSubpartition[] allPartitions = partition.getAllPartitions();
+		for (ResultSubpartition part : allPartitions) {
+			int size = part.unsynchronizedGetNumberOfQueuedBuffers();
+			total += size;
+		}
+
+		return total / (float) allPartitions.length;
 	}
 
 	// ------------------------------------------------------------------------
 	//  Gauges to access the stats
 	// ------------------------------------------------------------------------
 
+	private Gauge<Long> getTotalQueueLenGauge() {
+		return new Gauge<Long>() {
+			@Override
+			public Long getValue() {
+				return refreshAndGetTotal();
+			}
+		};
+	}
+
 	private Gauge<Integer> getMinQueueLenGauge() {
 		return new Gauge<Integer>() {
 			@Override
@@ -129,8 +161,9 @@ public Float getValue() {
 	public static void registerQueueLengthMetrics(MetricGroup group, ResultPartition partition) {
 		ResultPartitionMetrics metrics = new ResultPartitionMetrics(partition);
 
-		group.gauge("min-queue-len", metrics.getMinQueueLenGauge());
-		group.gauge("max-queue-len", metrics.getMaxQueueLenGauge());
-		group.gauge("avg-queue-len", metrics.getAvgQueueLenGauge());
+		group.gauge("totalQueueLen", metrics.getTotalQueueLenGauge());
+		group.gauge("minQueueLen", metrics.getMinQueueLenGauge());
+		group.gauge("maxQueueLen", metrics.getMaxQueueLenGauge());
+		group.gauge("avgQueueLen", metrics.getAvgQueueLenGauge());
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java
index b5eb8601029a8..c9fed9bb93139 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateMetrics.java
@@ -21,20 +21,17 @@
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.metrics.MetricGroup;
 
+import java.util.Collection;
+
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
+/**
+ * Collects metrics of an input gate.
+ */
 public class InputGateMetrics {
 
 	private final SingleInputGate inputGate;
 
-	private long lastTotal = -1;
-
-	private int lastMin = -1;
-
-	private int lastMax = -1;
-
-	private float lastAvg = -1.0f;
-
 	// ------------------------------------------------------------------------
 
 	private InputGateMetrics(SingleInputGate inputGate) {
@@ -45,71 +42,95 @@ private InputGateMetrics(SingleInputGate inputGate) {
 
 	// these methods are package private to make access from the nested classes faster 
 
+	/**
+	 * Iterates over all input channels and collects the total number of queued buffers in a
+	 * best-effort way.
+	 *
+	 * @return total number of queued buffers
+	 */
 	long refreshAndGetTotal() {
-		long total;
-		if ((total = lastTotal) == -1) {
-			refresh();
-			total = lastTotal;
+		long total = 0;
+
+		for (InputChannel channel : inputGate.getInputChannels().values()) {
+			if (channel instanceof RemoteInputChannel) {
+				RemoteInputChannel rc = (RemoteInputChannel) channel;
+
+				total += rc.unsynchronizedGetNumberOfQueuedBuffers();
+			}
 		}
 
-		lastTotal = -1;
 		return total;
 	}
 
+	/**
+	 * Iterates over all input channels and collects the minimum number of queued buffers in a
+	 * channel in a best-effort way.
+	 *
+	 * @return minimum number of queued buffers per channel (<tt>0</tt> if no channels exist)
+	 */
 	int refreshAndGetMin() {
-		int min;
-		if ((min = lastMin) == -1) {
-			refresh();
-			min = lastMin;
+		int min = Integer.MAX_VALUE;
+
+		Collection<InputChannel> channels = inputGate.getInputChannels().values();
+		if (channels.isEmpty()) {
+			// meaningful value when no channels exist:
+			return 0;
+		}
+
+		for (InputChannel channel : channels) {
+			if (channel instanceof RemoteInputChannel) {
+				RemoteInputChannel rc = (RemoteInputChannel) channel;
+
+				int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
+				min = Math.min(min, size);
+			}
 		}
 
-		lastMin = -1;
 		return min;
 	}
 
+	/**
+	 * Iterates over all input channels and collects the maximum number of queued buffers in a
+	 * channel in a best-effort way.
+	 *
+	 * @return maximum number of queued buffers per channel
+	 */
 	int refreshAndGetMax() {
-		int max;
-		if ((max = lastMax) == -1) {
-			refresh();
-			max = lastMax;
-		}
+		int max = 0;
 
-		lastMax = -1;
-		return max;
-	}
+		for (InputChannel channel : inputGate.getInputChannels().values()) {
+			if (channel instanceof RemoteInputChannel) {
+				RemoteInputChannel rc = (RemoteInputChannel) channel;
 
-	float refreshAndGetAvg() {
-		float avg;
-		if ((avg = lastAvg) < 0.0f) {
-			refresh();
-			avg = lastAvg;
+				int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
+				max = Math.max(max, size);
+			}
 		}
 
-		lastAvg = -1.0f;
-		return avg;
+		return max;
 	}
 
-	private void refresh() {
+	/**
+	 * Iterates over all input channels and collects the average number of queued buffers in a
+	 * channel in a best-effort way.
+	 *
+	 * @return average number of queued buffers per channel
+	 */
+	float refreshAndGetAvg() {
 		long total = 0;
-		int min = Integer.MAX_VALUE;
-		int max = 0;
 		int count = 0;
 
 		for (InputChannel channel : inputGate.getInputChannels().values()) {
-			if (channel.getClass() == RemoteInputChannel.class) {
+			if (channel instanceof RemoteInputChannel) {
 				RemoteInputChannel rc = (RemoteInputChannel) channel;
 
 				int size = rc.unsynchronizedGetNumberOfQueuedBuffers();
 				total += size;
-				min = Math.min(min, size);
-				max = Math.max(max, size);
-				count++;
+				++count;
 			}
 		}
 
-		this.lastMin = min;
-		this.lastMax = max;
-		this.lastAvg = total / (float) count;
+		return total / (float) count;
 	}
 
 	// ------------------------------------------------------------------------
@@ -159,9 +180,9 @@ public Float getValue() {
 	public static void registerQueueLengthMetrics(MetricGroup group, SingleInputGate gate) {
 		InputGateMetrics metrics = new InputGateMetrics(gate);
 
-		group.gauge("total-queue-len", metrics.getTotalQueueLenGauge());
-		group.gauge("min-queue-len", metrics.getMinQueueLenGauge());
-		group.gauge("max-queue-len", metrics.getMaxQueueLenGauge());
-		group.gauge("avg-queue-len", metrics.getAvgQueueLenGauge());
+		group.gauge("totalQueueLen", metrics.getTotalQueueLenGauge());
+		group.gauge("minQueueLen", metrics.getMinQueueLenGauge());
+		group.gauge("maxQueueLen", metrics.getMaxQueueLenGauge());
+		group.gauge("avgQueueLen", metrics.getAvgQueueLenGauge());
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 5a402fd666933..dab0f9505a1cd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -23,11 +23,11 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.TaskInfo;
 import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.fs.FileSystemSafetyNet;
 import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.blob.BlobKey;
 import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
@@ -390,27 +390,10 @@ public Task(
 			++counter;
 		}
 
-		// register detailed network metrics, if configured
-		if (tmConfig.getBoolean(ConfigConstants.NETWORK_DETAILED_METRICS_KEY, false)) {
-			// output metrics
-			for (int i = 0; i < producedPartitions.length; i++) {
-				ResultPartitionMetrics.registerQueueLengthMetrics(
-						metricGroup.addGroup("netout_" + i), producedPartitions[i]);
-			}
-
-			for (int i = 0; i < inputGates.length; i++) {
-				InputGateMetrics.registerQueueLengthMetrics(
-						metricGroup.addGroup("netin_" + i), inputGates[i]);
-			}
-		}
-
 		invokableHasBeenCanceled = new AtomicBoolean(false);
 
 		// finally, create the executing thread, but do not start it
 		executingThread = new Thread(TASK_THREADS_GROUP, this, taskNameWithSubtask);
-
-		// add metrics for buffers
-		this.metrics.getIOMetricGroup().initializeBufferMetrics(this);
 	}
 
 	// ------------------------------------------------------------------------
@@ -616,6 +599,28 @@ else if (current == ExecutionState.CANCELING) {
 
 			network.registerTask(this);
 
+			// add metrics for buffers
+			this.metrics.getIOMetricGroup().initializeBufferMetrics(this);
+
+			// register detailed network metrics, if configured
+			if (taskManagerConfig.getConfiguration().getBoolean(TaskManagerOptions.NETWORK_DETAILED_METRICS)) {
+				// similar to MetricUtils.instantiateNetworkMetrics() but inside this IOMetricGroup
+				MetricGroup networkGroup = this.metrics.getIOMetricGroup().addGroup("Network");
+				MetricGroup outputGroup = networkGroup.addGroup("Output");
+				MetricGroup inputGroup = networkGroup.addGroup("Input");
+
+				// output metrics
+				for (int i = 0; i < producedPartitions.length; i++) {
+					ResultPartitionMetrics.registerQueueLengthMetrics(
+						outputGroup.addGroup(i), producedPartitions[i]);
+				}
+
+				for (int i = 0; i < inputGates.length; i++) {
+					InputGateMetrics.registerQueueLengthMetrics(
+						inputGroup.addGroup(i), inputGates[i]);
+				}
+			}
+
 			// next, kick off the background copying of files for the distributed cache
 			try {
 				for (Map.Entry<String, DistributedCache.DistributedCacheEntry> entry :

From 25096d5aaf6446a5a2639c818dcdd92fec69d386 Mon Sep 17 00:00:00 2001
From: sunjincheng121 <sunjincheng121@gmail.com>
Date: Fri, 31 Mar 2017 16:31:18 +0800
Subject: [PATCH 071/253] [FLINK-6247] [table] Put flink-table.jar with all
 dependencies into ./opt folder.

This closes #3666.
---
 .../flink-connector-kafka-0.10/pom.xml        |  9 ++++++++
 flink-connectors/flink-hbase/pom.xml          | 22 +++++++++++++++++++
 flink-dist/pom.xml                            |  7 ++++++
 flink-dist/src/main/assemblies/opt.xml        |  8 +++++++
 flink-examples/flink-examples-table/pom.xml   |  9 ++++++++
 flink-libraries/flink-table/pom.xml           | 10 +++++++--
 6 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml
index ab60946b9e758..9fb0e613bc18b 100644
--- a/flink-connectors/flink-connector-kafka-0.10/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml
@@ -56,6 +56,15 @@ under the License.
 			</exclusions>
 		</dependency>
 
+		<!-- streaming-java dependencies -->
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-streaming-java_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
+
 		<!-- Add Kafka 0.10.x as a dependency -->
 
 		<dependency>
diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml
index 00c08ef9f2beb..4fc05568d3fe6 100644
--- a/flink-connectors/flink-hbase/pom.xml
+++ b/flink-connectors/flink-hbase/pom.xml
@@ -76,6 +76,8 @@ under the License.
 			<scope>provided</scope>
 		</dependency>
 
+		<!-- flink-java dependencies -->
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-java</artifactId>
@@ -83,6 +85,25 @@ under the License.
 			<scope>provided</scope>
 		</dependency>
 
+		<!-- flink-scala dependencies -->
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-scala_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
+
+		<!-- streaming-scala dependencies -->
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
+
+		<!-- table dependencies -->
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
@@ -93,6 +114,7 @@ under the License.
 			won't depend on flink-table. -->
 			<optional>true</optional>
 		</dependency>
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-shaded-hadoop2</artifactId>
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 76df09e4f1769..a700870624d41 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -239,6 +239,13 @@ under the License.
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-table_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
 		<!-- end optional Flink libraries -->
 	</dependencies>
 
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 3622ece29d348..0282315fc9854 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -59,6 +59,14 @@
 			<fileMode>0644</fileMode>
 		</file>
 
+		<!-- TableAPI-->
+		<file>
+			<source>../flink-libraries/flink-table/target/flink-table_2.10-${project.version}.jar</source>
+			<outputDirectory>opt/</outputDirectory>
+			<destName>flink-table_2.10-${project.version}.jar</destName>
+			<fileMode>0644</fileMode>
+		</file>
+
 		<!-- ML -->
 		<file>
 			<source>../flink-libraries/flink-ml/target/flink-ml_2.10-${project.version}-jar-with-dependencies.jar</source>
diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml
index de050d742b004..684d883f3d599 100644
--- a/flink-examples/flink-examples-table/pom.xml
+++ b/flink-examples/flink-examples-table/pom.xml
@@ -34,10 +34,19 @@ under the License.
 	<packaging>jar</packaging>
 
 	<dependencies>
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-table_2.10</artifactId>
 			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
 		</dependency>
 	</dependencies>
 
diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml
index 49eb45165c7a9..c7440ab456786 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -40,6 +40,7 @@ under the License.
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-streaming-scala_2.10</artifactId>
 			<version>${project.version}</version>
+			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
@@ -195,13 +196,17 @@ under the License.
 						<configuration>
 							<filters>
 								<filter>
+									<artifact>*:*</artifact>
 									<excludes>
-									    	<exclude>org-apache-calcite-jdbc.properties</exclude>
+										<exclude>org-apache-calcite-jdbc.properties</exclude>
 										<exclude>mozilla/**</exclude>
 										<exclude>codegen/**</exclude>
 										<exclude>google/**</exclude>
+										<exclude>META-INF/*.SF</exclude>
+										<exclude>META-INF/*.DSA</exclude>
+										<exclude>META-INF/*.RSA</exclude>
 									</excludes>
-								 </filter>
+								</filter>
 							</filters>
 							<artifactSet>
 								<includes combine.children="append">
@@ -209,6 +214,7 @@ under the License.
 									<include>org.apache.calcite.avatica:*</include>
 									<include>net.hydromatic:*</include>
                                     <include>org.reflections:*</include>
+									<include>org.codehaus.janino:*</include>
                                 </includes>
 							</artifactSet>
 							<relocations>

From 0e90115ddc8e10b397a438eed1659fc1400aa805 Mon Sep 17 00:00:00 2001
From: sunjincheng121 <sunjincheng121@gmail.com>
Date: Thu, 27 Apr 2017 09:58:18 +0800
Subject: [PATCH 072/253] [FLINK-6392] [table] Improve group window API to make
 alias mandatory.

This closes #3786.
---
 .../apache/flink/table/api/java/windows.scala |  18 +-
 .../flink/table/api/scala/windows.scala       |  18 +-
 .../org/apache/flink/table/api/table.scala    |   5 +-
 .../org/apache/flink/table/api/windows.scala  | 297 +++++++++++-------
 .../table/plan/logical/LogicalWindow.scala    |   7 +-
 .../table/plan/logical/groupWindows.scala     |  28 +-
 .../flink/table/plan/logical/operators.scala  |   2 +-
 .../common/WindowStartEndPropertiesRule.scala |   4 +-
 .../scala/batch/sql/WindowAggregateTest.scala |  12 +-
 .../batch/table/FieldProjectionTest.scala     |   4 +-
 .../scala/batch/table/GroupWindowTest.scala   |  18 +-
 .../stream/sql/WindowAggregateTest.scala      |   6 +-
 .../scala/stream/table/GroupWindowTest.scala  |  48 +--
 13 files changed, 274 insertions(+), 193 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
index 1e9d65a523336..15208ceb6a981 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
@@ -18,7 +18,7 @@
 
 package org.apache.flink.table.api.java
 
-import org.apache.flink.table.api.{OverWindowPredefined, SessionWindow, SlideWithSize, TumblingWindow}
+import org.apache.flink.table.api.{TumbleWithSize, OverWindowWithOrderBy, SlideWithSize, SessionWithGap}
 import org.apache.flink.table.expressions.{Expression, ExpressionParser}
 
 /**
@@ -34,9 +34,9 @@ object Tumble {
     * elements in 5 minutes intervals.
     *
     * @param size the size of the window as time or row-count interval.
-    * @return a tumbling window
+    * @return a partially defined tumbling window
     */
-  def over(size: String): TumblingWindow = new TumblingWindow(size)
+  def over(size: String): TumbleWithSize = new TumbleWithSize(size)
 }
 
 /**
@@ -79,9 +79,9 @@ object Session {
     *
     * @param gap specifies how long (as interval of milliseconds) to wait for new data before
     *            closing the session window.
-    * @return a session window
+    * @return a partially defined session window
     */
-  def withGap(gap: String): SessionWindow = new SessionWindow(gap)
+  def withGap(gap: String): SessionWithGap = new SessionWithGap(gap)
 }
 
 /**
@@ -96,9 +96,9 @@ object Over {
     *
     * For batch tables, refer to a timestamp or long attribute.
     */
-  def orderBy(orderBy: String): OverWindowPredefined = {
+  def orderBy(orderBy: String): OverWindowWithOrderBy = {
     val orderByExpr = ExpressionParser.parseExpression(orderBy)
-    new OverWindowPredefined(Seq[Expression](), orderByExpr)
+    new OverWindowWithOrderBy(Seq[Expression](), orderByExpr)
   }
 
   /**
@@ -122,8 +122,8 @@ class PartitionedOver(private val partitionByExpr: Array[Expression]) {
     *
     * For batch tables, refer to a timestamp or long attribute.
     */
-  def orderBy(orderBy: String): OverWindowPredefined = {
+  def orderBy(orderBy: String): OverWindowWithOrderBy = {
     val orderByExpr = ExpressionParser.parseExpression(orderBy)
-    new OverWindowPredefined(partitionByExpr, orderByExpr)
+    new OverWindowWithOrderBy(partitionByExpr, orderByExpr)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
index 12300276c01b8..5e7044053f38b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
@@ -19,7 +19,7 @@
 package org.apache.flink.table.api.scala
 
 import org.apache.flink.table.expressions.Expression
-import org.apache.flink.table.api.{OverWindowPredefined, SessionWindow, SlideWithSize, TumblingWindow}
+import org.apache.flink.table.api.{TumbleWithSize, OverWindowWithOrderBy, SlideWithSize, SessionWithGap}
 
 /**
   * Helper object for creating a tumbling window. Tumbling windows are consecutive, non-overlapping
@@ -34,9 +34,9 @@ object Tumble {
     * elements in 5 minutes intervals.
     *
     * @param size the size of the window as time or row-count interval.
-    * @return a tumbling window
+    * @return a partially defined tumbling window
     */
-  def over(size: Expression): TumblingWindow = new TumblingWindow(size)
+  def over(size: Expression): TumbleWithSize = new TumbleWithSize(size)
 }
 
 /**
@@ -79,9 +79,9 @@ object Session {
     *
     * @param gap specifies how long (as interval of milliseconds) to wait for new data before
     *            closing the session window.
-    * @return a session window
+    * @return a partially defined session window
     */
-  def withGap(gap: Expression): SessionWindow = new SessionWindow(gap)
+  def withGap(gap: Expression): SessionWithGap = new SessionWithGap(gap)
 }
 
 /**
@@ -96,8 +96,8 @@ object Over {
     *
     * For batch tables, refer to a timestamp or long attribute.
     */
-  def orderBy(orderBy: Expression): OverWindowPredefined = {
-    new OverWindowPredefined(Seq[Expression](), orderBy)
+  def orderBy(orderBy: Expression): OverWindowWithOrderBy = {
+    new OverWindowWithOrderBy(Seq[Expression](), orderBy)
   }
 
   /**
@@ -120,7 +120,7 @@ case class PartitionedOver(partitionBy: Array[Expression]) {
     *
     * For batch tables, refer to a timestamp or long attribute.
     */
-  def orderBy(orderBy: Expression): OverWindowPredefined = {
-    new OverWindowPredefined(partitionBy, orderBy)
+  def orderBy(orderBy: Expression): OverWindowWithOrderBy = {
+    new OverWindowWithOrderBy(partitionBy, orderBy)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index c688aabfb6a62..095361152aa26 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -805,9 +805,6 @@ class Table(
     * @return A windowed table.
     */
   def window(window: Window): WindowedTable = {
-    if (window.alias.isEmpty) {
-      throw new ValidationException("An alias must be specified for the window.")
-    }
     new WindowedTable(this, window)
   }
 
@@ -936,7 +933,7 @@ class WindowedTable(
     * }}}
     */
   def groupBy(fields: Expression*): WindowGroupedTable = {
-    val fieldsWithoutWindow = fields.filterNot(window.alias.get.equals(_))
+    val fieldsWithoutWindow = fields.filterNot(window.alias.equals(_))
     if (fields.size != fieldsWithoutWindow.size + 1) {
       throw new ValidationException("GroupBy must contain exactly one window alias.")
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
index 577d77c377b47..80260f7d80e22 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
@@ -35,32 +35,40 @@ case class OverWindow(
 
 case class CurrentRow() extends Expression {
   override private[flink] def resultType = RowIntervalTypeInfo.INTERVAL_ROWS
+
   override private[flink] def children = Seq()
+
   override def toString = "CURRENT ROW"
 }
 
 case class CurrentRange() extends Expression {
   override private[flink] def resultType = TimeIntervalTypeInfo.INTERVAL_MILLIS
+
   override private[flink] def children = Seq()
+
   override def toString = "CURRENT RANGE"
 }
 
 case class UnboundedRow() extends Expression {
   override private[flink] def resultType = RowIntervalTypeInfo.INTERVAL_ROWS
+
   override private[flink] def children = Seq()
+
   override def toString = "UNBOUNDED ROW"
 }
 
 case class UnboundedRange() extends Expression {
   override private[flink] def resultType = TimeIntervalTypeInfo.INTERVAL_MILLIS
+
   override private[flink] def children = Seq()
+
   override def toString = "UNBOUNDED RANGE"
 }
 
 /**
-  * An over window predefined  specification.
+  * A partially defined over window.
   */
-class OverWindowPredefined(
+class OverWindowWithOrderBy(
     private val partitionBy: Seq[Expression],
     private val orderBy: Expression) {
 
@@ -100,7 +108,7 @@ class OverWindowPredefined(
     * @param preceding preceding offset relative to the current row.
     * @return this over window
     */
-  def preceding(preceding: String): OverWindowPredefined = {
+  def preceding(preceding: String): OverWindowWithOrderBy = {
     this.preceding(ExpressionParser.parseExpression(preceding))
   }
 
@@ -110,7 +118,7 @@ class OverWindowPredefined(
     * @param preceding preceding offset relative to the current row.
     * @return this over window
     */
-  def preceding(preceding: Expression): OverWindowPredefined = {
+  def preceding(preceding: Expression): OverWindowWithOrderBy = {
     this.preceding = preceding
     this
   }
@@ -121,7 +129,7 @@ class OverWindowPredefined(
     * @param following following offset that relative to the current row.
     * @return this over window
     */
-  def following(following: String): OverWindowPredefined = {
+  def following(following: String): OverWindowWithOrderBy = {
     this.following(ExpressionParser.parseExpression(following))
   }
 
@@ -131,7 +139,7 @@ class OverWindowPredefined(
     * @param following following offset that relative to the current row.
     * @return this over window
     */
-  def following(following: Expression): OverWindowPredefined = {
+  def following(following: Expression): OverWindowWithOrderBy = {
     this.following = following
     this
   }
@@ -149,15 +157,20 @@ class OverWindowPredefined(
   *
   * For finite batch tables, window provides shortcuts for time-based groupBy.
   *
+  * @param alias The expression of alias for this Window
   */
-abstract class Window {
+abstract class Window(val alias: Expression) {
 
-  // The expression of alias for this Window
-  private[flink] var alias: Option[Expression] = None
   /**
     * Converts an API class to a logical window for planning.
     */
   private[flink] def toLogicalWindow: LogicalWindow
+}
+
+/**
+  * A window specification without alias.
+  */
+abstract class WindowWithoutAlias {
 
   /**
     * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
@@ -166,10 +179,7 @@ abstract class Window {
     * @param alias alias for this window
     * @return this window
     */
-  def as(alias: Expression): Window = {
-    this.alias = Some(alias)
-    this
-  }
+  def as(alias: Expression): Window
 
   /**
     * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
@@ -182,36 +192,22 @@ abstract class Window {
 }
 
 /**
-  * A window operating on event-time.
-  *
-  * @param timeField defines the time mode for streaming tables. For batch table it defines the
-  *                  time attribute on which is grouped.
-  */
-abstract class EventTimeWindow(val timeField: Expression) extends Window
-
-// ------------------------------------------------------------------------------------------------
-// Tumbling windows
-// ------------------------------------------------------------------------------------------------
-
-/**
-  * Tumbling window.
-  *
-  * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows are
-  * grouped by processing-time.
-  *
-  * @param size the size of the window either as time or row-count interval.
+  * A predefined specification of window on processing-time
   */
-class TumblingWindow(size: Expression) extends Window {
+abstract class ProcTimeWindowWithoutAlias extends WindowWithoutAlias {
 
   /**
-    * Tumbling window.
+    * Specifies the time attribute on which rows are grouped.
     *
     * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
     * are grouped by processing-time.
     *
-    * @param size the size of the window either as time or row-count interval.
+    * For batch tables, refer to a timestamp or long attribute.
+    *
+    * @param timeField time mode for streaming tables and time attribute for batch tables
+    * @return a predefined window on event-time
     */
-  def this(size: String) = this(ExpressionParser.parseExpression(size))
+  def on(timeField: Expression): WindowWithoutAlias
 
   /**
     * Specifies the time attribute on which rows are grouped.
@@ -222,24 +218,85 @@ class TumblingWindow(size: Expression) extends Window {
     * For batch tables, refer to a timestamp or long attribute.
     *
     * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a tumbling window on event-time
+    * @return a predefined window on event-time
     */
-  def on(timeField: Expression): TumblingEventTimeWindow =
-    new TumblingEventTimeWindow(timeField, size)
+  def on(timeField: String): WindowWithoutAlias =
+    on(ExpressionParser.parseExpression(timeField))
+}
+
+/**
+  * A window operating on event-time.
+  *
+  * For streaming tables call on('rowtime) to specify grouping by event-time.
+  * Otherwise rows are grouped by processing-time.
+  *
+  * For batch tables, refer to a timestamp or long attribute.
+  *
+  * @param timeField time mode for streaming tables and time attribute for batch tables
+  */
+abstract class EventTimeWindow(alias: Expression, val timeField: Expression) extends Window(alias)
+
+// ------------------------------------------------------------------------------------------------
+// Tumbling windows
+// ------------------------------------------------------------------------------------------------
+
+/**
+  * A partial specification of a tumbling window.
+  *
+  * @param size the size of the window either a time or a row-count interval.
+  */
+class TumbleWithSize(size: Expression) extends ProcTimeWindowWithoutAlias {
+
+  def this(size: String) = this(ExpressionParser.parseExpression(size))
 
   /**
     * Specifies the time attribute on which rows are grouped.
     *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
+    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time.
+    * Otherwise rows are grouped by processing-time.
     *
     * For batch tables, refer to a timestamp or long attribute.
     *
     * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a tumbling window on event-time
+    * @return a predefined window on event-time
     */
-  def on(timeField: String): TumblingEventTimeWindow =
-    on(ExpressionParser.parseExpression(timeField))
+  override def on(timeField: Expression): WindowWithoutAlias =
+    new TumbleWithoutAlias(timeField, size)
+
+  /**
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
+    *
+    * @param alias alias for this window
+    * @return this window
+    */
+  override def as(alias: Expression) = new TumblingWindow(alias, size)
+}
+
+/**
+  * A tumbling window on event-time without alias.
+  */
+class TumbleWithoutAlias(
+    time: Expression,
+    size: Expression) extends WindowWithoutAlias {
+
+  /**
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
+    *
+    * @param alias alias for this window
+    * @return this window
+    */
+  override def as(alias: Expression): Window = new TumblingEventTimeWindow(alias, time, size)
+}
+
+/**
+  * Tumbling window on processing-time.
+  *
+  * @param alias the alias of the window.
+  * @param size the size of the window either a time or a row-count interval.
+  */
+class TumblingWindow(alias: Expression, size: Expression) extends Window(alias) {
 
   override private[flink] def toLogicalWindow: LogicalWindow =
     ProcessingTimeTumblingGroupWindow(alias, size)
@@ -249,9 +306,9 @@ class TumblingWindow(size: Expression) extends Window {
   * Tumbling window on event-time.
   */
 class TumblingEventTimeWindow(
+    alias: Expression,
     time: Expression,
-    size: Expression)
-  extends EventTimeWindow(time) {
+    size: Expression) extends EventTimeWindow(alias, time) {
 
   override private[flink] def toLogicalWindow: LogicalWindow =
     EventTimeTumblingGroupWindow(alias, time, size)
@@ -262,16 +319,16 @@ class TumblingEventTimeWindow(
 // ------------------------------------------------------------------------------------------------
 
 /**
-  * Partially specified sliding window.
+  * A partially specified sliding window.
   *
-  * @param size the size of the window either as time or row-count interval.
+  * @param size the size of the window either a time or a row-count interval.
   */
 class SlideWithSize(size: Expression) {
 
   /**
-    * Partially specified sliding window.
+    * A partially specified sliding window.
     *
-    * @param size the size of the window either as time or row-count interval.
+    * @param size the size of the window either a time or a row-count interval.
     */
   def this(size: String) = this(ExpressionParser.parseExpression(size))
 
@@ -286,9 +343,9 @@ class SlideWithSize(size: Expression) {
     * windows.
     *
     * @param slide the slide of the window either as time or row-count interval.
-    * @return a sliding window
+    * @return a predefined sliding window.
     */
-  def every(slide: Expression): SlidingWindow = new SlidingWindow(size, slide)
+  def every(slide: Expression): SlideWithSlide = new SlideWithSlide(size, slide)
 
   /**
     * Specifies the window's slide as time or row-count interval.
@@ -301,24 +358,17 @@ class SlideWithSize(size: Expression) {
     * windows.
     *
     * @param slide the slide of the window either as time or row-count interval.
-    * @return a sliding window
+    * @return a predefined sliding window.
     */
-  def every(slide: String): SlidingWindow = every(ExpressionParser.parseExpression(slide))
+  def every(slide: String): WindowWithoutAlias = every(ExpressionParser.parseExpression(slide))
 }
 
 /**
-  * Sliding window.
-  *
-  * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows are
-  * grouped by processing-time.
-  *
-  * @param size the size of the window either as time or row-count interval.
+  * A partially defined sliding window.
   */
-class SlidingWindow(
+class SlideWithSlide(
     size: Expression,
-    slide: Expression)
-  extends Window {
-
+    slide: Expression) extends ProcTimeWindowWithoutAlias {
   /**
     * Specifies the time attribute on which rows are grouped.
     *
@@ -328,37 +378,65 @@ class SlidingWindow(
     * For batch tables, refer to a timestamp or long attribute.
     *
     * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a sliding window on event-time
+    * @return a predefined Sliding window on event-time.
     */
-  def on(timeField: Expression): SlidingEventTimeWindow =
-    new SlidingEventTimeWindow(timeField, size, slide)
+  override def on(timeField: Expression): SlideWithoutAlias =
+    new SlideWithoutAlias(timeField, size, slide)
 
   /**
-    * Specifies the time attribute on which rows are grouped.
-    *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
     *
-    * For batch tables, refer to a timestamp or long attribute.
+    * @param alias alias for this window
+    * @return this window
+    */
+  override def as(alias: Expression): Window = new SlidingWindow(alias, size, slide)
+}
+
+/**
+  * A partially defined sliding window on event-time without alias.
+  */
+class SlideWithoutAlias(
+    timeField: Expression,
+    size: Expression,
+    slide: Expression) extends WindowWithoutAlias {
+  /**
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
     *
-    * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a sliding window on event-time
+    * @param alias alias for this window
+    * @return this window
     */
-  def on(timeField: String): SlidingEventTimeWindow =
-    on(ExpressionParser.parseExpression(timeField))
+  override def as(alias: Expression): Window =
+    new SlidingEventTimeWindow(alias, timeField, size, slide)
+}
+
+/**
+  * A sliding window on processing-time.
+  *
+  * @param alias the alias of the window.
+  * @param size the size of the window either a time or a row-count interval.
+  * @param slide the interval by which the window slides.
+  */
+class SlidingWindow(
+  alias: Expression,
+  size: Expression,
+  slide: Expression)
+  extends Window(alias) {
 
   override private[flink] def toLogicalWindow: LogicalWindow =
     ProcessingTimeSlidingGroupWindow(alias, size, slide)
 }
 
 /**
-  * Sliding window on event-time.
+  * A sliding window on event-time.
   */
 class SlidingEventTimeWindow(
+    alias: Expression,
     timeField: Expression,
     size: Expression,
     slide: Expression)
-  extends EventTimeWindow(timeField) {
+  extends EventTimeWindow(alias, timeField) {
 
   override private[flink] def toLogicalWindow: LogicalWindow =
     EventTimeSlidingGroupWindow(alias, timeField, size, slide)
@@ -369,23 +447,10 @@ class SlidingEventTimeWindow(
 // ------------------------------------------------------------------------------------------------
 
 /**
-  * Session window.
-  *
-  * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows are
-  * grouped by processing-time.
-  *
-  * @param gap the time interval of inactivity before a window is closed.
+  * A partially defined session window.
   */
-class SessionWindow(gap: Expression) extends Window {
+class SessionWithGap(gap: Expression) extends ProcTimeWindowWithoutAlias {
 
-  /**
-    * Session window.
-    *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
-    *
-    * @param gap the time interval of inactivity before a window is closed.
-    */
   def this(gap: String) = this(ExpressionParser.parseExpression(gap))
 
   /**
@@ -397,36 +462,56 @@ class SessionWindow(gap: Expression) extends Window {
     * For batch tables, refer to a timestamp or long attribute.
     *
     * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a session window on event-time
+    * @return an on event-time session window on event-time
     */
-  def on(timeField: Expression): SessionEventTimeWindow =
-    new SessionEventTimeWindow(timeField, gap)
+  override def on(timeField: Expression): SessionWithoutAlias =
+    new SessionWithoutAlias(timeField, gap)
 
   /**
-    * Specifies the time attribute on which rows are grouped.
-    *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
     *
-    * For batch tables, refer to a timestamp or long attribute.
+    * @param alias alias for this window
+    * @return this window
+    */
+  override def as(alias: Expression): Window = new SessionWindow(alias, gap)
+}
+
+/**
+  * A partially defined session window on event-time without alias.
+  */
+class SessionWithoutAlias(
+    timeField: Expression,
+    gap: Expression) extends WindowWithoutAlias {
+  /**
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
     *
-    * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a session window on event-time
+    * @param alias alias for this window
+    * @return this window
     */
-  def on(timeField: String): SessionEventTimeWindow =
-    on(ExpressionParser.parseExpression(timeField))
+  override def as(alias: Expression): Window = new SessionEventTimeWindow(alias, timeField, gap)
+}
+
+/**
+  * A session window on processing-time.
+  *
+  * @param gap the time interval of inactivity before a window is closed.
+  */
+class SessionWindow(alias: Expression, gap: Expression) extends Window(alias) {
 
   override private[flink] def toLogicalWindow: LogicalWindow =
     ProcessingTimeSessionGroupWindow(alias, gap)
 }
 
 /**
-  * Session window on event-time.
+  * A session window on event-time.
   */
 class SessionEventTimeWindow(
+    alias: Expression,
     timeField: Expression,
     gap: Expression)
-  extends EventTimeWindow(timeField) {
+  extends EventTimeWindow(alias, timeField) {
 
   override private[flink] def toLogicalWindow: LogicalWindow =
     EventTimeSessionGroupWindow(alias, timeField, gap)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
index 1264566954ff7..1884e543077a5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
@@ -22,14 +22,13 @@ import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.expressions.{Expression, WindowReference}
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
-abstract class LogicalWindow(val alias: Option[Expression]) extends Resolvable[LogicalWindow] {
+abstract class LogicalWindow(val alias: Expression) extends Resolvable[LogicalWindow] {
 
   def resolveExpressions(resolver: (Expression) => Expression): LogicalWindow = this
 
   def validate(tableEnv: TableEnvironment): ValidationResult = alias match {
-    case Some(WindowReference(_)) => ValidationSuccess
-    case Some(_) => ValidationFailure("Window reference for window expected.")
-    case None => ValidationSuccess
+    case WindowReference(_) => ValidationSuccess
+    case _ => ValidationFailure("Window reference for window expected.")
   }
 
   override def toString: String = getClass.getSimpleName
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
index f1f058e97802d..576756d174c0f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
@@ -25,7 +25,7 @@ import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeIn
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
 abstract class EventTimeGroupWindow(
-    alias: Option[Expression],
+    alias: Expression,
     time: Expression)
   extends LogicalWindow(alias) {
 
@@ -55,7 +55,7 @@ abstract class EventTimeGroupWindow(
   }
 }
 
-abstract class ProcessingTimeGroupWindow(alias: Option[Expression]) extends LogicalWindow(alias) {
+abstract class ProcessingTimeGroupWindow(alias: Expression) extends LogicalWindow(alias) {
   override def validate(tableEnv: TableEnvironment): ValidationResult = {
     val valid = super.validate(tableEnv)
     if (valid.isFailure) {
@@ -88,13 +88,13 @@ object TumblingGroupWindow {
 }
 
 case class ProcessingTimeTumblingGroupWindow(
-    override val alias: Option[Expression],
+    override val alias: Expression,
     size: Expression)
   extends ProcessingTimeGroupWindow(alias) {
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
     ProcessingTimeTumblingGroupWindow(
-      alias.map(resolve),
+      resolve(alias),
       resolve(size))
 
   override def validate(tableEnv: TableEnvironment): ValidationResult =
@@ -104,7 +104,7 @@ case class ProcessingTimeTumblingGroupWindow(
 }
 
 case class EventTimeTumblingGroupWindow(
-    override val alias: Option[Expression],
+    override val alias: Expression,
     timeField: Expression,
     size: Expression)
   extends EventTimeGroupWindow(
@@ -113,7 +113,7 @@ case class EventTimeTumblingGroupWindow(
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
     EventTimeTumblingGroupWindow(
-      alias.map(resolve),
+      resolve(alias),
       resolve(timeField),
       resolve(size))
 
@@ -177,14 +177,14 @@ object SlidingGroupWindow {
 }
 
 case class ProcessingTimeSlidingGroupWindow(
-    override val alias: Option[Expression],
+    override val alias: Expression,
     size: Expression,
     slide: Expression)
   extends ProcessingTimeGroupWindow(alias) {
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
     ProcessingTimeSlidingGroupWindow(
-      alias.map(resolve),
+      resolve(alias),
       resolve(size),
       resolve(slide))
 
@@ -195,7 +195,7 @@ case class ProcessingTimeSlidingGroupWindow(
 }
 
 case class EventTimeSlidingGroupWindow(
-    override val alias: Option[Expression],
+    override val alias: Expression,
     timeField: Expression,
     size: Expression,
     slide: Expression)
@@ -203,7 +203,7 @@ case class EventTimeSlidingGroupWindow(
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
     EventTimeSlidingGroupWindow(
-      alias.map(resolve),
+      resolve(alias),
       resolve(timeField),
       resolve(size),
       resolve(slide))
@@ -240,13 +240,13 @@ object SessionGroupWindow {
 }
 
 case class ProcessingTimeSessionGroupWindow(
-    override val alias: Option[Expression],
+    override val alias: Expression,
     gap: Expression)
   extends ProcessingTimeGroupWindow(alias) {
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
     ProcessingTimeSessionGroupWindow(
-      alias.map(resolve),
+      resolve(alias),
       resolve(gap))
 
   override def validate(tableEnv: TableEnvironment): ValidationResult =
@@ -256,7 +256,7 @@ case class ProcessingTimeSessionGroupWindow(
 }
 
 case class EventTimeSessionGroupWindow(
-    override val alias: Option[Expression],
+    override val alias: Expression,
     timeField: Expression,
     gap: Expression)
   extends EventTimeGroupWindow(
@@ -265,7 +265,7 @@ case class EventTimeSessionGroupWindow(
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
     EventTimeSessionGroupWindow(
-      alias.map(resolve),
+      resolve(alias),
       resolve(timeField),
       resolve(gap))
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index 559bd7596985b..f1bb644c720d1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -568,7 +568,7 @@ case class WindowAggregate(
     case _ =>
       window.alias match {
         // resolve reference to this window's alias
-        case Some(UnresolvedFieldReference(alias)) if name == alias =>
+        case UnresolvedFieldReference(alias) if name == alias =>
           // check if reference can already be resolved by input fields
           val found = super.resolveReference(tableEnv, name)
           if (found.isDefined) {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala
index c68da0444d8a2..4da2da995eb2b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala
@@ -53,8 +53,8 @@ class WindowStartEndPropertiesRule
     transformed.push(LogicalWindowAggregate.create(
       agg.getWindow,
       Seq(
-        NamedWindowProperty("w$start", WindowStart(agg.getWindow.alias.get)),
-        NamedWindowProperty("w$end", WindowEnd(agg.getWindow.alias.get))
+        NamedWindowProperty("w$start", WindowStart(agg.getWindow.alias)),
+        NamedWindowProperty("w$end", WindowEnd(agg.getWindow.alias))
       ), agg)
     )
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
index cd6c77df7a44f..9d165365bac9b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
@@ -46,7 +46,7 @@ class WindowAggregateTest extends TableTestBase {
           batchTableNode(0),
           term("select", "ts, a, b")
         ),
-        term("window", EventTimeTumblingGroupWindow(Some('w$), 'ts, 7200000.millis)),
+        term("window", EventTimeTumblingGroupWindow('w$, 'ts, 7200000.millis)),
         term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
       )
 
@@ -75,7 +75,7 @@ class WindowAggregateTest extends TableTestBase {
           "DataSetWindowAggregate",
           batchTableNode(0),
           term("groupBy", "c"),
-          term("window", EventTimeTumblingGroupWindow(Some('w$), 'ts, 240000.millis)),
+          term("window", EventTimeTumblingGroupWindow('w$, 'ts, 240000.millis)),
           term("select", "c, SUM(a) AS sumA, MIN(b) AS minB, " +
             "start('w$) AS w$start, end('w$) AS w$end")
         ),
@@ -104,7 +104,7 @@ class WindowAggregateTest extends TableTestBase {
           term("select", "ts, a, b")
         ),
         term("window",
-          EventTimeSlidingGroupWindow(Some('w$), 'ts, 5400000.millis, 900000.millis)),
+          EventTimeSlidingGroupWindow('w$, 'ts, 5400000.millis, 900000.millis)),
         term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
       )
 
@@ -134,7 +134,7 @@ class WindowAggregateTest extends TableTestBase {
           batchTableNode(0),
           term("groupBy", "c, d"),
           term("window",
-            EventTimeSlidingGroupWindow(Some('w$), 'ts, 10800000.millis, 3600000.millis)),
+            EventTimeSlidingGroupWindow('w$, 'ts, 10800000.millis, 3600000.millis)),
           term("select", "c, d, SUM(a) AS sumA, AVG(b) AS avgB, " +
             "start('w$) AS w$start, end('w$) AS w$end")
         ),
@@ -160,7 +160,7 @@ class WindowAggregateTest extends TableTestBase {
           batchTableNode(0),
           term("select", "ts")
         ),
-        term("window", EventTimeSessionGroupWindow(Some('w$), 'ts, 1800000.millis)),
+        term("window", EventTimeSessionGroupWindow('w$, 'ts, 1800000.millis)),
         term("select", "COUNT(*) AS cnt")
       )
 
@@ -189,7 +189,7 @@ class WindowAggregateTest extends TableTestBase {
           "DataSetWindowAggregate",
           batchTableNode(0),
           term("groupBy", "c, d"),
-          term("window", EventTimeSessionGroupWindow(Some('w$), 'ts, 43200000.millis)),
+          term("window", EventTimeSessionGroupWindow('w$, 'ts, 43200000.millis)),
           term("select", "c, d, SUM(a) AS sumA, MIN(b) AS minB, " +
             "start('w$) AS w$start, end('w$) AS w$end")
         ),
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
index 4d0d9aaaea6cb..6ebfec073bd76 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
@@ -239,7 +239,7 @@ class FieldProjectionTest extends TableTestBase {
         ),
         term("window",
           EventTimeTumblingGroupWindow(
-            Some(WindowReference("w")),
+           WindowReference("w"),
             RowtimeAttribute(),
             5.millis)),
         term("select", "COUNT($f2) AS TMP_0", "SUM(a) AS TMP_1")
@@ -268,7 +268,7 @@ class FieldProjectionTest extends TableTestBase {
           term("groupBy", "b"),
           term("window",
             EventTimeTumblingGroupWindow(
-              Some(WindowReference("w")),
+             WindowReference("w"),
               RowtimeAttribute(),
               5.millis)),
           term("select", "b", "COUNT($f3) AS TMP_0", "SUM(a) AS TMP_1")
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
index a778b40e0f47d..8f4eea54fd017 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
@@ -97,7 +97,7 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeTumblingGroupWindow(Some(WindowReference("w")), 'long, 2.rows)),
+      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 2.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -118,7 +118,7 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeTumblingGroupWindow(Some(WindowReference("w")), 'long, 5.milli)),
+      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -164,7 +164,7 @@ class GroupWindowTest extends TableTestBase {
         batchTableNode(0),
         term("select", "int", "long")
       ),
-      term("window", EventTimeTumblingGroupWindow(Some(WindowReference("w")), 'long, 5.milli)),
+      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -188,7 +188,7 @@ class GroupWindowTest extends TableTestBase {
         batchTableNode(0),
         term("select", "int", "long")
       ),
-      term("window", EventTimeTumblingGroupWindow(Some(WindowReference("w")), 'long, 2.rows)),
+      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 2.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -236,7 +236,7 @@ class GroupWindowTest extends TableTestBase {
       batchTableNode(0),
       term("groupBy", "string"),
       term("window",
-        EventTimeSlidingGroupWindow(Some(WindowReference("w")), 'long, 8.milli, 10.milli)),
+        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -258,7 +258,7 @@ class GroupWindowTest extends TableTestBase {
       batchTableNode(0),
       term("groupBy", "string"),
       term("window",
-        EventTimeSlidingGroupWindow(Some(WindowReference("w")), 'long, 2.rows, 1.rows)),
+        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 2.rows, 1.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -294,7 +294,7 @@ class GroupWindowTest extends TableTestBase {
         term("select", "int", "long")
       ),
       term("window",
-        EventTimeSlidingGroupWindow(Some(WindowReference("w")), 'long, 8.milli, 10.milli)),
+        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -319,7 +319,7 @@ class GroupWindowTest extends TableTestBase {
         term("select", "int", "long")
       ),
       term("window",
-        EventTimeSlidingGroupWindow(Some(WindowReference("w")), 'long, 2.rows, 1.rows)),
+        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 2.rows, 1.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -344,7 +344,7 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeSessionGroupWindow(Some(WindowReference("w")), 'long, 7.milli)),
+      term("window", EventTimeSessionGroupWindow(WindowReference("w"), 'long, 7.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
index 324b4d6740b7c..578a6a84d123a 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
@@ -102,7 +102,7 @@ class WindowAggregateTest extends TableTestBase {
             streamTableNode(0),
             term("select", "1970-01-01 00:00:00 AS $f0")
           ),
-          term("window", EventTimeTumblingGroupWindow(Some('w$), 'rowtime, 900000.millis)),
+          term("window", EventTimeTumblingGroupWindow('w$, 'rowtime, 900000.millis)),
           term("select", "COUNT(*) AS EXPR$0, start('w$) AS w$start, end('w$) AS w$end")
         ),
         term("select", "EXPR$0, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
@@ -128,7 +128,7 @@ class WindowAggregateTest extends TableTestBase {
             streamTableNode(0),
             term("select", "1970-01-01 00:00:00 AS $f0")
           ),
-          term("window", ProcessingTimeSlidingGroupWindow(Some('w$),
+          term("window", ProcessingTimeSlidingGroupWindow('w$,
             3600000.millis, 900000.millis)),
           term("select", "COUNT(*) AS EXPR$0, start('w$) AS w$start, end('w$) AS w$end")
         ),
@@ -156,7 +156,7 @@ class WindowAggregateTest extends TableTestBase {
             streamTableNode(0),
             term("select", "1970-01-01 00:00:00 AS $f0")
           ),
-          term("window", ProcessingTimeSessionGroupWindow(Some('w$), 900000.millis)),
+          term("window", ProcessingTimeSessionGroupWindow('w$, 900000.millis)),
           term("select", "COUNT(*) AS EXPR$0, start('w$) AS w$start, end('w$) AS w$end")
         ),
         term("select", "EXPR$0, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index fde7682cea68d..402ebbeb781ab 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -195,7 +195,7 @@ class GroupWindowTest extends TableTestBase {
           term(
             "window",
             ProcessingTimeTumblingGroupWindow(
-              Some(WindowReference("w1")),
+              WindowReference("w1"),
               50.milli)),
           term("select", "string", "COUNT(int) AS TMP_0")
         ),
@@ -204,7 +204,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeSlidingGroupWindow(
-          Some(WindowReference("w2")),
+          WindowReference("w2"),
           20.milli, 10.milli)),
       term("select", "COUNT(string) AS TMP_1")
     )
@@ -232,7 +232,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           50.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -261,7 +261,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeTumblingGroupWindow(
-          Some(WindowReference("w")), 2.rows)),
+          WindowReference("w"), 2.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -289,7 +289,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(),
           5.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
@@ -316,7 +316,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 2.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -345,7 +345,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           50.milli, 50.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -374,7 +374,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           2.rows, 1.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -403,7 +403,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 8.milli, 10.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -429,7 +429,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 2.rows, 1.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -458,7 +458,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeSessionGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 7.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -487,7 +487,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           50.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -515,7 +515,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           2.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -543,7 +543,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 5.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -572,7 +572,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 2.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -601,7 +601,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           50.milli, 50.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -629,7 +629,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         ProcessingTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           2.rows, 1.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -657,7 +657,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 8.milli, 10.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -686,7 +686,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 2.rows, 1.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -714,7 +714,7 @@ class GroupWindowTest extends TableTestBase {
       term(
         "window",
         EventTimeSessionGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(), 7.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -742,7 +742,7 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term("window",
         EventTimeTumblingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(),
           5.milli)),
       term("select",
@@ -775,7 +775,7 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term("window",
         EventTimeSlidingGroupWindow(
-          Some(WindowReference("w")),
+          WindowReference("w"),
           RowtimeAttribute(),
           10.milli,
           5.milli)),
@@ -811,7 +811,7 @@ class GroupWindowTest extends TableTestBase {
         term("groupBy", "string"),
         term("window",
           EventTimeSessionGroupWindow(
-            Some(WindowReference("w")),
+            WindowReference("w"),
             RowtimeAttribute(),
             3.milli)),
         term("select",
@@ -849,7 +849,7 @@ class GroupWindowTest extends TableTestBase {
         term("groupBy", "string"),
         term("window",
           EventTimeTumblingGroupWindow(
-            Some(WindowReference("w")),
+            WindowReference("w"),
             RowtimeAttribute(),
             5.millis)),
         term("select",

From 7f4577e2ae37623877278b932fc28f5e150112b3 Mon Sep 17 00:00:00 2001
From: Zhijiang <wangzhijiang999@aliyun.com>
Date: Thu, 30 Mar 2017 00:30:29 +0800
Subject: [PATCH 073/253] [FLINK-6120] [heartbeat] Implement heartbeat logic
 between JobManager and ResourceManager

This closes #3645.
---
 .../flink/runtime/jobmaster/JobMaster.java    | 153 ++++++++++++++----
 .../runtime/jobmaster/JobMasterGateway.java   |   7 +
 .../JobMasterRegistrationSuccess.java         |  14 +-
 .../resourcemanager/ResourceManager.java      | 133 ++++++++++++---
 .../ResourceManagerGateway.java               |  18 ++-
 .../registration/JobManagerRegistration.java  |   9 ++
 .../clusterframework/ResourceManagerTest.java | 116 ++++++++++++-
 .../runtime/jobmaster/JobMasterTest.java      |  89 +++++++++-
 .../ResourceManagerJobMasterTest.java         |  42 ++++-
 .../slotmanager/SlotProtocolTest.java         |   6 +-
 .../taskexecutor/TaskExecutorITCase.java      |   8 +-
 11 files changed, 521 insertions(+), 74 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 6fe8cb3c0d13a..ab4357729e5dc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -155,7 +155,10 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
 	private final MetricGroup jobMetricGroup;
 
 	/** The heartbeat manager with task managers */
-	private final HeartbeatManager<Void, Void> heartbeatManager;
+	private final HeartbeatManager<Void, Void> taskManagerHeartbeatManager;
+
+	/** The heartbeat manager with resource manager */
+	private final HeartbeatManager<Void, Void> resourceManagerHeartbeatManager;
 
 	/** The execution context which is used to execute futures */
 	private final Executor executor;
@@ -218,12 +221,18 @@ public JobMaster(
 		this.errorHandler = checkNotNull(errorHandler);
 		this.userCodeLoader = checkNotNull(userCodeLoader);
 
-		this.heartbeatManager = heartbeatServices.createHeartbeatManagerSender(
+		this.taskManagerHeartbeatManager = heartbeatServices.createHeartbeatManagerSender(
 			resourceId,
 			new TaskManagerHeartbeatListener(),
 			rpcService.getScheduledExecutor(),
 			log);
 
+		this.resourceManagerHeartbeatManager = heartbeatServices.createHeartbeatManager(
+				resourceId,
+				new ResourceManagerHeartbeatListener(),
+				rpcService.getScheduledExecutor(),
+				log);
+
 		final String jobName = jobGraph.getName();
 		final JobID jid = jobGraph.getJobID();
 
@@ -309,7 +318,8 @@ public void start(final UUID leaderSessionID) throws Exception {
 	 */
 	@Override
 	public void shutDown() throws Exception {
-		heartbeatManager.stop();
+		taskManagerHeartbeatManager.stop();
+		resourceManagerHeartbeatManager.stop();
 
 		// make sure there is a graceful exit
 		getSelf().suspendExecution(new Exception("JobManager is shutting down."));
@@ -407,7 +417,7 @@ public void suspendExecution(final Throwable cause) {
 		slotPoolGateway.suspend();
 
 		// disconnect from resource manager:
-		closeResourceManagerConnection();
+		closeResourceManagerConnection(new Exception("Execution was suspended.", cause));
 	}
 
 	//----------------------------------------------------------------------------------------------
@@ -534,7 +544,7 @@ public Acknowledge scheduleOrUpdateConsumers(
 
 	@RpcMethod
 	public void disconnectTaskManager(final ResourceID resourceID, final Exception cause) {
-		heartbeatManager.unmonitorTarget(resourceID);
+		taskManagerHeartbeatManager.unmonitorTarget(resourceID);
 		slotPoolGateway.releaseTaskManager(resourceID);
 
 		Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManagerConnection = registeredTaskManagers.remove(resourceID);
@@ -766,7 +776,7 @@ public RegistrationResponse apply(final TaskExecutorGateway taskExecutorGateway,
 					registeredTaskManagers.put(taskManagerId, Tuple2.of(taskManagerLocation, taskExecutorGateway));
 
 					// monitor the task manager as heartbeat target
-					heartbeatManager.monitorTarget(taskManagerId, new HeartbeatTarget<Void>() {
+					taskManagerHeartbeatManager.monitorTarget(taskManagerId, new HeartbeatTarget<Void>() {
 						@Override
 						public void receiveHeartbeat(ResourceID resourceID, Void payload) {
 							// the task manager will not request heartbeat, so this method will never be called currently
@@ -788,13 +798,24 @@ public void requestHeartbeat(ResourceID resourceID, Void payload) {
 	public void disconnectResourceManager(
 			final UUID jobManagerLeaderId,
 			final UUID resourceManagerLeaderId,
-			final Exception cause) {
-		// TODO: Implement disconnect behaviour
+			final Exception cause) throws Exception {
+
+		validateLeaderSessionId(jobManagerLeaderId);
+
+		if (resourceManagerConnection != null
+				&& resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderId)) {
+			closeResourceManagerConnection(cause);
+		}
 	}
 
 	@RpcMethod
 	public void heartbeatFromTaskManager(final ResourceID resourceID) {
-		heartbeatManager.receiveHeartbeat(resourceID, null);
+		taskManagerHeartbeatManager.receiveHeartbeat(resourceID, null);
+	}
+
+	@RpcMethod
+	public void heartbeatFromResourceManager(final ResourceID resourceID) {
+		resourceManagerHeartbeatManager.requestHeartbeat(resourceID, null);
 	}
 
 	//----------------------------------------------------------------------------------------------
@@ -872,56 +893,79 @@ private void jobStatusChanged(final JobStatus newJobStatus, long timestamp, fina
 		}
 	}
 
-	private void notifyOfNewResourceManagerLeader(
-			final String resourceManagerAddress, final UUID resourceManagerLeaderId)
-	{
-		validateRunsInMainThread();
-
+	private void notifyOfNewResourceManagerLeader(final String resourceManagerAddress, final UUID resourceManagerLeaderId) {
 		if (resourceManagerConnection != null) {
 			if (resourceManagerAddress != null) {
 				if (resourceManagerAddress.equals(resourceManagerConnection.getTargetAddress())
-						&& resourceManagerLeaderId.equals(resourceManagerConnection.getTargetLeaderId())) {
+					&& resourceManagerLeaderId.equals(resourceManagerConnection.getTargetLeaderId())) {
 					// both address and leader id are not changed, we can keep the old connection
 					return;
 				}
+
+				closeResourceManagerConnection(new Exception(
+					"ResourceManager leader changed to new address " + resourceManagerAddress));
+
 				log.info("ResourceManager leader changed from {} to {}. Registering at new leader.",
-						resourceManagerConnection.getTargetAddress(), resourceManagerAddress);
+					resourceManagerConnection.getTargetAddress(), resourceManagerAddress);
 			} else {
 				log.info("Current ResourceManager {} lost leader status. Waiting for new ResourceManager leader.",
-						resourceManagerConnection.getTargetAddress());
+					resourceManagerConnection.getTargetAddress());
 			}
 		}
 
-		closeResourceManagerConnection();
-
 		if (resourceManagerAddress != null) {
 			log.info("Attempting to register at ResourceManager {}", resourceManagerAddress);
+
 			resourceManagerConnection = new ResourceManagerConnection(
-					log, jobGraph.getJobID(), getAddress(), leaderSessionID,
-					resourceManagerAddress, resourceManagerLeaderId, executor);
+				log,
+				jobGraph.getJobID(),
+				resourceId,
+				getAddress(),
+				leaderSessionID,
+				resourceManagerAddress,
+				resourceManagerLeaderId,
+				executor);
+
 			resourceManagerConnection.start();
 		}
 	}
 
-	private void onResourceManagerRegistrationSuccess(final JobMasterRegistrationSuccess success) {
-		validateRunsInMainThread();
+	private void establishResourceManagerConnection(final JobMasterRegistrationSuccess success) {
+		final UUID resourceManagerLeaderId = success.getResourceManagerLeaderId();
 	
 		// verify the response with current connection
 		if (resourceManagerConnection != null
-				&& resourceManagerConnection.getTargetLeaderId().equals(success.getResourceManagerLeaderId()))
-		{
-			log.info("JobManager successfully registered at ResourceManager, leader id: {}.",
-					success.getResourceManagerLeaderId());
+				&& resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderId)) {
 
-			slotPoolGateway.connectToResourceManager(
-					success.getResourceManagerLeaderId(), resourceManagerConnection.getTargetGateway());
+			log.info("JobManager successfully registered at ResourceManager, leader id: {}.", resourceManagerLeaderId);
+
+			final ResourceManagerGateway resourceManagerGateway = resourceManagerConnection.getTargetGateway();
+
+			slotPoolGateway.connectToResourceManager(resourceManagerLeaderId, resourceManagerGateway);
+
+			resourceManagerHeartbeatManager.monitorTarget(success.getResourceManagerResourceId(), new HeartbeatTarget<Void>() {
+				@Override
+				public void receiveHeartbeat(ResourceID resourceID, Void payload) {
+					resourceManagerGateway.heartbeatFromJobManager(resourceID);
+				}
+
+				@Override
+				public void requestHeartbeat(ResourceID resourceID, Void payload) {
+					// request heartbeat will never be called on the job manager side
+				}
+			});
 		}
 	}
 
-	private void closeResourceManagerConnection() {
-		validateRunsInMainThread();
-
+	private void closeResourceManagerConnection(Exception cause) {
 		if (resourceManagerConnection != null) {
+			log.info("Close ResourceManager connection {}.", resourceManagerConnection.getResourceManagerResourceID(), cause);
+
+			resourceManagerHeartbeatManager.unmonitorTarget(resourceManagerConnection.getResourceManagerResourceID());
+
+			ResourceManagerGateway resourceManagerGateway = resourceManagerConnection.getTargetGateway();
+			resourceManagerGateway.disconnectJobManager(resourceManagerConnection.getJobID(), cause);
+
 			resourceManagerConnection.close();
 			resourceManagerConnection = null;
 		}
@@ -964,13 +1008,18 @@ private class ResourceManagerConnection
 	{
 		private final JobID jobID;
 
+		private final ResourceID jobManagerResourceID;
+
 		private final String jobManagerRpcAddress;
 
 		private final UUID jobManagerLeaderID;
 
+		private ResourceID resourceManagerResourceID;
+
 		ResourceManagerConnection(
 				final Logger log,
 				final JobID jobID,
+				final ResourceID jobManagerResourceID,
 				final String jobManagerRpcAddress,
 				final UUID jobManagerLeaderID,
 				final String resourceManagerAddress,
@@ -979,6 +1028,7 @@ private class ResourceManagerConnection
 		{
 			super(log, resourceManagerAddress, resourceManagerLeaderID, executor);
 			this.jobID = checkNotNull(jobID);
+			this.jobManagerResourceID = checkNotNull(jobManagerResourceID);
 			this.jobManagerRpcAddress = checkNotNull(jobManagerRpcAddress);
 			this.jobManagerLeaderID = checkNotNull(jobManagerLeaderID);
 		}
@@ -998,6 +1048,7 @@ protected Future<RegistrationResponse> invokeRegistration(
 					return gateway.registerJobManager(
 						leaderId,
 						jobManagerLeaderID,
+						jobManagerResourceID,
 						jobManagerRpcAddress,
 						jobID,
 						timeout);
@@ -1010,7 +1061,8 @@ protected void onRegistrationSuccess(final JobMasterRegistrationSuccess success)
 			runAsync(new Runnable() {
 				@Override
 				public void run() {
-					onResourceManagerRegistrationSuccess(success);
+					resourceManagerResourceID = success.getResourceManagerResourceId();
+					establishResourceManagerConnection(success);
 				}
 			});
 		}
@@ -1019,6 +1071,14 @@ public void run() {
 		protected void onRegistrationFailure(final Throwable failure) {
 			handleFatalError(failure);
 		}
+
+		public ResourceID getResourceManagerResourceID() {
+			return resourceManagerResourceID;
+		}
+
+		public JobID getJobID() {
+			return jobID;
+		}
 	}
 
 	//----------------------------------------------------------------------------------------------
@@ -1063,4 +1123,31 @@ public Future<Void> retrievePayload() {
 			return FlinkCompletableFuture.completed(null);
 		}
 	}
+
+	private class ResourceManagerHeartbeatListener implements HeartbeatListener<Void, Void> {
+
+		@Override
+		public void notifyHeartbeatTimeout(final ResourceID resourceId) {
+			runAsync(new Runnable() {
+				@Override
+				public void run() {
+					log.info("The heartbeat of ResourceManager with id {} timed out.", resourceId);
+
+					closeResourceManagerConnection(
+						new TimeoutException(
+							"The heartbeat of ResourceManager with id " + resourceId + " timed out."));
+				}
+			});
+		}
+
+		@Override
+		public void reportPayload(ResourceID resourceID, Void payload) {
+			// nothing to do since the payload is of type Void
+		}
+
+		@Override
+		public Future<Void> retrievePayload() {
+			return FlinkCompletableFuture.completed(null);
+		}
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 13a7372607442..5a271f90c383e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -226,4 +226,11 @@ Future<RegistrationResponse> registerTaskManager(
 	 * @param resourceID unique id of the task manager
 	 */
 	void heartbeatFromTaskManager(final ResourceID resourceID);
+
+	/**
+	 * Heartbeat request from the resource manager
+	 *
+	 * @param resourceID unique id of the resource manager
+	 */
+	void heartbeatFromResourceManager(final ResourceID resourceID);
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java
index 4058452ffefd3..a7a622465eccb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.jobmaster;
 
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.registration.RegistrationResponse;
 
 import java.util.UUID;
@@ -35,9 +36,15 @@ public class JobMasterRegistrationSuccess extends RegistrationResponse.Success {
 
 	private final UUID resourceManagerLeaderId;
 
-	public JobMasterRegistrationSuccess(final long heartbeatInterval, final UUID resourceManagerLeaderId) {
+	private final ResourceID resourceManagerResourceId;
+
+	public JobMasterRegistrationSuccess(
+			final long heartbeatInterval,
+			final UUID resourceManagerLeaderId,
+			final ResourceID resourceManagerResourceId) {
 		this.heartbeatInterval = heartbeatInterval;
 		this.resourceManagerLeaderId = checkNotNull(resourceManagerLeaderId);
+		this.resourceManagerResourceId = checkNotNull(resourceManagerResourceId);
 	}
 
 	/**
@@ -53,11 +60,16 @@ public UUID getResourceManagerLeaderId() {
 		return resourceManagerLeaderId;
 	}
 
+	public ResourceID getResourceManagerResourceId() {
+		return resourceManagerResourceId;
+	}
+
 	@Override
 	public String toString() {
 		return "JobMasterRegistrationSuccess{" +
 			"heartbeatInterval=" + heartbeatInterval +
 			", resourceManagerLeaderId=" + resourceManagerLeaderId +
+			", resourceManagerResourceId=" + resourceManagerResourceId +
 			'}';
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index c0ff41292de9f..f17dbe5905225 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -79,7 +79,7 @@
  *
  * It offers the following methods as part of its rpc interface to interact with him remotely:
  * <ul>
- *     <li>{@link #registerJobManager(UUID, UUID, String, JobID)} registers a {@link JobMaster} at the resource manager</li>
+ *     <li>{@link #registerJobManager(UUID, UUID, String, JobID, ResourceID)} registers a {@link JobMaster} at the resource manager</li>
  *     <li>{@link #requestSlot(UUID, UUID, SlotRequest)} requests a slot from the resource manager</li>
  * </ul>
  */
@@ -96,6 +96,9 @@ public abstract class ResourceManager<WorkerType extends Serializable>
 	/** All currently registered JobMasterGateways scoped by JobID. */
 	private final Map<JobID, JobManagerRegistration> jobManagerRegistrations;
 
+	/** All currently registered JobMasterGateways scoped by ResourceID. */
+	private final Map<ResourceID, JobManagerRegistration> jmResourceIdRegistrations;
+
 	/** Service to retrieve the job leader ids */
 	private final JobLeaderIdService jobLeaderIdService;
 
@@ -108,6 +111,9 @@ public abstract class ResourceManager<WorkerType extends Serializable>
 	/** The heartbeat manager with task managers. */
 	private final HeartbeatManager<Void, Void> taskManagerHeartbeatManager;
 
+	/** The heartbeat manager with job managers. */
+	private final HeartbeatManager<Void, Void> jobManagerHeartbeatManager;
+
 	/** The factory to construct the SlotManager. */
 	private final SlotManagerFactory slotManagerFactory;
 
@@ -152,12 +158,19 @@ public ResourceManager(
 		this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
 
 		this.taskManagerHeartbeatManager = heartbeatServices.createHeartbeatManagerSender(
-				resourceId,
-				new TaskManagerHeartbeatListener(),
-				rpcService.getScheduledExecutor(),
-				log);
+			resourceId,
+			new TaskManagerHeartbeatListener(),
+			rpcService.getScheduledExecutor(),
+			log);
+
+		this.jobManagerHeartbeatManager = heartbeatServices.createHeartbeatManagerSender(
+			resourceId,
+			new JobManagerHeartbeatListener(),
+			rpcService.getScheduledExecutor(),
+			log);
 
 		this.jobManagerRegistrations = new HashMap<>(4);
+		this.jmResourceIdRegistrations = new HashMap<>(4);
 		this.taskExecutors = new HashMap<>(8);
 		this.leaderSessionId = null;
 		infoMessageListeners = new ConcurrentHashMap<>(8);
@@ -204,6 +217,8 @@ public void shutDown() throws Exception {
 
 		taskManagerHeartbeatManager.stop();
 
+		jobManagerHeartbeatManager.stop();
+
 		try {
 			super.shutDown();
 		} catch (Exception e) {
@@ -231,11 +246,13 @@ public void shutDown() throws Exception {
 	public Future<RegistrationResponse> registerJobManager(
 			final UUID resourceManagerLeaderId,
 			final UUID jobManagerLeaderId,
+			final ResourceID jobManagerResourceId,
 			final String jobManagerAddress,
 			final JobID jobId) {
 
 		checkNotNull(resourceManagerLeaderId);
 		checkNotNull(jobManagerLeaderId);
+		checkNotNull(jobManagerResourceId);
 		checkNotNull(jobManagerAddress);
 		checkNotNull(jobId);
 
@@ -276,7 +293,7 @@ public Future<RegistrationResponse> registerJobManager(
 
 			Future<RegistrationResponse> registrationResponseFuture = jobMasterGatewayFuture.thenCombineAsync(jobLeaderIdFuture, new BiFunction<JobMasterGateway, UUID, RegistrationResponse>() {
 				@Override
-				public RegistrationResponse apply(JobMasterGateway jobMasterGateway, UUID jobLeaderId) {
+				public RegistrationResponse apply(final JobMasterGateway jobMasterGateway, UUID jobLeaderId) {
 					if (isValid(resourceManagerLeaderId)) {
 						if (jobLeaderId.equals(jobManagerLeaderId)) {
 							if (jobManagerRegistrations.containsKey(jobId)) {
@@ -291,21 +308,43 @@ public RegistrationResponse apply(JobMasterGateway jobMasterGateway, UUID jobLea
 										oldJobManagerRegistration.getJobID(),
 										new Exception("New job leader for job " + jobId + " found."));
 
-									JobManagerRegistration jobManagerRegistration = new JobManagerRegistration(jobId, jobLeaderId, jobMasterGateway);
+									JobManagerRegistration jobManagerRegistration = new JobManagerRegistration(
+										jobId,
+										jobManagerResourceId,
+										jobLeaderId,
+										jobMasterGateway);
 									jobManagerRegistrations.put(jobId, jobManagerRegistration);
+									jmResourceIdRegistrations.put(jobManagerResourceId, jobManagerRegistration);
 								}
 							} else {
 								// new registration for the job
-								JobManagerRegistration jobManagerRegistration = new JobManagerRegistration(jobId, jobLeaderId, jobMasterGateway);
-
+								JobManagerRegistration jobManagerRegistration = new JobManagerRegistration(
+									jobId,
+									jobManagerResourceId,
+									jobLeaderId,
+									jobMasterGateway);
 								jobManagerRegistrations.put(jobId, jobManagerRegistration);
+								jmResourceIdRegistrations.put(jobManagerResourceId, jobManagerRegistration);
 							}
 
 							log.info("Registered job manager {}@{} for job {}.", jobManagerLeaderId, jobManagerAddress, jobId);
 
+							jobManagerHeartbeatManager.monitorTarget(jobManagerResourceId, new HeartbeatTarget<Void>() {
+								@Override
+								public void receiveHeartbeat(ResourceID resourceID, Void payload) {
+									// the ResourceManager will always send heartbeat requests to the JobManager
+								}
+
+								@Override
+								public void requestHeartbeat(ResourceID resourceID, Void payload) {
+									jobMasterGateway.heartbeatFromResourceManager(resourceID);
+								}
+							});
+
 							return new JobMasterRegistrationSuccess(
 								resourceManagerConfiguration.getHeartbeatInterval().toMilliseconds(),
-								getLeaderSessionId());
+								getLeaderSessionId(),
+								resourceId);
 
 						} else {
 							log.debug("The job manager leader id {} did not match the job " +
@@ -422,11 +461,21 @@ public void heartbeatFromTaskManager(final ResourceID resourceID) {
 		taskManagerHeartbeatManager.receiveHeartbeat(resourceID, null);
 	}
 
+	@RpcMethod
+	public void heartbeatFromJobManager(final ResourceID resourceID) {
+		jobManagerHeartbeatManager.receiveHeartbeat(resourceID, null);
+	}
+
 	@RpcMethod
 	public void disconnectTaskManager(final ResourceID resourceId, final Exception cause) {
 		closeTaskManagerConnection(resourceId, cause);
 	}
 
+	@RpcMethod
+	public void disconnectJobManager(final JobID jobId, final Exception cause) {
+		closeJobManagerConnection(jobId, cause);
+	}
+
 	/**
 	 * Requests a slot from the resource manager.
 	 *
@@ -577,6 +626,7 @@ UUID getLeaderSessionId() {
 
 	private void clearState() {
 		jobManagerRegistrations.clear();
+		jmResourceIdRegistrations.clear();
 		taskExecutors.clear();
 		slotManager.clearState();
 
@@ -590,23 +640,31 @@ private void clearState() {
 	}
 
 	/**
-	 * Disconnects the job manager which is connected for the given job from the resource manager.
+	 * This method should be called by the framework once it detects that a currently registered
+	 * job manager has failed.
 	 *
-	 * @param jobId identifying the job whose leader shall be disconnected
+	 * @param jobId identifying the job whose leader shall be disconnected.
+	 * @param cause The exception which cause the JobManager failed.
 	 */
-	protected void disconnectJobManager(JobID jobId, Exception cause) {
+	protected void closeJobManagerConnection(JobID jobId, Exception cause) {
 		JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.remove(jobId);
 
 		if (jobManagerRegistration != null) {
+			final ResourceID jobManagerResourceId = jobManagerRegistration.getJobManagerResourceID();
+			final JobMasterGateway jobMasterGateway = jobManagerRegistration.getJobManagerGateway();
+			final UUID jobManagerLeaderId = jobManagerRegistration.getLeaderID();
+
 			log.info("Disconnect job manager {}@{} for job {} from the resource manager.",
-				jobManagerRegistration.getLeaderID(),
-				jobManagerRegistration.getJobManagerGateway().getAddress(),
+				jobManagerLeaderId,
+				jobMasterGateway.getAddress(),
 				jobId);
 
-			JobMasterGateway jobMasterGateway = jobManagerRegistration.getJobManagerGateway();
+			jobManagerHeartbeatManager.unmonitorTarget(jobManagerResourceId);
+
+			jmResourceIdRegistrations.remove(jobManagerResourceId);
 
 			// tell the job manager about the disconnect
-			jobMasterGateway.disconnectResourceManager(jobManagerRegistration.getLeaderID(), getLeaderSessionId(), cause);
+			jobMasterGateway.disconnectResourceManager(jobManagerLeaderId, getLeaderSessionId(), cause);
 		} else {
 			log.debug("There was no registered job manager for job {}.", jobId);
 		}
@@ -882,14 +940,47 @@ private class TaskManagerHeartbeatListener implements HeartbeatListener<Void, Vo
 
 		@Override
 		public void notifyHeartbeatTimeout(final ResourceID resourceID) {
-			log.info("The heartbeat of TaskManager with id {} timed out.", resourceID);
-
 			runAsync(new Runnable() {
 				@Override
 				public void run() {
+					log.info("The heartbeat of TaskManager with id {} timed out.", resourceID);
+
 					closeTaskManagerConnection(
-						resourceID,
-						new TimeoutException("Task manager with id " + resourceID + " heartbeat timed out."));
+							resourceID,
+							new TimeoutException("The heartbeat of TaskManager with id " + resourceID + "  timed out."));
+				}
+			});
+		}
+
+		@Override
+		public void reportPayload(ResourceID resourceID, Void payload) {
+			// nothing to do since there is no payload
+		}
+
+		@Override
+		public Future<Void> retrievePayload() {
+			return FlinkCompletableFuture.completed(null);
+		}
+	}
+
+	private class JobManagerHeartbeatListener implements HeartbeatListener<Void, Void> {
+
+		@Override
+		public void notifyHeartbeatTimeout(final ResourceID resourceID) {
+			runAsync(new Runnable() {
+				@Override
+				public void run() {
+					log.info("The heartbeat of JobManager with id {} timed out.", resourceID);
+
+					if (jmResourceIdRegistrations.containsKey(resourceID)) {
+						JobManagerRegistration jobManagerRegistration = jmResourceIdRegistrations.get(resourceID);
+
+						if (jobManagerRegistration != null) {
+							closeJobManagerConnection(
+								jobManagerRegistration.getJobID(),
+								new TimeoutException("The heartbeat of JobManager with id " + resourceID + " timed out."));
+						}
+					}
 				}
 			});
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index cda4a7c8dfa1f..530113f815049 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -44,6 +44,7 @@ public interface ResourceManagerGateway extends RpcGateway {
 	 *
 	 * @param resourceManagerLeaderId The fencing token for the ResourceManager leader
 	 * @param jobMasterLeaderId The fencing token for the JobMaster leader
+	 * @param jobMasterResourceId   The resource ID of the JobMaster that registers
 	 * @param jobMasterAddress        The address of the JobMaster that registers
 	 * @param jobID                   The Job ID of the JobMaster that registers
 	 * @param timeout                 Timeout for the future to complete
@@ -52,11 +53,11 @@ public interface ResourceManagerGateway extends RpcGateway {
 	Future<RegistrationResponse> registerJobManager(
 		UUID resourceManagerLeaderId,
 		UUID jobMasterLeaderId,
+		ResourceID jobMasterResourceId,
 		String jobMasterAddress,
 		JobID jobID,
 		@RpcTimeout Time timeout);
 
-
 	/**
 	 * Requests a slot from the resource manager.
 	 *
@@ -138,6 +139,13 @@ void notifySlotAvailable(
 	 */
 	void heartbeatFromTaskManager(final ResourceID heartbeatOrigin);
 
+	/**
+	 * Sends the heartbeat to resource manager from job manager
+	 *
+	 * @param heartbeatOrigin unique id of the job manager
+	 */
+	void heartbeatFromJobManager(final ResourceID heartbeatOrigin);
+
 	/**
 	 * Disconnects a TaskManager specified by the given resourceID from the {@link ResourceManager}.
 	 *
@@ -145,4 +153,12 @@ void notifySlotAvailable(
 	 * @param cause for the disconnection of the TaskManager
 	 */
 	void disconnectTaskManager(ResourceID resourceID, Exception cause);
+
+	/**
+	 * Disconnects a JobManager specified by the given resourceID from the {@link ResourceManager}.
+	 *
+	 * @param jobId JobID for which the JobManager was the leader
+	 * @param cause for the disconnection of the JobManager
+	 */
+	void disconnectJobManager(JobID jobId, Exception cause);
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java
index a1deb65cfc0bd..df3a39fad2856 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.resourcemanager.registration;
 
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.jobmaster.JobMasterGateway;
 import org.apache.flink.util.Preconditions;
 
@@ -30,15 +31,19 @@
 public class JobManagerRegistration {
 	private final JobID jobID;
 
+	private final ResourceID jobManagerResourceID;
+
 	private final UUID leaderID;
 
 	private final JobMasterGateway jobManagerGateway;
 
 	public JobManagerRegistration(
 			JobID jobID,
+			ResourceID jobManagerResourceID,
 			UUID leaderID,
 			JobMasterGateway jobManagerGateway) {
 		this.jobID = Preconditions.checkNotNull(jobID);
+		this.jobManagerResourceID = Preconditions.checkNotNull(jobManagerResourceID);
 		this.leaderID = Preconditions.checkNotNull(leaderID);
 		this.jobManagerGateway = Preconditions.checkNotNull(jobManagerGateway);
 	}
@@ -47,6 +52,10 @@ public JobID getJobID() {
 		return jobID;
 	}
 
+	public ResourceID getJobManagerResourceID() {
+		return jobManagerResourceID;
+	}
+
 	public UUID getLeaderID() {
 		return leaderID;
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
index 3464129f388b9..e4e20b90d78e6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
@@ -20,6 +20,7 @@
 
 import akka.actor.ActorSystem;
 import akka.testkit.JavaTestKit;
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
@@ -37,7 +38,10 @@
 import org.apache.flink.runtime.heartbeat.TestingHeartbeatServices;
 import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
 import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
+import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.metrics.MetricRegistry;
@@ -412,27 +416,32 @@ public void testHeartbeatTimeoutWithTaskExecutor() throws Exception {
 
 			final SlotReport slotReport = new SlotReport();
 			// test registration response successful and it will trigger monitor heartbeat target, schedule heartbeat request at interval time
-			Future<RegistrationResponse> successfulFuture =
-					resourceManager.registerTaskExecutor(rmLeaderSessionId, taskManagerAddress, taskManagerResourceID, slotReport);
+			Future<RegistrationResponse> successfulFuture = resourceManager.registerTaskExecutor(
+				rmLeaderSessionId,
+				taskManagerAddress,
+				taskManagerResourceID,
+				slotReport);
 			RegistrationResponse response = successfulFuture.get(5, TimeUnit.SECONDS);
 			assertTrue(response instanceof TaskExecutorRegistrationSuccess);
 
 			ArgumentCaptor<Runnable> heartbeatRunnableCaptor = ArgumentCaptor.forClass(Runnable.class);
-			verify(scheduledExecutor, times(1)).scheduleAtFixedRate(
+			verify(scheduledExecutor, times(2)).scheduleAtFixedRate(
 				heartbeatRunnableCaptor.capture(),
 				eq(0L),
 				eq(heartbeatInterval),
 				eq(TimeUnit.MILLISECONDS));
 
-			Runnable heartbeatRunnable = heartbeatRunnableCaptor.getValue();
+			List<Runnable> heartbeatRunnable = heartbeatRunnableCaptor.getAllValues();
 
 			ArgumentCaptor<Runnable> timeoutRunnableCaptor = ArgumentCaptor.forClass(Runnable.class);
 			verify(scheduledExecutor).schedule(timeoutRunnableCaptor.capture(), eq(heartbeatTimeout), eq(TimeUnit.MILLISECONDS));
 
 			Runnable timeoutRunnable = timeoutRunnableCaptor.getValue();
 
-			// run the first heartbeat request
-			heartbeatRunnable.run();
+			// run all the heartbeat requests
+			for (Runnable runnable : heartbeatRunnable) {
+				runnable.run();
+			}
 
 			verify(taskExecutorGateway, times(1)).heartbeatFromResourceManager(eq(resourceManagerResourceID));
 
@@ -445,4 +454,99 @@ public void testHeartbeatTimeoutWithTaskExecutor() throws Exception {
 			rpcService.stopService();
 		}
 	}
+
+	@Test
+	public void testHeartbeatTimeoutWithJobManager() throws Exception {
+		final String jobMasterAddress = "jm";
+		final ResourceID jmResourceId = new ResourceID(jobMasterAddress);
+		final ResourceID rmResourceId = ResourceID.generate();
+		final UUID rmLeaderId = UUID.randomUUID();
+		final UUID jmLeaderId = UUID.randomUUID();
+		final JobID jobId = new JobID();
+
+		final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
+
+		final TestingSerialRpcService rpcService = new TestingSerialRpcService();
+		rpcService.registerGateway(jobMasterAddress, jobMasterGateway);
+
+		final ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(
+			Time.seconds(5L),
+			Time.seconds(5L));
+
+		final TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
+		final TestingLeaderRetrievalService jmLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jmLeaderId);
+		final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
+		highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
+		highAvailabilityServices.setJobMasterLeaderRetriever(jobId, jmLeaderRetrievalService);
+
+		final long heartbeatInterval = 1L;
+		final long heartbeatTimeout = 5L;
+		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
+		final HeartbeatServices heartbeatServices = new TestingHeartbeatServices(heartbeatInterval, heartbeatTimeout, scheduledExecutor);
+
+		final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
+		final MetricRegistry metricRegistry = mock(MetricRegistry.class);
+		final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
+			highAvailabilityServices,
+			rpcService.getScheduledExecutor(),
+			Time.minutes(5L));
+		final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+
+		try {
+			final StandaloneResourceManager resourceManager = new StandaloneResourceManager(
+				rpcService,
+				FlinkResourceManager.RESOURCE_MANAGER_NAME,
+				rmResourceId,
+				resourceManagerConfiguration,
+				highAvailabilityServices,
+				heartbeatServices,
+				slotManagerFactory,
+				metricRegistry,
+				jobLeaderIdService,
+				testingFatalErrorHandler);
+
+			resourceManager.start();
+
+			rmLeaderElectionService.isLeader(rmLeaderId);
+
+			// test registration response successful and it will trigger monitor heartbeat target, schedule heartbeat request at interval time
+			Future<RegistrationResponse> successfulFuture = resourceManager.registerJobManager(
+				rmLeaderId,
+				jmLeaderId,
+				jmResourceId,
+				jobMasterAddress,
+				jobId);
+			RegistrationResponse response = successfulFuture.get(5, TimeUnit.SECONDS);
+			assertTrue(response instanceof JobMasterRegistrationSuccess);
+
+			ArgumentCaptor<Runnable> heartbeatRunnableCaptor = ArgumentCaptor.forClass(Runnable.class);
+			verify(scheduledExecutor, times(2)).scheduleAtFixedRate(
+				heartbeatRunnableCaptor.capture(),
+				eq(0L),
+				eq(heartbeatInterval),
+				eq(TimeUnit.MILLISECONDS));
+
+			List<Runnable> heartbeatRunnable = heartbeatRunnableCaptor.getAllValues();
+
+			ArgumentCaptor<Runnable> timeoutRunnableCaptor = ArgumentCaptor.forClass(Runnable.class);
+			verify(scheduledExecutor).schedule(timeoutRunnableCaptor.capture(), eq(heartbeatTimeout), eq(TimeUnit.MILLISECONDS));
+
+			Runnable timeoutRunnable = timeoutRunnableCaptor.getValue();
+
+			// run all the heartbeat requests
+			for (Runnable runnable : heartbeatRunnable) {
+				runnable.run();
+			}
+
+			verify(jobMasterGateway, times(1)).heartbeatFromResourceManager(eq(rmResourceId));
+
+			// run the timeout runnable to simulate a heartbeat timeout
+			timeoutRunnable.run();
+
+			verify(jobMasterGateway).disconnectResourceManager(eq(jmLeaderId), eq(rmLeaderId), any(TimeoutException.class));
+
+		} finally {
+			rpcService.stopService();
+		}
+	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
index 8b9b800644896..0b25e6cdb3854 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
@@ -18,11 +18,13 @@
 
 package org.apache.flink.runtime.jobmaster;
 
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.ScheduledExecutor;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
 import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
 import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoader;
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
@@ -32,6 +34,8 @@
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobmanager.OnCompletionActions;
 import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
+import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
@@ -52,9 +56,8 @@
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.when;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(BlobLibraryCacheManager.class)
@@ -144,5 +147,85 @@ public void testHeartbeatTimeoutWithTaskManager() throws Exception {
 		}
 	}
 
+	@Test
+	public void testHeartbeatTimeoutWithResourceManager() throws Exception {
+		final String resourceManagerAddress = "rm";
+		final String jobManagerAddress = "jm";
+		final UUID rmLeaderId = UUID.randomUUID();
+		final UUID jmLeaderId = UUID.randomUUID();
+		final ResourceID rmResourceId = new ResourceID(resourceManagerAddress);
+		final ResourceID jmResourceId = new ResourceID(jobManagerAddress);
+		final JobGraph jobGraph = new JobGraph();
+
+		final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
+		final TestingLeaderRetrievalService rmLeaderRetrievalService = new TestingLeaderRetrievalService(
+			null,
+			null);
+		haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
+		haServices.setCheckpointRecoveryFactory(mock(CheckpointRecoveryFactory.class));
+
+		final long heartbeatInterval = 1L;
+		final long heartbeatTimeout = 5L;
+		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
+		final HeartbeatServices heartbeatServices = new TestingHeartbeatServices(heartbeatInterval, heartbeatTimeout, scheduledExecutor);
+
+		final ResourceManagerGateway resourceManagerGateway = mock(ResourceManagerGateway.class);
+		when(resourceManagerGateway.registerJobManager(
+			any(UUID.class),
+			any(UUID.class),
+			any(ResourceID.class),
+			anyString(),
+			any(JobID.class),
+			any(Time.class)
+		)).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JobMasterRegistrationSuccess(
+			heartbeatInterval, rmLeaderId, rmResourceId)));
+
+		final TestingSerialRpcService rpc = new TestingSerialRpcService();
+		rpc.registerGateway(resourceManagerAddress, resourceManagerGateway);
+
+		final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+
+		try {
+			final JobMaster jobMaster = new JobMaster(
+				rpc,
+				jmResourceId,
+				jobGraph,
+				new Configuration(),
+				haServices,
+				heartbeatServices,
+				Executors.newScheduledThreadPool(1),
+				mock(BlobLibraryCacheManager.class),
+				mock(RestartStrategyFactory.class),
+				Time.of(10, TimeUnit.SECONDS),
+				null,
+				mock(OnCompletionActions.class),
+				testingFatalErrorHandler,
+				new FlinkUserCodeClassLoader(new URL[0]));
+
+			jobMaster.start(jmLeaderId);
+
+			// define a leader and see that a registration happens
+			rmLeaderRetrievalService.notifyListener(resourceManagerAddress, rmLeaderId);
+
+			// register job manager success will trigger monitor heartbeat target between jm and rm
+			verify(resourceManagerGateway).registerJobManager(
+				eq(rmLeaderId),
+				eq(jmLeaderId),
+				eq(jmResourceId),
+				anyString(),
+				eq(jobGraph.getJobID()),
+				any(Time.class));
+
+			// heartbeat timeout should trigger disconnect JobManager from ResourceManager
+			verify(resourceManagerGateway, timeout(heartbeatTimeout * 50L)).disconnectJobManager(eq(jobGraph.getJobID()), any(TimeoutException.class));
+
+			// check if a concurrent error occurred
+			testingFatalErrorHandler.rethrowError();
+
+		} finally {
+			rpc.stopService();
+		}
+	}
+
 
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
index b6b86141ab4d6..6a151ac8f6df6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
@@ -70,13 +70,19 @@ public void testRegisterJobMaster() throws Exception {
 		JobID jobID = mockJobMaster(jobMasterAddress);
 		TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService();
 		UUID jmLeaderID = UUID.randomUUID();
+		final ResourceID jmResourceId = new ResourceID(jobMasterAddress);
 		TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jmLeaderID);
 		TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
 		final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
 		final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
 
 		// test response successful
-		Future<RegistrationResponse> successfulFuture = resourceManager.registerJobManager(rmLeaderSessionId, jmLeaderID, jobMasterAddress, jobID);
+		Future<RegistrationResponse> successfulFuture = resourceManager.registerJobManager(
+			rmLeaderSessionId,
+			jmLeaderID,
+			jmResourceId,
+			jobMasterAddress,
+			jobID);
 		RegistrationResponse response = successfulFuture.get(5L, TimeUnit.SECONDS);
 		assertTrue(response instanceof JobMasterRegistrationSuccess);
 
@@ -94,6 +100,7 @@ public void testRegisterJobMasterWithUnmatchedLeaderSessionId1() throws Exceptio
 		JobID jobID = mockJobMaster(jobMasterAddress);
 		TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService();
 		UUID jmLeaderID = UUID.randomUUID();
+		final ResourceID jmResourceId = new ResourceID(jobMasterAddress);
 		TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jmLeaderID);
 		TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
 		final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
@@ -101,7 +108,12 @@ public void testRegisterJobMasterWithUnmatchedLeaderSessionId1() throws Exceptio
 
 		// test throw exception when receive a registration from job master which takes unmatched leaderSessionId
 		UUID differentLeaderSessionID = UUID.randomUUID();
-		Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobManager(differentLeaderSessionID, jmLeaderID, jobMasterAddress, jobID);
+		Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobManager(
+			differentLeaderSessionID,
+			jmLeaderID,
+			jmResourceId,
+			jobMasterAddress,
+			jobID);
 		assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
 
 		if (testingFatalErrorHandler.hasExceptionOccurred()) {
@@ -124,10 +136,16 @@ public void testRegisterJobMasterWithUnmatchedLeaderSessionId2() throws Exceptio
 		final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
 		final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
 		final UUID jmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
+		final ResourceID jmResourceId = new ResourceID(jobMasterAddress);
 
 		// test throw exception when receive a registration from job master which takes unmatched leaderSessionId
 		UUID differentLeaderSessionID = UUID.randomUUID();
-		Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobManager(rmLeaderSessionId, differentLeaderSessionID, jobMasterAddress, jobID);
+		Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobManager(
+			rmLeaderSessionId,
+			differentLeaderSessionID,
+			jmResourceId,
+			jobMasterAddress,
+			jobID);
 		assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
 
 		if (testingFatalErrorHandler.hasExceptionOccurred()) {
@@ -150,10 +168,16 @@ public void testRegisterJobMasterFromInvalidAddress() throws Exception {
 		final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
 		final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
 		final UUID jmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
+		final ResourceID jmResourceId = new ResourceID(jobMasterAddress);
 
 		// test throw exception when receive a registration from job master which takes invalid address
 		String invalidAddress = "/jobMasterAddress2";
-		Future<RegistrationResponse> invalidAddressFuture = resourceManager.registerJobManager(rmLeaderSessionId, jmLeaderSessionId, invalidAddress, jobID);
+		Future<RegistrationResponse> invalidAddressFuture = resourceManager.registerJobManager(
+			rmLeaderSessionId,
+			jmLeaderSessionId,
+			jmResourceId,
+			invalidAddress,
+			jobID);
 		assertTrue(invalidAddressFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
 
 		if (testingFatalErrorHandler.hasExceptionOccurred()) {
@@ -176,10 +200,16 @@ public void testRegisterJobMasterWithFailureLeaderListener() throws Exception {
 		final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
 		final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
 		final UUID jmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
+		final ResourceID jmResourceId = new ResourceID(jobMasterAddress);
 
 		JobID unknownJobIDToHAServices = new JobID();
 		// verify return RegistrationResponse.Decline when failed to start a job master Leader retrieval listener
-		Future<RegistrationResponse> declineFuture = resourceManager.registerJobManager(rmLeaderSessionId, jmLeaderSessionId, jobMasterAddress, unknownJobIDToHAServices);
+		Future<RegistrationResponse> declineFuture = resourceManager.registerJobManager(
+			rmLeaderSessionId,
+			jmLeaderSessionId,
+			jmResourceId,
+			jobMasterAddress,
+			unknownJobIDToHAServices);
 		RegistrationResponse response = declineFuture.get(5, TimeUnit.SECONDS);
 		assertTrue(response instanceof RegistrationResponse.Decline);
 
@@ -205,7 +235,7 @@ private ResourceManager createAndStartResourceManager(
 		highAvailabilityServices.setResourceManagerLeaderElectionService(resourceManagerLeaderElectionService);
 		highAvailabilityServices.setJobMasterLeaderRetriever(jobID, jobMasterLeaderRetrievalService);
 
-		HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
+		HeartbeatServices heartbeatServices = new HeartbeatServices(5L, 5L);
 
 		ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(
 			Time.seconds(5L),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 4d2309a1e5f63..37690b546f5e2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -101,6 +101,7 @@ public void testSlotsUnavailableRequest() throws Exception {
 		final String jmAddress = "/jm1";
 		final JobID jobID = new JobID();
 		final ResourceID rmResourceId = new ResourceID(rmAddress);
+		final ResourceID jmResourceId = new ResourceID(jmAddress);
 
 		testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
 
@@ -138,7 +139,7 @@ public void testSlotsUnavailableRequest() throws Exception {
 		rmLeaderElectionService.isLeader(rmLeaderID);
 
 		Future<RegistrationResponse> registrationFuture =
-			resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
+			resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmResourceId, jmAddress, jobID);
 		try {
 			registrationFuture.get(5, TimeUnit.SECONDS);
 		} catch (Exception e) {
@@ -207,6 +208,7 @@ public void testSlotAvailableRequest() throws Exception {
 		final String tmAddress = "/tm1";
 		final JobID jobID = new JobID();
 		final ResourceID rmResourceId = new ResourceID(rmAddress);
+		final ResourceID jmResourceId = new ResourceID(jmAddress);
 
 		testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
 
@@ -254,7 +256,7 @@ public void testSlotAvailableRequest() throws Exception {
 		Thread.sleep(1000);
 
 		Future<RegistrationResponse> registrationFuture =
-			resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
+			resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmResourceId, jmAddress, jobID);
 		try {
 			registrationFuture.get(5L, TimeUnit.SECONDS);
 		} catch (Exception e) {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
index fa326b5735a25..579ca3a9cac0e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
@@ -90,6 +90,7 @@ public void testSlotAllocation() throws Exception {
 		final String jmAddress = "jm";
 		final UUID jmLeaderId = UUID.randomUUID();
 		final ResourceID rmResourceId = new ResourceID(rmAddress);
+		final ResourceID jmResourceId = new ResourceID(jmAddress);
 		final JobID jobId = new JobID();
 		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
 
@@ -180,7 +181,12 @@ public void testSlotAllocation() throws Exception {
 			// notify the TM about the new RM leader
 			rmLeaderRetrievalService.notifyListener(rmAddress, rmLeaderId);
 
-			Future<RegistrationResponse> registrationResponseFuture = resourceManager.registerJobManager(rmLeaderId, jmLeaderId, jmAddress, jobId);
+			Future<RegistrationResponse> registrationResponseFuture = resourceManager.registerJobManager(
+				rmLeaderId,
+				jmLeaderId,
+				jmResourceId,
+				jmAddress,
+				jobId);
 
 			RegistrationResponse registrationResponse = registrationResponseFuture.get();
 

From d2e5c3a57d42226bc43b72ab719cad51464f19bd Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 27 Apr 2017 22:31:10 +0200
Subject: [PATCH 074/253] [FLINK-6404] [FLINK-6400] Ensure PendingCheckpoint is
 registered when calling Checkpoint Hooks

---
 .../checkpoint/CheckpointCoordinator.java     | 24 ++++------
 .../runtime/checkpoint/PendingCheckpoint.java | 28 ++++++-----
 .../CheckpointCoordinatorMasterHooksTest.java | 46 +++++++++++++++++++
 3 files changed, 70 insertions(+), 28 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 23a38d4847f2c..fb6cc72a9c561 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -41,7 +41,6 @@
 import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
 import org.apache.flink.runtime.state.TaskStateHandles;
 import org.apache.flink.runtime.taskmanager.DispatcherThreadFactory;
-import org.apache.flink.util.FlinkException;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.StringUtils;
 
@@ -540,20 +539,6 @@ CheckpointTriggerResult triggerCheckpoint(
 				checkpoint.setStatsCallback(callback);
 			}
 
-			// trigger the master hooks for the checkpoint
-			try {
-				List<MasterState> masterStates = MasterHooks.triggerMasterHooks(masterHooks.values(),
-						checkpointID, timestamp, executor, Time.milliseconds(checkpointTimeout));
-
-				for (MasterState s : masterStates) {
-					checkpoint.addMasterState(s);
-				}
-			}
-			catch (FlinkException e) {
-				checkpoint.abortError(e);
-				return new CheckpointTriggerResult(CheckpointDeclineReason.EXCEPTION);
-			}
-
 			// schedule the timer that will clean up the expired checkpoints
 			final Runnable canceller = new Runnable() {
 				@Override
@@ -628,6 +613,13 @@ else if (!props.forceCheckpoint()) {
 						// checkpoint is already disposed!
 						cancellerHandle.cancel(false);
 					}
+
+					// trigger the master hooks for the checkpoint
+					final List<MasterState> masterStates = MasterHooks.triggerMasterHooks(masterHooks.values(),
+							checkpointID, timestamp, executor, Time.milliseconds(checkpointTimeout));
+					for (MasterState s : masterStates) {
+						checkpoint.addMasterState(s);
+					}
 				}
 				// end of lock scope
 
@@ -656,7 +648,7 @@ else if (!props.forceCheckpoint()) {
 				LOG.warn("Failed to trigger checkpoint (" + numUnsuccessful + " consecutive failed attempts so far)", t);
 
 				if (!checkpoint.isDiscarded()) {
-					checkpoint.abortError(new Exception("Failed to trigger checkpoint"));
+					checkpoint.abortError(new Exception("Failed to trigger checkpoint", t));
 				}
 				return new CheckpointTriggerResult(CheckpointDeclineReason.EXCEPTION);
 			}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
index ce97edc676ca1..cc3dce2e6622e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
@@ -350,17 +350,6 @@ private CompletedCheckpoint finalizeInternal(
 		return completed;
 	}
 
-	/**
-	 * Adds a master state (state generated on the checkpoint coordinator) to
-	 * the pending checkpoint.
-	 * 
-	 * @param state The state to add
-	 */
-	public void addMasterState(MasterState state) {
-		checkNotNull(state);
-		masterState.add(state);
-	}
-
 	/**
 	 * Acknowledges the task with the given execution attempt id and the given subtask state.
 	 *
@@ -453,7 +442,22 @@ public TaskAcknowledgeResult acknowledgeTask(
 		}
 	}
 
-	
+	/**
+	 * Adds a master state (state generated on the checkpoint coordinator) to
+	 * the pending checkpoint.
+	 *
+	 * @param state The state to add
+	 */
+	public void addMasterState(MasterState state) {
+		checkNotNull(state);
+
+		synchronized (lock) {
+			if (!discarded) {
+				masterState.add(state);
+			}
+		}
+	}
+
 
 	// ------------------------------------------------------------------------
 	//  Cancellation
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
index 0ec46065b3606..7c271a7f297b4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
@@ -21,6 +21,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.core.io.SimpleVersionedSerializer;
 import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
@@ -31,6 +32,9 @@
 
 import org.junit.Test;
 
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -46,6 +50,7 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -312,6 +317,47 @@ public void checkUnMatchedStateOnRestore() throws Exception {
 		verify(statelessHook, times(1)).restoreCheckpoint(eq(checkpointId), isNull(Void.class));
 	}
 
+	// ------------------------------------------------------------------------
+	//  failure scenarios
+	// ------------------------------------------------------------------------
+
+	/**
+	 * This test makes sure that the checkpoint is already registered by the time
+	 * that the hooks are called
+	 */
+	@Test
+	public void ensureRegisteredAtHookTime() throws Exception {
+		final String id = "id";
+
+		// create the checkpoint coordinator
+		final JobID jid = new JobID();
+		final ExecutionAttemptID execId = new ExecutionAttemptID();
+		final ExecutionVertex ackVertex = mockExecutionVertex(execId);
+		final CheckpointCoordinator cc = instantiateCheckpointCoordinator(jid, ackVertex);
+
+		final MasterTriggerRestoreHook<Void> hook = mockGeneric(MasterTriggerRestoreHook.class);
+		when(hook.getIdentifier()).thenReturn(id);
+		when(hook.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class))).thenAnswer(
+				new Answer<Future<Void>>() {
+
+					@Override
+					public Future<Void> answer(InvocationOnMock invocation) throws Throwable {
+						assertEquals(1, cc.getNumberOfPendingCheckpoints());
+
+						long checkpointId = (Long) invocation.getArguments()[0];
+						assertNotNull(cc.getPendingCheckpoints().get(checkpointId));
+						return null;
+					}
+				}
+		);
+
+		cc.addMasterHook(hook);
+
+		// trigger a checkpoint
+		assertTrue(cc.triggerCheckpoint(System.currentTimeMillis(), false));
+	}
+
+
 	// ------------------------------------------------------------------------
 	//  failure scenarios
 	// ------------------------------------------------------------------------

From 2ab2e05ae68b46d42257dfe7b7ea0c3116e9bc5b Mon Sep 17 00:00:00 2001
From: Addison Higham <ahigham@instructure.com>
Date: Mon, 6 Mar 2017 23:40:16 -0700
Subject: [PATCH 075/253] [FLINK-5975] Add volume support to flink-mesos

When using containerization, specifically, docker, it is useful to be
able to attach additional volumes, such as an NFS share.

This adds support for volumes to be attached via specifying a new config
values `mesos.resourcemanager.tasks.container.volumes`. This is comma
delimited string of `[host_path:]container_path[:RO|RW]`.

It is modeled after the spark mesos framework

[FLINK-5975] Address code review, simplify handling of modes

[FLINK-5975] Fix volume config key to match actual

[FLINK-5975] make building volume info eager

[FLINK-5975] add volume info before setting container info

[FLINK-5975] Always set volumes even in no image

In the event that no container is used, we should still be able to use
volumes with the mesos containerizer.

This closes #3481.
---
 docs/setup/config.md                          |   8 +-
 docs/setup/mesos.md                           |   2 +
 .../LaunchableMesosWorker.java                |  46 ++++----
 .../MesosTaskManagerParameters.java           | 101 ++++++++++++++++--
 .../MesosFlinkResourceManagerTest.java        |   8 +-
 .../MesosTaskManagerParametersTest.java       |  77 +++++++++++++
 6 files changed, 205 insertions(+), 37 deletions(-)
 create mode 100644 flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java

diff --git a/docs/setup/config.md b/docs/setup/config.md
index e36f1492d47c1..d8a70820cbc02 100644
--- a/docs/setup/config.md
+++ b/docs/setup/config.md
@@ -152,7 +152,7 @@ Below is a list of currently first-class supported connectors or components by F
 
 - Zookeeper (for HA): see [here]({{site.baseurl}}/setup/jobmanager_high_availability.html#configuring-for-zookeeper-security) for details on Zookeeper security configuration to work with the Kerberos-based security configurations mentioned here.
 
-For more information on how Flink security internally setups Kerberos authentication, please see [here]({{site.baseurl}}/ops/security-kerberos.html). 
+For more information on how Flink security internally setups Kerberos authentication, please see [here]({{site.baseurl}}/ops/security-kerberos.html).
 
 ### Other
 
@@ -196,7 +196,7 @@ will be used under the directory specified by jobmanager.web.tmpdir.
 
 - `blob.service.ssl.enabled`: Flag to enable ssl for the blob client/server communication. This is applicable only when the global ssl flag security.ssl.enabled is set to true (DEFAULT: true).
 
-- `restart-strategy`: Default [restart strategy]({{site.baseurl}}/dev/restart_strategies.html) to use in case no 
+- `restart-strategy`: Default [restart strategy]({{site.baseurl}}/dev/restart_strategies.html) to use in case no
 restart strategy has been specified for the job.
 The options are:
     - fixed delay strategy: `fixed-delay`.
@@ -209,7 +209,7 @@ The options are:
 Default value is 1, unless "fixed-delay" was activated by enabling checkpoints, in which case the default is `Integer.MAX_VALUE`.
 
 - `restart-strategy.fixed-delay.delay`: Delay between restart attempts, used if the default restart strategy is set to "fixed-delay".
-Default value is the `akka.ask.timeout`, unless "fixed-delay" was activated by enabling checkpoints, in which case 
+Default value is the `akka.ask.timeout`, unless "fixed-delay" was activated by enabling checkpoints, in which case
 the default is 10s.
 
 - `restart-strategy.failure-rate.max-failures-per-interval`: Maximum number of restarts in given time interval before failing a job in "failure-rate" strategy.
@@ -484,6 +484,8 @@ May be set to -1 to disable this feature.
 
 - `mesos.resourcemanager.tasks.container.image.name`: Image name to use for the container (**NO DEFAULT**)
 
+- `mesos.resourcemanager.tasks.container.volumes`: A comma seperated list of [host_path:]container_path[:RO|RW]. This allows for mounting additional volumes into your container. (**NO DEFAULT**)
+
 - `high-availability.zookeeper.path.mesos-workers`: The ZooKeeper root path for persisting the Mesos worker information.
 
 ### High Availability (HA)
diff --git a/docs/setup/mesos.md b/docs/setup/mesos.md
index 2ccee65c9781e..62168e76cae23 100644
--- a/docs/setup/mesos.md
+++ b/docs/setup/mesos.md
@@ -257,3 +257,5 @@ May be set to -1 to disable this feature.
 `mesos.resourcemanager.tasks.container.type`: Type of the containerization used: "mesos" or "docker" (DEFAULT: mesos);
 
 `mesos.resourcemanager.tasks.container.image.name`: Image name to use for the container (**NO DEFAULT**)
+
+`mesos.resourcemanager.tasks.container.volumes`: A comma seperated list of [host_path:]container_path[:RO|RW]. This allows for mounting additional volumes into your container. (**NO DEFAULT**)
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
index bfe9be818579e..ff5472e8bd581 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
@@ -28,7 +28,10 @@
 import org.apache.flink.mesos.util.MesosArtifactResolver;
 import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
 import org.apache.flink.runtime.clusterframework.ContainerSpecification;
+import org.apache.flink.util.Preconditions;
 import org.apache.mesos.Protos;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Collections;
 import java.util.List;
@@ -44,10 +47,11 @@
  * Implements the launch of a Mesos worker.
  *
  * Translates the abstract {@link ContainerSpecification} into a concrete
- * Mesos-specific {@link org.apache.mesos.Protos.TaskInfo}.
+ * Mesos-specific {@link Protos.TaskInfo}.
  */
 public class LaunchableMesosWorker implements LaunchableTask {
 
+	protected static final Logger LOG = LoggerFactory.getLogger(LaunchableMesosWorker.class);
 	/**
 	 * The set of configuration keys to be dynamically configured with a port allocated from Mesos.
 	 */
@@ -69,12 +73,14 @@ public class LaunchableMesosWorker implements LaunchableTask {
 	 * @param taskID the taskID for this worker.
 	 */
 	public LaunchableMesosWorker(
-		MesosArtifactResolver resolver, MesosTaskManagerParameters params,
-		ContainerSpecification containerSpec, Protos.TaskID taskID) {
-		this.resolver = resolver;
-		this.params = params;
-		this.containerSpec = containerSpec;
-		this.taskID = taskID;
+			MesosArtifactResolver resolver,
+			MesosTaskManagerParameters params,
+			ContainerSpecification containerSpec,
+			Protos.TaskID taskID) {
+		this.resolver = Preconditions.checkNotNull(resolver);
+		this.params = Preconditions.checkNotNull(params);
+		this.containerSpec = Preconditions.checkNotNull(containerSpec);
+		this.taskID = Preconditions.checkNotNull(taskID);
 		this.taskRequest = new Request();
 	}
 
@@ -232,23 +238,25 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assig
 		cmd.setValue(launchCommand.toString());
 
 		// build the container info
-		Protos.ContainerInfo.Builder containerInfo = null;
+		Protos.ContainerInfo.Builder containerInfo = Protos.ContainerInfo.newBuilder();
+		// in event that no docker image or mesos image name is specified, we must still
+		// set type to MESOS
+		containerInfo.setType(Protos.ContainerInfo.Type.MESOS);
 		switch(params.containerType()) {
 			case MESOS:
 				if(params.containerImageName().isDefined()) {
-					containerInfo = Protos.ContainerInfo.newBuilder()
-						.setType(Protos.ContainerInfo.Type.MESOS)
+					containerInfo
 						.setMesos(Protos.ContainerInfo.MesosInfo.newBuilder()
-						.setImage(Protos.Image.newBuilder()
-							.setType(Protos.Image.Type.DOCKER)
-							.setDocker(Protos.Image.Docker.newBuilder()
-								.setName(params.containerImageName().get()))));
+							.setImage(Protos.Image.newBuilder()
+								.setType(Protos.Image.Type.DOCKER)
+								.setDocker(Protos.Image.Docker.newBuilder()
+									.setName(params.containerImageName().get()))));
 				}
 				break;
 
 			case DOCKER:
 				assert(params.containerImageName().isDefined());
-				containerInfo = Protos.ContainerInfo.newBuilder()
+					containerInfo
 					.setType(Protos.ContainerInfo.Type.DOCKER)
 					.setDocker(Protos.ContainerInfo.DockerInfo.newBuilder()
 						.setNetwork(Protos.ContainerInfo.DockerInfo.Network.HOST)
@@ -258,9 +266,11 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assig
 			default:
 				throw new IllegalStateException("unsupported container type");
 		}
-		if(containerInfo != null) {
-			taskInfo.setContainer(containerInfo);
-		}
+
+		// add any volumes to the containerInfo
+		containerInfo.addAllVolumes(params.containerVolumes());
+		taskInfo.setContainer(containerInfo);
+
 
 		return taskInfo.build();
 	}
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
index 044bffe2aad3a..6064c46bb17dd 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
@@ -23,9 +23,14 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.util.Preconditions;
+import org.apache.mesos.Protos;
 import scala.Option;
 
-import static java.util.Objects.requireNonNull;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
 import static org.apache.flink.configuration.ConfigOptions.key;
 
 /**
@@ -56,6 +61,9 @@ public class MesosTaskManagerParameters {
 		key("mesos.resourcemanager.tasks.container.image.name")
 			.noDefaultValue();
 
+	public static final ConfigOption<String> MESOS_RM_CONTAINER_VOLUMES =
+		key("mesos.resourcemanager.tasks.container.volumes")
+			.noDefaultValue();
 	/**
 	 * Value for {@code MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE} setting. Tells to use the Mesos containerizer.
 	 */
@@ -73,19 +81,24 @@ public class MesosTaskManagerParameters {
 
 	private final ContaineredTaskManagerParameters containeredParameters;
 
+	private final List<Protos.Volume> containerVolumes;
+
 	public MesosTaskManagerParameters(
-		double cpus,
-		ContainerType containerType,
-		Option<String> containerImageName,
-		ContaineredTaskManagerParameters containeredParameters) {
-		requireNonNull(containeredParameters);
+			double cpus,
+			ContainerType containerType,
+			Option<String> containerImageName,
+			ContaineredTaskManagerParameters containeredParameters,
+			List<Protos.Volume> containerVolumes) {
+
 		this.cpus = cpus;
-		this.containerType = containerType;
-		this.containerImageName = containerImageName;
-		this.containeredParameters = containeredParameters;
+		this.containerType = Preconditions.checkNotNull(containerType);
+		this.containerImageName = Preconditions.checkNotNull(containerImageName);
+		this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
+		this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
 	}
 
-	/**
+
+    /**
 	 * Get the CPU units to use for the TaskManager process.
      */
 	public double cpus() {
@@ -115,6 +128,13 @@ public ContaineredTaskManagerParameters containeredParameters() {
 		return containeredParameters;
 	}
 
+	/**
+	 * Get the container volumes string
+	 */
+	public List<Protos.Volume> containerVolumes() {
+		return containerVolumes;
+	}
+
 	@Override
 	public String toString() {
 		return "MesosTaskManagerParameters{" +
@@ -122,6 +142,7 @@ public String toString() {
 			", containerType=" + containerType +
 			", containerImageName=" + containerImageName +
 			", containeredParameters=" + containeredParameters +
+			", containerVolumes=" + containerVolumes +
 			'}';
 	}
 
@@ -162,11 +183,69 @@ public static MesosTaskManagerParameters create(Configuration flinkConfig) {
 				throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
 		}
 
+		Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));
+
+		List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);
+
 		return new MesosTaskManagerParameters(
 			cpus,
 			containerType,
 			Option.apply(imageName),
-			containeredParameters);
+			containeredParameters,
+			containerVolumes);
+	}
+
+	/**
+	 * Used to build volume specs for mesos. This allows for mounting additional volumes into a container
+	 *
+	 * @param containerVolumes a comma delimited optional string of [host_path:]container_path[:RO|RW] that
+	 *                         defines mount points for a container volume. If None or empty string, returns
+	 *                         an empty iterator
+	 */
+	public static List<Protos.Volume> buildVolumes(Option<String> containerVolumes) {
+		if (containerVolumes.isEmpty()) {
+			return Collections.emptyList();
+		} else {
+			String[] volumeSpecifications = containerVolumes.get().split(",");
+
+			List<Protos.Volume> volumes = new ArrayList<>(volumeSpecifications.length);
+
+			for (String volumeSpecification : volumeSpecifications) {
+				if (!volumeSpecification.trim().isEmpty()) {
+					Protos.Volume.Builder volume = Protos.Volume.newBuilder();
+					volume.setMode(Protos.Volume.Mode.RW);
+
+					String[] parts = volumeSpecification.split(":");
+
+					switch (parts.length) {
+						case 1:
+							volume.setContainerPath(parts[0]);
+							break;
+						case 2:
+							try {
+								Protos.Volume.Mode mode = Protos.Volume.Mode.valueOf(parts[1].trim().toUpperCase());
+								volume.setMode(mode)
+									.setContainerPath(parts[0]);
+							} catch (IllegalArgumentException e) {
+								volume.setHostPath(parts[0])
+									.setContainerPath(parts[1]);
+							}
+							break;
+						case 3:
+							Protos.Volume.Mode mode = Protos.Volume.Mode.valueOf(parts[2].trim().toUpperCase());
+							volume.setMode(mode)
+								.setHostPath(parts[0])
+								.setContainerPath(parts[1]);
+							break;
+						default:
+							throw new IllegalArgumentException("volume specification is invalid, given: " + volumeSpecification);
+					}
+
+					volumes.add(volume.build());
+				}
+			}
+			return volumes;
+		}
 	}
 
 	public enum ContainerType {
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
index c854a1713ef87..745d35fecef77 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
@@ -58,11 +58,9 @@
 import org.slf4j.LoggerFactory;
 import scala.Option;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.*;
+
 import static java.util.Collections.singletonList;
-import java.util.HashMap;
 
 import static org.apache.flink.mesos.runtime.clusterframework.MesosFlinkResourceManager.extractGoalState;
 import static org.apache.flink.mesos.runtime.clusterframework.MesosFlinkResourceManager.extractResourceID;
@@ -202,7 +200,7 @@ public void initialize() {
 			ContaineredTaskManagerParameters containeredParams =
 				new ContaineredTaskManagerParameters(1024, 768, 256, 4, new HashMap<String, String>());
 			MesosTaskManagerParameters tmParams = new MesosTaskManagerParameters(
-				1.0, MesosTaskManagerParameters.ContainerType.MESOS, Option.<String>empty(), containeredParams);
+				1.0, MesosTaskManagerParameters.ContainerType.MESOS, Option.<String>empty(), containeredParams, new ArrayList<Protos.Volume>());
 
 			TestActorRef<TestingMesosFlinkResourceManager> resourceManagerRef =
 				TestActorRef.create(system, MesosFlinkResourceManager.createActorProps(
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
new file mode 100644
index 0000000000000..c4a7bcb453618
--- /dev/null
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.mesos.runtime.clusterframework;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.TestLogger;
+import org.apache.mesos.Protos;
+import org.junit.Test;
+import scala.Option;
+
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class MesosTaskManagerParametersTest extends TestLogger {
+
+	@Test
+	public void testBuildVolumes() throws Exception {
+		List<Protos.Volume> vols;
+		assertEquals(MesosTaskManagerParameters.buildVolumes(Option.<String>apply(null)).size(), 0);
+		String spec1 = "/host/path:/container/path:RO,/container/path:ro,/host/path:/container/path,/container/path";
+		vols = MesosTaskManagerParameters.buildVolumes(Option.<String>apply(spec1));
+		assertEquals(vols.size(), 4);
+		assertEquals("/container/path", vols.get(0).getContainerPath());
+		assertEquals("/host/path", vols.get(0).getHostPath());
+		assertEquals(Protos.Volume.Mode.RO, vols.get(0).getMode());
+		assertEquals("/container/path", vols.get(1).getContainerPath());
+		assertEquals(Protos.Volume.Mode.RO, vols.get(1).getMode());
+		assertEquals("/container/path", vols.get(2).getContainerPath());
+		assertEquals("/host/path", vols.get(2).getHostPath());
+		assertEquals(Protos.Volume.Mode.RW, vols.get(2).getMode());
+		assertEquals("/container/path", vols.get(3).getContainerPath());
+		assertEquals(Protos.Volume.Mode.RW, vols.get(3).getMode());
+
+		// should handle empty strings, but not error
+		assertEquals(0, MesosTaskManagerParameters.buildVolumes(Option.<String>apply("")).size());
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testBuildVolumesBadMode() throws Exception {
+		MesosTaskManagerParameters.buildVolumes(Option.<String>apply("/hp:/cp:RF"));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testBuildVolumesMalformed() throws Exception {
+		MesosTaskManagerParameters.buildVolumes(Option.<String>apply("/hp:/cp:ro:extra"));
+	}
+
+	@Test
+	public void testContainerVolumes() throws Exception {
+		Configuration config = new Configuration();
+		config.setString(MesosTaskManagerParameters.MESOS_RM_CONTAINER_VOLUMES, "/host/path:/container/path:ro");
+
+		MesosTaskManagerParameters params = MesosTaskManagerParameters.create(config);
+		assertEquals(1, params.containerVolumes().size());
+		assertEquals("/container/path", params.containerVolumes().get(0).getContainerPath());
+		assertEquals("/host/path", params.containerVolumes().get(0).getHostPath());
+		assertEquals(Protos.Volume.Mode.RO, params.containerVolumes().get(0).getMode());
+	}
+
+}

From a86961853466addf008b0dbc90af49ec404590b1 Mon Sep 17 00:00:00 2001
From: Stephen Gran <stephen.gran@piksel.com>
Date: Thu, 20 Apr 2017 10:22:08 +0100
Subject: [PATCH 076/253] [FLINK-6336] [mesos] Add mesos placement constraints

This adds a HostAttrValueConstraint evaluator for fenzo, allowing tasks
to be placed on agents with matching attributes.

Signed-off-by: Stephen Gran <stephen.gran@piksel.com>

This closes #3744.
---
 docs/setup/config.md                          |  2 +
 docs/setup/mesos.md                           |  4 ++
 .../LaunchableMesosWorker.java                |  2 +-
 .../MesosTaskManagerParameters.java           | 59 ++++++++++++++++-
 .../MesosFlinkResourceManagerTest.java        |  3 +-
 .../MesosTaskManagerParametersTest.java       | 65 +++++++++++++++++++
 6 files changed, 130 insertions(+), 5 deletions(-)

diff --git a/docs/setup/config.md b/docs/setup/config.md
index d8a70820cbc02..250d92ff31fd7 100644
--- a/docs/setup/config.md
+++ b/docs/setup/config.md
@@ -450,6 +450,8 @@ use the `env.java.opts` setting, which is the `%jvmopts%` variable in the String
 
 - `mesos.initial-tasks`: The initial workers to bring up when the master starts (**DEFAULT**: The number of workers specified at cluster startup).
 
+- `mesos.constraints.hard.hostattribute`: Constraints for task placement on mesos (**DEFAULT**: None).
+
 - `mesos.maximum-failed-tasks`: The maximum number of failed workers before the cluster fails (**DEFAULT**: Number of initial workers).
 May be set to -1 to disable this feature.
 
diff --git a/docs/setup/mesos.md b/docs/setup/mesos.md
index 62168e76cae23..4c72fbcc461bd 100644
--- a/docs/setup/mesos.md
+++ b/docs/setup/mesos.md
@@ -222,6 +222,10 @@ When running Flink with Marathon, the whole Flink cluster including the job mana
 
 `mesos.initial-tasks`: The initial workers to bring up when the master starts (**DEFAULT**: The number of workers specified at cluster startup).
 
+`mesos.constraints.hard.hostattribute`: Constraints for task placement on mesos based on agent attributes (**DEFAULT**: None).
+Takes a comma-separated list of key:value pairs corresponding to the attributes exposed by the target
+mesos agents.  Example: `az:eu-west-1a,series:t2`
+
 `mesos.maximum-failed-tasks`: The maximum number of failed workers before the cluster fails (**DEFAULT**: Number of initial workers).
 May be set to -1 to disable this feature.
 
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
index ff5472e8bd581..3d5350adfe062 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
@@ -138,7 +138,7 @@ public Map<String, NamedResourceSetRequest> getCustomNamedResources() {
 
 		@Override
 		public List<? extends ConstraintEvaluator> getHardConstraints() {
-			return null;
+			return params.constraints();
 		}
 
 		@Override
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
index 6064c46bb17dd..aaab027faaf55 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.mesos.runtime.clusterframework;
 
+import com.netflix.fenzo.ConstraintEvaluator;
+import com.netflix.fenzo.functions.Func1;
+import com.netflix.fenzo.plugins.HostAttrValueConstraint;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.Configuration;
@@ -63,7 +66,12 @@ public class MesosTaskManagerParameters {
 
 	public static final ConfigOption<String> MESOS_RM_CONTAINER_VOLUMES =
 		key("mesos.resourcemanager.tasks.container.volumes")
+		.noDefaultValue();
+	
+	public static final ConfigOption<String> MESOS_CONSTRAINTS_HARD_HOSTATTR =
+		key("mesos.constraints.hard.hostattribute")
 			.noDefaultValue();
+
 	/**
 	 * Value for {@code MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE} setting. Tells to use the Mesos containerizer.
 	 */
@@ -82,19 +90,23 @@ public class MesosTaskManagerParameters {
 	private final ContaineredTaskManagerParameters containeredParameters;
 
 	private final List<Protos.Volume> containerVolumes;
+	
+	private final List<ConstraintEvaluator> constraints;
 
 	public MesosTaskManagerParameters(
 			double cpus,
 			ContainerType containerType,
 			Option<String> containerImageName,
 			ContaineredTaskManagerParameters containeredParameters,
-			List<Protos.Volume> containerVolumes) {
+			List<Protos.Volume> containerVolumes,
+			List<ConstraintEvaluator> constraints) {
 
 		this.cpus = cpus;
 		this.containerType = Preconditions.checkNotNull(containerType);
 		this.containerImageName = Preconditions.checkNotNull(containerImageName);
 		this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
 		this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
+		this.constraints = Preconditions.checkNotNull(constraints);
 	}
 
 
@@ -135,6 +147,13 @@ public List<Protos.Volume> containerVolumes() {
 		return containerVolumes;
 	}
 
+	/**
+	 * Get the placement constraints
+	 */
+	public List<ConstraintEvaluator> constraints() {
+		return constraints;
+	}
+
 	@Override
 	public String toString() {
 		return "MesosTaskManagerParameters{" +
@@ -143,6 +162,7 @@ public String toString() {
 			", containerImageName=" + containerImageName +
 			", containeredParameters=" + containeredParameters +
 			", containerVolumes=" + containerVolumes +
+			", constraints=" + constraints +
 			'}';
 	}
 
@@ -152,6 +172,7 @@ public String toString() {
      */
 	public static MesosTaskManagerParameters create(Configuration flinkConfig) {
 
+		List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
 		// parse the common parameters
 		ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
 			flinkConfig,
@@ -191,10 +212,42 @@ public static MesosTaskManagerParameters create(Configuration flinkConfig) {
 			cpus,
 			containerType,
 			Option.apply(imageName),
-			containeredParameters,
-			containerVolumes);
+			containeredParameters,			
+			containerVolumes,
+			constraints);
+	}
+
+	private static List<ConstraintEvaluator> parseConstraints(String mesosConstraints) {
+
+		if (mesosConstraints == null || mesosConstraints.isEmpty()) {
+			return Collections.emptyList();
+		} else {
+			List<ConstraintEvaluator> constraints = new ArrayList<>();
+
+			for (String constraint : mesosConstraints.split(",")) {
+				if (constraint.isEmpty()) {
+					continue;
+				}
+				final String[] constraintList = constraint.split(":");
+				if (constraintList.length != 2) {
+					continue;
+				}
+				addHostAttrValueConstraint(constraints, constraintList[0], constraintList[1]);
+			}
+
+			return constraints;
+		}
 	}
 
+	private static void addHostAttrValueConstraint(List<ConstraintEvaluator> constraints, String constraintKey, final String constraintValue) {
+		constraints.add(new HostAttrValueConstraint(constraintKey, new Func1<String, String>() {
+			@Override
+			public String call(String s) {
+				return constraintValue;
+			}
+		}));
+	}
+	
 	/**
 	 * Used to build volume specs for mesos. This allows for mounting additional volumes into a container
 	 *
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
index 745d35fecef77..7915e8ae03f8b 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
@@ -24,6 +24,7 @@
 import akka.testkit.JavaTestKit;
 import akka.testkit.TestActorRef;
 import akka.testkit.TestProbe;
+import com.netflix.fenzo.ConstraintEvaluator;
 import junit.framework.AssertionFailedError;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
@@ -200,7 +201,7 @@ public void initialize() {
 			ContaineredTaskManagerParameters containeredParams =
 				new ContaineredTaskManagerParameters(1024, 768, 256, 4, new HashMap<String, String>());
 			MesosTaskManagerParameters tmParams = new MesosTaskManagerParameters(
-				1.0, MesosTaskManagerParameters.ContainerType.MESOS, Option.<String>empty(), containeredParams, new ArrayList<Protos.Volume>());
+				1.0, MesosTaskManagerParameters.ContainerType.MESOS, Option.<String>empty(), containeredParams, Collections.<Protos.Volume>emptyList(), Collections.<ConstraintEvaluator>emptyList());
 
 			TestActorRef<TestingMesosFlinkResourceManager> resourceManagerRef =
 				TestActorRef.create(system, MesosFlinkResourceManager.createActorProps(
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
index c4a7bcb453618..bc303e91ff34d 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParametersTest.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.mesos.runtime.clusterframework;
 
+import com.netflix.fenzo.ConstraintEvaluator;
+import com.netflix.fenzo.functions.Func1;
+import com.netflix.fenzo.plugins.HostAttrValueConstraint;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.util.TestLogger;
 import org.apache.mesos.Protos;
@@ -26,6 +29,7 @@
 
 import java.util.List;
 
+import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.*;
 
 public class MesosTaskManagerParametersTest extends TestLogger {
@@ -73,5 +77,66 @@ public void testContainerVolumes() throws Exception {
 		assertEquals("/host/path", params.containerVolumes().get(0).getHostPath());
 		assertEquals(Protos.Volume.Mode.RO, params.containerVolumes().get(0).getMode());
 	}
+	
+	@Test
+    public void givenTwoConstraintsInConfigShouldBeParsed() throws Exception {
+
+        MesosTaskManagerParameters mesosTaskManagerParameters = MesosTaskManagerParameters.create(withHardHostAttrConstraintConfiguration("cluster:foo,az:eu-west-1"));
+        assertThat(mesosTaskManagerParameters.constraints().size(), is(2));
+        ConstraintEvaluator firstConstraintEvaluator = new HostAttrValueConstraint("cluster", new Func1<String, String>() {
+            @Override
+            public String call(String s) {
+                return "foo";
+            }
+        });
+        ConstraintEvaluator secondConstraintEvaluator = new HostAttrValueConstraint("az", new Func1<String, String>() {
+            @Override
+            public String call(String s) {
+                return "foo";
+            }
+        });
+        assertThat(mesosTaskManagerParameters.constraints().get(0).getName(), is(firstConstraintEvaluator.getName()));
+        assertThat(mesosTaskManagerParameters.constraints().get(1).getName(), is(secondConstraintEvaluator.getName()));
+
+    }
+
+    @Test
+    public void givenOneConstraintInConfigShouldBeParsed() throws Exception {
+
+        MesosTaskManagerParameters mesosTaskManagerParameters = MesosTaskManagerParameters.create(withHardHostAttrConstraintConfiguration("cluster:foo"));
+        assertThat(mesosTaskManagerParameters.constraints().size(), is(1));
+        ConstraintEvaluator firstConstraintEvaluator = new HostAttrValueConstraint("cluster", new Func1<String, String>() {
+            @Override
+            public String call(String s) {
+                return "foo";
+            }
+        });
+        assertThat(mesosTaskManagerParameters.constraints().get(0).getName(), is(firstConstraintEvaluator.getName()));
+    }
+
+    @Test
+    public void givenEmptyConstraintInConfigShouldBeParsed() throws Exception {
+
+        MesosTaskManagerParameters mesosTaskManagerParameters = MesosTaskManagerParameters.create(withHardHostAttrConstraintConfiguration(""));
+        assertThat(mesosTaskManagerParameters.constraints().size(), is(0));
+    }
+
+    @Test
+    public void givenInvalidConstraintInConfigShouldBeParsed() throws Exception {
+
+        MesosTaskManagerParameters mesosTaskManagerParameters = MesosTaskManagerParameters.create(withHardHostAttrConstraintConfiguration(",:,"));
+        assertThat(mesosTaskManagerParameters.constraints().size(), is(0));
+    }
+
+
+    private static Configuration withHardHostAttrConstraintConfiguration(final String configuration) {
+        return new Configuration() {
+            private static final long serialVersionUID = -3249384117909445760L;
+
+            {
+                setString(MesosTaskManagerParameters.MESOS_CONSTRAINTS_HARD_HOSTATTR, configuration);
+            }
+        };
+    }
 
 }

From f3c797afca4dad61515079694f75f137b41578c1 Mon Sep 17 00:00:00 2001
From: Kurt Young <ykt836@gmail.com>
Date: Thu, 27 Apr 2017 23:37:21 +0800
Subject: [PATCH 077/253] [FLINK-6398] RowSerializer's duplicate should always
 return a new instance

This closes #3794.
---
 .../api/java/typeutils/runtime/RowSerializer.java   | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
index 827c2fa7ec585..dbd5d3a549925 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
@@ -53,22 +53,11 @@ public boolean isImmutableType() {
 
 	@Override
 	public TypeSerializer<Row> duplicate() {
-		boolean stateful = false;
 		TypeSerializer<?>[] duplicateFieldSerializers = new TypeSerializer[fieldSerializers.length];
-
 		for (int i = 0; i < fieldSerializers.length; i++) {
 			duplicateFieldSerializers[i] = fieldSerializers[i].duplicate();
-			if (duplicateFieldSerializers[i] != fieldSerializers[i]) {
-				// at least one of them is stateful
-				stateful = true;
-			}
-		}
-
-		if (stateful) {
-			return new RowSerializer(duplicateFieldSerializers);
-		} else {
-			return this;
 		}
+		return new RowSerializer(duplicateFieldSerializers);
 	}
 
 	@Override

From fb57e5d4b859f887bcc4c9eabd5bfb031661285d Mon Sep 17 00:00:00 2001
From: Bowen Li <bowenli86@gmail.com>
Date: Tue, 25 Apr 2017 19:55:25 -0700
Subject: [PATCH 078/253] [FLINK-6386] Missing bracket in 'Compiler Limitation'
 section

This closes #3775.
---
 docs/dev/java8.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/dev/java8.md b/docs/dev/java8.md
index e98f7489e433a..1912fb1e1c708 100644
--- a/docs/dev/java8.md
+++ b/docs/dev/java8.md
@@ -104,7 +104,7 @@ input.filter(line -> !line.contains("not"))
 Currently, Flink only supports jobs containing Lambda Expressions completely if they are **compiled with the Eclipse JDT compiler contained in Eclipse Luna 4.4.2 (and above)**.
 
 Only the Eclipse JDT compiler preserves the generic type information necessary to use the entire Lambda Expressions feature type-safely.
-Other compilers such as the OpenJDK's and Oracle JDK's `javac` throw away all generic parameters related to Lambda Expressions. This means that types such as `Tuple2<String,Integer` or `Collector<String>` declared as a Lambda function input or output parameter will be pruned to `Tuple2` or `Collector` in the compiled `.class` files, which is too little information for the Flink Compiler.
+Other compilers such as the OpenJDK's and Oracle JDK's `javac` throw away all generic parameters related to Lambda Expressions. This means that types such as `Tuple2<String, Integer>` or `Collector<String>` declared as a Lambda function input or output parameter will be pruned to `Tuple2` or `Collector` in the compiled `.class` files, which is too little information for the Flink compiler.
 
 How to compile a Flink job that contains Lambda Expressions with the JDT compiler will be covered in the next section.
 

From 325aa9eba3319696afac010a5417a91a39e12fd6 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Thu, 27 Oct 2016 00:24:12 +0200
Subject: [PATCH 079/253] [FLINK-3347] [akka] Add QuarantineMonitor which shuts
 a quarantined actor system and JVM down

The QuarantineMonitor subscribes to the actor system's event bus and listens to
AssociationErrorEvents. These are the events which are generated when the actor system
has quarantined another actor system or if it has been quarantined by another actor
system. In case of the quarantined state, the actor system will be shutdown killing
all actors and then the JVM is terminated.

Disable the quarantine monitor per default

This closes #2696.
---
 .../configuration/TaskManagerOptions.java     |   9 +
 .../akka/DefaultQuarantineHandler.java        |  76 ++++
 .../flink/runtime/akka/QuarantineHandler.java |  46 +++
 .../flink/runtime/akka/QuarantineMonitor.java | 100 ++++++
 .../runtime/taskmanager/TaskManager.scala     |  20 +-
 .../runtime/akka/QuarantineMonitorTest.java   | 326 ++++++++++++++++++
 6 files changed, 572 insertions(+), 5 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/akka/DefaultQuarantineHandler.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineHandler.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineMonitor.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/akka/QuarantineMonitorTest.java

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
index 777ee215957f6..e915c0bde6117 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
@@ -156,6 +156,15 @@ public class TaskManagerOptions {
 			key("task.checkpoint.alignment.max-size")
 			.defaultValue(-1L);
 
+	/**
+	 * Whether the quarantine monitor for task managers shall be started. The quarantine monitor
+	 * shuts down the actor system if it detects that it has quarantined another actor system
+	 * or if it has been quarantined by another actor system.
+	 */
+	public static final ConfigOption<Boolean> EXIT_ON_FATAL_AKKA_ERROR =
+		key("taskmanager.exit-on-fatal-akka-error")
+		.defaultValue(false);
+
 	// ------------------------------------------------------------------------
 
 	/** Not intended to be instantiated */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/akka/DefaultQuarantineHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/DefaultQuarantineHandler.java
new file mode 100644
index 0000000000000..378cb25f0f5eb
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/DefaultQuarantineHandler.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.akka;
+
+import akka.actor.ActorSystem;
+import akka.actor.Address;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import scala.concurrent.duration.FiniteDuration;
+
+/**
+ * Default quarantine handler which logs the quarantining events, then shuts down the given
+ * actor system by sending Kill to all actors and then shutting the JVM down with the given
+ * exit code.
+ */
+public class DefaultQuarantineHandler implements QuarantineHandler {
+
+	private final FiniteDuration timeout;
+	private final int exitCode;
+	private final Logger log;
+
+	public DefaultQuarantineHandler(Time timeout, int exitCode, Logger log) {
+		Preconditions.checkNotNull(timeout);
+		this.timeout = new FiniteDuration(timeout.getSize(), timeout.getUnit());
+		this.exitCode = exitCode;
+		this.log = Preconditions.checkNotNull(log);
+	}
+
+	@Override
+	public void wasQuarantinedBy(String remoteSystem, ActorSystem actorSystem) {
+		Address actorSystemAddress = AkkaUtils.getAddress(actorSystem);
+		log.error("The actor system {} has been quarantined by {}. Shutting the actor system " +
+			"down to be able to reestablish a connection!", actorSystemAddress, remoteSystem);
+
+		shutdownActorSystem(actorSystem);
+	}
+
+	@Override
+	public void hasQuarantined(String remoteSystem, ActorSystem actorSystem) {
+		Address actorSystemAddress = AkkaUtils.getAddress(actorSystem);
+		log.error("The actor system {} has quarantined the remote actor system {}. Shutting " +
+			"the actor system down to be able to reestablish a connection!", actorSystemAddress, remoteSystem);
+
+		shutdownActorSystem(actorSystem);
+	}
+
+	private void shutdownActorSystem(ActorSystem actorSystem) {
+		// shut the actor system down
+		actorSystem.shutdown();
+
+		try {
+			// give it some time to complete the shutdown
+			actorSystem.awaitTermination(timeout);
+		} finally {
+			// now let's crash the JVM
+			System.exit(exitCode);
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineHandler.java
new file mode 100644
index 0000000000000..21623e82081d6
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineHandler.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.akka;
+
+import akka.actor.ActorSystem;
+
+/**
+ * Callback interface for the {@link QuarantineMonitor} which is called in case the actor system
+ * has been quarantined or quarantined another system.
+ */
+public interface QuarantineHandler {
+
+	/**
+	 * Callback when the given actor system was quarantined by the given remote actor system.
+	 *
+	 * @param remoteSystem is the address of the remote actor system which has quarantined this
+	 *                     actor system
+	 * @param actorSystem which has been quarantined
+	 */
+	void wasQuarantinedBy(final String remoteSystem, final ActorSystem actorSystem);
+
+	/**
+	 * Callback when the given actor system has quarantined the given remote actor system.
+	 *
+	 * @param remoteSystem is the address of the remote actor system which has been quarantined
+	 *                     by our actor system
+	 * @param actorSystem which has quarantined the other actor system
+	 */
+	void hasQuarantined(final String remoteSystem, final ActorSystem actorSystem);
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineMonitor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineMonitor.java
new file mode 100644
index 0000000000000..de82f291568d2
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/QuarantineMonitor.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.akka;
+
+import akka.actor.UntypedActor;
+import akka.remote.AssociationErrorEvent;
+import akka.remote.transport.Transport;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * The quarantine monitor subscribes to the event bus of the actor system in which it was started.
+ * It listens to {@link AssociationErrorEvent} which contain information if we got quarantined
+ * or quarantine another remote actor system. If the actor detects that the actor system has been
+ * quarantined or quarantined another system, then the {@link QuarantineHandler} is called.
+ *
+ * IMPORTANT: The implementation if highly specific for Akka 2.3.7. With different version the
+ * quarantine state might be detected differently.
+ */
+public class QuarantineMonitor extends UntypedActor {
+
+	private static final Pattern pattern = Pattern.compile("^Invalid address:\\s+(.*)$");
+
+	private static final String QUARANTINE_MSG = "The remote system has a UID that has been quarantined. Association aborted.";
+	private static final String QUARANTINED_MSG = "The remote system has quarantined this system. No further associations to the remote system are possible until this system is restarted.";
+
+	private final QuarantineHandler handler;
+	private final Logger log;
+
+	public QuarantineMonitor(QuarantineHandler handler, Logger log) {
+		this.handler = Preconditions.checkNotNull(handler);
+		this.log = Preconditions.checkNotNull(log);
+	}
+
+	@Override
+	public void preStart() {
+		getContext().system().eventStream().subscribe(getSelf(), AssociationErrorEvent.class);
+	}
+
+	@Override
+	public void onReceive(Object message) throws Exception {
+		if (message instanceof AssociationErrorEvent) {
+			AssociationErrorEvent associationErrorEvent = (AssociationErrorEvent) message;
+
+			// IMPORTANT: The check for the quarantining event is highly specific to Akka 2.3.7
+			// and can change with a different Akka version.
+			// It assumes the following:
+			// AssociationErrorEvent(InvalidAssociation(InvalidAssociationException(QUARANTINE(D)_MSG))
+			if (associationErrorEvent.getCause() != null) {
+				Throwable invalidAssociation = associationErrorEvent.getCause();
+				Matcher matcher = pattern.matcher(invalidAssociation.getMessage());
+
+				final String remoteSystem;
+
+				if (matcher.find()) {
+					remoteSystem = matcher.group(1);
+				} else {
+					remoteSystem = "Unknown";
+				}
+
+				if (invalidAssociation.getCause() instanceof Transport.InvalidAssociationException) {
+					Transport.InvalidAssociationException invalidAssociationException = (Transport.InvalidAssociationException) invalidAssociation.getCause();
+
+					// don't hate the player, hate the game! That's the only way to find out if we
+					// got quarantined or quarantined another actor system in Akka 2.3.7
+					if (QUARANTINE_MSG.equals(invalidAssociationException.getMessage())) {
+						handler.hasQuarantined(remoteSystem, getContext().system());
+					} else if (QUARANTINED_MSG.equals(invalidAssociationException.getMessage())) {
+						handler.wasQuarantinedBy(remoteSystem, getContext().system());
+					} else {
+						log.debug("The invalid association exception's message could not be matched.", associationErrorEvent);
+					}
+				} else {
+					log.debug("The association error event's root cause is not of type {}.", Transport.InvalidAssociationException.class.getSimpleName(), associationErrorEvent);
+				}
+			} else {
+				log.debug("Received association error event which did not contain a cause.", associationErrorEvent);
+			}
+		}
+	}
+}
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 2e8a6fa19132a..97e55f0452e23 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -28,15 +28,13 @@ import java.util.{Collections, UUID}
 import _root_.akka.actor._
 import _root_.akka.pattern.ask
 import _root_.akka.util.Timeout
-
 import grizzled.slf4j.Logger
-
 import org.apache.commons.lang3.exception.ExceptionUtils
-
+import org.apache.flink.api.common.time.Time
 import org.apache.flink.configuration._
 import org.apache.flink.core.fs.FileSystem
 import org.apache.flink.runtime.accumulators.AccumulatorSnapshot
-import org.apache.flink.runtime.akka.AkkaUtils
+import org.apache.flink.runtime.akka.{AkkaUtils, DefaultQuarantineHandler, QuarantineMonitor}
 import org.apache.flink.runtime.blob.{BlobCache, BlobClient, BlobService}
 import org.apache.flink.runtime.broadcast.BroadcastVariableManager
 import org.apache.flink.runtime.clusterframework.messages.StopCluster
@@ -66,7 +64,7 @@ import org.apache.flink.runtime.metrics.{MetricRegistry => FlinkMetricRegistry}
 import org.apache.flink.runtime.process.ProcessReaper
 import org.apache.flink.runtime.security.SecurityUtils
 import org.apache.flink.runtime.security.SecurityUtils.SecurityConfiguration
-import org.apache.flink.runtime.taskexecutor.{TaskManagerServices, TaskManagerServicesConfiguration, TaskManagerConfiguration}
+import org.apache.flink.runtime.taskexecutor.{TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
 import org.apache.flink.runtime.util._
 import org.apache.flink.runtime.{FlinkActor, LeaderSessionMessageFilter, LogMessages}
 
@@ -1794,6 +1792,18 @@ object TaskManager {
         Props(classOf[ProcessReaper], taskManager, LOG.logger, RUNTIME_FAILURE_RETURN_CODE),
         "TaskManager_Process_Reaper")
 
+      if (configuration.getBoolean(TaskManagerOptions.EXIT_ON_FATAL_AKKA_ERROR)) {
+        val quarantineHandler = new DefaultQuarantineHandler(
+          Time.milliseconds(AkkaUtils.getTimeout(configuration).toMillis),
+          RUNTIME_FAILURE_RETURN_CODE,
+          LOG.logger)
+
+        LOG.debug("Starting TaskManager quarantine monitor")
+        taskManagerSystem.actorOf(
+          Props(classOf[QuarantineMonitor], quarantineHandler, LOG.logger)
+        )
+      }
+
       // if desired, start the logging daemon that periodically logs the
       // memory usage information
       if (LOG.isInfoEnabled && configuration.getBoolean(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/akka/QuarantineMonitorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/akka/QuarantineMonitorTest.java
new file mode 100644
index 0000000000000..97309a415bc08
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/akka/QuarantineMonitorTest.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.akka;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.Address;
+import akka.actor.Props;
+import akka.actor.UntypedActor;
+import akka.dispatch.OnComplete;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.TestLogger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+public class QuarantineMonitorTest extends TestLogger {
+
+	private static final Logger LOG = LoggerFactory.getLogger(QuarantineMonitorTest.class);
+
+	private static final FiniteDuration zeroDelay = new FiniteDuration(0L, TimeUnit.SECONDS);
+
+	// we need two actor systems because we're quarantining one of them
+	private static ActorSystem actorSystem1;
+	private ActorSystem actorSystem2;
+
+	@BeforeClass
+	public static void setup() {
+		Properties properties = new Properties();
+		properties.setProperty("akka.remote.watch-failure-detector.threshold", "0.00001");
+		properties.setProperty("akka.remote.watch-failure-detector.heartbeat-interval", "1 ms");
+		properties.setProperty("akka.remote.watch-failure-detector.acceptable-heartbeat-pause", "1 ms");
+		Config deathWatch = ConfigFactory.parseProperties(properties);
+		Config defaultConfig = AkkaUtils.getDefaultAkkaConfig();
+
+		actorSystem1 = AkkaUtils.createActorSystem(deathWatch.withFallback(defaultConfig));
+	}
+
+	@AfterClass
+	public static void tearDown() {
+		if (actorSystem1 != null) {
+			actorSystem1.shutdown();
+			actorSystem1.awaitTermination();
+		}
+	}
+
+	@Before
+	public void setupTest() {
+		actorSystem2 = AkkaUtils.createDefaultActorSystem();
+	}
+
+	@After
+	public void tearDownTest() {
+		if (actorSystem2 != null) {
+			actorSystem2.shutdown();
+			actorSystem2.awaitTermination();
+		}
+	}
+
+	/**
+	 * Tests that the quarantine monitor detects if an actor system has been quarantined by another
+	 * actor system.
+	 */
+	@Test(timeout = 5000L)
+	public void testWatcheeQuarantined() throws ExecutionException, InterruptedException {
+		TestingQuarantineHandler quarantineHandler = new TestingQuarantineHandler();
+
+		ActorRef watchee = null;
+		ActorRef watcher = null;
+		ActorRef monitor = null;
+
+		FiniteDuration timeout = new FiniteDuration(5, TimeUnit.SECONDS);
+		FiniteDuration interval = new FiniteDuration(200, TimeUnit.MILLISECONDS);
+
+		try {
+			// start the quarantine monitor in the watchee actor system
+			monitor = actorSystem2.actorOf(getQuarantineMonitorProps(quarantineHandler), "quarantineMonitor");
+
+			watchee = actorSystem2.actorOf(getWatcheeProps(timeout, interval, quarantineHandler), "watchee");
+			watcher = actorSystem1.actorOf(getWatcherProps(timeout, interval, quarantineHandler), "watcher");
+
+			final Address actorSystem1Address = AkkaUtils.getAddress(actorSystem1);
+			final String watcheeAddress = AkkaUtils.getAkkaURL(actorSystem2, watchee);
+			final String watcherAddress = AkkaUtils.getAkkaURL(actorSystem1, watcher);
+
+			// ping the watcher continuously
+			watchee.tell(new Ping(watcherAddress), ActorRef.noSender());
+			// start watching the watchee
+			watcher.tell(new Watch(watcheeAddress), ActorRef.noSender());
+
+			Future<String> quarantineFuture = quarantineHandler.getWasQuarantinedByFuture();
+
+			Assert.assertEquals(actorSystem1Address.toString(), quarantineFuture.get());
+		} finally {
+			TestingUtils.stopActor(watchee);
+			TestingUtils.stopActor(watcher);
+			TestingUtils.stopActor(monitor);
+		}
+	}
+
+	/**
+	 * Tests that the quarantine monitor detects if an actor system quarantines another actor
+	 * system.
+	 */
+	@Test(timeout = 5000L)
+	public void testWatcherQuarantining() throws ExecutionException, InterruptedException {
+		TestingQuarantineHandler quarantineHandler = new TestingQuarantineHandler();
+
+		ActorRef watchee = null;
+		ActorRef watcher = null;
+		ActorRef monitor = null;
+
+		FiniteDuration timeout = new FiniteDuration(5, TimeUnit.SECONDS);
+		FiniteDuration interval = new FiniteDuration(200, TimeUnit.MILLISECONDS);
+
+		try {
+			// start the quarantine monitor in the watcher actor system
+			monitor = actorSystem1.actorOf(getQuarantineMonitorProps(quarantineHandler), "quarantineMonitor");
+
+			watchee = actorSystem2.actorOf(getWatcheeProps(timeout, interval, quarantineHandler), "watchee");
+			watcher = actorSystem1.actorOf(getWatcherProps(timeout, interval, quarantineHandler), "watcher");
+
+			final Address actorSystem1Address = AkkaUtils.getAddress(actorSystem2);
+			final String watcheeAddress = AkkaUtils.getAkkaURL(actorSystem2, watchee);
+			final String watcherAddress = AkkaUtils.getAkkaURL(actorSystem1, watcher);
+
+			// ping the watcher continuously
+			watchee.tell(new Ping(watcherAddress), ActorRef.noSender());
+			// start watching the watchee
+			watcher.tell(new Watch(watcheeAddress), ActorRef.noSender());
+
+			Future<String> quarantineFuture = quarantineHandler.getHasQuarantinedFuture();
+
+			Assert.assertEquals(actorSystem1Address.toString(), quarantineFuture.get());
+		} finally {
+			TestingUtils.stopActor(watchee);
+			TestingUtils.stopActor(watcher);
+			TestingUtils.stopActor(monitor);
+		}
+	}
+
+	private static class TestingQuarantineHandler implements QuarantineHandler, ErrorHandler {
+
+		private final CompletableFuture<String> wasQuarantinedByFuture;
+		private final CompletableFuture<String> hasQuarantinedFuture;
+
+		public TestingQuarantineHandler() {
+			this.wasQuarantinedByFuture = new FlinkCompletableFuture<>();
+			this.hasQuarantinedFuture = new FlinkCompletableFuture<>();
+		}
+
+		@Override
+		public void wasQuarantinedBy(String remoteSystem, ActorSystem actorSystem) {
+			wasQuarantinedByFuture.complete(remoteSystem);
+		}
+
+		@Override
+		public void hasQuarantined(String remoteSystem, ActorSystem actorSystem) {
+			hasQuarantinedFuture.complete(remoteSystem);
+		}
+
+		public Future<String> getWasQuarantinedByFuture() {
+			return wasQuarantinedByFuture;
+		}
+
+		public Future<String> getHasQuarantinedFuture() {
+			return hasQuarantinedFuture;
+		}
+
+		@Override
+		public void handleError(Throwable failure) {
+			wasQuarantinedByFuture.completeExceptionally(failure);
+			hasQuarantinedFuture.completeExceptionally(failure);
+		}
+	}
+
+	private interface ErrorHandler {
+		void handleError(Throwable failure);
+	}
+
+	static class Watcher extends UntypedActor {
+
+		private final FiniteDuration timeout;
+		private final FiniteDuration interval;
+		private final ErrorHandler errorHandler;
+
+		Watcher(FiniteDuration timeout, FiniteDuration interval, ErrorHandler errorHandler) {
+			this.timeout = Preconditions.checkNotNull(timeout);
+			this.interval = Preconditions.checkNotNull(interval);
+			this.errorHandler = Preconditions.checkNotNull(errorHandler);
+		}
+
+		@Override
+		public void onReceive(Object message) throws Exception {
+			if (message instanceof Watch) {
+				Watch watch = (Watch) message;
+
+				getContext().actorSelection(watch.getTarget()).resolveOne(timeout).onComplete(new OnComplete<ActorRef>() {
+					@Override
+					public void onComplete(Throwable failure, ActorRef success) throws Throwable {
+						if (success != null) {
+							getContext().watch(success);
+							// constantly ping the watchee
+							getContext().system().scheduler().schedule(
+								zeroDelay,
+								interval,
+								success,
+								"Watcher message",
+								getContext().dispatcher(),
+								getSelf());
+						} else {
+							errorHandler.handleError(failure);
+						}
+					}
+				}, getContext().dispatcher());
+			}
+		}
+	}
+
+	static class Watchee extends UntypedActor {
+
+		private final FiniteDuration timeout;
+		private final FiniteDuration interval;
+		private final ErrorHandler errorHandler;
+
+		Watchee(FiniteDuration timeout, FiniteDuration interval, ErrorHandler errorHandler) {
+			this.timeout = Preconditions.checkNotNull(timeout);
+			this.interval = Preconditions.checkNotNull(interval);
+			this.errorHandler = Preconditions.checkNotNull(errorHandler);
+		}
+
+		@Override
+		public void onReceive(Object message) throws Exception {
+			if (message instanceof Ping) {
+				final Ping ping = (Ping) message;
+
+				getContext().actorSelection(ping.getTarget()).resolveOne(timeout).onComplete(new OnComplete<ActorRef>() {
+					@Override
+					public void onComplete(Throwable failure, ActorRef success) throws Throwable {
+						if (success != null) {
+							// constantly ping the target
+							getContext().system().scheduler().schedule(
+								zeroDelay,
+								interval,
+								success,
+								"Watchee message",
+								getContext().dispatcher(),
+								getSelf());
+						} else {
+							errorHandler.handleError(failure);
+						}
+					}
+				}, getContext().dispatcher());
+			}
+		}
+	}
+
+	static class Watch {
+		private final String target;
+
+		Watch(String target) {
+			this.target = target;
+		}
+
+		public String getTarget() {
+			return target;
+		}
+	}
+
+	static class Ping {
+		private final String target;
+
+		Ping(String target) {
+			this.target = target;
+		}
+
+		public String getTarget() {
+			return target;
+		}
+	}
+
+	static Props getWatcheeProps(FiniteDuration timeout, FiniteDuration interval, ErrorHandler errorHandler) {
+		return Props.create(Watchee.class, timeout, interval, errorHandler);
+	}
+
+	static Props getWatcherProps(FiniteDuration timeout, FiniteDuration interval, ErrorHandler errorHandler) {
+		return Props.create(Watcher.class, timeout, interval, errorHandler);
+	}
+
+	static Props getQuarantineMonitorProps(QuarantineHandler handler) {
+		return Props.create(QuarantineMonitor.class, handler, LOG);
+	}
+
+}

From ed61ecda9a59ae7e7a345d2a3014da80bd254c32 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 28 Apr 2017 11:43:28 +0200
Subject: [PATCH 080/253] [FLINK-6410] [build] Fix Gelly Example packaging to
 support different Scala versions

---
 flink-dist/src/main/assemblies/bin.xml | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index d3f2005e2d6c5..c3385e17d0e89 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -54,19 +54,12 @@ under the License.
 			<fileMode>0644</fileMode>
 		</file>
 
+		<!-- copy the config file -->
 		<file>
 			<source>src/main/resources/flink-conf.yaml</source>
 			<outputDirectory>conf</outputDirectory>
 			<fileMode>0644</fileMode>
 		</file>
-
-		<!-- copy jar file of the Gelly examples -->
-		<file>
-			<source>../flink-libraries/flink-gelly-examples/target/flink-gelly-examples_2.10-${project.version}.jar</source>
-			<outputDirectory>examples/</outputDirectory>
-			<destName>flink-gelly-examples_2.10-${project.version}.jar</destName>
-			<fileMode>0644</fileMode>
-		</file>
 	</files>
 
 	<fileSets>
@@ -181,6 +174,20 @@ under the License.
 			</excludes>
 		</fileSet>
 
+		<!-- copy jar files of the gelly examples -->
+		<fileSet>
+			<directory>../flink-libraries/flink-gelly-examples/target</directory>
+			<outputDirectory>examples/gelly</outputDirectory>
+			<fileMode>0644</fileMode>
+			<includes>
+				<include>*.jar</include>
+			</includes>
+			<excludes>
+				<exclude>original-*.jar</exclude>
+				<exclude>*-tests.jar</exclude>
+			</excludes>
+		</fileSet>
+
 		<!-- copy python package -->
 		<fileSet>
 			<directory>../flink-libraries/flink-python/src/main/python/org/apache/flink/python/api</directory>

From 614c91516485118339adb2a244ab642611d322e7 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Fri, 28 Apr 2017 11:42:30 +0200
Subject: [PATCH 081/253] [hotfix] Change checkstyle/suppression file
 references to be absolute

The relative path caused problems with "mvn deploy".
---
 flink-streaming-java/pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml
index 72c2133722a28..013e958a7c1fd 100644
--- a/flink-streaming-java/pom.xml
+++ b/flink-streaming-java/pom.xml
@@ -137,8 +137,8 @@ under the License.
 					</dependency>
 				</dependencies>
 				<configuration>
-					<configLocation>../tools/maven/strict-checkstyle.xml</configLocation>
-					<suppressionsLocation>../tools/maven/suppressions.xml</suppressionsLocation>
+					<configLocation>/tools/maven/strict-checkstyle.xml</configLocation>
+					<suppressionsLocation>/tools/maven/suppressions.xml</suppressionsLocation>
 					<logViolationsToConsole>true</logViolationsToConsole>
 					<failOnViolation>true</failOnViolation>
 				</configuration>

From 48ee4b5588c21d2f2deb5345880cc6b232793bee Mon Sep 17 00:00:00 2001
From: Dawid Wysakowicz <dawid@getindata.com>
Date: Wed, 26 Apr 2017 11:57:53 +0200
Subject: [PATCH 082/253] [FLINK-6208] [cep] Implement skip till next match
 strategy

---
 docs/dev/libs/cep.md                          |  19 +-
 .../cep/scala/pattern/FollowedByPattern.scala |  44 -
 .../flink/cep/scala/pattern/Pattern.scala     |  31 +-
 .../flink/cep/scala/pattern/package.scala     |   3 +-
 .../flink/cep/scala/pattern/PatternTest.scala |  52 +-
 .../flink/cep/nfa/compiler/NFACompiler.java   | 192 ++---
 .../flink/cep/pattern/FollowedByPattern.java  |  33 -
 .../org/apache/flink/cep/pattern/Pattern.java |  42 +-
 .../apache/flink/cep/pattern/Quantifier.java  |  82 +-
 .../java/org/apache/flink/cep/CEPITCase.java  |  30 +-
 .../org/apache/flink/cep/nfa/NFAITCase.java   | 760 +++++++++---------
 .../cep/nfa/compiler/NFACompilerTest.java     |  39 -
 .../flink/cep/operator/CEPOperatorTest.java   |   5 +-
 .../apache/flink/cep/pattern/PatternTest.java |   9 +-
 14 files changed, 656 insertions(+), 685 deletions(-)
 delete mode 100644 flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/FollowedByPattern.scala
 delete mode 100644 flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/FollowedByPattern.java

diff --git a/docs/dev/libs/cep.md b/docs/dev/libs/cep.md
index 15afdf542aad4..b379615afc485 100644
--- a/docs/dev/libs/cep.md
+++ b/docs/dev/libs/cep.md
@@ -268,7 +268,7 @@ val strictNext: Pattern[Event, _] = start.next("middle")
 </div>
 
 Non-strict contiguity means that other events are allowed to occur in-between two matching events.
-A non-strict contiguity pattern state can be created via the `followedBy` method.
+A non-strict contiguity pattern state can be created via the `followedBy` or `followedByAny` method.
 
 <div class="codetabs" markdown="1">
 <div data-lang="java" markdown="1">
@@ -282,6 +282,23 @@ Pattern<Event, ?> nonStrictNext = start.followedBy("middle");
 val nonStrictNext : Pattern[Event, _] = start.followedBy("middle")
 {% endhighlight %}
 </div>
+
+For non-strict contiguity one can specify if only the first succeeding matching event will be matched, or
+all. In the latter case multiple matches will be emitted for the same beginning.
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+Pattern<Event, ?> nonStrictNext = start.followedByAny("middle");
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+val nonStrictNext : Pattern[Event, _] = start.followedByAny("middle")
+{% endhighlight %}
+</div>
+
 </div>
 It is also possible to define a temporal constraint for the pattern to be valid.
 For example, one can define that a pattern should occur within 10 seconds via the `within` method. 
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/FollowedByPattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/FollowedByPattern.scala
deleted file mode 100644
index 4bda08f7c8d64..0000000000000
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/FollowedByPattern.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.cep.scala.pattern
-
-import org.apache.flink.cep.pattern.{FollowedByPattern => JFollowedByPattern}
-
-object FollowedByPattern {
-  /**
-    * Constructs a new Pattern by wrapping a given Java API Pattern
-    *
-    * @param jfbPattern Underlying Java API Pattern.
-    * @tparam T Base type of the elements appearing in the pattern
-    * @tparam F Subtype of T to which the current pattern operator is constrained
-    * @return New wrapping FollowedByPattern object
-    */
-  def apply[T, F <: T](jfbPattern: JFollowedByPattern[T, F]) =
-    new FollowedByPattern[T, F](jfbPattern)
-}
-
-/**
-  * Pattern operator which signifies that the there is a non-strict temporal contiguity between
-  * itself and its preceding pattern operator. This means that there might be events in between
-  * two matching events. These events are then simply ignored.
-  *
-  * @tparam T Base type of the events
-  * @tparam F Subtype of T to which the operator is currently constrained
-  */
-class FollowedByPattern[T, F <: T](jfbPattern: JFollowedByPattern[T, F])
-  extends Pattern[T, F](jfbPattern)
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
index 3bdbfcf46f923..65b7ab0a31ce0 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
@@ -46,22 +46,22 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
   /**
     * @return The previous pattern
     */
-  def getPrevious(): Option[Pattern[T, _ <: T]] = {
-    wrapPattern(jPattern.getPrevious())
+  def getPrevious: Option[Pattern[T, _ <: T]] = {
+    wrapPattern(jPattern.getPrevious)
   }
 
   /**
     *
     * @return Name of the pattern operator
     */
-  def getName(): String = jPattern.getName()
+  def getName: String = jPattern.getName
 
   /**
     *
     * @return Window length in which the pattern match has to occur
     */
-  def getWindowTime(): Option[Time] = {
-    Option(jPattern.getWindowTime())
+  def getWindowTime: Option[Time] = {
+    Option(jPattern.getWindowTime)
   }
 
   /**
@@ -70,8 +70,8 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
     */
   def getQuantifier: Quantifier = jPattern.getQuantifier
 
-  def getCondition(): Option[IterativeCondition[F]] = {
-    Option(jPattern.getCondition())
+  def getCondition: Option[IterativeCondition[F]] = {
+    Option(jPattern.getCondition)
   }
 
   /**
@@ -208,7 +208,22 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
     * @param name Name of the new pattern
     * @return A new pattern which is appended to this one
     */
-  def followedBy(name: String) = FollowedByPattern(jPattern.followedBy(name))
+  def followedBy(name: String): Pattern[T, T] = {
+    Pattern[T, T](jPattern.followedBy(name))
+  }
+
+  /**
+    * Appends a new pattern to the existing one. The new pattern enforces non-strict
+    * temporal contiguity. This means that a matching event of this pattern and the
+    * preceding matching event might be interleaved with other events which are ignored.
+    *
+    * @param name Name of the new pattern
+    * @return A new pattern which is appended to this one
+    */
+  def followedByAny(name: String): Pattern[T, T] = {
+    Pattern[T, T](jPattern.followedByAny(name))
+  }
+
 
   /**
     * Specifies that this pattern is optional for a final match of the pattern
diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
index 382c160a3c33b..26355a5e49566 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/package.scala
@@ -17,7 +17,7 @@
  */
 package org.apache.flink.cep.scala
 
-import org.apache.flink.cep.pattern.{FollowedByPattern => JFollowedByPattern, Pattern => JPattern}
+import org.apache.flink.cep.pattern.{Pattern => JPattern}
 
 package object pattern {
   /**
@@ -31,7 +31,6 @@ package object pattern {
     */
   private[flink] def wrapPattern[T, F <: T](javaPattern: JPattern[T, F])
   : Option[Pattern[T, F]] = javaPattern match {
-    case f: JFollowedByPattern[T, F] => Some(FollowedByPattern[T, F](f))
     case p: JPattern[T, F] => Some(Pattern[T, F](p))
     case _ => None
   }
diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
index a95dddd460c3b..d5745131c1b12 100644
--- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
+++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/pattern/PatternTest.scala
@@ -22,7 +22,7 @@ import org.junit.Assert._
 import org.junit.Test
 import org.apache.flink.cep.Event
 import org.apache.flink.cep.SubEvent
-import org.apache.flink.cep.pattern.conditions.IterativeCondition.Context
+import org.apache.flink.cep.pattern.Quantifier.ConsumingStrategy
 import org.apache.flink.cep.pattern.conditions._
 
 class PatternTest {
@@ -33,7 +33,7 @@ class PatternTest {
     */
 
   @Test
-  def testStrictContiguity: Unit = {
+  def testStrictContiguity(): Unit = {
     val pattern = Pattern.begin[Event]("start").next("next").next("end")
     val jPattern = JPattern.begin[Event]("start").next("next").next("end")
 
@@ -56,7 +56,7 @@ class PatternTest {
 
 
   @Test
-  def testNonStrictContiguity: Unit = {
+  def testNonStrictContiguity(): Unit = {
     val pattern = Pattern.begin[Event]("start").followedBy("next").followedBy("end")
     val jPattern = JPattern.begin[Event]("start").followedBy("next").followedBy("end")
 
@@ -69,8 +69,8 @@ class PatternTest {
     assertTrue(previous.getPrevious.isDefined)
     assertFalse(preprevious.getPrevious.isDefined)
 
-    assertTrue(pattern.isInstanceOf[FollowedByPattern[_, _]])
-    assertTrue(previous.isInstanceOf[FollowedByPattern[_, _]])
+    assertEquals(ConsumingStrategy.SKIP_TILL_NEXT, pattern.getQuantifier.getConsumingStrategy)
+    assertEquals(ConsumingStrategy.SKIP_TILL_NEXT, previous.getQuantifier.getConsumingStrategy)
 
     assertEquals(pattern.getName, "end")
     assertEquals(previous.getName, "next")
@@ -78,25 +78,25 @@ class PatternTest {
   }
 
   @Test
-  def testStrictContiguityWithCondition: Unit = {
+  def testStrictContiguityWithCondition(): Unit = {
     val pattern = Pattern.begin[Event]("start")
       .next("next")
-      .where((value: Event, ctx: Context[Event]) => value.getName() == "foobar")
+      .where((value: Event, _) => value.getName == "foobar")
       .next("end")
-      .where((value: Event, ctx: Context[Event]) => value.getId() == 42)
+      .where((value: Event, _) => value.getId == 42)
 
     val jPattern = JPattern.begin[Event]("start")
       .next("next")
       .where(new SimpleCondition[Event]() {
         @throws[Exception]
         def filter(value: Event): Boolean = {
-          return value.getName() == "foobar"
+          value.getName == "foobar"
         }
       }).next("end")
       .where(new SimpleCondition[Event]() {
         @throws[Exception]
         def filter(value: Event): Boolean = {
-          return value.getId() == 42
+          value.getId == 42
         }
       })
 
@@ -120,7 +120,7 @@ class PatternTest {
   }
 
   @Test
-  def testPatternWithSubtyping: Unit = {
+  def testPatternWithSubtyping(): Unit = {
     val pattern = Pattern.begin[Event]("start")
       .next("subevent")
       .subtype(classOf[SubEvent])
@@ -150,11 +150,11 @@ class PatternTest {
   }
 
   @Test
-  def testPatternWithSubtypingAndFilter: Unit = {
+  def testPatternWithSubtypingAndFilter(): Unit = {
     val pattern = Pattern.begin[Event]("start")
       .next("subevent")
       .subtype(classOf[SubEvent])
-      .where((value: SubEvent) => false)
+      .where(_ => false)
       .followedBy("end")
 
     val jpattern = JPattern.begin[Event]("start")
@@ -163,7 +163,7 @@ class PatternTest {
       .where(new SimpleCondition[SubEvent]() {
         @throws[Exception]
         def filter(value: SubEvent): Boolean = {
-          return false
+          false
         }
       }).followedBy("end")
 
@@ -178,8 +178,8 @@ class PatternTest {
     assertTrue(previous.getPrevious.isDefined)
     assertFalse(preprevious.getPrevious.isDefined)
 
-    assertTrue(pattern.isInstanceOf[FollowedByPattern[_, _]])
-    assertTrue(previous.getCondition().isDefined)
+    assertEquals(ConsumingStrategy.SKIP_TILL_NEXT, pattern.getQuantifier.getConsumingStrategy)
+    assertTrue(previous.getCondition.isDefined)
 
     assertEquals(pattern.getName, "end")
     assertEquals(previous.getName, "subevent")
@@ -194,25 +194,25 @@ class PatternTest {
       && threeWayEquals(
       pattern.getName,
       pattern.wrappedPattern.getName,
-      jPattern.getName())
+      jPattern.getName)
       //check equal time windows
       && threeWayEquals(
       pattern.getWindowTime.orNull,
       pattern.wrappedPattern.getWindowTime,
-      jPattern.getWindowTime())
+      jPattern.getWindowTime)
       //check congruent class names / types
       && threeWayEquals(
       pattern.getClass.getSimpleName,
       pattern.wrappedPattern.getClass.getSimpleName,
-      jPattern.getClass().getSimpleName())
+      jPattern.getClass.getSimpleName)
       //best effort to confirm congruent filter functions
       && compareFilterFunctions(
-      pattern.getCondition().orNull,
-      jPattern.getCondition())
+      pattern.getCondition.orNull,
+      jPattern.getCondition)
       //recursively check previous patterns
       && checkCongruentRepresentations(
       pattern.getPrevious.orNull,
-      jPattern.getPrevious()))
+      jPattern.getPrevious))
   }
 
   def threeWayEquals(a: AnyRef, b: AnyRef, c: AnyRef): Boolean = {
@@ -233,15 +233,15 @@ class PatternTest {
     (sFilter, jFilter) match {
       //matching types: and-filter; branch and recurse for inner filters
       case (saf: AndCondition[_], jaf: AndCondition[_])
-      => (compareFilterFunctions(saf.getLeft(), jaf.getLeft())
-        && compareFilterFunctions(saf.getRight(), jaf.getRight()))
+      => (compareFilterFunctions(saf.getLeft, jaf.getLeft)
+        && compareFilterFunctions(saf.getRight, jaf.getRight))
       //matching types: subtype-filter
-      case (saf: SubtypeCondition[_], jaf: SubtypeCondition[_]) => true
+      case (_: SubtypeCondition[_], _: SubtypeCondition[_]) => true
       //mismatch: one-sided and/subtype-filter
       case (_: AndCondition[_] | _: SubtypeCondition[_], _) => false
       case (_, _: AndCondition[_] | _: SubtypeCondition[_]) => false
       //from here we can only check mutual presence or absence of a function
-      case (s: IterativeCondition[_], j: IterativeCondition[_]) => true
+      case (_: IterativeCondition[_], _: IterativeCondition[_]) => true
       case (null, null) => true
       case _ => false
     }
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
index b4e05573b1913..0ca0e14c52e79 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
@@ -20,32 +20,30 @@
 
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterators;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.cep.nfa.NFA;
 import org.apache.flink.cep.nfa.State;
 import org.apache.flink.cep.nfa.StateTransition;
 import org.apache.flink.cep.nfa.StateTransitionAction;
-import org.apache.flink.cep.pattern.conditions.BooleanConditions;
-import org.apache.flink.cep.pattern.FollowedByPattern;
 import org.apache.flink.cep.pattern.MalformedPatternException;
-import org.apache.flink.cep.pattern.conditions.NotCondition;
 import org.apache.flink.cep.pattern.Pattern;
-import org.apache.flink.cep.pattern.Quantifier.QuantifierProperty;
+import org.apache.flink.cep.pattern.Quantifier;
+import org.apache.flink.cep.pattern.conditions.BooleanConditions;
 import org.apache.flink.cep.pattern.conditions.IterativeCondition;
+import org.apache.flink.cep.pattern.conditions.NotCondition;
 import org.apache.flink.streaming.api.windowing.time.Time;
 
-import javax.annotation.Nullable;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 /**
  * Compiler class containing methods to compile a {@link Pattern} into a {@link NFA} or a
  * {@link NFAFactory}.
@@ -159,25 +157,7 @@ private State<T> createMiddleStates(final State<T> sinkState) {
 
 			State<T> lastSink = sinkState;
 			while (currentPattern.getPrevious() != null) {
-				checkPatternNameUniqueness();
-				usedNames.add(currentPattern.getName());
-
-				if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.LOOPING)) {
-					final State<T> looping = createLooping(lastSink);
-
-					if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL)) {
-						lastSink = createFirstMandatoryStateOfLoop(looping);
-					} else if (currentPattern instanceof FollowedByPattern &&
-								currentPattern.getQuantifier().hasProperty(QuantifierProperty.CONSECUTIVE)) {
-						lastSink = createWaitingStateForZeroOrMore(looping, lastSink);
-					} else {
-						lastSink = looping;
-					}
-				} else if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.TIMES)) {
-					lastSink = createTimesState(lastSink, currentPattern.getTimes());
-				} else {
-					lastSink = createSingletonState(lastSink);
-				}
+				lastSink = convertPattern(lastSink);
 				currentPattern = currentPattern.getPrevious();
 
 				final Time currentWindowTime = currentPattern.getWindowTime();
@@ -190,6 +170,29 @@ private State<T> createMiddleStates(final State<T> sinkState) {
 			return lastSink;
 		}
 
+		private State<T> convertPattern(final State<T> sinkState) {
+			final State<T> lastSink;
+			checkPatternNameUniqueness();
+			usedNames.add(currentPattern.getName());
+
+			final Quantifier quantifier = currentPattern.getQuantifier();
+			if (quantifier.hasProperty(Quantifier.QuantifierProperty.LOOPING)) {
+				final State<T> looping = createLooping(sinkState);
+
+				if (!quantifier.hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
+					lastSink = createFirstMandatoryStateOfLoop(looping);
+				} else {
+					lastSink = createWaitingStateForZeroOrMore(looping, sinkState);
+				}
+			} else if (quantifier.hasProperty(Quantifier.QuantifierProperty.TIMES)) {
+				lastSink = createTimesState(sinkState, currentPattern.getTimes());
+			} else {
+				lastSink = createSingletonState(sinkState);
+			}
+
+			return lastSink;
+		}
+
 		/**
 		 * Creates a pair of states that enables relaxed strictness before a zeroOrMore looping state.
 		 *
@@ -197,19 +200,21 @@ private State<T> createMiddleStates(final State<T> sinkState) {
 		 * @param lastSink     the state that the looping one points to
 		 * @return the newly created state
 		 */
+		@SuppressWarnings("unchecked")
 		private State<T> createWaitingStateForZeroOrMore(final State<T> loopingState, final State<T> lastSink) {
-			final State<T> followByState = createNormalState();
-			final State<T> followByStateWithoutProceed = createNormalState();
-
 			final IterativeCondition<T> currentFunction = (IterativeCondition<T>)currentPattern.getCondition();
-			final IterativeCondition<T> ignoreFunction = getIgnoreCondition(currentPattern);
 
+			final State<T> followByState = createNormalState();
 			followByState.addProceed(lastSink, BooleanConditions.<T>trueFunction());
-			followByState.addIgnore(followByStateWithoutProceed, ignoreFunction);
 			followByState.addTake(loopingState, currentFunction);
 
-			followByStateWithoutProceed.addIgnore(ignoreFunction);
-			followByStateWithoutProceed.addTake(loopingState, currentFunction);
+			final IterativeCondition<T> ignoreFunction = getIgnoreCondition(currentPattern);
+			if (ignoreFunction != null) {
+				final State<T> followByStateWithoutProceed = createNormalState();
+				followByState.addIgnore(followByStateWithoutProceed, ignoreFunction);
+				followByStateWithoutProceed.addIgnore(ignoreFunction);
+				followByStateWithoutProceed.addTake(loopingState, currentFunction);
+			}
 
 			return followByState;
 		}
@@ -230,23 +235,7 @@ private void checkPatternNameUniqueness() {
 		 */
 		@SuppressWarnings("unchecked")
 		private State<T> createStartState(State<T> sinkState) {
-			checkPatternNameUniqueness();
-			usedNames.add(currentPattern.getName());
-
-			final State<T> beginningState;
-			if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.LOOPING)) {
-				final State<T> loopingState = createLooping(sinkState);
-				if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL)) {
-					beginningState = createFirstMandatoryStateOfLoop(loopingState);
-				} else {
-					beginningState = loopingState;
-				}
-			} else if (currentPattern.getQuantifier().hasProperty(QuantifierProperty.TIMES)) {
-				beginningState = createTimesState(sinkState, currentPattern.getTimes());
-			} else {
-				beginningState = createSingletonState(sinkState);
-			}
-
+			final State<T> beginningState = convertPattern(sinkState);
 			beginningState.makeStart();
 
 			return beginningState;
@@ -263,29 +252,26 @@ private State<T> createStartState(State<T> sinkState) {
 		private State<T> createTimesState(final State<T> sinkState, int times) {
 			State<T> lastSink = sinkState;
 			for (int i = 0; i < times - 1; i++) {
-				lastSink = createSingletonState(
-					lastSink,
-					!currentPattern.getQuantifier().hasProperty(QuantifierProperty.CONSECUTIVE),
-					false);
+				lastSink = createSingletonState(lastSink, getInnerIgnoreCondition(currentPattern), false);
 			}
 
+			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
+			final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
+
 			// we created the intermediate states in the loop, now we create the start of the loop.
-			if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL)) {
-				return createSingletonState(lastSink, currentPattern instanceof FollowedByPattern, false);
+			if (!currentPattern.getQuantifier().hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
+				return createSingletonState(lastSink, ignoreCondition, false);
 			}
 
-			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
-			final IterativeCondition<T> trueFunction = BooleanConditions.trueFunction();
-
 			final State<T> singletonState = createNormalState();
 			singletonState.addTake(lastSink, currentFilterFunction);
-			singletonState.addProceed(sinkState, trueFunction);
+			singletonState.addProceed(sinkState, BooleanConditions.<T>trueFunction());
 
-			if (currentPattern instanceof FollowedByPattern) {
+			if (ignoreCondition != null) {
 				State<T> ignoreState = createNormalState();
 				ignoreState.addTake(lastSink, currentFilterFunction);
-				ignoreState.addIgnore(trueFunction);
-				singletonState.addIgnore(ignoreState, trueFunction);
+				ignoreState.addIgnore(ignoreCondition);
+				singletonState.addIgnore(ignoreState, ignoreCondition);
 			}
 			return singletonState;
 		}
@@ -300,8 +286,10 @@ private State<T> createTimesState(final State<T> sinkState, int times) {
 		 */
 		@SuppressWarnings("unchecked")
 		private State<T> createSingletonState(final State<T> sinkState) {
-			return createSingletonState(sinkState, currentPattern instanceof FollowedByPattern,
-					currentPattern.getQuantifier().hasProperty(QuantifierProperty.OPTIONAL));
+			return createSingletonState(
+				sinkState,
+				getIgnoreCondition(currentPattern),
+				currentPattern.getQuantifier().hasProperty(Quantifier.QuantifierProperty.OPTIONAL));
 		}
 
 		/**
@@ -309,12 +297,12 @@ private State<T> createSingletonState(final State<T> sinkState) {
 		 * of a similar state without the PROCEED edge, so that for each PROCEED transition branches
 		 * in computation state graph  can be created only once.
 		 *
-		 * @param addIgnore if any IGNORE should be added
+		 * @param ignoreCondition condition that should be applied to IGNORE transition
 		 * @param sinkState state that the state being converted should point to
 		 * @return the created state
 		 */
 		@SuppressWarnings("unchecked")
-		private State<T> createSingletonState(final State<T> sinkState, boolean addIgnore, boolean isOptional) {
+		private State<T> createSingletonState(final State<T> sinkState, final IterativeCondition<T> ignoreCondition, final boolean isOptional) {
 			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
 			final IterativeCondition<T> trueFunction = BooleanConditions.trueFunction();
 
@@ -325,7 +313,7 @@ private State<T> createSingletonState(final State<T> sinkState, boolean addIgnor
 				singletonState.addProceed(sinkState, trueFunction);
 			}
 
-			if (addIgnore) {
+			if (ignoreCondition != null) {
 				final State<T> ignoreState;
 				if (isOptional) {
 					ignoreState = createNormalState();
@@ -333,7 +321,7 @@ private State<T> createSingletonState(final State<T> sinkState, boolean addIgnor
 				} else {
 					ignoreState = singletonState;
 				}
-				singletonState.addIgnore(ignoreState, getIgnoreCondition(currentPattern));
+				singletonState.addIgnore(ignoreState, ignoreCondition);
 			}
 			return singletonState;
 		}
@@ -352,8 +340,8 @@ private State<T> createFirstMandatoryStateOfLoop(final State<T> sinkState) {
 			final State<T> firstState = createNormalState();
 
 			firstState.addTake(sinkState, currentFilterFunction);
-			if (currentPattern instanceof FollowedByPattern) {
-				final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
+			final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
+			if (ignoreCondition != null) {
 				firstState.addIgnore(ignoreCondition);
 			}
 			return firstState;
@@ -369,18 +357,16 @@ private State<T> createFirstMandatoryStateOfLoop(final State<T> sinkState) {
 		 */
 		@SuppressWarnings("unchecked")
 		private State<T> createLooping(final State<T> sinkState) {
-
-			final State<T> loopingState = createNormalState();
 			final IterativeCondition<T> filterFunction = (IterativeCondition<T>) currentPattern.getCondition();
+			final IterativeCondition<T> ignoreCondition = getInnerIgnoreCondition(currentPattern);
 			final IterativeCondition<T> trueFunction = BooleanConditions.trueFunction();
 
+			final State<T> loopingState = createNormalState();
 			loopingState.addProceed(sinkState, trueFunction);
 			loopingState.addTake(filterFunction);
-			if (!currentPattern.getQuantifier().hasProperty(QuantifierProperty.CONSECUTIVE)) {
-				final State<T> ignoreState = createNormalState();
-
-				final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
 
+			if (ignoreCondition != null) {
+				final State<T> ignoreState = createNormalState();
 				ignoreState.addTake(loopingState, filterFunction);
 				ignoreState.addIgnore(ignoreCondition);
 				loopingState.addIgnore(ignoreState, ignoreCondition);
@@ -403,15 +389,37 @@ private State<T> createNormalState() {
 
 		/**
 		 * @return The {@link IterativeCondition condition} for the {@code IGNORE} edge
-		 * that corresponds to the specified {@link Pattern}. If the pattern is
-		 * {@link QuantifierProperty#EAGER}, the negated user-specified condition is
-		 * returned. In other case, a condition that always evaluated to {@code true} is
-		 * returned.
+		 * that corresponds to the specified {@link Pattern}. It is applicable only for inner states of a complex
+		 * state like looping or times.
 		 */
+		@SuppressWarnings("unchecked")
+		private IterativeCondition<T> getInnerIgnoreCondition(Pattern<T, ?> pattern) {
+			switch (pattern.getQuantifier().getInnerConsumingStrategy()) {
+				case STRICT:
+					return null;
+				case SKIP_TILL_NEXT:
+					return new NotCondition<>((IterativeCondition<T>) pattern.getCondition());
+				case SKIP_TILL_ANY:
+					return BooleanConditions.trueFunction();
+			}
+			return null;
+		}
+
+		/**
+		 * @return The {@link IterativeCondition condition} for the {@code IGNORE} edge
+		 * that corresponds to the specified {@link Pattern}. For more on strategy see {@link Quantifier}
+		 */
+		@SuppressWarnings("unchecked")
 		private IterativeCondition<T> getIgnoreCondition(Pattern<T, ?> pattern) {
-			return pattern.getQuantifier().hasProperty(QuantifierProperty.EAGER)
-					? new NotCondition<>((IterativeCondition<T>) pattern.getCondition())
-					: BooleanConditions.<T>trueFunction();
+			switch (pattern.getQuantifier().getConsumingStrategy()) {
+				case STRICT:
+					return null;
+				case SKIP_TILL_NEXT:
+					return new NotCondition<>((IterativeCondition<T>) pattern.getCondition());
+				case SKIP_TILL_ANY:
+					return BooleanConditions.trueFunction();
+			}
+			return null;
 		}
 	}
 
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/FollowedByPattern.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/FollowedByPattern.java
deleted file mode 100644
index 266451c65b093..0000000000000
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/FollowedByPattern.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cep.pattern;
-
-/**
- * Pattern operator which signifies that the there is a non-strict temporal contiguity between
- * itself and its preceding pattern operator. This means that there might be events in between
- * two matching events. These events are then simply ignored.
- *
- * @param <T> Base type of the events
- * @param <F> Subtype of T to which the operator is currently constrained
- */
-public class FollowedByPattern<T, F extends T> extends Pattern<T, F> {
-	FollowedByPattern(final String name, Pattern<T, ?> previous) {
-		super(name, previous);
-	}
-}
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
index 3ce0c739e102d..cef0f85b7a6ea 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.api.java.ClosureCleaner;
 import org.apache.flink.cep.nfa.NFA;
+import org.apache.flink.cep.pattern.Quantifier.ConsumingStrategy;
 import org.apache.flink.cep.pattern.conditions.AndCondition;
 import org.apache.flink.cep.pattern.conditions.IterativeCondition;
 import org.apache.flink.cep.pattern.conditions.OrCondition;
@@ -56,8 +57,8 @@ public class Pattern<T, F extends T> {
 	/** Window length in which the pattern match has to occur. */
 	private Time windowTime;
 
-	/** A quantifier for the pattern. By default set to {@link Quantifier#ONE()}. */
-	private Quantifier quantifier = Quantifier.ONE();
+	/** A quantifier for the pattern. By default set to {@link Quantifier#ONE(ConsumingStrategy)}. */
+	private Quantifier quantifier = Quantifier.ONE(ConsumingStrategy.STRICT);
 
 	/**
 	 * Applicable to a {@code times} pattern, and holds
@@ -70,6 +71,15 @@ protected Pattern(final String name, final Pattern<T, ? extends T> previous) {
 		this.previous = previous;
 	}
 
+	protected Pattern(
+			final String name,
+			final Pattern<T, ? extends T> previous,
+			final ConsumingStrategy consumingStrategy) {
+		this.name = name;
+		this.previous = previous;
+		this.quantifier = Quantifier.ONE(consumingStrategy);
+	}
+
 	public Pattern<T, ? extends T> getPrevious() {
 		return previous;
 	}
@@ -195,7 +205,19 @@ public Pattern<T, F> within(Time windowTime) {
 	 * @return A new pattern which is appended to this one
 	 */
 	public Pattern<T, T> next(final String name) {
-		return new Pattern<T, T>(name, this);
+		return new Pattern<T, T>(name, this, ConsumingStrategy.STRICT);
+	}
+
+	/**
+	 * Appends a new pattern to the existing one. The new pattern enforces non-strict
+	 * temporal contiguity. This means that a matching event of this pattern and the
+	 * preceding matching event might be interleaved with other events which are ignored.
+	 *
+	 * @param name Name of the new pattern
+	 * @return A new pattern which is appended to this one
+	 */
+	public Pattern<T, T> followedBy(final String name) {
+		return new Pattern<>(name, this, ConsumingStrategy.SKIP_TILL_NEXT);
 	}
 
 	/**
@@ -206,8 +228,8 @@ public Pattern<T, T> next(final String name) {
 	 * @param name Name of the new pattern
 	 * @return A new pattern which is appended to this one
 	 */
-	public FollowedByPattern<T, T> followedBy(final String name) {
-		return new FollowedByPattern<T, T>(name, this);
+	public Pattern<T, T> followedByAny(final String name) {
+		return new Pattern<>(name, this, ConsumingStrategy.SKIP_TILL_ANY);
 	}
 
 	/**
@@ -232,12 +254,12 @@ public Pattern<T, F> optional() {
 	 * {@code A1 A2 B} appears, this will generate patterns:
 	 * {@code A1 B} and {@code A1 A2 B}. See also {@link #allowCombinations()}.
 	 *
-	 * @return The same pattern with a {@link Quantifier#ONE_OR_MORE()} quantifier applied.
+	 * @return The same pattern with a {@link Quantifier#ONE_OR_MORE(ConsumingStrategy)} quantifier applied.
 	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
 	public Pattern<T, F> oneOrMore() {
 		checkIfQuantifierApplied();
-		this.quantifier = Quantifier.ONE_OR_MORE();
+		this.quantifier = Quantifier.ONE_OR_MORE(quantifier.getConsumingStrategy());
 		return this;
 	}
 
@@ -252,14 +274,14 @@ public Pattern<T, F> oneOrMore() {
 	public Pattern<T, F> times(int times) {
 		checkIfQuantifierApplied();
 		Preconditions.checkArgument(times > 0, "You should give a positive number greater than 0.");
-		this.quantifier = Quantifier.TIMES();
+		this.quantifier = Quantifier.TIMES(quantifier.getConsumingStrategy());
 		this.times = times;
 		return this;
 	}
 
 	/**
-	 * Applicable only to {@link Quantifier#ONE_OR_MORE()} and {@link Quantifier#TIMES()} patterns,
-	 * this option allows more flexibility to the matching events.
+	 * Applicable only to {@link Quantifier#ONE_OR_MORE(ConsumingStrategy)} and
+	 * {@link Quantifier#TIMES(ConsumingStrategy)} patterns, this option allows more flexibility to the matching events.
 	 *
 	 * <p>If {@code allowCombinations()} is not applied for a
 	 * pattern {@code A.oneOrMore().followedBy(B)} and a sequence of events
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
index 0332ed493ad22..b0f882c4ef7d5 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
@@ -24,48 +24,62 @@ public class Quantifier {
 
 	private final EnumSet<QuantifierProperty> properties;
 
-	private Quantifier(final QuantifierProperty first, final QuantifierProperty... rest) {
+	private final ConsumingStrategy consumingStrategy;
+
+	private ConsumingStrategy innerConsumingStrategy = ConsumingStrategy.SKIP_TILL_NEXT;
+
+	private Quantifier(
+			final ConsumingStrategy consumingStrategy,
+			final QuantifierProperty first,
+			final QuantifierProperty... rest) {
 		this.properties = EnumSet.of(first, rest);
+		this.consumingStrategy = consumingStrategy;
 	}
 
-	public static Quantifier ONE() {
-		return new Quantifier(QuantifierProperty.SINGLE);
+	public static Quantifier ONE(final ConsumingStrategy consumingStrategy) {
+		return new Quantifier(consumingStrategy, QuantifierProperty.SINGLE);
 	}
 
-	public static Quantifier ONE_OR_MORE() {
-		return new Quantifier(QuantifierProperty.LOOPING, QuantifierProperty.EAGER);
+	public static Quantifier ONE_OR_MORE(final ConsumingStrategy consumingStrategy) {
+		return new Quantifier(consumingStrategy, QuantifierProperty.LOOPING);
 	}
 
-	public static Quantifier TIMES() {
-		return new Quantifier(QuantifierProperty.TIMES, QuantifierProperty.EAGER);
+	public static Quantifier TIMES(final ConsumingStrategy consumingStrategy) {
+		return new Quantifier(consumingStrategy, QuantifierProperty.TIMES);
 	}
 
 	public boolean hasProperty(QuantifierProperty property) {
 		return properties.contains(property);
 	}
 
-	public void combinations() {
-		if (!hasProperty(QuantifierProperty.SINGLE) && !hasProperty(Quantifier.QuantifierProperty.EAGER)) {
-			throw new MalformedPatternException("Combinations already allowed!");
-		}
+	public ConsumingStrategy getConsumingStrategy() {
+		return consumingStrategy;
+	}
 
-		if (hasProperty(Quantifier.QuantifierProperty.LOOPING) || hasProperty(Quantifier.QuantifierProperty.TIMES)) {
-			properties.remove(Quantifier.QuantifierProperty.EAGER);
-		} else {
-			throw new MalformedPatternException("Combinations not applicable to " + this + "!");
+	public ConsumingStrategy getInnerConsumingStrategy() {
+		return innerConsumingStrategy;
+	}
+
+	private static void checkPattern(boolean condition, Object errorMessage) {
+		if (!condition) {
+			throw new MalformedPatternException(String.valueOf(errorMessage));
 		}
 	}
 
+	public void combinations() {
+		checkPattern(!hasProperty(QuantifierProperty.SINGLE), "Combinations not applicable to " + this + "!");
+		checkPattern(innerConsumingStrategy != ConsumingStrategy.STRICT, "You can apply apply either combinations or consecutive, not both!");
+		checkPattern(innerConsumingStrategy != ConsumingStrategy.SKIP_TILL_ANY, "Combinations already applied!");
+
+		innerConsumingStrategy = ConsumingStrategy.SKIP_TILL_ANY;
+	}
+
 	public void consecutive() {
-		if (!hasProperty(QuantifierProperty.SINGLE) && hasProperty(Quantifier.QuantifierProperty.CONSECUTIVE)) {
-			throw new MalformedPatternException("Strict continuity already applied!");
-		}
+		checkPattern(hasProperty(QuantifierProperty.LOOPING) || hasProperty(QuantifierProperty.TIMES), "Combinations not applicable to " + this + "!");
+		checkPattern(innerConsumingStrategy != ConsumingStrategy.SKIP_TILL_ANY, "You can apply apply either combinations or consecutive, not both!");
+		checkPattern(innerConsumingStrategy != ConsumingStrategy.STRICT, "Combinations already applied!");
 
-		if (hasProperty(Quantifier.QuantifierProperty.LOOPING) || hasProperty(Quantifier.QuantifierProperty.TIMES)) {
-			properties.add(Quantifier.QuantifierProperty.CONSECUTIVE);
-		} else {
-			throw new MalformedPatternException("Strict continuity not applicable to " + this + "!");
-		}
+		innerConsumingStrategy = ConsumingStrategy.STRICT;
 	}
 
 	public void optional() {
@@ -76,13 +90,21 @@ public void optional() {
 	}
 
 	@Override
-	public boolean equals(Object obj) {
-		return obj instanceof Quantifier && properties.equals(((Quantifier)obj).properties);
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+		Quantifier that = (Quantifier) o;
+		return Objects.equals(properties, that.properties) &&
+				consumingStrategy == that.consumingStrategy;
 	}
 
 	@Override
 	public int hashCode() {
-		return Objects.hashCode(properties);
+		return Objects.hash(properties, consumingStrategy);
 	}
 
 	/**
@@ -92,9 +114,13 @@ public enum QuantifierProperty {
 		SINGLE,
 		LOOPING,
 		TIMES,
-		EAGER,
-		CONSECUTIVE,
 		OPTIONAL
 	}
 
+	public enum ConsumingStrategy {
+		STRICT,
+		SKIP_TILL_NEXT,
+		SKIP_TILL_ANY
+	}
+
 }
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/CEPITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/CEPITCase.java
index 3a3217508d238..f62c686d4d2fb 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/CEPITCase.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/CEPITCase.java
@@ -96,7 +96,7 @@ public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
 		})
-		.followedBy("middle").subtype(SubEvent.class).where(
+		.followedByAny("middle").subtype(SubEvent.class).where(
 				new SimpleCondition<SubEvent>() {
 
 					@Override
@@ -105,7 +105,7 @@ public boolean filter(SubEvent value) throws Exception {
 					}
 				}
 			)
-		.followedBy("end").where(new SimpleCondition<Event>() {
+		.followedByAny("end").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
@@ -171,7 +171,7 @@ public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
 		})
-			.followedBy("middle").subtype(SubEvent.class).where(
+			.followedByAny("middle").subtype(SubEvent.class).where(
 				new SimpleCondition<SubEvent>() {
 
 					@Override
@@ -180,7 +180,7 @@ public boolean filter(SubEvent value) throws Exception {
 					}
 				}
 			)
-			.followedBy("end").where(new SimpleCondition<Event>() {
+			.followedByAny("end").where(new SimpleCondition<Event>() {
 
 				@Override
 				public boolean filter(Event value) throws Exception {
@@ -250,13 +250,13 @@ public Event map(Tuple2<Event, Long> value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("middle");
 			}
-		}).followedBy("end").where(new SimpleCondition<Event>() {
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
@@ -339,13 +339,13 @@ public Integer getKey(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("middle");
 			}
-		}).followedBy("end").where(new SimpleCondition<Event>() {
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
@@ -416,7 +416,7 @@ public void testProcessingTimeWithWindow() throws Exception {
 
 		DataStream<Integer> input = env.fromElements(1, 2);
 
-		Pattern<Integer, ?> pattern = Pattern.<Integer>begin("start").followedBy("end").within(Time.days(1));
+		Pattern<Integer, ?> pattern = Pattern.<Integer>begin("start").followedByAny("end").within(Time.days(1));
 
 		DataStream<Integer> result = CEP.pattern(input, pattern).select(new PatternSelectFunction<Integer, Integer>() {
 			@Override
@@ -470,13 +470,13 @@ public Event map(Tuple2<Event, Long> value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("middle");
 			}
-		}).followedBy("end").where(new SimpleCondition<Event>() {
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
@@ -538,7 +538,7 @@ public boolean filter(Event value) throws Exception {
 					return value.getName().equals("start");
 				}
 			})
-			.followedBy("middle")
+			.followedByAny("middle")
 			.where(new SimpleCondition<Event>() {
 				@Override
 				public boolean filter(Event value) throws Exception {
@@ -551,7 +551,7 @@ public boolean filter(Event value) throws Exception {
 					return value.getPrice() == 5.0;
 				}
 			})
-			.followedBy("end").where(new SimpleCondition<Event>() {
+			.followedByAny("end").where(new SimpleCondition<Event>() {
 
 				@Override
 				public boolean filter(Event value) throws Exception {
@@ -623,13 +623,13 @@ public Event map(Tuple2<Event, Long> value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("middle");
 			}
-		}).followedBy("end").where(new SimpleCondition<Event>() {
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
 
 			@Override
 			public boolean filter(Event value) throws Exception {
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index f4bf647ff934a..4a00c1e00f7d8 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -26,6 +26,7 @@
 import org.apache.flink.cep.SubEvent;
 import org.apache.flink.cep.nfa.compiler.NFACompiler;
 import org.apache.flink.cep.pattern.Pattern;
+import org.apache.flink.cep.pattern.Quantifier;
 import org.apache.flink.cep.pattern.conditions.IterativeCondition;
 import org.apache.flink.cep.pattern.conditions.SimpleCondition;
 import org.apache.flink.streaming.api.windowing.time.Time;
@@ -46,6 +47,7 @@
 
 import static org.junit.Assert.assertEquals;
 
+@SuppressWarnings("unchecked")
 public class NFAITCase extends TestLogger {
 
 	@Test
@@ -56,12 +58,12 @@ public void testSimplePatternNFA() {
 		SubEvent middleEvent = new SubEvent(42, "foo", 1.0, 10.0);
 		Event endEvent = new Event(43, "end", 1.0);
 
-		inputEvents.add(new StreamRecord<Event>(startEvent, 1));
-		inputEvents.add(new StreamRecord<Event>(new Event(43, "foobar", 1.0), 2));
+		inputEvents.add(new StreamRecord<>(startEvent, 1));
+		inputEvents.add(new StreamRecord<>(new Event(43, "foobar", 1.0), 2));
 		inputEvents.add(new StreamRecord<Event>(new SubEvent(41, "barfoo", 1.0, 5.0), 3));
 		inputEvents.add(new StreamRecord<Event>(middleEvent, 3));
-		inputEvents.add(new StreamRecord<Event>(new Event(43, "start", 1.0), 4));
-		inputEvents.add(new StreamRecord<Event>(endEvent, 5));
+		inputEvents.add(new StreamRecord<>(new Event(43, "start", 1.0), 4));
+		inputEvents.add(new StreamRecord<>(endEvent, 5));
 
 		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
@@ -212,12 +214,12 @@ public void testSimplePatternWithTimeWindowNFA() {
 		final Event middleEvent;
 		final Event endEvent;
 
-		events.add(new StreamRecord<Event>(new Event(1, "start", 1.0), 1));
-		events.add(new StreamRecord<Event>(startEvent = new Event(2, "start", 1.0), 2));
-		events.add(new StreamRecord<Event>(middleEvent = new Event(3, "middle", 1.0), 3));
-		events.add(new StreamRecord<Event>(new Event(4, "foobar", 1.0), 4));
-		events.add(new StreamRecord<Event>(endEvent = new Event(5, "end", 1.0), 11));
-		events.add(new StreamRecord<Event>(new Event(6, "end", 1.0), 13));
+		events.add(new StreamRecord<>(new Event(1, "start", 1.0), 1));
+		events.add(new StreamRecord<>(startEvent = new Event(2, "start", 1.0), 2));
+		events.add(new StreamRecord<>(middleEvent = new Event(3, "middle", 1.0), 3));
+		events.add(new StreamRecord<>(new Event(4, "foobar", 1.0), 4));
+		events.add(new StreamRecord<>(endEvent = new Event(5, "end", 1.0), 11));
+		events.add(new StreamRecord<>(new Event(6, "end", 1.0), 13));
 
 
 		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
@@ -273,12 +275,12 @@ public void testSimplePatternWithTimeoutHandling() {
 		Set<Tuple2<Map<String, Event>, Long>> resultingTimeoutPatterns = new HashSet<>();
 		Set<Tuple2<Map<String, Event>, Long>> expectedTimeoutPatterns = new HashSet<>();
 
-		events.add(new StreamRecord<Event>(new Event(1, "start", 1.0), 1));
-		events.add(new StreamRecord<Event>(new Event(2, "start", 1.0), 2));
-		events.add(new StreamRecord<Event>(new Event(3, "middle", 1.0), 3));
-		events.add(new StreamRecord<Event>(new Event(4, "foobar", 1.0), 4));
-		events.add(new StreamRecord<Event>(new Event(5, "end", 1.0), 11));
-		events.add(new StreamRecord<Event>(new Event(6, "end", 1.0), 13));
+		events.add(new StreamRecord<>(new Event(1, "start", 1.0), 1));
+		events.add(new StreamRecord<>(new Event(2, "start", 1.0), 2));
+		events.add(new StreamRecord<>(new Event(3, "middle", 1.0), 3));
+		events.add(new StreamRecord<>(new Event(4, "foobar", 1.0), 4));
+		events.add(new StreamRecord<>(new Event(5, "end", 1.0), 11));
+		events.add(new StreamRecord<>(new Event(6, "end", 1.0), 13));
 
 		Map<String, Event> timeoutPattern1 = new HashMap<>();
 		timeoutPattern1.put("start", new Event(1, "start", 1.0));
@@ -306,14 +308,14 @@ public void testSimplePatternWithTimeoutHandling() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = -3268741540234334074L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("middle");
 			}
-		}).followedBy("end").where(new SimpleCondition<Event>() {
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = -8995174172182138608L;
 
 			@Override
@@ -353,13 +355,13 @@ public void testBranchingPattern() {
 		SubEvent nextOne2 = new SubEvent(45, "next-one", 1.0, 2.0);
 		Event endEvent=  new Event(46, "end", 1.0);
 
-		inputEvents.add(new StreamRecord<Event>(startEvent, 1));
+		inputEvents.add(new StreamRecord<>(startEvent, 1));
 		inputEvents.add(new StreamRecord<Event>(middleEvent1, 3));
 		inputEvents.add(new StreamRecord<Event>(middleEvent2, 4));
 		inputEvents.add(new StreamRecord<Event>(middleEvent3, 5));
 		inputEvents.add(new StreamRecord<Event>(nextOne1, 6));
 		inputEvents.add(new StreamRecord<Event>(nextOne2, 7));
-		inputEvents.add(new StreamRecord<Event>(endEvent, 8));
+		inputEvents.add(new StreamRecord<>(endEvent, 8));
 
 		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
@@ -368,21 +370,21 @@ public void testBranchingPattern() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).followedBy("middle-first").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+		}).followedByAny("middle-first").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
 			private static final long serialVersionUID = 6215754202506583964L;
 
 			@Override
 			public boolean filter(SubEvent value) throws Exception {
 				return value.getVolume() > 5.0;
 			}
-		}).followedBy("middle-second").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+		}).followedByAny("middle-second").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
 			private static final long serialVersionUID = 6215754202506583964L;
 
 			@Override
 			public boolean filter(SubEvent value) throws Exception {
 				return value.getName().equals("next-one");
 			}
-		}).followedBy("end").where(new SimpleCondition<Event>() {
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 7056763917392056548L;
 
 			@Override
@@ -449,28 +451,28 @@ public void testComplexBranchingAfterZeroOrMore() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore().allowCombinations().optional().followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedByAny("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("b");
 			}
-		}).followedBy("end2").where(new SimpleCondition<Event>() {
+		}).followedByAny("end2").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("d");
 			}
-		}).followedBy("end3").where(new SimpleCondition<Event>() {
+		}).followedByAny("end3").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -537,7 +539,7 @@ public void testZeroOrMore() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -555,27 +557,14 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		Set<Set<Event>> resultingPatterns = new HashSet<>();
-		List<Collection<Event>> allPatterns = new ArrayList<>();
-
-		for (StreamRecord<Event> inputEvent : inputEvents) {
-			Collection<Map<String, Event>> patterns = nfa.process(
-				inputEvent.getValue(),
-				inputEvent.getTimestamp()).f0;
-
-			for (Map<String, Event> foundPattern : patterns) {
-				resultingPatterns.add(new HashSet<>(foundPattern.values()));
-				allPatterns.add(foundPattern.values());
-			}
-		}
+		final List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
 
-		assertEquals(4, allPatterns.size());
-		assertEquals(Sets.newHashSet(
-			Sets.newHashSet(startEvent, middleEvent1, middleEvent2, end1),
-			Sets.newHashSet(startEvent, middleEvent1, end1),
-			Sets.newHashSet(startEvent, middleEvent2, end1),
-			Sets.newHashSet(startEvent, end1)
-		), resultingPatterns);
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(startEvent, middleEvent1, middleEvent2, end1),
+			Lists.newArrayList(startEvent, middleEvent1, end1),
+			Lists.newArrayList(startEvent, middleEvent2, end1),
+			Lists.newArrayList(startEvent, end1)
+		));
 	}
 
 	@Test
@@ -724,14 +713,14 @@ public void testZeroOrMoreAfterZeroOrMore() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle-first").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle-first").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore().allowCombinations().optional().followedBy("middle-second").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional().followedByAny("middle-second").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -803,21 +792,21 @@ public void testZeroOrMoreAfterBranching() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("branching").where(new SimpleCondition<Event>() {
+		}).followedByAny("branching").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).followedBy("merging").where(new SimpleCondition<Event>() {
+		}).followedByAny("merging").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("f");
 			}
-		}).followedBy("kleene").where(new SimpleCondition<Event>() {
+		}).followedByAny("kleene").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -940,7 +929,7 @@ public void testStrictContinuityResultsAfterZeroOrMore() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("d");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -958,25 +947,12 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		Set<Set<Event>> resultingPatterns = new HashSet<>();
-		List<Collection<Event>> allPatterns = new ArrayList<>();
-
-		for (StreamRecord<Event> inputEvent : inputEvents) {
-			Collection<Map<String, Event>> patterns = nfa.process(
-				inputEvent.getValue(),
-				inputEvent.getTimestamp()).f0;
-
-			for (Map<String, Event> foundPattern : patterns) {
-				resultingPatterns.add(new HashSet<>(foundPattern.values()));
-				allPatterns.add(foundPattern.values());
-			}
-		}
+		final List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
 
-		assertEquals(2, allPatterns.size());
-		assertEquals(Sets.newHashSet(
-			Sets.newHashSet(start, middleEvent1, middleEvent2, end),
-			Sets.newHashSet(start, middleEvent2, end)
-		), resultingPatterns);
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(start, middleEvent1, middleEvent2, end),
+			Lists.newArrayList(start, middleEvent2, end)
+		));
 	}
 
 	@Test
@@ -1000,14 +976,14 @@ public void testAtLeastOne() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore().allowCombinations().followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().followedByAny("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1183,14 +1159,14 @@ public void testAtLeastOneEager() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore().followedBy("end1").where(new SimpleCondition<Event>() {
+		}).oneOrMore().followedByAny("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1201,26 +1177,16 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		Set<Set<Event>> resultingPatterns = new HashSet<>();
-		List<Collection<Event>> allPatterns = new ArrayList<>();
+		final List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
 
-		for (StreamRecord<Event> inputEvent : inputEvents) {
-			Collection<Map<String, Event>> patterns = nfa.process(
-				inputEvent.getValue(),
-				inputEvent.getTimestamp()).f0;
-
-			for (Map<String, Event> foundPattern : patterns) {
-				resultingPatterns.add(new HashSet<>(foundPattern.values()));
-				allPatterns.add(foundPattern.values());
-			}
-		}
-
-		assertEquals(3, allPatterns.size());
-		assertEquals(Sets.newHashSet(
-			Sets.newHashSet(startEvent, middleEvent1, middleEvent2, middleEvent3, end1),
-			Sets.newHashSet(startEvent, middleEvent1, middleEvent2, end1),
-			Sets.newHashSet(startEvent, middleEvent1, end1)
-		), resultingPatterns);
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(startEvent, middleEvent1, middleEvent2, middleEvent3, end1),
+			Lists.newArrayList(startEvent, middleEvent1, middleEvent2, end1),
+			Lists.newArrayList(startEvent, middleEvent2, middleEvent3, end1),
+			Lists.newArrayList(startEvent, middleEvent3, end1),
+			Lists.newArrayList(startEvent, middleEvent2, end1),
+			Lists.newArrayList(startEvent, middleEvent1, end1)
+		));
 	}
 
 	@Test
@@ -1322,25 +1288,12 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		Set<Set<Event>> resultingPatterns = new HashSet<>();
-		List<Collection<Event>> allPatterns = new ArrayList<>();
+		final List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
 
-		for (StreamRecord<Event> inputEvent : inputEvents) {
-			Collection<Map<String, Event>> patterns = nfa.process(
-				inputEvent.getValue(),
-				inputEvent.getTimestamp()).f0;
-
-			for (Map<String, Event> foundPattern : patterns) {
-				resultingPatterns.add(new HashSet<>(foundPattern.values()));
-				allPatterns.add(foundPattern.values());
-			}
-		}
-
-		assertEquals(2, allPatterns.size());
-		assertEquals(Sets.<Set<Event>>newHashSet(
-			Sets.newHashSet(startEvent, middleEvent1, middleEvent2, end1),
-			Sets.newHashSet(startEvent, middleEvent1, middleEvent3, end1)
-		), resultingPatterns);
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(startEvent, middleEvent1, middleEvent2, end1),
+			Lists.newArrayList(startEvent, middleEvent1, middleEvent3, end1)
+		));
 	}
 
 	@Test
@@ -1375,25 +1328,13 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		Set<Set<Event>> resultingPatterns = new HashSet<>();
-		List<Collection<Event>> allPatterns = new ArrayList<>();
-
-		for (StreamRecord<Event> inputEvent : inputEvents) {
-			Collection<Map<String, Event>> patterns = nfa.process(
-				inputEvent.getValue(),
-				inputEvent.getTimestamp()).f0;
+		final List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
 
-			for (Map<String, Event> foundPattern : patterns) {
-				resultingPatterns.add(new HashSet<>(foundPattern.values()));
-				allPatterns.add(foundPattern.values());
-			}
-		}
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(middleEvent1, middleEvent2, end1),
+			Lists.newArrayList(middleEvent2, middleEvent3, end1)
+		));
 
-		assertEquals(2, allPatterns.size());
-		assertEquals(Sets.<Set<Event>>newHashSet(
-			Sets.newHashSet(middleEvent1, middleEvent2, end1),
-			Sets.newHashSet(middleEvent2, middleEvent3, end1)
-		), resultingPatterns);
 	}
 
 	@Test
@@ -1441,49 +1382,6 @@ public boolean filter(Event value) throws Exception {
 		));
 	}
 
-	@Test
-	public void testTimesStrictWithFollowedBy() {
-		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
-
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 2));
-		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 3));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
-
-		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("c");
-			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("a");
-			}
-		}).times(2).allowCombinations().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("b");
-			}
-		});
-
-		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
-
-		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
-
-		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent2, ConsecutiveData.end)
-		));
-	}
-
 	@Test
 	public void testTimesNotStrictWithFollowedByEager() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
@@ -1543,7 +1441,7 @@ public void testTimesNotStrictWithFollowedByNotEager() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -1570,49 +1468,6 @@ public boolean filter(Event value) throws Exception {
 		));
 	}
 
-	@Test
-	public void testTimesStrictWithFollowedByNotEager() {
-		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
-
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 2));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
-
-		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("c");
-			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("a");
-			}
-		}).times(2).allowCombinations().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("b");
-			}
-		});
-
-		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
-
-		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
-
-		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
-				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
-				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end)
-		));
-	}
-
 	@Test
 	public void testTimesStrictWithNextAndConsecutive() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
@@ -1789,29 +1644,16 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		Set<Set<Event>> resultingPatterns = new HashSet<>();
-		List<Collection<Event>> allPatterns = new ArrayList<>();
-
-		for (StreamRecord<Event> inputEvent : inputEvents) {
-			Collection<Map<String, Event>> patterns = nfa.process(
-				inputEvent.getValue(),
-				inputEvent.getTimestamp()).f0;
+		final List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
 
-			for (Map<String, Event> foundPattern : patterns) {
-				resultingPatterns.add(new HashSet<>(foundPattern.values()));
-				allPatterns.add(foundPattern.values());
-			}
-		}
-
-		assertEquals(6, allPatterns.size());
-		assertEquals(Sets.newHashSet(
-			Sets.newHashSet(middleEvent1,  middleEvent2, middleEvent3),
-			Sets.newHashSet(middleEvent1,  middleEvent2),
-			Sets.newHashSet(middleEvent1),
-			Sets.newHashSet(middleEvent2,  middleEvent3),
-			Sets.newHashSet(middleEvent2),
-			Sets.newHashSet(middleEvent3)
-		), resultingPatterns);
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(middleEvent1,  middleEvent2, middleEvent3),
+			Lists.newArrayList(middleEvent1,  middleEvent2),
+			Lists.newArrayList(middleEvent1),
+			Lists.newArrayList(middleEvent2,  middleEvent3),
+			Lists.newArrayList(middleEvent2),
+			Lists.newArrayList(middleEvent3)
+		));
 	}
 
 	@Test
@@ -1895,26 +1737,13 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		Set<Set<Event>> resultingPatterns = new HashSet<>();
-		List<Collection<Event>> allPatterns = new ArrayList<>();
-
-		for (StreamRecord<Event> inputEvent : inputEvents) {
-			Collection<Map<String, Event>> patterns = nfa.process(
-				inputEvent.getValue(),
-				inputEvent.getTimestamp()).f0;
-
-			for (Map<String, Event> foundPattern : patterns) {
-				resultingPatterns.add(new HashSet<>(foundPattern.values()));
-				allPatterns.add(foundPattern.values());
-			}
-		}
+		final List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
 
-		assertEquals(3, allPatterns.size());
-		assertEquals(Sets.newHashSet(
-			Sets.newHashSet(startEvent,  middleEvent1, middleEvent2, middleEvent3),
-			Sets.newHashSet(startEvent,  middleEvent1, middleEvent2),
-			Sets.newHashSet(startEvent,  middleEvent1)
-		), resultingPatterns);
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(startEvent,  middleEvent1, middleEvent2, middleEvent3),
+			Lists.newArrayList(startEvent,  middleEvent1, middleEvent2),
+			Lists.newArrayList(startEvent,  middleEvent1)
+		));
 	}
 
 	///////////////////////////////         Optional           ////////////////////////////////////////
@@ -1978,7 +1807,7 @@ public void testTimesNonStrictOptional2() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2025,7 +1854,7 @@ public void testTimesNonStrictOptional3() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2071,7 +1900,7 @@ public void testTimesStrictOptional() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2097,6 +1926,54 @@ public boolean filter(Event value) throws Exception {
 		));
 	}
 
+	@Test
+	public void testOneOrMoreStrictOptional() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 2));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 3));
+		inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 5));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).oneOrMore().consecutive().optional().followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+				Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
+
 	@Test
 	public void testTimesStrictOptional1() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
@@ -2201,94 +2078,92 @@ private ConsecutiveData() {
 	}
 
 	@Test
-	public void testStrictCombinationsOneOrMore() {
-		List<List<Event>> resultingPatterns = testStrictOneOrMore(false);
+	public void testStrictOneOrMore() {
+		List<List<Event>> resultingPatterns = testOneOrMore(Quantifier.ConsumingStrategy.STRICT);
 
 		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end),
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.end),
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent3, ConsecutiveData.end),
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent4, ConsecutiveData.end)
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end)
+		));
+	}
+
+	@Test
+	public void testSkipTillNextOneOrMore() {
+		List<List<Event>> resultingPatterns = testOneOrMore(Quantifier.ConsumingStrategy.SKIP_TILL_NEXT);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent4, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end)
 		));
 	}
 
 	@Test
-	public void testStrictEagerOneOrMore() {
-		List<List<Event>> resultingPatterns = testStrictOneOrMore(true);
+	public void testSkipTillAnyOneOrMore() {
+		List<List<Event>> resultingPatterns = testOneOrMore(Quantifier.ConsumingStrategy.SKIP_TILL_ANY);
 
 		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent4, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent4, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent4, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent4, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end)
 		));
 	}
 
-	private List<List<Event>> testStrictOneOrMore(boolean eager) {
+	private List<List<Event>> testOneOrMore(Quantifier.ConsumingStrategy strategy) {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
 
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
 		inputEvents.add(new StreamRecord<>(new Event(50, "d", 6.0), 2));
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 3));
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 5));
-		inputEvents.add(new StreamRecord<>(new Event(50, "d", 6.0), 6));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent4, 7));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 8));
-
-		Pattern<Event, ?> pattern;
-		if (eager) {
-			pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-				private static final long serialVersionUID = 5726188262756267490L;
-
-				@Override
-				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("c");
-				}
-			}).followedBy("middle").where(new SimpleCondition<Event>() {
-				private static final long serialVersionUID = 5726188262756267490L;
-
-				@Override
-				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("a");
-				}
-			}).oneOrMore().consecutive()
-					.followedBy("end1").where(new SimpleCondition<Event>() {
-						private static final long serialVersionUID = 5726188262756267490L;
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 4));
+		inputEvents.add(new StreamRecord<>(new Event(50, "d", 6.0), 5));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent4, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
 
-						@Override
-						public boolean filter(Event value) throws Exception {
-							return value.getName().equals("b");
-						}
-					});
-		} else {
-			pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-				private static final long serialVersionUID = 5726188262756267490L;
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
 
-				@Override
-				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("c");
-				}
-			}).followedBy("middle").where(new SimpleCondition<Event>() {
-				private static final long serialVersionUID = 5726188262756267490L;
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
 
-				@Override
-				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("a");
-				}
-			}).oneOrMore().allowCombinations().consecutive()
-					.followedBy("end1").where(new SimpleCondition<Event>() {
-						private static final long serialVersionUID = 5726188262756267490L;
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).oneOrMore();
 
-						@Override
-						public boolean filter(Event value) throws Exception {
-							return value.getName().equals("b");
-						}
-					});
+		switch (strategy) {
+			case STRICT:
+				pattern = pattern.consecutive();
+				break;
+			case SKIP_TILL_NEXT:
+				break;
+			case SKIP_TILL_ANY:
+				pattern = pattern.allowCombinations();
+				break;
 		}
 
+		pattern = pattern.followedBy("end1").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
 		return feedNFA(inputEvents, nfa);
@@ -2296,9 +2171,10 @@ public boolean filter(Event value) throws Exception {
 
 	@Test
 	public void testStrictEagerZeroOrMore() {
-		List<List<Event>> resultingPatterns = testStrictZeroOrMore(true);
+		List<List<Event>> resultingPatterns = testZeroOrMore(Quantifier.ConsumingStrategy.STRICT);
 
 		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
@@ -2306,67 +2182,75 @@ public void testStrictEagerZeroOrMore() {
 	}
 
 	@Test
-	public void testStrictCombinationsZeroOrMore() {
-		List<List<Event>> resultingPatterns = testStrictZeroOrMore(false);
+	public void testSkipTillAnyZeroOrMore() {
+		List<List<Event>> resultingPatterns = testZeroOrMore(Quantifier.ConsumingStrategy.SKIP_TILL_ANY);
 
 		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent4, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent4, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent4, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent4, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent3, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end),
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent2, ConsecutiveData.end),
-			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent3, ConsecutiveData.end),
 			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
 		));
 	}
 
-	private List<List<Event>> testStrictZeroOrMore(boolean eager) {
+	@Test
+	public void testSkipTillNextZeroOrMore() {
+		List<List<Event>> resultingPatterns = testZeroOrMore(Quantifier.ConsumingStrategy.SKIP_TILL_NEXT);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.middleEvent4, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.middleEvent3, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.middleEvent2, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.middleEvent1, ConsecutiveData.end),
+			Lists.newArrayList(ConsecutiveData.startEvent, ConsecutiveData.end)
+		));
+	}
+
+	private List<List<Event>> testZeroOrMore(Quantifier.ConsumingStrategy strategy) {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
 
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent, 1));
 		inputEvents.add(new StreamRecord<>(new Event(50, "d", 6.0), 2));
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent1, 3));
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent2, 4));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 4));
 		inputEvents.add(new StreamRecord<>(new Event(50, "d", 6.0), 5));
-		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+		inputEvents.add(new StreamRecord<>(ConsecutiveData.middleEvent4, 6));
 		inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
 
-		Pattern<Event, ?> pattern = eager
-				? Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-					private static final long serialVersionUID = 5726188262756267490L;
-
-					@Override
-					public boolean filter(Event value) throws Exception {
-						return value.getName().equals("c");
-					}
-				}).followedBy("middle").where(new SimpleCondition<Event>() {
-					private static final long serialVersionUID = 5726188262756267490L;
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
 
-					@Override
-					public boolean filter(Event value) throws Exception {
-						return value.getName().equals("a");
-					}
-				}).oneOrMore().optional().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
-					private static final long serialVersionUID = 5726188262756267490L;
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
 
-					@Override
-					public boolean filter(Event value) throws Exception {
-						return value.getName().equals("b");
-					}
-				})
-				: Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
-					private static final long serialVersionUID = 5726188262756267490L;
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).oneOrMore().optional();
 
-					@Override
-					public boolean filter(Event value) throws Exception {
-						return value.getName().equals("c");
-					}
-				}).followedBy("middle").where(new SimpleCondition<Event>() {
-					private static final long serialVersionUID = 5726188262756267490L;
+		switch (strategy) {
+			case STRICT:
+				pattern = pattern.consecutive();
+				break;
+			case SKIP_TILL_NEXT:
+				break;
+			case SKIP_TILL_ANY:
+				pattern = pattern.allowCombinations();
+				break;
+		}
 
-					@Override
-					public boolean filter(Event value) throws Exception {
-						return value.getName().equals("a");
-					}
-				}).oneOrMore().allowCombinations().optional().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
+		pattern = pattern.followedBy("end1").where(new SimpleCondition<Event>() {
 					private static final long serialVersionUID = 5726188262756267490L;
 
 					@Override
@@ -2380,7 +2264,6 @@ public boolean filter(Event value) throws Exception {
 		return feedNFA(inputEvents, nfa);
 	}
 
-
 	@Test
 	public void testTimesStrict() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
@@ -2400,14 +2283,14 @@ public void testTimesStrict() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).times(2).allowCombinations().consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
+		}).times(2).consecutive().followedBy("end1").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2444,7 +2327,7 @@ public void testTimesNonStrict() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("c");
 			}
-		}).followedBy("middle").where(new SimpleCondition<Event>() {
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -2733,16 +2616,10 @@ public void testIterativeWithBranchingPatternCombinations() {
 				Lists.<List<Event>>newArrayList(
 						Lists.newArrayList(startEvent1, endEvent, middleEvent1, middleEvent2, middleEvent4),
 						Lists.newArrayList(startEvent1, endEvent, middleEvent2, middleEvent1),
-						Lists.newArrayList(startEvent1, endEvent, middleEvent4, middleEvent3),
-						Lists.newArrayList(startEvent1, endEvent, middleEvent4, middleEvent2),
 						Lists.newArrayList(startEvent1, endEvent, middleEvent3, middleEvent1),
 						Lists.newArrayList(startEvent2, endEvent, middleEvent3, middleEvent4),
 						Lists.newArrayList(startEvent1, endEvent, middleEvent4, middleEvent1),
-						Lists.newArrayList(startEvent1, endEvent, middleEvent4),
 						Lists.newArrayList(startEvent1, endEvent, middleEvent1),
-						Lists.newArrayList(startEvent1, endEvent, middleEvent2),
-						Lists.newArrayList(startEvent1, endEvent, middleEvent3),
-						Lists.newArrayList(startEvent2, endEvent, middleEvent4),
 						Lists.newArrayList(startEvent2, endEvent, middleEvent3)
 				)
 		);
@@ -2798,9 +2675,7 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
-
-		return resultingPatterns;
+		return feedNFA(inputEvents, nfa);
 	}
 
 	private static class MySubeventIterCondition extends IterativeCondition<SubEvent> {
@@ -2887,9 +2762,7 @@ public boolean filter(Event value) throws Exception {
 
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
 
-		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
-
-		return resultingPatterns;
+		return feedNFA(inputEvents, nfa);
 	}
 
 	private static class MyEventIterCondition extends IterativeCondition<Event> {
@@ -2978,7 +2851,7 @@ public void testIterativeWithABACPattern() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).followedBy("middle1").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+		}).followedByAny("middle1").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
 			private static final long serialVersionUID = 2178338526904474690L;
 
 			@Override
@@ -3043,14 +2916,14 @@ public void testIterativeWithPrevPatternDependencyAfterBranching() {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("start");
 			}
-		}).oneOrMore().followedBy("middle1").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+		}).oneOrMore().followedByAny("middle1").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
 			private static final long serialVersionUID = 2178338526904474690L;
 
 			@Override
 			public boolean filter(SubEvent value) throws Exception {
 				return value.getName().startsWith("foo");
 			}
-		}).followedBy("end").where(new IterativeCondition<Event>() {
+		}).followedByAny("end").where(new IterativeCondition<Event>() {
 			private static final long serialVersionUID = 7056763917392056548L;
 
 			@Override
@@ -3084,6 +2957,131 @@ public boolean filter(Event value, Context<Event> ctx) throws Exception {
 		);
 	}
 
+
+	///////////////////////////////////////   Skip till next     /////////////////////////////
+
+	@Test
+	public void testBranchingPatternSkipTillNext() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event startEvent = new Event(40, "start", 1.0);
+		SubEvent middleEvent1 = new SubEvent(41, "foo1", 1.0, 10.0);
+		SubEvent middleEvent2 = new SubEvent(42, "foo2", 1.0, 10.0);
+		SubEvent middleEvent3 = new SubEvent(43, "foo3", 1.0, 10.0);
+		SubEvent nextOne1 = new SubEvent(44, "next-one", 1.0, 2.0);
+		SubEvent nextOne2 = new SubEvent(45, "next-one", 1.0, 2.0);
+		Event endEvent=  new Event(46, "end", 1.0);
+
+		inputEvents.add(new StreamRecord<>(startEvent, 1));
+		inputEvents.add(new StreamRecord<Event>(middleEvent1, 3));
+		inputEvents.add(new StreamRecord<Event>(middleEvent2, 4));
+		inputEvents.add(new StreamRecord<Event>(middleEvent3, 5));
+		inputEvents.add(new StreamRecord<Event>(nextOne1, 6));
+		inputEvents.add(new StreamRecord<Event>(nextOne2, 7));
+		inputEvents.add(new StreamRecord<>(endEvent, 8));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("start");
+			}
+		}).followedBy("middle-first").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+			private static final long serialVersionUID = 6215754202506583964L;
+
+			@Override
+			public boolean filter(SubEvent value) throws Exception {
+				return value.getVolume() > 5.0;
+			}
+		}).followedBy("middle-second").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+			private static final long serialVersionUID = 6215754202506583964L;
+
+			@Override
+			public boolean filter(SubEvent value) throws Exception {
+				return value.getName().equals("next-one");
+			}
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 7056763917392056548L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("end");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> patterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(patterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(startEvent, middleEvent1, nextOne1, endEvent)
+		));
+	}
+
+	@Test
+	public void testBranchingPatternMixedFollowedBy() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event startEvent = new Event(40, "start", 1.0);
+		SubEvent middleEvent1 = new SubEvent(41, "foo1", 1.0, 10.0);
+		SubEvent middleEvent2 = new SubEvent(42, "foo2", 1.0, 10.0);
+		SubEvent middleEvent3 = new SubEvent(43, "foo3", 1.0, 10.0);
+		SubEvent nextOne1 = new SubEvent(44, "next-one", 1.0, 2.0);
+		SubEvent nextOne2 = new SubEvent(45, "next-one", 1.0, 2.0);
+		Event endEvent=  new Event(46, "end", 1.0);
+
+		inputEvents.add(new StreamRecord<>(startEvent, 1));
+		inputEvents.add(new StreamRecord<Event>(middleEvent1, 3));
+		inputEvents.add(new StreamRecord<Event>(middleEvent2, 4));
+		inputEvents.add(new StreamRecord<Event>(middleEvent3, 5));
+		inputEvents.add(new StreamRecord<Event>(nextOne1, 6));
+		inputEvents.add(new StreamRecord<Event>(nextOne2, 7));
+		inputEvents.add(new StreamRecord<>(endEvent, 8));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("start");
+			}
+		}).followedByAny("middle-first").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+			private static final long serialVersionUID = 6215754202506583964L;
+
+			@Override
+			public boolean filter(SubEvent value) throws Exception {
+				return value.getVolume() > 5.0;
+			}
+		}).followedBy("middle-second").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+			private static final long serialVersionUID = 6215754202506583964L;
+
+			@Override
+			public boolean filter(SubEvent value) throws Exception {
+				return value.getName().equals("next-one");
+			}
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 7056763917392056548L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("end");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> patterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(patterns, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(startEvent, middleEvent1, nextOne1, endEvent),
+			Lists.newArrayList(startEvent, middleEvent2, nextOne1, endEvent),
+			Lists.newArrayList(startEvent, middleEvent3, nextOne1, endEvent)
+		));
+	}
+
+
+	/////////////////////////////////////////    Utility      ////////////////////////////////
 	private List<List<Event>> feedNFA(List<StreamRecord<Event>> inputEvents, NFA<Event> nfa) {
 		List<List<Event>> resultingPatterns = new ArrayList<>();
 
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
index da1f9c82fdee9..ced9efe93db10 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
@@ -149,45 +149,6 @@ public void testNFACompilerWithSimplePattern() {
 		assertEquals(0, endingState.getStateTransitions().size());
 	}
 
-	@Test
-	public void testNFACompilerWithKleeneStar() {
-
-		Pattern<Event, Event> pattern = Pattern.<Event>begin("start").where(startFilter)
-			.followedBy("middle").subtype(SubEvent.class).oneOrMore().optional()
-			.followedBy("end").where(endFilter);
-
-		NFA<Event> nfa = NFACompiler.compile(pattern, serializer, false);
-
-		Set<State<Event>> states = nfa.getStates();
-		assertEquals(5, states.size());
-
-
-		Set<Tuple2<String, Set<Tuple2<String, StateTransitionAction>>>> stateMap = new HashSet<>();
-		for (State<Event> state : states) {
-			stateMap.add(Tuple2.of(state.getName(), unfoldTransitions(state)));
-		}
-
-		assertEquals(stateMap, newHashSet(
-			Tuple2.of("start", newHashSet(Tuple2.of("middle", StateTransitionAction.TAKE))),
-			Tuple2.of("middle", newHashSet(
-				Tuple2.of("middle", StateTransitionAction.IGNORE),
-				Tuple2.of("middle", StateTransitionAction.TAKE)
-			)),
-		    Tuple2.of("middle", newHashSet(
-			    Tuple2.of("middle", StateTransitionAction.IGNORE),
-			    Tuple2.of("middle", StateTransitionAction.TAKE),
-			    Tuple2.of("end", StateTransitionAction.PROCEED)
-		    )),
-			Tuple2.of("end", newHashSet(
-				Tuple2.of(NFACompiler.ENDING_STATE_NAME, StateTransitionAction.TAKE),
-				Tuple2.of("end", StateTransitionAction.IGNORE)
-			)),
-		    Tuple2.of(NFACompiler.ENDING_STATE_NAME, Sets.newHashSet())
-		));
-
-	}
-
-
 	private <T> Set<Tuple2<String, StateTransitionAction>> unfoldTransitions(final State<T> state) {
 		final Set<Tuple2<String, StateTransitionAction>> transitions = new HashSet<>();
 		for (StateTransition<T> transition : state.getStateTransitions()) {
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java
index d599ec9c70b6b..8465bc34832ec 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java
@@ -208,6 +208,7 @@ public void testKeyedCEPOperatorCheckpointingWithRocksDB() throws Exception {
 	 * Tests that the internal time of a CEP operator advances only given watermarks. See FLINK-5033
 	 */
 	@Test
+	@SuppressWarnings("unchecked")
 	public void testKeyedAdvancingTimeWithoutElements() throws Exception {
 		final KeySelector<Event, Integer> keySelector = new TestKeySelector();
 
@@ -522,7 +523,7 @@ public boolean filter(Event value) throws Exception {
 							return value.getName().equals("start");
 						}
 					})
-					.followedBy("middle").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
+					.followedByAny("middle").subtype(SubEvent.class).where(new SimpleCondition<SubEvent>() {
 						private static final long serialVersionUID = 6215754202506583964L;
 
 						@Override
@@ -530,7 +531,7 @@ public boolean filter(SubEvent value) throws Exception {
 							return value.getVolume() > 5.0;
 						}
 					})
-					.followedBy("end").where(new SimpleCondition<Event>() {
+					.followedByAny("end").where(new SimpleCondition<Event>() {
 						private static final long serialVersionUID = 7056763917392056548L;
 
 						@Override
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
index 8c4304aea0fc3..3c1da2ed3b7b8 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.cep.Event;
 import org.apache.flink.cep.SubEvent;
+import org.apache.flink.cep.pattern.Quantifier.ConsumingStrategy;
 import org.apache.flink.cep.pattern.conditions.OrCondition;
 import org.apache.flink.cep.pattern.conditions.SimpleCondition;
 import org.apache.flink.cep.pattern.conditions.SubtypeCondition;
@@ -58,8 +59,8 @@ public void testNonStrictContiguity() {
 		assertNotNull(previous2 = previous.getPrevious());
 		assertNull(previous2.getPrevious());
 
-		assertTrue(pattern instanceof FollowedByPattern);
-		assertTrue(previous instanceof FollowedByPattern);
+		assertEquals(ConsumingStrategy.SKIP_TILL_NEXT, pattern.getQuantifier().getConsumingStrategy());
+		assertEquals(ConsumingStrategy.SKIP_TILL_NEXT, previous.getQuantifier().getConsumingStrategy());
 
 		assertEquals(pattern.getName(), "end");
 		assertEquals(previous.getName(), "next");
@@ -137,7 +138,7 @@ public boolean filter(SubEvent value) throws Exception {
 		assertNotNull(previous2 = previous.getPrevious());
 		assertNull(previous2.getPrevious());
 
-		assertTrue(pattern instanceof FollowedByPattern);
+		assertEquals(ConsumingStrategy.SKIP_TILL_NEXT, pattern.getQuantifier().getConsumingStrategy());
 		assertNotNull(previous.getCondition());
 
 		assertEquals(pattern.getName(), "end");
@@ -177,7 +178,7 @@ public boolean filter(Event value) throws Exception {
 		assertNotNull(previous2 = previous.getPrevious());
 		assertNull(previous2.getPrevious());
 
-		assertTrue(pattern instanceof FollowedByPattern);
+		assertEquals(ConsumingStrategy.SKIP_TILL_NEXT, pattern.getQuantifier().getConsumingStrategy());
 		assertFalse(previous.getCondition() instanceof OrCondition);
 		assertTrue(previous2.getCondition() instanceof OrCondition);
 

From 0b4a334842ebc6a06b41e377c1c6435079cf8869 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Fri, 28 Apr 2017 10:04:57 +0200
Subject: [PATCH 083/253] [FLINK-6293] [tests] Harden JobManagerITCase

One of the unit tests in JobManagerITCase starts a MiniCluster and sends a
LeaderSessionMessage to the JobManager without waiting until the JobManager
has gained leadership. This can lead to a dropped TriggerSavepoint message
which will cause the test to deadlock.

This PR fixes the problem by explicitly waiting for the JobManager to become
the leader.

This closes #3796.
---
 .../apache/flink/runtime/jobmanager/JobManagerITCase.scala   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
index ce8517ef4ebd8..5fb9ddfc39e59 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
@@ -765,6 +765,11 @@ class JobManagerITCase(_system: ActorSystem)
           val jobManager = flinkCluster
             .getLeaderGateway(deadline.timeLeft)
 
+          // we have to make sure that the job manager knows also that he is the leader
+          // in case of standalone leader retrieval this can happen after the getLeaderGateway call
+          val leaderFuture = jobManager.ask(NotifyWhenLeader, timeout.duration)
+          Await.ready(leaderFuture, timeout.duration)
+
           val jobId = new JobID()
 
           // Trigger savepoint for non-existing job

From 21996e1eb714faa53d9e31c94e6fb2f34affa2bc Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Thu, 9 Feb 2017 11:59:45 +0100
Subject: [PATCH 084/253] [FLINK-5810] [flip-6] Introduce a hardened slot
 manager

Harden the slot manager so that it better deals with lost and out of order messages
from the TaskManager. The basic idea is that the TaskManager are considered the ground
truth and the SlotManager tries to maintain a consistent view of what is reported to it
by the TaskManagers. This has the assumption that the TaskManagers regularly report their
slot status to the SlotManager piggy backed on the heartbeat signals to the ResourceManager.
That way it is possible to handle lost and out of order messages because the SlotManager
will eventually converge on a consistent view of the actual slot allocation.

Additionally, the hardened SlotManager registers for idle TaskManagers and pending slot
requests a timeout. If the timoeut expires, then the TaskManagers are released and the
slot request is failed. This prevents resource leaks and wasteful resource allocation.

This closes #3394.
---
 ...ResourceSlot.java => TaskManagerSlot.java} |   68 +-
 .../flink/runtime/concurrent/Executors.java   |    2 +
 .../flink/runtime/instance/SlotPool.java      |   42 +-
 .../flink/runtime/jobmaster/JobMaster.java    |    2 +-
 .../resourcemanager/ResourceManager.java      |  142 +-
 .../ResourceManagerGateway.java               |    9 +-
 .../ResourceManagerRunner.java                |    2 +-
 .../ResourceManagerRuntimeServices.java       |   25 +-
 ...ceManagerRuntimeServicesConfiguration.java |   14 +-
 .../runtime/resourcemanager/SlotRequest.java  |   14 +-
 .../StandaloneResourceManager.java            |   12 +-
 .../jobmanager/RMSlotRequestRegistered.java   |   33 -
 .../jobmanager/RMSlotRequestRejected.java     |   34 -
 .../taskexecutor/SlotAvailableReply.java      |   47 -
 .../taskexecutor/TMSlotRequestRejected.java   |   35 -
 .../taskexecutor/TMSlotRequestReply.java      |   58 -
 ...ation.java => TaskExecutorConnection.java} |    8 +-
 .../registration/WorkerRegistration.java      |    6 +-
 .../slotmanager/DefaultSlotManager.java       |   69 -
 .../slotmanager/PendingSlotRequest.java       |   93 ++
 .../ResourceManagerActions.java}              |   19 +-
 .../slotmanager/SlotManager.java              | 1086 +++++++++-----
 .../slotmanager/SlotManagerConfiguration.java |   74 +
 ...Factory.java => SlotManagerException.java} |   21 +-
 .../slotmanager/TaskManagerRegistration.java  |   89 ++
 .../runtime/taskexecutor/SlotReport.java      |   15 +-
 .../runtime/taskexecutor/SlotStatus.java      |   19 +-
 .../runtime/taskexecutor/TaskExecutor.java    |   69 +-
 .../taskexecutor/TaskExecutorGateway.java     |    3 +-
 .../exceptions/SlotOccupiedException.java}    |   30 +-
 .../taskexecutor/slot/TaskSlotTable.java      |   10 +
 .../clusterframework/ResourceManagerTest.java |   10 +-
 .../runtime/instance/SlotPoolRpcTest.java     |    2 +-
 .../flink/runtime/instance/SlotPoolTest.java  |    4 +-
 .../ResourceManagerHATest.java                |   13 +-
 .../ResourceManagerJobMasterTest.java         |   12 +-
 .../ResourceManagerTaskExecutorTest.java      |   11 +-
 .../resourcemanager/TestingSlotManager.java   |   86 --
 .../TestingSlotManagerFactory.java            |   30 -
 .../slotmanager/SlotManagerTest.java          | 1254 ++++++++++++-----
 .../slotmanager/SlotProtocolTest.java         |  328 +----
 .../taskexecutor/TaskExecutorITCase.java      |   14 +-
 .../taskexecutor/TaskExecutorTest.java        |   38 +-
 .../runtime/testingUtils/TestingUtils.scala   |   12 +
 .../YarnFlinkApplicationMasterRunner.java     |    2 +-
 .../flink/yarn/YarnResourceManager.java       |   12 +-
 46 files changed, 2360 insertions(+), 1618 deletions(-)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/{ResourceSlot.java => TaskManagerSlot.java} (51%)
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java
 rename flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/{TaskExecutorRegistration.java => TaskExecutorConnection.java} (85%)
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
 rename flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/{messages/taskexecutor/TMSlotRequestRegistered.java => slotmanager/ResourceManagerActions.java} (63%)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java
 rename flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/{SlotManagerFactory.java => SlotManagerException.java} (69%)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
 rename flink-runtime/src/main/java/org/apache/flink/runtime/{resourcemanager/messages/jobmanager/RMSlotRequestReply.java => taskexecutor/exceptions/SlotOccupiedException.java} (51%)
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/TaskManagerSlot.java
similarity index 51%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/TaskManagerSlot.java
index 0b9367d5aa956..cb9d349551894 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/TaskManagerSlot.java
@@ -18,15 +18,17 @@
 
 package org.apache.flink.runtime.clusterframework.types;
 
-import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection;
+import org.apache.flink.runtime.resourcemanager.slotmanager.PendingSlotRequest;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
- * A ResourceSlot represents a slot located in TaskManager from ResourceManager's view. It has a unique
- * identification and resource profile which we can compare to the resource request.
+ * A TaskManagerSlot represents a slot located in a TaskManager. It has a unique identification and
+ * resource profile associated.
  */
-public class ResourceSlot implements ResourceIDRetrievable {
+public class TaskManagerSlot {
 
 	/** The unique identification of this slot */
 	private final SlotID slotId;
@@ -35,17 +37,25 @@ public class ResourceSlot implements ResourceIDRetrievable {
 	private final ResourceProfile resourceProfile;
 
 	/** Gateway to the TaskExecutor which owns the slot */
-	private final TaskExecutorRegistration taskExecutorRegistration;
+	private final TaskExecutorConnection taskManagerConnection;
 
-	public ResourceSlot(SlotID slotId, ResourceProfile resourceProfile, TaskExecutorRegistration taskExecutorRegistration) {
+	/** Allocation id for which this slot has been allocated */
+	private AllocationID allocationId;
+
+	/** Assigned slot request if there is currently an ongoing request */
+	private PendingSlotRequest assignedSlotRequest;
+
+	public TaskManagerSlot(
+			SlotID slotId,
+			ResourceProfile resourceProfile,
+			TaskExecutorConnection taskManagerConnection,
+			AllocationID allocationId) {
 		this.slotId = checkNotNull(slotId);
 		this.resourceProfile = checkNotNull(resourceProfile);
-		this.taskExecutorRegistration = checkNotNull(taskExecutorRegistration);
-	}
+		this.taskManagerConnection = checkNotNull(taskManagerConnection);
 
-	@Override
-	public ResourceID getResourceID() {
-		return slotId.getResourceID();
+		this.allocationId = allocationId;
+		this.assignedSlotRequest = null;
 	}
 
 	public SlotID getSlotId() {
@@ -56,8 +66,28 @@ public ResourceProfile getResourceProfile() {
 		return resourceProfile;
 	}
 
-	public TaskExecutorRegistration getTaskExecutorRegistration() {
-		return taskExecutorRegistration;
+	public TaskExecutorConnection getTaskManagerConnection() {
+		return taskManagerConnection;
+	}
+
+	public AllocationID getAllocationId() {
+		return allocationId;
+	}
+
+	public void setAllocationId(AllocationID allocationId) {
+		this.allocationId = allocationId;
+	}
+
+	public PendingSlotRequest getAssignedSlotRequest() {
+		return assignedSlotRequest;
+	}
+
+	public void setAssignedSlotRequest(PendingSlotRequest assignedSlotRequest) {
+		this.assignedSlotRequest = assignedSlotRequest;
+	}
+
+	public InstanceID getInstanceId() {
+		return taskManagerConnection.getInstanceID();
 	}
 
 	/**
@@ -69,4 +99,16 @@ public TaskExecutorRegistration getTaskExecutorRegistration() {
 	public boolean isMatchingRequirement(ResourceProfile required) {
 		return resourceProfile.isMatching(required);
 	}
+
+	public boolean isFree() {
+		return !isAllocated() && !hasPendingSlotRequest();
+	}
+
+	public boolean isAllocated() {
+		return null != allocationId;
+	}
+
+	public boolean hasPendingSlotRequest() {
+		return null != assignedSlotRequest;
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/Executors.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/Executors.java
index 63b6a2550ba9f..e8a9be9584125 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/Executors.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/Executors.java
@@ -92,6 +92,8 @@ public static void gracefulShutdown(long timeout, TimeUnit unit, ExecutorService
 					executorService.shutdownNow();
 
 					wasInterrupted = true;
+
+					Thread.currentThread().interrupt();
 				}
 
 				timeLeft = endTime - System.currentTimeMillis();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
index 6d482b4514dc9..8cf6a9bed85d9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
@@ -35,11 +35,9 @@
 import org.apache.flink.runtime.jobmanager.slots.AllocatedSlot;
 import org.apache.flink.runtime.jobmanager.slots.SlotAndLocality;
 import org.apache.flink.runtime.jobmanager.slots.SlotOwner;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
 import org.apache.flink.runtime.rpc.RpcEndpoint;
 import org.apache.flink.runtime.rpc.RpcMethod;
 import org.apache.flink.runtime.rpc.RpcService;
@@ -129,6 +127,8 @@ public class SlotPool extends RpcEndpoint<SlotPoolGateway> {
 	/** The gateway to communicate with resource manager */
 	private ResourceManagerGateway resourceManagerGateway;
 
+	private String jobManagerAddress;
+
 	// ------------------------------------------------------------------------
 
 	public SlotPool(RpcService rpcService, JobID jobId) {
@@ -172,10 +172,12 @@ public void start() {
 	/**
 	 * Start the slot pool to accept RPC calls.
 	 *
-	 * @param jobManagerLeaderId The necessary leader id for running the job.
+	 * @param newJobManagerLeaderId The necessary leader id for running the job.
+	 * @param newJobManagerAddress for the slot requests which are sent to the resource manager
 	 */
-	public void start(UUID jobManagerLeaderId) throws Exception {
-		this.jobManagerLeaderId = jobManagerLeaderId;
+	public void start(UUID newJobManagerLeaderId, String newJobManagerAddress) throws Exception {
+		this.jobManagerLeaderId = checkNotNull(newJobManagerLeaderId);
+		this.jobManagerAddress = checkNotNull(newJobManagerAddress);
 
 		// TODO - start should not throw an exception
 		try {
@@ -315,33 +317,15 @@ private void requestSlotFromResourceManager(
 
 		pendingRequests.put(allocationID, new PendingRequest(allocationID, future, resources));
 
-		Future<RMSlotRequestReply> rmResponse = resourceManagerGateway.requestSlot(
+		Future<Acknowledge> rmResponse = resourceManagerGateway.requestSlot(
 				jobManagerLeaderId, resourceManagerLeaderId,
-				new SlotRequest(jobId, allocationID, resources),
+				new SlotRequest(jobId, allocationID, resources, jobManagerAddress),
 				resourceManagerRequestsTimeout);
 
-		// on success, trigger let the slot pool know
-		Future<Void> slotRequestProcessingFuture = rmResponse.thenAcceptAsync(new AcceptFunction<RMSlotRequestReply>() {
+		Future<Void> slotRequestProcessingFuture = rmResponse.thenAcceptAsync(new AcceptFunction<Acknowledge>() {
 			@Override
-			public void accept(RMSlotRequestReply reply) {
-				if (reply.getAllocationID() != null && reply.getAllocationID().equals(allocationID)) {
-					if (reply instanceof RMSlotRequestRegistered) {
-						slotRequestToResourceManagerSuccess(allocationID);
-					}
-					else if (reply instanceof RMSlotRequestRejected) {
-						slotRequestToResourceManagerFailed(allocationID,
-								new Exception("ResourceManager rejected slot request"));
-					}
-					else {
-						slotRequestToResourceManagerFailed(allocationID, 
-								new Exception("Unknown ResourceManager response: " + reply));
-					}
-				}
-				else {
-					future.completeExceptionally(new Exception(String.format(
-							"Bug: ResourceManager response had wrong AllocationID. Request: %s , Response: %s", 
-							allocationID, reply.getAllocationID())));
-				}
+			public void accept(Acknowledge value) {
+				slotRequestToResourceManagerSuccess(allocationID);
 			}
 		}, getMainThreadExecutor());
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index ab4357729e5dc..99dbc8692c28f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -345,7 +345,7 @@ public void startJobExecution() {
 		try {
 			// start the slot pool make sure the slot pool now accepts messages for this leader
 			log.debug("Staring SlotPool component");
-			slotPool.start(leaderSessionID);
+			slotPool.start(leaderSessionID, getAddress());
 		} catch (Exception e) {
 			log.error("Faild to start job {} ({})", jobGraph.getName(), jobGraph.getJobID(), e);
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index f17dbe5905225..2ef827648f482 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -22,6 +22,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.clusterframework.ApplicationStatus;
 import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
@@ -40,14 +41,13 @@
 import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
 import org.apache.flink.runtime.leaderelection.LeaderContender;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
 import org.apache.flink.runtime.resourcemanager.registration.JobManagerRegistration;
 import org.apache.flink.runtime.resourcemanager.registration.WorkerRegistration;
 import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.ResourceManagerActions;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcEndpoint;
 import org.apache.flink.runtime.rpc.RpcMethod;
@@ -56,6 +56,7 @@
 import org.apache.flink.runtime.jobmaster.JobMasterGateway;
 import org.apache.flink.runtime.registration.RegistrationResponse;
 
+import org.apache.flink.runtime.rpc.exceptions.LeaderSessionIDException;
 import org.apache.flink.runtime.taskexecutor.SlotReport;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
@@ -68,7 +69,6 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Executor;
 import java.util.concurrent.TimeoutException;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -114,17 +114,14 @@ public abstract class ResourceManager<WorkerType extends Serializable>
 	/** The heartbeat manager with job managers. */
 	private final HeartbeatManager<Void, Void> jobManagerHeartbeatManager;
 
-	/** The factory to construct the SlotManager. */
-	private final SlotManagerFactory slotManagerFactory;
-
 	/** Registry to use for metrics */
 	private final MetricRegistry metricRegistry;
 
 	/** Fatal error handler */
 	private final FatalErrorHandler fatalErrorHandler;
 
-	/** The SlotManager created by the slotManagerFactory when the ResourceManager is started. */
-	private SlotManager slotManager;
+	/** The slot manager maintains the available slots */
+	private final SlotManager slotManager;
 
 	/** The service to elect a ResourceManager leader. */
 	private LeaderElectionService leaderElectionService;
@@ -142,7 +139,7 @@ public ResourceManager(
 			ResourceManagerConfiguration resourceManagerConfiguration,
 			HighAvailabilityServices highAvailabilityServices,
 			HeartbeatServices heartbeatServices,
-			SlotManagerFactory slotManagerFactory,
+			SlotManager slotManager,
 			MetricRegistry metricRegistry,
 			JobLeaderIdService jobLeaderIdService,
 			FatalErrorHandler fatalErrorHandler) {
@@ -152,7 +149,7 @@ public ResourceManager(
 		this.resourceId = checkNotNull(resourceId);
 		this.resourceManagerConfiguration = checkNotNull(resourceManagerConfiguration);
 		this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
-		this.slotManagerFactory = checkNotNull(slotManagerFactory);
+		this.slotManager = checkNotNull(slotManager);
 		this.metricRegistry = checkNotNull(metricRegistry);
 		this.jobLeaderIdService = checkNotNull(jobLeaderIdService);
 		this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
@@ -187,13 +184,6 @@ public void start() throws Exception {
 		// start a leader
 		super.start();
 
-		try {
-			// SlotManager should start first
-			slotManager = slotManagerFactory.create(createResourceManagerServices());
-		} catch (Exception e) {
-			throw new ResourceManagerException("Could not create the slot manager.", e);
-		}
-
 		leaderElectionService = highAvailabilityServices.getResourceManagerLeaderElectionService();
 
 		try {
@@ -220,19 +210,25 @@ public void shutDown() throws Exception {
 		jobManagerHeartbeatManager.stop();
 
 		try {
-			super.shutDown();
+			slotManager.close();
 		} catch (Exception e) {
 			exception = ExceptionUtils.firstOrSuppressed(e, exception);
 		}
 
-		clearState();
-
 		try {
 			leaderElectionService.stop();
 		} catch (Exception e) {
 			exception = ExceptionUtils.firstOrSuppressed(e, exception);
 		}
 
+		try {
+			super.shutDown();
+		} catch (Exception e) {
+			exception = ExceptionUtils.firstOrSuppressed(e, exception);
+		}
+
+		clearState();
+
 		if (exception != null) {
 			ExceptionUtils.rethrowException(exception, "Error while shutting the ResourceManager down.");
 		}
@@ -397,10 +393,10 @@ public RegistrationResponse apply(RegistrationResponse registrationResponse, Thr
 	 */
 	@RpcMethod
 	public Future<RegistrationResponse> registerTaskExecutor(
-		final UUID resourceManagerLeaderId,
-		final String taskExecutorAddress,
+			final UUID resourceManagerLeaderId,
+			final String taskExecutorAddress,
 		final ResourceID taskExecutorResourceId,
-		final SlotReport slotReport) {
+			final SlotReport slotReport) {
 
 		if (Objects.equals(leaderSessionId, resourceManagerLeaderId)) {
 			Future<TaskExecutorGateway> taskExecutorGatewayFuture = getRpcService().connect(taskExecutorAddress, TaskExecutorGateway.class);
@@ -415,6 +411,9 @@ public RegistrationResponse apply(final TaskExecutorGateway taskExecutorGateway,
 						if (oldRegistration != null) {
 							// TODO :: suggest old taskExecutor to stop itself
 							log.info("Replacing old instance of worker for ResourceID {}", taskExecutorResourceId);
+
+							// remove old task manager registration from slot manager
+							slotManager.unregisterTaskManager(oldRegistration.getInstanceID());
 						}
 
 						WorkerType newWorker = workerStarted(taskExecutorResourceId);
@@ -422,7 +421,8 @@ public RegistrationResponse apply(final TaskExecutorGateway taskExecutorGateway,
 							new WorkerRegistration<>(taskExecutorGateway, newWorker);
 
 						taskExecutors.put(taskExecutorResourceId, registration);
-						slotManager.registerTaskExecutor(taskExecutorResourceId, registration, slotReport);
+
+						slotManager.registerTaskManager(registration, slotReport);
 
 						taskManagerHeartbeatManager.monitorTarget(taskExecutorResourceId, new HeartbeatTarget<Void>() {
 							@Override
@@ -483,26 +483,34 @@ public void disconnectJobManager(final JobID jobId, final Exception cause) {
 	 * @return Slot assignment
 	 */
 	@RpcMethod
-	public RMSlotRequestReply requestSlot(
+	public Acknowledge requestSlot(
 			UUID jobMasterLeaderID,
 			UUID resourceManagerLeaderID,
-			SlotRequest slotRequest) {
+			SlotRequest slotRequest) throws ResourceManagerException, LeaderSessionIDException {
 
-		log.info("Request slot with profile {} for job {} with allocation id {}.",
-			slotRequest.getResourceProfile(),
-			slotRequest.getJobId(),
-			slotRequest.getAllocationId());
+		if (!Objects.equals(resourceManagerLeaderID, leaderSessionId)) {
+			throw new LeaderSessionIDException(resourceManagerLeaderID, leaderSessionId);
+		}
 
 		JobID jobId = slotRequest.getJobId();
 		JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.get(jobId);
 
-		if (jobManagerRegistration != null
-				&& jobMasterLeaderID.equals(jobManagerRegistration.getLeaderID())
-				&& resourceManagerLeaderID.equals(leaderSessionId)) {
-			return slotManager.requestSlot(slotRequest);
+		if (null != jobManagerRegistration) {
+			if (Objects.equals(jobMasterLeaderID, jobManagerRegistration.getLeaderID())) {
+				log.info("Request slot with profile {} for job {} with allocation id {}.",
+					slotRequest.getResourceProfile(),
+					slotRequest.getJobId(),
+					slotRequest.getAllocationId());
+
+				slotManager.registerSlotRequest(slotRequest);
+
+				return Acknowledge.get();
+			} else {
+				throw new LeaderSessionIDException(jobMasterLeaderID, jobManagerRegistration.getLeaderID());
+			}
+
 		} else {
-			log.info("Ignoring slot request for unknown JobMaster with JobID {}", jobId);
-			return new RMSlotRequestRejected(slotRequest.getAllocationId());
+			throw new ResourceManagerException("Could not find registered job manager for job " + jobId + '.');
 		}
 	}
 
@@ -511,23 +519,23 @@ public RMSlotRequestReply requestSlot(
 	 * @param resourceManagerLeaderId TaskExecutor's resource manager leader id
 	 * @param instanceID TaskExecutor's instance id
 	 * @param slotId The slot id of the available slot
-	 * @return SlotAvailableReply
 	 */
 	@RpcMethod
 	public void notifySlotAvailable(
 			final UUID resourceManagerLeaderId,
 			final InstanceID instanceID,
-			final SlotID slotId) {
+			final SlotID slotId,
+			final AllocationID allocationId) {
 
-		if (resourceManagerLeaderId.equals(leaderSessionId)) {
+		if (Objects.equals(resourceManagerLeaderId, leaderSessionId)) {
 			final ResourceID resourceId = slotId.getResourceID();
 			WorkerRegistration<WorkerType> registration = taskExecutors.get(resourceId);
 
 			if (registration != null) {
 				InstanceID registrationId = registration.getInstanceID();
 
-				if (registrationId.equals(instanceID)) {
-					slotManager.notifySlotAvailable(resourceId, slotId);
+				if (Objects.equals(registrationId, instanceID)) {
+					slotManager.freeSlot(slotId, allocationId);
 				} else {
 					log.debug("Invalid registration id for slot available message. This indicates an" +
 						" outdated request.");
@@ -587,8 +595,8 @@ public void unRegisterInfoMessageListener(final String address) {
 	/**
 	 * Cleanup application and shut down cluster
 	 *
-	 * @param finalStatus
-	 * @param optionalDiagnostics
+	 * @param finalStatus of the Flink application
+	 * @param optionalDiagnostics for the Flink application
 	 */
 	@RpcMethod
 	public void shutDownCluster(final ApplicationStatus finalStatus, final String optionalDiagnostics) {
@@ -597,12 +605,12 @@ public void shutDownCluster(final ApplicationStatus finalStatus, final String op
 	}
 
 	@RpcMethod
-	public Integer getNumberOfRegisteredTaskManagers(UUID leaderSessionId) throws LeaderIdMismatchException {
-		if (this.leaderSessionId != null && this.leaderSessionId.equals(leaderSessionId)) {
+	public Integer getNumberOfRegisteredTaskManagers(UUID requestLeaderSessionId) throws LeaderIdMismatchException {
+		if (Objects.equals(leaderSessionId, requestLeaderSessionId)) {
 			return taskExecutors.size();
 		}
 		else {
-			throw new LeaderIdMismatchException(this.leaderSessionId, leaderSessionId);
+			throw new LeaderIdMismatchException(leaderSessionId, requestLeaderSessionId);
 		}
 	}
 
@@ -628,7 +636,6 @@ private void clearState() {
 		jobManagerRegistrations.clear();
 		jmResourceIdRegistrations.clear();
 		taskExecutors.clear();
-		slotManager.clearState();
 
 		try {
 			jobLeaderIdService.clear();
@@ -686,7 +693,7 @@ protected void closeTaskManagerConnection(final ResourceID resourceID, final Exc
 			log.info("Task manager {} failed because {}.", resourceID, cause);
 
 			// TODO :: suggest failed task executor to stop itself
-			slotManager.notifyTaskManagerFailure(resourceID);
+			slotManager.unregisterTaskManager(workerRegistration.getInstanceID());
 
 			workerRegistration.getTaskExecutorGateway().disconnectResourceManager(cause);
 		} else {
@@ -702,11 +709,7 @@ protected void closeTaskManagerConnection(final ResourceID resourceID, final Exc
 	 * @return True if the given leader id matches the actual leader id and is not null; otherwise false
 	 */
 	protected boolean isValid(UUID resourceManagerLeaderId) {
-		if (resourceManagerLeaderId == null) {
-			return false;
-		} else {
-			return resourceManagerLeaderId.equals(leaderSessionId);
-		}
+		return Objects.equals(resourceManagerLeaderId, leaderSessionId);
 	}
 
 	protected void removeJob(JobID jobId) {
@@ -725,7 +728,7 @@ protected void jobLeaderLostLeadership(JobID jobId, UUID oldJobLeaderId) {
 		if (jobManagerRegistrations.containsKey(jobId)) {
 			JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.get(jobId);
 
-			if (jobManagerRegistration.getLeaderID().equals(oldJobLeaderId)) {
+			if (Objects.equals(jobManagerRegistration.getLeaderID(), oldJobLeaderId)) {
 				disconnectJobManager(jobId, new Exception("Job leader lost leadership."));
 			} else {
 				log.debug("Discarding job leader lost leadership, because a new job leader was found for job {}. ", jobId);
@@ -806,6 +809,8 @@ public void run() {
 
 				leaderSessionId = newLeaderSessionID;
 
+				slotManager.start(leaderSessionId, getMainThreadExecutor(), new ResourceManagerActionsImpl());
+
 				getRpcService().execute(new Runnable() {
 					@Override
 					public void run() {
@@ -829,6 +834,8 @@ public void run() {
 
 				clearState();
 
+				slotManager.suspend();
+
 				leaderSessionId = null;
 			}
 		});
@@ -874,6 +881,8 @@ public void handleError(final Exception exception) {
 	@VisibleForTesting
 	public abstract void startNewWorker(ResourceProfile resourceProfile);
 
+	public abstract void stopWorker(InstanceID instanceId);
+
 	/**
 	 * Callback when a worker was started.
 	 * @param resourceID The worker resource id
@@ -884,30 +893,21 @@ public void handleError(final Exception exception) {
 	//  Static utility classes
 	// ------------------------------------------------------------------------
 
-	protected ResourceManagerServices createResourceManagerServices() {
-		return new DefaultResourceManagerServices();
-	}
-
-	private class DefaultResourceManagerServices implements ResourceManagerServices {
-
-		@Override
-		public UUID getLeaderID() {
-			return ResourceManager.this.leaderSessionId;
-		}
+	private class ResourceManagerActionsImpl implements ResourceManagerActions {
 
 		@Override
-		public void allocateResource(ResourceProfile resourceProfile) {
-			ResourceManager.this.startNewWorker(resourceProfile);
+		public void releaseResource(InstanceID instanceId) {
+			stopWorker(instanceId);
 		}
 
 		@Override
-		public Executor getAsyncExecutor() {
-			return ResourceManager.this.getRpcService().getExecutor();
+		public void allocateResource(ResourceProfile resourceProfile) throws ResourceManagerException {
+			startNewWorker(resourceProfile);
 		}
 
 		@Override
-		public Executor getMainThreadExecutor() {
-			return ResourceManager.this.getMainThreadExecutor();
+		public void notifyAllocationFailure(JobID jobId, AllocationID allocationId, Exception cause) {
+			log.info("Slot request with allocation id {} for job {} failed.", allocationId, jobId, cause);
 		}
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index 530113f815049..fcbedcbba61a3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -21,11 +21,12 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.instance.InstanceID;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.rpc.RpcGateway;
 import org.apache.flink.runtime.rpc.RpcTimeout;
 import org.apache.flink.runtime.jobmaster.JobMaster;
@@ -66,7 +67,7 @@ Future<RegistrationResponse> registerJobManager(
 	 * @param slotRequest The slot to request
 	 * @return The confirmation that the slot gets allocated
 	 */
-	Future<RMSlotRequestReply> requestSlot(
+	Future<Acknowledge> requestSlot(
 		UUID resourceManagerLeaderID,
 		UUID jobMasterLeaderID,
 		SlotRequest slotRequest,
@@ -96,11 +97,13 @@ Future<RegistrationResponse> registerTaskExecutor(
 	 * @param resourceManagerLeaderId The ResourceManager leader id
 	 * @param instanceId TaskExecutor's instance id
 	 * @param slotID The SlotID of the freed slot
+	 * @param oldAllocationId to which the slot has been allocated
 	 */
 	void notifySlotAvailable(
 		UUID resourceManagerLeaderId,
 		InstanceID instanceId,
-		SlotID slotID);
+		SlotID slotID,
+		AllocationID oldAllocationId);
 
 	/**
 	 * Registers an infoMessage listener
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
index c6d64db167e80..d0c411ceea6bb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
@@ -76,7 +76,7 @@ public ResourceManagerRunner(
 			resourceManagerConfiguration,
 			highAvailabilityServices,
 			heartbeatServices,
-			resourceManagerRuntimeServices.getSlotManagerFactory(),
+			resourceManagerRuntimeServices.getSlotManager(),
 			metricRegistry,
 			resourceManagerRuntimeServices.getJobLeaderIdService(),
 			this);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java
index 56edde4c1a370..ed8f1e0cd9689 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java
@@ -20,8 +20,8 @@
 
 import org.apache.flink.runtime.concurrent.ScheduledExecutor;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.resourcemanager.slotmanager.DefaultSlotManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerConfiguration;
 import org.apache.flink.util.Preconditions;
 
 /**
@@ -29,16 +29,16 @@
  */
 public class ResourceManagerRuntimeServices {
 
-	private final SlotManagerFactory slotManagerFactory;
+	private final SlotManager slotManager;
 	private final JobLeaderIdService jobLeaderIdService;
 
-	public ResourceManagerRuntimeServices(SlotManagerFactory slotManagerFactory, JobLeaderIdService jobLeaderIdService) {
-		this.slotManagerFactory = Preconditions.checkNotNull(slotManagerFactory);
+	public ResourceManagerRuntimeServices(SlotManager slotManager, JobLeaderIdService jobLeaderIdService) {
+		this.slotManager = Preconditions.checkNotNull(slotManager);
 		this.jobLeaderIdService = Preconditions.checkNotNull(jobLeaderIdService);
 	}
 
-	public SlotManagerFactory getSlotManagerFactory() {
-		return slotManagerFactory;
+	public SlotManager getSlotManager() {
+		return slotManager;
 	}
 
 	public JobLeaderIdService getJobLeaderIdService() {
@@ -58,12 +58,19 @@ public static ResourceManagerRuntimeServices fromConfiguration(
 			HighAvailabilityServices highAvailabilityServices,
 			ScheduledExecutor scheduledExecutor) throws Exception {
 
-		final SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
+		final SlotManagerConfiguration slotManagerConfiguration = configuration.getSlotManagerConfiguration();
+
+		final SlotManager slotManager = new SlotManager(
+			scheduledExecutor,
+			slotManagerConfiguration.getTaskManagerRequestTimeout(),
+			slotManagerConfiguration.getSlotRequestTimeout(),
+			slotManagerConfiguration.getTaskManagerTimeout());
+
 		final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
 			highAvailabilityServices,
 			scheduledExecutor,
 			configuration.getJobTimeout());
 
-		return new ResourceManagerRuntimeServices(slotManagerFactory, jobLeaderIdService);
+		return new ResourceManagerRuntimeServices(slotManager, jobLeaderIdService);
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java
index 6de5f4d615357..79d1c02a60271 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java
@@ -22,6 +22,7 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.ResourceManagerOptions;
 import org.apache.flink.runtime.resourcemanager.exceptions.ConfigurationException;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerConfiguration;
 import org.apache.flink.util.Preconditions;
 import scala.concurrent.duration.Duration;
 
@@ -32,14 +33,21 @@ public class ResourceManagerRuntimeServicesConfiguration {
 
 	private final Time jobTimeout;
 
-	public ResourceManagerRuntimeServicesConfiguration(Time jobTimeout) {
+	private final SlotManagerConfiguration slotManagerConfiguration;
+
+	public ResourceManagerRuntimeServicesConfiguration(Time jobTimeout, SlotManagerConfiguration slotManagerConfiguration) {
 		this.jobTimeout = Preconditions.checkNotNull(jobTimeout);
+		this.slotManagerConfiguration = Preconditions.checkNotNull(slotManagerConfiguration);
 	}
 
 	public Time getJobTimeout() {
 		return jobTimeout;
 	}
 
+	public SlotManagerConfiguration getSlotManagerConfiguration() {
+		return slotManagerConfiguration;
+	}
+
 	// ---------------------------- Static methods ----------------------------------
 
 	public static ResourceManagerRuntimeServicesConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {
@@ -54,6 +62,8 @@ public static ResourceManagerRuntimeServicesConfiguration fromConfiguration(Conf
 				"value " + ResourceManagerOptions.JOB_TIMEOUT + '.', e);
 		}
 
-		return new ResourceManagerRuntimeServicesConfiguration(jobTimeout);
+		final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);
+
+		return new ResourceManagerRuntimeServicesConfiguration(jobTimeout, slotManagerConfiguration);
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequest.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequest.java
index 896421ba7f4ac..779e9135bd4eb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequest.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequest.java
@@ -42,10 +42,18 @@ public class SlotRequest implements Serializable {
 	/** The resource profile of the required slot */
 	private final ResourceProfile resourceProfile;
 
-	public SlotRequest(JobID jobId, AllocationID allocationId, ResourceProfile resourceProfile) {
+	/** Address of the emitting job manager */
+	private final String targetAddress;
+
+	public SlotRequest(
+			JobID jobId,
+			AllocationID allocationId,
+			ResourceProfile resourceProfile,
+			String targetAddress) {
 		this.jobId = checkNotNull(jobId);
 		this.allocationId = checkNotNull(allocationId);
 		this.resourceProfile = checkNotNull(resourceProfile);
+		this.targetAddress = checkNotNull(targetAddress);
 	}
 
 	/**
@@ -71,4 +79,8 @@ public AllocationID getAllocationId() {
 	public ResourceProfile getResourceProfile() {
 		return resourceProfile;
 	}
+
+	public String getTargetAddress() {
+		return targetAddress;
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
index 389ca5c82666f..a921a29183ba8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
@@ -23,9 +23,10 @@
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
 
@@ -44,7 +45,7 @@ public StandaloneResourceManager(
 			ResourceManagerConfiguration resourceManagerConfiguration,
 			HighAvailabilityServices highAvailabilityServices,
 			HeartbeatServices heartbeatServices,
-			SlotManagerFactory slotManagerFactory,
+			SlotManager slotManager,
 			MetricRegistry metricRegistry,
 			JobLeaderIdService jobLeaderIdService,
 			FatalErrorHandler fatalErrorHandler) {
@@ -55,7 +56,7 @@ public StandaloneResourceManager(
 			resourceManagerConfiguration,
 			highAvailabilityServices,
 			heartbeatServices,
-			slotManagerFactory,
+			slotManager,
 			metricRegistry,
 			jobLeaderIdService,
 			fatalErrorHandler);
@@ -74,6 +75,11 @@ protected void shutDownApplication(ApplicationStatus finalStatus, String optiona
 	public void startNewWorker(ResourceProfile resourceProfile) {
 	}
 
+	@Override
+	public void stopWorker(InstanceID instanceId) {
+
+	}
+
 	@Override
 	protected ResourceID workerStarted(ResourceID resourceID) {
 		return resourceID;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java
deleted file mode 100644
index 01bc5326458d4..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.messages.jobmanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-
-/**
- * Acknowledgment by the ResourceManager for a SlotRequest from the JobManager
- */
-public class RMSlotRequestRegistered extends RMSlotRequestReply {
-
-	private static final long serialVersionUID = 4760320859275256855L;
-
-	public RMSlotRequestRegistered(AllocationID allocationID) {
-		super(allocationID);
-	}
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java
deleted file mode 100644
index 649d61cc68f8a..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.messages.jobmanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-
-/**
- * Rejection message by the ResourceManager for a SlotRequest from the JobManager
- */
-public class RMSlotRequestRejected extends RMSlotRequestReply {
-
-	private static final long serialVersionUID = 9049346740895325144L;
-
-	public RMSlotRequestRejected(AllocationID allocationID) {
-		super(allocationID);
-	}
-
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java
deleted file mode 100644
index f2e0105e025bd..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
-
-import org.apache.flink.runtime.clusterframework.types.SlotID;
-
-import java.io.Serializable;
-import java.util.UUID;
-
-/**
- * Sent by the ResourceManager to the TaskExecutor confirm receipt of
- * {@code org.apache.flink.runtime.resourcemanager.ResourceManagerGateway.notifySlotAvailable}.
- */
-public class SlotAvailableReply implements Serializable {
-
-	private final UUID resourceManagerLeaderID;
-
-	private final SlotID slotID;
-
-	public SlotAvailableReply(UUID resourceManagerLeaderID, SlotID slotID) {
-		this.resourceManagerLeaderID = resourceManagerLeaderID;
-		this.slotID = slotID;
-	}
-
-	public UUID getResourceManagerLeaderID() {
-		return resourceManagerLeaderID;
-	}
-
-	public SlotID getSlotID() {
-		return slotID;
-	}
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java
deleted file mode 100644
index 9b10a350544de..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.instance.InstanceID;
-
-/**
- * Rejection by the TaskExecutor for a SlotRequest from the ResourceManager
- */
-public class TMSlotRequestRejected extends TMSlotRequestReply {
-
-	private static final long serialVersionUID = 9049346740895325144L;
-
-	public TMSlotRequestRejected(InstanceID instanceID, ResourceID resourceID, AllocationID allocationID) {
-		super(instanceID, resourceID, allocationID);
-	}
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java
deleted file mode 100644
index b23b6e9a5e05b..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.instance.InstanceID;
-
-import java.io.Serializable;
-
-/**
- * Acknowledgment by the TaskExecutor for a SlotRequest from the ResourceManager
- */
-public abstract class TMSlotRequestReply implements Serializable {
-
-	private static final long serialVersionUID = 42;
-
-	private final InstanceID instanceID;
-
-	private final ResourceID resourceID;
-
-	private final AllocationID allocationID;
-
-	protected TMSlotRequestReply(InstanceID instanceID, ResourceID resourceID, AllocationID allocationID) {
-		this.instanceID = instanceID;
-		this.resourceID = resourceID;
-		this.allocationID = allocationID;
-	}
-
-	public InstanceID getInstanceID() {
-		return instanceID;
-	}
-
-	public ResourceID getResourceID() {
-		return resourceID;
-	}
-
-	public AllocationID getAllocationID() {
-		return allocationID;
-	}
-
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorConnection.java
similarity index 85%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorConnection.java
index bfa9c0019e732..e4522f2351da6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorConnection.java
@@ -21,21 +21,17 @@
 import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 
-import java.io.Serializable;
-
 /**
  * This class is responsible for grouping the TaskExecutorGateway and the InstanceID
  * of a registered task executor.
  */
-public class TaskExecutorRegistration implements Serializable {
-
-	private static final long serialVersionUID = -2062957799469434614L;
+public class TaskExecutorConnection {
 
 	private final InstanceID instanceID;
 
 	private TaskExecutorGateway taskExecutorGateway;
 
-	public TaskExecutorRegistration(TaskExecutorGateway taskExecutorGateway) {
+	public TaskExecutorConnection(TaskExecutorGateway taskExecutorGateway) {
 		this.instanceID = new InstanceID();
 		this.taskExecutorGateway = taskExecutorGateway;
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
index 7ee7a1fea76a3..8f949b09d2632 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
@@ -24,11 +24,9 @@
 import java.io.Serializable;
 
 /**
- * This class extends the {@link TaskExecutorRegistration}, adding the worker information.
+ * This class extends the {@link TaskExecutorConnection}, adding the worker information.
  */
-public class WorkerRegistration<WorkerType extends Serializable> extends TaskExecutorRegistration {
-
-	private static final long serialVersionUID = -2062957799469434614L;
+public class WorkerRegistration<WorkerType extends Serializable> extends TaskExecutorConnection {
 
 	private final WorkerType worker;
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java
deleted file mode 100644
index 950893600a300..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.slotmanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
-import org.apache.flink.runtime.clusterframework.types.SlotID;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
-import org.apache.flink.runtime.resourcemanager.SlotRequest;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * A slot manager that answers requests with slots without any special logic. The first slot
- * in the maps to match a request is chosen.
- */
-public class DefaultSlotManager extends SlotManager {
-
-	public DefaultSlotManager(ResourceManagerServices rmServices) {
-		super(rmServices);
-	}
-
-	@Override
-	protected ResourceSlot chooseSlotToUse(SlotRequest request, Map<SlotID, ResourceSlot> freeSlots) {
-		final Iterator<ResourceSlot> slotIterator = freeSlots.values().iterator();
-		if (slotIterator.hasNext()) {
-			return slotIterator.next();
-		} else {
-			return null;
-		}
-	}
-
-	@Override
-	protected SlotRequest chooseRequestToFulfill(ResourceSlot offeredSlot, Map<AllocationID, SlotRequest> pendingRequests) {
-		final Iterator<SlotRequest> requestIterator = pendingRequests.values().iterator();
-		if (requestIterator.hasNext()) {
-			return requestIterator.next();
-		} else {
-			return null;
-		}
-	}
-
-	// ------------------------------------------------------------------------
-
-	public static class Factory implements SlotManagerFactory {
-
-		@Override
-		public SlotManager create(ResourceManagerServices rmServices) {
-			return new DefaultSlotManager(rmServices);
-		}
-	}
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
new file mode 100644
index 0000000000000..894f14639ae19
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.slotmanager;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
+import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.resourcemanager.SlotRequest;
+import org.apache.flink.util.Preconditions;
+
+import java.util.UUID;
+import java.util.concurrent.ScheduledFuture;
+
+public class PendingSlotRequest {
+
+	private final SlotRequest slotRequest;
+
+	private CompletableFuture<Acknowledge> requestFuture;
+
+	private UUID timeoutIdentifier;
+
+	private ScheduledFuture<?> timeoutFuture;
+
+	public PendingSlotRequest(SlotRequest slotRequest) {
+		this.slotRequest = Preconditions.checkNotNull(slotRequest);
+	}
+
+	public AllocationID getAllocationId() {
+		return slotRequest.getAllocationId();
+	}
+
+	public ResourceProfile getResourceProfile() {
+		return slotRequest.getResourceProfile();
+	}
+
+	public UUID getTimeoutIdentifier() {
+		return timeoutIdentifier;
+	}
+
+	public JobID getJobId() {
+		return slotRequest.getJobId();
+	}
+
+	public String getTargetAddress() {
+		return slotRequest.getTargetAddress();
+	}
+
+	public boolean isAssigned() {
+		return null != requestFuture;
+	}
+
+	public void setRequestFuture(CompletableFuture<Acknowledge> requestFuture) {
+		this.requestFuture = requestFuture;
+	}
+
+	public CompletableFuture<Acknowledge> getRequestFuture() {
+		return requestFuture;
+	}
+
+	public void cancelTimeout() {
+		if (timeoutFuture != null) {
+			timeoutFuture.cancel(true);
+
+			timeoutIdentifier = null;
+			timeoutFuture = null;
+		}
+	}
+
+	public void registerTimeout(ScheduledFuture<?> newTimeoutFuture, UUID newTimeoutIdentifier) {
+		cancelTimeout();
+
+		timeoutFuture = newTimeoutFuture;
+		timeoutIdentifier = newTimeoutIdentifier;
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRegistered.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/ResourceManagerActions.java
similarity index 63%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRegistered.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/ResourceManagerActions.java
index c0f0f4906a1f1..c8b288ead9f97 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRegistered.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/ResourceManagerActions.java
@@ -16,20 +16,19 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
+package org.apache.flink.runtime.resourcemanager.slotmanager;
 
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
 
-/**
- * Acknowledgment by the TaskExecutor for a SlotRequest from the ResourceManager
- */
-public class TMSlotRequestRegistered extends TMSlotRequestReply {
+public interface ResourceManagerActions {
+
+	void releaseResource(InstanceID instanceId);
 
-	private static final long serialVersionUID = 4760320859275256855L;
+	void allocateResource(ResourceProfile resourceProfile) throws ResourceManagerException;
 
-	public TMSlotRequestRegistered(InstanceID instanceID, ResourceID resourceID, AllocationID allocationID) {
-		super(instanceID, resourceID, allocationID);
-	}
+	void notifyAllocationFailure(JobID jobId, AllocationID allocationId, Exception cause);
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index f055971c4bec5..31edbf38e6973 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -21,519 +21,893 @@
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.clusterframework.types.TaskManagerSlot;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
 import org.apache.flink.runtime.concurrent.BiFunction;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
 import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
+import org.apache.flink.runtime.concurrent.ScheduledExecutor;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection;
 import org.apache.flink.runtime.taskexecutor.SlotReport;
 import org.apache.flink.runtime.taskexecutor.SlotStatus;
+import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotOccupiedException;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 /**
- * SlotManager is responsible for receiving slot requests and do slot allocations. It allows to request
- * slots from registered TaskManagers and issues container allocation requests in case of there are not
- * enough available slots. Besides, it should sync its slot allocation with TaskManager's heartbeat.
- * <p>
- * The main operation principle of SlotManager is:
- * <ul>
- * <li>1. All slot allocation status should be synced with TaskManager, which is the ground truth.</li>
- * <li>2. All slots that have registered must be tracked, either by free pool or allocated pool.</li>
- * <li>3. All slot requests will be handled by best efforts, there is no guarantee that one request will be
- * fulfilled in time or correctly allocated. Conflicts or timeout or some special error will happen, it should
- * be handled outside SlotManager. SlotManager will make each decision based on the information it currently
- * holds.</li>
- * </ul>
- * <b>IMPORTANT:</b> This class is <b>Not Thread-safe</b>.
+ * The slot manager is responsible for maintaining a view on all registered task manager slots,
+ * their allocation and all pending slot requests. Whenever a new slot is registered or and
+ * allocated slot is freed, then it tries to fulfill another pending slot request. Whenever there
+ * are not enough slots available the slot manager will notify the resource manager about it via
+ * {@link ResourceManagerActions#allocateResource(ResourceProfile)}.
+ *
+ * In order to free resources and avoid resource leaks, idling task managers (task managers whose
+ * slots are currently not used) and not fulfilled pending slot requests time out triggering their
+ * release and failure, respectively.
  */
-public abstract class SlotManager {
+public class SlotManager implements AutoCloseable {
+
+	private static final Logger LOG = LoggerFactory.getLogger(SlotManager.class);
+
+	/** Scheduled executor for timeouts */
+	private final ScheduledExecutor scheduledExecutor;
+
+	/** Timeout for slot requests to the task manager */
+	private final Time taskManagerRequestTimeout;
+
+	/** Timeout after which an allocation is discarded */
+	private final Time slotRequestTimeout;
+
+	/** Timeout after which an unused TaskManager is released */
+	private final Time taskManagerTimeout;
+
+	/** Map for all registered slots */
+	private final HashMap<SlotID, TaskManagerSlot> slots;
 
-	protected final Logger LOG = LoggerFactory.getLogger(getClass());
+	/** Index of all currently free slots */
+	private final LinkedHashMap<SlotID, TaskManagerSlot> freeSlots;
 
-	/** The Resource allocation provider */
-	protected final ResourceManagerServices rmServices;
+	/** All currently registered task managers */
+	private final HashMap<InstanceID, TaskManagerRegistration> taskManagerRegistrations;
 
-	/** All registered task managers with ResourceID and gateway. */
-	private final Map<ResourceID, TaskExecutorRegistration> taskManagers;
+	/** Map of fulfilled and active allocations for request deduplication purposes */
+	private final HashMap<AllocationID, SlotID> fulfilledSlotRequests;
 
-	/** All registered slots, including free and allocated slots */
-	private final Map<ResourceID, Map<SlotID, ResourceSlot>> registeredSlots;
+	/** Map of pending/unfulfilled slot allocation requests */
+	private final HashMap<AllocationID, PendingSlotRequest> pendingSlotRequests;
 
-	/** All pending slot requests, waiting available slots to fulfil */
-	private final Map<AllocationID, SlotRequest> pendingSlotRequests;
+	/** Leader id of the containing component */
+	private UUID leaderId;
 
-	/** All free slots that can be used to be allocated */
-	private final Map<SlotID, ResourceSlot> freeSlots;
+	/** Executor for future callbacks which have to be "synchronized" */
+	private Executor mainThreadExecutor;
 
-	/** All allocations, we can lookup allocations either by SlotID or AllocationID */
-	private final AllocationMap allocationMap;
+	/** Callbacks for resource (de-)allocations */
+	private ResourceManagerActions resourceManagerActions;
 
-	private final Time timeout;
+	/** True iff the component has been started */
+	private boolean started;
 
-	public SlotManager(ResourceManagerServices rmServices) {
-		this.rmServices = checkNotNull(rmServices);
-		this.registeredSlots = new HashMap<>(16);
-		this.pendingSlotRequests = new LinkedHashMap<>(16);
-		this.freeSlots = new HashMap<>(16);
-		this.allocationMap = new AllocationMap();
-		this.taskManagers = new HashMap<>();
-		this.timeout = Time.seconds(10);
+	public SlotManager(
+			ScheduledExecutor scheduledExecutor,
+			Time taskManagerRequestTimeout,
+			Time slotRequestTimeout,
+			Time taskManagerTimeout) {
+		this.scheduledExecutor = Preconditions.checkNotNull(scheduledExecutor);
+		this.taskManagerRequestTimeout = Preconditions.checkNotNull(taskManagerRequestTimeout);
+		this.slotRequestTimeout = Preconditions.checkNotNull(slotRequestTimeout);
+		this.taskManagerTimeout = Preconditions.checkNotNull(taskManagerTimeout);
+
+		slots = new HashMap<>(16);
+		freeSlots = new LinkedHashMap<>(16);
+		taskManagerRegistrations = new HashMap<>(4);
+		fulfilledSlotRequests = new HashMap<>(16);
+		pendingSlotRequests = new HashMap<>(16);
+
+		leaderId = null;
+		resourceManagerActions = null;
+		started = false;
 	}
 
-	// ------------------------------------------------------------------------
-	//  slot managements
-	// ------------------------------------------------------------------------
+	// ---------------------------------------------------------------------------------------------
+	// Component lifecycle methods
+	// ---------------------------------------------------------------------------------------------
 
 	/**
-	 * Request a slot with requirements, we may either fulfill the request or pending it. Trigger container
-	 * allocation if we don't have enough resource. If we have free slot which can match the request, record
-	 * this allocation and forward the request to TaskManager through ResourceManager (we want this done by
-	 * RPC's main thread to avoid race condition).
+	 * Starts the slot manager with the given leader id and resource manager actions.
 	 *
-	 * @param request The detailed request of the slot
-	 * @return RMSlotRequestRegistered The confirmation message to be send to the caller
+	 * @param newLeaderId to use for communication with the task managers
+	 * @param newResourceManagerActions to use for resource (de-)allocations
+	 */
+	public void start(UUID newLeaderId, Executor newMainThreadExecutor, ResourceManagerActions newResourceManagerActions) {
+		leaderId = Preconditions.checkNotNull(newLeaderId);
+		mainThreadExecutor = Preconditions.checkNotNull(newMainThreadExecutor);
+		resourceManagerActions = Preconditions.checkNotNull(newResourceManagerActions);
+
+		started = true;
+	}
+
+	/**
+	 * Suspends the component. This clears the internal state of the slot manager.
 	 */
-	public RMSlotRequestRegistered requestSlot(final SlotRequest request) {
-		final AllocationID allocationId = request.getAllocationId();
-		if (isRequestDuplicated(request)) {
-			LOG.warn("Duplicated slot request, AllocationID:{}", allocationId);
-			return new RMSlotRequestRegistered(allocationId);
+	public void suspend() {
+		for (PendingSlotRequest pendingSlotRequest : pendingSlotRequests.values()) {
+			cancelPendingSlotRequest(pendingSlotRequest);
 		}
 
-		// try to fulfil the request with current free slots
-		final ResourceSlot slot = chooseSlotToUse(request, freeSlots);
-		if (slot != null) {
-			LOG.info("Assigning SlotID({}) to AllocationID({}), JobID:{}", slot.getSlotId(),
-				allocationId, request.getJobId());
+		pendingSlotRequests.clear();
+
+		ArrayList<InstanceID> registeredTaskManagers = new ArrayList<>(taskManagerRegistrations.keySet());
+
+		for (InstanceID registeredTaskManager : registeredTaskManagers) {
+			unregisterTaskManager(registeredTaskManager);
+		}
 
-			// record this allocation in bookkeeping
-			allocationMap.addAllocation(slot.getSlotId(), allocationId);
-			// remove selected slot from free pool
-			freeSlots.remove(slot.getSlotId());
+		leaderId = null;
+		resourceManagerActions = null;
+		started = false;
+	}
+
+	/**
+	 * Closes the slot manager.
+	 *
+	 * @throws Exception if the close operation fails
+	 */
+	@Override
+	public void close() throws Exception {
+		suspend();
+	}
+
+	// ---------------------------------------------------------------------------------------------
+	// Public API
+	// ---------------------------------------------------------------------------------------------
+
+	/**
+	 * Requests a slot with the respective resource profile.
+	 *
+	 * @param slotRequest specifying the requested slot specs
+	 * @return true if the slot request was registered; false if the request is a duplicate
+	 * @throws SlotManagerException if the slot request failed (e.g. not enough resources left)
+	 */
+	public boolean registerSlotRequest(SlotRequest slotRequest) throws SlotManagerException {
+		checkInit();
 
-			sendSlotRequest(slot, request);
+		if (checkDuplicateRequest(slotRequest.getAllocationId())) {
+			LOG.debug("Ignoring a duplicate slot request with allocation id {}.", slotRequest.getAllocationId());
+
+			return false;
 		} else {
-			LOG.info("Cannot fulfil slot request, try to allocate a new container for it, " +
-				"AllocationID:{}, JobID:{}", allocationId, request.getJobId());
-			Preconditions.checkState(rmServices != null,
-				"Attempted to allocate resources but no ResourceManagerServices set.");
-			rmServices.allocateResource(request.getResourceProfile());
-			pendingSlotRequests.put(allocationId, request);
+			PendingSlotRequest pendingSlotRequest = new PendingSlotRequest(slotRequest);
+
+			pendingSlotRequests.put(slotRequest.getAllocationId(), pendingSlotRequest);
+
+			try {
+				internalRequestSlot(pendingSlotRequest);
+			} catch (ResourceManagerException e) {
+				// requesting the slot failed --> remove pending slot request
+				pendingSlotRequests.remove(slotRequest.getAllocationId());
+
+				throw new SlotManagerException("Could not fulfill slot request " + slotRequest.getAllocationId() + '.', e);
+			}
+
+			return true;
 		}
+	}
+
+	/**
+	 * Cancels and removes a pending slot request with the given allocation id. If there is no such
+	 * pending request, then nothing is done.
+	 *
+	 * @param allocationId identifying the pending slot request
+	 * @return True if a pending slot request was found; otherwise false
+	 */
+	public boolean unregisterSlotRequest(AllocationID allocationId) {
+		checkInit();
+
+		PendingSlotRequest pendingSlotRequest = pendingSlotRequests.remove(allocationId);
+
+		if (null != pendingSlotRequest) {
+			cancelPendingSlotRequest(pendingSlotRequest);
+
+			return true;
+		} else {
+			LOG.debug("No pending slot request with allocation id {} found.", allocationId);
 
-		return new RMSlotRequestRegistered(allocationId);
+			return false;
+		}
 	}
 
 	/**
-	 * Notifies the SlotManager that a slot is available again after being allocated.
-	 * @param slotID slot id of available slot
+	 * Registers a new task manager at the slot manager. This will make the task managers slots
+	 * known and, thus, available for allocation.
+	 *
+	 * @param taskExecutorConnection for the new task manager
+	 * @param initialSlotReport for the new task manager
 	 */
-	public void notifySlotAvailable(ResourceID resourceID, SlotID slotID) {
-		if (!allocationMap.isAllocated(slotID)) {
-			throw new IllegalStateException("Slot was not previously allocated but " +
-				"TaskManager reports it as available again");
+	public void registerTaskManager(final TaskExecutorConnection taskExecutorConnection, SlotReport initialSlotReport) {
+		checkInit();
+
+		// we identify task managers by their instance id
+		if (!taskManagerRegistrations.containsKey(taskExecutorConnection.getInstanceID())) {
+			TaskManagerRegistration taskManagerRegistration = new TaskManagerRegistration(taskExecutorConnection);
+			taskManagerRegistrations.put(taskExecutorConnection.getInstanceID(), taskManagerRegistration);
 		}
-		allocationMap.removeAllocation(slotID);
-		final Map<SlotID, ResourceSlot> slots = registeredSlots.get(resourceID);
-		ResourceSlot freeSlot = slots.get(slotID);
-		if (freeSlot == null) {
-			throw new IllegalStateException("Slot was not registered with SlotManager but " +
-				"TaskManager reported it to be available.");
+
+		reportSlotStatus(taskExecutorConnection.getInstanceID(), initialSlotReport);
+	}
+
+	/**
+	 * Unregisters the task manager identified by the given instance id and its associated slots
+	 * from the slot manager.
+	 *
+	 * @param instanceId identifying the task manager to unregister
+	 * @return True if there existed a registered task manager with the given instance id
+	 */
+	public boolean unregisterTaskManager(InstanceID instanceId) {
+		checkInit();
+
+		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.remove(instanceId);
+
+		if (null != taskManagerRegistration) {
+			removeSlots(taskManagerRegistration.getSlots());
+
+			taskManagerRegistration.cancelTimeout();
+
+			return true;
+		} else {
+			LOG.debug("There is no task manager registered with instance ID {}. Ignoring this message.", instanceId);
+
+			return false;
 		}
-		handleFreeSlot(freeSlot);
 	}
 
 	/**
-	 * The slot request to TaskManager may be either failed by rpc communication (timeout, network error, etc.)
-	 * or really rejected by TaskManager. We shall retry this request by:
-	 * <ul>
-	 * <li>1. verify and clear all the previous allocate information for this request
-	 * <li>2. try to request slot again
-	 * </ul>
-	 * <p>
-	 * This may cause some duplicate allocation, e.g. the slot request to TaskManager is successful but the response
-	 * is lost somehow, so we may request a slot in another TaskManager, this causes two slots assigned to one request,
-	 * but it can be taken care of by rejecting registration at JobManager.
+	 * Reports the current slot allocations for a task manager identified by the given instance id.
 	 *
-	 * @param originalRequest The original slot request
-	 * @param slotId          The target SlotID
+	 * @param instanceId identifying the task manager for which to report the slot status
+	 * @param slotReport containing the status for all of its slots
+	 * @return true if the slot status has been updated successfully, otherwise false
 	 */
-	void handleSlotRequestFailedAtTaskManager(final SlotRequest originalRequest, final SlotID slotId) {
-		final AllocationID originalAllocationId = originalRequest.getAllocationId();
-		LOG.info("Slot request failed at TaskManager, SlotID:{}, AllocationID:{}, JobID:{}",
-			slotId, originalAllocationId, originalRequest.getJobId());
-
-		if (allocationMap.isAllocated(slotId)) {
-			final AllocationID expectedAllocationId = allocationMap.getAllocationID(slotId);
-
-			// check whether we have an agreement on whom this slot belongs to
-			if (originalAllocationId.equals(expectedAllocationId)) {
-				LOG.info("De-allocate this request and retry");
-				allocationMap.removeAllocation(expectedAllocationId);
-				pendingSlotRequests.put(originalRequest.getAllocationId(), originalRequest);
-				ResourceSlot slot = checkNotNull(getRegisteredSlot(slotId));
-				// treat this slot as empty and retry with a different request
-				handleFreeSlot(slot);
-			} else {
-				LOG.error("Slot request failed for slot {} with allocation id {}:" +
-						" Allocation id did not match the expected allocation id {}.",
-					slotId, originalAllocationId, expectedAllocationId);
+	public boolean reportSlotStatus(InstanceID instanceId, SlotReport slotReport) {
+		checkInit();
+
+		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(instanceId);
+
+		if (null != taskManagerRegistration) {
+			ArrayList<SlotID> slotsToRemove = new ArrayList<>(taskManagerRegistration.getSlots());
+			boolean idle = true;
+
+			for (SlotStatus slotStatus : slotReport) {
+				if (slotsToRemove.remove(slotStatus.getSlotID())) {
+					// slot which was already registered
+					updateSlot(slotStatus.getSlotID(), slotStatus.getAllocationID());
+				} else {
+					// new slot
+					registerSlot(
+						taskManagerRegistration,
+						slotStatus.getSlotID(),
+						slotStatus.getAllocationID(),
+						slotStatus.getResourceProfile(),
+						taskManagerRegistration.getTaskManagerConnection());
+				}
+
+				TaskManagerSlot slot = slots.get(slotStatus.getSlotID());
+
+				idle &= slot.isFree();
+			}
+
+			// remove the slots for which we haven't received a slot status message
+			removeSlots(slotsToRemove);
+
+			if (idle) {
+				// no slot of this task manager is being used --> register timer to free this resource
+				registerTaskManagerTimeout(taskManagerRegistration);
 			}
+
+			return true;
 		} else {
-			LOG.error("Slot request failed for slot {} with allocation id {}: " +
-					"Slot was not previously registered.",
-				slotId, originalAllocationId);
+			LOG.debug("Received slot report for unknown task manager with instance id {}. Ignoring this report.", instanceId);
+
+			return false;
 		}
 	}
 
 	/**
-	 * Registers a TaskExecutor
-	 * @param resourceID TaskExecutor's ResourceID
-	 * @param registration TaskExecutor's registration
-	 * @param slotReport TaskExecutor's free and allocated slots
+	 * Free the given slot from the given allocation. If the slot is still allocated by the given
+	 * allocation id, then the slot will be marked as free and will be subject to new slot requests.
+	 *
+	 * @param slotId identifying the slot to free
+	 * @param allocationId with which the slot is presumably allocated
 	 */
-	public void registerTaskExecutor(
-			ResourceID resourceID,
-			TaskExecutorRegistration registration,
-			SlotReport slotReport) {
+	public void freeSlot(SlotID slotId, AllocationID allocationId) {
+		checkInit();
 
-		if (taskManagers.get(resourceID) != null) {
-			notifyTaskManagerFailure(resourceID);
-		}
+		TaskManagerSlot slot = slots.get(slotId);
+
+		if (null != slot) {
+			if (slot.isAllocated()) {
+				if (Objects.equals(allocationId, slot.getAllocationId())) {
+					// free the slot
+					slot.setAllocationId(null);
+					fulfilledSlotRequests.remove(allocationId);
 
-		this.taskManagers.put(resourceID, registration);
+					if (slot.isFree()) {
+						handleFreeSlot(slot);
+					}
 
-		for (SlotStatus slotStatus : slotReport.getSlotsStatus()) {
-			final SlotID slotId = slotStatus.getSlotID();
+					TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(slot.getInstanceId());
 
-			final TaskExecutorRegistration taskExecutorRegistration = taskManagers.get(slotId.getResourceID());
-			if (taskExecutorRegistration == null) {
-				LOG.info("Received SlotStatus but ResourceID {} is unknown to the SlotManager",
-					slotId.getResourceID());
-				return;
+					if (null != taskManagerRegistration && !anySlotUsed(taskManagerRegistration.getSlots())) {
+						registerTaskManagerTimeout(taskManagerRegistration);
+					}
+				} else {
+					LOG.debug("Received request to free slot {} with expected allocation id {}, " +
+						"but actual allocation id {} differs. Ignoring the request.", slotId, allocationId, slot.getAllocationId());
+				}
+			} else {
+				LOG.debug("Slot {} has not been allocated.", allocationId);
 			}
+		} else {
+			LOG.debug("Trying to free a slot {} which has not been registered. Ignoring this message.", slotId);
+		}
+	}
 
-			final ResourceSlot slot = new ResourceSlot(slotId, slotStatus.getProfiler(), taskExecutorRegistration);
+	// ---------------------------------------------------------------------------------------------
+	// Behaviour methods
+	// ---------------------------------------------------------------------------------------------
 
-			registerNewSlot(slot);
-			LOG.info("New slot appeared, SlotID:{}, AllocationID:{}", slotId, slotStatus.getAllocationID());
+	/**
+	 * Finds a matching slot request for a given resource profile. If there is no such request,
+	 * the method returns null.
+	 *
+	 * Note: If you want to change the behaviour of the slot manager wrt slot allocation and
+	 * request fulfillment, then you should override this method.
+	 *
+	 * @param slotResourceProfile defining the resources of an available slot
+	 * @return A matching slot request which can be deployed in a slot with the given resource
+	 * profile. Null if there is no such slot request pending.
+	 */
+	protected PendingSlotRequest findMatchingRequest(ResourceProfile slotResourceProfile) {
 
-			if (slotStatus.getAllocationID() != null) {
-				// slot in use, record this in bookkeeping
-				allocationMap.addAllocation(slotId, slotStatus.getAllocationID());
-			} else {
-				handleFreeSlot(slot);
+		for (PendingSlotRequest pendingSlotRequest : pendingSlotRequests.values()) {
+			if (!pendingSlotRequest.isAssigned() && slotResourceProfile.isMatching(pendingSlotRequest.getResourceProfile())) {
+				return pendingSlotRequest;
 			}
 		}
+
+		return null;
 	}
 
 	/**
-	 * Callback for TaskManager failures. In case that a TaskManager fails, we have to clean up all its slots.
+	 * Finds a matching slot for a given resource profile. A matching slot has at least as many
+	 * resources available as the given resource profile. If there is no such slot available, then
+	 * the method returns null.
 	 *
-	 * @param resourceId The ResourceID of the TaskManager
+	 * Note: If you want to change the behaviour of the slot manager wrt slot allocation and
+	 * request fulfillment, then you should override this method.
+	 *
+	 * @param requestResourceProfile specifying the resource requirements for the a slot request
+	 * @return A matching slot which fulfills the given resource profile. Null if there is no such
+	 * slot available.
 	 */
-	public void notifyTaskManagerFailure(final ResourceID resourceId) {
-		LOG.info("Resource:{} been notified failure", resourceId);
-		taskManagers.remove(resourceId);
-		final Map<SlotID, ResourceSlot> slotIdsToRemove = registeredSlots.remove(resourceId);
-		if (slotIdsToRemove != null) {
-			for (SlotID slotId : slotIdsToRemove.keySet()) {
-				LOG.info("Removing Slot: {} upon resource failure", slotId);
-				if (freeSlots.containsKey(slotId)) {
-					freeSlots.remove(slotId);
-				} else if (allocationMap.isAllocated(slotId)) {
-					allocationMap.removeAllocation(slotId);
-				} else {
-					LOG.error("BUG! {} is neither in free pool nor in allocated pool", slotId);
-				}
+	protected TaskManagerSlot findMatchingSlot(ResourceProfile requestResourceProfile) {
+		Iterator<Map.Entry<SlotID, TaskManagerSlot>> iterator = freeSlots.entrySet().iterator();
+
+		while (iterator.hasNext()) {
+			TaskManagerSlot taskManagerSlot = iterator.next().getValue();
+
+			// sanity check
+			Preconditions.checkState(taskManagerSlot.isFree());
+
+			if (taskManagerSlot.getResourceProfile().isMatching(requestResourceProfile)) {
+				iterator.remove();
+				return taskManagerSlot;
 			}
 		}
+
+		return null;
 	}
 
-	// ------------------------------------------------------------------------
-	//  internal behaviors
-	// ------------------------------------------------------------------------
+	// ---------------------------------------------------------------------------------------------
+	// Internal slot operations
+	// ---------------------------------------------------------------------------------------------
 
 	/**
-	 * When we have a free slot, try to fulfill the pending request first. If any request can be fulfilled,
-	 * record this allocation in bookkeeping and send slot request to TaskManager, else we just add this slot
-	 * to the free pool.
+	 * Registers a slot for the given task manager at the slot manager. The slot is identified by
+	 * the given slot id. The given resource profile defines the available resources for the slot.
+	 * The task manager connection can be used to communicate with the task manager.
 	 *
-	 * @param freeSlot The free slot
+	 * @param taskManagerRegistration Task manager for which to register the given slot
+	 * @param slotId identifying the slot on the task manager
+	 * @param allocationId which is currently deployed in the slot
+	 * @param resourceProfile of the slot
+	 * @param taskManagerConnection to communicate with the remote task manager
 	 */
-	private void handleFreeSlot(final ResourceSlot freeSlot) {
-		SlotRequest chosenRequest = chooseRequestToFulfill(freeSlot, pendingSlotRequests);
+	private void registerSlot(
+			TaskManagerRegistration taskManagerRegistration,
+			SlotID slotId,
+			AllocationID allocationId,
+			ResourceProfile resourceProfile,
+			TaskExecutorConnection taskManagerConnection) {
+
+		Preconditions.checkNotNull(taskManagerRegistration);
+
+		TaskManagerSlot slot = new TaskManagerSlot(
+			slotId,
+			resourceProfile,
+			taskManagerConnection,
+			allocationId);
+
+		slots.put(slotId, slot);
+
+		taskManagerRegistration.addSlot(slotId);
+
+		if (slot.isFree()) {
+			handleFreeSlot(slot);
+		}
 
-		if (chosenRequest != null) {
-			final AllocationID allocationId = chosenRequest.getAllocationId();
-			final SlotRequest slotRequest = pendingSlotRequests.remove(allocationId);
+		if (slot.isAllocated()) {
+			fulfilledSlotRequests.put(slot.getAllocationId(), slotId);
+		}
+	}
+
+	/**
+	 * Updates a slot with the given allocation id.
+	 *
+	 * @param slotId to update
+	 * @param allocationId specifying the current allocation of the slot
+	 */
+	private void updateSlot(SlotID slotId, AllocationID allocationId) {
+		TaskManagerSlot slot = slots.get(slotId);
+
+		if (null != slot) {
+			// we assume the given allocation id to be the ground truth (coming from the TM)
+			slot.setAllocationId(allocationId);
+
+			if (null != allocationId) {
+				if (slot.hasPendingSlotRequest()){
+					// we have a pending slot request --> check whether we have to reject it
+					PendingSlotRequest pendingSlotRequest = slot.getAssignedSlotRequest();
+
+					if (Objects.equals(pendingSlotRequest.getAllocationId(), allocationId)) {
+						// we can cancel the slot request because it has been fulfilled
+						cancelPendingSlotRequest(pendingSlotRequest);
+
+						// remove the pending slot request, since it has been completed
+						pendingSlotRequests.remove(pendingSlotRequest.getAllocationId());
+					} else {
+						// this will try to find a new slot for the request
+						rejectPendingSlotRequest(
+							pendingSlotRequest,
+							new Exception("Task manager reported slot " + slotId + " being already allocated."));
+					}
+
+					slot.setAssignedSlotRequest(null);
+				}
 
-			LOG.info("Assigning SlotID({}) to AllocationID({}), JobID:{}", freeSlot.getSlotId(),
-				allocationId, chosenRequest.getJobId());
-			allocationMap.addAllocation(freeSlot.getSlotId(), allocationId);
+				fulfilledSlotRequests.put(allocationId, slotId);
 
-			sendSlotRequest(freeSlot, slotRequest);
+				TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(slot.getInstanceId());
+
+				if (null != taskManagerRegistration) {
+					// disable any registered time out for the task manager
+					taskManagerRegistration.cancelTimeout();
+				}
+			}
 		} else {
-			freeSlots.put(freeSlot.getSlotId(), freeSlot);
+			LOG.debug("Trying to update unknown slot with slot id {}.", slotId);
 		}
 	}
 
-	private void sendSlotRequest(final ResourceSlot freeSlot, final SlotRequest slotRequest) {
+	/**
+	 * Tries to allocate a slot for the given slot request. If there is no slot available, the
+	 * resource manager is informed to allocate more resources and a timeout for the request is
+	 * registered.
+	 *
+	 * @param pendingSlotRequest to allocate a slot for
+	 * @throws ResourceManagerException if the resource manager cannot allocate more resource
+	 */
+	private void internalRequestSlot(PendingSlotRequest pendingSlotRequest) throws ResourceManagerException {
+		TaskManagerSlot taskManagerSlot = findMatchingSlot(pendingSlotRequest.getResourceProfile());
 
-		final AllocationID allocationID = slotRequest.getAllocationId();
-		final TaskExecutorRegistration registration = freeSlot.getTaskExecutorRegistration();
-		final Future<TMSlotRequestReply> slotRequestReplyFuture =
-			registration.getTaskExecutorGateway()
-				.requestSlot(
-					freeSlot.getSlotId(),
-					slotRequest.getJobId(),
-					allocationID,
-					"foobar", // TODO: set proper JM address
-					rmServices.getLeaderID(),
-					timeout);
+		if (taskManagerSlot != null) {
+			allocateSlot(taskManagerSlot, pendingSlotRequest);
+		} else {
+			final UUID timeoutIdentifier = UUID.randomUUID();
+			final AllocationID allocationId = pendingSlotRequest.getAllocationId();
+
+			// register timeout for slot request
+			ScheduledFuture<?> timeoutFuture = scheduledExecutor.schedule(new Runnable() {
+				@Override
+				public void run() {
+					mainThreadExecutor.execute(new Runnable() {
+						@Override
+						public void run() {
+							timeoutSlotRequest(allocationId, timeoutIdentifier);
+						}
+					});
+				}
+			}, slotRequestTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
+
+			pendingSlotRequest.registerTimeout(timeoutFuture, timeoutIdentifier);
 
-		slotRequestReplyFuture.handleAsync(new BiFunction<TMSlotRequestReply, Throwable, Void>() {
+			resourceManagerActions.allocateResource(pendingSlotRequest.getResourceProfile());
+		}
+	}
+
+	/**
+	 * Allocates the given slot for the given slot request. This entails sending a registration
+	 * message to the task manager and treating failures.
+	 *
+	 * @param taskManagerSlot to allocate for the given slot request
+	 * @param pendingSlotRequest to allocate the given slot for
+	 */
+	private void allocateSlot(TaskManagerSlot taskManagerSlot, PendingSlotRequest pendingSlotRequest) {
+		TaskExecutorConnection taskExecutorConnection = taskManagerSlot.getTaskManagerConnection();
+		TaskExecutorGateway gateway = taskExecutorConnection.getTaskExecutorGateway();
+
+		final CompletableFuture<Acknowledge> completableFuture = new FlinkCompletableFuture<>();
+		final AllocationID allocationId = pendingSlotRequest.getAllocationId();
+		final SlotID slotId = taskManagerSlot.getSlotId();
+
+		taskManagerSlot.setAssignedSlotRequest(pendingSlotRequest);
+		pendingSlotRequest.setRequestFuture(completableFuture);
+
+		// RPC call to the task manager
+		Future<Acknowledge> requestFuture = gateway.requestSlot(
+			slotId,
+			pendingSlotRequest.getJobId(),
+			allocationId,
+			pendingSlotRequest.getTargetAddress(),
+			leaderId,
+			taskManagerRequestTimeout);
+
+		requestFuture.handle(new BiFunction<Acknowledge, Throwable, Void>() {
+			@Override
+			public Void apply(Acknowledge acknowledge, Throwable throwable) {
+				if (acknowledge != null) {
+					completableFuture.complete(acknowledge);
+				} else {
+					completableFuture.completeExceptionally(throwable);
+				}
+
+				return null;
+			}
+		});
+
+		completableFuture.handleAsync(new BiFunction<Acknowledge, Throwable, Void>() {
 			@Override
-			public Void apply(TMSlotRequestReply slotRequestReply, Throwable throwable) {
-				TaskExecutorRegistration current = taskManagers.get(slotRequestReply.getResourceID());
-				if (current != null && current.getInstanceID().equals(slotRequestReply.getInstanceID())) {
-					if (throwable != null || slotRequestReply instanceof TMSlotRequestRejected) {
-						handleSlotRequestFailedAtTaskManager(slotRequest, freeSlot.getSlotId());
+			public Void apply(Acknowledge acknowledge, Throwable throwable) {
+				if (acknowledge != null) {
+					updateSlot(slotId, allocationId);
+				} else {
+					if (throwable instanceof SlotOccupiedException) {
+						SlotOccupiedException exception = (SlotOccupiedException) throwable;
+						updateSlot(slotId, exception.getAllocationId());
 					} else {
-						LOG.debug("Successfully registered slot {} ", freeSlot.getSlotId());
+						removeSlotRequestFromSlot(slotId, allocationId);
+					}
+
+					if (!(throwable instanceof CancellationException)) {
+						handleFailedSlotRequest(slotId, allocationId, throwable);
+					} else {
+						LOG.debug("Slot allocation request {} has been cancelled.", allocationId, throwable);
 					}
-				} else {
-					LOG.debug("Discarding message from obsolete TaskExecutor with InstanceID {}",
-						slotRequestReply.getInstanceID());
 				}
+
 				return null;
 			}
-		}, rmServices.getMainThreadExecutor());
+		}, mainThreadExecutor);
 	}
 
 	/**
-	 * Check whether the request is duplicated. We use AllocationID to identify slot request, for each
-	 * formerly received slot request, it is either in pending list or already been allocated.
+	 * Handles a free slot. It first tries to find a pending slot request which can be fulfilled.
+	 * If there is no such request, then it will add the slot to the set of free slots.
 	 *
-	 * @param request The slot request
-	 * @return <tt>true</tt> if the request is duplicated
+	 * @param freeSlot to find a new slot request for
 	 */
-	private boolean isRequestDuplicated(final SlotRequest request) {
-		final AllocationID allocationId = request.getAllocationId();
-		return pendingSlotRequests.containsKey(allocationId)
-			|| allocationMap.isAllocated(allocationId);
+	private void handleFreeSlot(TaskManagerSlot freeSlot) {
+		PendingSlotRequest pendingSlotRequest = findMatchingRequest(freeSlot.getResourceProfile());
+
+		if (null != pendingSlotRequest) {
+			allocateSlot(freeSlot, pendingSlotRequest);
+		} else {
+			freeSlots.put(freeSlot.getSlotId(), freeSlot);
+		}
 	}
 
 	/**
-	 * Registers a new slot with the SlotManager.
+	 * Removes the given set of slots from the slot manager.
 	 *
-	 * @param slot The ResourceSlot which will be registered
+	 * @param slotsToRemove identifying the slots to remove from the slot manager
 	 */
-	private void registerNewSlot(final ResourceSlot slot) {
-		final SlotID slotId = slot.getSlotId();
-		final ResourceID resourceId = slotId.getResourceID();
-		if (!registeredSlots.containsKey(resourceId)) {
-			registeredSlots.put(resourceId, new HashMap<SlotID, ResourceSlot>());
+	private void removeSlots(Iterable<SlotID> slotsToRemove) {
+		for (SlotID slotId : slotsToRemove) {
+			removeSlot(slotId);
 		}
-		registeredSlots.get(resourceId).put(slotId, slot);
 	}
 
-	private ResourceSlot getRegisteredSlot(final SlotID slotId) {
-		final ResourceID resourceId = slotId.getResourceID();
-		if (!registeredSlots.containsKey(resourceId)) {
-			return null;
+	/**
+	 * Removes the given slot from the slot manager.
+	 *
+	 * @param slotId identifying the slot to remove
+	 */
+	private void removeSlot(SlotID slotId) {
+		TaskManagerSlot slot = slots.remove(slotId);
+
+		if (null != slot) {
+			freeSlots.remove(slotId);
+
+			if (slot.hasPendingSlotRequest()) {
+				// reject the pending slot request --> triggering a new allocation attempt
+				rejectPendingSlotRequest(
+					slot.getAssignedSlotRequest(),
+					new Exception("The assigned slot " + slot.getSlotId() + " was removed."));
+			}
+
+			AllocationID oldAllocationId = slot.getAllocationId();
+
+			fulfilledSlotRequests.remove(oldAllocationId);
+
+			TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(slot.getInstanceId());
+
+			if (null != taskManagerRegistration) {
+				taskManagerRegistration.removeSlot(slotId);
+			}
+		} else {
+			LOG.debug("There was no slot registered with slot id {}.", slotId);
 		}
-		return registeredSlots.get(resourceId).get(slotId);
 	}
 
-	// ------------------------------------------------------------------------
-	//  Framework specific behavior
-	// ------------------------------------------------------------------------
+	// ---------------------------------------------------------------------------------------------
+	// Internal request handling methods
+	// ---------------------------------------------------------------------------------------------
 
 	/**
-	 * Choose a slot to use among all free slots, the behavior is framework specified.
+	 * Removes a pending slot request identified by the given allocation id from a slot identified
+	 * by the given slot id.
 	 *
-	 * @param request   The slot request
-	 * @param freeSlots All slots which can be used
-	 * @return The slot we choose to use, <tt>null</tt> if we did not find a match
+	 * @param slotId identifying the slot
+	 * @param allocationId identifying the presumable assigned pending slot request
 	 */
-	protected abstract ResourceSlot chooseSlotToUse(final SlotRequest request,
-		final Map<SlotID, ResourceSlot> freeSlots);
+	private void removeSlotRequestFromSlot(SlotID slotId, AllocationID allocationId) {
+		TaskManagerSlot taskManagerSlot = slots.get(slotId);
+
+		if (null != taskManagerSlot) {
+			if (taskManagerSlot.hasPendingSlotRequest() && Objects.equals(allocationId, taskManagerSlot.getAssignedSlotRequest().getAllocationId())) {
+				taskManagerSlot.setAssignedSlotRequest(null);
+			}
+
+			if (taskManagerSlot.isFree()) {
+				handleFreeSlot(taskManagerSlot);
+			}
+
+			TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(taskManagerSlot.getInstanceId());
+
+			if (null != taskManagerRegistration && !anySlotUsed(taskManagerRegistration.getSlots())) {
+				registerTaskManagerTimeout(taskManagerRegistration);
+			}
+		} else {
+			LOG.debug("There was no slot with {} registered. Probably this slot has been already freed.", slotId);
+		}
+	}
 
 	/**
-	 * Choose a pending request to fulfill when we have a free slot, the behavior is framework specified.
+	 * Handles a failed slot request. The slot manager tries to find a new slot fulfilling
+	 * the resource requirements for the failed slot request.
 	 *
-	 * @param offeredSlot     The free slot
-	 * @param pendingRequests All the pending slot requests
-	 * @return The chosen SlotRequest, <tt>null</tt> if we did not find a match
+	 * @param slotId identifying the slot which was assigned to the slot request before
+	 * @param allocationId identifying the failed slot request
+	 * @param cause of the failure
 	 */
-	protected abstract SlotRequest chooseRequestToFulfill(final ResourceSlot offeredSlot,
-		final Map<AllocationID, SlotRequest> pendingRequests);
+	private void handleFailedSlotRequest(SlotID slotId, AllocationID allocationId, Throwable cause) {
+		PendingSlotRequest pendingSlotRequest = pendingSlotRequests.get(allocationId);
+
+		LOG.debug("Slot request with allocation id {} failed for slot {}.", allocationId, slotId, cause);
+
+		if (null != pendingSlotRequest) {
+			pendingSlotRequest.setRequestFuture(null);
+
+			try {
+				internalRequestSlot(pendingSlotRequest);
+			} catch (ResourceManagerException e) {
+				pendingSlotRequests.remove(allocationId);
 
-	// ------------------------------------------------------------------------
-	//  Helper classes
-	// ------------------------------------------------------------------------
+				resourceManagerActions.notifyAllocationFailure(
+					pendingSlotRequest.getJobId(),
+					allocationId,
+					e);
+			}
+		} else {
+			LOG.debug("There was not pending slot request with allocation id {}. Probably the request has been fulfilled or cancelled.", allocationId);
+		}
+	}
 
 	/**
-	 * We maintain all the allocations with SlotID and AllocationID. We are able to get or remove the allocation info
-	 * either by SlotID or AllocationID.
+	 * Rejects the pending slot request by failing the request future with a
+	 * {@link SlotAllocationException}.
+	 *
+	 * @param pendingSlotRequest to reject
+	 * @param cause of the rejection
 	 */
-	private static class AllocationMap {
+	private void rejectPendingSlotRequest(PendingSlotRequest pendingSlotRequest, Exception cause) {
+		CompletableFuture<Acknowledge> request = pendingSlotRequest.getRequestFuture();
 
-		/** All allocated slots (by SlotID) */
-		private final Map<SlotID, AllocationID> allocatedSlots;
+		if (null != request) {
+			request.completeExceptionally(new SlotAllocationException(cause));
+		} else {
+			LOG.debug("Cannot reject pending slot request {}, since no request has been sent.", pendingSlotRequest.getAllocationId());
+		}
+	}
 
-		/** All allocated slots (by AllocationID), it'a a inverse view of allocatedSlots */
-		private final Map<AllocationID, SlotID> allocatedSlotsByAllocationId;
+	/**
+	 * Cancels the given slot request.
+	 *
+	 * @param pendingSlotRequest to cancel
+	 */
+	private void cancelPendingSlotRequest(PendingSlotRequest pendingSlotRequest) {
+		pendingSlotRequest.cancelTimeout();
 
-		AllocationMap() {
-			this.allocatedSlots = new HashMap<>(16);
-			this.allocatedSlotsByAllocationId = new HashMap<>(16);
-		}
+		CompletableFuture<Acknowledge> request = pendingSlotRequest.getRequestFuture();
 
-		/**
-		 * Add a allocation
-		 *
-		 * @param slotId       The slot id
-		 * @param allocationId The allocation id
-		 */
-		void addAllocation(final SlotID slotId, final AllocationID allocationId) {
-			allocatedSlots.put(slotId, allocationId);
-			allocatedSlotsByAllocationId.put(allocationId, slotId);
+		if (null != request) {
+			request.cancel(false);
 		}
+	}
+
+	// ---------------------------------------------------------------------------------------------
+	// Internal timeout methods
+	// ---------------------------------------------------------------------------------------------
+
+	private void timeoutTaskManager(InstanceID instanceId, UUID timeoutIdentifier) {
+		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.remove(instanceId);
 
-		/**
-		 * De-allocation with slot id
-		 *
-		 * @param slotId The slot id
-		 */
-		void removeAllocation(final SlotID slotId) {
-			if (allocatedSlots.containsKey(slotId)) {
-				final AllocationID allocationId = allocatedSlots.get(slotId);
-				allocatedSlots.remove(slotId);
-				allocatedSlotsByAllocationId.remove(allocationId);
+		if (null != taskManagerRegistration) {
+			if (Objects.equals(timeoutIdentifier, taskManagerRegistration.getTimeoutIdentifier())) {
+				if (anySlotUsed(taskManagerRegistration.getSlots())) {
+					LOG.debug("Cannot release the task manager with instance id {}, because some " +
+						"of its slots are still being used.", instanceId);
+				} else {
+					unregisterTaskManager(instanceId);
+
+					resourceManagerActions.releaseResource(instanceId);
+				}
+			} else {
+				taskManagerRegistrations.put(instanceId, taskManagerRegistration);
+
+				LOG.debug("Expected timeout identifier {} differs from the task manager's " +
+					"timeout identifier {}. Ignoring the task manager timeout call.",
+					timeoutIdentifier, taskManagerRegistration.getTimeoutIdentifier());
 			}
+		} else {
+			LOG.debug("Could not find a registered task manager with instance id {}. Ignoring the task manager timeout call.", instanceId);
 		}
+	}
 
-		/**
-		 * De-allocation with allocation id
-		 *
-		 * @param allocationId The allocation id
-		 */
-		void removeAllocation(final AllocationID allocationId) {
-			if (allocatedSlotsByAllocationId.containsKey(allocationId)) {
-				SlotID slotId = allocatedSlotsByAllocationId.get(allocationId);
-				allocatedSlotsByAllocationId.remove(allocationId);
-				allocatedSlots.remove(slotId);
+	private void timeoutSlotRequest(AllocationID allocationId, UUID timeoutIdentifier) {
+		PendingSlotRequest pendingSlotRequest = pendingSlotRequests.remove(allocationId);
+
+		if (null != pendingSlotRequest) {
+			if (Objects.equals(timeoutIdentifier, pendingSlotRequest.getTimeoutIdentifier())) {
+				if (!pendingSlotRequest.isAssigned()) {
+
+					resourceManagerActions.notifyAllocationFailure(
+						pendingSlotRequest.getJobId(),
+						allocationId,
+						new TimeoutException("The allocation could not be fulfilled in time."));
+				} else {
+					LOG.debug("Cannot fail pending slot request {} because it has been assigned.", allocationId);
+				}
+			} else {
+				pendingSlotRequests.put(allocationId, pendingSlotRequest);
+
+				LOG.debug("Expected timeout identifier {} differs from the pending slot request's " +
+					"timeout identifier {}. Ignoring the slot request timeout call.",
+					timeoutIdentifier, pendingSlotRequest.getTimeoutIdentifier());
 			}
+		} else {
+			LOG.debug("Could not find pending slot request with allocation id {}. Ignoring the slot request timeout call.", allocationId);
 		}
+	}
 
-		/**
-		 * Check whether allocation exists by slot id
-		 *
-		 * @param slotId The slot id
-		 * @return true if the allocation exists
-		 */
-		boolean isAllocated(final SlotID slotId) {
-			return allocatedSlots.containsKey(slotId);
-		}
+	// ---------------------------------------------------------------------------------------------
+	// Internal utility methods
+	// ---------------------------------------------------------------------------------------------
 
-		/**
-		 * Check whether allocation exists by allocation id
-		 *
-		 * @param allocationId The allocation id
-		 * @return true if the allocation exists
-		 */
-		boolean isAllocated(final AllocationID allocationId) {
-			return allocatedSlotsByAllocationId.containsKey(allocationId);
-		}
+	private boolean checkDuplicateRequest(AllocationID allocationId) {
+		return pendingSlotRequests.containsKey(allocationId) || fulfilledSlotRequests.containsKey(allocationId);
+	}
 
-		AllocationID getAllocationID(final SlotID slotId) {
-			return allocatedSlots.get(slotId);
-		}
+	private boolean anySlotUsed(Iterable<SlotID> slotsToCheck) {
 
-		SlotID getSlotID(final AllocationID allocationId) {
-			return allocatedSlotsByAllocationId.get(allocationId);
-		}
+		if (null != slotsToCheck) {
+			boolean idle = true;
 
-		public int size() {
-			return allocatedSlots.size();
-		}
+			for (SlotID slotId : slotsToCheck) {
+				TaskManagerSlot taskManagerSlot = slots.get(slotId);
 
-		public void clear() {
-			allocatedSlots.clear();
-			allocatedSlotsByAllocationId.clear();
+				if (null != taskManagerSlot) {
+					idle &= taskManagerSlot.isFree();
+				}
+			}
+
+			return !idle;
+		} else {
+			return false;
 		}
 	}
 
-	/**
-	 * Clears the state of the SlotManager after leadership revokal
-	 */
-	public void clearState() {
-		taskManagers.clear();
-		registeredSlots.clear();
-		pendingSlotRequests.clear();
-		freeSlots.clear();
-		allocationMap.clear();
-	}
+	private void registerTaskManagerTimeout(final TaskManagerRegistration taskManagerRegistration) {
+		final UUID timeoutIdentifier = UUID.randomUUID();
 
-	// ------------------------------------------------------------------------
-	//  Testing utilities
-	// ------------------------------------------------------------------------
+		ScheduledFuture<?> timeoutFuture = scheduledExecutor.schedule(new Runnable() {
+			@Override
+			public void run() {
+				mainThreadExecutor.execute(new Runnable() {
+					@Override
+					public void run() {
+						timeoutTaskManager(taskManagerRegistration.getInstanceId(), timeoutIdentifier);
+					}
+				});
+			}
+		}, taskManagerTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
 
-	@VisibleForTesting
-	boolean isAllocated(final SlotID slotId) {
-		return allocationMap.isAllocated(slotId);
+		taskManagerRegistration.registerTimeout(timeoutFuture, timeoutIdentifier);
 	}
 
-	@VisibleForTesting
-	boolean isAllocated(final AllocationID allocationId) {
-		return allocationMap.isAllocated(allocationId);
+	private void checkInit() {
+		Preconditions.checkState(started, "The slot manager has not been started.");
 	}
 
-	/**
-	 * Add free slots directly to the free pool, this will not trigger pending requests allocation
-	 *
-	 * @param slot The resource slot
-	 */
-	@VisibleForTesting
-	void addFreeSlot(final ResourceSlot slot) {
-		final ResourceID resourceId = slot.getResourceID();
-		final SlotID slotId = slot.getSlotId();
+	// ---------------------------------------------------------------------------------------------
+	// Testing methods
+	// ---------------------------------------------------------------------------------------------
 
-		if (!registeredSlots.containsKey(resourceId)) {
-			registeredSlots.put(resourceId, new HashMap<SlotID, ResourceSlot>());
-		}
-		registeredSlots.get(resourceId).put(slot.getSlotId(), slot);
-		freeSlots.put(slotId, slot);
+	@VisibleForTesting
+	TaskManagerSlot getSlot(SlotID slotId) {
+		return slots.get(slotId);
 	}
 
 	@VisibleForTesting
-	int getAllocatedSlotCount() {
-		return allocationMap.size();
+	int getNumberRegisteredSlots() {
+		return slots.size();
 	}
 
 	@VisibleForTesting
-	int getFreeSlotCount() {
-		return freeSlots.size();
+	PendingSlotRequest getSlotRequest(AllocationID allocationId) {
+		return pendingSlotRequests.get(allocationId);
 	}
 
 	@VisibleForTesting
-	int getPendingRequestCount() {
-		return pendingSlotRequests.size();
+	boolean hasTimeoutRegistered(InstanceID instanceId) {
+		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(instanceId);
+
+		if (null != taskManagerRegistration) {
+			return taskManagerRegistration.getTimeoutIdentifier() != null;
+		} else {
+			return false;
+		}
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java
new file mode 100644
index 0000000000000..d21c25167abb3
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.slotmanager;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.resourcemanager.exceptions.ConfigurationException;
+import org.apache.flink.util.Preconditions;
+import scala.concurrent.duration.Duration;
+
+public class SlotManagerConfiguration {
+
+	private final Time taskManagerRequestTimeout;
+	private final Time slotRequestTimeout;
+	private final Time taskManagerTimeout;
+
+	public SlotManagerConfiguration(
+			Time taskManagerRequestTimeout,
+			Time slotRequestTimeout,
+			Time taskManagerTimeout) {
+		this.taskManagerRequestTimeout = Preconditions.checkNotNull(taskManagerRequestTimeout);
+		this.slotRequestTimeout = Preconditions.checkNotNull(slotRequestTimeout);
+		this.taskManagerTimeout = Preconditions.checkNotNull(taskManagerTimeout);
+	}
+
+	public Time getTaskManagerRequestTimeout() {
+		return taskManagerRequestTimeout;
+	}
+
+	public Time getSlotRequestTimeout() {
+		return slotRequestTimeout;
+	}
+
+	public Time getTaskManagerTimeout() {
+		return taskManagerTimeout;
+	}
+
+	public static SlotManagerConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {
+		ConfigOption<String> timeoutOption = ConfigOptions
+			.key(ConfigConstants.AKKA_ASK_TIMEOUT)
+			.defaultValue(ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT);
+
+		final String strTimeout = configuration.getString(timeoutOption);
+		final Time timeout;
+
+		try {
+			timeout = Time.milliseconds(Duration.apply(strTimeout).toMillis());
+		} catch (NumberFormatException e) {
+			throw new ConfigurationException("Could not parse the resource manager's timeout " +
+				"value " + timeoutOption + '.', e);
+		}
+
+		return new SlotManagerConfiguration(timeout, timeout, timeout);
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerException.java
similarity index 69%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerFactory.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerException.java
index b4e9c99250670..c322c817f1dce 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerException.java
@@ -15,17 +15,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.flink.runtime.resourcemanager.slotmanager;
 
-import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
 
-/**
- * Factory to create a SlotManager and provide it with dependencies.
- */
-public interface SlotManagerFactory {
+public class SlotManagerException extends ResourceManagerException {
+
+	private static final long serialVersionUID = -3723028616920379071L;
+
+	public SlotManagerException(String message) {
+		super(message);
+	}
 
-	/**
-	 * Creates a SlotManager and provides it with ResourceManager services.
-	 */
-	SlotManager create(ResourceManagerServices rmServices);
+	public SlotManagerException(String message, Throwable cause) {
+		super(message, cause);
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
new file mode 100644
index 0000000000000..8e23dbb44615f
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.slotmanager;
+
+import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection;
+import org.apache.flink.util.Preconditions;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ScheduledFuture;
+
+public class TaskManagerRegistration {
+
+	private final TaskExecutorConnection taskManagerConnection;
+
+	private final HashSet<SlotID> slots;
+
+	private UUID timeoutIdentifier;
+
+	private ScheduledFuture<?> timeoutFuture;
+
+	public TaskManagerRegistration(TaskExecutorConnection taskManagerConnection) {
+		this.taskManagerConnection = Preconditions.checkNotNull(taskManagerConnection);
+
+		slots = new HashSet<>(4);
+
+		timeoutIdentifier = null;
+		timeoutFuture = null;
+	}
+
+	public TaskExecutorConnection getTaskManagerConnection() {
+		return taskManagerConnection;
+	}
+
+	public InstanceID getInstanceId() {
+		return taskManagerConnection.getInstanceID();
+	}
+
+	public UUID getTimeoutIdentifier() {
+		return timeoutIdentifier;
+	}
+
+	public Set<SlotID> getSlots() {
+		return slots;
+	}
+
+	public boolean removeSlot(SlotID slotId) {
+		return slots.remove(slotId);
+	}
+
+	public void addSlot(SlotID slotId) {
+		slots.add(slotId);
+	}
+
+	public void cancelTimeout() {
+		if (null != timeoutFuture) {
+			timeoutFuture.cancel(false);
+
+			timeoutFuture = null;
+			timeoutIdentifier = null;
+		}
+	}
+
+	public void registerTimeout(ScheduledFuture<?> newTimeoutFuture, UUID newTimeoutIdentifier) {
+		cancelTimeout();
+
+		timeoutFuture = newTimeoutFuture;
+		timeoutIdentifier = newTimeoutIdentifier;
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
index 54adce6016690..7a9da284570bc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
@@ -19,8 +19,9 @@
 package org.apache.flink.runtime.taskexecutor;
 
 import java.io.Serializable;
+import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
+import java.util.Iterator;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
@@ -29,12 +30,12 @@
  * which slots are available and allocated, and what jobs (JobManagers) the allocated slots
  * have been allocated to.
  */
-public class SlotReport implements Serializable {
+public class SlotReport implements Serializable, Iterable<SlotStatus> {
 
 	private static final long serialVersionUID = -3150175198722481689L;
 
 	/** The slots status of the TaskManager */
-	private final List<SlotStatus> slotsStatus;
+	private final Collection<SlotStatus> slotsStatus;
 
 	public SlotReport() {
 		this(Collections.<SlotStatus>emptyList());
@@ -44,12 +45,16 @@ public SlotReport(SlotStatus slotStatus) {
 		this(Collections.singletonList(slotStatus));
 	}
 
-	public SlotReport(final List<SlotStatus> slotsStatus) {
+	public SlotReport(final Collection<SlotStatus> slotsStatus) {
 		this.slotsStatus = checkNotNull(slotsStatus);
 	}
 
-	public List<SlotStatus> getSlotsStatus() {
+	public Collection<SlotStatus> getSlotsStatus() {
 		return slotsStatus;
 	}
 
+	@Override
+	public Iterator<SlotStatus> iterator() {
+		return slotsStatus.iterator();
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java
index 0f57bb15cdd8c..a3bc4a6af4a9e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotStatus.java
@@ -38,7 +38,7 @@ public class SlotStatus implements Serializable {
 	private final SlotID slotID;
 
 	/** the resource profile of the slot */
-	private final ResourceProfile profiler;
+	private final ResourceProfile resourceProfile;
 
 	/** if the slot is allocated, allocationId identify its allocation; else, allocationId is null */
 	private final AllocationID allocationID;
@@ -46,16 +46,17 @@ public class SlotStatus implements Serializable {
 	/** if the slot is allocated, jobId identify which job this slot is allocated to; else, jobId is null */
 	private final JobID jobID;
 
-	public SlotStatus(SlotID slotID, ResourceProfile profiler) {
-		this(slotID, profiler, null, null);
+	public SlotStatus(SlotID slotID, ResourceProfile resourceProfile) {
+		this(slotID, resourceProfile, null, null);
 	}
 
 	public SlotStatus(
-			SlotID slotID, ResourceProfile profiler,
+			SlotID slotID,
+			ResourceProfile resourceProfile,
 			JobID jobID,
 			AllocationID allocationID) {
 		this.slotID = checkNotNull(slotID, "slotID cannot be null");
-		this.profiler = checkNotNull(profiler, "profile cannot be null");
+		this.resourceProfile = checkNotNull(resourceProfile, "profile cannot be null");
 		this.allocationID = allocationID;
 		this.jobID = jobID;
 	}
@@ -74,8 +75,8 @@ public SlotID getSlotID() {
 	 *
 	 * @return The resource profile
 	 */
-	public ResourceProfile getProfiler() {
-		return profiler;
+	public ResourceProfile getResourceProfile() {
+		return resourceProfile;
 	}
 
 	/**
@@ -110,7 +111,7 @@ public boolean equals(Object o) {
 		if (!slotID.equals(that.slotID)) {
 			return false;
 		}
-		if (!profiler.equals(that.profiler)) {
+		if (!resourceProfile.equals(that.resourceProfile)) {
 			return false;
 		}
 		if (allocationID != null ? !allocationID.equals(that.allocationID) : that.allocationID != null) {
@@ -123,7 +124,7 @@ public boolean equals(Object o) {
 	@Override
 	public int hashCode() {
 		int result = slotID.hashCode();
-		result = 31 * result + profiler.hashCode();
+		result = 31 * result + resourceProfile.hashCode();
 		result = 31 * result + (allocationID != null ? allocationID.hashCode() : 0);
 		result = 31 * result + (jobID != null ? jobID.hashCode() : 0);
 		return result;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index d04cabbac93ae..5b8c8ee2e2e25 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -61,8 +61,6 @@
 import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
 import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
 import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcEndpoint;
 import org.apache.flink.runtime.rpc.RpcMethod;
@@ -71,6 +69,7 @@
 import org.apache.flink.runtime.taskexecutor.exceptions.CheckpointException;
 import org.apache.flink.runtime.taskexecutor.exceptions.PartitionException;
 import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotOccupiedException;
 import org.apache.flink.runtime.taskexecutor.exceptions.TaskException;
 import org.apache.flink.runtime.taskexecutor.exceptions.TaskSubmissionException;
 import org.apache.flink.runtime.taskexecutor.rpc.RpcCheckpointResponder;
@@ -99,6 +98,7 @@
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
 import java.util.concurrent.TimeoutException;
 
@@ -315,7 +315,7 @@ public Acknowledge submitTask(TaskDeploymentDescriptor tdd, UUID jobManagerLeade
 			throw new TaskSubmissionException(message);
 		}
 
-		if (!jobManagerConnection.getLeaderId().equals(jobManagerLeaderId)) {
+		if (!Objects.equals(jobManagerConnection.getLeaderId(), jobManagerLeaderId)) {
 			final String message = "Rejecting the task submission because the job manager leader id " +
 				jobManagerLeaderId + " does not match the expected job manager leader id " +
 				jobManagerConnection.getLeaderId() + '.';
@@ -355,30 +355,30 @@ public Acknowledge submitTask(TaskDeploymentDescriptor tdd, UUID jobManagerLeade
 		PartitionProducerStateChecker partitionStateChecker = jobManagerConnection.getPartitionStateChecker();
 
 		Task task = new Task(
-				jobInformation,
-				taskInformation,
-				tdd.getExecutionAttemptId(),
-				tdd.getAllocationId(),
-				tdd.getSubtaskIndex(),
-				tdd.getAttemptNumber(),
-				tdd.getProducedPartitions(),
-				tdd.getInputGates(),
-				tdd.getTargetSlotNumber(),
-				tdd.getTaskStateHandles(),
-				memoryManager,
-				ioManager,
-				networkEnvironment,
-				broadcastVariableManager,
-				taskManagerActions,
-				inputSplitProvider,
-				checkpointResponder,
-				libraryCache,
-				fileCache,
-				taskManagerConfiguration,
-				taskMetricGroup,
-				resultPartitionConsumableNotifier,
-				partitionStateChecker,
-				getRpcService().getExecutor());
+			jobInformation,
+			taskInformation,
+			tdd.getExecutionAttemptId(),
+			tdd.getAllocationId(),
+			tdd.getSubtaskIndex(),
+			tdd.getAttemptNumber(),
+			tdd.getProducedPartitions(),
+			tdd.getInputGates(),
+			tdd.getTargetSlotNumber(),
+			tdd.getTaskStateHandles(),
+			memoryManager,
+			ioManager,
+			networkEnvironment,
+			broadcastVariableManager,
+			taskManagerActions,
+			inputSplitProvider,
+			checkpointResponder,
+			libraryCache,
+			fileCache,
+			taskManagerConfiguration,
+			taskMetricGroup,
+			resultPartitionConsumableNotifier,
+			partitionStateChecker,
+			getRpcService().getExecutor());
 
 		log.info("Received task {}.", task.getTaskInfo().getTaskNameWithSubtasks());
 
@@ -561,7 +561,6 @@ public Acknowledge confirmCheckpoint(ExecutionAttemptID executionAttemptID, long
 	// ----------------------------------------------------------------------
 
 	/**
-	 * /**
 	 * Requests a slot from the TaskManager
 	 *
 	 * @param slotId identifying the requested slot
@@ -573,12 +572,14 @@ public Acknowledge confirmCheckpoint(ExecutionAttemptID executionAttemptID, long
 	 * @return answer to the slot request
 	 */
 	@RpcMethod
-	public TMSlotRequestReply requestSlot(
+	public Acknowledge requestSlot(
 		final SlotID slotId,
 		final JobID jobId,
 		final AllocationID allocationId,
 		final String targetAddress,
 		final UUID rmLeaderId) throws SlotAllocationException {
+		// TODO: Filter invalid requests from the resource manager by using the instance/registration Id
+
 		log.info("Receive slot request {} for job {} from resource manager with leader id {}.",
 			allocationId, jobId, rmLeaderId);
 
@@ -608,7 +609,8 @@ public TMSlotRequestReply requestSlot(
 			final String message = "The slot " + slotId + " has already been allocated for a different job.";
 
 			log.info(message);
-			throw new SlotAllocationException(message);
+
+			throw new SlotOccupiedException(message, taskSlotTable.getCurrentAllocation(slotId.getSlotNumber()));
 		}
 
 		if (jobManagerTable.contains(jobId)) {
@@ -635,7 +637,7 @@ public TMSlotRequestReply requestSlot(
 			}
 		}
 
-		return new TMSlotRequestRegistered(resourceManagerConnection.getRegistrationId(), getResourceID(), allocationId);
+		return Acknowledge.get();
 	}
 
 	// ----------------------------------------------------------------------
@@ -1046,7 +1048,8 @@ private void freeSlot(AllocationID allocationId, Throwable cause) {
 				resourceManagerGateway.notifySlotAvailable(
 					resourceManagerConnection.getTargetLeaderId(),
 					resourceManagerConnection.getRegistrationId(),
-					new SlotID(getResourceID(), freedSlotIndex));
+					new SlotID(getResourceID(), freedSlotIndex),
+					allocationId);
 			}
 		} catch (SlotNotFoundException e) {
 			log.debug("Could not free slot for allocation id {}.", allocationId, e);
@@ -1079,7 +1082,7 @@ private boolean isConnectedToResourceManager() {
 	private boolean isJobManagerConnectionValid(JobID jobId, UUID leaderId) {
 		JobManagerConnection jmConnection = jobManagerTable.get(jobId);
 
-		return jmConnection != null && jmConnection.getLeaderId().equals(leaderId);
+		return jmConnection != null && Objects.equals(jmConnection.getLeaderId(), leaderId);
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
index 2bbf0e62efe45..bedf8ecc5a114 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
@@ -25,7 +25,6 @@
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
 import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.apache.flink.runtime.executiongraph.PartitionInfo;
@@ -51,7 +50,7 @@ public interface TaskExecutorGateway extends RpcGateway {
 	 * @throws SlotAllocationException if the slot allocation fails
 	 * @return answer to the slot request
 	 */
-	Future<TMSlotRequestReply> requestSlot(
+	Future<Acknowledge> requestSlot(
 		SlotID slotId,
 		JobID jobId,
 		AllocationID allocationId,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestReply.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/SlotOccupiedException.java
similarity index 51%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestReply.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/SlotOccupiedException.java
index 66e1911592c77..93e67a851d665 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestReply.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/SlotOccupiedException.java
@@ -16,26 +16,32 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.resourcemanager.messages.jobmanager;
+package org.apache.flink.runtime.taskexecutor.exceptions;
 
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.util.Preconditions;
 
-import java.io.Serializable;
+public class SlotOccupiedException extends SlotAllocationException {
+	private static final long serialVersionUID = -3986333914244338888L;
 
-/**
- * Acknowledgment by the ResourceManager for a SlotRequest from the JobManager
- */
-public abstract class RMSlotRequestReply implements Serializable {
+	private final AllocationID allocationId;
 
-	private static final long serialVersionUID = 42;
+	public SlotOccupiedException(String message, AllocationID allocationId) {
+		super(message);
+		this.allocationId = Preconditions.checkNotNull(allocationId);
+	}
 
-	private final AllocationID allocationID;
+	public SlotOccupiedException(String message, Throwable cause, AllocationID allocationId) {
+		super(message, cause);
+		this.allocationId = Preconditions.checkNotNull(allocationId);
+	}
 
-	public RMSlotRequestReply(AllocationID allocationID) {
-		this.allocationID = allocationID;
+	public SlotOccupiedException(Throwable cause, AllocationID allocationId) {
+		super(cause);
+		this.allocationId = Preconditions.checkNotNull(allocationId);
 	}
 
-	public AllocationID getAllocationID() {
-		return allocationID;
+	public AllocationID getAllocationId() {
+		return allocationId;
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
index 081d8f8a477b2..5c51c7c359b01 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
@@ -509,6 +509,16 @@ public Iterator<Task> getTasks(JobID jobId) {
 		return new TaskIterator(jobId);
 	}
 
+	/**
+	 * Get the current allocation for the task slot with the given index.
+	 *
+	 * @param index identifying the slot for which the allocation id shall be retrieved
+	 * @return Allocation id of the specified slot if allocated; otherwise null
+	 */
+	public AllocationID getCurrentAllocation(int index) {
+		return taskSlots.get(index).getAllocationId();
+	}
+
 	// ---------------------------------------------------------------------
 	// TimeoutListener methods
 	// ---------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
index e4e20b90d78e6..41c2e1661cdf8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
@@ -49,7 +49,7 @@
 import org.apache.flink.runtime.resourcemanager.JobLeaderIdService;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
 import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
-import org.apache.flink.runtime.resourcemanager.TestingSlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
 import org.apache.flink.runtime.taskexecutor.SlotReport;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
@@ -391,10 +391,14 @@ public void testHeartbeatTimeoutWithTaskExecutor() throws Exception {
 		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
 		final HeartbeatServices heartbeatServices = new TestingHeartbeatServices(heartbeatInterval, heartbeatTimeout, scheduledExecutor);
 
-		final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
 		final MetricRegistry metricRegistry = mock(MetricRegistry.class);
 		final JobLeaderIdService jobLeaderIdService = mock(JobLeaderIdService.class);
 		final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+		final SlotManager slotManager = new SlotManager(
+			rpcService.getScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime());
 
 		try {
 			final StandaloneResourceManager resourceManager = new StandaloneResourceManager(
@@ -404,7 +408,7 @@ public void testHeartbeatTimeoutWithTaskExecutor() throws Exception {
 				resourceManagerConfiguration,
 				highAvailabilityServices,
 				heartbeatServices,
-				slotManagerFactory,
+				slotManager,
 				metricRegistry,
 				jobLeaderIdService,
 				testingFatalErrorHandler);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolRpcTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolRpcTest.java
index 89fd22f4bbc5b..b4446400ed71d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolRpcTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolRpcTest.java
@@ -80,7 +80,7 @@ public void testSlotAllocationNoResourceManager() throws Exception {
 				Time.days(1), Time.days(1),
 				Time.milliseconds(100) // this is the timeout for the request tested here
 		);
-		pool.start(UUID.randomUUID());
+		pool.start(UUID.randomUUID(), "foobar");
 
 		Future<SimpleSlot> future = pool.allocateSlot(mock(ScheduledUnit.class), DEFAULT_TESTING_PROFILE, null);
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java
index b4149b292a492..cf95461a12bd9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java
@@ -78,7 +78,9 @@ public void setUp() throws Exception {
 
 		mainThreadValidatorUtil.enterMainThread();
 
-		slotPool.start(UUID.randomUUID());
+		final String jobManagerAddress = "foobar";
+
+		slotPool.start(UUID.randomUUID(), jobManagerAddress);
 
 		this.resourceManagerGateway = mock(ResourceManagerGateway.class);
 		when(resourceManagerGateway
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
index 49bc5703825e2..986f8480aadda 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
@@ -25,9 +25,10 @@
 import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
 import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerConfiguration;
 import org.apache.flink.runtime.rpc.RpcService;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.util.TestingFatalErrorHandler;
 import org.apache.flink.util.TestLogger;
 import org.junit.Assert;
@@ -57,13 +58,17 @@ public void testGrantAndRevokeLeadership() throws Exception {
 			Time.seconds(5L),
 			Time.seconds(5L));
 
-		ResourceManagerRuntimeServicesConfiguration resourceManagerRuntimeServicesConfiguration = new ResourceManagerRuntimeServicesConfiguration(Time.seconds(5L));
+		ResourceManagerRuntimeServicesConfiguration resourceManagerRuntimeServicesConfiguration = new ResourceManagerRuntimeServicesConfiguration(
+			Time.seconds(5L),
+			new SlotManagerConfiguration(
+				TestingUtils.infiniteTime(),
+				TestingUtils.infiniteTime(),
+				TestingUtils.infiniteTime()));
 		ResourceManagerRuntimeServices resourceManagerRuntimeServices = ResourceManagerRuntimeServices.fromConfiguration(
 			resourceManagerRuntimeServicesConfiguration,
 			highAvailabilityServices,
 			rpcService.getScheduledExecutor());
 
-		SlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
 		MetricRegistry metricRegistry = mock(MetricRegistry.class);
 
 		TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
@@ -76,7 +81,7 @@ public void testGrantAndRevokeLeadership() throws Exception {
 				resourceManagerConfiguration,
 				highAvailabilityServices,
 				heartbeatServices,
-				slotManagerFactory,
+				resourceManagerRuntimeServices.getSlotManager(),
 				metricRegistry,
 				resourceManagerRuntimeServices.getJobLeaderIdService(),
 				testingFatalErrorHandler);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
index 6a151ac8f6df6..4836f74ac128d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
@@ -31,10 +31,11 @@
 import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
 import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
 import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
 import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.util.TestingFatalErrorHandler;
 import org.apache.flink.util.TestLogger;
 import org.junit.After;
@@ -240,12 +241,17 @@ private ResourceManager createAndStartResourceManager(
 		ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(
 			Time.seconds(5L),
 			Time.seconds(5L));
-		SlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
 		MetricRegistry metricRegistry = mock(MetricRegistry.class);
 		JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
 			highAvailabilityServices,
 			rpcService.getScheduledExecutor(),
 			Time.minutes(5L));
+			
+		final SlotManager slotManager = new SlotManager(
+			rpcService.getScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime());
 
 		ResourceManager resourceManager = new StandaloneResourceManager(
 			rpcService,
@@ -254,7 +260,7 @@ private ResourceManager createAndStartResourceManager(
 			resourceManagerConfiguration,
 			highAvailabilityServices,
 			heartbeatServices,
-			slotManagerFactory,
+			slotManager,
 			metricRegistry,
 			jobLeaderIdService,
 			fatalErrorHandler);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
index bae7086cd6dc8..85b7eb47e18c7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
@@ -26,12 +26,14 @@
 import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
 import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
 import org.apache.flink.runtime.registration.RegistrationResponse;
 import org.apache.flink.runtime.taskexecutor.SlotReport;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.util.TestingFatalErrorHandler;
 import org.apache.flink.util.TestLogger;
 import org.junit.After;
@@ -152,10 +154,15 @@ private StandaloneResourceManager createAndStartResourceManager(TestingLeaderEle
 		TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
 		HeartbeatServices heartbeatServices = new HeartbeatServices(5L, 5L);
 		highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
-		TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
 		ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(
 			Time.seconds(5L),
 			Time.seconds(5L));
+			
+		SlotManager slotManager = new SlotManager(
+			rpcService.getScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime());
 
 		MetricRegistry metricRegistry = mock(MetricRegistry.class);
 		JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
@@ -171,7 +178,7 @@ private StandaloneResourceManager createAndStartResourceManager(TestingLeaderEle
 				resourceManagerConfiguration,
 				highAvailabilityServices,
 				heartbeatServices,
-				slotManagerFactory,
+				slotManager,
 				metricRegistry,
 				jobLeaderIdService,
 				fatalErrorHandler);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
deleted file mode 100644
index 67b208d9b1ecb..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.resourcemanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
-import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
-import org.apache.flink.runtime.clusterframework.types.SlotID;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
-import org.mockito.Mockito;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Executor;
-
-public class TestingSlotManager extends SlotManager {
-
-	public TestingSlotManager() {
-		this(new TestingResourceManagerServices());
-	}
-
-	public TestingSlotManager(ResourceManagerServices rmServices) {
-		super(rmServices);
-	}
-
-	@Override
-	protected ResourceSlot chooseSlotToUse(SlotRequest request, Map<SlotID, ResourceSlot> freeSlots) {
-		final Iterator<ResourceSlot> slotIterator = freeSlots.values().iterator();
-		if (slotIterator.hasNext()) {
-			return slotIterator.next();
-		} else {
-			return null;
-		}
-	}
-
-	@Override
-	protected SlotRequest chooseRequestToFulfill(ResourceSlot offeredSlot, Map<AllocationID, SlotRequest> pendingRequests) {
-		final Iterator<SlotRequest> requestIterator = pendingRequests.values().iterator();
-		if (requestIterator.hasNext()) {
-			return requestIterator.next();
-		} else {
-			return null;
-		}
-	}
-
-	private static class TestingResourceManagerServices implements ResourceManagerServices {
-
-		private final UUID leaderID = UUID.randomUUID();
-
-		@Override
-		public UUID getLeaderID() {
-			return leaderID;
-		}
-
-		@Override
-		public void allocateResource(ResourceProfile resourceProfile) {
-
-		}
-
-		@Override
-		public Executor getAsyncExecutor() {
-			return Mockito.mock(Executor.class);
-		}
-
-		@Override
-		public Executor getMainThreadExecutor() {
-			return Mockito.mock(Executor.class);
-		}
-	}
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java
deleted file mode 100644
index 6b5f6b224e058..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager;
-
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
-
-public class TestingSlotManagerFactory implements SlotManagerFactory {
-
-	@Override
-	public SlotManager create(ResourceManagerServices rmServices) {
-		return new TestingSlotManager(rmServices);
-	}
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
index 041747d4b6a54..b0b5d326d1fdd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
@@ -23,510 +23,1058 @@
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
-import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.clusterframework.types.TaskManagerSlot;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.concurrent.ScheduledExecutor;
 import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection;
 import org.apache.flink.runtime.taskexecutor.SlotReport;
 import org.apache.flink.runtime.taskexecutor.SlotStatus;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
-import org.junit.BeforeClass;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotOccupiedException;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.TestLogger;
 import org.junit.Test;
-import org.mockito.Mockito;
+import org.mockito.ArgumentCaptor;
 
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.Arrays;
 import java.util.UUID;
 import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class SlotManagerTest extends TestLogger {
 
-public class SlotManagerTest {
-
-	private static final double DEFAULT_TESTING_CPU_CORES = 1.0;
+	/**
+	 * Tests that we can register task manager and their slots at the slot manager.
+	 */
+	@Test
+	public void testTaskManagerRegistration() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 
-	private static final int DEFAULT_TESTING_MEMORY = 512;
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
 
-	private static final ResourceProfile DEFAULT_TESTING_PROFILE =
-		new ResourceProfile(DEFAULT_TESTING_CPU_CORES, DEFAULT_TESTING_MEMORY);
+		ResourceID resourceId = ResourceID.generate();
+		final SlotID slotId1 = new SlotID(resourceId, 0);
+		final SlotID slotId2 = new SlotID(resourceId, 1);
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile);
+		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
+		final SlotReport slotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
 
-	private static final ResourceProfile DEFAULT_TESTING_BIG_PROFILE =
-		new ResourceProfile(DEFAULT_TESTING_CPU_CORES * 2, DEFAULT_TESTING_MEMORY * 2);
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
 
-	private static TaskExecutorRegistration taskExecutorRegistration;
+			assertTrue("The number registered slots does not equal the expected number.",2 == slotManager.getNumberRegisteredSlots());
 
-	@BeforeClass
-	public static void setUp() {
-		taskExecutorRegistration = Mockito.mock(TaskExecutorRegistration.class);
-		TaskExecutorGateway gateway = Mockito.mock(TaskExecutorGateway.class);
-		Mockito.when(taskExecutorRegistration.getTaskExecutorGateway()).thenReturn(gateway);
-		Mockito.when(gateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class)))
-			.thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
+			assertNotNull(slotManager.getSlot(slotId1));
+			assertNotNull(slotManager.getSlot(slotId2));
+		}
 	}
 
 	/**
-	 * Tests that there are no free slots when we request, need to allocate from cluster manager master
+	 * Tests that un-registration of task managers will free and remove all registered slots.
 	 */
 	@Test
-	public void testRequestSlotWithoutFreeSlot() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
-
-		assertEquals(0, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(1, slotManager.getPendingRequestCount());
-		assertEquals(1, slotManager.getAllocatedContainers().size());
-		assertEquals(DEFAULT_TESTING_PROFILE, slotManager.getAllocatedContainers().get(0));
+	public void testTaskManagerUnregistration() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final JobID jobId = new JobID();
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			any(SlotID.class),
+			any(JobID.class),
+			any(AllocationID.class),
+			anyString(),
+			eq(leaderId),
+			any(Time.class))).thenReturn(new FlinkCompletableFuture<Acknowledge>());
+
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		ResourceID resourceId = ResourceID.generate();
+		final SlotID slotId1 = new SlotID(resourceId, 0);
+		final SlotID slotId2 = new SlotID(resourceId, 1);
+		final AllocationID allocationId1 = new AllocationID();
+		final AllocationID allocationId2 = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile, jobId, allocationId1);
+		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
+		final SlotReport slotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
+
+		final SlotRequest slotRequest = new SlotRequest(
+			new JobID(),
+			allocationId2,
+			resourceProfile,
+			"foobar");
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
+
+			assertTrue("The number registered slots does not equal the expected number.",2 == slotManager.getNumberRegisteredSlots());
+
+			TaskManagerSlot slot1 = slotManager.getSlot(slotId1);
+			TaskManagerSlot slot2 = slotManager.getSlot(slotId2);
+
+			assertTrue(slot1.isAllocated());
+			assertTrue(slot2.isFree());
+
+			assertTrue(slotManager.registerSlotRequest(slotRequest));
+
+			assertFalse(slot2.isFree());
+			assertTrue(slot2.hasPendingSlotRequest());
+
+			PendingSlotRequest pendingSlotRequest = slotManager.getSlotRequest(allocationId2);
+
+			assertTrue("The pending slot request should have been assigned to slot 2", pendingSlotRequest.isAssigned());
+
+			slotManager.unregisterTaskManager(taskManagerConnection.getInstanceID());
+
+			assertTrue(0 == slotManager.getNumberRegisteredSlots());
+			assertFalse(pendingSlotRequest.isAssigned());
+		}
 	}
 
 	/**
-	 * Tests that there are some free slots when we request, and the request is fulfilled immediately
+	 * Tests that a slot request with no free slots will trigger the resource allocation
 	 */
 	@Test
-	public void testRequestSlotWithFreeSlot() {
-		TestingSlotManager slotManager = new TestingSlotManager();
+	public void testSlotRequestWithoutFreeSlots() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotRequest slotRequest = new SlotRequest(
+			new JobID(),
+			new AllocationID(),
+			resourceProfile,
+			"localhost");
 
-		directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 1);
-		assertEquals(1, slotManager.getFreeSlotCount());
+		ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-		assertEquals(0, slotManager.getAllocatedContainers().size());
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+
+			slotManager.registerSlotRequest(slotRequest);
+
+			verify(resourceManagerActions).allocateResource(eq(resourceProfile));
+		}
 	}
 
 	/**
-	 * Tests that there are some free slots when we request, but none of them are suitable
+	 * Tests that the slot request fails if we cannot allocate more resources.
 	 */
 	@Test
-	public void testRequestSlotWithoutSuitableSlot() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-
-		directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 2);
-		assertEquals(2, slotManager.getFreeSlotCount());
-
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE));
-		assertEquals(0, slotManager.getAllocatedSlotCount());
-		assertEquals(2, slotManager.getFreeSlotCount());
-		assertEquals(1, slotManager.getPendingRequestCount());
-		assertEquals(1, slotManager.getAllocatedContainers().size());
-		assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
+	public void testSlotRequestWithResourceAllocationFailure() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotRequest slotRequest = new SlotRequest(
+			new JobID(),
+			new AllocationID(),
+			resourceProfile,
+			"localhost");
+
+		ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		doThrow(new ResourceManagerException("Test exception")).when(resourceManagerActions).allocateResource(any(ResourceProfile.class));
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+
+			slotManager.registerSlotRequest(slotRequest);
+
+			fail("The slot request should have failed with a ResourceManagerException.");
+
+		} catch (ResourceManagerException e) {
+			// expected exception
+		}
 	}
 
 	/**
-	 * Tests that we send duplicated slot request
+	 * Tests that a slot request which can be fulfilled will trigger a slot allocation.
 	 */
 	@Test
-	public void testDuplicatedSlotRequest() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 1);
-
-		SlotRequest request1 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
-		SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE);
-
-		slotManager.requestSlot(request1);
-		slotManager.requestSlot(request2);
-		slotManager.requestSlot(request2);
-		slotManager.requestSlot(request1);
-
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(1, slotManager.getPendingRequestCount());
-		assertEquals(1, slotManager.getAllocatedContainers().size());
-		assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
+	public void testSlotRequestWithFreeSlot() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceID resourceID = ResourceID.generate();
+		final JobID jobId = new JobID();
+		final SlotID slotId = new SlotID(resourceID, 0);
+		final String targetAddress = "localhost";
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotRequest slotRequest = new SlotRequest(
+			jobId,
+			allocationId,
+			resourceProfile,
+			targetAddress);
+
+		ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+
+			// accept an incoming slot request
+			final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+			when(taskExecutorGateway.requestSlot(
+				eq(slotId),
+				eq(jobId),
+				eq(allocationId),
+				anyString(),
+				eq(leaderId),
+				any(Time.class))).thenReturn(FlinkCompletableFuture.completed(Acknowledge.get()));
+
+			final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+			final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
+			final SlotReport slotReport = new SlotReport(slotStatus);
+
+			slotManager.registerTaskManager(
+				taskExecutorConnection,
+				slotReport);
+
+			assertTrue("The slot request should be accepted", slotManager.registerSlotRequest(slotRequest));
+
+			verify(taskExecutorGateway).requestSlot(eq(slotId), eq(jobId), eq(allocationId), eq(targetAddress), eq(leaderId), any(Time.class));
+
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
+
+			assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
+		}
 	}
 
 	/**
-	 * Tests that we send multiple slot requests
+	 * Checks that un-registering a pending slot request will cancel it, removing it from all
+	 * assigned task manager slots and then remove it from the slot manager.
 	 */
 	@Test
-	public void testRequestMultipleSlots() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 5);
+	public void testUnregisterPendingSlotRequest() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final SlotID slotId = new SlotID(ResourceID.generate(), 0);
+		final AllocationID allocationId = new AllocationID();
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			any(SlotID.class),
+			any(JobID.class),
+			any(AllocationID.class),
+			anyString(),
+			eq(leaderId),
+			any(Time.class))).thenReturn(new FlinkCompletableFuture<Acknowledge>());
+
+		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
+		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
+		final SlotReport slotReport = new SlotReport(slotStatus);
 
-		// request 3 normal slots
-		for (int i = 0; i < 3; ++i) {
-			slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
-		}
+		final SlotRequest slotRequest = new SlotRequest(new JobID(), allocationId, resourceProfile, "foobar");
 
-		// request 2 big slots
-		for (int i = 0; i < 2; ++i) {
-			slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE));
-		}
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
 
-		// request 1 normal slot again
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
 
-		assertEquals(4, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
-		assertEquals(2, slotManager.getPendingRequestCount());
-		assertEquals(2, slotManager.getAllocatedContainers().size());
-		assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
-		assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(1));
+			slotManager.registerSlotRequest(slotRequest);
+
+			assertNotNull(slotManager.getSlotRequest(allocationId));
+
+			assertTrue(slot.hasPendingSlotRequest());
+
+			slotManager.unregisterSlotRequest(allocationId);
+
+			assertNull(slotManager.getSlotRequest(allocationId));
+
+			slot = slotManager.getSlot(slotId);
+			assertTrue(slot.isFree());
+		}
 	}
 
 	/**
-	 * Tests that a new slot appeared in SlotReport, and we used it to fulfill a pending request
+	 * Tests that pending slot requests are tried to be fulfilled upon new slot registrations.
 	 */
 	@Test
-	public void testNewlyAppearedFreeSlotFulfillPendingRequest() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
-		assertEquals(1, slotManager.getPendingRequestCount());
-
-		SlotID slotId = SlotID.generate();
-		SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
-		SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
-		slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
-
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-		assertTrue(slotManager.isAllocated(slotId));
+	public void testFulfillingPendingSlotRequest() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceID resourceID = ResourceID.generate();
+		final JobID jobId = new JobID();
+		final SlotID slotId = new SlotID(resourceID, 0);
+		final String targetAddress = "localhost";
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotRequest slotRequest = new SlotRequest(
+			jobId,
+			allocationId,
+			resourceProfile,
+			targetAddress);
+
+		ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+
+		// accept an incoming slot request
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			eq(slotId),
+			eq(jobId),
+			eq(allocationId),
+			anyString(),
+			eq(leaderId),
+			any(Time.class))).thenReturn(FlinkCompletableFuture.completed(Acknowledge.get()));
+
+		final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
+		final SlotReport slotReport = new SlotReport(slotStatus);
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+
+			assertTrue("The slot request should be accepted", slotManager.registerSlotRequest(slotRequest));
+
+			verify(resourceManagerActions, times(1)).allocateResource(eq(resourceProfile));
+
+			slotManager.registerTaskManager(
+				taskExecutorConnection,
+				slotReport);
+
+			verify(taskExecutorGateway).requestSlot(eq(slotId), eq(jobId), eq(allocationId), eq(targetAddress), eq(leaderId), any(Time.class));
+
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
+
+			assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
+		}
 	}
 
 	/**
-	 * Tests that a new slot appeared in SlotReport, but we have no pending request
+	 * Tests that freeing a slot will correctly reset the slot and mark it as a free slot
 	 */
 	@Test
-	public void testNewlyAppearedFreeSlot() {
-		TestingSlotManager slotManager = new TestingSlotManager();
+	public void testFreeSlot() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceID resourceID = ResourceID.generate();
+		final JobID jobId = new JobID();
+		final SlotID slotId = new SlotID(resourceID, 0);
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+
+		ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+
+		// accept an incoming slot request
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
 
-		SlotID slotId = SlotID.generate();
-		SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
-		SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
-		slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
+		final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskExecutorGateway);
 
-		assertEquals(0, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
+		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile, jobId, allocationId);
+		final SlotReport slotReport = new SlotReport(slotStatus);
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+
+			slotManager.registerTaskManager(
+				taskExecutorConnection,
+				slotReport);
+
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
+
+			assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
+
+			// this should be ignored since the allocation id does not match
+			slotManager.freeSlot(slotId, new AllocationID());
+
+			assertTrue(slot.isAllocated());
+			assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
+
+			slotManager.freeSlot(slotId, allocationId);
+
+			assertTrue(slot.isFree());
+			assertNull(slot.getAllocationId());
+		}
 	}
 
 	/**
-	 * Tests that a new slot appeared in SlotReport, but it't not suitable for all the pending requests
+	 * Tests that a second pending slot request is detected as a duplicate if the allocation ids are
+	 * the same.
 	 */
 	@Test
-	public void testNewlyAppearedFreeSlotNotMatchPendingRequests() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE));
-		assertEquals(1, slotManager.getPendingRequestCount());
-
-		SlotID slotId = SlotID.generate();
-		SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
-		SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
-		slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
-
-		assertEquals(0, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
-		assertEquals(1, slotManager.getPendingRequestCount());
-		assertFalse(slotManager.isAllocated(slotId));
+	public void testDuplicatePendingSlotRequest() throws Exception {
+
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile1 = new ResourceProfile(1.0, 2);
+		final ResourceProfile resourceProfile2 = new ResourceProfile(2.0, 1);
+		final SlotRequest slotRequest1 = new SlotRequest(new JobID(), allocationId, resourceProfile1, "foobar");
+		final SlotRequest slotRequest2 = new SlotRequest(new JobID(), allocationId, resourceProfile2, "barfoo");
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			assertTrue(slotManager.registerSlotRequest(slotRequest1));
+			assertFalse(slotManager.registerSlotRequest(slotRequest2));
+		}
+
+		// check that we have only called the resource allocation only for the first slot request,
+		// since the second request is a duplicate
+		verify(resourceManagerActions, times(1)).allocateResource(any(ResourceProfile.class));
 	}
 
 	/**
-	 * Tests that a new slot appeared in SlotReport, and it's been reported using by some job
+	 * Tests that if we have received a slot report with some allocated slots, then we don't accept
+	 * slot requests with allocated allocation ids.
 	 */
 	@Test
-	public void testNewlyAppearedInUseSlot() {
-		TestingSlotManager slotManager = new TestingSlotManager();
+	public void testDuplicatePendingSlotRequestAfterSlotReport() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final JobID jobId = new JobID();
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
+		final SlotID slotId = new SlotID(ResourceID.generate(), 0);
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile, jobId, allocationId);
+		final SlotReport slotReport = new SlotReport(slotStatus);
 
-		SlotID slotId = SlotID.generate();
-		SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE, new JobID(), new AllocationID());
-		SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
-		slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
+		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertTrue(slotManager.isAllocated(slotId));
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
+
+			assertFalse(slotManager.registerSlotRequest(slotRequest));
+		}
 	}
 
 	/**
-	 * Tests that we had a slot in-use and is freed again subsequently.
+	 * Tests that duplicate slot requests (requests with an already registered allocation id) are
+	 * also detected after a pending slot request has been fulfilled but not yet freed.
 	 */
 	@Test
-	public void testExistingInUseSlotUpdateStatus() {
-		TestingSlotManager slotManager = new TestingSlotManager();
+	public void testDuplicatePendingSlotRequestAfterSuccessfulAllocation() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile1 = new ResourceProfile(1.0, 2);
+		final ResourceProfile resourceProfile2 = new ResourceProfile(2.0, 1);
+		final SlotRequest slotRequest1 = new SlotRequest(new JobID(), allocationId, resourceProfile1, "foobar");
+		final SlotRequest slotRequest2 = new SlotRequest(new JobID(), allocationId, resourceProfile2, "barfoo");
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			any(SlotID.class),
+			any(JobID.class),
+			any(AllocationID.class),
+			anyString(),
+			eq(leaderId),
+			any(Time.class))).thenReturn(FlinkCompletableFuture.completed(Acknowledge.get()));
+
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		final SlotID slotId = new SlotID(ResourceID.generate(), 0);
+		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile1);
+		final SlotReport slotReport = new SlotReport(slotStatus);
 
-		SlotID slotId = SlotID.generate();
-		SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE, new JobID(), new AllocationID());
-		SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
-		slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
+			assertTrue(slotManager.registerSlotRequest(slotRequest1));
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertTrue(slotManager.isAllocated(slotId));
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
 
-		// slot is freed again
-		slotManager.notifySlotAvailable(slotId.getResourceID(), slotId);
+			assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
 
-		assertEquals(0, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
-		assertFalse(slotManager.isAllocated(slotId));
+			assertFalse(slotManager.registerSlotRequest(slotRequest2));
+		}
+
+		// check that we have only called the resource allocation only for the first slot request,
+		// since the second request is a duplicate
+		verify(resourceManagerActions, never()).allocateResource(any(ResourceProfile.class));
 	}
 
 	/**
-	 * Tests multiple slot requests with one slots.
+	 * Tests that an already registered allocation id can be reused after the initial slot request
+	 * has been freed.
 	 */
 	@Test
-	public void testMultipleSlotRequestsWithOneSlot() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		final AllocationID allocationID = new AllocationID();
+	public void testAcceptingDuplicateSlotRequestAfterAllocationRelease() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile1 = new ResourceProfile(1.0, 2);
+		final ResourceProfile resourceProfile2 = new ResourceProfile(2.0, 1);
+		final SlotRequest slotRequest1 = new SlotRequest(new JobID(), allocationId, resourceProfile1, "foobar");
+		final SlotRequest slotRequest2 = new SlotRequest(new JobID(), allocationId, resourceProfile2, "barfoo");
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			any(SlotID.class),
+			any(JobID.class),
+			any(AllocationID.class),
+			anyString(),
+			eq(leaderId),
+			any(Time.class))).thenReturn(FlinkCompletableFuture.completed(Acknowledge.get()));
+
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		final SlotID slotId = new SlotID(ResourceID.generate(), 0);
+		final SlotStatus slotStatus = new SlotStatus(slotId, new ResourceProfile(2.0, 2));
+		final SlotReport slotReport = new SlotReport(slotStatus);
 
-		SlotRequest request1 = new SlotRequest(new JobID(), allocationID, DEFAULT_TESTING_PROFILE);
-		slotManager.requestSlot(request1);
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
+			assertTrue(slotManager.registerSlotRequest(slotRequest1));
 
-		final ResourceID resourceID = ResourceID.generate();
-		final SlotStatus slotStatus = new SlotStatus(new SlotID(resourceID, 0), DEFAULT_TESTING_PROFILE);
-		final SlotReport slotReport = new SlotReport(slotStatus);
-		slotManager.registerTaskExecutor(resourceID, taskExecutorRegistration, slotReport);
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
 
-		// another request pending
-		SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
-		slotManager.requestSlot(request2);
+			assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(1, slotManager.getPendingRequestCount());
-		assertTrue(slotManager.isAllocated(allocationID));
-		assertTrue(slotManager.isAllocated(request1.getAllocationId()));
+			slotManager.freeSlot(slotId, allocationId);
 
-		// but slot is reported empty in a report in the meantime which shouldn't affect the state
-		slotManager.notifySlotAvailable(resourceID, slotStatus.getSlotID());
+			// check that the slot has been freed
+			assertTrue(slot.isFree());
+			assertNull(slot.getAllocationId());
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-		assertTrue(slotManager.isAllocated(slotStatus.getSlotID()));
-		assertTrue(slotManager.isAllocated(request2.getAllocationId()));
+			assertTrue(slotManager.registerSlotRequest(slotRequest2));
 
-		// but slot is reported empty in a report in the meantime which shouldn't affect the state
-		slotManager.notifySlotAvailable(resourceID, slotStatus.getSlotID());
+			assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
+		}
 
-		assertEquals(0, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
+		// check that we have only called the resource allocation only for the first slot request,
+		// since the second request is a duplicate
+		verify(resourceManagerActions, never()).allocateResource(any(ResourceProfile.class));
 	}
 
 	/**
-	 * Tests that we did some allocation but failed / rejected by TaskManager, request will retry
+	 * Tests that the slot manager ignores slot reports of unknown origin (not registered
+	 * task managers).
 	 */
 	@Test
-	public void testSlotAllocationFailedAtTaskManager() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		ResourceSlot slot = new ResourceSlot(SlotID.generate(), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
-		slotManager.addFreeSlot(slot);
+	public void testReceivingUnknownSlotReport() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 
-		SlotRequest request = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
-		slotManager.requestSlot(request);
+		final InstanceID unknownInstanceID = new InstanceID();
+		final SlotID unknownSlotId = new SlotID(ResourceID.generate(), 0);
+		final ResourceProfile unknownResourceProfile = new ResourceProfile(1.0, 1);
+		final SlotStatus unknownSlotStatus = new SlotStatus(unknownSlotId, unknownResourceProfile);
+		final SlotReport unknownSlotReport = new SlotReport(unknownSlotStatus);
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-		assertTrue(slotManager.isAllocated(slot.getSlotId()));
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			// check that we don't have any slots registered
+			assertTrue(0 == slotManager.getNumberRegisteredSlots());
 
-		slotManager.handleSlotRequestFailedAtTaskManager(request, slot.getSlotId());
+			// this should not update anything since the instance id is not known to the slot manager
+			assertFalse(slotManager.reportSlotStatus(unknownInstanceID, unknownSlotReport));
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
+			assertTrue(0 == slotManager.getNumberRegisteredSlots());
+		}
 	}
 
-
 	/**
-	 * Tests that we did some allocation but failed / rejected by TaskManager, and slot is occupied by another request
-	 * This can only occur after reconnect of the TaskExecutor.
+	 * Tests that slots are updated with respect to the latest incoming slot report. This means that
+	 * slot for which not report has been received will be removed and those for which a report was
+	 * received are updated accordingly.
 	 */
 	@Test
-	public void testSlotAllocationFailedAtTaskManagerOccupiedByOther() {
-		TestingSlotManager slotManager = new TestingSlotManager();
-		final SlotID slotID = SlotID.generate();
-		SlotStatus slot = new SlotStatus(slotID, DEFAULT_TESTING_PROFILE);
-		SlotReport slotReport = new SlotReport(slot);
-		slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorRegistration, slotReport);
-
-		SlotRequest request = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
-		slotManager.requestSlot(request);
-
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-
-		// slot is set empty by a reconnect of the TaskExecutor
-		slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorRegistration, slotReport);
-
-		assertEquals(0, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-
-		// another request takes the slot
-		SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
-		slotManager.requestSlot(request2);
-
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-		assertFalse(slotManager.isAllocated(request.getAllocationId()));
-		assertTrue(slotManager.isAllocated(request2.getAllocationId()));
-
-		// original request should be retried
-		slotManager.handleSlotRequestFailedAtTaskManager(request, slotID);
-
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-		assertFalse(slotManager.isAllocated(request.getAllocationId()));
-		assertTrue(slotManager.isAllocated(request2.getAllocationId()));
+	public void testUpdateSlotReport() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+
+		final JobID jobId = new JobID();
+		final AllocationID allocationId = new AllocationID();
+
+		final ResourceID resourceId = ResourceID.generate();
+		final SlotID slotId1 = new SlotID(resourceId, 0);
+		final SlotID slotId2 = new SlotID(resourceId, 1);
+		final SlotID slotId3 = new SlotID(resourceId, 2);
+
+
+		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
+		final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile);
+		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
+
+		final SlotStatus newSlotStatus2 = new SlotStatus(slotId2, resourceProfile, jobId, allocationId);
+		final SlotStatus slotStatus3 = new SlotStatus(slotId3, resourceProfile);
+
+		final SlotReport slotReport1 = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
+		final SlotReport slotReport2 = new SlotReport(Arrays.asList(newSlotStatus2, slotStatus3));
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+			// check that we don't have any slots registered
+			assertTrue(0 == slotManager.getNumberRegisteredSlots());
+
+			slotManager.registerTaskManager(taskManagerConnection, slotReport1);
+
+			TaskManagerSlot slot = slotManager.getSlot(slotId2);
+
+			assertTrue(2 == slotManager.getNumberRegisteredSlots());
+
+			assertTrue(slot.isFree());
+
+			assertTrue(slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), slotReport2));
+
+			assertTrue(2 == slotManager.getNumberRegisteredSlots());
+
+			// the slot manager should have removed slotId1
+			assertNull(slotManager.getSlot(slotId1));
+
+			assertNotNull(slotManager.getSlot(slotId3));
+
+			// slotId2 should have been allocated for allocationId
+			assertEquals(allocationId, slotManager.getSlot(slotId2).getAllocationId());
+		}
 	}
 
+	/**
+	 * Tests that idle task managers time out after the configured timeout. A timed out task manager
+	 * will be removed from the slot manager and the resource manager will be notified about the
+	 * timeout.
+	 */
 	@Test
-	public void testNotifyTaskManagerFailure() {
-		TestingSlotManager slotManager = new TestingSlotManager();
+	public void testTaskManagerTimeout() throws Exception {
+		final long tmTimeout = 50L;
 
-		ResourceID resource1 = ResourceID.generate();
-		ResourceID resource2 = ResourceID.generate();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final UUID leaderId = UUID.randomUUID();
 
-		ResourceSlot slot11 = new ResourceSlot(new SlotID(resource1, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
-		ResourceSlot slot12 = new ResourceSlot(new SlotID(resource1, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
-		ResourceSlot slot21 = new ResourceSlot(new SlotID(resource2, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
-		ResourceSlot slot22 = new ResourceSlot(new SlotID(resource2, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
 
-		slotManager.addFreeSlot(slot11);
-		slotManager.addFreeSlot(slot21);
+		final SlotID slotId = new SlotID(ResourceID.generate(), 0);
+		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
+		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
+		final SlotReport slotReport = new SlotReport(slotStatus);
 
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
-		slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
+		final Executor mainThreadExecutor = mock(Executor.class);
 
-		assertEquals(2, slotManager.getAllocatedSlotCount());
-		assertEquals(0, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
+		try (SlotManager slotManager = new SlotManager(
+			TestingUtils.defaultScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			Time.milliseconds(tmTimeout))) {
 
-		slotManager.addFreeSlot(slot12);
-		slotManager.addFreeSlot(slot22);
+			slotManager.start(leaderId, mainThreadExecutor, resourceManagerActions);
 
-		assertEquals(2, slotManager.getAllocatedSlotCount());
-		assertEquals(2, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
 
-		slotManager.notifyTaskManagerFailure(resource2);
+			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
+			verify(mainThreadExecutor, timeout(tmTimeout * 10L)).execute(runnableArgumentCaptor.capture());
 
-		// notify an not exist resource failure
-		slotManager.notifyTaskManagerFailure(ResourceID.generate());
+			// the only runnable being executed by the main thread executor should be the timeout runnable
+			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
 
-		assertEquals(1, slotManager.getAllocatedSlotCount());
-		assertEquals(1, slotManager.getFreeSlotCount());
-		assertEquals(0, slotManager.getPendingRequestCount());
-	}
+			timeoutRunnable.run();
 
-	// ------------------------------------------------------------------------
-	//  testing utilities
-	// ------------------------------------------------------------------------
-
-	private void directlyProvideFreeSlots(
-		final SlotManager slotManager,
-		final ResourceProfile resourceProfile,
-		final int freeSlotNum)
-	{
-		for (int i = 0; i < freeSlotNum; ++i) {
-			slotManager.addFreeSlot(new ResourceSlot(SlotID.generate(), new ResourceProfile(resourceProfile), taskExecutorRegistration));
+			verify(resourceManagerActions, times(1)).releaseResource(eq(taskManagerConnection.getInstanceID()));
 		}
 	}
 
-	// ------------------------------------------------------------------------
-	//  testing classes
-	// ------------------------------------------------------------------------
+	/**
+	 * Tests that slot requests time out after the specified request timeout. If a slot request
+	 * times out, then the request is cancelled, removed from the slot manager and the resourc
+	 * manager is notified about the failed allocation.
+	 */
+	@Test
+	public void testSlotRequestTimeout() throws Exception {
+		final long allocationTimeout = 50L;
+
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final UUID leaderId = UUID.randomUUID();
+		final JobID jobId = new JobID();
+		final AllocationID allocationId = new AllocationID();
+
+		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
+		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
 
-	private static class TestingSlotManager extends SlotManager {
+		final Executor mainThreadExecutor = mock(Executor.class);
 
-		private static TestingRmServices testingRmServices = new TestingRmServices();
+		try (SlotManager slotManager = new SlotManager(
+			TestingUtils.defaultScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			Time.milliseconds(allocationTimeout),
+			TestingUtils.infiniteTime())) {
 
-		TestingSlotManager() {
-			super(testingRmServices);
-			testingRmServices.allocatedContainers.clear();
+			slotManager.start(leaderId, mainThreadExecutor, resourceManagerActions);
+
+			assertTrue(slotManager.registerSlotRequest(slotRequest));
+
+			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
+
+			verify(mainThreadExecutor, timeout(allocationTimeout * 10L)).execute(runnableArgumentCaptor.capture());
+
+			// the only runnable being executed by the main thread executor should be the timeout runnable
+			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
+
+			timeoutRunnable.run();
+
+			verify(resourceManagerActions, times(1)).notifyAllocationFailure(
+				eq(jobId),
+				eq(allocationId),
+				any(TimeoutException.class));
 		}
+	}
 
-		/**
-		 * Choose slot randomly if it matches requirement
-		 *
-		 * @param request   The slot request
-		 * @param freeSlots All slots which can be used
-		 * @return The chosen slot or null if cannot find a match
-		 */
-		@Override
-		protected ResourceSlot chooseSlotToUse(SlotRequest request, Map<SlotID, ResourceSlot> freeSlots) {
-			for (ResourceSlot slot : freeSlots.values()) {
-				if (slot.isMatchingRequirement(request.getResourceProfile())) {
-					return slot;
-				}
+	/**
+	 * Tests that a slot request is retried if it times out on the task manager side
+	 */
+	@Test
+	@SuppressWarnings("unchecked")
+	public void testTaskManagerSlotRequestTimeoutHandling() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+
+		final JobID jobId = new JobID();
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
+		final FlinkCompletableFuture<Acknowledge> slotRequestFuture1 = new FlinkCompletableFuture<>();
+		final FlinkCompletableFuture<Acknowledge> slotRequestFuture2 = new FlinkCompletableFuture<>();
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			any(SlotID.class),
+			any(JobID.class),
+			eq(allocationId),
+			anyString(),
+			any(UUID.class),
+			any(Time.class))).thenReturn(slotRequestFuture1, slotRequestFuture2);
+
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		final ResourceID resourceId = ResourceID.generate();
+		final SlotID slotId1 = new SlotID(resourceId, 0);
+		final SlotID slotId2 = new SlotID(resourceId, 1);
+		final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile);
+		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
+		final SlotReport slotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
+
+		try (SlotManager slotManager = createSlotManager(leaderId, resourceManagerActions)) {
+
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
+
+			slotManager.registerSlotRequest(slotRequest);
+
+			ArgumentCaptor<SlotID> slotIdCaptor = ArgumentCaptor.forClass(SlotID.class);
+
+			verify(taskExecutorGateway, times(1)).requestSlot(
+				slotIdCaptor.capture(),
+				eq(jobId),
+				eq(allocationId),
+				anyString(),
+				eq(leaderId),
+				any(Time.class));
+
+			TaskManagerSlot failedSlot = slotManager.getSlot(slotIdCaptor.getValue());
+
+			// let the first attempt fail --> this should trigger a second attempt
+			slotRequestFuture1.completeExceptionally(new SlotAllocationException("Test exception."));
+
+			verify(taskExecutorGateway, times(2)).requestSlot(
+				slotIdCaptor.capture(),
+				eq(jobId),
+				eq(allocationId),
+				anyString(),
+				eq(leaderId),
+				any(Time.class));
+
+			// the second attempt succeeds
+			slotRequestFuture2.complete(Acknowledge.get());
+
+			TaskManagerSlot slot = slotManager.getSlot(slotIdCaptor.getValue());
+
+			assertTrue(slot.isAllocated());
+			assertEquals(allocationId, slot.getAllocationId());
+
+			if (!failedSlot.getSlotId().equals(slot.getSlotId())) {
+				assertTrue(failedSlot.isFree());
 			}
-			return null;
 		}
+	}
 
-		/**
-		 * Choose request randomly if offered slot can match its requirement
-		 *
-		 * @param offeredSlot     The free slot
-		 * @param pendingRequests All the pending slot requests
-		 * @return The chosen request's AllocationID or null if cannot find a match
-		 */
-		@Override
-		protected SlotRequest chooseRequestToFulfill(ResourceSlot offeredSlot,
-			Map<AllocationID, SlotRequest> pendingRequests)
-		{
-			for (Map.Entry<AllocationID, SlotRequest> pendingRequest : pendingRequests.entrySet()) {
-				if (offeredSlot.isMatchingRequirement(pendingRequest.getValue().getResourceProfile())) {
-					return pendingRequest.getValue();
-				}
-			}
-			return null;
+	/**
+	 * Tests that pending slot requests are rejected if a slot report with a different allocation
+	 * is received.
+	 */
+	@Test
+	@SuppressWarnings("unchecked")
+	public void testSlotReportWhileActiveSlotRequest() throws Exception {
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+
+		final JobID jobId = new JobID();
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(42.0, 1337);
+		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
+		final FlinkCompletableFuture<Acknowledge> slotRequestFuture1 = new FlinkCompletableFuture<>();
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			any(SlotID.class),
+			any(JobID.class),
+			eq(allocationId),
+			anyString(),
+			any(UUID.class),
+			any(Time.class))).thenReturn(slotRequestFuture1, FlinkCompletableFuture.completed(Acknowledge.get()));
+
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		final ResourceID resourceId = ResourceID.generate();
+		final SlotID slotId1 = new SlotID(resourceId, 0);
+		final SlotID slotId2 = new SlotID(resourceId, 1);
+		final SlotID slotId3 = new SlotID(resourceId, 2);
+		final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile);
+		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
+		final SlotReport slotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
+
+		// we have to manually trigger the future call backs to simulate the main thread executor behaviour
+		final Executor mainThreadExecutorMock = mock(Executor.class);
+
+		try (SlotManager slotManager = new SlotManager(
+			TestingUtils.defaultScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime())) {
+
+			slotManager.start(leaderId, mainThreadExecutorMock, resourceManagerActions);
+
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
+
+			slotManager.registerSlotRequest(slotRequest);
+
+			ArgumentCaptor<SlotID> slotIdCaptor = ArgumentCaptor.forClass(SlotID.class);
+
+			verify(taskExecutorGateway, times(1)).requestSlot(
+				slotIdCaptor.capture(),
+				eq(jobId),
+				eq(allocationId),
+				anyString(),
+				eq(leaderId),
+				any(Time.class));
+
+			final SlotStatus newSlotStatus1 = new SlotStatus(slotIdCaptor.getValue(), resourceProfile, new JobID(), new AllocationID());
+			final SlotStatus newSlotStatus2 = new SlotStatus(slotId3, resourceProfile);
+			final SlotReport newSlotReport = new SlotReport(Arrays.asList(newSlotStatus1, newSlotStatus2));
+
+			// this should remove the unused slot, replacing it with slotId3 and retry the pending slot request
+			slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), newSlotReport);
+
+			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
+			verify(mainThreadExecutorMock).execute(runnableArgumentCaptor.capture());
+
+			Runnable requestFailureRunnable = runnableArgumentCaptor.getValue();
+
+			requestFailureRunnable.run();
+
+			verify(taskExecutorGateway, times(2)).requestSlot(
+				slotIdCaptor.capture(),
+				eq(jobId),
+				eq(allocationId),
+				anyString(),
+				eq(leaderId),
+				any(Time.class));
+
+			verify(mainThreadExecutorMock, times(2)).execute(runnableArgumentCaptor.capture());
+			Runnable requestSuccessRunnable = runnableArgumentCaptor.getValue();
+
+			requestSuccessRunnable.run();
+
+			final SlotID requestedSlotId = slotIdCaptor.getValue();
+
+			assertEquals(slotId3, requestedSlotId);
+
+			TaskManagerSlot slot = slotManager.getSlot(requestedSlotId);
+
+			assertTrue(slot.isAllocated());
+			assertEquals(allocationId, slot.getAllocationId());
 		}
+	}
+
+	/**
+	 * Tests that formerly used task managers can again timeout after all of their slots have
+	 * been freed.
+	 */
+	@Test
+	public void testTimeoutForUnusedTaskManager() throws Exception {
+		final long taskManagerTimeout = 123456L;
+
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
+		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
+
+		final ResourceID resourceId = ResourceID.generate();
+
+		final JobID jobId = new JobID();
+		final AllocationID allocationId = new AllocationID();
+		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
+		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
+
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			any(SlotID.class),
+			eq(jobId),
+			eq(allocationId),
+			anyString(),
+			eq(leaderId),
+			any(Time.class))).thenReturn(FlinkCompletableFuture.completed(Acknowledge.get()));
+
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
+
+		final SlotID slotId1 = new SlotID(resourceId, 0);
+		final SlotID slotId2 = new SlotID(resourceId, 1);
+		final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile);
+		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
+		final SlotReport initialSlotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
+
+		try (SlotManager slotManager = new SlotManager(
+			scheduledExecutor,
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			Time.of(taskManagerTimeout, TimeUnit.MILLISECONDS))) {
+
+			slotManager.start(leaderId, Executors.directExecutor(), resourceManagerActions);
+
+			slotManager.registerSlotRequest(slotRequest);
+
+			slotManager.registerTaskManager(taskManagerConnection, initialSlotReport);
+
+			ArgumentCaptor<SlotID> slotIdArgumentCaptor = ArgumentCaptor.forClass(SlotID.class);
+
+			verify(taskExecutorGateway).requestSlot(
+				slotIdArgumentCaptor.capture(),
+				eq(jobId),
+				eq(allocationId),
+				anyString(),
+				eq(leaderId),
+				any(Time.class));
+
+			assertFalse(slotManager.hasTimeoutRegistered(taskManagerConnection.getInstanceID()));
+
+			SlotID slotId = slotIdArgumentCaptor.getValue();
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
+
+			assertTrue(slot.isAllocated());
+			assertEquals(allocationId, slot.getAllocationId());
+
+			slotManager.freeSlot(slotId, allocationId);
+
+			assertTrue(slotManager.hasTimeoutRegistered(taskManagerConnection.getInstanceID()));
+
+			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
+
+			// filter out the schedule call for the task manager which will be registered using the
+			// taskManagerTimeout value
+			verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), eq(taskManagerTimeout), eq(TimeUnit.MILLISECONDS));
+
+			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
 
-		List<ResourceProfile> getAllocatedContainers() {
-			return testingRmServices.allocatedContainers;
+			timeoutRunnable.run();
+
+			verify(resourceManagerActions, times(1)).releaseResource(eq(taskManagerConnection.getInstanceID()));
 		}
+	}
 
+	/**
+	 * Tests that the slot manager re-registers a timeout for a rejected slot request.
+	 */
+	@Test
+	public void testTimeoutForRejectedSlotRequest() throws Exception {
 
-		private static class TestingRmServices implements ResourceManagerServices {
+		final long slotRequestTimeout = 1337L;
+		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
 
-			private final UUID leaderID;
+		final ResourceID resourceId = ResourceID.generate();
+		final SlotID slotId = new SlotID(resourceId, 0);
+		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
+		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
+		final SlotReport slotReport = new SlotReport(slotStatus);
 
-			private final List<ResourceProfile> allocatedContainers;
+		final UUID leaderId = UUID.randomUUID();
+		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 
-			public TestingRmServices() {
-				this.leaderID = UUID.randomUUID();
-				this.allocatedContainers = new LinkedList<>();
-			}
+		final JobID jobId = new JobID();
+		final AllocationID allocationId = new AllocationID();
+		final AllocationID allocationId2 = new AllocationID();
+		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
 
-			@Override
-			public UUID getLeaderID() {
-				return leaderID;
-			}
+		CompletableFuture<Acknowledge> requestFuture = new FlinkCompletableFuture<>();
 
-			@Override
-			public void allocateResource(ResourceProfile resourceProfile) {
-				allocatedContainers.add(resourceProfile);
-			}
+		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+		when(taskExecutorGateway.requestSlot(
+			eq(slotId),
+			eq(jobId),
+			eq(allocationId),
+			anyString(),
+			eq(leaderId),
+			any(Time.class))).thenReturn(requestFuture);
 
-			@Override
-			public Executor getAsyncExecutor() {
-				return Mockito.mock(Executor.class);
-			}
+		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
 
-			@Override
-			public Executor getMainThreadExecutor() {
-				return Mockito.mock(Executor.class);
-			}
+		try (SlotManager slotManager = new SlotManager(
+			scheduledExecutor,
+			TestingUtils.infiniteTime(),
+			Time.milliseconds(slotRequestTimeout),
+			TestingUtils.infiniteTime())) {
+
+			slotManager.start(leaderId, Executors.directExecutor(), resourceManagerActions);
+
+			slotManager.registerTaskManager(taskManagerConnection, slotReport);
 
+			slotManager.registerSlotRequest(slotRequest);
+
+			verify(taskExecutorGateway).requestSlot(
+				eq(slotId),
+				eq(jobId),
+				eq(allocationId),
+				anyString(),
+				eq(leaderId),
+				any(Time.class));
+
+			requestFuture.completeExceptionally(new SlotOccupiedException("Slot is already occupied", allocationId2));
+
+			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
+			verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), eq(slotRequestTimeout), eq(TimeUnit.MILLISECONDS));
+
+			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
+
+			timeoutRunnable.run();
+
+			verify(resourceManagerActions).notifyAllocationFailure(eq(jobId), eq(allocationId), any(Exception.class));
+
+			TaskManagerSlot slot = slotManager.getSlot(slotId);
+
+			assertTrue(slot.isAllocated());
+			assertEquals(allocationId2, slot.getAllocationId());
 		}
 	}
+
+	private SlotManager createSlotManager(UUID leaderId, ResourceManagerActions resourceManagerActions) {
+		SlotManager slotManager = new SlotManager(
+			TestingUtils.defaultScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime());
+
+		slotManager.start(leaderId, Executors.directExecutor(), resourceManagerActions);
+
+		return slotManager;
+	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 37690b546f5e2..a72969ee1864b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -19,47 +19,29 @@
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
-import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
-import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
-import org.apache.flink.runtime.heartbeat.HeartbeatServices;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
-import org.apache.flink.runtime.jobmaster.JobMasterGateway;
-import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
-import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
-import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.registration.RegistrationResponse;
-import org.apache.flink.runtime.resourcemanager.JobLeaderIdService;
-import org.apache.flink.runtime.resourcemanager.ResourceManager;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter;
+import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
-import org.apache.flink.runtime.resourcemanager.TestingSlotManager;
-import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
-import org.apache.flink.runtime.rpc.FatalErrorHandler;
-import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.TestingSerialRpcService;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection;
 import org.apache.flink.runtime.taskexecutor.SlotReport;
 import org.apache.flink.runtime.taskexecutor.SlotStatus;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
 
 import java.util.Collections;
 import java.util.UUID;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
 import static org.mockito.Matchers.any;
@@ -70,128 +52,75 @@
 
 public class SlotProtocolTest extends TestLogger {
 
-	private static TestingSerialRpcService testRpcService;
+	private static final long timeout = 10000L;
+
+
+	private static ScheduledExecutorService scheduledExecutorService;
 
 	@BeforeClass
 	public static void beforeClass() {
-		testRpcService = new TestingSerialRpcService();
+		scheduledExecutorService = new ScheduledThreadPoolExecutor(1);
 	}
 
 	@AfterClass
 	public static void afterClass() {
-		testRpcService.stopService();
-		testRpcService = null;
-	}
-
-	@Before
-	public void beforeTest(){
-		testRpcService.clearGateways();
+		Executors.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, scheduledExecutorService);
 	}
 
 	/**
 	 * Tests whether
-	 * 1) SlotRequest is routed to the SlotManager
-	 * 2) SlotRequest is confirmed
-	 * 3) SlotRequest leads to a container allocation
-	 * 4) Slot becomes available and TaskExecutor gets a SlotRequest
+	 * 1) SlotManager accepts a slot request
+	 * 2) SlotRequest leads to a container allocation
+	 * 3) Slot becomes available and TaskExecutor gets a SlotRequest
 	 */
 	@Test
 	public void testSlotsUnavailableRequest() throws Exception {
-		final String rmAddress = "/rm1";
-		final String jmAddress = "/jm1";
 		final JobID jobID = new JobID();
-		final ResourceID rmResourceId = new ResourceID(rmAddress);
 		final ResourceID jmResourceId = new ResourceID(jmAddress);
 
-		testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
-
-		final TestingHighAvailabilityServices testingHaServices = new TestingHighAvailabilityServices();
 		final UUID rmLeaderID = UUID.randomUUID();
-		final UUID jmLeaderID = UUID.randomUUID();
-		TestingLeaderElectionService rmLeaderElectionService =
-			configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
-
-		ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(
-			Time.seconds(5L),
-			Time.seconds(5L));
-
-		JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
-			testingHaServices,
-			testRpcService.getScheduledExecutor(),
-			Time.seconds(5L));
-
-		final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
-
-		final HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
-
-		SpiedResourceManager resourceManager =
-			new SpiedResourceManager(
-				rmResourceId,
-				testRpcService,
-				resourceManagerConfiguration,
-				testingHaServices,
-				heartbeatServices,
-				slotManagerFactory,
-				mock(MetricRegistry.class),
-				jobLeaderIdService,
-				mock(FatalErrorHandler.class));
-		resourceManager.start();
-		rmLeaderElectionService.isLeader(rmLeaderID);
-
-		Future<RegistrationResponse> registrationFuture =
-			resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmResourceId, jmAddress, jobID);
-		try {
-			registrationFuture.get(5, TimeUnit.SECONDS);
-		} catch (Exception e) {
-			Assert.fail("JobManager registration Future didn't become ready.");
-		}
 
-		final SlotManager slotManager = slotManagerFactory.slotManager;
+		try (SlotManager slotManager = new SlotManager(
+			new ScheduledExecutorServiceAdapter(scheduledExecutorService),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime())) {
 
-		final AllocationID allocationID = new AllocationID();
-		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
+			ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 
-		SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
-		RMSlotRequestReply slotRequestReply =
-			resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
+			slotManager.start(rmLeaderID, Executors.directExecutor(), resourceManagerActions);
 
-		// 1) SlotRequest is routed to the SlotManager
-		verify(slotManager).requestSlot(slotRequest);
+			final AllocationID allocationID = new AllocationID();
+			final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
+			final String targetAddress = "foobar";
 
-		// 2) SlotRequest is confirmed
-		Assert.assertEquals(
-			slotRequestReply.getAllocationID(),
-			allocationID);
+			SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile, targetAddress);
 
-		// 3) SlotRequest leads to a container allocation
-		Assert.assertEquals(1, resourceManager.startNewWorkerCalled);
+			slotManager.registerSlotRequest(slotRequest);
 
-		Assert.assertFalse(slotManager.isAllocated(allocationID));
+			verify(resourceManagerActions).allocateResource(eq(slotRequest.getResourceProfile()));
 
-		// slot becomes available
-		final String tmAddress = "/tm1";
-		TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
-		Mockito
-			.when(
+			// slot becomes available
+			TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+			Mockito.when(
 				taskExecutorGateway
 					.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class)))
-			.thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
-		testRpcService.registerGateway(tmAddress, taskExecutorGateway);
-
-		final ResourceID resourceID = ResourceID.generate();
-		final SlotID slotID = new SlotID(resourceID, 0);
-
-		final SlotStatus slotStatus =
-			new SlotStatus(slotID, resourceProfile);
-		final SlotReport slotReport =
-			new SlotReport(Collections.singletonList(slotStatus));
-		// register slot at SlotManager
-		slotManager.registerTaskExecutor(
-			resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
-
-		// 4) Slot becomes available and TaskExecutor gets a SlotRequest
-		verify(taskExecutorGateway, timeout(5000))
-			.requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
+				.thenReturn(mock(FlinkFuture.class));
+
+			final ResourceID resourceID = ResourceID.generate();
+			final SlotID slotID = new SlotID(resourceID, 0);
+
+			final SlotStatus slotStatus =
+				new SlotStatus(slotID, resourceProfile);
+			final SlotReport slotReport =
+				new SlotReport(Collections.singletonList(slotStatus));
+			// register slot at SlotManager
+			slotManager.registerTaskManager(new TaskExecutorConnection(taskExecutorGateway), slotReport);
+
+			// 4) Slot becomes available and TaskExecutor gets a SlotRequest
+			verify(taskExecutorGateway, timeout(5000L))
+				.requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
+		}
 	}
 
 	/**
@@ -203,159 +132,48 @@ public void testSlotsUnavailableRequest() throws Exception {
 	 */
 	@Test
 	public void testSlotAvailableRequest() throws Exception {
-		final String rmAddress = "/rm1";
-		final String jmAddress = "/jm1";
-		final String tmAddress = "/tm1";
 		final JobID jobID = new JobID();
-		final ResourceID rmResourceId = new ResourceID(rmAddress);
 		final ResourceID jmResourceId = new ResourceID(jmAddress);
 
-		testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
-
-		final TestingHighAvailabilityServices testingHaServices = new TestingHighAvailabilityServices();
 		final UUID rmLeaderID = UUID.randomUUID();
-		final UUID jmLeaderID = UUID.randomUUID();
-		TestingLeaderElectionService rmLeaderElectionService =
-			configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
 
 		TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
 		Mockito.when(
 			taskExecutorGateway
 				.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class)))
-			.thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
-		testRpcService.registerGateway(tmAddress, taskExecutorGateway);
-
-		ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(
-			Time.seconds(5L),
-			Time.seconds(5L));
-
-		JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
-			testingHaServices,
-			testRpcService.getScheduledExecutor(),
-			Time.seconds(5L));
-
-		TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
-
-		HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
-
-		ResourceManager<ResourceID> resourceManager =
-			Mockito.spy(new StandaloneResourceManager(
-				testRpcService,
-				FlinkResourceManager.RESOURCE_MANAGER_NAME,
-				rmResourceId,
-				resourceManagerConfiguration,
-				testingHaServices,
-				heartbeatServices,
-				slotManagerFactory,
-				mock(MetricRegistry.class),
-				jobLeaderIdService,
-				mock(FatalErrorHandler.class)));
-		resourceManager.start();
-		rmLeaderElectionService.isLeader(rmLeaderID);
-
-		Thread.sleep(1000);
-
-		Future<RegistrationResponse> registrationFuture =
-			resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmResourceId, jmAddress, jobID);
-		try {
-			registrationFuture.get(5L, TimeUnit.SECONDS);
-		} catch (Exception e) {
-			Assert.fail("JobManager registration Future didn't become ready.");
-		}
-
-		final SlotManager slotManager = slotManagerFactory.slotManager;
+			.thenReturn(mock(FlinkFuture.class));
 
-		final ResourceID resourceID = ResourceID.generate();
-		final AllocationID allocationID = new AllocationID();
-		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
-		final SlotID slotID = new SlotID(resourceID, 0);
+		try (SlotManager slotManager = new SlotManager(
+			new ScheduledExecutorServiceAdapter(scheduledExecutorService),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime())) {
 
-		final SlotStatus slotStatus =
-			new SlotStatus(slotID, resourceProfile);
-		final SlotReport slotReport =
-			new SlotReport(Collections.singletonList(slotStatus));
-		// register slot at SlotManager
-		slotManager.registerTaskExecutor(
-			resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
+			ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 
-		SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
-		RMSlotRequestReply slotRequestReply =
-			resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
+			slotManager.start(rmLeaderID, Executors.directExecutor(), resourceManagerActions);
 
-		// 1) a SlotRequest is routed to the SlotManager
-		verify(slotManager).requestSlot(slotRequest);
+			final ResourceID resourceID = ResourceID.generate();
+			final AllocationID allocationID = new AllocationID();
+			final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
+			final SlotID slotID = new SlotID(resourceID, 0);
 
-		// 2) a SlotRequest is confirmed
-		Assert.assertEquals(
-			slotRequestReply.getAllocationID(),
-			allocationID);
-
-		// 3) a SlotRequest leads to an allocation of a registered slot
-		Assert.assertTrue(slotManager.isAllocated(slotID));
-		Assert.assertTrue(slotManager.isAllocated(allocationID));
-
-		// 4) a SlotRequest is routed to the TaskExecutor
-		verify(taskExecutorGateway, timeout(5000))
-			.requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
-	}
-
-	private static TestingLeaderElectionService configureHA(
-			TestingHighAvailabilityServices testingHA, JobID jobID, String rmAddress, UUID rmID, String jmAddress, UUID jmID) {
-		final TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
-		testingHA.setResourceManagerLeaderElectionService(rmLeaderElectionService);
-		final TestingLeaderRetrievalService rmLeaderRetrievalService = new TestingLeaderRetrievalService(rmAddress, rmID);
-		testingHA.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
-
-		final TestingLeaderElectionService jmLeaderElectionService = new TestingLeaderElectionService();
-		testingHA.setJobMasterLeaderElectionService(jobID, jmLeaderElectionService);
-		final TestingLeaderRetrievalService jmLeaderRetrievalService = new TestingLeaderRetrievalService(jmAddress, jmID);
-		testingHA.setJobMasterLeaderRetriever(jobID, jmLeaderRetrievalService);
-
-		return rmLeaderElectionService;
-	}
-
-	private static class SpiedResourceManager extends StandaloneResourceManager {
-
-		private int startNewWorkerCalled = 0;
-
-		public SpiedResourceManager(
-				ResourceID resourceId,
-				RpcService rpcService,
-				ResourceManagerConfiguration resourceManagerConfiguration,
-				HighAvailabilityServices highAvailabilityServices,
-				HeartbeatServices heartbeatServices,
-				SlotManagerFactory slotManagerFactory,
-				MetricRegistry metricRegistry,
-				JobLeaderIdService jobLeaderIdService,
-				FatalErrorHandler fatalErrorHandler) {
-			super(
-				rpcService,
-				FlinkResourceManager.RESOURCE_MANAGER_NAME,
-				resourceId,
-				resourceManagerConfiguration,
-				highAvailabilityServices,
-				heartbeatServices,
-				slotManagerFactory,
-				metricRegistry,
-				jobLeaderIdService,
-				fatalErrorHandler);
-		}
-
-
-		@Override
-		public void startNewWorker(ResourceProfile resourceProfile) {
-			startNewWorkerCalled++;
-		}
-	}
+			final SlotStatus slotStatus =
+				new SlotStatus(slotID, resourceProfile);
+			final SlotReport slotReport =
+				new SlotReport(Collections.singletonList(slotStatus));
+			// register slot at SlotManager
+			slotManager.registerTaskManager(
+				new TaskExecutorConnection(taskExecutorGateway), slotReport);
 
-	private static class TestingSlotManagerFactory implements SlotManagerFactory {
+			final String targetAddress = "foobar";
 
-		private SlotManager slotManager;
+			SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile, targetAddress);
+			slotManager.registerSlotRequest(slotRequest);
 
-		@Override
-		public SlotManager create(ResourceManagerServices rmServices) {
-			this.slotManager = Mockito.spy(new TestingSlotManager(rmServices));
-			return this.slotManager;
+			// a SlotRequest is routed to the TaskExecutor
+			verify(taskExecutorGateway, timeout(5000))
+				.requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
 		}
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
index 579ca3a9cac0e..6a0bd8712d258 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java
@@ -47,13 +47,13 @@
 import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
 import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.DefaultSlotManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
 import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
 import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
 import org.apache.flink.runtime.taskexecutor.slot.TimerService;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.util.TestingFatalErrorHandler;
 import org.apache.flink.util.TestLogger;
 import org.hamcrest.Matchers;
@@ -102,7 +102,6 @@ public void testSlotAllocation() throws Exception {
 		ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(
 			Time.milliseconds(500L),
 			Time.milliseconds(500L));
-		SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
 		JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
 			testingHAServices,
 			rpcService.getScheduledExecutor(),
@@ -121,6 +120,11 @@ public void testSlotAllocation() throws Exception {
 		final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(resourceProfile), new TimerService<AllocationID>(scheduledExecutorService, 100L));
 		final JobManagerTable jobManagerTable = new JobManagerTable();
 		final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
+		final SlotManager slotManager = new SlotManager(
+			rpcService.getScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime());
 
 		ResourceManager<ResourceID> resourceManager = new StandaloneResourceManager(
 			rpcService,
@@ -129,7 +133,7 @@ public void testSlotAllocation() throws Exception {
 			resourceManagerConfiguration,
 			testingHAServices,
 			heartbeatServices,
-			slotManagerFactory,
+			slotManager,
 			metricRegistry,
 			jobLeaderIdService,
 			testingFatalErrorHandler);
@@ -168,7 +172,7 @@ public void testSlotAllocation() throws Exception {
 		rpcService.registerGateway(jmAddress, jmGateway);
 
 		final AllocationID allocationId = new AllocationID();
-		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile);
+		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, jmAddress);
 		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, resourceProfile);
 
 		try {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 1d1840e8b5ca8..d3d4d43485e29 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -65,10 +65,8 @@
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
 import org.apache.flink.runtime.registration.RegistrationResponse;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
 import org.apache.flink.runtime.rpc.TestingSerialRpcService;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
 import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
 import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
 import org.apache.flink.runtime.taskexecutor.slot.TimerService;
@@ -101,6 +99,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.Mockito.*;
 
 public class TaskExecutorTest extends TestLogger {
@@ -703,10 +702,7 @@ public void testJobLeaderDetection() throws Exception {
 			resourceManagerLeaderRetrievalService.notifyListener(resourceManagerAddress, resourceManagerLeaderId);
 
 			// request slots from the task manager under the given allocation id
-			TMSlotRequestReply reply = taskManager.requestSlot(slotId, jobId, allocationId, jobManagerAddress, resourceManagerLeaderId);
-
-			// this is hopefully successful :-)
-			assertTrue(reply instanceof TMSlotRequestRegistered);
+			taskManager.requestSlot(slotId, jobId, allocationId, jobManagerAddress, resourceManagerLeaderId);
 
 			// now inform the task manager about the new job leader
 			jobManagerLeaderRetrievalService.notifyListener(jobManagerAddress, jobManagerLeaderId);
@@ -820,7 +816,11 @@ public void testSlotAcceptance() throws Exception {
 			// been properly started.
 			jobLeaderService.addJob(jobId, jobManagerAddress);
 
-			verify(resourceManagerGateway).notifySlotAvailable(eq(resourceManagerLeaderId), eq(registrationId), eq(new SlotID(resourceId, 1)));
+			verify(resourceManagerGateway).notifySlotAvailable(
+				eq(resourceManagerLeaderId),
+				eq(registrationId),
+				eq(new SlotID(resourceId, 1)),
+				eq(allocationId2));
 
 			assertTrue(taskSlotTable.existsActiveSlot(jobId, allocationId1));
 			assertFalse(taskSlotTable.existsActiveSlot(jobId, allocationId2));
@@ -903,15 +903,19 @@ public void testRejectAllocationRequestsForOutOfSyncSlots() throws Exception {
 
 			// test that allocating a slot works
 			final SlotID slotID = new SlotID(resourceID, 0);
-			TMSlotRequestReply tmSlotRequestReply = taskManager.requestSlot(slotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
-			assertTrue(tmSlotRequestReply instanceof TMSlotRequestRegistered);
+			taskManager.requestSlot(slotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
 
 			// TODO: Figure out the concrete allocation behaviour between RM and TM. Maybe we don't need the SlotID...
 			// test that we can't allocate slots which are blacklisted due to pending confirmation of the RM
 			final SlotID unconfirmedFreeSlotID = new SlotID(resourceID, 1);
-			TMSlotRequestReply tmSlotRequestReply2 =
+
+			try {
 				taskManager.requestSlot(unconfirmedFreeSlotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
-			assertTrue(tmSlotRequestReply2 instanceof TMSlotRequestRejected);
+
+				fail("The slot request should have failed.");
+			} catch (SlotAllocationException e) {
+				// expected
+			}
 
 			// re-register
 			verify(rmGateway1).registerTaskExecutor(
@@ -920,9 +924,7 @@ public void testRejectAllocationRequestsForOutOfSyncSlots() throws Exception {
 
 			// now we should be successful because the slots status has been synced
 			// test that we can't allocate slots which are blacklisted due to pending confirmation of the RM
-			TMSlotRequestReply tmSlotRequestReply3 =
-				taskManager.requestSlot(unconfirmedFreeSlotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
-			assertTrue(tmSlotRequestReply3 instanceof TMSlotRequestRegistered);
+			taskManager.requestSlot(unconfirmedFreeSlotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
 
 			// check if a concurrent error occurred
 			testingFatalErrorHandler.rethrowError();
@@ -1100,7 +1102,11 @@ public void testSubmitTaskBeforeAcceptSlot() throws Exception {
 			// acknowledge the offered slots
 			offerResultFuture.complete(Collections.singleton(offer1));
 
-			verify(resourceManagerGateway).notifySlotAvailable(eq(resourceManagerLeaderId), eq(registrationId), eq(new SlotID(resourceId, 1)));
+			verify(resourceManagerGateway).notifySlotAvailable(
+				eq(resourceManagerLeaderId),
+				eq(registrationId),
+				eq(new SlotID(resourceId, 1)),
+				any(AllocationID.class));
 
 			assertTrue(taskSlotTable.existsActiveSlot(jobId, allocationId1));
 			assertFalse(taskSlotTable.existsActiveSlot(jobId, allocationId2));
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index f58faf2a3cbd6..03b5172226a25 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -28,11 +28,13 @@ import com.google.common.util.concurrent.MoreExecutors
 import com.typesafe.config.ConfigFactory
 import grizzled.slf4j.Logger
 import org.apache.flink.api.common.JobExecutionResult
+import org.apache.flink.api.common.time.Time
 import org.apache.flink.configuration.{ConfigConstants, Configuration, HighAvailabilityOptions, TaskManagerOptions}
 import org.apache.flink.runtime.akka.AkkaUtils
 import org.apache.flink.runtime.client.JobClient
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager
 import org.apache.flink.runtime.clusterframework.types.ResourceID
+import org.apache.flink.runtime.concurrent.{ScheduledExecutor, ScheduledExecutorServiceAdapter}
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices
 import org.apache.flink.runtime.instance.{ActorGateway, AkkaActorGateway}
 import org.apache.flink.runtime.jobgraph.JobGraph
@@ -78,6 +80,10 @@ object TestingUtils {
   }
 
   def getDefaultTestingActorSystemConfig = testConfig
+
+  def infiniteTime: Time = {
+    Time.milliseconds(Long.MaxValue);
+  }
   
 
   def startTestingCluster(numSlots: Int, numTMs: Int = 1,
@@ -114,6 +120,12 @@ object TestingUtils {
     }
   }
 
+  def defaultScheduledExecutor: ScheduledExecutor = {
+    val scheduledExecutorService = defaultExecutor
+
+    new ScheduledExecutorServiceAdapter(scheduledExecutorService)
+  }
+
   /** Returns an [[ExecutionContext]] which uses the current thread to execute the runnable.
     *
     * @return Direct [[ExecutionContext]] which executes runnables directly
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
index 65d12b5010bee..a6b66d75dd720 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
@@ -208,7 +208,7 @@ private ResourceManager<?> createResourceManager(Configuration config) throws Ex
 			resourceManagerConfiguration,
 			haServices,
 			heartbeatServices,
-			resourceManagerRuntimeServices.getSlotManagerFactory(),
+			resourceManagerRuntimeServices.getSlotManager(),
 			metricRegistry,
 			resourceManagerRuntimeServices.getJobLeaderIdService(),
 			this);
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
index 74359c8f9fc8f..63e6a4c285de3 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
@@ -28,12 +28,13 @@
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.resourcemanager.JobLeaderIdService;
 import org.apache.flink.runtime.resourcemanager.ResourceManager;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
 import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -115,7 +116,7 @@ public YarnResourceManager(
 			ResourceManagerConfiguration resourceManagerConfiguration,
 			HighAvailabilityServices highAvailabilityServices,
 			HeartbeatServices heartbeatServices,
-			SlotManagerFactory slotManagerFactory,
+			SlotManager slotManager,
 			MetricRegistry metricRegistry,
 			JobLeaderIdService jobLeaderIdService,
 			FatalErrorHandler fatalErrorHandler) {
@@ -126,7 +127,7 @@ public YarnResourceManager(
 			resourceManagerConfiguration,
 			highAvailabilityServices,
 			heartbeatServices,
-			slotManagerFactory,
+			slotManager,
 			metricRegistry,
 			jobLeaderIdService,
 			fatalErrorHandler);
@@ -222,6 +223,11 @@ public void startNewWorker(ResourceProfile resourceProfile) {
 		requestYarnContainer(capability, priority);
 	}
 
+	@Override
+	public void stopWorker(InstanceID instanceId) {
+		// TODO: Implement to stop the worker
+	}
+
 	@Override
 	protected ResourceID workerStarted(ResourceID resourceID) {
 		return resourceID;

From 155792ac4b5c698cfcd195478f851a1ff351cf0f Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 16 Mar 2017 18:53:21 +0100
Subject: [PATCH 085/253] [FLINK-5810] [flip-6] Multiple small cleanups across
 Resource Manager related code

---
 .../types/ResourceIDRetrievable.java          |  4 +++-
 .../clusterframework/types/SlotID.java        | 20 ++++++-------------
 .../runtime/concurrent/ScheduledExecutor.java |  2 +-
 .../flink/runtime/messages/Acknowledge.java   |  3 ++-
 .../registration/TaskExecutorConnection.java  |  7 ++++---
 .../slotmanager/PendingSlotRequest.java       | 10 +++++++++-
 .../taskexecutor/TaskExecutorGateway.java     |  2 --
 .../slotmanager/SlotProtocolTest.java         | 16 +++++++--------
 8 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
index b45d53c960fc4..e65840dcdc562 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
@@ -22,6 +22,8 @@
  */
 public interface ResourceIDRetrievable {
 
+	/**
+	 * Gets the ResourceID of the object.
+	 */
 	ResourceID getResourceID();
-
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
index d6409b610fabe..83c28b4cae236 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
@@ -24,7 +24,11 @@
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
- * Unique identifier for a slot which located in TaskManager.
+ * Unique identifier for a slot on a TaskManager. This ID is constant across the
+ * life time of the TaskManager.
+ * 
+ * <p>In contrast, the {@link AllocationID} represents the a slot allocation and changes
+ * every time the slot is allocated by a JobManager.
  */
 public class SlotID implements ResourceIDRetrievable, Serializable {
 
@@ -66,10 +70,7 @@ public boolean equals(Object o) {
 
 		SlotID slotID = (SlotID) o;
 
-		if (slotNumber != slotID.slotNumber) {
-			return false;
-		}
-		return resourceId.equals(slotID.resourceId);
+		return slotNumber == slotID.slotNumber && resourceId.equals(slotID.resourceId);
 	}
 
 	@Override
@@ -83,13 +84,4 @@ public int hashCode() {
 	public String toString() {
 		return resourceId + "_" + slotNumber;
 	}
-
-	/**
-	 * Generate a random slot id.
-	 *
-	 * @return A random slot id.
-	 */
-	public static SlotID generate() {
-		return new SlotID(ResourceID.generate(), 0);
-	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
index c1b47e213253c..d09cfc3e48364 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
@@ -82,7 +82,7 @@ ScheduledFuture<?> scheduleAtFixedRate(
 	 * @param delay the time between the end of the current and the start of the next execution
 	 * @param unit the time unit of the initial delay and the delay parameter
 	 * @return a ScheduledFuture representing the repeatedly executed task. This future never
-	 * completes unless th exectuion of the given task fails or if the future is cancelled
+	 * completes unless the execution of the given task fails or if the future is cancelled
 	 */
 	ScheduledFuture<?> scheduleWithFixedDelay(
 		Runnable command,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/Acknowledge.java b/flink-runtime/src/main/java/org/apache/flink/runtime/messages/Acknowledge.java
index 4bbc50a4666dd..2c08ec94c2f51 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/Acknowledge.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/messages/Acknowledge.java
@@ -62,8 +62,9 @@ public String toString() {
 
 	/**
 	 * Read resolve to preserve the singleton object property.
+	 * (per best practices, this should have visibility 'protected')
 	 */
-	private Object readResolve() throws java.io.ObjectStreamException {
+	protected Object readResolve() throws java.io.ObjectStreamException {
 		return INSTANCE;
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorConnection.java
index e4522f2351da6..babe5b96d0ecb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorConnection.java
@@ -21,6 +21,8 @@
 import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 /**
  * This class is responsible for grouping the TaskExecutorGateway and the InstanceID
  * of a registered task executor.
@@ -29,11 +31,11 @@ public class TaskExecutorConnection {
 
 	private final InstanceID instanceID;
 
-	private TaskExecutorGateway taskExecutorGateway;
+	private final TaskExecutorGateway taskExecutorGateway;
 
 	public TaskExecutorConnection(TaskExecutorGateway taskExecutorGateway) {
 		this.instanceID = new InstanceID();
-		this.taskExecutorGateway = taskExecutorGateway;
+		this.taskExecutorGateway = checkNotNull(taskExecutorGateway);
 	}
 
 	public InstanceID getInstanceID() {
@@ -43,5 +45,4 @@ public InstanceID getInstanceID() {
 	public TaskExecutorGateway getTaskExecutorGateway() {
 		return taskExecutorGateway;
 	}
-
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
index 894f14639ae19..1195791647d44 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
@@ -26,6 +26,7 @@
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
 import org.apache.flink.util.Preconditions;
 
+import javax.annotation.Nullable;
 import java.util.UUID;
 import java.util.concurrent.ScheduledFuture;
 
@@ -33,16 +34,21 @@ public class PendingSlotRequest {
 
 	private final SlotRequest slotRequest;
 
+	@Nullable
 	private CompletableFuture<Acknowledge> requestFuture;
 
+	@Nullable
 	private UUID timeoutIdentifier;
 
+	@Nullable
 	private ScheduledFuture<?> timeoutFuture;
 
 	public PendingSlotRequest(SlotRequest slotRequest) {
 		this.slotRequest = Preconditions.checkNotNull(slotRequest);
 	}
 
+	// ------------------------------------------------------------------------
+
 	public AllocationID getAllocationId() {
 		return slotRequest.getAllocationId();
 	}
@@ -51,6 +57,7 @@ public ResourceProfile getResourceProfile() {
 		return slotRequest.getResourceProfile();
 	}
 
+	@Nullable
 	public UUID getTimeoutIdentifier() {
 		return timeoutIdentifier;
 	}
@@ -67,10 +74,11 @@ public boolean isAssigned() {
 		return null != requestFuture;
 	}
 
-	public void setRequestFuture(CompletableFuture<Acknowledge> requestFuture) {
+	public void setRequestFuture(@Nullable CompletableFuture<Acknowledge> requestFuture) {
 		this.requestFuture = requestFuture;
 	}
 
+	@Nullable
 	public CompletableFuture<Acknowledge> getRequestFuture() {
 		return requestFuture;
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
index bedf8ecc5a114..d4afdbd56f1f0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
@@ -31,7 +31,6 @@
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.rpc.RpcGateway;
 import org.apache.flink.runtime.rpc.RpcTimeout;
-import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
 import org.apache.flink.runtime.taskmanager.Task;
 
 import java.util.UUID;
@@ -47,7 +46,6 @@ public interface TaskExecutorGateway extends RpcGateway {
 	 * @param slotId slot id for the request
 	 * @param allocationId id for the request
 	 * @param resourceManagerLeaderId current leader id of the ResourceManager
-	 * @throws SlotAllocationException if the slot allocation fails
 	 * @return answer to the slot request
 	 */
 	Future<Acknowledge> requestSlot(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index a72969ee1864b..c09316cc9be9e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -24,6 +24,7 @@
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
 import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.concurrent.ScheduledExecutor;
 import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter;
 import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
@@ -33,8 +34,8 @@
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.TestLogger;
+
 import org.junit.AfterClass;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
 
@@ -54,13 +55,12 @@ public class SlotProtocolTest extends TestLogger {
 
 	private static final long timeout = 10000L;
 
+	private static final ScheduledExecutorService scheduledExecutorService = 
+			new ScheduledThreadPoolExecutor(1);
 
-	private static ScheduledExecutorService scheduledExecutorService;
 
-	@BeforeClass
-	public static void beforeClass() {
-		scheduledExecutorService = new ScheduledThreadPoolExecutor(1);
-	}
+	private static final ScheduledExecutor scheduledExecutor = 
+			new ScheduledExecutorServiceAdapter(scheduledExecutorService);
 
 	@AfterClass
 	public static void afterClass() {
@@ -81,7 +81,7 @@ public void testSlotsUnavailableRequest() throws Exception {
 		final UUID rmLeaderID = UUID.randomUUID();
 
 		try (SlotManager slotManager = new SlotManager(
-			new ScheduledExecutorServiceAdapter(scheduledExecutorService),
+			scheduledExecutor,
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime())) {
@@ -144,7 +144,7 @@ public void testSlotAvailableRequest() throws Exception {
 			.thenReturn(mock(FlinkFuture.class));
 
 		try (SlotManager slotManager = new SlotManager(
-			new ScheduledExecutorServiceAdapter(scheduledExecutorService),
+			scheduledExecutor,
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime())) {

From 827dea7718d5763ea46afc130a3e45bdb48cd696 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Thu, 27 Apr 2017 15:53:36 +0200
Subject: [PATCH 086/253] [FLINK-5810] [flip-6] Make slot registration static

---
 .../slotmanager/SlotManager.java              | 76 +++++++++++--------
 .../slotmanager/TaskManagerRegistration.java  | 22 +++---
 .../runtime/taskexecutor/SlotReport.java      |  4 -
 .../slotmanager/SlotManagerTest.java          | 30 ++++----
 4 files changed, 70 insertions(+), 62 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index 31edbf38e6973..f09b73aae72b1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -250,12 +250,33 @@ public void registerTaskManager(final TaskExecutorConnection taskExecutorConnect
 		checkInit();
 
 		// we identify task managers by their instance id
-		if (!taskManagerRegistrations.containsKey(taskExecutorConnection.getInstanceID())) {
-			TaskManagerRegistration taskManagerRegistration = new TaskManagerRegistration(taskExecutorConnection);
+		if (taskManagerRegistrations.containsKey(taskExecutorConnection.getInstanceID())) {
+			reportSlotStatus(taskExecutorConnection.getInstanceID(), initialSlotReport);
+		} else {
+			// first register the TaskManager
+			ArrayList<SlotID> reportedSlots = new ArrayList<>();
+
+			for (SlotStatus slotStatus : initialSlotReport) {
+				reportedSlots.add(slotStatus.getSlotID());
+			}
+
+			TaskManagerRegistration taskManagerRegistration = new TaskManagerRegistration(taskExecutorConnection, reportedSlots);
 			taskManagerRegistrations.put(taskExecutorConnection.getInstanceID(), taskManagerRegistration);
+
+			// next register the new slots
+			for (SlotStatus slotStatus : initialSlotReport) {
+				registerSlot(
+					slotStatus.getSlotID(),
+					slotStatus.getAllocationID(),
+					slotStatus.getResourceProfile(),
+					taskExecutorConnection);
+			}
+
+			if (!anySlotUsed(taskManagerRegistration.getSlots())) {
+				registerTaskManagerTimeout(taskManagerRegistration);
+			}
 		}
 
-		reportSlotStatus(taskExecutorConnection.getInstanceID(), initialSlotReport);
 	}
 
 	/**
@@ -296,31 +317,22 @@ public boolean reportSlotStatus(InstanceID instanceId, SlotReport slotReport) {
 		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(instanceId);
 
 		if (null != taskManagerRegistration) {
-			ArrayList<SlotID> slotsToRemove = new ArrayList<>(taskManagerRegistration.getSlots());
 			boolean idle = true;
 
 			for (SlotStatus slotStatus : slotReport) {
-				if (slotsToRemove.remove(slotStatus.getSlotID())) {
-					// slot which was already registered
-					updateSlot(slotStatus.getSlotID(), slotStatus.getAllocationID());
+
+				// We assume that the slots of a TaskManager don't change over its lifetime and they are registered
+				// once when the TaskManager is registered
+				if (taskManagerRegistration.containsSlot(slotStatus.getSlotID()) && updateSlot(slotStatus.getSlotID(), slotStatus.getAllocationID())) {
+					TaskManagerSlot slot = slots.get(slotStatus.getSlotID());
+					idle &= slot.isFree();
 				} else {
-					// new slot
-					registerSlot(
-						taskManagerRegistration,
-						slotStatus.getSlotID(),
-						slotStatus.getAllocationID(),
-						slotStatus.getResourceProfile(),
-						taskManagerRegistration.getTaskManagerConnection());
+					// sanity check to guarantee that slots of a TaskManager don't change
+					throw new IllegalStateException("Reported a slot status for slot " +  slotStatus.getSlotID() +
+						" which has not been registered.");
 				}
-
-				TaskManagerSlot slot = slots.get(slotStatus.getSlotID());
-
-				idle &= slot.isFree();
 			}
 
-			// remove the slots for which we haven't received a slot status message
-			removeSlots(slotsToRemove);
-
 			if (idle) {
 				// no slot of this task manager is being used --> register timer to free this resource
 				registerTaskManagerTimeout(taskManagerRegistration);
@@ -439,20 +451,21 @@ protected TaskManagerSlot findMatchingSlot(ResourceProfile requestResourceProfil
 	 * the given slot id. The given resource profile defines the available resources for the slot.
 	 * The task manager connection can be used to communicate with the task manager.
 	 *
-	 * @param taskManagerRegistration Task manager for which to register the given slot
 	 * @param slotId identifying the slot on the task manager
 	 * @param allocationId which is currently deployed in the slot
 	 * @param resourceProfile of the slot
 	 * @param taskManagerConnection to communicate with the remote task manager
 	 */
 	private void registerSlot(
-			TaskManagerRegistration taskManagerRegistration,
 			SlotID slotId,
 			AllocationID allocationId,
 			ResourceProfile resourceProfile,
 			TaskExecutorConnection taskManagerConnection) {
 
-		Preconditions.checkNotNull(taskManagerRegistration);
+		if (slots.containsKey(slotId)) {
+			// remove the old slot first
+			removeSlot(slotId);
+		}
 
 		TaskManagerSlot slot = new TaskManagerSlot(
 			slotId,
@@ -462,8 +475,6 @@ private void registerSlot(
 
 		slots.put(slotId, slot);
 
-		taskManagerRegistration.addSlot(slotId);
-
 		if (slot.isFree()) {
 			handleFreeSlot(slot);
 		}
@@ -478,8 +489,9 @@ private void registerSlot(
 	 *
 	 * @param slotId to update
 	 * @param allocationId specifying the current allocation of the slot
+	 * @return True if the slot could be updated; otherwise false
 	 */
-	private void updateSlot(SlotID slotId, AllocationID allocationId) {
+	private boolean updateSlot(SlotID slotId, AllocationID allocationId) {
 		TaskManagerSlot slot = slots.get(slotId);
 
 		if (null != slot) {
@@ -516,8 +528,12 @@ private void updateSlot(SlotID slotId, AllocationID allocationId) {
 					taskManagerRegistration.cancelTimeout();
 				}
 			}
+
+			return true;
 		} else {
 			LOG.debug("Trying to update unknown slot with slot id {}.", slotId);
+
+			return false;
 		}
 	}
 
@@ -670,12 +686,6 @@ private void removeSlot(SlotID slotId) {
 			AllocationID oldAllocationId = slot.getAllocationId();
 
 			fulfilledSlotRequests.remove(oldAllocationId);
-
-			TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(slot.getInstanceId());
-
-			if (null != taskManagerRegistration) {
-				taskManagerRegistration.removeSlot(slotId);
-			}
 		} else {
 			LOG.debug("There was no slot registered with slot id {}.", slotId);
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
index 8e23dbb44615f..3a15cb38ad218 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
@@ -23,8 +23,8 @@
 import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection;
 import org.apache.flink.util.Preconditions;
 
+import java.util.Collection;
 import java.util.HashSet;
-import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ScheduledFuture;
 
@@ -38,10 +38,14 @@ public class TaskManagerRegistration {
 
 	private ScheduledFuture<?> timeoutFuture;
 
-	public TaskManagerRegistration(TaskExecutorConnection taskManagerConnection) {
-		this.taskManagerConnection = Preconditions.checkNotNull(taskManagerConnection);
+	public TaskManagerRegistration(
+		TaskExecutorConnection taskManagerConnection,
+		Collection<SlotID> slots) {
 
-		slots = new HashSet<>(4);
+		this.taskManagerConnection = Preconditions.checkNotNull(taskManagerConnection, "taskManagerConnection");
+		Preconditions.checkNotNull(slots, "slots");
+
+		this.slots = new HashSet<>(slots);
 
 		timeoutIdentifier = null;
 		timeoutFuture = null;
@@ -59,16 +63,12 @@ public UUID getTimeoutIdentifier() {
 		return timeoutIdentifier;
 	}
 
-	public Set<SlotID> getSlots() {
+	public Iterable<SlotID> getSlots() {
 		return slots;
 	}
 
-	public boolean removeSlot(SlotID slotId) {
-		return slots.remove(slotId);
-	}
-
-	public void addSlot(SlotID slotId) {
-		slots.add(slotId);
+	public boolean containsSlot(SlotID slotId) {
+		return slots.contains(slotId);
 	}
 
 	public void cancelTimeout() {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
index 7a9da284570bc..94ecaffa4a820 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
@@ -49,10 +49,6 @@ public SlotReport(final Collection<SlotStatus> slotsStatus) {
 		this.slotsStatus = checkNotNull(slotsStatus);
 	}
 
-	public Collection<SlotStatus> getSlotsStatus() {
-		return slotsStatus;
-	}
-
 	@Override
 	public Iterator<SlotStatus> iterator() {
 		return slotsStatus.iterator();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
index b0b5d326d1fdd..fff28299dc248 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
@@ -587,8 +587,7 @@ public void testReceivingUnknownSlotReport() throws Exception {
 
 	/**
 	 * Tests that slots are updated with respect to the latest incoming slot report. This means that
-	 * slot for which not report has been received will be removed and those for which a report was
-	 * received are updated accordingly.
+	 * slots for which a report was received are updated accordingly.
 	 */
 	@Test
 	public void testUpdateSlotReport() throws Exception {
@@ -601,7 +600,6 @@ public void testUpdateSlotReport() throws Exception {
 		final ResourceID resourceId = ResourceID.generate();
 		final SlotID slotId1 = new SlotID(resourceId, 0);
 		final SlotID slotId2 = new SlotID(resourceId, 1);
-		final SlotID slotId3 = new SlotID(resourceId, 2);
 
 
 		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
@@ -609,10 +607,9 @@ public void testUpdateSlotReport() throws Exception {
 		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
 
 		final SlotStatus newSlotStatus2 = new SlotStatus(slotId2, resourceProfile, jobId, allocationId);
-		final SlotStatus slotStatus3 = new SlotStatus(slotId3, resourceProfile);
 
 		final SlotReport slotReport1 = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
-		final SlotReport slotReport2 = new SlotReport(Arrays.asList(newSlotStatus2, slotStatus3));
+		final SlotReport slotReport2 = new SlotReport(Arrays.asList(newSlotStatus2, slotStatus1));
 
 		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
 		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
@@ -623,20 +620,21 @@ public void testUpdateSlotReport() throws Exception {
 
 			slotManager.registerTaskManager(taskManagerConnection, slotReport1);
 
-			TaskManagerSlot slot = slotManager.getSlot(slotId2);
+			TaskManagerSlot slot1 = slotManager.getSlot(slotId1);
+			TaskManagerSlot slot2 = slotManager.getSlot(slotId2);
 
 			assertTrue(2 == slotManager.getNumberRegisteredSlots());
 
-			assertTrue(slot.isFree());
+			assertTrue(slot1.isFree());
+			assertTrue(slot2.isFree());
 
 			assertTrue(slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), slotReport2));
 
 			assertTrue(2 == slotManager.getNumberRegisteredSlots());
 
 			// the slot manager should have removed slotId1
-			assertNull(slotManager.getSlot(slotId1));
-
-			assertNotNull(slotManager.getSlot(slotId3));
+			assertNotNull(slotManager.getSlot(slotId1));
+			assertNotNull(slotManager.getSlot(slotId2));
 
 			// slotId2 should have been allocated for allocationId
 			assertEquals(allocationId, slotManager.getSlot(slotId2).getAllocationId());
@@ -840,7 +838,6 @@ public void testSlotReportWhileActiveSlotRequest() throws Exception {
 		final ResourceID resourceId = ResourceID.generate();
 		final SlotID slotId1 = new SlotID(resourceId, 0);
 		final SlotID slotId2 = new SlotID(resourceId, 1);
-		final SlotID slotId3 = new SlotID(resourceId, 2);
 		final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile);
 		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
 		final SlotReport slotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
@@ -870,11 +867,16 @@ public void testSlotReportWhileActiveSlotRequest() throws Exception {
 				eq(leaderId),
 				any(Time.class));
 
+			final SlotID requestedSlotdId = slotIdCaptor.getValue();
+			final SlotID freeSlotId = requestedSlotdId.equals(slotId1) ? slotId2 : slotId1;
+
+			assertTrue(slotManager.getSlot(freeSlotId).isFree());
+
 			final SlotStatus newSlotStatus1 = new SlotStatus(slotIdCaptor.getValue(), resourceProfile, new JobID(), new AllocationID());
-			final SlotStatus newSlotStatus2 = new SlotStatus(slotId3, resourceProfile);
+			final SlotStatus newSlotStatus2 = new SlotStatus(freeSlotId, resourceProfile);
 			final SlotReport newSlotReport = new SlotReport(Arrays.asList(newSlotStatus1, newSlotStatus2));
 
-			// this should remove the unused slot, replacing it with slotId3 and retry the pending slot request
+			// this should update the slot with the pending slot request triggering the reassignment of it
 			slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), newSlotReport);
 
 			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
@@ -899,7 +901,7 @@ public void testSlotReportWhileActiveSlotRequest() throws Exception {
 
 			final SlotID requestedSlotId = slotIdCaptor.getValue();
 
-			assertEquals(slotId3, requestedSlotId);
+			assertEquals(slotId2, requestedSlotId);
 
 			TaskManagerSlot slot = slotManager.getSlot(requestedSlotId);
 

From 7b3630b32d0174879738d085af780b837cafcb06 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Thu, 27 Apr 2017 17:46:29 +0200
Subject: [PATCH 087/253] [FLINK-5810] [flip-6] Use single timeout task for
 SlotManager

---
 .../slotmanager/PendingSlotRequest.java       |  35 +--
 .../slotmanager/SlotManager.java              | 219 +++++++------
 .../slotmanager/TaskManagerRegistration.java  |  38 +--
 .../clusterframework/ResourceManagerTest.java |   8 +-
 .../slotmanager/SlotManagerTest.java          | 291 +++++++++---------
 .../slotmanager/SlotProtocolTest.java         |   2 -
 .../src/test/resources/log4j-test.properties  |   2 +-
 .../runtime/testingUtils/TestingUtils.scala   |   4 +-
 8 files changed, 295 insertions(+), 304 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
index 1195791647d44..ffe1bfc006cca 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/PendingSlotRequest.java
@@ -27,8 +27,6 @@
 import org.apache.flink.util.Preconditions;
 
 import javax.annotation.Nullable;
-import java.util.UUID;
-import java.util.concurrent.ScheduledFuture;
 
 public class PendingSlotRequest {
 
@@ -37,14 +35,12 @@ public class PendingSlotRequest {
 	@Nullable
 	private CompletableFuture<Acknowledge> requestFuture;
 
-	@Nullable
-	private UUID timeoutIdentifier;
-
-	@Nullable
-	private ScheduledFuture<?> timeoutFuture;
+	/** Timestamp when this pending slot request has been created. */
+	private final long creationTimestamp;
 
 	public PendingSlotRequest(SlotRequest slotRequest) {
 		this.slotRequest = Preconditions.checkNotNull(slotRequest);
+		creationTimestamp = System.currentTimeMillis();
 	}
 
 	// ------------------------------------------------------------------------
@@ -57,11 +53,6 @@ public ResourceProfile getResourceProfile() {
 		return slotRequest.getResourceProfile();
 	}
 
-	@Nullable
-	public UUID getTimeoutIdentifier() {
-		return timeoutIdentifier;
-	}
-
 	public JobID getJobId() {
 		return slotRequest.getJobId();
 	}
@@ -70,6 +61,10 @@ public String getTargetAddress() {
 		return slotRequest.getTargetAddress();
 	}
 
+	public long getCreationTimestamp() {
+		return creationTimestamp;
+	}
+
 	public boolean isAssigned() {
 		return null != requestFuture;
 	}
@@ -82,20 +77,4 @@ public void setRequestFuture(@Nullable CompletableFuture<Acknowledge> requestFut
 	public CompletableFuture<Acknowledge> getRequestFuture() {
 		return requestFuture;
 	}
-
-	public void cancelTimeout() {
-		if (timeoutFuture != null) {
-			timeoutFuture.cancel(true);
-
-			timeoutIdentifier = null;
-			timeoutFuture = null;
-		}
-	}
-
-	public void registerTimeout(ScheduledFuture<?> newTimeoutFuture, UUID newTimeoutIdentifier) {
-		cancelTimeout();
-
-		timeoutFuture = newTimeoutFuture;
-		timeoutIdentifier = newTimeoutIdentifier;
-	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index f09b73aae72b1..829a06dfb5c55 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -64,11 +64,10 @@
  * {@link ResourceManagerActions#allocateResource(ResourceProfile)}.
  *
  * In order to free resources and avoid resource leaks, idling task managers (task managers whose
- * slots are currently not used) and not fulfilled pending slot requests time out triggering their
- * release and failure, respectively.
+ * slots are currently not used) and pending slot requests time out triggering their release and
+ * failure, respectively.
  */
 public class SlotManager implements AutoCloseable {
-
 	private static final Logger LOG = LoggerFactory.getLogger(SlotManager.class);
 
 	/** Scheduled executor for timeouts */
@@ -107,6 +106,10 @@ public class SlotManager implements AutoCloseable {
 	/** Callbacks for resource (de-)allocations */
 	private ResourceManagerActions resourceManagerActions;
 
+	private ScheduledFuture<?> taskManagerTimeoutCheck;
+
+	private ScheduledFuture<?> slotRequestTimeoutCheck;
+
 	/** True iff the component has been started */
 	private boolean started;
 
@@ -128,6 +131,10 @@ public SlotManager(
 
 		leaderId = null;
 		resourceManagerActions = null;
+		mainThreadExecutor = null;
+		taskManagerTimeoutCheck = null;
+		slotRequestTimeoutCheck = null;
+
 		started = false;
 	}
 
@@ -142,17 +149,52 @@ public SlotManager(
 	 * @param newResourceManagerActions to use for resource (de-)allocations
 	 */
 	public void start(UUID newLeaderId, Executor newMainThreadExecutor, ResourceManagerActions newResourceManagerActions) {
+		LOG.info("Starting the SlotManager.");
+
 		leaderId = Preconditions.checkNotNull(newLeaderId);
 		mainThreadExecutor = Preconditions.checkNotNull(newMainThreadExecutor);
 		resourceManagerActions = Preconditions.checkNotNull(newResourceManagerActions);
 
 		started = true;
+
+		taskManagerTimeoutCheck = scheduledExecutor.scheduleWithFixedDelay(new Runnable() {
+			@Override
+			public void run() {
+				mainThreadExecutor.execute(new Runnable() {
+					@Override
+					public void run() {
+						checkTaskManagerTimeouts();
+					}
+				});
+			}
+		}, 0L, taskManagerTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
+
+		slotRequestTimeoutCheck = scheduledExecutor.scheduleWithFixedDelay(new Runnable() {
+			@Override
+			public void run() {
+				mainThreadExecutor.execute(new Runnable() {
+					@Override
+					public void run() {
+						checkSlotRequestTimeouts();
+					}
+				});
+			}
+		}, 0L, slotRequestTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
 	}
 
 	/**
 	 * Suspends the component. This clears the internal state of the slot manager.
 	 */
 	public void suspend() {
+		LOG.info("Suspending the SlotManager.");
+
+		// stop the timeout checks for the TaskManagers and the SlotRequests
+		taskManagerTimeoutCheck.cancel(false);
+		slotRequestTimeoutCheck.cancel(false);
+
+		taskManagerTimeoutCheck = null;
+		slotRequestTimeoutCheck = null;
+
 		for (PendingSlotRequest pendingSlotRequest : pendingSlotRequests.values()) {
 			cancelPendingSlotRequest(pendingSlotRequest);
 		}
@@ -177,6 +219,8 @@ public void suspend() {
 	 */
 	@Override
 	public void close() throws Exception {
+		LOG.info("Closing the SlotManager.");
+
 		suspend();
 	}
 
@@ -249,6 +293,8 @@ public boolean unregisterSlotRequest(AllocationID allocationId) {
 	public void registerTaskManager(final TaskExecutorConnection taskExecutorConnection, SlotReport initialSlotReport) {
 		checkInit();
 
+		LOG.info("Register TaskManager {} at the SlotManager.", taskExecutorConnection.getInstanceID());
+
 		// we identify task managers by their instance id
 		if (taskManagerRegistrations.containsKey(taskExecutorConnection.getInstanceID())) {
 			reportSlotStatus(taskExecutorConnection.getInstanceID(), initialSlotReport);
@@ -272,8 +318,13 @@ public void registerTaskManager(final TaskExecutorConnection taskExecutorConnect
 					taskExecutorConnection);
 			}
 
-			if (!anySlotUsed(taskManagerRegistration.getSlots())) {
-				registerTaskManagerTimeout(taskManagerRegistration);
+			// determine if the task manager is idle or not
+			boolean idle = !anySlotUsed(taskManagerRegistration.getSlots());
+
+			if (idle) {
+				taskManagerRegistration.markIdle();
+			} else {
+				taskManagerRegistration.markUsed();
 			}
 		}
 
@@ -292,9 +343,7 @@ public boolean unregisterTaskManager(InstanceID instanceId) {
 		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.remove(instanceId);
 
 		if (null != taskManagerRegistration) {
-			removeSlots(taskManagerRegistration.getSlots());
-
-			taskManagerRegistration.cancelTimeout();
+			internalUnregisterTaskManager(taskManagerRegistration);
 
 			return true;
 		} else {
@@ -334,8 +383,11 @@ public boolean reportSlotStatus(InstanceID instanceId, SlotReport slotReport) {
 			}
 
 			if (idle) {
-				// no slot of this task manager is being used --> register timer to free this resource
-				registerTaskManagerTimeout(taskManagerRegistration);
+				// no slot of this task manager is being used --> mark this task manager to be idle which allows it to
+				// time out
+				taskManagerRegistration.markIdle();
+			} else {
+				taskManagerRegistration.markUsed();
 			}
 
 			return true;
@@ -371,9 +423,14 @@ public void freeSlot(SlotID slotId, AllocationID allocationId) {
 
 					TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(slot.getInstanceId());
 
-					if (null != taskManagerRegistration && !anySlotUsed(taskManagerRegistration.getSlots())) {
-						registerTaskManagerTimeout(taskManagerRegistration);
+					if (null != taskManagerRegistration) {
+						if (anySlotUsed(taskManagerRegistration.getSlots())) {
+							taskManagerRegistration.markUsed();
+						} else {
+							taskManagerRegistration.markIdle();
+						}
 					}
+
 				} else {
 					LOG.debug("Received request to free slot {} with expected allocation id {}, " +
 						"but actual allocation id {} differs. Ignoring the request.", slotId, allocationId, slot.getAllocationId());
@@ -524,8 +581,8 @@ private boolean updateSlot(SlotID slotId, AllocationID allocationId) {
 				TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(slot.getInstanceId());
 
 				if (null != taskManagerRegistration) {
-					// disable any registered time out for the task manager
-					taskManagerRegistration.cancelTimeout();
+					// mark this TaskManager to be used to exempt it from timing out
+					taskManagerRegistration.markUsed();
 				}
 			}
 
@@ -551,24 +608,6 @@ private void internalRequestSlot(PendingSlotRequest pendingSlotRequest) throws R
 		if (taskManagerSlot != null) {
 			allocateSlot(taskManagerSlot, pendingSlotRequest);
 		} else {
-			final UUID timeoutIdentifier = UUID.randomUUID();
-			final AllocationID allocationId = pendingSlotRequest.getAllocationId();
-
-			// register timeout for slot request
-			ScheduledFuture<?> timeoutFuture = scheduledExecutor.schedule(new Runnable() {
-				@Override
-				public void run() {
-					mainThreadExecutor.execute(new Runnable() {
-						@Override
-						public void run() {
-							timeoutSlotRequest(allocationId, timeoutIdentifier);
-						}
-					});
-				}
-			}, slotRequestTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
-
-			pendingSlotRequest.registerTimeout(timeoutFuture, timeoutIdentifier);
-
 			resourceManagerActions.allocateResource(pendingSlotRequest.getResourceProfile());
 		}
 	}
@@ -591,6 +630,16 @@ private void allocateSlot(TaskManagerSlot taskManagerSlot, PendingSlotRequest pe
 		taskManagerSlot.setAssignedSlotRequest(pendingSlotRequest);
 		pendingSlotRequest.setRequestFuture(completableFuture);
 
+		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(taskManagerSlot.getInstanceId());
+
+		if (taskManagerRegistration != null) {
+			// mark the task manager to be used since we have a pending slot request assigned ot one of its slots
+			taskManagerRegistration.markUsed();
+		} else {
+			throw new IllegalStateException("Could not find a registered task manager for instance id " +
+				taskManagerSlot.getInstanceId() + '.');
+		}
+
 		// RPC call to the task manager
 		Future<Acknowledge> requestFuture = gateway.requestSlot(
 			slotId,
@@ -717,7 +766,7 @@ private void removeSlotRequestFromSlot(SlotID slotId, AllocationID allocationId)
 			TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(taskManagerSlot.getInstanceId());
 
 			if (null != taskManagerRegistration && !anySlotUsed(taskManagerRegistration.getSlots())) {
-				registerTaskManagerTimeout(taskManagerRegistration);
+				taskManagerRegistration.markIdle();
 			}
 		} else {
 			LOG.debug("There was no slot with {} registered. Probably this slot has been already freed.", slotId);
@@ -778,8 +827,6 @@ private void rejectPendingSlotRequest(PendingSlotRequest pendingSlotRequest, Exc
 	 * @param pendingSlotRequest to cancel
 	 */
 	private void cancelPendingSlotRequest(PendingSlotRequest pendingSlotRequest) {
-		pendingSlotRequest.cancelTimeout();
-
 		CompletableFuture<Acknowledge> request = pendingSlotRequest.getRequestFuture();
 
 		if (null != request) {
@@ -791,54 +838,50 @@ private void cancelPendingSlotRequest(PendingSlotRequest pendingSlotRequest) {
 	// Internal timeout methods
 	// ---------------------------------------------------------------------------------------------
 
-	private void timeoutTaskManager(InstanceID instanceId, UUID timeoutIdentifier) {
-		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.remove(instanceId);
+	private void checkTaskManagerTimeouts() {
+		if (!taskManagerRegistrations.isEmpty()) {
+			long currentTime = System.currentTimeMillis();
+
+			Iterator<Map.Entry<InstanceID, TaskManagerRegistration>> taskManagerRegistrationIterator = taskManagerRegistrations.entrySet().iterator();
+
+			while (taskManagerRegistrationIterator.hasNext()) {
+				TaskManagerRegistration taskManagerRegistration = taskManagerRegistrationIterator.next().getValue();
 
-		if (null != taskManagerRegistration) {
-			if (Objects.equals(timeoutIdentifier, taskManagerRegistration.getTimeoutIdentifier())) {
 				if (anySlotUsed(taskManagerRegistration.getSlots())) {
-					LOG.debug("Cannot release the task manager with instance id {}, because some " +
-						"of its slots are still being used.", instanceId);
-				} else {
-					unregisterTaskManager(instanceId);
+					taskManagerRegistration.markUsed();
+				} else if (currentTime - taskManagerRegistration.getIdleSince() >= taskManagerTimeout.toMilliseconds()) {
+					taskManagerRegistrationIterator.remove();
 
-					resourceManagerActions.releaseResource(instanceId);
-				}
-			} else {
-				taskManagerRegistrations.put(instanceId, taskManagerRegistration);
+					internalUnregisterTaskManager(taskManagerRegistration);
 
-				LOG.debug("Expected timeout identifier {} differs from the task manager's " +
-					"timeout identifier {}. Ignoring the task manager timeout call.",
-					timeoutIdentifier, taskManagerRegistration.getTimeoutIdentifier());
+					resourceManagerActions.releaseResource(taskManagerRegistration.getInstanceId());
+				}
 			}
-		} else {
-			LOG.debug("Could not find a registered task manager with instance id {}. Ignoring the task manager timeout call.", instanceId);
 		}
 	}
 
-	private void timeoutSlotRequest(AllocationID allocationId, UUID timeoutIdentifier) {
-		PendingSlotRequest pendingSlotRequest = pendingSlotRequests.remove(allocationId);
+	private void checkSlotRequestTimeouts() {
+		if (!pendingSlotRequests.isEmpty()) {
+			long currentTime = System.currentTimeMillis();
 
-		if (null != pendingSlotRequest) {
-			if (Objects.equals(timeoutIdentifier, pendingSlotRequest.getTimeoutIdentifier())) {
-				if (!pendingSlotRequest.isAssigned()) {
+			Iterator<Map.Entry<AllocationID, PendingSlotRequest>> slotRequestIterator = pendingSlotRequests.entrySet().iterator();
+
+			while (slotRequestIterator.hasNext()) {
+				PendingSlotRequest slotRequest = slotRequestIterator.next().getValue();
+
+				if (currentTime - slotRequest.getCreationTimestamp() >= slotRequestTimeout.toMilliseconds()) {
+					slotRequestIterator.remove();
+
+					if (slotRequest.isAssigned()) {
+						cancelPendingSlotRequest(slotRequest);
+					}
 
 					resourceManagerActions.notifyAllocationFailure(
-						pendingSlotRequest.getJobId(),
-						allocationId,
+						slotRequest.getJobId(),
+						slotRequest.getAllocationId(),
 						new TimeoutException("The allocation could not be fulfilled in time."));
-				} else {
-					LOG.debug("Cannot fail pending slot request {} because it has been assigned.", allocationId);
 				}
-			} else {
-				pendingSlotRequests.put(allocationId, pendingSlotRequest);
-
-				LOG.debug("Expected timeout identifier {} differs from the pending slot request's " +
-					"timeout identifier {}. Ignoring the slot request timeout call.",
-					timeoutIdentifier, pendingSlotRequest.getTimeoutIdentifier());
 			}
-		} else {
-			LOG.debug("Could not find pending slot request with allocation id {}. Ignoring the slot request timeout call.", allocationId);
 		}
 	}
 
@@ -846,6 +889,12 @@ private void timeoutSlotRequest(AllocationID allocationId, UUID timeoutIdentifie
 	// Internal utility methods
 	// ---------------------------------------------------------------------------------------------
 
+	private void internalUnregisterTaskManager(TaskManagerRegistration taskManagerRegistration) {
+		Preconditions.checkNotNull(taskManagerRegistration);
+
+		removeSlots(taskManagerRegistration.getSlots());
+	}
+
 	private boolean checkDuplicateRequest(AllocationID allocationId) {
 		return pendingSlotRequests.containsKey(allocationId) || fulfilledSlotRequests.containsKey(allocationId);
 	}
@@ -853,38 +902,18 @@ private boolean checkDuplicateRequest(AllocationID allocationId) {
 	private boolean anySlotUsed(Iterable<SlotID> slotsToCheck) {
 
 		if (null != slotsToCheck) {
-			boolean idle = true;
-
 			for (SlotID slotId : slotsToCheck) {
 				TaskManagerSlot taskManagerSlot = slots.get(slotId);
 
 				if (null != taskManagerSlot) {
-					idle &= taskManagerSlot.isFree();
+					if (taskManagerSlot.isAllocated()) {
+						return true;
+					}
 				}
 			}
-
-			return !idle;
-		} else {
-			return false;
 		}
-	}
-
-	private void registerTaskManagerTimeout(final TaskManagerRegistration taskManagerRegistration) {
-		final UUID timeoutIdentifier = UUID.randomUUID();
-
-		ScheduledFuture<?> timeoutFuture = scheduledExecutor.schedule(new Runnable() {
-			@Override
-			public void run() {
-				mainThreadExecutor.execute(new Runnable() {
-					@Override
-					public void run() {
-						timeoutTaskManager(taskManagerRegistration.getInstanceId(), timeoutIdentifier);
-					}
-				});
-			}
-		}, taskManagerTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
 
-		taskManagerRegistration.registerTimeout(timeoutFuture, timeoutIdentifier);
+		return false;
 	}
 
 	private void checkInit() {
@@ -911,11 +940,11 @@ PendingSlotRequest getSlotRequest(AllocationID allocationId) {
 	}
 
 	@VisibleForTesting
-	boolean hasTimeoutRegistered(InstanceID instanceId) {
+	boolean isTaskManagerIdle(InstanceID instanceId) {
 		TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(instanceId);
 
 		if (null != taskManagerRegistration) {
-			return taskManagerRegistration.getTimeoutIdentifier() != null;
+			return taskManagerRegistration.isIdle();
 		} else {
 			return false;
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
index 3a15cb38ad218..7d3764c792c7d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/TaskManagerRegistration.java
@@ -25,8 +25,6 @@
 
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.UUID;
-import java.util.concurrent.ScheduledFuture;
 
 public class TaskManagerRegistration {
 
@@ -34,9 +32,8 @@ public class TaskManagerRegistration {
 
 	private final HashSet<SlotID> slots;
 
-	private UUID timeoutIdentifier;
-
-	private ScheduledFuture<?> timeoutFuture;
+	/** Timestamp when the last time becoming idle. Otherwise Long.MAX_VALUE. */
+	private long idleSince;
 
 	public TaskManagerRegistration(
 		TaskExecutorConnection taskManagerConnection,
@@ -47,8 +44,7 @@ public TaskManagerRegistration(
 
 		this.slots = new HashSet<>(slots);
 
-		timeoutIdentifier = null;
-		timeoutFuture = null;
+		idleSince = Long.MAX_VALUE;
 	}
 
 	public TaskExecutorConnection getTaskManagerConnection() {
@@ -59,31 +55,27 @@ public InstanceID getInstanceId() {
 		return taskManagerConnection.getInstanceID();
 	}
 
-	public UUID getTimeoutIdentifier() {
-		return timeoutIdentifier;
-	}
-
 	public Iterable<SlotID> getSlots() {
 		return slots;
 	}
 
-	public boolean containsSlot(SlotID slotId) {
-		return slots.contains(slotId);
+	public long getIdleSince() {
+		return idleSince;
 	}
 
-	public void cancelTimeout() {
-		if (null != timeoutFuture) {
-			timeoutFuture.cancel(false);
+	public boolean isIdle() {
+		return idleSince != Long.MAX_VALUE;
+	}
 
-			timeoutFuture = null;
-			timeoutIdentifier = null;
-		}
+	public void markIdle() {
+		idleSince = System.currentTimeMillis();
 	}
 
-	public void registerTimeout(ScheduledFuture<?> newTimeoutFuture, UUID newTimeoutIdentifier) {
-		cancelTimeout();
+	public void markUsed() {
+		idleSince = Long.MAX_VALUE;
+	}
 
-		timeoutFuture = newTimeoutFuture;
-		timeoutIdentifier = newTimeoutIdentifier;
+	public boolean containsSlot(SlotID slotId) {
+		return slots.contains(slotId);
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
index 41c2e1661cdf8..c7405187d3bb8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
@@ -488,13 +488,17 @@ public void testHeartbeatTimeoutWithJobManager() throws Exception {
 		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
 		final HeartbeatServices heartbeatServices = new TestingHeartbeatServices(heartbeatInterval, heartbeatTimeout, scheduledExecutor);
 
-		final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
 		final MetricRegistry metricRegistry = mock(MetricRegistry.class);
 		final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
 			highAvailabilityServices,
 			rpcService.getScheduledExecutor(),
 			Time.minutes(5L));
 		final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+		final SlotManager slotManager = new SlotManager(
+			TestingUtils.defaultScheduledExecutor(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime());
 
 		try {
 			final StandaloneResourceManager resourceManager = new StandaloneResourceManager(
@@ -504,7 +508,7 @@ public void testHeartbeatTimeoutWithJobManager() throws Exception {
 				resourceManagerConfiguration,
 				highAvailabilityServices,
 				heartbeatServices,
-				slotManagerFactory,
+				slotManager,
 				metricRegistry,
 				jobLeaderIdService,
 				testingFatalErrorHandler);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
index fff28299dc248..39c5f25735156 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
@@ -25,10 +25,9 @@
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.clusterframework.types.SlotID;
 import org.apache.flink.runtime.clusterframework.types.TaskManagerSlot;
-import org.apache.flink.runtime.concurrent.CompletableFuture;
-import org.apache.flink.runtime.concurrent.Executors;
-import org.apache.flink.runtime.concurrent.ScheduledExecutor;
+import org.apache.flink.runtime.concurrent.*;
 import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
 import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.resourcemanager.SlotRequest;
@@ -38,7 +37,6 @@
 import org.apache.flink.runtime.taskexecutor.SlotStatus;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
-import org.apache.flink.runtime.taskexecutor.exceptions.SlotOccupiedException;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
@@ -46,9 +44,11 @@
 
 import java.util.Arrays;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -648,7 +648,7 @@ public void testUpdateSlotReport() throws Exception {
 	 */
 	@Test
 	public void testTaskManagerTimeout() throws Exception {
-		final long tmTimeout = 50L;
+		final long tmTimeout = 500L;
 
 		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 		final UUID leaderId = UUID.randomUUID();
@@ -661,7 +661,7 @@ public void testTaskManagerTimeout() throws Exception {
 		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
 		final SlotReport slotReport = new SlotReport(slotStatus);
 
-		final Executor mainThreadExecutor = mock(Executor.class);
+		final Executor mainThreadExecutor = TestingUtils.defaultExecutor();
 
 		try (SlotManager slotManager = new SlotManager(
 			TestingUtils.defaultScheduledExecutor(),
@@ -671,24 +671,21 @@ public void testTaskManagerTimeout() throws Exception {
 
 			slotManager.start(leaderId, mainThreadExecutor, resourceManagerActions);
 
-			slotManager.registerTaskManager(taskManagerConnection, slotReport);
-
-			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
-
-			verify(mainThreadExecutor, timeout(tmTimeout * 10L)).execute(runnableArgumentCaptor.capture());
+			mainThreadExecutor.execute(new Runnable() {
+				@Override
+				public void run() {
+					slotManager.registerTaskManager(taskManagerConnection, slotReport);
+				}
+			});
 
-			// the only runnable being executed by the main thread executor should be the timeout runnable
-			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
-
-			timeoutRunnable.run();
-
-			verify(resourceManagerActions, times(1)).releaseResource(eq(taskManagerConnection.getInstanceID()));
+			verify(resourceManagerActions, timeout(100L * tmTimeout).times(1))
+				.releaseResource(eq(taskManagerConnection.getInstanceID()));
 		}
 	}
 
 	/**
 	 * Tests that slot requests time out after the specified request timeout. If a slot request
-	 * times out, then the request is cancelled, removed from the slot manager and the resourc
+	 * times out, then the request is cancelled, removed from the slot manager and the resource
 	 * manager is notified about the failed allocation.
 	 */
 	@Test
@@ -703,7 +700,7 @@ public void testSlotRequestTimeout() throws Exception {
 		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
 		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
 
-		final Executor mainThreadExecutor = mock(Executor.class);
+		final Executor mainThreadExecutor = TestingUtils.defaultExecutor();
 
 		try (SlotManager slotManager = new SlotManager(
 			TestingUtils.defaultScheduledExecutor(),
@@ -713,21 +710,27 @@ public void testSlotRequestTimeout() throws Exception {
 
 			slotManager.start(leaderId, mainThreadExecutor, resourceManagerActions);
 
-			assertTrue(slotManager.registerSlotRequest(slotRequest));
-
-			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
-
-			verify(mainThreadExecutor, timeout(allocationTimeout * 10L)).execute(runnableArgumentCaptor.capture());
-
-			// the only runnable being executed by the main thread executor should be the timeout runnable
-			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
+			final AtomicReference<Exception> atomicException = new AtomicReference<>(null);
 
-			timeoutRunnable.run();
+			mainThreadExecutor.execute(new Runnable() {
+				@Override
+				public void run() {
+					try {
+						assertTrue(slotManager.registerSlotRequest(slotRequest));
+					} catch (Exception e) {
+						atomicException.compareAndSet(null, e);
+					}
+				}
+			});
 
-			verify(resourceManagerActions, times(1)).notifyAllocationFailure(
+			verify(resourceManagerActions, timeout(100L * allocationTimeout).times(1)).notifyAllocationFailure(
 				eq(jobId),
 				eq(allocationId),
 				any(TimeoutException.class));
+
+			if (atomicException.get() != null) {
+				throw atomicException.get();
+			}
 		}
 	}
 
@@ -815,6 +818,7 @@ public void testTaskManagerSlotRequestTimeoutHandling() throws Exception {
 	@Test
 	@SuppressWarnings("unchecked")
 	public void testSlotReportWhileActiveSlotRequest() throws Exception {
+		final long verifyTimeout = 1000L;
 		final UUID leaderId = UUID.randomUUID();
 		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
 
@@ -842,20 +846,37 @@ public void testSlotReportWhileActiveSlotRequest() throws Exception {
 		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
 		final SlotReport slotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
 
-		// we have to manually trigger the future call backs to simulate the main thread executor behaviour
-		final Executor mainThreadExecutorMock = mock(Executor.class);
+		final Executor mainThreadExecutor = TestingUtils.defaultExecutor();
 
-		try (SlotManager slotManager = new SlotManager(
+		try (final SlotManager slotManager = new SlotManager(
 			TestingUtils.defaultScheduledExecutor(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime())) {
 
-			slotManager.start(leaderId, mainThreadExecutorMock, resourceManagerActions);
-
-			slotManager.registerTaskManager(taskManagerConnection, slotReport);
+			slotManager.start(leaderId, mainThreadExecutor, resourceManagerActions);
 
-			slotManager.registerSlotRequest(slotRequest);
+			Future<Void> registrationFuture = FlinkFuture.supplyAsync(new Callable<Void>() {
+				@Override
+				public Void call() throws Exception {
+					slotManager.registerTaskManager(taskManagerConnection, slotReport);
+
+					return null;
+				}
+			}, mainThreadExecutor)
+			.thenAccept(new AcceptFunction<Void>() {
+				@Override
+				public void accept(Void value) {
+					try {
+						slotManager.registerSlotRequest(slotRequest);
+					} catch (SlotManagerException e) {
+						throw new RuntimeException("Could not register slots.", e);
+					}
+				}
+			});
+
+			// check that no exception has been thrown
+			registrationFuture.get();
 
 			ArgumentCaptor<SlotID> slotIdCaptor = ArgumentCaptor.forClass(SlotID.class);
 
@@ -867,26 +888,33 @@ public void testSlotReportWhileActiveSlotRequest() throws Exception {
 				eq(leaderId),
 				any(Time.class));
 
-			final SlotID requestedSlotdId = slotIdCaptor.getValue();
-			final SlotID freeSlotId = requestedSlotdId.equals(slotId1) ? slotId2 : slotId1;
+			final SlotID requestedSlotId = slotIdCaptor.getValue();
+			final SlotID freeSlotId = requestedSlotId.equals(slotId1) ? slotId2 : slotId1;
+
+			Future<Boolean> freeSlotFuture = FlinkFuture.supplyAsync(new Callable<Boolean>() {
+				@Override
+				public Boolean call() throws Exception {
+					return slotManager.getSlot(freeSlotId).isFree();
+				}
+			}, mainThreadExecutor);
 
-			assertTrue(slotManager.getSlot(freeSlotId).isFree());
+			assertTrue(freeSlotFuture.get());
 
 			final SlotStatus newSlotStatus1 = new SlotStatus(slotIdCaptor.getValue(), resourceProfile, new JobID(), new AllocationID());
 			final SlotStatus newSlotStatus2 = new SlotStatus(freeSlotId, resourceProfile);
 			final SlotReport newSlotReport = new SlotReport(Arrays.asList(newSlotStatus1, newSlotStatus2));
 
-			// this should update the slot with the pending slot request triggering the reassignment of it
-			slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), newSlotReport);
+			FlinkFuture.supplyAsync(new Callable<Void>() {
+				@Override
+				public Void call() throws Exception {
+					// this should update the slot with the pending slot request triggering the reassignment of it
+					slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), newSlotReport);
 
-			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
-			verify(mainThreadExecutorMock).execute(runnableArgumentCaptor.capture());
+					return null;
+				}
+			}, mainThreadExecutor);
 
-			Runnable requestFailureRunnable = runnableArgumentCaptor.getValue();
-
-			requestFailureRunnable.run();
-
-			verify(taskExecutorGateway, times(2)).requestSlot(
+			verify(taskExecutorGateway, timeout(verifyTimeout).times(2)).requestSlot(
 				slotIdCaptor.capture(),
 				eq(jobId),
 				eq(allocationId),
@@ -894,16 +922,18 @@ public void testSlotReportWhileActiveSlotRequest() throws Exception {
 				eq(leaderId),
 				any(Time.class));
 
-			verify(mainThreadExecutorMock, times(2)).execute(runnableArgumentCaptor.capture());
-			Runnable requestSuccessRunnable = runnableArgumentCaptor.getValue();
+			final SlotID requestedSlotId2 = slotIdCaptor.getValue();
 
-			requestSuccessRunnable.run();
-
-			final SlotID requestedSlotId = slotIdCaptor.getValue();
+			assertEquals(slotId2, requestedSlotId2);
 
-			assertEquals(slotId2, requestedSlotId);
+			Future<TaskManagerSlot> requestedSlotFuture = FlinkFuture.supplyAsync(new Callable<TaskManagerSlot>() {
+				@Override
+				public TaskManagerSlot call() throws Exception {
+					return slotManager.getSlot(requestedSlotId2);
+				}
+			}, mainThreadExecutor);
 
-			TaskManagerSlot slot = slotManager.getSlot(requestedSlotId);
+			TaskManagerSlot slot = requestedSlotFuture.get();
 
 			assertTrue(slot.isAllocated());
 			assertEquals(allocationId, slot.getAllocationId());
@@ -916,11 +946,12 @@ public void testSlotReportWhileActiveSlotRequest() throws Exception {
 	 */
 	@Test
 	public void testTimeoutForUnusedTaskManager() throws Exception {
-		final long taskManagerTimeout = 123456L;
+		final long taskManagerTimeout = 50L;
+		final long verifyTimeout = taskManagerTimeout * 10L;
 
 		final UUID leaderId = UUID.randomUUID();
 		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
-		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
+		final ScheduledExecutor scheduledExecutor = TestingUtils.defaultScheduledExecutor();
 
 		final ResourceID resourceId = ResourceID.generate();
 
@@ -946,21 +977,34 @@ public void testTimeoutForUnusedTaskManager() throws Exception {
 		final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
 		final SlotReport initialSlotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));
 
-		try (SlotManager slotManager = new SlotManager(
+		final Executor mainThreadExecutor = TestingUtils.defaultExecutor();
+
+		try (final SlotManager slotManager = new SlotManager(
 			scheduledExecutor,
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
 			Time.of(taskManagerTimeout, TimeUnit.MILLISECONDS))) {
 
-			slotManager.start(leaderId, Executors.directExecutor(), resourceManagerActions);
-
-			slotManager.registerSlotRequest(slotRequest);
+			slotManager.start(leaderId, mainThreadExecutor, resourceManagerActions);
 
-			slotManager.registerTaskManager(taskManagerConnection, initialSlotReport);
+			FlinkFuture.supplyAsync(new Callable<Void>() {
+				@Override
+				public Void call() throws Exception {
+					slotManager.registerSlotRequest(slotRequest);
+
+					return null;
+				}
+			}, mainThreadExecutor)
+			.thenAccept(new AcceptFunction<Void>() {
+				@Override
+				public void accept(Void value) {
+					slotManager.registerTaskManager(taskManagerConnection, initialSlotReport);
+				}
+			});
 
 			ArgumentCaptor<SlotID> slotIdArgumentCaptor = ArgumentCaptor.forClass(SlotID.class);
 
-			verify(taskExecutorGateway).requestSlot(
+			verify(taskExecutorGateway, timeout(verifyTimeout)).requestSlot(
 				slotIdArgumentCaptor.capture(),
 				eq(jobId),
 				eq(allocationId),
@@ -968,103 +1012,48 @@ public void testTimeoutForUnusedTaskManager() throws Exception {
 				eq(leaderId),
 				any(Time.class));
 
-			assertFalse(slotManager.hasTimeoutRegistered(taskManagerConnection.getInstanceID()));
-
-			SlotID slotId = slotIdArgumentCaptor.getValue();
-			TaskManagerSlot slot = slotManager.getSlot(slotId);
-
-			assertTrue(slot.isAllocated());
-			assertEquals(allocationId, slot.getAllocationId());
-
-			slotManager.freeSlot(slotId, allocationId);
+			Future<Boolean> idleFuture = FlinkFuture.supplyAsync(new Callable<Boolean>() {
+				@Override
+				public Boolean call() throws Exception {
+					return slotManager.isTaskManagerIdle(taskManagerConnection.getInstanceID());
+				}
+			}, mainThreadExecutor);
 
-			assertTrue(slotManager.hasTimeoutRegistered(taskManagerConnection.getInstanceID()));
+			// check that the TaskManaer is not idle
+			assertFalse(idleFuture.get());
 
-			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
+			final SlotID slotId = slotIdArgumentCaptor.getValue();
 
-			// filter out the schedule call for the task manager which will be registered using the
-			// taskManagerTimeout value
-			verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), eq(taskManagerTimeout), eq(TimeUnit.MILLISECONDS));
+			Future<TaskManagerSlot> slotFuture = FlinkFuture.supplyAsync(new Callable<TaskManagerSlot>() {
+				@Override
+				public TaskManagerSlot call() throws Exception {
+					return slotManager.getSlot(slotId);
+				}
+			}, mainThreadExecutor);
 
-			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
+			TaskManagerSlot slot = slotFuture.get();
 
-			timeoutRunnable.run();
-
-			verify(resourceManagerActions, times(1)).releaseResource(eq(taskManagerConnection.getInstanceID()));
-		}
-	}
-
-	/**
-	 * Tests that the slot manager re-registers a timeout for a rejected slot request.
-	 */
-	@Test
-	public void testTimeoutForRejectedSlotRequest() throws Exception {
-
-		final long slotRequestTimeout = 1337L;
-		final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
-
-		final ResourceID resourceId = ResourceID.generate();
-		final SlotID slotId = new SlotID(resourceId, 0);
-		final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
-		final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
-		final SlotReport slotReport = new SlotReport(slotStatus);
-
-		final UUID leaderId = UUID.randomUUID();
-		final ResourceManagerActions resourceManagerActions = mock(ResourceManagerActions.class);
-
-		final JobID jobId = new JobID();
-		final AllocationID allocationId = new AllocationID();
-		final AllocationID allocationId2 = new AllocationID();
-		final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
-
-		CompletableFuture<Acknowledge> requestFuture = new FlinkCompletableFuture<>();
-
-		final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
-		when(taskExecutorGateway.requestSlot(
-			eq(slotId),
-			eq(jobId),
-			eq(allocationId),
-			anyString(),
-			eq(leaderId),
-			any(Time.class))).thenReturn(requestFuture);
-
-		final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(taskExecutorGateway);
-
-		try (SlotManager slotManager = new SlotManager(
-			scheduledExecutor,
-			TestingUtils.infiniteTime(),
-			Time.milliseconds(slotRequestTimeout),
-			TestingUtils.infiniteTime())) {
-
-			slotManager.start(leaderId, Executors.directExecutor(), resourceManagerActions);
-
-			slotManager.registerTaskManager(taskManagerConnection, slotReport);
-
-			slotManager.registerSlotRequest(slotRequest);
-
-			verify(taskExecutorGateway).requestSlot(
-				eq(slotId),
-				eq(jobId),
-				eq(allocationId),
-				anyString(),
-				eq(leaderId),
-				any(Time.class));
-
-			requestFuture.completeExceptionally(new SlotOccupiedException("Slot is already occupied", allocationId2));
-
-			ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
-			verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), eq(slotRequestTimeout), eq(TimeUnit.MILLISECONDS));
+			assertTrue(slot.isAllocated());
+			assertEquals(allocationId, slot.getAllocationId());
 
-			Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
+			Future<Boolean> idleFuture2 = FlinkFuture.supplyAsync(new Callable<Void>() {
+				@Override
+				public Void call() throws Exception {
+					slotManager.freeSlot(slotId, allocationId);
 
-			timeoutRunnable.run();
+					return null;
+				}
+			}, mainThreadExecutor)
+			.thenApply(new ApplyFunction<Void, Boolean>() {
+				@Override
+				public Boolean apply(Void value) {
+					return slotManager.isTaskManagerIdle(taskManagerConnection.getInstanceID());
+				}
+			});
 
-			verify(resourceManagerActions).notifyAllocationFailure(eq(jobId), eq(allocationId), any(Exception.class));
+			assertTrue(idleFuture2.get());
 
-			TaskManagerSlot slot = slotManager.getSlot(slotId);
-
-			assertTrue(slot.isAllocated());
-			assertEquals(allocationId2, slot.getAllocationId());
+			verify(resourceManagerActions, timeout(verifyTimeout).times(1)).releaseResource(eq(taskManagerConnection.getInstanceID()));
 		}
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index c09316cc9be9e..a1ab1ab0dabd1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -76,7 +76,6 @@ public static void afterClass() {
 	@Test
 	public void testSlotsUnavailableRequest() throws Exception {
 		final JobID jobID = new JobID();
-		final ResourceID jmResourceId = new ResourceID(jmAddress);
 
 		final UUID rmLeaderID = UUID.randomUUID();
 
@@ -133,7 +132,6 @@ public void testSlotsUnavailableRequest() throws Exception {
 	@Test
 	public void testSlotAvailableRequest() throws Exception {
 		final JobID jobID = new JobID();
-		final ResourceID jmResourceId = new ResourceID(jmAddress);
 
 		final UUID rmLeaderID = UUID.randomUUID();
 
diff --git a/flink-runtime/src/test/resources/log4j-test.properties b/flink-runtime/src/test/resources/log4j-test.properties
index 7ba163366a631..812a256d5ca2a 100644
--- a/flink-runtime/src/test/resources/log4j-test.properties
+++ b/flink-runtime/src/test/resources/log4j-test.properties
@@ -16,7 +16,7 @@
 # limitations under the License.
 ################################################################################
 
-log4j.rootLogger=OFF, console
+log4j.rootLogger=DEBUG, console
 
 # -----------------------------------------------------------------------------
 # Console (use 'console')
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index 03b5172226a25..876e26b1bb6db 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -82,7 +82,7 @@ object TestingUtils {
   def getDefaultTestingActorSystemConfig = testConfig
 
   def infiniteTime: Time = {
-    Time.milliseconds(Long.MaxValue);
+    Time.milliseconds(Integer.MAX_VALUE);
   }
   
 
@@ -113,7 +113,7 @@ object TestingUtils {
   def defaultExecutor: ScheduledExecutorService = {
     synchronized {
       if (sharedExecutorInstance == null || sharedExecutorInstance.isShutdown) {
-        sharedExecutorInstance = Executors.newSingleThreadScheduledExecutor()
+        sharedExecutorInstance = Executors.newSingleThreadScheduledExecutor();
       }
 
       sharedExecutorInstance

From 2cbde0cac52bfc7d241e2df6e05755b90526c1c0 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Fri, 28 Apr 2017 12:17:42 +0200
Subject: [PATCH 088/253] [FLINK-6411] [flip-6] Remove job removal from
 RunningJobsRegistry in YarnFlinkApplicationMasterRunner.shutdown

The YarnFlinkApplicationMasterRunner should not be concerned with removing jobs from
the RunningJobsRegistry. This is the responsibility of the JobManagerRunner.

This PR removes the job removal from the RunningJobRegistry from the
YarnFlinkApplicationMasterRunner.shutdown method.

This closes #3797.
---
 .../flink/yarn/YarnFlinkApplicationMasterRunner.java | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
index a6b66d75dd720..4e178e6024568 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
@@ -103,9 +103,6 @@ public class YarnFlinkApplicationMasterRunner extends AbstractYarnFlinkApplicati
 	@GuardedBy("lock")
 	private JobManagerRunner jobManagerRunner;
 
-	@GuardedBy("lock")
-	private JobGraph jobGraph;
-
 	// ------------------------------------------------------------------------
 	//  Program entry point
 	// ------------------------------------------------------------------------
@@ -217,7 +214,7 @@ private ResourceManager<?> createResourceManager(Configuration config) throws Ex
 	private JobManagerRunner createJobManagerRunner(Configuration config) throws Exception{
 		// first get JobGraph from local resources
 		//TODO: generate the job graph from user's jar
-		jobGraph = loadJobGraph(config);
+		JobGraph jobGraph = loadJobGraph(config);
 
 		// now the JobManagerRunner
 		return new JobManagerRunner(
@@ -232,13 +229,6 @@ private JobManagerRunner createJobManagerRunner(Configuration config) throws Exc
 	}
 
 	protected void shutdown(ApplicationStatus status, String msg) {
-		// Need to clear the job state in the HA services before shutdown
-		try {
-			haServices.getRunningJobsRegistry().clearJob(jobGraph.getJobID());
-		}
-		catch (Throwable t) {
-			LOG.warn("Could not clear the job at the high-availability services", t);
-		}
 
 		synchronized (lock) {
 			if (jobManagerRunner != null) {

From 6fad093566fcdad1106c36afa624c160499a799d Mon Sep 17 00:00:00 2001
From: WangTaoTheTonic <wangtao111@huawei.com>
Date: Thu, 20 Apr 2017 20:28:10 +0800
Subject: [PATCH 089/253] [FLINK-6341] [jm] Don't let JM fall into infinite
 loop

This closes #3745.
---
 .../messages/ReconnectResourceManager.java            | 11 ++++++++++-
 .../apache/flink/runtime/jobmanager/JobManager.scala  |  7 +++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
index 6f6f878a39c5d..d02193e33dc27 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
@@ -22,6 +22,8 @@
 import org.apache.flink.runtime.messages.RequiresLeaderSessionID;
 import org.apache.flink.util.Preconditions;
 
+import java.util.UUID;
+
 /**
  * This message signals that the ResourceManager should reconnect to the JobManager. It is processed
  * by the JobManager if it fails to register resources with the ResourceManager. The JobManager wants
@@ -33,14 +35,21 @@ public class ReconnectResourceManager implements RequiresLeaderSessionID, java.i
 
 	private final ActorRef resourceManager;
 
-	public ReconnectResourceManager(ActorRef resourceManager) {
+	private final UUID currentConnID;
+
+	public ReconnectResourceManager(ActorRef resourceManager, UUID currentConnID) {
 		this.resourceManager = Preconditions.checkNotNull(resourceManager);
+		this.currentConnID = Preconditions.checkNotNull(currentConnID);
 	}
 	
 	public ActorRef resourceManager() {
 		return resourceManager;
 	}
 
+	public UUID connID() {
+		return currentConnID;
+	}
+
 	@Override
 	public String toString() {
 		return "ReconnectResourceManager " + resourceManager.path();
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 2fc3ef4bfbf93..da9df2b09018d 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -178,6 +178,8 @@ class JobManager(
   /** The resource manager actor responsible for allocating and managing task manager resources. */
   var currentResourceManager: Option[ActorRef] = None
 
+  var currentRMConnID: UUID = null
+
   val taskManagerMap = mutable.Map[ActorRef, InstanceID]()
 
   /**
@@ -337,6 +339,7 @@ class JobManager(
 
       // ditch current resource manager (if any)
       currentResourceManager = Option(msg.resourceManager())
+      currentRMConnID = UUID.randomUUID()
 
       val taskManagerResources = instanceManager.getAllRegisteredInstances.asScala.map(
         instance => instance.getTaskManagerID).toList.asJava
@@ -361,7 +364,7 @@ class JobManager(
       }
 
       currentResourceManager match {
-        case Some(rm) if rm.equals(msg.resourceManager()) =>
+        case Some(rm) if rm.equals(msg.resourceManager()) && currentRMConnID.equals(msg.connID()) =>
           // we should ditch the current resource manager
           log.debug(s"Disconnecting resource manager $rm and forcing a reconnect.")
           currentResourceManager = None
@@ -396,7 +399,7 @@ class JobManager(
                 case _ =>
                   log.warn("Failure while asking ResourceManager for RegisterResource. Retrying", t)
               }
-              self ! decorateMessage(new ReconnectResourceManager(rm))
+              self ! decorateMessage(new ReconnectResourceManager(rm, currentRMConnID))
           }(context.dispatcher)
 
         case None =>

From f730291ca41efa2f3fb663cd22344762ba2963da Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Fri, 28 Apr 2017 15:21:56 +0200
Subject: [PATCH 090/253] [FLINK-6341] [jm] Add test case to guard against RM
 registration loop

---
 .../configuration/JobManagerOptions.java      |  10 ++
 .../messages/ReconnectResourceManager.java    |  18 +--
 .../flink/runtime/jobmanager/JobManager.scala |  24 ++--
 .../runtime/jobmanager/JobManagerTest.java    | 105 +++++++++++++++++-
 4 files changed, 138 insertions(+), 19 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index d129405e06a60..10d9e16235708 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -72,6 +72,16 @@ public class JobManagerOptions {
 			.defaultValue(16)
 			.withDeprecatedKeys("job-manager.max-attempts-history-size");
 
+	/**
+	 * This option specifies the interval in order to trigger a resource manager reconnection if the connection
+	 * to the resource manager has been lost.
+	 *
+	 * This option is only intended for internal use.
+	 */
+	public static final ConfigOption<Long> RESOURCE_MANAGER_RECONNECT_INTERVAL =
+		key("jobmanager.resourcemanager.reconnect-interval")
+		.defaultValue(2000L);
+
 	// ------------------------------------------------------------------------
 	//  JobManager web UI
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
index d02193e33dc27..1f852e891fd40 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
@@ -22,7 +22,7 @@
 import org.apache.flink.runtime.messages.RequiresLeaderSessionID;
 import org.apache.flink.util.Preconditions;
 
-import java.util.UUID;
+import java.io.Serializable;
 
 /**
  * This message signals that the ResourceManager should reconnect to the JobManager. It is processed
@@ -30,28 +30,30 @@
  * the ResourceManager to go through the reconciliation phase to sync up with the JobManager bookkeeping.
  * This is done by forcing the ResourceManager to reconnect.
  */
-public class ReconnectResourceManager implements RequiresLeaderSessionID, java.io.Serializable {
+public class ReconnectResourceManager implements RequiresLeaderSessionID, Serializable {
 	private static final long serialVersionUID = 1L;
 
 	private final ActorRef resourceManager;
 
-	private final UUID currentConnID;
+	private final long connectionId;
 
-	public ReconnectResourceManager(ActorRef resourceManager, UUID currentConnID) {
+	public ReconnectResourceManager(ActorRef resourceManager, long connectionId) {
 		this.resourceManager = Preconditions.checkNotNull(resourceManager);
-		this.currentConnID = Preconditions.checkNotNull(currentConnID);
+		this.connectionId = Preconditions.checkNotNull(connectionId);
 	}
 	
 	public ActorRef resourceManager() {
 		return resourceManager;
 	}
 
-	public UUID connID() {
-		return currentConnID;
+	public long getConnectionId() {
+		return connectionId;
 	}
 
 	@Override
 	public String toString() {
-		return "ReconnectResourceManager " + resourceManager.path();
+		return "ReconnectResourceManager(" +
+			resourceManager.path() + ", " +
+			connectionId + ')';
 	}
 }
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index da9df2b09018d..b01ddc086b166 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -178,10 +178,14 @@ class JobManager(
   /** The resource manager actor responsible for allocating and managing task manager resources. */
   var currentResourceManager: Option[ActorRef] = None
 
-  var currentRMConnID: UUID = null
+  var currentResourceManagerConnectionId: Long = 0
 
   val taskManagerMap = mutable.Map[ActorRef, InstanceID]()
 
+  val triggerResourceManagerReconnectInterval = new FiniteDuration(
+    flinkConfiguration.getLong(JobManagerOptions.RESOURCE_MANAGER_RECONNECT_INTERVAL),
+    TimeUnit.MILLISECONDS)
+
   /**
    * Run when the job manager is started. Simply logs an informational message.
    * The method also starts the leader election service.
@@ -339,7 +343,7 @@ class JobManager(
 
       // ditch current resource manager (if any)
       currentResourceManager = Option(msg.resourceManager())
-      currentRMConnID = UUID.randomUUID()
+      currentResourceManagerConnectionId += 1
 
       val taskManagerResources = instanceManager.getAllRegisteredInstances.asScala.map(
         instance => instance.getTaskManagerID).toList.asJava
@@ -358,24 +362,25 @@ class JobManager(
       def reconnectRepeatedly(): Unit = {
         msg.resourceManager() ! decorateMessage(new TriggerRegistrationAtJobManager(self))
         // try again after some delay
-        context.system.scheduler.scheduleOnce(2 seconds) {
+        context.system.scheduler.scheduleOnce(triggerResourceManagerReconnectInterval) {
           self ! decorateMessage(msg)
         }(context.dispatcher)
       }
 
       currentResourceManager match {
-        case Some(rm) if rm.equals(msg.resourceManager()) && currentRMConnID.equals(msg.connID()) =>
+        case Some(rm) if rm.equals(msg.resourceManager()) &&
+          currentResourceManagerConnectionId == msg.getConnectionId =>
           // we should ditch the current resource manager
           log.debug(s"Disconnecting resource manager $rm and forcing a reconnect.")
           currentResourceManager = None
           reconnectRepeatedly()
-        case Some(rm) =>
-          // we have registered with another ResourceManager in the meantime, stop sending
-          // TriggerRegistrationAtJobManager messages to the old ResourceManager
         case None =>
           log.warn(s"No resource manager ${msg.resourceManager()} connected. " +
             s"Telling old ResourceManager to register again.")
           reconnectRepeatedly()
+        case _ =>
+        // we have established a new connection to a ResourceManager in the meantime, stop sending
+        // TriggerRegistrationAtJobManager messages to the old ResourceManager
       }
 
     case msg @ RegisterTaskManager(
@@ -399,7 +404,10 @@ class JobManager(
                 case _ =>
                   log.warn("Failure while asking ResourceManager for RegisterResource. Retrying", t)
               }
-              self ! decorateMessage(new ReconnectResourceManager(rm, currentRMConnID))
+              self ! decorateMessage(
+                new ReconnectResourceManager(
+                  rm,
+                  currentResourceManagerConnectionId))
           }(context.dispatcher)
 
         case None =>
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
index d7fc71dabb766..6316bfd4cb54b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
@@ -18,18 +18,22 @@
 
 package org.apache.flink.runtime.jobmanager;
 
-import akka.actor.ActorRef;
-import akka.actor.ActorSystem;
-import akka.actor.PoisonPill;
+import akka.actor.*;
 import akka.testkit.JavaTestKit;
+import akka.testkit.TestProbe;
 import com.typesafe.config.Config;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.akka.ListeningBehaviour;
 import org.apache.flink.runtime.checkpoint.CheckpointDeclineReason;
+import org.apache.flink.runtime.clusterframework.messages.NotifyResourceStarted;
+import org.apache.flink.runtime.clusterframework.messages.RegisterResourceManager;
+import org.apache.flink.runtime.clusterframework.messages.RegisterResourceManagerSuccessful;
+import org.apache.flink.runtime.clusterframework.messages.TriggerRegistrationAtJobManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -40,6 +44,7 @@
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
+import org.apache.flink.runtime.instance.HardwareDescription;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
 import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
 import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -64,6 +69,7 @@
 import org.apache.flink.runtime.messages.JobManagerMessages.SubmitJob;
 import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepoint;
 import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepointSuccess;
+import org.apache.flink.runtime.messages.RegistrationMessages;
 import org.apache.flink.runtime.query.KvStateID;
 import org.apache.flink.runtime.query.KvStateLocation;
 import org.apache.flink.runtime.query.KvStateMessage.LookupKvStateLocation;
@@ -73,6 +79,7 @@
 import org.apache.flink.runtime.query.UnknownKvStateLocation;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.testingUtils.TestingCluster;
 import org.apache.flink.runtime.testingUtils.TestingJobManager;
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
@@ -89,11 +96,15 @@
 import org.apache.flink.runtime.testtasks.NoOpInvokable;
 import org.apache.flink.runtime.testutils.StoppableInvokable;
 import org.apache.flink.util.TestLogger;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
 import scala.Option;
 import scala.Some;
 import scala.Tuple2;
@@ -107,6 +118,7 @@
 import java.net.InetAddress;
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import static org.apache.flink.runtime.io.network.partition.ResultPartitionType.PIPELINED;
 import static org.apache.flink.runtime.messages.JobManagerMessages.JobResultSuccess;
@@ -121,6 +133,9 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
 
 public class JobManagerTest extends TestLogger {
 
@@ -1261,4 +1276,88 @@ public void testSavepointRestoreSettings() throws Exception {
 			}
 		}
 	}
+
+	/**
+	 * This tests makes sure that triggering a reconnection from the ResourceManager will stop after a new
+	 * ResourceManager has connected. Furthermore it makes sure that there is not endless loop of reconnection
+	 * commands (see FLINK-6341).
+	 */
+	@Test
+	public void testResourceManagerConnection() throws TimeoutException, InterruptedException {
+		FiniteDuration testTimeout = new FiniteDuration(30L, TimeUnit.SECONDS);
+		final long reconnectionInterval = 200L;
+
+		final Configuration configuration = new Configuration();
+		configuration.setLong(JobManagerOptions.RESOURCE_MANAGER_RECONNECT_INTERVAL, reconnectionInterval);
+
+
+		final ActorSystem actorSystem = AkkaUtils.createLocalActorSystem(configuration);
+
+		try {
+			final ActorGateway jmGateway = TestingUtils.createJobManager(
+				actorSystem,
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				configuration);
+
+			final TestProbe probe = TestProbe.apply(actorSystem);
+			final AkkaActorGateway rmGateway = new AkkaActorGateway(probe.ref(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+			// wait for the JobManager to become the leader
+			Future<?> leaderFuture = jmGateway.ask(TestingJobManagerMessages.getNotifyWhenLeader(), testTimeout);
+			Await.ready(leaderFuture, testTimeout);
+
+			jmGateway.tell(new RegisterResourceManager(probe.ref()), rmGateway);
+
+			JobManagerMessages.LeaderSessionMessage leaderSessionMessage = probe.expectMsgClass(JobManagerMessages.LeaderSessionMessage.class);
+
+			assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, leaderSessionMessage.leaderSessionID());
+			assertTrue(leaderSessionMessage.message() instanceof RegisterResourceManagerSuccessful);
+
+			jmGateway.tell(
+				new RegistrationMessages.RegisterTaskManager(
+					ResourceID.generate(),
+					mock(TaskManagerLocation.class),
+					new HardwareDescription(1, 1L, 1L, 1L),
+					1));
+			leaderSessionMessage = probe.expectMsgClass(JobManagerMessages.LeaderSessionMessage.class);
+
+			assertTrue(leaderSessionMessage.message() instanceof NotifyResourceStarted);
+
+			// fail the NotifyResourceStarted so that we trigger the reconnection process on the JobManager's side
+			probe.lastSender().tell(new Status.Failure(new Exception("Test exception")), ActorRef.noSender());
+
+			Deadline reconnectionDeadline = new FiniteDuration(5L * reconnectionInterval, TimeUnit.MILLISECONDS).fromNow();
+			boolean registered = false;
+
+			while (reconnectionDeadline.hasTimeLeft()) {
+				try {
+					leaderSessionMessage = probe.expectMsgClass(reconnectionDeadline.timeLeft(), JobManagerMessages.LeaderSessionMessage.class);
+				} catch (AssertionError ignored) {
+					// expected timeout after the reconnectionDeadline has been exceeded
+					continue;
+				}
+
+				if (leaderSessionMessage.message() instanceof TriggerRegistrationAtJobManager) {
+					if (registered) {
+						fail("A successful registration should not be followed by another TriggerRegistrationAtJobManager message.");
+					}
+
+					jmGateway.tell(new RegisterResourceManager(probe.ref()), rmGateway);
+				} else if (leaderSessionMessage.message() instanceof RegisterResourceManagerSuccessful) {
+					// now we should no longer receive TriggerRegistrationAtJobManager messages
+					registered = true;
+				} else {
+					fail("Received unknown message: " + leaderSessionMessage.message() + '.');
+				}
+			}
+
+			assertTrue(registered);
+
+		} finally {
+			// cleanup the actor system and with it all of the started actors if not already terminated
+			actorSystem.shutdown();
+			actorSystem.awaitTermination();
+		}
+	}
 }

From fbe55e9c5ef29a3c9a73c5594535e4ae312286bc Mon Sep 17 00:00:00 2001
From: "guowei.mgw" <guowei.mgw@gmail.com>
Date: Mon, 24 Apr 2017 11:47:47 +0200
Subject: [PATCH 091/253] [FLINK-5892] Add new StateAssignmentOperationV2

---
 .../StateAssignmentOperationV2.java           | 458 ++++++++++++++++++
 1 file changed, 458 insertions(+)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
new file mode 100644
index 0000000000000..83c188c78dae2
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.checkpoint;
+
+//import com.google.common.collect.Lists;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyGroupsStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.runtime.state.TaskStateHandles;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class encapsulates the operation of assigning restored state when restoring from a checkpoint that works on the
+ * granularity of operators. This is the case for checkpoints that were created either with a Flink version >= 1.3 or
+ * 1.2 if the savepoint only contains {@link SubtaskState}s for which the length of contained
+ * {@link ChainedStateHandle}s is equal to 1.
+ */
+public class StateAssignmentOperationV2 {
+
+	private final Logger logger;
+	private final Map<JobVertexID, ExecutionJobVertex> tasks;
+	private final Map<JobVertexID, TaskState> taskStates;
+	private final boolean allowNonRestoredState;
+
+	public StateAssignmentOperationV2(
+			Logger logger,
+			Map<JobVertexID, ExecutionJobVertex> tasks,
+			Map<JobVertexID, TaskState> taskStates,
+			boolean allowNonRestoredState) {
+
+		this.logger = Preconditions.checkNotNull(logger);
+		this.tasks = Preconditions.checkNotNull(tasks);
+		this.taskStates = Preconditions.checkNotNull(taskStates);
+		this.allowNonRestoredState = allowNonRestoredState;
+	}
+
+	public boolean assignStates() throws Exception {
+		Map<JobVertexID, TaskState> localStates = new HashMap<>(taskStates);
+		Map<JobVertexID, ExecutionJobVertex> localTasks = this.tasks;
+
+		Set<JobVertexID> allOperatorIDs = new HashSet<>();
+		for (ExecutionJobVertex executionJobVertex : tasks.values()) {
+			//allOperatorIDs.addAll(Lists.newArrayList(executionJobVertex.getOperatorIDs()));
+		}
+		for (Map.Entry<JobVertexID, TaskState> taskGroupStateEntry : taskStates.entrySet()) {
+			TaskState taskState = taskGroupStateEntry.getValue();
+			//----------------------------------------find operator for state---------------------------------------------
+
+			if (!allOperatorIDs.contains(taskGroupStateEntry.getKey())) {
+				if (allowNonRestoredState) {
+					logger.info("Skipped checkpoint state for operator {}.", taskState.getJobVertexID());
+					continue;
+				} else {
+					throw new IllegalStateException("There is no operator for the state " + taskState.getJobVertexID());
+				}
+			}
+		}
+
+		for (Map.Entry<JobVertexID, ExecutionJobVertex> task : localTasks.entrySet()) {
+			final ExecutionJobVertex executionJobVertex = task.getValue();
+
+			// find the states of all operators belonging to this task
+			JobVertexID[] operatorIDs = null;//executionJobVertex.getOperatorIDs();
+			JobVertexID[] altOperatorIDs = null;//executionJobVertex.getUserDefinedOperatorIDs();
+			List<TaskState> operatorStates = new ArrayList<>();
+			boolean statelessTask = true;
+			for (int x = 0; x < operatorIDs.length; x++) {
+				JobVertexID operatorID = altOperatorIDs[x] == null
+					? operatorIDs[x]
+					: altOperatorIDs[x];
+
+				TaskState operatorState = localStates.remove(operatorID);
+				if (operatorState == null) {
+					operatorState = new TaskState(
+						operatorID,
+						executionJobVertex.getParallelism(),
+						executionJobVertex.getMaxParallelism(),
+						1);
+				} else {
+					statelessTask = false;
+				}
+				operatorStates.add(operatorState);
+			}
+			if (statelessTask) { // skip tasks where no operator has any state
+				continue;
+			}
+
+			assignAttemptState(task.getValue(), operatorStates);
+		}
+
+		return true;
+	}
+
+	private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<TaskState> operatorStates) {
+
+		JobVertexID[] operatorIDs = null;//executionJobVertex.getOperatorIDs();
+
+		//1. first compute the new parallelism
+		checkParallelismPreconditions(operatorStates, executionJobVertex);
+
+		int newParallelism = executionJobVertex.getParallelism();
+
+		List<KeyGroupRange> keyGroupPartitions = null;//StateAssignmentOperationUtils.createKeyGroupPartitions(
+			//executionJobVertex.getMaxParallelism(),
+			//newParallelism);
+
+		//2. Redistribute the operator state.
+		/**
+		 *
+		 * Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
+		 *
+		 * The old ManagedOperatorStates with old parallelism 3:
+		 *
+		 * 		parallelism0 parallelism1 parallelism2
+		 * op0   states0,0    state0,1	   state0,2
+		 * op1
+		 * op2   states2,0    state2,1	   state1,2
+		 * op3   states3,0    state3,1     state3,2
+		 *
+		 * The new ManagedOperatorStates with new parallelism 4:
+		 *
+		 * 		parallelism0 parallelism1 parallelism2 parallelism3
+		 * op0   state0,0	  state0,1 	   state0,2		state0,3
+		 * op1
+		 * op2   state2,0	  state2,1 	   state2,2		state2,3
+		 * op3   state3,0	  state3,1 	   state3,2		state3,3
+		 */
+		List<List<Collection<OperatorStateHandle>>> newManagedOperatorStates = new ArrayList<>();
+		List<List<Collection<OperatorStateHandle>>> newRawOperatorStates = new ArrayList<>();
+
+		reDistributePartitionableStates(operatorStates, newParallelism, newManagedOperatorStates, newRawOperatorStates);
+
+
+		//3. Compute TaskStateHandles of every subTask in the executionJobVertex
+		/**
+		 *  An executionJobVertex's all state handles needed to restore are something like a matrix
+		 *
+		 * 		parallelism0 parallelism1 parallelism2 parallelism3
+		 * op0   sh(0,0)     sh(0,1)       sh(0,2)	    sh(0,3)
+		 * op1   sh(1,0)	 sh(1,1)	   sh(1,2)	    sh(1,3)
+		 * op2   sh(2,0)	 sh(2,1)	   sh(2,2)		sh(2,3)
+		 * op3   sh(3,0)	 sh(3,1)	   sh(3,2)		sh(3,3)
+		 *
+		 * we will compute the state handles column by column.
+		 *
+		 */
+		for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
+
+			Execution currentExecutionAttempt = executionJobVertex.getTaskVertices()[subTaskIndex]
+				.getCurrentExecutionAttempt();
+
+			List<StreamStateHandle> subNonPartitionableState = new ArrayList<>();
+
+			Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> subKeyedState = null;
+
+			List<Collection<OperatorStateHandle>> subManagedOperatorState = new ArrayList<>();
+			List<Collection<OperatorStateHandle>> subRawOperatorState = new ArrayList<>();
+
+
+			for (int operatorIndex = 0; operatorIndex < operatorIDs.length; operatorIndex++) {
+				TaskState operatorState = operatorStates.get(operatorIndex);
+				int oldParallelism = operatorState.getParallelism();
+
+				// NonPartitioned State
+
+				reAssignSubNonPartitionedStates(
+					operatorState,
+					subTaskIndex,
+					newParallelism,
+					oldParallelism,
+					subNonPartitionableState);
+
+				// PartitionedState
+				reAssignSubPartitionableState(newManagedOperatorStates,
+					newRawOperatorStates,
+					subTaskIndex,
+					operatorIndex,
+					subManagedOperatorState,
+					subRawOperatorState);
+
+				// KeyedState
+				if (operatorIndex == operatorIDs.length - 1) {
+					subKeyedState = reAssignSubKeyedStates(operatorState,
+						keyGroupPartitions,
+						subTaskIndex,
+						newParallelism,
+						oldParallelism);
+
+				}
+			}
+
+
+			// check if a stateless task
+			if (!allElementsAreNull(subNonPartitionableState) ||
+				!allElementsAreNull(subManagedOperatorState) ||
+				!allElementsAreNull(subRawOperatorState) ||
+				subKeyedState != null) {
+
+				TaskStateHandles taskStateHandles = new TaskStateHandles(
+
+					new ChainedStateHandle<>(subNonPartitionableState),
+					subManagedOperatorState,
+					subRawOperatorState,
+					subKeyedState != null ? subKeyedState.f0 : null,
+					subKeyedState != null ? subKeyedState.f1 : null);
+
+				currentExecutionAttempt.setInitialState(taskStateHandles);
+			}
+		}
+	}
+
+
+	public void checkParallelismPreconditions(List<TaskState> operatorStates, ExecutionJobVertex executionJobVertex) {
+
+		for (TaskState taskState : operatorStates) {
+			//StateAssignmentOperation.checkParallelismPreconditions(taskState, executionJobVertex, this.logger);
+		}
+	}
+
+
+	private void reAssignSubPartitionableState(
+			List<List<Collection<OperatorStateHandle>>> newMangedOperatorStates,
+			List<List<Collection<OperatorStateHandle>>> newRawOperatorStates,
+			int subTaskIndex, int operatorIndex,
+			List<Collection<OperatorStateHandle>> subManagedOperatorState,
+			List<Collection<OperatorStateHandle>> subRawOperatorState) {
+
+		if (newMangedOperatorStates.get(operatorIndex) != null) {
+			subManagedOperatorState.add(newMangedOperatorStates.get(operatorIndex).get(subTaskIndex));
+		} else {
+			subManagedOperatorState.add(null);
+		}
+		if (newRawOperatorStates.get(operatorIndex) != null) {
+			subRawOperatorState.add(newRawOperatorStates.get(operatorIndex).get(subTaskIndex));
+		} else {
+			subRawOperatorState.add(null);
+		}
+
+
+	}
+
+	private Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> reAssignSubKeyedStates(
+			TaskState operatorState,
+			List<KeyGroupRange> keyGroupPartitions,
+			int subTaskIndex,
+			int newParallelism,
+			int oldParallelism) {
+
+		Collection<KeyedStateHandle> subManagedKeyedState;
+		Collection<KeyedStateHandle> subRawKeyedState;
+
+		if (newParallelism == oldParallelism) {
+			if (operatorState.getState(subTaskIndex) != null) {
+				KeyedStateHandle oldSubManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState();
+				KeyedStateHandle oldSubRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState();
+				subManagedKeyedState = oldSubManagedKeyedState != null ? Collections.singletonList(
+					oldSubManagedKeyedState) : null;
+				subRawKeyedState = oldSubRawKeyedState != null ? Collections.singletonList(
+					oldSubRawKeyedState) : null;
+			} else {
+				subManagedKeyedState = null;
+				subRawKeyedState = null;
+			}
+		} else {
+			subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex));
+			subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex));
+		}
+		if (subManagedKeyedState == null && subRawKeyedState == null) {
+			return null;
+		}
+		return new Tuple2<>(subManagedKeyedState, subRawKeyedState);
+	}
+
+
+	private <X> boolean allElementsAreNull(List<X> nonPartitionableStates) {
+		for (Object streamStateHandle : nonPartitionableStates) {
+			if (streamStateHandle != null) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+
+	private void reAssignSubNonPartitionedStates(
+			TaskState operatorState,
+			int subTaskIndex,
+			int newParallelism,
+			int oldParallelism,
+			List<StreamStateHandle> subNonPartitionableState) {
+		if (oldParallelism == newParallelism) {
+			if (operatorState.getState(subTaskIndex) != null &&
+				!operatorState.getState(subTaskIndex).getLegacyOperatorState().isEmpty()) {
+				subNonPartitionableState.add(operatorState.getState(subTaskIndex).getLegacyOperatorState().get(0));
+			} else {
+				subNonPartitionableState.add(null);
+			}
+		} else {
+			subNonPartitionableState.add(null);
+		}
+	}
+
+	private void reDistributePartitionableStates(
+			List<TaskState> operatorStates, int newParallelism,
+			List<List<Collection<OperatorStateHandle>>> newManagedOperatorStates,
+			List<List<Collection<OperatorStateHandle>>> newRawOperatorStates) {
+
+		//collect the old partitionalbe state
+		List<List<OperatorStateHandle>> oldManagedOperatorStates = new ArrayList<>();
+		List<List<OperatorStateHandle>> oldRawOperatorStates = new ArrayList<>();
+
+		collectPartionableStates(operatorStates, oldManagedOperatorStates, oldRawOperatorStates);
+
+
+		//redistribute
+		OperatorStateRepartitioner opStateRepartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
+
+		for (int operatorIndex = 0; operatorIndex < operatorStates.size(); operatorIndex++) {
+			int oldParallelism = operatorStates.get(operatorIndex).getParallelism();
+			//newManagedOperatorStates.add(StateAssignmentOperationUtils.applyRepartitioner(opStateRepartitioner,
+			//	oldManagedOperatorStates.get(operatorIndex), oldParallelism, newParallelism));
+			//newRawOperatorStates.add(StateAssignmentOperationUtils.applyRepartitioner(opStateRepartitioner,
+			//	oldRawOperatorStates.get(operatorIndex), oldParallelism, newParallelism));
+
+		}
+	}
+
+
+	private void collectPartionableStates(
+			List<TaskState> operatorStates,
+			List<List<OperatorStateHandle>> managedOperatorStates,
+			List<List<OperatorStateHandle>> rawOperatorStates) {
+
+		for (TaskState operatorState : operatorStates) {
+			List<OperatorStateHandle> managedOperatorState = null;
+			List<OperatorStateHandle> rawOperatorState = null;
+
+			for (int i = 0; i < operatorState.getParallelism(); i++) {
+				SubtaskState subtaskState = operatorState.getState(i);
+				if (subtaskState != null) {
+					if (subtaskState.getManagedOperatorState() != null &&
+						subtaskState.getManagedOperatorState().getLength() > 0 &&
+						subtaskState.getManagedOperatorState().get(0) != null) {
+						if (managedOperatorState == null) {
+							managedOperatorState = new ArrayList<>();
+						}
+						managedOperatorState.add(subtaskState.getManagedOperatorState().get(0));
+					}
+
+					if (subtaskState.getRawOperatorState() != null &&
+						subtaskState.getRawOperatorState().getLength() > 0 &&
+						subtaskState.getRawOperatorState().get(0) != null) {
+						if (rawOperatorState == null) {
+							rawOperatorState = new ArrayList<>();
+						}
+						rawOperatorState.add(subtaskState.getRawOperatorState().get(0));
+					}
+				}
+
+			}
+			managedOperatorStates.add(managedOperatorState);
+			rawOperatorStates.add(rawOperatorState);
+		}
+	}
+
+
+	/**
+	 * Collect {@link KeyGroupsStateHandle  managedKeyedStateHandles} which have intersection with given
+	 * {@link KeyGroupRange} from {@link TaskState operatorState}
+	 *
+	 * @param operatorState        all state handles of a operator
+	 * @param subtaskKeyGroupRange the KeyGroupRange of a subtask
+	 * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange
+	 */
+	public static List<KeyedStateHandle> getManagedKeyedStateHandles(
+			TaskState operatorState,
+			KeyGroupRange subtaskKeyGroupRange) {
+
+		List<KeyedStateHandle> subtaskKeyedStateHandles = null;
+
+		for (int i = 0; i < operatorState.getParallelism(); i++) {
+			if (operatorState.getState(i) != null && operatorState.getState(i).getManagedKeyedState() != null) {
+				KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getManagedKeyedState().getIntersection(subtaskKeyGroupRange);
+
+				if (intersectedKeyedStateHandle != null) {
+					if (subtaskKeyedStateHandles == null) {
+						subtaskKeyedStateHandles = new ArrayList<>();
+					}
+					subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
+				}
+			}
+		}
+
+		return subtaskKeyedStateHandles;
+	}
+
+	/**
+	 * Collect {@link KeyGroupsStateHandle  rawKeyedStateHandles} which have intersection with given
+	 * {@link KeyGroupRange} from {@link TaskState operatorState}
+	 *
+	 * @param operatorState        all state handles of a operator
+	 * @param subtaskKeyGroupRange the KeyGroupRange of a subtask
+	 * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange
+	 */
+	public static List<KeyedStateHandle> getRawKeyedStateHandles(
+			TaskState operatorState,
+			KeyGroupRange subtaskKeyGroupRange) {
+
+		List<KeyedStateHandle> subtaskKeyedStateHandles = null;
+
+		for (int i = 0; i < operatorState.getParallelism(); i++) {
+			if (operatorState.getState(i) != null && operatorState.getState(i).getRawKeyedState() != null) {
+				KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getRawKeyedState().getIntersection(subtaskKeyGroupRange);
+
+				if (intersectedKeyedStateHandle != null) {
+					if (subtaskKeyedStateHandles == null) {
+						subtaskKeyedStateHandles = new ArrayList<>();
+					}
+					subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
+				}
+			}
+		}
+
+		return subtaskKeyedStateHandles;
+	}
+}

From ae87cec7e2ecfc45104168e5c0ad2867a4c80474 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Fri, 28 Apr 2017 19:40:20 +0200
Subject: [PATCH 092/253] [Flink-5892] Restore state on operator level

---
 docs/ops/upgrading.md                         |   4 +-
 docs/setup/savepoints.md                      |   5 -
 .../checkpoint/savepoint/SavepointV0.java     |   6 +
 .../checkpoint/CheckpointCoordinator.java     |   9 +-
 .../checkpoint/CompletedCheckpoint.java       |  57 +-
 .../runtime/checkpoint/OperatorState.java     | 183 ++++++
 .../checkpoint/OperatorSubtaskState.java      | 229 +++++++
 .../runtime/checkpoint/PendingCheckpoint.java |  87 +--
 .../checkpoint/StateAssignmentOperation.java  | 611 ++++++++++++------
 .../StateAssignmentOperationV2.java           | 458 -------------
 .../flink/runtime/checkpoint/TaskState.java   |   4 +
 .../checkpoint/savepoint/Savepoint.java       |  12 +
 .../checkpoint/savepoint/SavepointLoader.java |  54 +-
 .../checkpoint/savepoint/SavepointV1.java     |   6 +
 .../savepoint/SavepointV1Serializer.java      |   4 +-
 .../checkpoint/savepoint/SavepointV2.java     | 155 ++++-
 .../savepoint/SavepointV2Serializer.java      | 114 ++--
 .../executiongraph/ExecutionJobVertex.java    |  63 ++
 .../runtime/jobgraph/InputFormatVertex.java   |   4 +-
 .../flink/runtime/jobgraph/JobVertex.java     |  25 +-
 .../flink/runtime/jobgraph/OperatorID.java    |  45 ++
 ...oordinatorExternalizedCheckpointsTest.java |   5 +-
 .../CheckpointCoordinatorFailureTest.java     |  35 +-
 .../CheckpointCoordinatorMasterHooksTest.java |   5 +-
 .../checkpoint/CheckpointCoordinatorTest.java | 236 +++++--
 .../CheckpointStateRestoreTest.java           |  55 +-
 .../CompletedCheckpointStoreTest.java         |  59 +-
 .../checkpoint/CompletedCheckpointTest.java   |  47 +-
 .../checkpoint/PendingCheckpointTest.java     |  24 +-
 ...tandaloneCompletedCheckpointStoreTest.java |  10 +-
 ...oKeeperCompletedCheckpointStoreITCase.java |  14 +-
 ...ZooKeeperCompletedCheckpointStoreTest.java |   2 +-
 .../savepoint/CheckpointTestUtils.java        |  97 ++-
 .../savepoint/SavepointLoaderTest.java        |  24 +-
 .../savepoint/SavepointStoreTest.java         |  23 +-
 .../savepoint/SavepointV2SerializerTest.java  |  22 +-
 .../checkpoint/savepoint/SavepointV2Test.java |  14 +-
 .../executiongraph/LegacyJobVertexIdTest.java |   4 +-
 .../RecoverableCompletedCheckpointStore.java  |   2 +-
 .../api/graph/StreamGraphHasherV1.java        |  13 -
 .../api/graph/StreamGraphHasherV2.java        |  13 -
 .../api/graph/StreamGraphUserHashHasher.java  |   9 -
 .../api/graph/StreamingJobGraphGenerator.java |  46 +-
 ...treamingJobGraphGeneratorNodeHashTest.java |  18 +-
 .../test/checkpointing/SavepointITCase.java   |  54 +-
 .../AbstractOperatorRestoreTestBase.java      | 261 ++++++++
 .../state/operator/restore/ExecutionMode.java |  31 +
 .../restore/keyed/KeyedComplexChainTest.java  |  61 ++
 .../operator/restore/keyed/KeyedJob.java      | 243 +++++++
 ...stractNonKeyedOperatorRestoreTestBase.java |  59 ++
 .../restore/unkeyed/ChainBreakTest.java       |  55 ++
 .../unkeyed/ChainLengthDecreaseTest.java      |  51 ++
 .../unkeyed/ChainLengthIncreaseTest.java      |  56 ++
 .../restore/unkeyed/ChainOrderTest.java       |  54 ++
 .../restore/unkeyed/ChainUnionTest.java       |  53 ++
 .../operator/restore/unkeyed/NonKeyedJob.java | 198 ++++++
 .../operatorstate/complexKeyed/_metadata      | Bin 0 -> 137490 bytes
 .../operatorstate/nonKeyed/_metadata          | Bin 0 -> 3212 bytes
 58 files changed, 2971 insertions(+), 1117 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorState.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/OperatorID.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/ExecutionMode.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
 create mode 100644 flink-tests/src/test/resources/operatorstate/complexKeyed/_metadata
 create mode 100644 flink-tests/src/test/resources/operatorstate/nonKeyed/_metadata

diff --git a/docs/ops/upgrading.md b/docs/ops/upgrading.md
index 194d8af2ccc98..7259a6bcdf2f7 100644
--- a/docs/ops/upgrading.md
+++ b/docs/ops/upgrading.md
@@ -73,6 +73,8 @@ val mappedEvents: DataStream[(Int, Long)] = events
 
 **Note:** Since the operator IDs stored in a savepoint and IDs of operators in the application to start must be equal, it is highly recommended to assign unique IDs to all operators of an application that might be upgraded in the future. This advice applies to all operators, i.e., operators with and without explicitly declared operator state, because some operators have internal state that is not visible to the user. Upgrading an application without assigned operator IDs is significantly more difficult and may only be possible via a low-level workaround using the `setUidHash()` method.
 
+**Important:** As of 1.3.0 this also applies to operators that are part of a chain.
+
 By default all state stored in a savepoint must be matched to the operators of a starting application. However, users can explicitly agree to skip (and thereby discard) state that cannot be matched to an operator when starting a application from a savepoint. Stateful operators for which no state is found in the savepoint are initialized with their default state.
 
 ### Stateful Operators and User Functions
@@ -105,7 +107,7 @@ When upgrading an application by changing its topology, a few things need to be
 * **Adding a stateful operator:** The state of the operator will be initialized with the default state unless it takes over the state of another operator.
 * **Removing a stateful operator:** The state of the removed operator is lost unless another operator takes it over. When starting the upgraded application, you have to explicitly agree to discard the state.
 * **Changing of input and output types of operators:** When adding a new operator before or behind an operator with internal state, you have to ensure that the input or output type of the stateful operator is not modified to preserve the data type of the internal operator state (see above for details).
-* **Changing operator chaining:** Operators can be chained together for improved performance. However, chaining can limit the ability of an application to be upgraded if a chain contains a stateful operator that is not the first operator of the chain. In such a case, it is not possible to break the chain such that the stateful operator is moved out of the chain. It is also not possible to append or inject an existing stateful operator into a chain. The chaining behavior can be changed by modifying the parallelism of a chained operator or by adding or removing explicit operator chaining instructions. 
+* **Changing operator chaining:** Operators can be chained together for improved performance. When restoring from a savepoint taken since 1.3.0 it is possible to modify chains while preversing state consistency. It is possible a break the chain such that a stateful operator is moved out of the chain. It is also possible to append or inject a new or existing stateful operator into a chain, or to modify the operator order within a chain. However, when upgrading a savepoint to 1.3.0 it is paramount that the topology did not change in regards to chaining. All operators that are part of a chain should be assigned an ID as described in the [Matching Operator State](#Matching Operator State) section above.
 
 ## Upgrading the Flink Framework Version
 
diff --git a/docs/setup/savepoints.md b/docs/setup/savepoints.md
index 4bdc43f6cf617..eada9b4309c19 100644
--- a/docs/setup/savepoints.md
+++ b/docs/setup/savepoints.md
@@ -185,8 +185,3 @@ If you did not assign IDs, the auto generated IDs of the stateful operators will
 If the savepoint was triggered with Flink >= 1.2.0 and using no deprecated state API like `Checkpointed`, you can simply restore the program from a savepoint and specify a new parallelism.
 
 If you are resuming from a savepoint triggered with Flink < 1.2.0 or using now deprecated APIs you first have to migrate your job and savepoint to Flink 1.2.0 before being able to change the parallelism. See the [upgrading jobs and Flink versions guide]({{ site.baseurl }}/ops/upgrading.html).
-
-## Current limitations
-
-- **Chaining**: Chained operators are identified by the ID of the first task. It's not possible to manually assign an ID to an intermediate chained task, e.g. in the chain `[  a -> b -> c ]` only **a** can have its ID assigned manually, but not **b** or **c**. To work around this, you can [manually define the task chains](index.html#task-chaining-and-resource-groups). If you rely on the automatic ID assignment, a change in the chaining behaviour will also change the IDs.
-
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java
index f3ec1cf2f4b5f..7888d2fe6774d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/savepoint/SavepointV0.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.migration.runtime.checkpoint.TaskState;
 import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 import org.apache.flink.runtime.checkpoint.savepoint.Savepoint;
 import org.apache.flink.util.Preconditions;
 
@@ -71,6 +72,11 @@ public Collection<MasterState> getMasterStates() {
 		throw new UnsupportedOperationException();
 	}
 
+	@Override
+	public Collection<OperatorState> getOperatorStates() {
+		return null;
+	}
+
 	@Override
 	public void dispose() throws Exception {
 		//NOP
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index fb6cc72a9c561..96add0681fd0d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -36,6 +36,7 @@
 import org.apache.flink.runtime.executiongraph.JobStatusListener;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
@@ -892,7 +893,7 @@ public void run() {
 		if (LOG.isDebugEnabled()) {
 			StringBuilder builder = new StringBuilder();
 			builder.append("Checkpoint state: ");
-			for (TaskState state : completedCheckpoint.getTaskStates().values()) {
+			for (OperatorState state : completedCheckpoint.getOperatorStates().values()) {
 				builder.append(state);
 				builder.append(", ");
 			}
@@ -1017,11 +1018,11 @@ public boolean restoreLatestCheckpointedState(
 			LOG.info("Restoring from latest valid checkpoint: {}.", latest);
 
 			// re-assign the task states
-
-			final Map<JobVertexID, TaskState> taskStates = latest.getTaskStates();
+			final Map<OperatorID, OperatorState> operatorStates = latest.getOperatorStates();
 
 			StateAssignmentOperation stateAssignmentOperation =
-					new StateAssignmentOperation(LOG, tasks, taskStates, allowNonRestoredState);
+					new StateAssignmentOperation(tasks, operatorStates, allowNonRestoredState);
+
 			stateAssignmentOperation.assignStates();
 
 			// call master hooks for restore
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
index bb49b45637b8b..1ab5b41f0cdc2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java
@@ -18,10 +18,9 @@
 
 package org.apache.flink.runtime.checkpoint;
 
-import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.runtime.state.StreamStateHandle;
@@ -94,8 +93,8 @@ public class CompletedCheckpoint implements Serializable {
 	/** The duration of the checkpoint (completion timestamp - trigger timestamp). */
 	private final long duration;
 
-	/** States of the different task groups belonging to this checkpoint */
-	private final HashMap<JobVertexID, TaskState> taskStates;
+	/** States of the different operator groups belonging to this checkpoint */
+	private final Map<OperatorID, OperatorState> operatorStates;
 
 	/** Properties for this checkpoint. */
 	private final CheckpointProperties props;
@@ -117,38 +116,12 @@ public class CompletedCheckpoint implements Serializable {
 
 	// ------------------------------------------------------------------------
 
-	@VisibleForTesting
-	CompletedCheckpoint(
-			JobID job,
-			long checkpointID,
-			long timestamp,
-			long completionTimestamp,
-			Map<JobVertexID, TaskState> taskStates) {
-
-		this(job, checkpointID, timestamp, completionTimestamp, taskStates,
-				Collections.<MasterState>emptyList(),
-				CheckpointProperties.forStandardCheckpoint());
-	}
-
-	public CompletedCheckpoint(
-			JobID job,
-			long checkpointID,
-			long timestamp,
-			long completionTimestamp,
-			Map<JobVertexID, TaskState> taskStates,
-			@Nullable Collection<MasterState> masterHookStates,
-			CheckpointProperties props) {
-
-		this(job, checkpointID, timestamp, completionTimestamp, taskStates, 
-				masterHookStates, props, null, null);
-	}
-
 	public CompletedCheckpoint(
 			JobID job,
 			long checkpointID,
 			long timestamp,
 			long completionTimestamp,
-			Map<JobVertexID, TaskState> taskStates,
+			Map<OperatorID, OperatorState> operatorStates,
 			@Nullable Collection<MasterState> masterHookStates,
 			CheckpointProperties props,
 			@Nullable StreamStateHandle externalizedMetadata,
@@ -171,7 +144,7 @@ public CompletedCheckpoint(
 
 		// we create copies here, to make sure we have no shared mutable
 		// data structure with the "outside world"
-		this.taskStates = new HashMap<>(checkNotNull(taskStates));
+		this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
 		this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
 				Collections.<MasterState>emptyList() :
 				new ArrayList<>(masterHookStates);
@@ -239,19 +212,15 @@ public boolean discardOnShutdown(JobStatus jobStatus, SharedStateRegistry shared
 	public long getStateSize() {
 		long result = 0L;
 
-		for (TaskState taskState : taskStates.values()) {
-			result += taskState.getStateSize();
+		for (OperatorState operatorState : operatorStates.values()) {
+			result += operatorState.getStateSize();
 		}
 
 		return result;
 	}
 
-	public Map<JobVertexID, TaskState> getTaskStates() {
-		return Collections.unmodifiableMap(taskStates);
-	}
-
-	public TaskState getTaskState(JobVertexID jobVertexID) {
-		return taskStates.get(jobVertexID);
+	public Map<OperatorID, OperatorState> getOperatorStates() {
+		return operatorStates;
 	}
 
 	public Collection<MasterState> getMasterHookStates() {
@@ -288,7 +257,7 @@ void setDiscardCallback(@Nullable CompletedCheckpointStats.DiscardCallback disca
 	 * @param sharedStateRegistry The registry where shared states are registered
 	 */
 	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
-		sharedStateRegistry.registerAll(taskStates.values());
+		sharedStateRegistry.registerAll(operatorStates.values());
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -338,7 +307,7 @@ protected void doDiscardExternalizedMetaData() {
 		protected void doDiscardPrivateState() {
 			// discard private state objects
 			try {
-				StateUtil.bestEffortDiscardAllStateObjects(taskStates.values());
+				StateUtil.bestEffortDiscardAllStateObjects(operatorStates.values());
 			} catch (Exception e) {
 				storedException = ExceptionUtils.firstOrSuppressed(e, storedException);
 			}
@@ -353,7 +322,7 @@ protected void doReportStoredExceptions() throws Exception {
 		}
 
 		protected void clearTaskStatesAndNotifyDiscardCompleted() {
-			taskStates.clear();
+			operatorStates.clear();
 			// to be null-pointer safe, copy reference to stack
 			CompletedCheckpointStats.DiscardCallback discardCallback =
 				CompletedCheckpoint.this.discardCallback;
@@ -392,7 +361,7 @@ public StoredDiscardStrategy(SharedStateRegistry sharedStateRegistry) {
 
 		@Override
 		protected void doDiscardSharedState() {
-			sharedStateRegistry.unregisterAll(taskStates.values());
+			sharedStateRegistry.unregisterAll(operatorStates.values());
 		}
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorState.java
new file mode 100644
index 0000000000000..aa676e71f7ce0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorState.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.state.CompositeStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
+import org.apache.flink.util.Preconditions;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Simple container class which contains the raw/managed/legacy operator state and key-group state handles for the sub
+ * tasks of an operator.
+ */
+public class OperatorState implements CompositeStateHandle {
+
+	private static final long serialVersionUID = -4845578005863201810L;
+
+	/** id of the operator */
+	private final OperatorID operatorID;
+
+	/** handles to non-partitioned states, subtaskindex -> subtaskstate */
+	private final Map<Integer, OperatorSubtaskState> operatorSubtaskStates;
+
+	/** parallelism of the operator when it was checkpointed */
+	private final int parallelism;
+
+	/** maximum parallelism of the operator when the job was first created */
+	private final int maxParallelism;
+
+	public OperatorState(OperatorID operatorID, int parallelism, int maxParallelism) {
+		Preconditions.checkArgument(
+			parallelism <= maxParallelism,
+			"Parallelism " + parallelism + " is not smaller or equal to max parallelism " + maxParallelism + ".");
+
+		this.operatorID = operatorID;
+
+		this.operatorSubtaskStates = new HashMap<>(parallelism);
+
+		this.parallelism = parallelism;
+		this.maxParallelism = maxParallelism;
+	}
+
+	public OperatorID getOperatorID() {
+		return operatorID;
+	}
+
+	public void putState(int subtaskIndex, OperatorSubtaskState subtaskState) {
+		Preconditions.checkNotNull(subtaskState);
+
+		if (subtaskIndex < 0 || subtaskIndex >= parallelism) {
+			throw new IndexOutOfBoundsException("The given sub task index " + subtaskIndex +
+				" exceeds the maximum number of sub tasks " + operatorSubtaskStates.size());
+		} else {
+			operatorSubtaskStates.put(subtaskIndex, subtaskState);
+		}
+	}
+
+	public OperatorSubtaskState getState(int subtaskIndex) {
+		if (subtaskIndex < 0 || subtaskIndex >= parallelism) {
+			throw new IndexOutOfBoundsException("The given sub task index " + subtaskIndex +
+				" exceeds the maximum number of sub tasks " + operatorSubtaskStates.size());
+		} else {
+			return operatorSubtaskStates.get(subtaskIndex);
+		}
+	}
+
+	public Collection<OperatorSubtaskState> getStates() {
+		return operatorSubtaskStates.values();
+	}
+
+	public int getNumberCollectedStates() {
+		return operatorSubtaskStates.size();
+	}
+
+	public int getParallelism() {
+		return parallelism;
+	}
+
+	public int getMaxParallelism() {
+		return maxParallelism;
+	}
+
+	public boolean hasNonPartitionedState() {
+		for (OperatorSubtaskState sts : operatorSubtaskStates.values()) {
+			if (sts != null && sts.getLegacyOperatorState() != null) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	@Override
+	public void discardState() throws Exception {
+		for (OperatorSubtaskState operatorSubtaskState : operatorSubtaskStates.values()) {
+			operatorSubtaskState.discardState();
+		}
+	}
+
+	@Override
+	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
+		for (OperatorSubtaskState operatorSubtaskState : operatorSubtaskStates.values()) {
+			operatorSubtaskState.registerSharedStates(sharedStateRegistry);
+		}
+	}
+
+	@Override
+	public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
+		for (OperatorSubtaskState operatorSubtaskState : operatorSubtaskStates.values()) {
+			operatorSubtaskState.unregisterSharedStates(sharedStateRegistry);
+		}
+	}
+
+	@Override
+	public long getStateSize() {
+		long result = 0L;
+
+		for (int i = 0; i < parallelism; i++) {
+			OperatorSubtaskState operatorSubtaskState = operatorSubtaskStates.get(i);
+			if (operatorSubtaskState != null) {
+				result += operatorSubtaskState.getStateSize();
+			}
+		}
+
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj instanceof OperatorState) {
+			OperatorState other = (OperatorState) obj;
+
+			return operatorID.equals(other.operatorID)
+				&& parallelism == other.parallelism
+				&& operatorSubtaskStates.equals(other.operatorSubtaskStates);
+		} else {
+			return false;
+		}
+	}
+
+	@Override
+	public int hashCode() {
+		return parallelism + 31 * Objects.hash(operatorID, operatorSubtaskStates);
+	}
+
+	public Map<Integer, OperatorSubtaskState> getSubtaskStates() {
+		return Collections.unmodifiableMap(operatorSubtaskStates);
+	}
+
+	@Override
+	public String toString() {
+		// KvStates are always null in 1.1. Don't print this as it might
+		// confuse users that don't care about how we store it internally.
+		return "OperatorState(" +
+			"operatorID: " + operatorID +
+			", parallelism: " + parallelism +
+			", maxParallelism: " + maxParallelism +
+			", sub task states: " + operatorSubtaskStates.size() +
+			", total size (bytes): " + getStateSize() +
+			')';
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java
new file mode 100644
index 0000000000000..863816a2b2353
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.runtime.state.CompositeStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
+import org.apache.flink.runtime.state.StateObject;
+import org.apache.flink.runtime.state.StateUtil;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+
+/**
+ * Container for the state of one parallel subtask of an operator. This is part of the {@link OperatorState}.
+ */
+public class OperatorSubtaskState implements CompositeStateHandle {
+
+	private static final Logger LOG = LoggerFactory.getLogger(OperatorSubtaskState.class);
+
+	private static final long serialVersionUID = -2394696997971923995L;
+
+	/**
+	 * Legacy (non-repartitionable) operator state.
+	 *
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
+	 */
+	@Deprecated
+	private final StreamStateHandle legacyOperatorState;
+
+	/**
+	 * Snapshot from the {@link org.apache.flink.runtime.state.OperatorStateBackend}.
+	 */
+	private final OperatorStateHandle managedOperatorState;
+
+	/**
+	 * Snapshot written using {@link org.apache.flink.runtime.state.OperatorStateCheckpointOutputStream}.
+	 */
+	private final OperatorStateHandle rawOperatorState;
+
+	/**
+	 * Snapshot from {@link org.apache.flink.runtime.state.KeyedStateBackend}.
+	 */
+	private final KeyedStateHandle managedKeyedState;
+
+	/**
+	 * Snapshot written using {@link org.apache.flink.runtime.state.KeyedStateCheckpointOutputStream}.
+	 */
+	private final KeyedStateHandle rawKeyedState;
+
+	/**
+	 * The state size. This is also part of the deserialized state handle.
+	 * We store it here in order to not deserialize the state handle when
+	 * gathering stats.
+	 */
+	private final long stateSize;
+
+	public OperatorSubtaskState(
+		StreamStateHandle legacyOperatorState,
+		OperatorStateHandle managedOperatorState,
+		OperatorStateHandle rawOperatorState,
+		KeyedStateHandle managedKeyedState,
+		KeyedStateHandle rawKeyedState) {
+
+		this.legacyOperatorState = legacyOperatorState;
+		this.managedOperatorState = managedOperatorState;
+		this.rawOperatorState = rawOperatorState;
+		this.managedKeyedState = managedKeyedState;
+		this.rawKeyedState = rawKeyedState;
+
+		try {
+			long calculateStateSize = getSizeNullSafe(legacyOperatorState);
+			calculateStateSize += getSizeNullSafe(managedOperatorState);
+			calculateStateSize += getSizeNullSafe(rawOperatorState);
+			calculateStateSize += getSizeNullSafe(managedKeyedState);
+			calculateStateSize += getSizeNullSafe(rawKeyedState);
+			stateSize = calculateStateSize;
+		} catch (Exception e) {
+			throw new RuntimeException("Failed to get state size.", e);
+		}
+	}
+
+	private static long getSizeNullSafe(StateObject stateObject) throws Exception {
+		return stateObject != null ? stateObject.getStateSize() : 0L;
+	}
+
+	// --------------------------------------------------------------------------------------------
+
+	/**
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
+	 */
+	@Deprecated
+	public StreamStateHandle getLegacyOperatorState() {
+		return legacyOperatorState;
+	}
+
+	public OperatorStateHandle getManagedOperatorState() {
+		return managedOperatorState;
+	}
+
+	public OperatorStateHandle getRawOperatorState() {
+		return rawOperatorState;
+	}
+
+	public KeyedStateHandle getManagedKeyedState() {
+		return managedKeyedState;
+	}
+
+	public KeyedStateHandle getRawKeyedState() {
+		return rawKeyedState;
+	}
+
+	@Override
+	public void discardState() {
+		try {
+			StateUtil.bestEffortDiscardAllStateObjects(
+				Arrays.asList(
+					legacyOperatorState,
+					managedOperatorState,
+					rawOperatorState,
+					managedKeyedState,
+					rawKeyedState));
+		} catch (Exception e) {
+			LOG.warn("Error while discarding operator states.", e);
+		}
+	}
+
+	@Override
+	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
+		// No shared states
+	}
+
+	@Override
+	public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
+		// No shared states
+	}
+
+	@Override
+	public long getStateSize() {
+		return stateSize;
+	}
+
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+
+		OperatorSubtaskState that = (OperatorSubtaskState) o;
+
+		if (stateSize != that.stateSize) {
+			return false;
+		}
+
+		if (legacyOperatorState != null ?
+			!legacyOperatorState.equals(that.legacyOperatorState)
+			: that.legacyOperatorState != null) {
+			return false;
+		}
+		if (managedOperatorState != null ?
+			!managedOperatorState.equals(that.managedOperatorState)
+			: that.managedOperatorState != null) {
+			return false;
+		}
+		if (rawOperatorState != null ?
+			!rawOperatorState.equals(that.rawOperatorState)
+			: that.rawOperatorState != null) {
+			return false;
+		}
+		if (managedKeyedState != null ?
+			!managedKeyedState.equals(that.managedKeyedState)
+			: that.managedKeyedState != null) {
+			return false;
+		}
+		return rawKeyedState != null ?
+			rawKeyedState.equals(that.rawKeyedState)
+			: that.rawKeyedState == null;
+
+	}
+
+	@Override
+	public int hashCode() {
+		int result = legacyOperatorState != null ? legacyOperatorState.hashCode() : 0;
+		result = 31 * result + (managedOperatorState != null ? managedOperatorState.hashCode() : 0);
+		result = 31 * result + (rawOperatorState != null ? rawOperatorState.hashCode() : 0);
+		result = 31 * result + (managedKeyedState != null ? managedKeyedState.hashCode() : 0);
+		result = 31 * result + (rawKeyedState != null ? rawKeyedState.hashCode() : 0);
+		result = 31 * result + (int) (stateSize ^ (stateSize >>> 32));
+		return result;
+	}
+
+	@Override
+	public String toString() {
+		return "SubtaskState{" +
+			"legacyState=" + legacyOperatorState +
+			", operatorStateFromBackend=" + managedOperatorState +
+			", operatorStateFromStream=" + rawOperatorState +
+			", keyedStateFromBackend=" + managedKeyedState +
+			", keyedStateFromStream=" + rawKeyedState +
+			", stateSize=" + stateSize +
+			'}';
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
index cc3dce2e6622e..370032aa5363a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
@@ -26,8 +26,9 @@
 import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
 import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.runtime.state.StreamStateHandle;
@@ -87,7 +88,7 @@ public enum TaskAcknowledgeResult {
 
 	private final long checkpointTimestamp;
 
-	private final Map<JobVertexID, TaskState> taskStates;
+	private final Map<OperatorID, OperatorState> operatorStates;
 
 	private final Map<ExecutionAttemptID, ExecutionVertex> notYetAcknowledgedTasks;
 
@@ -146,7 +147,7 @@ public PendingCheckpoint(
 		this.targetDirectory = targetDirectory;
 		this.executor = Preconditions.checkNotNull(executor);
 
-		this.taskStates = new HashMap<>();
+		this.operatorStates = new HashMap<>();
 		this.masterState = new ArrayList<>();
 		this.acknowledgedTasks = new HashSet<>(verticesToConfirm.size());
 		this.onCompletionPromise = new FlinkCompletableFuture<>();
@@ -178,8 +179,8 @@ public int getNumberOfAcknowledgedTasks() {
 		return numAcknowledgedTasks;
 	}
 
-	public Map<JobVertexID, TaskState> getTaskStates() {
-		return taskStates;
+	public Map<OperatorID, OperatorState> getOperatorStates() {
+		return operatorStates;
 	}
 
 	public boolean isFullyAcknowledged() {
@@ -261,7 +262,7 @@ public CompletedCheckpoint finalizeCheckpointExternalized() throws IOException {
 			// make sure we fulfill the promise with an exception if something fails
 			try {
 				// externalize the metadata
-				final Savepoint savepoint = new SavepointV2(checkpointId, taskStates.values());
+				final Savepoint savepoint = new SavepointV2(checkpointId, operatorStates.values(), masterState);
 
 				// TEMP FIX - The savepoint store is strictly typed to file systems currently
 				//            but the checkpoints think more generic. we need to work with file handles
@@ -326,7 +327,7 @@ private CompletedCheckpoint finalizeInternal(
 				checkpointId,
 				checkpointTimestamp,
 				System.currentTimeMillis(),
-				taskStates,
+				operatorStates,
 				masterState,
 				props,
 				externalMetadata,
@@ -380,41 +381,53 @@ public TaskAcknowledgeResult acknowledgeTask(
 				acknowledgedTasks.add(executionAttemptId);
 			}
 
-			JobVertexID jobVertexID = vertex.getJobvertexId();
+			List<OperatorID> operatorIDs = vertex.getJobVertex().getOperatorIDs();
 			int subtaskIndex = vertex.getParallelSubtaskIndex();
 			long ackTimestamp = System.currentTimeMillis();
 
 			long stateSize = 0;
-			if (null != subtaskState) {
-				TaskState taskState = taskStates.get(jobVertexID);
-
-				if (null == taskState) {
-					@SuppressWarnings("deprecation")
-					ChainedStateHandle<StreamStateHandle> nonPartitionedState = 
-							subtaskState.getLegacyOperatorState();
-					ChainedStateHandle<OperatorStateHandle> partitioneableState =
-							subtaskState.getManagedOperatorState();
-					//TODO this should go away when we remove chained state, assigning state to operators directly instead
-					int chainLength;
-					if (nonPartitionedState != null) {
-						chainLength = nonPartitionedState.getLength();
-					} else if (partitioneableState != null) {
-						chainLength = partitioneableState.getLength();
-					} else {
-						chainLength = 1;
-					}
+			if (subtaskState != null) {
+				stateSize = subtaskState.getStateSize();
 
-					taskState = new TaskState(
-							jobVertexID,
+				@SuppressWarnings("deprecation")
+				ChainedStateHandle<StreamStateHandle> nonPartitionedState =
+					subtaskState.getLegacyOperatorState();
+				ChainedStateHandle<OperatorStateHandle> partitioneableState =
+					subtaskState.getManagedOperatorState();
+				ChainedStateHandle<OperatorStateHandle> rawOperatorState =
+					subtaskState.getRawOperatorState();
+
+				// break task state apart into separate operator states
+				for (int x = 0; x < operatorIDs.size(); x++) {
+					OperatorID operatorID = operatorIDs.get(x);
+					OperatorState operatorState = operatorStates.get(operatorID);
+
+					if (operatorState == null) {
+						operatorState = new OperatorState(
+							operatorID,
 							vertex.getTotalNumberOfParallelSubtasks(),
-							vertex.getMaxParallelism(),
-							chainLength);
+							vertex.getMaxParallelism());
+						operatorStates.put(operatorID, operatorState);
+					}
 
-					taskStates.put(jobVertexID, taskState);
-				}
+					KeyedStateHandle managedKeyedState = null;
+					KeyedStateHandle rawKeyedState = null;
 
-				taskState.putState(subtaskIndex, subtaskState);
-				stateSize = subtaskState.getStateSize();
+					// only the head operator retains the keyed state
+					if (x == operatorIDs.size() - 1) {
+						managedKeyedState = subtaskState.getManagedKeyedState();
+						rawKeyedState = subtaskState.getRawKeyedState();
+					}
+
+					OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState(
+							nonPartitionedState != null ? nonPartitionedState.get(x) : null,
+							partitioneableState != null ? partitioneableState.get(x) : null,
+							rawOperatorState != null ? rawOperatorState.get(x) : null,
+							managedKeyedState,
+							rawKeyedState);
+
+					operatorState.putState(subtaskIndex, operatorSubtaskState);
+				}
 			}
 
 			++numAcknowledgedTasks;
@@ -435,7 +448,7 @@ public TaskAcknowledgeResult acknowledgeTask(
 					metrics.getBytesBufferedInAlignment(),
 					alignmentDurationMillis);
 
-				statsCallback.reportSubtaskStats(jobVertexID, subtaskStateStats);
+				statsCallback.reportSubtaskStats(vertex.getJobvertexId(), subtaskStateStats);
 			}
 
 			return TaskAcknowledgeResult.SUCCESS;
@@ -530,12 +543,12 @@ public void run() {
 							// discard the private states.
 							// unregistered shared states are still considered private at this point.
 							try {
-								StateUtil.bestEffortDiscardAllStateObjects(taskStates.values());
+								StateUtil.bestEffortDiscardAllStateObjects(operatorStates.values());
 							} catch (Throwable t) {
 								LOG.warn("Could not properly dispose the private states in the pending checkpoint {} of job {}.",
 									checkpointId, jobId, t);
 							} finally {
-								taskStates.clear();
+								operatorStates.clear();
 							}
 						}
 					});
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java
index ac70e1aefc088..1042d5aedeb20 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperation.java
@@ -18,9 +18,11 @@
 
 package org.apache.flink.runtime.checkpoint;
 
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.executiongraph.Execution;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.ChainedStateHandle;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
@@ -31,277 +33,400 @@
 import org.apache.flink.runtime.state.TaskStateHandles;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * This class encapsulates the operation of assigning restored state when restoring from a checkpoint.
  */
 public class StateAssignmentOperation {
 
-	private final Logger logger;
+	private static final Logger LOG = LoggerFactory.getLogger(StateAssignmentOperation.class);
+
 	private final Map<JobVertexID, ExecutionJobVertex> tasks;
-	private final Map<JobVertexID, TaskState> taskStates;
+	private final Map<OperatorID, OperatorState> operatorStates;
 	private final boolean allowNonRestoredState;
 
 	public StateAssignmentOperation(
-			Logger logger,
 			Map<JobVertexID, ExecutionJobVertex> tasks,
-			Map<JobVertexID, TaskState> taskStates,
+			Map<OperatorID, OperatorState> operatorStates,
 			boolean allowNonRestoredState) {
 
-		this.logger = Preconditions.checkNotNull(logger);
 		this.tasks = Preconditions.checkNotNull(tasks);
-		this.taskStates = Preconditions.checkNotNull(taskStates);
+		this.operatorStates = Preconditions.checkNotNull(operatorStates);
 		this.allowNonRestoredState = allowNonRestoredState;
 	}
 
 	public boolean assignStates() throws Exception {
-
-		// this tracks if we find missing node hash ids and already use secondary mappings
-		boolean expandedToLegacyIds = false;
-
+		Map<OperatorID, OperatorState> localOperators = new HashMap<>(operatorStates);
 		Map<JobVertexID, ExecutionJobVertex> localTasks = this.tasks;
 
-		for (Map.Entry<JobVertexID, TaskState> taskGroupStateEntry : taskStates.entrySet()) {
-
-			TaskState taskState = taskGroupStateEntry.getValue();
-
-			//----------------------------------------find vertex for state---------------------------------------------
-
-			ExecutionJobVertex executionJobVertex = localTasks.get(taskGroupStateEntry.getKey());
-
-			// on the first time we can not find the execution job vertex for an id, we also consider alternative ids,
-			// for example as generated from older flink versions, to provide backwards compatibility.
-			if (executionJobVertex == null && !expandedToLegacyIds) {
-				localTasks = ExecutionJobVertex.includeLegacyJobVertexIDs(localTasks);
-				executionJobVertex = localTasks.get(taskGroupStateEntry.getKey());
-				expandedToLegacyIds = true;
-				logger.info("Could not find ExecutionJobVertex. Including legacy JobVertexIDs in search.");
-			}
-
-			if (executionJobVertex == null) {
-				if (allowNonRestoredState) {
-					logger.info("Skipped checkpoint state for operator {}.", taskState.getJobVertexID());
-					continue;
+		checkStateMappingCompleteness(allowNonRestoredState, operatorStates, tasks);
+
+		for (Map.Entry<JobVertexID, ExecutionJobVertex> task : localTasks.entrySet()) {
+			final ExecutionJobVertex executionJobVertex = task.getValue();
+
+			// find the states of all operators belonging to this task
+			List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
+			List<OperatorID> altOperatorIDs = executionJobVertex.getUserDefinedOperatorIDs();
+			List<OperatorState> operatorStates = new ArrayList<>();
+			boolean statelessTask = true;
+			for (int x = 0; x < operatorIDs.size(); x++) {
+				OperatorID operatorID = altOperatorIDs.get(x) == null
+					? operatorIDs.get(x)
+					: altOperatorIDs.get(x);
+
+				OperatorState operatorState = localOperators.remove(operatorID);
+				if (operatorState == null) {
+					operatorState = new OperatorState(
+						operatorID,
+						executionJobVertex.getParallelism(),
+						executionJobVertex.getMaxParallelism());
 				} else {
-					throw new IllegalStateException("There is no execution job vertex for the job" +
-							" vertex ID " + taskGroupStateEntry.getKey());
+					statelessTask = false;
 				}
+				operatorStates.add(operatorState);
+			}
+			if (statelessTask) { // skip tasks where no operator has any state
+				continue;
 			}
 
-			checkParallelismPreconditions(taskState, executionJobVertex);
-
-			assignTaskStatesToOperatorInstances(taskState, executionJobVertex);
+			assignAttemptState(task.getValue(), operatorStates);
 		}
 
 		return true;
 	}
 
-	private void checkParallelismPreconditions(TaskState taskState, ExecutionJobVertex executionJobVertex) {
-		//----------------------------------------max parallelism preconditions-------------------------------------
+	private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {
 
-		// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
-		if (taskState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {
+		List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
 
-			if (!executionJobVertex.isMaxParallelismConfigured()) {
-				// if the max parallelism was not explicitly specified by the user, we derive it from the state
+		//1. first compute the new parallelism
+		checkParallelismPreconditions(operatorStates, executionJobVertex);
+
+		int newParallelism = executionJobVertex.getParallelism();
+
+		List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
+			executionJobVertex.getMaxParallelism(),
+			newParallelism);
+
+		//2. Redistribute the operator state.
+		/**
+		 *
+		 * Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
+		 *
+		 * The old ManagedOperatorStates with old parallelism 3:
+		 *
+		 * 		parallelism0 parallelism1 parallelism2
+		 * op0   states0,0    state0,1	   state0,2
+		 * op1
+		 * op2   states2,0    state2,1	   state1,2
+		 * op3   states3,0    state3,1     state3,2
+		 *
+		 * The new ManagedOperatorStates with new parallelism 4:
+		 *
+		 * 		parallelism0 parallelism1 parallelism2 parallelism3
+		 * op0   state0,0	  state0,1 	   state0,2		state0,3
+		 * op1
+		 * op2   state2,0	  state2,1 	   state2,2		state2,3
+		 * op3   state3,0	  state3,1 	   state3,2		state3,3
+		 */
+		List<List<Collection<OperatorStateHandle>>> newManagedOperatorStates = new ArrayList<>();
+		List<List<Collection<OperatorStateHandle>>> newRawOperatorStates = new ArrayList<>();
+
+		reDistributePartitionableStates(operatorStates, newParallelism, newManagedOperatorStates, newRawOperatorStates);
+
+
+		//3. Compute TaskStateHandles of every subTask in the executionJobVertex
+		/**
+		 *  An executionJobVertex's all state handles needed to restore are something like a matrix
+		 *
+		 * 		parallelism0 parallelism1 parallelism2 parallelism3
+		 * op0   sh(0,0)     sh(0,1)       sh(0,2)	    sh(0,3)
+		 * op1   sh(1,0)	 sh(1,1)	   sh(1,2)	    sh(1,3)
+		 * op2   sh(2,0)	 sh(2,1)	   sh(2,2)		sh(2,3)
+		 * op3   sh(3,0)	 sh(3,1)	   sh(3,2)		sh(3,3)
+		 *
+		 * we will compute the state handles column by column.
+		 *
+		 */
+		for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
+
+			Execution currentExecutionAttempt = executionJobVertex.getTaskVertices()[subTaskIndex]
+				.getCurrentExecutionAttempt();
+
+			List<StreamStateHandle> subNonPartitionableState = new ArrayList<>();
+
+			Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> subKeyedState = null;
+
+			List<Collection<OperatorStateHandle>> subManagedOperatorState = new ArrayList<>();
+			List<Collection<OperatorStateHandle>> subRawOperatorState = new ArrayList<>();
+
+
+			for (int operatorIndex = 0; operatorIndex < operatorIDs.size(); operatorIndex++) {
+				OperatorState operatorState = operatorStates.get(operatorIndex);
+				int oldParallelism = operatorState.getParallelism();
+
+				// NonPartitioned State
+
+				reAssignSubNonPartitionedStates(
+					operatorState,
+					subTaskIndex,
+					newParallelism,
+					oldParallelism,
+					subNonPartitionableState);
+
+				// PartitionedState
+				reAssignSubPartitionableState(newManagedOperatorStates,
+					newRawOperatorStates,
+					subTaskIndex,
+					operatorIndex,
+					subManagedOperatorState,
+					subRawOperatorState);
+
+				// KeyedState
+				if (operatorIndex == operatorIDs.size() - 1) {
+					subKeyedState = reAssignSubKeyedStates(operatorState,
+						keyGroupPartitions,
+						subTaskIndex,
+						newParallelism,
+						oldParallelism);
 
-				if (logger.isDebugEnabled()) {
-					logger.debug("Overriding maximum parallelism for JobVertex " + executionJobVertex.getJobVertexId()
-							+ " from " + executionJobVertex.getMaxParallelism() + " to " + taskState.getMaxParallelism());
 				}
+			}
 
-				executionJobVertex.setMaxParallelism(taskState.getMaxParallelism());
-			} else {
-				// if the max parallelism was explicitly specified, we complain on mismatch
-				throw new IllegalStateException("The maximum parallelism (" +
-						taskState.getMaxParallelism() + ") with which the latest " +
-						"checkpoint of the execution job vertex " + executionJobVertex +
-						" has been taken and the current maximum parallelism (" +
-						executionJobVertex.getMaxParallelism() + ") changed. This " +
-						"is currently not supported.");
+
+			// check if a stateless task
+			if (!allElementsAreNull(subNonPartitionableState) ||
+				!allElementsAreNull(subManagedOperatorState) ||
+				!allElementsAreNull(subRawOperatorState) ||
+				subKeyedState != null) {
+
+				TaskStateHandles taskStateHandles = new TaskStateHandles(
+
+					new ChainedStateHandle<>(subNonPartitionableState),
+					subManagedOperatorState,
+					subRawOperatorState,
+					subKeyedState != null ? subKeyedState.f0 : null,
+					subKeyedState != null ? subKeyedState.f1 : null);
+
+				currentExecutionAttempt.setInitialState(taskStateHandles);
 			}
 		}
+	}
 
-		//----------------------------------------parallelism preconditions-----------------------------------------
 
-		final int oldParallelism = taskState.getParallelism();
-		final int newParallelism = executionJobVertex.getParallelism();
+	public void checkParallelismPreconditions(List<OperatorState> operatorStates, ExecutionJobVertex executionJobVertex) {
 
-		if (taskState.hasNonPartitionedState() && (oldParallelism != newParallelism)) {
-			throw new IllegalStateException("Cannot restore the latest checkpoint because " +
-					"the operator " + executionJobVertex.getJobVertexId() + " has non-partitioned " +
-					"state and its parallelism changed. The operator " + executionJobVertex.getJobVertexId() +
-					" has parallelism " + newParallelism + " whereas the corresponding " +
-					"state object has a parallelism of " + oldParallelism);
+		for (OperatorState operatorState : operatorStates) {
+			checkParallelismPreconditions(operatorState, executionJobVertex);
 		}
 	}
 
-	private static void assignTaskStatesToOperatorInstances(
-			TaskState taskState, ExecutionJobVertex executionJobVertex) {
 
-		final int oldParallelism = taskState.getParallelism();
-		final int newParallelism = executionJobVertex.getParallelism();
+	private void reAssignSubPartitionableState(
+			List<List<Collection<OperatorStateHandle>>> newMangedOperatorStates,
+			List<List<Collection<OperatorStateHandle>>> newRawOperatorStates,
+			int subTaskIndex, int operatorIndex,
+			List<Collection<OperatorStateHandle>> subManagedOperatorState,
+			List<Collection<OperatorStateHandle>> subRawOperatorState) {
 
-		List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
-				executionJobVertex.getMaxParallelism(),
-				newParallelism);
-
-		final int chainLength = taskState.getChainLength();
+		if (newMangedOperatorStates.get(operatorIndex) != null) {
+			subManagedOperatorState.add(newMangedOperatorStates.get(operatorIndex).get(subTaskIndex));
+		} else {
+			subManagedOperatorState.add(null);
+		}
+		if (newRawOperatorStates.get(operatorIndex) != null) {
+			subRawOperatorState.add(newRawOperatorStates.get(operatorIndex).get(subTaskIndex));
+		} else {
+			subRawOperatorState.add(null);
+		}
 
-		// operator chain idx -> list of the stored op states from all parallel instances for this chain idx
-		@SuppressWarnings("unchecked")
-		List<OperatorStateHandle>[] parallelOpStatesBackend = new List[chainLength];
-		@SuppressWarnings("unchecked")
-		List<OperatorStateHandle>[] parallelOpStatesStream = new List[chainLength];
 
-		List<KeyedStateHandle> parallelKeyedStatesBackend = new ArrayList<>(oldParallelism);
-		List<KeyedStateHandle> parallelKeyedStateStream = new ArrayList<>(oldParallelism);
+	}
 
-		for (int p = 0; p < oldParallelism; ++p) {
-			SubtaskState subtaskState = taskState.getState(p);
+	private Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> reAssignSubKeyedStates(
+			OperatorState operatorState,
+			List<KeyGroupRange> keyGroupPartitions,
+			int subTaskIndex,
+			int newParallelism,
+			int oldParallelism) {
+
+		Collection<KeyedStateHandle> subManagedKeyedState;
+		Collection<KeyedStateHandle> subRawKeyedState;
+
+		if (newParallelism == oldParallelism) {
+			if (operatorState.getState(subTaskIndex) != null) {
+				KeyedStateHandle oldSubManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState();
+				KeyedStateHandle oldSubRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState();
+				subManagedKeyedState = oldSubManagedKeyedState != null ? Collections.singletonList(
+					oldSubManagedKeyedState) : null;
+				subRawKeyedState = oldSubRawKeyedState != null ? Collections.singletonList(
+					oldSubRawKeyedState) : null;
+			} else {
+				subManagedKeyedState = null;
+				subRawKeyedState = null;
+			}
+		} else {
+			subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex));
+			subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex));
+		}
+		if (subManagedKeyedState == null && subRawKeyedState == null) {
+			return null;
+		}
+		return new Tuple2<>(subManagedKeyedState, subRawKeyedState);
+	}
 
-			if (null != subtaskState) {
-				collectParallelStatesByChainOperator(
-						parallelOpStatesBackend, subtaskState.getManagedOperatorState());
 
-				collectParallelStatesByChainOperator(
-						parallelOpStatesStream, subtaskState.getRawOperatorState());
+	private <X> boolean allElementsAreNull(List<X> nonPartitionableStates) {
+		for (Object streamStateHandle : nonPartitionableStates) {
+			if (streamStateHandle != null) {
+				return false;
+			}
+		}
+		return true;
+	}
 
-				KeyedStateHandle keyedStateBackend = subtaskState.getManagedKeyedState();
-				if (null != keyedStateBackend) {
-					parallelKeyedStatesBackend.add(keyedStateBackend);
-				}
 
-				KeyedStateHandle keyedStateStream = subtaskState.getRawKeyedState();
-				if (null != keyedStateStream) {
-					parallelKeyedStateStream.add(keyedStateStream);
-				}
+	private void reAssignSubNonPartitionedStates(
+			OperatorState operatorState,
+			int subTaskIndex,
+			int newParallelism,
+			int oldParallelism,
+		List<StreamStateHandle> subNonPartitionableState) {
+		if (oldParallelism == newParallelism) {
+			if (operatorState.getState(subTaskIndex) != null) {
+				subNonPartitionableState.add(operatorState.getState(subTaskIndex).getLegacyOperatorState());
+			} else {
+				subNonPartitionableState.add(null);
 			}
+		} else {
+			subNonPartitionableState.add(null);
 		}
+	}
 
-		// operator chain index -> lists with collected states (one collection for each parallel subtasks)
-		@SuppressWarnings("unchecked")
-		List<Collection<OperatorStateHandle>>[] partitionedParallelStatesBackend = new List[chainLength];
+	private void reDistributePartitionableStates(
+			List<OperatorState> operatorStates, int newParallelism,
+			List<List<Collection<OperatorStateHandle>>> newManagedOperatorStates,
+			List<List<Collection<OperatorStateHandle>>> newRawOperatorStates) {
 
-		@SuppressWarnings("unchecked")
-		List<Collection<OperatorStateHandle>>[] partitionedParallelStatesStream = new List[chainLength];
+		//collect the old partitionalbe state
+		List<List<OperatorStateHandle>> oldManagedOperatorStates = new ArrayList<>();
+		List<List<OperatorStateHandle>> oldRawOperatorStates = new ArrayList<>();
 
-		//TODO here we can employ different redistribution strategies for state, e.g. union state.
-		// For now we only offer round robin as the default.
-		OperatorStateRepartitioner opStateRepartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
+		collectPartionableStates(operatorStates, oldManagedOperatorStates, oldRawOperatorStates);
 
-		for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {
 
-			List<OperatorStateHandle> chainOpParallelStatesBackend = parallelOpStatesBackend[chainIdx];
-			List<OperatorStateHandle> chainOpParallelStatesStream = parallelOpStatesStream[chainIdx];
+		//redistribute
+		OperatorStateRepartitioner opStateRepartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
 
-			partitionedParallelStatesBackend[chainIdx] = applyRepartitioner(
-					opStateRepartitioner,
-					chainOpParallelStatesBackend,
-					oldParallelism,
-					newParallelism);
+		for (int operatorIndex = 0; operatorIndex < operatorStates.size(); operatorIndex++) {
+			int oldParallelism = operatorStates.get(operatorIndex).getParallelism();
+			newManagedOperatorStates.add(applyRepartitioner(opStateRepartitioner,
+				oldManagedOperatorStates.get(operatorIndex), oldParallelism, newParallelism));
+			newRawOperatorStates.add(applyRepartitioner(opStateRepartitioner,
+				oldRawOperatorStates.get(operatorIndex), oldParallelism, newParallelism));
 
-			partitionedParallelStatesStream[chainIdx] = applyRepartitioner(
-					opStateRepartitioner,
-					chainOpParallelStatesStream,
-					oldParallelism,
-					newParallelism);
 		}
+	}
 
-		for (int subTaskIdx = 0; subTaskIdx < newParallelism; ++subTaskIdx) {
-			// non-partitioned state
-			ChainedStateHandle<StreamStateHandle> nonPartitionableState = null;
-
-			if (oldParallelism == newParallelism) {
-				if (taskState.getState(subTaskIdx) != null) {
-					nonPartitionableState = taskState.getState(subTaskIdx).getLegacyOperatorState();
-				}
-			}
 
-			// partitionable state
-			@SuppressWarnings("unchecked")
-			Collection<OperatorStateHandle>[] iab = new Collection[chainLength];
-			@SuppressWarnings("unchecked")
-			Collection<OperatorStateHandle>[] ias = new Collection[chainLength];
-			List<Collection<OperatorStateHandle>> operatorStateFromBackend = Arrays.asList(iab);
-			List<Collection<OperatorStateHandle>> operatorStateFromStream = Arrays.asList(ias);
+	private void collectPartionableStates(
+			List<OperatorState> operatorStates,
+			List<List<OperatorStateHandle>> managedOperatorStates,
+			List<List<OperatorStateHandle>> rawOperatorStates) {
 
-			for (int chainIdx = 0; chainIdx < partitionedParallelStatesBackend.length; ++chainIdx) {
-				List<Collection<OperatorStateHandle>> redistributedOpStateBackend =
-						partitionedParallelStatesBackend[chainIdx];
+		for (OperatorState operatorState : operatorStates) {
+			List<OperatorStateHandle> managedOperatorState = null;
+			List<OperatorStateHandle> rawOperatorState = null;
 
-				List<Collection<OperatorStateHandle>> redistributedOpStateStream =
-						partitionedParallelStatesStream[chainIdx];
+			for (int i = 0; i < operatorState.getParallelism(); i++) {
+				OperatorSubtaskState operatorSubtaskState = operatorState.getState(i);
+				if (operatorSubtaskState != null) {
+					if (operatorSubtaskState.getManagedOperatorState() != null) {
+						if (managedOperatorState == null) {
+							managedOperatorState = new ArrayList<>();
+						}
+						managedOperatorState.add(operatorSubtaskState.getManagedOperatorState());
+					}
 
-				if (redistributedOpStateBackend != null) {
-					operatorStateFromBackend.set(chainIdx, redistributedOpStateBackend.get(subTaskIdx));
+					if (operatorSubtaskState.getRawOperatorState() != null) {
+						if (rawOperatorState == null) {
+							rawOperatorState = new ArrayList<>();
+						}
+						rawOperatorState.add(operatorSubtaskState.getRawOperatorState());
+					}
 				}
 
-				if (redistributedOpStateStream != null) {
-					operatorStateFromStream.set(chainIdx, redistributedOpStateStream.get(subTaskIdx));
-				}
 			}
+			managedOperatorStates.add(managedOperatorState);
+			rawOperatorStates.add(rawOperatorState);
+		}
+	}
 
-			Execution currentExecutionAttempt = executionJobVertex
-					.getTaskVertices()[subTaskIdx]
-					.getCurrentExecutionAttempt();
-
-			List<KeyedStateHandle> newKeyedStatesBackend;
-			List<KeyedStateHandle> newKeyedStateStream;
-			if (oldParallelism == newParallelism) {
-				SubtaskState subtaskState = taskState.getState(subTaskIdx);
-				if (subtaskState != null) {
-					KeyedStateHandle oldKeyedStatesBackend = subtaskState.getManagedKeyedState();
-					KeyedStateHandle oldKeyedStatesStream = subtaskState.getRawKeyedState();
-					newKeyedStatesBackend = oldKeyedStatesBackend != null ? Collections.singletonList(
-							oldKeyedStatesBackend) : null;
-					newKeyedStateStream = oldKeyedStatesStream != null ? Collections.singletonList(
-							oldKeyedStatesStream) : null;
-				} else {
-					newKeyedStatesBackend = null;
-					newKeyedStateStream = null;
-				}
-			} else {
-				KeyGroupRange subtaskKeyGroupIds = keyGroupPartitions.get(subTaskIdx);
-				newKeyedStatesBackend = getKeyedStateHandles(parallelKeyedStatesBackend, subtaskKeyGroupIds);
-				newKeyedStateStream = getKeyedStateHandles(parallelKeyedStateStream, subtaskKeyGroupIds);
-			}
 
-			TaskStateHandles taskStateHandles = new TaskStateHandles(
-					nonPartitionableState,
-					operatorStateFromBackend,
-					operatorStateFromStream,
-					newKeyedStatesBackend,
-					newKeyedStateStream);
+	/**
+	 * Collect {@link KeyGroupsStateHandle  managedKeyedStateHandles} which have intersection with given
+	 * {@link KeyGroupRange} from {@link TaskState operatorState}
+	 *
+	 * @param operatorState        all state handles of a operator
+	 * @param subtaskKeyGroupRange the KeyGroupRange of a subtask
+	 * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange
+	 */
+	public static List<KeyedStateHandle> getManagedKeyedStateHandles(
+			OperatorState operatorState,
+			KeyGroupRange subtaskKeyGroupRange) {
+
+		List<KeyedStateHandle> subtaskKeyedStateHandles = null;
+
+		for (int i = 0; i < operatorState.getParallelism(); i++) {
+			if (operatorState.getState(i) != null && operatorState.getState(i).getManagedKeyedState() != null) {
+				KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getManagedKeyedState().getIntersection(subtaskKeyGroupRange);
 
-			currentExecutionAttempt.setInitialState(taskStateHandles);
+				if (intersectedKeyedStateHandle != null) {
+					if (subtaskKeyedStateHandles == null) {
+						subtaskKeyedStateHandles = new ArrayList<>();
+					}
+					subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
+				}
+			}
 		}
+
+		return subtaskKeyedStateHandles;
 	}
 
 	/**
-	 * Determine the subset of {@link KeyGroupsStateHandle KeyGroupsStateHandles} with correct
-	 * key group index for the given subtask {@link KeyGroupRange}.
-	 * <p>
-	 * <p>This is publicly visible to be used in tests.
+	 * Collect {@link KeyGroupsStateHandle  rawKeyedStateHandles} which have intersection with given
+	 * {@link KeyGroupRange} from {@link TaskState operatorState}
+	 *
+	 * @param operatorState        all state handles of a operator
+	 * @param subtaskKeyGroupRange the KeyGroupRange of a subtask
+	 * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange
 	 */
-	public static List<KeyedStateHandle> getKeyedStateHandles(
-			Collection<? extends KeyedStateHandle> keyedStateHandles,
-			KeyGroupRange subtaskKeyGroupRange) {
+	public static List<KeyedStateHandle> getRawKeyedStateHandles(
+		OperatorState operatorState,
+		KeyGroupRange subtaskKeyGroupRange) {
 
-		List<KeyedStateHandle> subtaskKeyedStateHandles = new ArrayList<>();
+		List<KeyedStateHandle> subtaskKeyedStateHandles = null;
 
-		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {
-			KeyedStateHandle intersectedKeyedStateHandle = keyedStateHandle.getIntersection(subtaskKeyGroupRange);
+		for (int i = 0; i < operatorState.getParallelism(); i++) {
+			if (operatorState.getState(i) != null && operatorState.getState(i).getRawKeyedState() != null) {
+				KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getRawKeyedState().getIntersection(subtaskKeyGroupRange);
 
-			if (intersectedKeyedStateHandle != null) {
-				subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
+				if (intersectedKeyedStateHandle != null) {
+					if (subtaskKeyedStateHandles == null) {
+						subtaskKeyedStateHandles = new ArrayList<>();
+					}
+					subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
+				}
 			}
 		}
 
@@ -331,37 +456,90 @@ public static List<KeyGroupRange> createKeyGroupPartitions(int numberKeyGroups,
 	}
 
 	/**
-	 * @param chainParallelOpStates array = chain ops, array[idx] = parallel states for this chain op.
-	 * @param chainOpState the operator chain
+	 * Verifies conditions in regards to parallelism and maxParallelism that must be met when restoring state.
+	 *
+	 * @param operatorState      state to restore
+	 * @param executionJobVertex task for which the state should be restored
 	 */
-	private static void collectParallelStatesByChainOperator(
-			List<OperatorStateHandle>[] chainParallelOpStates, ChainedStateHandle<OperatorStateHandle> chainOpState) {
+	private static void checkParallelismPreconditions(OperatorState operatorState, ExecutionJobVertex executionJobVertex) {
+		//----------------------------------------max parallelism preconditions-------------------------------------
 
-		if (null != chainOpState) {
+		// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
+		if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {
 
-			int chainLength = chainOpState.getLength();
-			Preconditions.checkState(chainLength >= chainParallelOpStates.length,
-					"Found more states than operators in the chain. Chain length: " + chainLength +
-							", States: " + chainParallelOpStates.length);
+			if (!executionJobVertex.isMaxParallelismConfigured()) {
+				// if the max parallelism was not explicitly specified by the user, we derive it from the state
 
-			for (int chainIdx = 0; chainIdx < chainParallelOpStates.length; ++chainIdx) {
-				OperatorStateHandle operatorState = chainOpState.get(chainIdx);
+				LOG.debug("Overriding maximum parallelism for JobVertex {} from {} to {}",
+					executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism());
 
-				if (null != operatorState) {
+				executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism());
+			} else {
+				// if the max parallelism was explicitly specified, we complain on mismatch
+				throw new IllegalStateException("The maximum parallelism (" +
+					operatorState.getMaxParallelism() + ") with which the latest " +
+					"checkpoint of the execution job vertex " + executionJobVertex +
+					" has been taken and the current maximum parallelism (" +
+					executionJobVertex.getMaxParallelism() + ") changed. This " +
+					"is currently not supported.");
+			}
+		}
 
-					List<OperatorStateHandle> opParallelStatesForOneChainOp = chainParallelOpStates[chainIdx];
+		//----------------------------------------parallelism preconditions-----------------------------------------
 
-					if (null == opParallelStatesForOneChainOp) {
-						opParallelStatesForOneChainOp = new ArrayList<>();
-						chainParallelOpStates[chainIdx] = opParallelStatesForOneChainOp;
-					}
-					opParallelStatesForOneChainOp.add(operatorState);
+		final int oldParallelism = operatorState.getParallelism();
+		final int newParallelism = executionJobVertex.getParallelism();
+
+		if (operatorState.hasNonPartitionedState() && (oldParallelism != newParallelism)) {
+			throw new IllegalStateException("Cannot restore the latest checkpoint because " +
+				"the operator " + executionJobVertex.getJobVertexId() + " has non-partitioned " +
+				"state and its parallelism changed. The operator " + executionJobVertex.getJobVertexId() +
+				" has parallelism " + newParallelism + " whereas the corresponding " +
+				"state object has a parallelism of " + oldParallelism);
+		}
+	}
+
+	/**
+	 * Verifies that all operator states can be mapped to an execution job vertex.
+	 *
+	 * @param allowNonRestoredState if false an exception will be thrown if a state could not be mapped
+	 * @param operatorStates operator states to map
+	 * @param tasks task to map to
+	 */
+	private static void checkStateMappingCompleteness(
+			boolean allowNonRestoredState,
+			Map<OperatorID, OperatorState> operatorStates,
+			Map<JobVertexID, ExecutionJobVertex> tasks) {
+
+		Set<OperatorID> allOperatorIDs = new HashSet<>();
+		for (ExecutionJobVertex executionJobVertex : tasks.values()) {
+			allOperatorIDs.addAll(executionJobVertex.getOperatorIDs());
+		}
+		for (Map.Entry<OperatorID, OperatorState> operatorGroupStateEntry : operatorStates.entrySet()) {
+			OperatorState operatorState = operatorGroupStateEntry.getValue();
+			//----------------------------------------find operator for state---------------------------------------------
+
+			if (!allOperatorIDs.contains(operatorGroupStateEntry.getKey())) {
+				if (allowNonRestoredState) {
+					LOG.info("Skipped checkpoint state for operator {}.", operatorState.getOperatorID());
+				} else {
+					throw new IllegalStateException("There is no operator for the state " + operatorState.getOperatorID());
 				}
 			}
 		}
 	}
 
-	private static List<Collection<OperatorStateHandle>> applyRepartitioner(
+	/**
+	 * Repartitions the given operator state using the given {@link OperatorStateRepartitioner} with respect to the new
+	 * parallelism.
+	 *
+	 * @param opStateRepartitioner  partitioner to use
+	 * @param chainOpParallelStates state to repartition
+	 * @param oldParallelism        parallelism with which the state is currently partitioned
+	 * @param newParallelism        parallelism with which the state should be partitioned
+	 * @return repartitioned state
+	 */
+	public static List<Collection<OperatorStateHandle>> applyRepartitioner(
 			OperatorStateRepartitioner opStateRepartitioner,
 			List<OperatorStateHandle> chainOpParallelStates,
 			int oldParallelism,
@@ -399,4 +577,27 @@ private static List<Collection<OperatorStateHandle>> applyRepartitioner(
 			return repackStream;
 		}
 	}
-}
\ No newline at end of file
+
+	/**
+	 * Determine the subset of {@link KeyGroupsStateHandle KeyGroupsStateHandles} with correct
+	 * key group index for the given subtask {@link KeyGroupRange}.
+	 * <p>
+	 * <p>This is publicly visible to be used in tests.
+	 */
+	public static List<KeyedStateHandle> getKeyedStateHandles(
+		Collection<? extends KeyedStateHandle> keyedStateHandles,
+		KeyGroupRange subtaskKeyGroupRange) {
+
+		List<KeyedStateHandle> subtaskKeyedStateHandles = new ArrayList<>();
+
+		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {
+			KeyedStateHandle intersectedKeyedStateHandle = keyedStateHandle.getIntersection(subtaskKeyGroupRange);
+
+			if (intersectedKeyedStateHandle != null) {
+				subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
+			}
+		}
+
+		return subtaskKeyedStateHandles;
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
deleted file mode 100644
index 83c188c78dae2..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.checkpoint;
-
-//import com.google.common.collect.Lists;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.executiongraph.Execution;
-import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.state.ChainedStateHandle;
-import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.KeyGroupsStateHandle;
-import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.OperatorStateHandle;
-import org.apache.flink.runtime.state.StreamStateHandle;
-import org.apache.flink.runtime.state.TaskStateHandles;
-import org.apache.flink.util.Preconditions;
-import org.slf4j.Logger;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * This class encapsulates the operation of assigning restored state when restoring from a checkpoint that works on the
- * granularity of operators. This is the case for checkpoints that were created either with a Flink version >= 1.3 or
- * 1.2 if the savepoint only contains {@link SubtaskState}s for which the length of contained
- * {@link ChainedStateHandle}s is equal to 1.
- */
-public class StateAssignmentOperationV2 {
-
-	private final Logger logger;
-	private final Map<JobVertexID, ExecutionJobVertex> tasks;
-	private final Map<JobVertexID, TaskState> taskStates;
-	private final boolean allowNonRestoredState;
-
-	public StateAssignmentOperationV2(
-			Logger logger,
-			Map<JobVertexID, ExecutionJobVertex> tasks,
-			Map<JobVertexID, TaskState> taskStates,
-			boolean allowNonRestoredState) {
-
-		this.logger = Preconditions.checkNotNull(logger);
-		this.tasks = Preconditions.checkNotNull(tasks);
-		this.taskStates = Preconditions.checkNotNull(taskStates);
-		this.allowNonRestoredState = allowNonRestoredState;
-	}
-
-	public boolean assignStates() throws Exception {
-		Map<JobVertexID, TaskState> localStates = new HashMap<>(taskStates);
-		Map<JobVertexID, ExecutionJobVertex> localTasks = this.tasks;
-
-		Set<JobVertexID> allOperatorIDs = new HashSet<>();
-		for (ExecutionJobVertex executionJobVertex : tasks.values()) {
-			//allOperatorIDs.addAll(Lists.newArrayList(executionJobVertex.getOperatorIDs()));
-		}
-		for (Map.Entry<JobVertexID, TaskState> taskGroupStateEntry : taskStates.entrySet()) {
-			TaskState taskState = taskGroupStateEntry.getValue();
-			//----------------------------------------find operator for state---------------------------------------------
-
-			if (!allOperatorIDs.contains(taskGroupStateEntry.getKey())) {
-				if (allowNonRestoredState) {
-					logger.info("Skipped checkpoint state for operator {}.", taskState.getJobVertexID());
-					continue;
-				} else {
-					throw new IllegalStateException("There is no operator for the state " + taskState.getJobVertexID());
-				}
-			}
-		}
-
-		for (Map.Entry<JobVertexID, ExecutionJobVertex> task : localTasks.entrySet()) {
-			final ExecutionJobVertex executionJobVertex = task.getValue();
-
-			// find the states of all operators belonging to this task
-			JobVertexID[] operatorIDs = null;//executionJobVertex.getOperatorIDs();
-			JobVertexID[] altOperatorIDs = null;//executionJobVertex.getUserDefinedOperatorIDs();
-			List<TaskState> operatorStates = new ArrayList<>();
-			boolean statelessTask = true;
-			for (int x = 0; x < operatorIDs.length; x++) {
-				JobVertexID operatorID = altOperatorIDs[x] == null
-					? operatorIDs[x]
-					: altOperatorIDs[x];
-
-				TaskState operatorState = localStates.remove(operatorID);
-				if (operatorState == null) {
-					operatorState = new TaskState(
-						operatorID,
-						executionJobVertex.getParallelism(),
-						executionJobVertex.getMaxParallelism(),
-						1);
-				} else {
-					statelessTask = false;
-				}
-				operatorStates.add(operatorState);
-			}
-			if (statelessTask) { // skip tasks where no operator has any state
-				continue;
-			}
-
-			assignAttemptState(task.getValue(), operatorStates);
-		}
-
-		return true;
-	}
-
-	private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<TaskState> operatorStates) {
-
-		JobVertexID[] operatorIDs = null;//executionJobVertex.getOperatorIDs();
-
-		//1. first compute the new parallelism
-		checkParallelismPreconditions(operatorStates, executionJobVertex);
-
-		int newParallelism = executionJobVertex.getParallelism();
-
-		List<KeyGroupRange> keyGroupPartitions = null;//StateAssignmentOperationUtils.createKeyGroupPartitions(
-			//executionJobVertex.getMaxParallelism(),
-			//newParallelism);
-
-		//2. Redistribute the operator state.
-		/**
-		 *
-		 * Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
-		 *
-		 * The old ManagedOperatorStates with old parallelism 3:
-		 *
-		 * 		parallelism0 parallelism1 parallelism2
-		 * op0   states0,0    state0,1	   state0,2
-		 * op1
-		 * op2   states2,0    state2,1	   state1,2
-		 * op3   states3,0    state3,1     state3,2
-		 *
-		 * The new ManagedOperatorStates with new parallelism 4:
-		 *
-		 * 		parallelism0 parallelism1 parallelism2 parallelism3
-		 * op0   state0,0	  state0,1 	   state0,2		state0,3
-		 * op1
-		 * op2   state2,0	  state2,1 	   state2,2		state2,3
-		 * op3   state3,0	  state3,1 	   state3,2		state3,3
-		 */
-		List<List<Collection<OperatorStateHandle>>> newManagedOperatorStates = new ArrayList<>();
-		List<List<Collection<OperatorStateHandle>>> newRawOperatorStates = new ArrayList<>();
-
-		reDistributePartitionableStates(operatorStates, newParallelism, newManagedOperatorStates, newRawOperatorStates);
-
-
-		//3. Compute TaskStateHandles of every subTask in the executionJobVertex
-		/**
-		 *  An executionJobVertex's all state handles needed to restore are something like a matrix
-		 *
-		 * 		parallelism0 parallelism1 parallelism2 parallelism3
-		 * op0   sh(0,0)     sh(0,1)       sh(0,2)	    sh(0,3)
-		 * op1   sh(1,0)	 sh(1,1)	   sh(1,2)	    sh(1,3)
-		 * op2   sh(2,0)	 sh(2,1)	   sh(2,2)		sh(2,3)
-		 * op3   sh(3,0)	 sh(3,1)	   sh(3,2)		sh(3,3)
-		 *
-		 * we will compute the state handles column by column.
-		 *
-		 */
-		for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
-
-			Execution currentExecutionAttempt = executionJobVertex.getTaskVertices()[subTaskIndex]
-				.getCurrentExecutionAttempt();
-
-			List<StreamStateHandle> subNonPartitionableState = new ArrayList<>();
-
-			Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> subKeyedState = null;
-
-			List<Collection<OperatorStateHandle>> subManagedOperatorState = new ArrayList<>();
-			List<Collection<OperatorStateHandle>> subRawOperatorState = new ArrayList<>();
-
-
-			for (int operatorIndex = 0; operatorIndex < operatorIDs.length; operatorIndex++) {
-				TaskState operatorState = operatorStates.get(operatorIndex);
-				int oldParallelism = operatorState.getParallelism();
-
-				// NonPartitioned State
-
-				reAssignSubNonPartitionedStates(
-					operatorState,
-					subTaskIndex,
-					newParallelism,
-					oldParallelism,
-					subNonPartitionableState);
-
-				// PartitionedState
-				reAssignSubPartitionableState(newManagedOperatorStates,
-					newRawOperatorStates,
-					subTaskIndex,
-					operatorIndex,
-					subManagedOperatorState,
-					subRawOperatorState);
-
-				// KeyedState
-				if (operatorIndex == operatorIDs.length - 1) {
-					subKeyedState = reAssignSubKeyedStates(operatorState,
-						keyGroupPartitions,
-						subTaskIndex,
-						newParallelism,
-						oldParallelism);
-
-				}
-			}
-
-
-			// check if a stateless task
-			if (!allElementsAreNull(subNonPartitionableState) ||
-				!allElementsAreNull(subManagedOperatorState) ||
-				!allElementsAreNull(subRawOperatorState) ||
-				subKeyedState != null) {
-
-				TaskStateHandles taskStateHandles = new TaskStateHandles(
-
-					new ChainedStateHandle<>(subNonPartitionableState),
-					subManagedOperatorState,
-					subRawOperatorState,
-					subKeyedState != null ? subKeyedState.f0 : null,
-					subKeyedState != null ? subKeyedState.f1 : null);
-
-				currentExecutionAttempt.setInitialState(taskStateHandles);
-			}
-		}
-	}
-
-
-	public void checkParallelismPreconditions(List<TaskState> operatorStates, ExecutionJobVertex executionJobVertex) {
-
-		for (TaskState taskState : operatorStates) {
-			//StateAssignmentOperation.checkParallelismPreconditions(taskState, executionJobVertex, this.logger);
-		}
-	}
-
-
-	private void reAssignSubPartitionableState(
-			List<List<Collection<OperatorStateHandle>>> newMangedOperatorStates,
-			List<List<Collection<OperatorStateHandle>>> newRawOperatorStates,
-			int subTaskIndex, int operatorIndex,
-			List<Collection<OperatorStateHandle>> subManagedOperatorState,
-			List<Collection<OperatorStateHandle>> subRawOperatorState) {
-
-		if (newMangedOperatorStates.get(operatorIndex) != null) {
-			subManagedOperatorState.add(newMangedOperatorStates.get(operatorIndex).get(subTaskIndex));
-		} else {
-			subManagedOperatorState.add(null);
-		}
-		if (newRawOperatorStates.get(operatorIndex) != null) {
-			subRawOperatorState.add(newRawOperatorStates.get(operatorIndex).get(subTaskIndex));
-		} else {
-			subRawOperatorState.add(null);
-		}
-
-
-	}
-
-	private Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> reAssignSubKeyedStates(
-			TaskState operatorState,
-			List<KeyGroupRange> keyGroupPartitions,
-			int subTaskIndex,
-			int newParallelism,
-			int oldParallelism) {
-
-		Collection<KeyedStateHandle> subManagedKeyedState;
-		Collection<KeyedStateHandle> subRawKeyedState;
-
-		if (newParallelism == oldParallelism) {
-			if (operatorState.getState(subTaskIndex) != null) {
-				KeyedStateHandle oldSubManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState();
-				KeyedStateHandle oldSubRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState();
-				subManagedKeyedState = oldSubManagedKeyedState != null ? Collections.singletonList(
-					oldSubManagedKeyedState) : null;
-				subRawKeyedState = oldSubRawKeyedState != null ? Collections.singletonList(
-					oldSubRawKeyedState) : null;
-			} else {
-				subManagedKeyedState = null;
-				subRawKeyedState = null;
-			}
-		} else {
-			subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex));
-			subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex));
-		}
-		if (subManagedKeyedState == null && subRawKeyedState == null) {
-			return null;
-		}
-		return new Tuple2<>(subManagedKeyedState, subRawKeyedState);
-	}
-
-
-	private <X> boolean allElementsAreNull(List<X> nonPartitionableStates) {
-		for (Object streamStateHandle : nonPartitionableStates) {
-			if (streamStateHandle != null) {
-				return false;
-			}
-		}
-		return true;
-	}
-
-
-	private void reAssignSubNonPartitionedStates(
-			TaskState operatorState,
-			int subTaskIndex,
-			int newParallelism,
-			int oldParallelism,
-			List<StreamStateHandle> subNonPartitionableState) {
-		if (oldParallelism == newParallelism) {
-			if (operatorState.getState(subTaskIndex) != null &&
-				!operatorState.getState(subTaskIndex).getLegacyOperatorState().isEmpty()) {
-				subNonPartitionableState.add(operatorState.getState(subTaskIndex).getLegacyOperatorState().get(0));
-			} else {
-				subNonPartitionableState.add(null);
-			}
-		} else {
-			subNonPartitionableState.add(null);
-		}
-	}
-
-	private void reDistributePartitionableStates(
-			List<TaskState> operatorStates, int newParallelism,
-			List<List<Collection<OperatorStateHandle>>> newManagedOperatorStates,
-			List<List<Collection<OperatorStateHandle>>> newRawOperatorStates) {
-
-		//collect the old partitionalbe state
-		List<List<OperatorStateHandle>> oldManagedOperatorStates = new ArrayList<>();
-		List<List<OperatorStateHandle>> oldRawOperatorStates = new ArrayList<>();
-
-		collectPartionableStates(operatorStates, oldManagedOperatorStates, oldRawOperatorStates);
-
-
-		//redistribute
-		OperatorStateRepartitioner opStateRepartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
-
-		for (int operatorIndex = 0; operatorIndex < operatorStates.size(); operatorIndex++) {
-			int oldParallelism = operatorStates.get(operatorIndex).getParallelism();
-			//newManagedOperatorStates.add(StateAssignmentOperationUtils.applyRepartitioner(opStateRepartitioner,
-			//	oldManagedOperatorStates.get(operatorIndex), oldParallelism, newParallelism));
-			//newRawOperatorStates.add(StateAssignmentOperationUtils.applyRepartitioner(opStateRepartitioner,
-			//	oldRawOperatorStates.get(operatorIndex), oldParallelism, newParallelism));
-
-		}
-	}
-
-
-	private void collectPartionableStates(
-			List<TaskState> operatorStates,
-			List<List<OperatorStateHandle>> managedOperatorStates,
-			List<List<OperatorStateHandle>> rawOperatorStates) {
-
-		for (TaskState operatorState : operatorStates) {
-			List<OperatorStateHandle> managedOperatorState = null;
-			List<OperatorStateHandle> rawOperatorState = null;
-
-			for (int i = 0; i < operatorState.getParallelism(); i++) {
-				SubtaskState subtaskState = operatorState.getState(i);
-				if (subtaskState != null) {
-					if (subtaskState.getManagedOperatorState() != null &&
-						subtaskState.getManagedOperatorState().getLength() > 0 &&
-						subtaskState.getManagedOperatorState().get(0) != null) {
-						if (managedOperatorState == null) {
-							managedOperatorState = new ArrayList<>();
-						}
-						managedOperatorState.add(subtaskState.getManagedOperatorState().get(0));
-					}
-
-					if (subtaskState.getRawOperatorState() != null &&
-						subtaskState.getRawOperatorState().getLength() > 0 &&
-						subtaskState.getRawOperatorState().get(0) != null) {
-						if (rawOperatorState == null) {
-							rawOperatorState = new ArrayList<>();
-						}
-						rawOperatorState.add(subtaskState.getRawOperatorState().get(0));
-					}
-				}
-
-			}
-			managedOperatorStates.add(managedOperatorState);
-			rawOperatorStates.add(rawOperatorState);
-		}
-	}
-
-
-	/**
-	 * Collect {@link KeyGroupsStateHandle  managedKeyedStateHandles} which have intersection with given
-	 * {@link KeyGroupRange} from {@link TaskState operatorState}
-	 *
-	 * @param operatorState        all state handles of a operator
-	 * @param subtaskKeyGroupRange the KeyGroupRange of a subtask
-	 * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange
-	 */
-	public static List<KeyedStateHandle> getManagedKeyedStateHandles(
-			TaskState operatorState,
-			KeyGroupRange subtaskKeyGroupRange) {
-
-		List<KeyedStateHandle> subtaskKeyedStateHandles = null;
-
-		for (int i = 0; i < operatorState.getParallelism(); i++) {
-			if (operatorState.getState(i) != null && operatorState.getState(i).getManagedKeyedState() != null) {
-				KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getManagedKeyedState().getIntersection(subtaskKeyGroupRange);
-
-				if (intersectedKeyedStateHandle != null) {
-					if (subtaskKeyedStateHandles == null) {
-						subtaskKeyedStateHandles = new ArrayList<>();
-					}
-					subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
-				}
-			}
-		}
-
-		return subtaskKeyedStateHandles;
-	}
-
-	/**
-	 * Collect {@link KeyGroupsStateHandle  rawKeyedStateHandles} which have intersection with given
-	 * {@link KeyGroupRange} from {@link TaskState operatorState}
-	 *
-	 * @param operatorState        all state handles of a operator
-	 * @param subtaskKeyGroupRange the KeyGroupRange of a subtask
-	 * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange
-	 */
-	public static List<KeyedStateHandle> getRawKeyedStateHandles(
-			TaskState operatorState,
-			KeyGroupRange subtaskKeyGroupRange) {
-
-		List<KeyedStateHandle> subtaskKeyedStateHandles = null;
-
-		for (int i = 0; i < operatorState.getParallelism(); i++) {
-			if (operatorState.getState(i) != null && operatorState.getState(i).getRawKeyedState() != null) {
-				KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getRawKeyedState().getIntersection(subtaskKeyGroupRange);
-
-				if (intersectedKeyedStateHandle != null) {
-					if (subtaskKeyedStateHandles == null) {
-						subtaskKeyedStateHandles = new ArrayList<>();
-					}
-					subtaskKeyedStateHandles.add(intersectedKeyedStateHandle);
-				}
-			}
-		}
-
-		return subtaskKeyedStateHandles;
-	}
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
index 4f5f536bcb065..aa5c516a6be29 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/TaskState.java
@@ -34,7 +34,11 @@
  * tasks of a {@link org.apache.flink.runtime.jobgraph.JobVertex}.
  *
  * This class basically groups all non-partitioned state and key-group state belonging to the same job vertex together.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
+@Deprecated
+@SuppressWarnings("deprecation")
 public class TaskState implements CompositeStateHandle {
 
 	private static final long serialVersionUID = -4845578005863201810L;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java
index 79ec59630bf73..a7cf4b5dd1226 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/Savepoint.java
@@ -21,6 +21,7 @@
 import org.apache.flink.core.io.Versioned;
 import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
 import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 
 import java.util.Collection;
@@ -54,8 +55,10 @@ public interface Savepoint extends Versioned {
 	 *
 	 * <p>These are used to restore the snapshot state.
 	 *
+	 * @deprecated Only kept for backwards-compatibility with versionS < 1.3. Will be removed in the future.
 	 * @return Snapshotted task states
 	 */
+	@Deprecated
 	Collection<TaskState> getTaskStates();
 
 	/**
@@ -63,6 +66,15 @@ public interface Savepoint extends Versioned {
 	 */
 	Collection<MasterState> getMasterStates();
 
+	/**
+	 * Returns the snapshotted operator states.
+	 *
+	 * <p>These are used to restore the snapshot state.
+	 *
+	 * @return Snapshotted operator states
+	 */
+	Collection<OperatorState> getOperatorStates();
+
 	/**
 	 * Disposes the savepoint.
 	 */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java
index 8ee38daaafd2f..38db7c24b99b5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoader.java
@@ -22,9 +22,10 @@
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.checkpoint.CheckpointProperties;
 import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
-import org.apache.flink.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.StreamStateHandle;
 
 import org.slf4j.Logger;
@@ -67,32 +68,42 @@ public static CompletedCheckpoint loadAndValidateSavepoint(
 		final Tuple2<Savepoint, StreamStateHandle> savepointAndHandle = 
 				SavepointStore.loadSavepointWithHandle(savepointPath, classLoader);
 
-		final Savepoint savepoint = savepointAndHandle.f0;
+		Savepoint savepoint = savepointAndHandle.f0;
 		final StreamStateHandle metadataHandle = savepointAndHandle.f1;
 
-		final Map<JobVertexID, TaskState> taskStates = new HashMap<>(savepoint.getTaskStates().size());
+		if (savepoint.getTaskStates() != null) {
+			savepoint = SavepointV2.convertToOperatorStateSavepointV2(tasks, savepoint);
+		}
+		// generate mapping from operator to task
+		Map<OperatorID, ExecutionJobVertex> operatorToJobVertexMapping = new HashMap<>();
+		for (ExecutionJobVertex task : tasks.values()) {
+			for (OperatorID operatorID : task.getOperatorIDs()) {
+				operatorToJobVertexMapping.put(operatorID, task);
+			}
+		}
 
+		// (2) validate it (parallelism, etc)
 		boolean expandedToLegacyIds = false;
 
-		// (2) validate it (parallelism, etc)
-		for (TaskState taskState : savepoint.getTaskStates()) {
+		HashMap<OperatorID, OperatorState> operatorStates = new HashMap<>(savepoint.getOperatorStates().size());
+		for (OperatorState operatorState : savepoint.getOperatorStates()) {
 
-			ExecutionJobVertex executionJobVertex = tasks.get(taskState.getJobVertexID());
+			ExecutionJobVertex executionJobVertex = operatorToJobVertexMapping.get(operatorState.getOperatorID());
 
 			// on the first time we can not find the execution job vertex for an id, we also consider alternative ids,
 			// for example as generated from older flink versions, to provide backwards compatibility.
 			if (executionJobVertex == null && !expandedToLegacyIds) {
-				tasks = ExecutionJobVertex.includeLegacyJobVertexIDs(tasks);
-				executionJobVertex = tasks.get(taskState.getJobVertexID());
+				operatorToJobVertexMapping = ExecutionJobVertex.includeAlternativeOperatorIDs(operatorToJobVertexMapping);
+				executionJobVertex = operatorToJobVertexMapping.get(operatorState.getOperatorID());
 				expandedToLegacyIds = true;
-				LOG.info("Could not find ExecutionJobVertex. Including legacy JobVertexIDs in search.");
+				LOG.info("Could not find ExecutionJobVertex. Including user-defined OperatorIDs in search.");
 			}
 
 			if (executionJobVertex != null) {
 
-				if (executionJobVertex.getMaxParallelism() == taskState.getMaxParallelism()
+				if (executionJobVertex.getMaxParallelism() == operatorState.getMaxParallelism()
 						|| !executionJobVertex.isMaxParallelismConfigured()) {
-					taskStates.put(taskState.getJobVertexID(), taskState);
+					operatorStates.put(operatorState.getOperatorID(), operatorState);
 				} else {
 					String msg = String.format("Failed to rollback to savepoint %s. " +
 									"Max parallelism mismatch between savepoint state and new program. " +
@@ -100,21 +111,21 @@ public static CompletedCheckpoint loadAndValidateSavepoint(
 									"max parallelism %d. This indicates that the program has been changed " +
 									"in a non-compatible way after the savepoint.",
 							savepoint,
-							taskState.getJobVertexID(),
-							taskState.getMaxParallelism(),
+							operatorState.getOperatorID(),
+							operatorState.getMaxParallelism(),
 							executionJobVertex.getMaxParallelism());
 
 					throw new IllegalStateException(msg);
 				}
 			} else if (allowNonRestoredState) {
-				LOG.info("Skipping savepoint state for operator {}.", taskState.getJobVertexID());
+				LOG.info("Skipping savepoint state for operator {}.", operatorState.getOperatorID());
 			} else {
 				String msg = String.format("Failed to rollback to savepoint %s. " +
 								"Cannot map savepoint state for operator %s to the new program, " +
 								"because the operator is not available in the new program. If " +
 								"you want to allow to skip this, you can set the --allowNonRestoredState " +
 								"option on the CLI.",
-						savepointPath, taskState.getJobVertexID());
+						savepointPath, operatorState.getOperatorID());
 
 				throw new IllegalStateException(msg);
 			}
@@ -122,8 +133,17 @@ public static CompletedCheckpoint loadAndValidateSavepoint(
 
 		// (3) convert to checkpoint so the system can fall back to it
 		CheckpointProperties props = CheckpointProperties.forStandardSavepoint();
-		return new CompletedCheckpoint(jobId, savepoint.getCheckpointId(), 0L, 0L,
-				taskStates, savepoint.getMasterStates(), props, metadataHandle, savepointPath);
+
+		return new CompletedCheckpoint(
+			jobId,
+			savepoint.getCheckpointId(),
+			0L,
+			0L,
+			operatorStates,
+			savepoint.getMasterStates(),
+			props,
+			metadataHandle,
+			savepointPath);
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java
index 196c8704d5a1a..daf5b7f4a93cb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.checkpoint.savepoint;
 
 import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.util.Preconditions;
 
@@ -67,6 +68,11 @@ public Collection<MasterState> getMasterStates() {
 		throw new UnsupportedOperationException();
 	}
 
+	@Override
+	public Collection<OperatorState> getOperatorStates() {
+		return null;
+	}
+
 	@Override
 	public void dispose() throws Exception {
 		// since checkpoints are never deserialized into this format,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
index ae9f4a9b1ea2d..aaa8cdd305686 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
@@ -19,7 +19,6 @@
 package org.apache.flink.runtime.checkpoint.savepoint;
 
 import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.SubtaskState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -38,7 +37,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -97,7 +95,7 @@ public SavepointV2 deserialize(DataInputStream dis, ClassLoader cl) throws IOExc
 			}
 		}
 
-		return new SavepointV2(checkpointId, taskStates, Collections.<MasterState>emptyList());
+		return new SavepointV2(checkpointId, taskStates);
 	}
 
 	public void serializeOld(SavepointV1 savepoint, DataOutputStream dos) throws IOException {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java
index 100982d80a3c8..6a3b57fc1feb2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2.java
@@ -19,10 +19,23 @@
 package org.apache.flink.runtime.checkpoint.savepoint;
 
 import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
+import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
+import org.apache.flink.runtime.checkpoint.SubtaskState;
 import org.apache.flink.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.StreamStateHandle;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
@@ -38,20 +51,48 @@ public class SavepointV2 implements Savepoint {
 	/** The checkpoint ID */
 	private final long checkpointId;
 
-	/** The task states */
+	/**
+	 * The task states 
+	 * @deprecated Only kept for backwards-compatibility with versions < 1.3. Will be removed in the future. 
+	 */
+	@Deprecated
 	private final Collection<TaskState> taskStates;
 
+	/** The operator states */
+	private final Collection<OperatorState> operatorStates;
+
 	/** The states generated by the CheckpointCoordinator */
 	private final Collection<MasterState> masterStates;
 
-
+	/** @deprecated Only kept for backwards-compatibility with versions < 1.3. Will be removed in the future. */
+	@Deprecated
 	public SavepointV2(long checkpointId, Collection<TaskState> taskStates) {
-		this(checkpointId, taskStates, Collections.<MasterState>emptyList());
+		this(
+			checkpointId, 
+			null,
+			checkNotNull(taskStates, "taskStates"),
+			Collections.<MasterState>emptyList()
+		);
+	}
+
+	public SavepointV2(long checkpointId, Collection<OperatorState> operatorStates, Collection<MasterState> masterStates) {
+		this(
+			checkpointId,
+			checkNotNull(operatorStates, "operatorStates"),
+			null,
+			masterStates
+		);
 	}
 
-	public SavepointV2(long checkpointId, Collection<TaskState> taskStates, Collection<MasterState> masterStates) {
+	private SavepointV2(
+			long checkpointId,
+			Collection<OperatorState> operatorStates,
+			Collection<TaskState> taskStates,
+			Collection<MasterState> masterStates) {
+
 		this.checkpointId = checkpointId;
-		this.taskStates = checkNotNull(taskStates, "taskStates");
+		this.operatorStates = operatorStates;
+		this.taskStates = taskStates;
 		this.masterStates = checkNotNull(masterStates, "masterStates");
 	}
 
@@ -65,6 +106,11 @@ public long getCheckpointId() {
 		return checkpointId;
 	}
 
+	@Override
+	public Collection<OperatorState> getOperatorStates() {
+		return operatorStates;
+	}
+
 	@Override
 	public Collection<TaskState> getTaskStates() {
 		return taskStates;
@@ -77,10 +123,10 @@ public Collection<MasterState> getMasterStates() {
 
 	@Override
 	public void dispose() throws Exception {
-		for (TaskState taskState : taskStates) {
-			taskState.discardState();
+		for (OperatorState operatorState : operatorStates) {
+			operatorState.discardState();
 		}
-		taskStates.clear();
+		operatorStates.clear();
 		masterStates.clear();
 	}
 
@@ -88,4 +134,97 @@ public void dispose() throws Exception {
 	public String toString() {
 		return "Checkpoint Metadata (version=" + VERSION + ')';
 	}
+
+	/**
+	 * Converts the {@link Savepoint} containing {@link TaskState TaskStates} to an equivalent savepoint containing
+	 * {@link OperatorState OperatorStates}.
+	 *
+	 * @param savepoint savepoint to convert
+	 * @param tasks     map of all vertices and their job vertex ids
+	 * @return converted completed checkpoint
+	 * @deprecated Only kept for backwards-compatibility with versions < 1.3. Will be removed in the future.
+	 * */
+	@Deprecated
+	public static Savepoint convertToOperatorStateSavepointV2(
+			Map<JobVertexID, ExecutionJobVertex> tasks,
+			Savepoint savepoint) {
+
+		if (savepoint.getOperatorStates() != null) {
+			return savepoint;
+		}
+
+		boolean expandedToLegacyIds = false;
+
+		Map<OperatorID, OperatorState> operatorStates = new HashMap<>(savepoint.getTaskStates().size() << 1);
+
+		for (TaskState taskState : savepoint.getTaskStates()) {
+			ExecutionJobVertex jobVertex = tasks.get(taskState.getJobVertexID());
+
+			// on the first time we can not find the execution job vertex for an id, we also consider alternative ids,
+			// for example as generated from older flink versions, to provide backwards compatibility.
+			if (jobVertex == null && !expandedToLegacyIds) {
+				tasks = ExecutionJobVertex.includeLegacyJobVertexIDs(tasks);
+				jobVertex = tasks.get(taskState.getJobVertexID());
+				expandedToLegacyIds = true;
+			}
+
+			List<OperatorID> operatorIDs = jobVertex.getOperatorIDs();
+
+			for (int subtaskIndex = 0; subtaskIndex < jobVertex.getParallelism(); subtaskIndex++) {
+				SubtaskState subtaskState = taskState.getState(subtaskIndex);
+
+				if (subtaskState == null) {
+					continue;
+				}
+
+				@SuppressWarnings("deprecation")
+				ChainedStateHandle<StreamStateHandle> nonPartitionedState =
+					subtaskState.getLegacyOperatorState();
+				ChainedStateHandle<OperatorStateHandle> partitioneableState =
+					subtaskState.getManagedOperatorState();
+				ChainedStateHandle<OperatorStateHandle> rawOperatorState =
+					subtaskState.getRawOperatorState();
+
+				for (int chainIndex = 0; chainIndex < taskState.getChainLength(); chainIndex++) {
+
+					// task consists of multiple operators so we have to break the state apart
+					for (int o = 0; o < operatorIDs.size(); o++) {
+						OperatorID operatorID = operatorIDs.get(o);
+						OperatorState operatorState = operatorStates.get(operatorID);
+
+						if (operatorState == null) {
+							operatorState = new OperatorState(
+								operatorID,
+								jobVertex.getParallelism(),
+								jobVertex.getMaxParallelism());
+							operatorStates.put(operatorID, operatorState);
+						}
+
+						KeyedStateHandle managedKeyedState = null;
+						KeyedStateHandle rawKeyedState = null;
+
+						// only the head operator retains the keyed state
+						if (o == operatorIDs.size() - 1) {
+							managedKeyedState = subtaskState.getManagedKeyedState();
+							rawKeyedState = subtaskState.getRawKeyedState();
+						}
+
+						OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState(
+							nonPartitionedState != null ? nonPartitionedState.get(o) : null,
+							partitioneableState != null ? partitioneableState.get(o) : null,
+							rawOperatorState != null ? rawOperatorState.get(o) : null,
+							managedKeyedState,
+							rawKeyedState);
+
+						operatorState.putState(subtaskIndex, operatorSubtaskState);
+					}
+				}
+			}
+		}
+
+		return new SavepointV2(
+			savepoint.getCheckpointId(),
+			operatorStates.values(),
+			savepoint.getMasterStates());
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
index 307ea1641fece..1b5f2c6f12aa9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
@@ -20,10 +20,9 @@
 
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.checkpoint.MasterState;
-import org.apache.flink.runtime.checkpoint.SubtaskState;
-import org.apache.flink.runtime.checkpoint.TaskState;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.checkpoint.OperatorState;
+import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyGroupRangeOffsets;
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
@@ -97,25 +96,25 @@ public void serialize(SavepointV2 checkpointMetadata, DataOutputStream dos) thro
 			serializeMasterState(ms, dos);
 		}
 
-		// third: task states
-		final Collection<TaskState> taskStates = checkpointMetadata.getTaskStates();
-		dos.writeInt(taskStates.size());
+		// third: operator states
+		Collection<OperatorState> operatorStates = checkpointMetadata.getOperatorStates();
+		dos.writeInt(operatorStates.size());
 
-		for (TaskState taskState : checkpointMetadata.getTaskStates()) {
-			// Vertex ID
-			dos.writeLong(taskState.getJobVertexID().getLowerPart());
-			dos.writeLong(taskState.getJobVertexID().getUpperPart());
+		for (OperatorState operatorState : operatorStates) {
+			// Operator ID
+			dos.writeLong(operatorState.getOperatorID().getLowerPart());
+			dos.writeLong(operatorState.getOperatorID().getUpperPart());
 
 			// Parallelism
-			int parallelism = taskState.getParallelism();
+			int parallelism = operatorState.getParallelism();
 			dos.writeInt(parallelism);
-			dos.writeInt(taskState.getMaxParallelism());
-			dos.writeInt(taskState.getChainLength());
+			dos.writeInt(operatorState.getMaxParallelism());
+			dos.writeInt(1);
 
 			// Sub task states
-			Map<Integer, SubtaskState> subtaskStateMap = taskState.getSubtaskStates();
+			Map<Integer, OperatorSubtaskState> subtaskStateMap = operatorState.getSubtaskStates();
 			dos.writeInt(subtaskStateMap.size());
-			for (Map.Entry<Integer, SubtaskState> entry : subtaskStateMap.entrySet()) {
+			for (Map.Entry<Integer, OperatorSubtaskState> entry : subtaskStateMap.entrySet()) {
 				dos.writeInt(entry.getKey());
 				serializeSubtaskState(entry.getValue(), dos);
 			}
@@ -147,31 +146,32 @@ else if (numMasterStates > 0) {
 			throw new IOException("invalid number of master states: " + numMasterStates);
 		}
 
-		// third: task states
-		final int numTaskStates = dis.readInt();
-		final ArrayList<TaskState> taskStates = new ArrayList<>(numTaskStates);
+		// third: operator states
+		int numTaskStates = dis.readInt();
+		List<OperatorState> operatorStates = new ArrayList<>(numTaskStates);
 
 		for (int i = 0; i < numTaskStates; i++) {
-			JobVertexID jobVertexId = new JobVertexID(dis.readLong(), dis.readLong());
+			OperatorID jobVertexId = new OperatorID(dis.readLong(), dis.readLong());
 			int parallelism = dis.readInt();
 			int maxParallelism = dis.readInt();
 			int chainLength = dis.readInt();
 
 			// Add task state
-			TaskState taskState = new TaskState(jobVertexId, parallelism, maxParallelism, chainLength);
-			taskStates.add(taskState);
+			OperatorState taskState = new OperatorState(jobVertexId, parallelism, maxParallelism);
+			operatorStates.add(taskState);
 
 			// Sub task states
 			int numSubTaskStates = dis.readInt();
 
 			for (int j = 0; j < numSubTaskStates; j++) {
 				int subtaskIndex = dis.readInt();
-				SubtaskState subtaskState = deserializeSubtaskState(dis);
+
+				OperatorSubtaskState subtaskState = deserializeSubtaskState(dis);
 				taskState.putState(subtaskIndex, subtaskState);
 			}
 		}
 
-		return new SavepointV2(checkpointId, taskStates, masterStates);
+		return new SavepointV2(checkpointId, operatorStates, masterStates);
 	}
 
 	// ------------------------------------------------------------------------
@@ -235,35 +235,32 @@ private MasterState deserializeMasterState(DataInputStream dis) throws IOExcepti
 	//  task state (de)serialization methods
 	// ------------------------------------------------------------------------
 
-	private static void serializeSubtaskState(SubtaskState subtaskState, DataOutputStream dos) throws IOException {
+	private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException {
 
 		dos.writeLong(-1);
 
-		ChainedStateHandle<StreamStateHandle> nonPartitionableState = subtaskState.getLegacyOperatorState();
+		StreamStateHandle nonPartitionableState = subtaskState.getLegacyOperatorState();
 
-		int len = nonPartitionableState != null ? nonPartitionableState.getLength() : 0;
+		int len = nonPartitionableState != null ? 1 : 0;
 		dos.writeInt(len);
-		for (int i = 0; i < len; ++i) {
-			StreamStateHandle stateHandle = nonPartitionableState.get(i);
-			serializeStreamStateHandle(stateHandle, dos);
+		if (len == 1) {
+			serializeStreamStateHandle(nonPartitionableState, dos);
 		}
 
-		ChainedStateHandle<OperatorStateHandle> operatorStateBackend = subtaskState.getManagedOperatorState();
+		OperatorStateHandle operatorStateBackend = subtaskState.getManagedOperatorState();
 
-		len = operatorStateBackend != null ? operatorStateBackend.getLength() : 0;
+		len = operatorStateBackend != null ? 1 : 0;
 		dos.writeInt(len);
-		for (int i = 0; i < len; ++i) {
-			OperatorStateHandle stateHandle = operatorStateBackend.get(i);
-			serializeOperatorStateHandle(stateHandle, dos);
+		if (len == 1) {
+			serializeOperatorStateHandle(operatorStateBackend, dos);
 		}
 
-		ChainedStateHandle<OperatorStateHandle> operatorStateFromStream = subtaskState.getRawOperatorState();
+		OperatorStateHandle operatorStateFromStream = subtaskState.getRawOperatorState();
 
-		len = operatorStateFromStream != null ? operatorStateFromStream.getLength() : 0;
+		len = operatorStateFromStream != null ? 1 : 0;
 		dos.writeInt(len);
-		for (int i = 0; i < len; ++i) {
-			OperatorStateHandle stateHandle = operatorStateFromStream.get(i);
-			serializeOperatorStateHandle(stateHandle, dos);
+		if (len == 1) {
+			serializeOperatorStateHandle(operatorStateFromStream, dos);
 		}
 
 		KeyedStateHandle keyedStateBackend = subtaskState.getManagedKeyedState();
@@ -273,49 +270,28 @@ private static void serializeSubtaskState(SubtaskState subtaskState, DataOutputS
 		serializeKeyedStateHandle(keyedStateStream, dos);
 	}
 
-	private static SubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException {
+	private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException {
 		// Duration field has been removed from SubtaskState
 		long ignoredDuration = dis.readLong();
 
 		int len = dis.readInt();
-		List<StreamStateHandle> nonPartitionableState = new ArrayList<>(len);
-		for (int i = 0; i < len; ++i) {
-			StreamStateHandle streamStateHandle = deserializeStreamStateHandle(dis);
-			nonPartitionableState.add(streamStateHandle);
-		}
-
+		StreamStateHandle nonPartitionableState = len == 0 ? null : deserializeStreamStateHandle(dis);
 
 		len = dis.readInt();
-		List<OperatorStateHandle> operatorStateBackend = new ArrayList<>(len);
-		for (int i = 0; i < len; ++i) {
-			OperatorStateHandle streamStateHandle = deserializeOperatorStateHandle(dis);
-			operatorStateBackend.add(streamStateHandle);
-		}
+		OperatorStateHandle operatorStateBackend = len == 0 ? null : deserializeOperatorStateHandle(dis);
 
 		len = dis.readInt();
-		List<OperatorStateHandle> operatorStateStream = new ArrayList<>(len);
-		for (int i = 0; i < len; ++i) {
-			OperatorStateHandle streamStateHandle = deserializeOperatorStateHandle(dis);
-			operatorStateStream.add(streamStateHandle);
-		}
+		OperatorStateHandle operatorStateStream = len == 0 ? null : deserializeOperatorStateHandle(dis);
 
 		KeyedStateHandle keyedStateBackend = deserializeKeyedStateHandle(dis);
 
 		KeyedStateHandle keyedStateStream = deserializeKeyedStateHandle(dis);
 
-		ChainedStateHandle<StreamStateHandle> nonPartitionableStateChain =
-				new ChainedStateHandle<>(nonPartitionableState);
-
-		ChainedStateHandle<OperatorStateHandle> operatorStateBackendChain =
-				new ChainedStateHandle<>(operatorStateBackend);
-
-		ChainedStateHandle<OperatorStateHandle> operatorStateStreamChain =
-				new ChainedStateHandle<>(operatorStateStream);
 
-		return new SubtaskState(
-				nonPartitionableStateChain,
-				operatorStateBackendChain,
-				operatorStateStreamChain,
+		return new OperatorSubtaskState(
+				nonPartitionableState,
+				operatorStateBackend,
+				operatorStateStream,
 				keyedStateBackend,
 				keyedStateStream);
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index 5fbce4df23937..2e5de64ca9298 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -41,6 +41,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
@@ -68,6 +69,24 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 	private final ExecutionGraph graph;
 	
 	private final JobVertex jobVertex;
+
+	/**
+	 * The IDs of all operators contained in this execution job vertex.
+	 *
+	 * The ID's are stored depth-first post-order; for the forking chain below the ID's would be stored as [D, E, B, C, A].
+	 *  A - B - D
+	 *   \    \
+	 *    C    E
+	 * This is the same order that operators are stored in the {@code StreamTask}.
+	 */
+	private final List<OperatorID> operatorIDs;
+
+	/**
+	 * The alternative IDs of all operators contained in this execution job vertex.
+	 *
+	 * The ID's are in the same order as {@link ExecutionJobVertex#operatorIDs}.
+	 */
+	private final List<OperatorID> userDefinedOperatorIds;
 	
 	private final ExecutionVertex[] taskVertices;
 
@@ -139,6 +158,8 @@ public ExecutionJobVertex(
 		this.serializedTaskInformation = null;
 
 		this.taskVertices = new ExecutionVertex[numTaskVertices];
+		this.operatorIDs = Collections.unmodifiableList(jobVertex.getOperatorIDs());
+		this.userDefinedOperatorIds = Collections.unmodifiableList(jobVertex.getUserDefinedOperatorIDs());
 		
 		this.inputs = new ArrayList<>(jobVertex.getInputs().size());
 		
@@ -214,6 +235,24 @@ public ExecutionJobVertex(
 		finishedSubtasks = new boolean[parallelism];
 	}
 
+	/**
+	 * Returns a list containing the IDs of all operators contained in this execution job vertex.
+	 *
+	 * @return list containing the IDs of all contained operators
+	 */
+	public List<OperatorID> getOperatorIDs() {
+		return operatorIDs;
+	}
+
+	/**
+	 * Returns a list containing the alternative IDs of all operators contained in this execution job vertex.
+	 *
+	 * @return list containing alternative the IDs of all contained operators
+	 */
+	public List<OperatorID> getUserDefinedOperatorIDs() {
+		return userDefinedOperatorIds;
+	}
+
 	public void setMaxParallelism(int maxParallelismDerived) {
 
 		Preconditions.checkState(!maxParallelismConfigured,
@@ -731,6 +770,30 @@ public static Map<JobVertexID, ExecutionJobVertex> includeLegacyJobVertexIDs(
 		return expanded;
 	}
 
+	public static Map<OperatorID, ExecutionJobVertex> includeAlternativeOperatorIDs(
+			Map<OperatorID, ExecutionJobVertex> operatorMapping) {
+
+		Map<OperatorID, ExecutionJobVertex> expanded = new HashMap<>(2 * operatorMapping.size());
+		// first include all existing ids
+		expanded.putAll(operatorMapping);
+
+		// now expand and add user-defined ids
+		for (ExecutionJobVertex executionJobVertex : operatorMapping.values()) {
+			if (executionJobVertex != null) {
+				JobVertex jobVertex = executionJobVertex.getJobVertex();
+				if (jobVertex != null) {
+					for (OperatorID operatorID : jobVertex.getUserDefinedOperatorIDs()) {
+						if (operatorID != null) {
+							expanded.put(operatorID, executionJobVertex);
+						}
+					}
+				}
+			}
+		}
+
+		return expanded;
+	}
+
 	@Override
 	public ArchivedExecutionJobVertex archive() {
 		return new ArchivedExecutionJobVertex(this);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/InputFormatVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/InputFormatVertex.java
index c4fc907fb4b17..5627ac7b9469f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/InputFormatVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/InputFormatVertex.java
@@ -39,8 +39,8 @@ public InputFormatVertex(String name, JobVertexID id) {
 		super(name, id);
 	}
 
-	public InputFormatVertex(String name, JobVertexID id, List<JobVertexID> alternativeIds) {
-		super(name, id, alternativeIds);
+	public InputFormatVertex(String name, JobVertexID id, List<JobVertexID> alternativeIds, List<OperatorID> operatorIds, List<OperatorID> alternativeOperatorIds) {
+		super(name, id, alternativeIds, operatorIds, alternativeOperatorIds);
 	}
 	
 	
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java
index 1180db4920103..4f52895dde792 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java
@@ -50,8 +50,15 @@ public class JobVertex implements java.io.Serializable {
 	/** The ID of the vertex. */
 	private final JobVertexID id;
 
+	/** The alternative IDs of the vertex. */
 	private final ArrayList<JobVertexID> idAlternatives = new ArrayList<>();
 
+	/** The IDs of all operators contained in this vertex. */
+	private final ArrayList<OperatorID> operatorIDs = new ArrayList<>();
+
+	/** The alternative IDs of all operators contained in this vertex. */
+	private final ArrayList<OperatorID> operatorIdsAlternatives = new ArrayList<>();
+
 	/** List of produced data sets, one per writer */
 	private final ArrayList<IntermediateDataSet> results = new ArrayList<IntermediateDataSet>();
 
@@ -125,6 +132,9 @@ public JobVertex(String name) {
 	public JobVertex(String name, JobVertexID id) {
 		this.name = name == null ? DEFAULT_NAME : name;
 		this.id = id == null ? new JobVertexID() : id;
+		// the id lists must have the same size
+		this.operatorIDs.add(OperatorID.fromJobVertexID(this.id));
+		this.operatorIdsAlternatives.add(null);
 	}
 
 	/**
@@ -133,11 +143,16 @@ public JobVertex(String name, JobVertexID id) {
 	 * @param name The name of the new job vertex.
 	 * @param primaryId The id of the job vertex.
 	 * @param alternativeIds The alternative ids of the job vertex.
+	 * @param operatorIds The ids of all operators contained in this job vertex.
+	 * @param alternativeOperatorIds The alternative ids of all operators contained in this job vertex-
 	 */
-	public JobVertex(String name, JobVertexID primaryId, List<JobVertexID> alternativeIds) {
+	public JobVertex(String name, JobVertexID primaryId, List<JobVertexID> alternativeIds, List<OperatorID> operatorIds, List<OperatorID> alternativeOperatorIds) {
+		Preconditions.checkArgument(operatorIds.size() == alternativeOperatorIds.size());
 		this.name = name == null ? DEFAULT_NAME : name;
 		this.id = primaryId == null ? new JobVertexID() : primaryId;
 		this.idAlternatives.addAll(alternativeIds);
+		this.operatorIDs.addAll(operatorIds);
+		this.operatorIdsAlternatives.addAll(alternativeOperatorIds);
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -196,6 +211,14 @@ public int getNumberOfInputs() {
 		return this.inputs.size();
 	}
 
+	public List<OperatorID> getOperatorIDs() {
+		return operatorIDs;
+	}
+
+	public List<OperatorID> getUserDefinedOperatorIDs() {
+		return operatorIdsAlternatives;
+	}
+
 	/**
 	 * Returns the vertex's configuration object which can be used to pass custom settings to the task at runtime.
 	 * 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/OperatorID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/OperatorID.java
new file mode 100644
index 0000000000000..0e378de4ca7f9
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/OperatorID.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph;
+
+import org.apache.flink.util.AbstractID;
+
+/**
+ * A class for statistically unique operator IDs.
+ */
+public class OperatorID extends AbstractID {
+
+	private static final long serialVersionUID = 1L;
+
+	public OperatorID() {
+		super();
+	}
+
+	public OperatorID(byte[] bytes) {
+		super(bytes);
+	}
+
+	public OperatorID(long lowerPart, long upperPart) {
+		super(lowerPart, upperPart);
+	}
+
+	public static OperatorID fromJobVertexID(JobVertexID id) {
+		return new OperatorID(id.getLowerPart(), id.getUpperPart());
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorExternalizedCheckpointsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorExternalizedCheckpointsTest.java
index 9f94f2fc795be..d293eea1c757e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorExternalizedCheckpointsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorExternalizedCheckpointsTest.java
@@ -34,6 +34,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
@@ -190,7 +191,9 @@ private static void verifyExternalizedCheckpointRestore(
 				false);
 
 		for (ExecutionVertex vertex : vertices) {
-			assertEquals(checkpoint.getTaskState(vertex.getJobvertexId()), loaded.getTaskState(vertex.getJobvertexId()));
+			for (OperatorID operatorID : vertex.getJobVertex().getOperatorIDs()) {
+				assertEquals(checkpoint.getOperatorStates().get(operatorID), loaded.getOperatorStates().get(operatorID));
+			}
 		}
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
index 90b7fe7113756..6e20be31044b6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorFailureTest.java
@@ -25,10 +25,13 @@
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
+import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
@@ -38,8 +41,10 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyInt;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(PendingCheckpoint.class)
@@ -85,8 +90,26 @@ public void testFailingCompletedCheckpointStoreAdd() throws Exception {
 		final long checkpointId = coord.getPendingCheckpoints().keySet().iterator().next();
 		
 		SubtaskState subtaskState = mock(SubtaskState.class);
-		PowerMockito.when(subtaskState.getLegacyOperatorState()).thenReturn(null);
-		PowerMockito.when(subtaskState.getManagedOperatorState()).thenReturn(null);
+
+		StreamStateHandle legacyHandle = mock(StreamStateHandle.class);
+		ChainedStateHandle<StreamStateHandle> chainedLegacyHandle = mock(ChainedStateHandle.class);
+		when(chainedLegacyHandle.get(anyInt())).thenReturn(legacyHandle);
+		when(subtaskState.getLegacyOperatorState()).thenReturn(chainedLegacyHandle);
+
+		OperatorStateHandle managedHandle = mock(OperatorStateHandle.class);
+		ChainedStateHandle<OperatorStateHandle> chainedManagedHandle = mock(ChainedStateHandle.class);
+		when(chainedManagedHandle.get(anyInt())).thenReturn(managedHandle);
+		when(subtaskState.getManagedOperatorState()).thenReturn(chainedManagedHandle);
+
+		OperatorStateHandle rawHandle = mock(OperatorStateHandle.class);
+		ChainedStateHandle<OperatorStateHandle> chainedRawHandle = mock(ChainedStateHandle.class);
+		when(chainedRawHandle.get(anyInt())).thenReturn(rawHandle);
+		when(subtaskState.getRawOperatorState()).thenReturn(chainedRawHandle);
+
+		KeyedStateHandle managedKeyedHandle = mock(KeyedStateHandle.class);
+		when(subtaskState.getRawKeyedState()).thenReturn(managedKeyedHandle);
+		KeyedStateHandle managedRawHandle = mock(KeyedStateHandle.class);
+		when(subtaskState.getManagedKeyedState()).thenReturn(managedRawHandle);
 		
 		AcknowledgeCheckpoint acknowledgeMessage = new AcknowledgeCheckpoint(jid, executionAttemptId, checkpointId, new CheckpointMetrics(), subtaskState);
 		
@@ -102,7 +125,11 @@ public void testFailingCompletedCheckpointStoreAdd() throws Exception {
 		assertTrue(pendingCheckpoint.isDiscarded());
 
 		// make sure that the subtask state has been discarded after we could not complete it.
-		verify(subtaskState).discardState();
+		verify(subtaskState.getLegacyOperatorState().get(0)).discardState();
+		verify(subtaskState.getManagedOperatorState().get(0)).discardState();
+		verify(subtaskState.getRawOperatorState().get(0)).discardState();
+		verify(subtaskState.getManagedKeyedState()).discardState();
+		verify(subtaskState.getRawKeyedState()).discardState();
 	}
 
 	private static final class FailingCompletedCheckpointStore implements CompletedCheckpointStore {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
index 7c271a7f297b4..d6daa4e4ea3c2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorMasterHooksTest.java
@@ -27,6 +27,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 
@@ -228,7 +229,7 @@ public void testHooksAreCalledOnRestore() throws Exception {
 
 		final CompletedCheckpoint checkpoint = new CompletedCheckpoint(
 				jid, checkpointId, 123L, 125L,
-				Collections.<JobVertexID, TaskState>emptyMap(),
+				Collections.<OperatorID, OperatorState>emptyMap(),
 				masterHookStates,
 				CheckpointProperties.forStandardCheckpoint(),
 				null,
@@ -282,7 +283,7 @@ public void checkUnMatchedStateOnRestore() throws Exception {
 
 		final CompletedCheckpoint checkpoint = new CompletedCheckpoint(
 				jid, checkpointId, 123L, 125L,
-				Collections.<JobVertexID, TaskState>emptyMap(),
+				Collections.<OperatorID, OperatorState>emptyMap(),
 				masterHookStates,
 				CheckpointProperties.forStandardCheckpoint(),
 				null,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index 24169f2a5b607..0d2e9031d1042 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.checkpoint;
 
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.api.java.tuple.Tuple2;
@@ -33,6 +34,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
@@ -168,7 +170,14 @@ public void testCheckpointAbortsIfTriggerTasksAreFinished() {
 			final ExecutionAttemptID triggerAttemptID1 = new ExecutionAttemptID();
 			final ExecutionAttemptID triggerAttemptID2 = new ExecutionAttemptID();
 			ExecutionVertex triggerVertex1 = mockExecutionVertex(triggerAttemptID1);
-			ExecutionVertex triggerVertex2 = mockExecutionVertex(triggerAttemptID2, new JobVertexID(), 1, 1, ExecutionState.FINISHED);
+			JobVertexID jobVertexID2 = new JobVertexID();
+			ExecutionVertex triggerVertex2 = mockExecutionVertex(
+				triggerAttemptID2,
+				jobVertexID2,
+				Lists.newArrayList(OperatorID.fromJobVertexID(jobVertexID2)),
+				1,
+				1,
+				ExecutionState.FINISHED);
 
 			// create some mock Execution vertices that need to ack the checkpoint
 			final ExecutionAttemptID ackAttemptID1 = new ExecutionAttemptID();
@@ -317,7 +326,7 @@ public void testTriggerAndDeclineCheckpointSimple() {
 			assertEquals(jid, checkpoint.getJobId());
 			assertEquals(2, checkpoint.getNumberOfNonAcknowledgedTasks());
 			assertEquals(0, checkpoint.getNumberOfAcknowledgedTasks());
-			assertEquals(0, checkpoint.getTaskStates().size());
+			assertEquals(0, checkpoint.getOperatorStates().size());
 			assertFalse(checkpoint.isDiscarded());
 			assertFalse(checkpoint.isFullyAcknowledged());
 
@@ -426,7 +435,7 @@ public void testTriggerAndDeclineCheckpointComplex() {
 			assertEquals(jid, checkpoint1.getJobId());
 			assertEquals(2, checkpoint1.getNumberOfNonAcknowledgedTasks());
 			assertEquals(0, checkpoint1.getNumberOfAcknowledgedTasks());
-			assertEquals(0, checkpoint1.getTaskStates().size());
+			assertEquals(0, checkpoint1.getOperatorStates().size());
 			assertFalse(checkpoint1.isDiscarded());
 			assertFalse(checkpoint1.isFullyAcknowledged());
 
@@ -436,7 +445,7 @@ public void testTriggerAndDeclineCheckpointComplex() {
 			assertEquals(jid, checkpoint2.getJobId());
 			assertEquals(2, checkpoint2.getNumberOfNonAcknowledgedTasks());
 			assertEquals(0, checkpoint2.getNumberOfAcknowledgedTasks());
-			assertEquals(0, checkpoint2.getTaskStates().size());
+			assertEquals(0, checkpoint2.getOperatorStates().size());
 			assertFalse(checkpoint2.isDiscarded());
 			assertFalse(checkpoint2.isFullyAcknowledged());
 
@@ -471,7 +480,7 @@ public void testTriggerAndDeclineCheckpointComplex() {
 			assertEquals(jid, checkpointNew.getJobId());
 			assertEquals(2, checkpointNew.getNumberOfNonAcknowledgedTasks());
 			assertEquals(0, checkpointNew.getNumberOfAcknowledgedTasks());
-			assertEquals(0, checkpointNew.getTaskStates().size());
+			assertEquals(0, checkpointNew.getOperatorStates().size());
 			assertFalse(checkpointNew.isDiscarded());
 			assertFalse(checkpointNew.isFullyAcknowledged());
 			assertNotEquals(checkpoint1.getCheckpointId(), checkpointNew.getCheckpointId());
@@ -539,10 +548,20 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			assertEquals(jid, checkpoint.getJobId());
 			assertEquals(2, checkpoint.getNumberOfNonAcknowledgedTasks());
 			assertEquals(0, checkpoint.getNumberOfAcknowledgedTasks());
-			assertEquals(0, checkpoint.getTaskStates().size());
+			assertEquals(0, checkpoint.getOperatorStates().size());
 			assertFalse(checkpoint.isDiscarded());
 			assertFalse(checkpoint.isFullyAcknowledged());
 
+			OperatorID opID1 = OperatorID.fromJobVertexID(vertex1.getJobvertexId());
+			OperatorID opID2 = OperatorID.fromJobVertexID(vertex2.getJobvertexId());
+
+			Map<OperatorID, OperatorState> operatorStates = checkpoint.getOperatorStates();
+
+			operatorStates.put(opID1, new SpyInjectingOperatorState(
+				opID1, vertex1.getTotalNumberOfParallelSubtasks(), vertex1.getMaxParallelism()));
+			operatorStates.put(opID2, new SpyInjectingOperatorState(
+				opID2, vertex2.getTotalNumberOfParallelSubtasks(), vertex2.getMaxParallelism()));
+
 			// check that the vertices received the trigger checkpoint message
 			{
 				verify(vertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId), eq(timestamp), any(CheckpointOptions.class));
@@ -550,9 +569,9 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			}
 
 			// acknowledge from one of the tasks
-			SubtaskState subtaskState2 = mock(SubtaskState.class);
-			AcknowledgeCheckpoint acknowledgeCheckpoint1 = new AcknowledgeCheckpoint(jid, attemptID2, checkpointId, new CheckpointMetrics(), subtaskState2);
+			AcknowledgeCheckpoint acknowledgeCheckpoint1 = new AcknowledgeCheckpoint(jid, attemptID2, checkpointId, new CheckpointMetrics(), mock(SubtaskState.class));
 			coord.receiveAcknowledgeMessage(acknowledgeCheckpoint1);
+			OperatorSubtaskState subtaskState2 = operatorStates.get(opID2).getState(vertex2.getParallelSubtaskIndex());
 			assertEquals(1, checkpoint.getNumberOfAcknowledgedTasks());
 			assertEquals(1, checkpoint.getNumberOfNonAcknowledgedTasks());
 			assertFalse(checkpoint.isDiscarded());
@@ -566,8 +585,8 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			verify(subtaskState2, never()).registerSharedStates(any(SharedStateRegistry.class));
 
 			// acknowledge the other task.
-			SubtaskState subtaskState1 = mock(SubtaskState.class);
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId, new CheckpointMetrics(), subtaskState1));
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId, new CheckpointMetrics(), mock(SubtaskState.class)));
+			OperatorSubtaskState subtaskState1 = operatorStates.get(opID1).getState(vertex1.getParallelSubtaskIndex());
 
 			// the checkpoint is internally converted to a successful checkpoint and the
 			// pending checkpoint object is disposed
@@ -596,7 +615,7 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			assertEquals(jid, success.getJobId());
 			assertEquals(timestamp, success.getTimestamp());
 			assertEquals(checkpoint.getCheckpointId(), success.getCheckpointID());
-			assertEquals(2, success.getTaskStates().size());
+			assertEquals(2, success.getOperatorStates().size());
 
 			// ---------------
 			// trigger another checkpoint and see that this one replaces the other checkpoint
@@ -606,7 +625,9 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 
 			long checkpointIdNew = coord.getPendingCheckpoints().entrySet().iterator().next().getKey();
 			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointIdNew));
+			subtaskState1 = operatorStates.get(opID1).getState(vertex1.getParallelSubtaskIndex());
 			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointIdNew));
+			subtaskState2 = operatorStates.get(opID2).getState(vertex2.getParallelSubtaskIndex());
 
 			assertEquals(0, coord.getNumberOfPendingCheckpoints());
 			assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
@@ -616,7 +637,7 @@ public void testTriggerAndConfirmSimpleCheckpoint() {
 			assertEquals(jid, successNew.getJobId());
 			assertEquals(timestampNew, successNew.getTimestamp());
 			assertEquals(checkpointIdNew, successNew.getCheckpointID());
-			assertTrue(successNew.getTaskStates().isEmpty());
+			assertTrue(successNew.getOperatorStates().isEmpty());
 
 			// validate that the subtask states in old savepoint have unregister their shared states
 			{
@@ -756,13 +777,13 @@ public void testMultipleConcurrentCheckpoints() {
 			assertEquals(checkpointId1, sc1.getCheckpointID());
 			assertEquals(timestamp1, sc1.getTimestamp());
 			assertEquals(jid, sc1.getJobId());
-			assertTrue(sc1.getTaskStates().isEmpty());
+			assertTrue(sc1.getOperatorStates().isEmpty());
 
 			CompletedCheckpoint sc2 = scs.get(1);
 			assertEquals(checkpointId2, sc2.getCheckpointID());
 			assertEquals(timestamp2, sc2.getTimestamp());
 			assertEquals(jid, sc2.getJobId());
-			assertTrue(sc2.getTaskStates().isEmpty());
+			assertTrue(sc2.getOperatorStates().isEmpty());
 
 			coord.shutdown(JobStatus.FINISHED);
 		}
@@ -830,10 +851,22 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			verify(triggerVertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId1), eq(timestamp1), any(CheckpointOptions.class));
 			verify(triggerVertex2.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId1), eq(timestamp1), any(CheckpointOptions.class));
 
-			// acknowledge one of the three tasks
-			SubtaskState subtaskState1_2 = mock(SubtaskState.class);
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId1, new CheckpointMetrics(), subtaskState1_2));
+			OperatorID opID1 = OperatorID.fromJobVertexID(ackVertex1.getJobvertexId());
+			OperatorID opID2 = OperatorID.fromJobVertexID(ackVertex2.getJobvertexId());
+			OperatorID opID3 = OperatorID.fromJobVertexID(ackVertex3.getJobvertexId());
+
+			Map<OperatorID, OperatorState> operatorStates1 = pending1.getOperatorStates();
+
+			operatorStates1.put(opID1, new SpyInjectingOperatorState(
+				opID1, ackVertex1.getTotalNumberOfParallelSubtasks(), ackVertex1.getMaxParallelism()));
+			operatorStates1.put(opID2, new SpyInjectingOperatorState(
+				opID2, ackVertex2.getTotalNumberOfParallelSubtasks(), ackVertex2.getMaxParallelism()));
+			operatorStates1.put(opID3, new SpyInjectingOperatorState(
+				opID3, ackVertex3.getTotalNumberOfParallelSubtasks(), ackVertex3.getMaxParallelism()));
 
+			// acknowledge one of the three tasks
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId1, new CheckpointMetrics(), mock(SubtaskState.class)));
+			OperatorSubtaskState subtaskState1_2 = operatorStates1.get(opID2).getState(ackVertex2.getParallelSubtaskIndex());
 			// start the second checkpoint
 			// trigger the first checkpoint. this should succeed
 			assertTrue(coord.triggerCheckpoint(timestamp2, false));
@@ -850,6 +883,15 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			}
 			long checkpointId2 = pending2.getCheckpointId();
 
+			Map<OperatorID, OperatorState> operatorStates2 = pending2.getOperatorStates();
+
+			operatorStates2.put(opID1, new SpyInjectingOperatorState(
+				opID1, ackVertex1.getTotalNumberOfParallelSubtasks(), ackVertex1.getMaxParallelism()));
+			operatorStates2.put(opID2, new SpyInjectingOperatorState(
+				opID2, ackVertex2.getTotalNumberOfParallelSubtasks(), ackVertex2.getMaxParallelism()));
+			operatorStates2.put(opID3, new SpyInjectingOperatorState(
+				opID3, ackVertex3.getTotalNumberOfParallelSubtasks(), ackVertex3.getMaxParallelism()));
+
 			// trigger messages should have been sent
 			verify(triggerVertex1.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId2), eq(timestamp2), any(CheckpointOptions.class));
 			verify(triggerVertex2.getCurrentExecutionAttempt(), times(1)).triggerCheckpoint(eq(checkpointId2), eq(timestamp2), any(CheckpointOptions.class));
@@ -857,17 +899,17 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			// we acknowledge one more task from the first checkpoint and the second
 			// checkpoint completely. The second checkpoint should then subsume the first checkpoint
 
-			SubtaskState subtaskState2_3 = mock(SubtaskState.class);
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId2, new CheckpointMetrics(), subtaskState2_3));
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId2, new CheckpointMetrics(), mock(SubtaskState.class)));
+			OperatorSubtaskState subtaskState2_3 = operatorStates2.get(opID3).getState(ackVertex3.getParallelSubtaskIndex());
 
-			SubtaskState subtaskState2_1 = mock(SubtaskState.class);
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId2, new CheckpointMetrics(), subtaskState2_1));
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId2, new CheckpointMetrics(), mock(SubtaskState.class)));
+			OperatorSubtaskState subtaskState2_1 = operatorStates2.get(opID1).getState(ackVertex1.getParallelSubtaskIndex());
 
-			SubtaskState subtaskState1_1 = mock(SubtaskState.class);
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId1, new CheckpointMetrics(), subtaskState1_1));
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpointId1, new CheckpointMetrics(), mock(SubtaskState.class)));
+			OperatorSubtaskState subtaskState1_1 = operatorStates1.get(opID1).getState(ackVertex1.getParallelSubtaskIndex());
 
-			SubtaskState subtaskState2_2 = mock(SubtaskState.class);
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId2, new CheckpointMetrics(), subtaskState2_2));
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID2, checkpointId2, new CheckpointMetrics(), mock(SubtaskState.class)));
+			OperatorSubtaskState subtaskState2_2 = operatorStates2.get(opID2).getState(ackVertex2.getParallelSubtaskIndex());
 
 			// now, the second checkpoint should be confirmed, and the first discarded
 			// actually both pending checkpoints are discarded, and the second has been transformed
@@ -896,7 +938,7 @@ public void testSuccessfulCheckpointSubsumesUnsuccessful() {
 			assertEquals(checkpointId2, success.getCheckpointID());
 			assertEquals(timestamp2, success.getTimestamp());
 			assertEquals(jid, success.getJobId());
-			assertEquals(3, success.getTaskStates().size());
+			assertEquals(3, success.getOperatorStates().size());
 
 			// the first confirm message should be out
 			verify(commitVertex.getCurrentExecutionAttempt(), times(1)).notifyCheckpointComplete(eq(checkpointId2), eq(timestamp2));
@@ -970,8 +1012,15 @@ public void testCheckpointTimeoutIsolated() {
 			PendingCheckpoint checkpoint = coord.getPendingCheckpoints().values().iterator().next();
 			assertFalse(checkpoint.isDiscarded());
 
-			SubtaskState subtaskState = mock(SubtaskState.class);
-			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpoint.getCheckpointId(), new CheckpointMetrics(), subtaskState));
+			OperatorID opID1 = OperatorID.fromJobVertexID(ackVertex1.getJobvertexId());
+
+			Map<OperatorID, OperatorState> operatorStates = checkpoint.getOperatorStates();
+
+			operatorStates.put(opID1, new SpyInjectingOperatorState(
+				opID1, ackVertex1.getTotalNumberOfParallelSubtasks(), ackVertex1.getMaxParallelism()));
+
+			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID1, checkpoint.getCheckpointId(), new CheckpointMetrics(), mock(SubtaskState.class)));
+			OperatorSubtaskState subtaskState = operatorStates.get(opID1).getState(ackVertex1.getParallelSubtaskIndex());
 
 			// wait until the checkpoint must have expired.
 			// we check every 250 msecs conservatively for 5 seconds
@@ -1106,13 +1155,25 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 
 		long checkpointId = pendingCheckpoint.getCheckpointId();
 
-		SubtaskState triggerSubtaskState = mock(SubtaskState.class);
+		OperatorID opIDtrigger = OperatorID.fromJobVertexID(triggerVertex.getJobvertexId());
+		OperatorID opID1 = OperatorID.fromJobVertexID(ackVertex1.getJobvertexId());
+		OperatorID opID2 = OperatorID.fromJobVertexID(ackVertex2.getJobvertexId());
+
+		Map<OperatorID, OperatorState> operatorStates = pendingCheckpoint.getOperatorStates();
+
+		operatorStates.put(opIDtrigger, new SpyInjectingOperatorState(
+			opIDtrigger, triggerVertex.getTotalNumberOfParallelSubtasks(), triggerVertex.getMaxParallelism()));
+		operatorStates.put(opID1, new SpyInjectingOperatorState(
+			opID1, ackVertex1.getTotalNumberOfParallelSubtasks(), ackVertex1.getMaxParallelism()));
+		operatorStates.put(opID2, new SpyInjectingOperatorState(
+			opID2, ackVertex2.getTotalNumberOfParallelSubtasks(), ackVertex2.getMaxParallelism()));
 
 		// acknowledge the first trigger vertex
-		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, triggerAttemptId, checkpointId, new CheckpointMetrics(), triggerSubtaskState));
+		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, triggerAttemptId, checkpointId, new CheckpointMetrics(), mock(SubtaskState.class)));
+		OperatorSubtaskState storedTriggerSubtaskState = operatorStates.get(opIDtrigger).getState(triggerVertex.getParallelSubtaskIndex());
 
-		// verify that the subtask state has registered its shared states at the registry
-		verify(triggerSubtaskState, never()).discardState();
+		// verify that the subtask state has not been discarded
+		verify(storedTriggerSubtaskState, never()).discardState();
 
 		SubtaskState unknownSubtaskState = mock(SubtaskState.class);
 
@@ -1131,20 +1192,20 @@ public void testStateCleanupForLateOrUnknownMessages() throws Exception {
 		verify(differentJobSubtaskState, never()).discardState();
 
 		// duplicate acknowledge message for the trigger vertex
-		reset(triggerSubtaskState);
+		SubtaskState triggerSubtaskState = mock(SubtaskState.class);
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jobId, triggerAttemptId, checkpointId, new CheckpointMetrics(), triggerSubtaskState));
 
 		// duplicate acknowledge messages for a known vertex should not trigger discarding the state
 		verify(triggerSubtaskState, never()).discardState();
 
 		// let the checkpoint fail at the first ack vertex
-		reset(triggerSubtaskState);
+		reset(storedTriggerSubtaskState);
 		coord.receiveDeclineMessage(new DeclineCheckpoint(jobId, ackAttemptId1, checkpointId));
 
 		assertTrue(pendingCheckpoint.isDiscarded());
 
 		// check that we've cleaned up the already acknowledged state
-		verify(triggerSubtaskState, times(1)).discardState();
+		verify(storedTriggerSubtaskState, times(1)).discardState();
 
 		SubtaskState ackSubtaskState = mock(SubtaskState.class);
 
@@ -1411,15 +1472,25 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		assertEquals(jid, pending.getJobId());
 		assertEquals(2, pending.getNumberOfNonAcknowledgedTasks());
 		assertEquals(0, pending.getNumberOfAcknowledgedTasks());
-		assertEquals(0, pending.getTaskStates().size());
+		assertEquals(0, pending.getOperatorStates().size());
 		assertFalse(pending.isDiscarded());
 		assertFalse(pending.isFullyAcknowledged());
 		assertFalse(pending.canBeSubsumed());
 
+		OperatorID opID1 = OperatorID.fromJobVertexID(vertex1.getJobvertexId());
+		OperatorID opID2 = OperatorID.fromJobVertexID(vertex2.getJobvertexId());
+
+		Map<OperatorID, OperatorState> operatorStates = pending.getOperatorStates();
+
+		operatorStates.put(opID1, new SpyInjectingOperatorState(
+			opID1, vertex1.getTotalNumberOfParallelSubtasks(), vertex1.getMaxParallelism()));
+		operatorStates.put(opID2, new SpyInjectingOperatorState(
+			opID2, vertex2.getTotalNumberOfParallelSubtasks(), vertex1.getMaxParallelism()));
+
 		// acknowledge from one of the tasks
-		SubtaskState subtaskState2 = mock(SubtaskState.class);
-		AcknowledgeCheckpoint acknowledgeCheckpoint2 = new AcknowledgeCheckpoint(jid, attemptID2, checkpointId, new CheckpointMetrics(), subtaskState2);
+		AcknowledgeCheckpoint acknowledgeCheckpoint2 = new AcknowledgeCheckpoint(jid, attemptID2, checkpointId, new CheckpointMetrics(), mock(SubtaskState.class));
 		coord.receiveAcknowledgeMessage(acknowledgeCheckpoint2);
+		OperatorSubtaskState subtaskState2 = operatorStates.get(opID2).getState(vertex2.getParallelSubtaskIndex());
 		assertEquals(1, pending.getNumberOfAcknowledgedTasks());
 		assertEquals(1, pending.getNumberOfNonAcknowledgedTasks());
 		assertFalse(pending.isDiscarded());
@@ -1433,8 +1504,8 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		assertFalse(savepointFuture.isDone());
 
 		// acknowledge the other task.
-		SubtaskState subtaskState1 = mock(SubtaskState.class);
-		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId, new CheckpointMetrics(), subtaskState1));
+		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointId, new CheckpointMetrics(), mock(SubtaskState.class)));
+		OperatorSubtaskState subtaskState1 = operatorStates.get(opID1).getState(vertex1.getParallelSubtaskIndex());
 
 		// the checkpoint is internally converted to a successful checkpoint and the
 		// pending checkpoint object is disposed
@@ -1461,7 +1532,7 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		assertEquals(jid, success.getJobId());
 		assertEquals(timestamp, success.getTimestamp());
 		assertEquals(pending.getCheckpointId(), success.getCheckpointID());
-		assertEquals(2, success.getTaskStates().size());
+		assertEquals(2, success.getOperatorStates().size());
 
 		// ---------------
 		// trigger another checkpoint and see that this one replaces the other checkpoint
@@ -1474,6 +1545,9 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID1, checkpointIdNew));
 		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointIdNew));
 
+		subtaskState1 = operatorStates.get(opID1).getState(vertex1.getParallelSubtaskIndex());
+		subtaskState2 = operatorStates.get(opID2).getState(vertex2.getParallelSubtaskIndex());
+
 		assertEquals(0, coord.getNumberOfPendingCheckpoints());
 		assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
 
@@ -1481,7 +1555,7 @@ public void testTriggerAndConfirmSimpleSavepoint() throws Exception {
 		assertEquals(jid, successNew.getJobId());
 		assertEquals(timestampNew, successNew.getTimestamp());
 		assertEquals(checkpointIdNew, successNew.getCheckpointID());
-		assertTrue(successNew.getTaskStates().isEmpty());
+		assertTrue(successNew.getOperatorStates().isEmpty());
 		assertTrue(savepointFuture.isDone());
 
 		// validate that the first savepoint does not discard its private states.
@@ -1969,6 +2043,18 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 		List<KeyGroupRange> keyGroupPartitions1 = StateAssignmentOperation.createKeyGroupPartitions(maxParallelism1, parallelism1);
 		List<KeyGroupRange> keyGroupPartitions2 = StateAssignmentOperation.createKeyGroupPartitions(maxParallelism2, parallelism2);
 
+		PendingCheckpoint pending = coord.getPendingCheckpoints().get(checkpointId);
+
+		OperatorID opID1 = OperatorID.fromJobVertexID(jobVertexID1);
+		OperatorID opID2 = OperatorID.fromJobVertexID(jobVertexID2);
+
+		Map<OperatorID, OperatorState> operatorStates = pending.getOperatorStates();
+
+		operatorStates.put(opID1, new SpyInjectingOperatorState(
+			opID1, jobVertex1.getParallelism(), jobVertex1.getMaxParallelism()));
+		operatorStates.put(opID2, new SpyInjectingOperatorState(
+			opID2, jobVertex2.getParallelism(), jobVertex2.getMaxParallelism()));
+
 		for (int index = 0; index < jobVertex1.getParallelism(); index++) {
 			SubtaskState subtaskState = mockSubtaskState(jobVertexID1, index, keyGroupPartitions1.get(index));
 
@@ -2004,8 +2090,8 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 
 		// All shared states should be unregistered once the store is shut down
 		for (CompletedCheckpoint completedCheckpoint : completedCheckpoints) {
-			for (TaskState taskState : completedCheckpoint.getTaskStates().values()) {
-				for (SubtaskState subtaskState : taskState.getStates()) {
+			for (OperatorState taskState : completedCheckpoint.getOperatorStates().values()) {
+				for (OperatorSubtaskState subtaskState : taskState.getStates()) {
 					verify(subtaskState, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
 				}
 			}
@@ -2021,8 +2107,8 @@ public void testRestoreLatestCheckpointedState() throws Exception {
 
 		// validate that all shared states are registered again after the recovery.
 		for (CompletedCheckpoint completedCheckpoint : completedCheckpoints) {
-			for (TaskState taskState : completedCheckpoint.getTaskStates().values()) {
-				for (SubtaskState subtaskState : taskState.getStates()) {
+			for (OperatorState taskState : completedCheckpoint.getOperatorStates().values()) {
+				for (OperatorSubtaskState subtaskState : taskState.getStates()) {
 					verify(subtaskState, times(2)).registerSharedStates(any(SharedStateRegistry.class));
 				}
 			}
@@ -2432,7 +2518,11 @@ private void testRestoreLatestCheckpointedStateWithChangingParallelism(boolean s
 
 			actualOpStatesBackend.add(opStateBackend);
 			actualOpStatesRaw.add(opStateRaw);
-			assertNull(operatorState);
+			// the 'non partition state' is not null because it is recombined.
+			assertNotNull(operatorState);
+			for (int index = 0; index < operatorState.getLength(); index++) {
+				assertNull(operatorState.get(index));
+			}
 			compareKeyedState(Collections.singletonList(originalKeyedStateBackend), keyedStateBackend);
 			compareKeyedState(Collections.singletonList(originalKeyedStateRaw), keyGroupStateRaw);
 		}
@@ -2681,8 +2771,22 @@ private static ChainedStateHandle<OperatorStateHandle> generateChainedPartitiona
 		return ChainedStateHandle.wrapSingleHandle(operatorStateHandle);
 	}
 
+	static ExecutionJobVertex mockExecutionJobVertex(
+			JobVertexID jobVertexID,
+			int parallelism,
+			int maxParallelism) {
+
+		return mockExecutionJobVertex(
+			jobVertexID,
+			Collections.singletonList(OperatorID.fromJobVertexID(jobVertexID)),
+			parallelism,
+			maxParallelism
+		);
+	}
+
 	static ExecutionJobVertex mockExecutionJobVertex(
 		JobVertexID jobVertexID,
+		List<OperatorID> jobVertexIDs,
 		int parallelism,
 		int maxParallelism) {
 		final ExecutionJobVertex executionJobVertex = mock(ExecutionJobVertex.class);
@@ -2693,6 +2797,7 @@ static ExecutionJobVertex mockExecutionJobVertex(
 			executionVertices[i] = mockExecutionVertex(
 				new ExecutionAttemptID(),
 				jobVertexID,
+				jobVertexIDs,
 				parallelism,
 				maxParallelism,
 				ExecutionState.RUNNING);
@@ -2705,14 +2810,18 @@ static ExecutionJobVertex mockExecutionJobVertex(
 		when(executionJobVertex.getParallelism()).thenReturn(parallelism);
 		when(executionJobVertex.getMaxParallelism()).thenReturn(maxParallelism);
 		when(executionJobVertex.isMaxParallelismConfigured()).thenReturn(true);
+		when(executionJobVertex.getOperatorIDs()).thenReturn(jobVertexIDs);
+		when(executionJobVertex.getUserDefinedOperatorIDs()).thenReturn(Arrays.asList(new OperatorID[jobVertexIDs.size()]));
 
 		return executionJobVertex;
 	}
 
 	static ExecutionVertex mockExecutionVertex(ExecutionAttemptID attemptID) {
+		JobVertexID jobVertexID = new JobVertexID();
 		return mockExecutionVertex(
 			attemptID,
-			new JobVertexID(),
+			jobVertexID,
+			Arrays.asList(OperatorID.fromJobVertexID(jobVertexID)),
 			1,
 			1,
 			ExecutionState.RUNNING);
@@ -2721,6 +2830,7 @@ static ExecutionVertex mockExecutionVertex(ExecutionAttemptID attemptID) {
 	private static ExecutionVertex mockExecutionVertex(
 		ExecutionAttemptID attemptID,
 		JobVertexID jobVertexID,
+		List<OperatorID> jobVertexIDs,
 		int parallelism,
 		int maxParallelism,
 		ExecutionState state,
@@ -2743,6 +2853,11 @@ private static ExecutionVertex mockExecutionVertex(
 		when(vertex.getTotalNumberOfParallelSubtasks()).thenReturn(parallelism);
 		when(vertex.getMaxParallelism()).thenReturn(maxParallelism);
 
+		ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+		when(jobVertex.getOperatorIDs()).thenReturn(jobVertexIDs);
+		
+		when(vertex.getJobVertex()).thenReturn(jobVertex);
+
 		return vertex;
 	}
 
@@ -3135,8 +3250,16 @@ public void testCheckpointStatsTrackerRestoreCallback() throws Exception {
 			null,
 			Executors.directExecutor());
 
-		store.addCheckpoint(
-			new CompletedCheckpoint(new JobID(), 0, 0, 0, Collections.<JobVertexID, TaskState>emptyMap()));
+		store.addCheckpoint(new CompletedCheckpoint(
+			new JobID(),
+			0,
+			0,
+			0,
+			Collections.<OperatorID, OperatorState>emptyMap(),
+			Collections.<MasterState>emptyList(),
+			CheckpointProperties.forStandardCheckpoint(),
+			null,
+			null));
 
 		CheckpointStatsTracker tracker = mock(CheckpointStatsTracker.class);
 		coord.setCheckpointStatsTracker(tracker);
@@ -3146,4 +3269,17 @@ public void testCheckpointStatsTrackerRestoreCallback() throws Exception {
 		verify(tracker, times(1))
 			.reportRestoredCheckpoint(any(RestoredCheckpointStats.class));
 	}
+
+	private static final class SpyInjectingOperatorState extends OperatorState {
+
+		private static final long serialVersionUID = -4004437428483663815L;
+
+		public SpyInjectingOperatorState(OperatorID taskID, int parallelism, int maxParallelism) {
+			super(taskID, parallelism, maxParallelism);
+		}
+
+		public void putState(int subtaskIndex, OperatorSubtaskState subtaskState) {
+			super.putState(subtaskIndex, spy(subtaskState));
+		}
+	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
index 2fc1de5cef3c7..7d2456881fc33 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
@@ -26,6 +26,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
 import org.apache.flink.runtime.state.ChainedStateHandle;
@@ -209,6 +210,8 @@ public void testNonRestoredState() throws Exception {
 		JobVertexID jobVertexId1 = new JobVertexID();
 		JobVertexID jobVertexId2 = new JobVertexID();
 
+		OperatorID operatorId1 = OperatorID.fromJobVertexID(jobVertexId1);
+
 		// 1st JobVertex
 		ExecutionVertex vertex11 = mockExecutionVertex(mockExecution(), jobVertexId1, 0, 3);
 		ExecutionVertex vertex12 = mockExecutionVertex(mockExecution(), jobVertexId1, 1, 3);
@@ -239,20 +242,30 @@ public void testNonRestoredState() throws Exception {
 			null,
 			Executors.directExecutor());
 
-		ChainedStateHandle<StreamStateHandle> serializedState = CheckpointCoordinatorTest
-				.generateChainedStateHandle(new SerializableObject());
+		StreamStateHandle serializedState = CheckpointCoordinatorTest
+				.generateChainedStateHandle(new SerializableObject())
+				.get(0);
 
 		// --- (2) Checkpoint misses state for a jobVertex (should work) ---
-		Map<JobVertexID, TaskState> checkpointTaskStates = new HashMap<>();
+		Map<OperatorID, OperatorState> checkpointTaskStates = new HashMap<>();
 		{
-			TaskState taskState = new TaskState(jobVertexId1, 3, 3, 1);
-			taskState.putState(0, new SubtaskState(serializedState, null, null, null, null));
-			taskState.putState(1, new SubtaskState(serializedState, null, null, null, null));
-			taskState.putState(2, new SubtaskState(serializedState, null, null, null, null));
+			OperatorState taskState = new OperatorState(operatorId1, 3, 3);
+			taskState.putState(0, new OperatorSubtaskState(serializedState, null, null, null, null));
+			taskState.putState(1, new OperatorSubtaskState(serializedState, null, null, null, null));
+			taskState.putState(2, new OperatorSubtaskState(serializedState, null, null, null, null));
 
-			checkpointTaskStates.put(jobVertexId1, taskState);
+			checkpointTaskStates.put(operatorId1, taskState);
 		}
-		CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 1, 2, new HashMap<>(checkpointTaskStates));
+		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
+			new JobID(),
+			0,
+			1,
+			2,
+			new HashMap<>(checkpointTaskStates),
+			Collections.<MasterState>emptyList(),
+			CheckpointProperties.forStandardCheckpoint(),
+			null,
+			null);
 
 		coord.getCheckpointStore().addCheckpoint(checkpoint);
 
@@ -261,16 +274,26 @@ public void testNonRestoredState() throws Exception {
 
 		// --- (3) JobVertex missing for task state that is part of the checkpoint ---
 		JobVertexID newJobVertexID = new JobVertexID();
+		OperatorID newOperatorID = OperatorID.fromJobVertexID(newJobVertexID);
 
 		// There is no task for this
 		{
-			TaskState taskState = new TaskState(jobVertexId1, 1, 1, 1);
-			taskState.putState(0, new SubtaskState(serializedState, null, null, null, null));
+			OperatorState taskState = new OperatorState(newOperatorID, 1, 1);
+			taskState.putState(0, new OperatorSubtaskState(serializedState, null, null, null, null));
 
-			checkpointTaskStates.put(newJobVertexID, taskState);
+			checkpointTaskStates.put(newOperatorID, taskState);
 		}
 
-		checkpoint = new CompletedCheckpoint(new JobID(), 1, 2, 3, new HashMap<>(checkpointTaskStates));
+		checkpoint = new CompletedCheckpoint(
+			new JobID(),
+			1,
+			2,
+			3,
+			new HashMap<>(checkpointTaskStates),
+			Collections.<MasterState>emptyList(),
+			CheckpointProperties.forStandardCheckpoint(),
+			null,
+			null);
 
 		coord.getCheckpointStore().addCheckpoint(checkpoint);
 
@@ -314,6 +337,12 @@ private ExecutionJobVertex mockExecutionJobVertex(JobVertexID id, ExecutionVerte
 		when(vertex.getMaxParallelism()).thenReturn(vertices.length);
 		when(vertex.getJobVertexId()).thenReturn(id);
 		when(vertex.getTaskVertices()).thenReturn(vertices);
+		when(vertex.getOperatorIDs()).thenReturn(Collections.singletonList(OperatorID.fromJobVertexID(id)));
+		when(vertex.getUserDefinedOperatorIDs()).thenReturn(Collections.<OperatorID>singletonList(null));
+
+		for (ExecutionVertex v : vertices) {
+			when(v.getJobVertex()).thenReturn(vertex);
+		}
 		return vertex;
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
index fc6e51664306e..94bd12f893fb4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
@@ -20,8 +20,7 @@
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
@@ -39,6 +38,7 @@
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
@@ -105,7 +105,7 @@ public void testAddCheckpointMoreThanMaxRetained() throws Exception {
 		assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
 
 		for (int i = 1; i < expected.length; i++) {
-			Collection<TaskState> taskStates = expected[i - 1].getTaskStates().values();
+			Collection<OperatorState> taskStates = expected[i - 1].getOperatorStates().values();
 
 			checkpoints.addCheckpoint(expected[i]);
 
@@ -114,8 +114,8 @@ public void testAddCheckpointMoreThanMaxRetained() throws Exception {
 			assertTrue(expected[i - 1].isDiscarded());
 			assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
 
-			for (TaskState taskState : taskStates) {
-				for (SubtaskState subtaskState : taskState.getStates()) {
+			for (OperatorState operatorState : taskStates) {
+				for (OperatorSubtaskState subtaskState : operatorState.getStates()) {
 					verify(subtaskState, times(1)).unregisterSharedStates(any(SharedStateRegistry.class));
 				}
 			}
@@ -198,51 +198,43 @@ public void testDiscardAllCheckpoints() throws Exception {
 	// ---------------------------------------------------------------------------------------------
 
 	protected TestCompletedCheckpoint createCheckpoint(int id) throws IOException {
-		return createCheckpoint(id, 4);
-	}
-
-	protected TestCompletedCheckpoint createCheckpoint(int id, int numberOfStates)
-			throws IOException {
-		return createCheckpoint(id, numberOfStates, CheckpointProperties.forStandardCheckpoint());
-	}
+		int numberOfStates = 4;
+		CheckpointProperties props = CheckpointProperties.forStandardCheckpoint();
 
-	protected TestCompletedCheckpoint createCheckpoint(int id, int numberOfStates, CheckpointProperties props)
-			throws IOException {
+		OperatorID operatorID = new OperatorID();
 
-		JobVertexID jvid = new JobVertexID();
-
-		Map<JobVertexID, TaskState> taskGroupStates = new HashMap<>();
-		TaskState taskState = new TaskState(jvid, numberOfStates, numberOfStates, 1);
-		taskGroupStates.put(jvid, taskState);
+		Map<OperatorID, OperatorState> operatorGroupState = new HashMap<>();
+		OperatorState operatorState = new OperatorState(operatorID, numberOfStates, numberOfStates);
+		operatorGroupState.put(operatorID, operatorState);
 
 		for (int i = 0; i < numberOfStates; i++) {
-			SubtaskState subtaskState = CheckpointCoordinatorTest.mockSubtaskState(jvid, i, new KeyGroupRange(i, i));
+			OperatorSubtaskState subtaskState = mock(OperatorSubtaskState.class);
 
-			taskState.putState(i, subtaskState);
+			operatorState.putState(i, subtaskState);
 		}
 
-		return new TestCompletedCheckpoint(new JobID(), id, 0, taskGroupStates, props);
+		return new TestCompletedCheckpoint(new JobID(), id, 0, operatorGroupState, props);
 	}
 
-	protected void resetCheckpoint(Collection<TaskState> taskStates) {
-		for (TaskState taskState : taskStates) {
-			for (SubtaskState subtaskState : taskState.getStates()) {
+	protected void resetCheckpoint(Collection<OperatorState> operatorStates) {
+		for (OperatorState operatorState : operatorStates) {
+			for (OperatorSubtaskState subtaskState : operatorState.getStates()) {
 				Mockito.reset(subtaskState);
 			}
 		}
 	}
 
-	protected void verifyCheckpointRegistered(Collection<TaskState> taskStates, SharedStateRegistry registry) {
-		for (TaskState taskState : taskStates) {
-			for (SubtaskState subtaskState : taskState.getStates()) {
+	protected void verifyCheckpointRegistered(Collection<OperatorState> operatorStates, SharedStateRegistry registry) {
+		for (OperatorState operatorState : operatorStates) {
+			for (OperatorSubtaskState subtaskState : operatorState.getStates()) {
 				verify(subtaskState, times(1)).registerSharedStates(eq(registry));
 			}
 		}
 	}
 
-	protected void verifyCheckpointDiscarded(Collection<TaskState> taskStates) {
-		for (TaskState taskState : taskStates) {
-			for (SubtaskState subtaskState : taskState.getStates()) {
+	protected void verifyCheckpointDiscarded(Collection<OperatorState> operatorStates) {
+		for (OperatorState operatorState : operatorStates) {
+			for (OperatorSubtaskState subtaskState : operatorState.getStates()) {
 				verify(subtaskState, times(1)).discardState();
 			}
 		}
@@ -270,10 +262,9 @@ public TestCompletedCheckpoint(
 			JobID jobId,
 			long checkpointId,
 			long timestamp,
-			Map<JobVertexID, TaskState> taskGroupStates,
+			Map<OperatorID, OperatorState> operatorGroupState,
 			CheckpointProperties props) {
-
-			super(jobId, checkpointId, timestamp, Long.MAX_VALUE, taskGroupStates, null, props);
+			super(jobId, checkpointId, timestamp, Long.MAX_VALUE, operatorGroupState, null, props, null, null);
 		}
 
 		@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
index 652cc767af366..589ff46863a35 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
@@ -23,7 +23,7 @@
 import org.apache.flink.core.testutils.CommonTestUtils;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.state.SharedStateHandle;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
 import org.junit.Rule;
@@ -37,7 +37,6 @@
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -55,9 +54,9 @@ public void testDiscard() throws Exception {
 		File file = tmpFolder.newFile();
 		assertEquals(true, file.exists());
 
-		TaskState state = mock(TaskState.class);
-		Map<JobVertexID, TaskState> taskStates = new HashMap<>();
-		taskStates.put(new JobVertexID(), state);
+		OperatorState state = mock(OperatorState.class);
+		Map<OperatorID, OperatorState> taskStates = new HashMap<>();
+		taskStates.put(new OperatorID(), state);
 
 		// Verify discard call is forwarded to state
 		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
@@ -78,18 +77,20 @@ public void testDiscard() throws Exception {
 	 */
 	@Test
 	public void testCleanUpOnSubsume() throws Exception {
-		TaskState state = mock(TaskState.class);
-		Map<JobVertexID, TaskState> taskStates = new HashMap<>();
-		taskStates.put(new JobVertexID(), state);
+		OperatorState state = mock(OperatorState.class);
+		Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
+		operatorStates.put(new OperatorID(), state);
 
 		boolean discardSubsumed = true;
 		CheckpointProperties props = new CheckpointProperties(false, false, discardSubsumed, true, true, true, true);
 		
 		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
 				new JobID(), 0, 0, 1,
-				taskStates,
+				operatorStates,
 				Collections.<MasterState>emptyList(),
-				props);
+				props,
+				null,
+				null);
 
 		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		checkpoint.registerSharedStates(sharedStateRegistry);
@@ -114,9 +115,9 @@ public void testCleanUpOnShutdown() throws Exception {
 				JobStatus.FINISHED, JobStatus.CANCELED, JobStatus.FAILED, JobStatus.SUSPENDED
 		};
 
-		TaskState state = mock(TaskState.class);
-		Map<JobVertexID, TaskState> taskStates = new HashMap<>();
-		taskStates.put(new JobVertexID(), state);
+		OperatorState state = mock(OperatorState.class);
+		Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
+		operatorStates.put(new OperatorID(), state);
 
 		for (JobStatus status : terminalStates) {
 			Mockito.reset(state);
@@ -125,7 +126,7 @@ public void testCleanUpOnShutdown() throws Exception {
 			CheckpointProperties props = new CheckpointProperties(false, true, false, false, false, false, false);
 			CompletedCheckpoint checkpoint = new CompletedCheckpoint(
 					new JobID(), 0, 0, 1,
-					new HashMap<>(taskStates),
+					new HashMap<>(operatorStates),
 					Collections.<MasterState>emptyList(),
 					props,
 					new FileStateHandle(new Path(file.toURI()), file.length()),
@@ -143,9 +144,11 @@ public void testCleanUpOnShutdown() throws Exception {
 			props = new CheckpointProperties(false, false, true, true, true, true, true);
 			checkpoint = new CompletedCheckpoint(
 					new JobID(), 0, 0, 1,
-					new HashMap<>(taskStates),
+					new HashMap<>(operatorStates),
 					Collections.<MasterState>emptyList(),
-					props);
+					props,
+					null,
+					null);
 
 			checkpoint.discardOnShutdown(status, sharedStateRegistry);
 			verify(state, times(1)).discardState();
@@ -158,18 +161,20 @@ public void testCleanUpOnShutdown() throws Exception {
 	 */
 	@Test
 	public void testCompletedCheckpointStatsCallbacks() throws Exception {
-		TaskState state = mock(TaskState.class);
-		Map<JobVertexID, TaskState> taskStates = new HashMap<>();
-		taskStates.put(new JobVertexID(), state);
+		OperatorState state = mock(OperatorState.class);
+		Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
+		operatorStates.put(new OperatorID(), state);
 
 		CompletedCheckpoint completed = new CompletedCheckpoint(
 			new JobID(),
 			0,
 			0,
 			1,
-			new HashMap<>(taskStates),
+			new HashMap<>(operatorStates),
 			Collections.<MasterState>emptyList(),
-			CheckpointProperties.forStandardCheckpoint());
+			CheckpointProperties.forStandardCheckpoint(),
+			null,
+			null);
 
 		CompletedCheckpointStats.DiscardCallback callback = mock(CompletedCheckpointStats.DiscardCallback.class);
 		completed.setDiscardCallback(callback);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
index 2dd1803cd9960..6df01a02b32ca 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
@@ -22,15 +22,15 @@
 import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.state.SharedStateHandle;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.SharedStateRegistry;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.mockito.Mock;
 import org.mockito.Mockito;
 
 import java.io.File;
@@ -50,9 +50,7 @@
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.powermock.api.mockito.PowerMockito.when;
@@ -63,9 +61,13 @@ public class PendingCheckpointTest {
 	private static final ExecutionAttemptID ATTEMPT_ID = new ExecutionAttemptID();
 
 	static {
+		ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+		when(jobVertex.getOperatorIDs()).thenReturn(Collections.singletonList(new OperatorID()));
+		
 		ExecutionVertex vertex = mock(ExecutionVertex.class);
 		when(vertex.getMaxParallelism()).thenReturn(128);
 		when(vertex.getTotalNumberOfParallelSubtasks()).thenReturn(1);
+		when(vertex.getJobVertex()).thenReturn(jobVertex);
 		ACK_TASKS.put(ATTEMPT_ID, vertex);
 	}
 
@@ -193,7 +195,7 @@ public void testAbortDiscardsState() throws Exception {
 		CheckpointProperties props = new CheckpointProperties(false, true, false, false, false, false, false);
 		QueueExecutor executor = new QueueExecutor();
 
-		TaskState state = mock(TaskState.class);
+		OperatorState state = mock(OperatorState.class);
 		doNothing().when(state).registerSharedStates(any(SharedStateRegistry.class));
 		doNothing().when(state).unregisterSharedStates(any(SharedStateRegistry.class));
 
@@ -311,7 +313,7 @@ public void testPendingCheckpointStatsCallbacks() throws Exception {
 	public void testNullSubtaskStateLeadsToStatelessTask() throws Exception {
 		PendingCheckpoint pending = createPendingCheckpoint(CheckpointProperties.forStandardCheckpoint(), null);
 		pending.acknowledgeTask(ATTEMPT_ID, null, mock(CheckpointMetrics.class));
-		Assert.assertTrue(pending.getTaskStates().isEmpty());
+		Assert.assertTrue(pending.getOperatorStates().isEmpty());
 	}
 
 	/**
@@ -324,7 +326,7 @@ public void testNullSubtaskStateLeadsToStatelessTask() throws Exception {
 	public void testNonNullSubtaskStateLeadsToStatefulTask() throws Exception {
 		PendingCheckpoint pending = createPendingCheckpoint(CheckpointProperties.forStandardCheckpoint(), null);
 		pending.acknowledgeTask(ATTEMPT_ID, mock(SubtaskState.class), mock(CheckpointMetrics.class));
-		Assert.assertFalse(pending.getTaskStates().isEmpty());
+		Assert.assertFalse(pending.getOperatorStates().isEmpty());
 	}
 
 	@Test
@@ -367,12 +369,12 @@ private static PendingCheckpoint createPendingCheckpoint(
 	}
 
 	@SuppressWarnings("unchecked")
-	static void setTaskState(PendingCheckpoint pending, TaskState state) throws NoSuchFieldException, IllegalAccessException {
-		Field field = PendingCheckpoint.class.getDeclaredField("taskStates");
+	static void setTaskState(PendingCheckpoint pending, OperatorState state) throws NoSuchFieldException, IllegalAccessException {
+		Field field = PendingCheckpoint.class.getDeclaredField("operatorStates");
 		field.setAccessible(true);
-		Map<JobVertexID, TaskState> taskStates = (Map<JobVertexID, TaskState>) field.get(pending);
+		Map<OperatorID, OperatorState> taskStates = (Map<OperatorID, OperatorState>) field.get(pending);
 
-		taskStates.put(new JobVertexID(), state);
+		taskStates.put(new OperatorID(), state);
 	}
 
 	private static final class QueueExecutor implements Executor {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
index 64aeebac41780..be9476225fb11 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java
@@ -54,16 +54,16 @@ protected AbstractCompletedCheckpointStore createCompletedCheckpoints(
 	public void testShutdownDiscardsCheckpoints() throws Exception {
 		AbstractCompletedCheckpointStore store = createCompletedCheckpoints(1);
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
-		Collection<TaskState> taskStates = checkpoint.getTaskStates().values();
+		Collection<OperatorState> operatorStates = checkpoint.getOperatorStates().values();
 
 		store.addCheckpoint(checkpoint);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
-		verifyCheckpointRegistered(taskStates, store.sharedStateRegistry);
+		verifyCheckpointRegistered(operatorStates, store.sharedStateRegistry);
 
 		store.shutdown(JobStatus.FINISHED);
 		assertEquals(0, store.getNumberOfRetainedCheckpoints());
 		assertTrue(checkpoint.isDiscarded());
-		verifyCheckpointDiscarded(taskStates);
+		verifyCheckpointDiscarded(operatorStates);
 	}
 
 	/**
@@ -74,7 +74,7 @@ public void testShutdownDiscardsCheckpoints() throws Exception {
 	public void testSuspendDiscardsCheckpoints() throws Exception {
 		AbstractCompletedCheckpointStore store = createCompletedCheckpoints(1);
 		TestCompletedCheckpoint checkpoint = createCheckpoint(0);
-		Collection<TaskState> taskStates = checkpoint.getTaskStates().values();
+		Collection<OperatorState> taskStates = checkpoint.getOperatorStates().values();
 
 		store.addCheckpoint(checkpoint);
 		assertEquals(1, store.getNumberOfRetainedCheckpoints());
@@ -99,7 +99,7 @@ public void testAddCheckpointWithFailedRemove() throws Exception {
 		for (long i = 0; i <= numCheckpointsToRetain; ++i) {
 			CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class);
 			doReturn(i).when(checkpointToAdd).getCheckpointID();
-			doReturn(Collections.emptyMap()).when(checkpointToAdd).getTaskStates();
+			doReturn(Collections.emptyMap()).when(checkpointToAdd).getOperatorStates();
 			doThrow(new IOException()).when(checkpointToAdd).discardOnSubsume(any(SharedStateRegistry.class));
 			
 			try {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
index 73fcf78298a40..73e0ed9e7bf09 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java
@@ -90,17 +90,17 @@ public void testRecover() throws Exception {
 		checkpoints.addCheckpoint(expected[1]);
 		checkpoints.addCheckpoint(expected[2]);
 
-		verifyCheckpointRegistered(expected[0].getTaskStates().values(), checkpoints.sharedStateRegistry);
-		verifyCheckpointRegistered(expected[1].getTaskStates().values(), checkpoints.sharedStateRegistry);
-		verifyCheckpointRegistered(expected[2].getTaskStates().values(), checkpoints.sharedStateRegistry);
+		verifyCheckpointRegistered(expected[0].getOperatorStates().values(), checkpoints.sharedStateRegistry);
+		verifyCheckpointRegistered(expected[1].getOperatorStates().values(), checkpoints.sharedStateRegistry);
+		verifyCheckpointRegistered(expected[2].getOperatorStates().values(), checkpoints.sharedStateRegistry);
 
 		// All three should be in ZK
 		assertEquals(3, ZooKeeper.getClient().getChildren().forPath(CheckpointsPath).size());
 		assertEquals(3, checkpoints.getNumberOfRetainedCheckpoints());
 
-		resetCheckpoint(expected[0].getTaskStates().values());
-		resetCheckpoint(expected[1].getTaskStates().values());
-		resetCheckpoint(expected[2].getTaskStates().values());
+		resetCheckpoint(expected[0].getOperatorStates().values());
+		resetCheckpoint(expected[1].getOperatorStates().values());
+		resetCheckpoint(expected[2].getOperatorStates().values());
 
 		// Recover TODO!!! clear registry!
 		checkpoints.recover();
@@ -121,7 +121,7 @@ public void testRecover() throws Exception {
 		assertEquals(expectedCheckpoints, actualCheckpoints);
 
 		for (CompletedCheckpoint actualCheckpoint : actualCheckpoints) {
-			verifyCheckpointRegistered(actualCheckpoint.getTaskStates().values(), checkpoints.sharedStateRegistry);
+			verifyCheckpointRegistered(actualCheckpoint.getOperatorStates().values(), checkpoints.sharedStateRegistry);
 		}
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
index 66ef232b0d31b..8fc0f02a86f3f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
@@ -227,7 +227,7 @@ public RetrievableStateHandle<CompletedCheckpoint> answer(InvocationOnMock invoc
 		for (long i = 0; i <= numCheckpointsToRetain; ++i) {
 			CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class);
 			doReturn(i).when(checkpointToAdd).getCheckpointID();
-			doReturn(Collections.emptyMap()).when(checkpointToAdd).getTaskStates();
+			doReturn(Collections.emptyMap()).when(checkpointToAdd).getOperatorStates();
 			
 			try {
 				zooKeeperCompletedCheckpointStore.addCheckpoint(checkpointToAdd);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
index 7d9874e96e2c2..ba77dbc0b6ca4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
@@ -20,9 +20,12 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
+import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
 import org.apache.flink.runtime.checkpoint.SubtaskState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.ChainedStateHandle;
 import org.apache.flink.runtime.state.KeyGroupRangeOffsets;
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
@@ -48,6 +51,94 @@
  */
 public class CheckpointTestUtils {
 
+	/**
+	 * Creates a random collection of OperatorState objects containing various types of state handles.
+	 */
+	public static Collection<OperatorState> createOperatorStates(int numTaskStates, int numSubtasksPerTask) {
+		return createOperatorStates(new Random(), numTaskStates, numSubtasksPerTask);
+	}
+
+	/**
+	 * Creates a random collection of OperatorState objects containing various types of state handles.
+	 */
+	public static Collection<OperatorState> createOperatorStates(
+			Random random,
+			int numTaskStates,
+			int numSubtasksPerTask) {
+
+		List<OperatorState> taskStates = new ArrayList<>(numTaskStates);
+
+		for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {
+
+			OperatorState taskState = new OperatorState(new OperatorID(), numSubtasksPerTask, 128);
+
+			boolean hasNonPartitionableState = random.nextBoolean();
+			boolean hasOperatorStateBackend = random.nextBoolean();
+			boolean hasOperatorStateStream = random.nextBoolean();
+
+			boolean hasKeyedBackend = random.nextInt(4) != 0;
+			boolean hasKeyedStream = random.nextInt(4) != 0;
+
+			for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {
+
+				StreamStateHandle nonPartitionableState = null;
+				StreamStateHandle operatorStateBackend =
+					new TestByteStreamStateHandleDeepCompare("b", ("Beautiful").getBytes(ConfigConstants.DEFAULT_CHARSET));
+				StreamStateHandle operatorStateStream =
+					new TestByteStreamStateHandleDeepCompare("b", ("Beautiful").getBytes(ConfigConstants.DEFAULT_CHARSET));
+
+				OperatorStateHandle operatorStateHandleBackend = null;
+				OperatorStateHandle operatorStateHandleStream = null;
+				
+				Map<String, StateMetaInfo> offsetsMap = new HashMap<>();
+				offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
+				offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
+				offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.BROADCAST));
+
+				if (hasNonPartitionableState) {
+					nonPartitionableState =
+						new TestByteStreamStateHandleDeepCompare("a", ("Hi").getBytes(ConfigConstants.DEFAULT_CHARSET));
+				}
+
+				if (hasOperatorStateBackend) {
+					operatorStateHandleBackend = new OperatorStateHandle(offsetsMap, operatorStateBackend);
+				}
+
+				if (hasOperatorStateStream) {
+					operatorStateHandleStream = new OperatorStateHandle(offsetsMap, operatorStateStream);
+				}
+
+				KeyGroupsStateHandle keyedStateBackend = null;
+				KeyGroupsStateHandle keyedStateStream = null;
+
+				if (hasKeyedBackend) {
+					keyedStateBackend = new KeyGroupsStateHandle(
+							new KeyGroupRangeOffsets(1, 1, new long[]{42}),
+							new TestByteStreamStateHandleDeepCompare("c", "Hello"
+									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+				}
+
+				if (hasKeyedStream) {
+					keyedStateStream = new KeyGroupsStateHandle(
+							new KeyGroupRangeOffsets(1, 1, new long[]{23}),
+							new TestByteStreamStateHandleDeepCompare("d", "World"
+									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+				}
+
+				taskState.putState(subtaskIdx, new OperatorSubtaskState(
+						nonPartitionableState,
+						operatorStateHandleBackend,
+						operatorStateHandleStream,
+						keyedStateStream,
+						keyedStateBackend));
+			}
+
+			taskStates.add(taskState);
+		}
+
+		return taskStates;
+	}
+
 	/**
 	 * Creates a random collection of TaskState objects containing various types of state handles.
 	 */
@@ -88,7 +179,7 @@ public static Collection<TaskState> createTaskStates(
 
 					StreamStateHandle nonPartitionableState =
 							new TestByteStreamStateHandleDeepCompare("a-" + chainIdx, ("Hi-" + chainIdx).getBytes(
-									ConfigConstants.DEFAULT_CHARSET));
+								ConfigConstants.DEFAULT_CHARSET));
 					StreamStateHandle operatorStateBackend =
 							new TestByteStreamStateHandleDeepCompare("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
 					StreamStateHandle operatorStateStream =
@@ -122,14 +213,14 @@ public static Collection<TaskState> createTaskStates(
 					keyedStateBackend = new KeyGroupsStateHandle(
 							new KeyGroupRangeOffsets(1, 1, new long[]{42}),
 							new TestByteStreamStateHandleDeepCompare("c", "Hello"
-									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+								.getBytes(ConfigConstants.DEFAULT_CHARSET)));
 				}
 
 				if (hasKeyedStream) {
 					keyedStateStream = new KeyGroupsStateHandle(
 							new KeyGroupRangeOffsets(1, 1, new long[]{23}),
 							new TestByteStreamStateHandleDeepCompare("d", "World"
-									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+								.getBytes(ConfigConstants.DEFAULT_CHARSET)));
 				}
 
 				taskState.putState(subtaskIdx, new SubtaskState(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java
index 20b1e5713bfa4..331621d165f60 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointLoaderTest.java
@@ -20,14 +20,17 @@
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
-import org.apache.flink.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -53,29 +56,30 @@ public void testLoadAndValidateSavepoint() throws Exception {
 
 		int parallelism = 128128;
 		long checkpointId = Integer.MAX_VALUE + 123123L;
-		JobVertexID vertexId = new JobVertexID();
+		JobVertexID jobVertexID = new JobVertexID();
+		OperatorID operatorID = OperatorID.fromJobVertexID(jobVertexID);
 
-		TaskState state = mock(TaskState.class);
+		OperatorState state = mock(OperatorState.class);
 		when(state.getParallelism()).thenReturn(parallelism);
-		when(state.getJobVertexID()).thenReturn(vertexId);
+		when(state.getOperatorID()).thenReturn(operatorID);
 		when(state.getMaxParallelism()).thenReturn(parallelism);
-		when(state.getChainLength()).thenReturn(1);
 
-		Map<JobVertexID, TaskState> taskStates = new HashMap<>();
-		taskStates.put(vertexId, state);
+		Map<OperatorID, OperatorState> taskStates = new HashMap<>();
+		taskStates.put(operatorID, state);
 
 		JobID jobId = new JobID();
 
 		// Store savepoint
-		SavepointV2 savepoint = new SavepointV2(checkpointId, taskStates.values());
+		SavepointV2 savepoint = new SavepointV2(checkpointId, taskStates.values(), Collections.<MasterState>emptyList());
 		String path = SavepointStore.storeSavepoint(tmp.getAbsolutePath(), savepoint);
 
 		ExecutionJobVertex vertex = mock(ExecutionJobVertex.class);
 		when(vertex.getParallelism()).thenReturn(parallelism);
 		when(vertex.getMaxParallelism()).thenReturn(parallelism);
+		when(vertex.getOperatorIDs()).thenReturn(Collections.singletonList(operatorID));
 
 		Map<JobVertexID, ExecutionJobVertex> tasks = new HashMap<>();
-		tasks.put(vertexId, vertex);
+		tasks.put(jobVertexID, vertex);
 
 		ClassLoader ucl = Thread.currentThread().getContextClassLoader();
 
@@ -97,7 +101,7 @@ public void testLoadAndValidateSavepoint() throws Exception {
 		}
 
 		// 3) Load and validate: missing vertex
-		assertNotNull(tasks.remove(vertexId));
+		assertNotNull(tasks.remove(jobVertexID));
 
 		try {
 			SavepointLoader.loadAndValidateSavepoint(jobId, tasks, path, ucl, false);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java
index cf79282797479..391102cb20dbd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointStoreTest.java
@@ -24,6 +24,7 @@
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.checkpoint.MasterState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 import org.apache.flink.runtime.checkpoint.TaskState;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
 import org.junit.Rule;
@@ -69,7 +70,10 @@ public void testStoreLoadDispose() throws Exception {
 
 		// Store
 		String savepointDirectory = SavepointStore.createSavepointDirectory(root, new JobID());
-		SavepointV2 stored = new SavepointV2(1929292, CheckpointTestUtils.createTaskStates(4, 24));
+		SavepointV2 stored = new SavepointV2(
+			1929292,
+			CheckpointTestUtils.createOperatorStates(4, 24),
+			Collections.<MasterState>emptyList());
 		String path = SavepointStore.storeSavepoint(savepointDirectory, stored);
 
 		list = rootFile.listFiles();
@@ -80,7 +84,7 @@ public void testStoreLoadDispose() throws Exception {
 		Savepoint loaded = SavepointStore.loadSavepoint(path, Thread.currentThread().getContextClassLoader());
 
 		assertEquals(stored.getCheckpointId(), loaded.getCheckpointId());
-		assertEquals(stored.getTaskStates(), loaded.getTaskStates());
+		assertEquals(stored.getOperatorStates(), loaded.getOperatorStates());
 		assertEquals(stored.getMasterStates(), loaded.getMasterStates());
 
 		loaded.dispose();
@@ -147,7 +151,10 @@ public void testMultipleSavepointVersions() throws Exception {
 
 		// Savepoint v0
 		String savepointDirectory2 = SavepointStore.createSavepointDirectory(root, new JobID());
-		SavepointV2 savepoint = new SavepointV2(checkpointId, CheckpointTestUtils.createTaskStates(4, 32));
+		SavepointV2 savepoint = new SavepointV2(
+			checkpointId,
+			CheckpointTestUtils.createOperatorStates(4, 32),
+			Collections.<MasterState>emptyList());
 		String pathSavepoint = SavepointStore.storeSavepoint(savepointDirectory2, savepoint);
 
 		list = rootFile.listFiles();
@@ -205,7 +212,10 @@ public void testStoreExternalizedCheckpointsToSameDirectory() throws Exception {
 		FileSystem fs = FileSystem.get(new Path(root).toUri());
 
 		// Store
-		SavepointV2 savepoint = new SavepointV2(1929292, CheckpointTestUtils.createTaskStates(4, 24));
+		SavepointV2 savepoint = new SavepointV2(
+			1929292,
+			CheckpointTestUtils.createOperatorStates(4, 24),
+			Collections.<MasterState>emptyList());
 
 		FileStateHandle store1 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
 		fs.exists(store1.getFilePath());
@@ -265,6 +275,11 @@ public Collection<MasterState> getMasterStates() {
 			return Collections.emptyList();
 		}
 
+		@Override
+		public Collection<OperatorState> getOperatorStates() {
+			return null;
+		}
+
 		@Override
 		public void dispose() {
 		}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
index deb14ddc85a8c..154d761dd025f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
@@ -23,7 +23,7 @@
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.runtime.checkpoint.MasterState;
-import org.apache.flink.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 
 import org.junit.Test;
 
@@ -48,7 +48,7 @@ public void testCheckpointWithNoState() throws Exception {
 
 		for (int i = 0; i < 100; ++i) {
 			final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
-			final Collection<TaskState> taskStates = Collections.emptyList();
+			final Collection<OperatorState> taskStates = Collections.emptyList();
 			final Collection<MasterState> masterStates = Collections.emptyList();
 
 			testCheckpointSerialization(checkpointId, taskStates, masterStates);
@@ -63,13 +63,13 @@ public void testCheckpointWithOnlyMasterState() throws Exception {
 		for (int i = 0; i < 100; ++i) {
 			final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
 
-			final Collection<TaskState> taskStates = Collections.emptyList();
+			final Collection<OperatorState> operatorStates = Collections.emptyList();
 
 			final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
 			final Collection<MasterState> masterStates = 
 					CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
 
-			testCheckpointSerialization(checkpointId, taskStates, masterStates);
+			testCheckpointSerialization(checkpointId, operatorStates, masterStates);
 		}
 	}
 
@@ -84,8 +84,8 @@ public void testCheckpointWithOnlyTaskState() throws Exception {
 
 			final int numTasks = rnd.nextInt(maxTaskStates) + 1;
 			final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
-			final Collection<TaskState> taskStates = 
-					CheckpointTestUtils.createTaskStates(rnd, numTasks, numSubtasks);
+			final Collection<OperatorState> taskStates = 
+					CheckpointTestUtils.createOperatorStates(rnd, numTasks, numSubtasks);
 
 			final Collection<MasterState> masterStates = Collections.emptyList();
 
@@ -106,8 +106,8 @@ public void testCheckpointWithMasterAndTaskState() throws Exception {
 
 			final int numTasks = rnd.nextInt(maxTaskStates) + 1;
 			final int numSubtasks = rnd.nextInt(maxNumSubtasks) + 1;
-			final Collection<TaskState> taskStates =
-					CheckpointTestUtils.createTaskStates(rnd, numTasks, numSubtasks);
+			final Collection<OperatorState> taskStates =
+					CheckpointTestUtils.createOperatorStates(rnd, numTasks, numSubtasks);
 
 			final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
 			final Collection<MasterState> masterStates =
@@ -119,7 +119,7 @@ public void testCheckpointWithMasterAndTaskState() throws Exception {
 
 	private void testCheckpointSerialization(
 			long checkpointId,
-			Collection<TaskState> taskStates,
+			Collection<OperatorState> operatorStates,
 			Collection<MasterState> masterStates) throws IOException {
 
 		SavepointV2Serializer serializer = SavepointV2Serializer.INSTANCE;
@@ -127,7 +127,7 @@ private void testCheckpointSerialization(
 		ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
 		DataOutputStream out = new DataOutputViewStreamWrapper(baos);
 
-		serializer.serialize(new SavepointV2(checkpointId, taskStates, masterStates), out);
+		serializer.serialize(new SavepointV2(checkpointId, operatorStates, masterStates), out);
 		out.close();
 
 		byte[] bytes = baos.toByteArray();
@@ -136,7 +136,7 @@ private void testCheckpointSerialization(
 		SavepointV2 deserialized = serializer.deserialize(in, getClass().getClassLoader());
 
 		assertEquals(checkpointId, deserialized.getCheckpointId());
-		assertEquals(taskStates, deserialized.getTaskStates());
+		assertEquals(operatorStates, deserialized.getOperatorStates());
 
 		assertEquals(masterStates.size(), deserialized.getMasterStates().size());
 		for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java
index 428a62a675c7c..6b6a6d4c22e50 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Test.java
@@ -19,7 +19,7 @@
 package org.apache.flink.runtime.checkpoint.savepoint;
 
 import org.apache.flink.runtime.checkpoint.MasterState;
-import org.apache.flink.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
 
 import org.junit.Test;
 
@@ -36,7 +36,7 @@ public class SavepointV2Test {
 	 * Simple test of savepoint methods.
 	 */
 	@Test
-	public void testSavepointV1() throws Exception {
+	public void testSavepointV2() throws Exception {
 		final Random rnd = new Random();
 
 		final long checkpointId = rnd.nextInt(Integer.MAX_VALUE) + 1;
@@ -44,8 +44,8 @@ public void testSavepointV1() throws Exception {
 		final int numSubtaskStates = 16;
 		final int numMasterStates = 7;
 
-		Collection<TaskState> taskStates =
-				CheckpointTestUtils.createTaskStates(rnd, numTaskStates, numSubtaskStates);
+		Collection<OperatorState> taskStates =
+				CheckpointTestUtils.createOperatorStates(rnd, numTaskStates, numSubtaskStates);
 
 		Collection<MasterState> masterStates =
 				CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
@@ -54,15 +54,15 @@ public void testSavepointV1() throws Exception {
 
 		assertEquals(2, checkpoint.getVersion());
 		assertEquals(checkpointId, checkpoint.getCheckpointId());
-		assertEquals(taskStates, checkpoint.getTaskStates());
+		assertEquals(taskStates, checkpoint.getOperatorStates());
 		assertEquals(masterStates, checkpoint.getMasterStates());
 
-		assertFalse(checkpoint.getTaskStates().isEmpty());
+		assertFalse(checkpoint.getOperatorStates().isEmpty());
 		assertFalse(checkpoint.getMasterStates().isEmpty());
 
 		checkpoint.dispose();
 
-		assertTrue(checkpoint.getTaskStates().isEmpty());
+		assertTrue(checkpoint.getOperatorStates().isEmpty());
 		assertTrue(checkpoint.getMasterStates().isEmpty());
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LegacyJobVertexIdTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LegacyJobVertexIdTest.java
index 89db3a14354d5..b5a67fdb9682c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LegacyJobVertexIdTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LegacyJobVertexIdTest.java
@@ -25,11 +25,13 @@
 import org.apache.flink.runtime.instance.SlotProvider;
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.util.SerializedValue;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -46,7 +48,7 @@ public void testIntroduceLegacyJobVertexIds() throws Exception {
 		JobVertexID legacyId1 = new JobVertexID();
 		JobVertexID legacyId2 = new JobVertexID();
 
-		JobVertex jobVertex = new JobVertex("test", defaultId, Arrays.asList(legacyId1, legacyId2));
+		JobVertex jobVertex = new JobVertex("test", defaultId, Arrays.asList(legacyId1, legacyId2), new ArrayList<OperatorID>(), new ArrayList<OperatorID>());
 		jobVertex.setInvokableClass(AbstractInvokable.class);
 
 		ExecutionGraph executionGraph = new ExecutionGraph(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
index a932c189c66f9..11a03cceb4b32 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/RecoverableCompletedCheckpointStore.java
@@ -76,7 +76,7 @@ public void shutdown(JobStatus jobStatus) throws Exception {
 			suspended.clear();
 
 			for (CompletedCheckpoint checkpoint : checkpoints) {
-				sharedStateRegistry.unregisterAll(checkpoint.getTaskStates().values());
+				sharedStateRegistry.unregisterAll(checkpoint.getOperatorStates().values());
 				suspended.add(checkpoint);
 			}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java
index 2fbfe1ca3b906..f468c938e8253 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/api/graph/StreamGraphHasherV1.java
@@ -153,19 +153,6 @@ private boolean generateNodeHash(
 
 			return true;
 		} else {
-			// Check that this node is not part of a chain. This is currently
-			// not supported, because the runtime takes the snapshots by the
-			// operator ID of the first vertex in a chain. It's OK if the node
-			// has chained outputs.
-			for (StreamEdge inEdge : node.getInEdges()) {
-				if (isChainable(inEdge, isChainingEnabled)) {
-					throw new UnsupportedOperationException("Cannot assign user-specified hash "
-							+ "to intermediate node in chain. This will be supported in future "
-							+ "versions of Flink. As a work around start new chain at task "
-							+ node.getOperatorName() + ".");
-				}
-			}
-
 			Hasher hasher = hashFunction.newHasher();
 			byte[] hash = generateUserSpecifiedHash(node, hasher);
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
index 3772e58df83b8..7c2416e0961fb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphHasherV2.java
@@ -169,19 +169,6 @@ private boolean generateNodeHash(
 
 			return true;
 		} else {
-			// Check that this node is not part of a chain. This is currently
-			// not supported, because the runtime takes the snapshots by the
-			// operator ID of the first vertex in a chain. It's OK if the node
-			// has chained outputs.
-			for (StreamEdge inEdge : node.getInEdges()) {
-				if (isChainable(inEdge, isChainingEnabled)) {
-					throw new UnsupportedOperationException("Cannot assign user-specified hash "
-							+ "to intermediate node in chain. This will be supported in future "
-							+ "versions of Flink. As a work around start new chain at task "
-							+ node.getOperatorName() + ".");
-				}
-			}
-
 			Hasher hasher = hashFunction.newHasher();
 			byte[] hash = generateUserSpecifiedHash(node, hasher);
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
index c1750a1cabfa3..8a8c8b0f5525b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphUserHashHasher.java
@@ -38,15 +38,6 @@ public Map<Integer, byte[]> traverseStreamGraphAndGenerateHashes(StreamGraph str
 			String userHash = streamNode.getUserHash();
 
 			if (null != userHash) {
-				for (StreamEdge inEdge : streamNode.getInEdges()) {
-					if (StreamingJobGraphGenerator.isChainable(inEdge, streamGraph)) {
-						throw new UnsupportedOperationException("Cannot assign user-specified hash "
-								+ "to intermediate node in chain. This will be supported in future "
-								+ "versions of Flink. As a work around start new chain at task "
-								+ streamNode.getOperatorName() + ".");
-					}
-				}
-
 				hashResult.put(streamNode.getId(), StringUtils.hexStringToByte(userHash));
 			}
 		}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 7f24cd358e353..aa0f08d20b9b1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -45,6 +45,7 @@
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.ScheduleMode;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
@@ -140,7 +141,9 @@ private JobGraph createJobGraph() {
 			legacyHashes.add(hasher.traverseStreamGraphAndGenerateHashes(streamGraph));
 		}
 
-		setChaining(hashes, legacyHashes);
+		Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes = new HashMap<>();
+
+		setChaining(hashes, legacyHashes, chainedOperatorHashes);
 
 		setPhysicalEdges();
 
@@ -190,9 +193,9 @@ private void setPhysicalEdges() {
 	 *
 	 * <p>This will recursively create all {@link JobVertex} instances.
 	 */
-	private void setChaining(Map<Integer, byte[]> hashes, List<Map<Integer, byte[]>> legacyHashes) {
+	private void setChaining(Map<Integer, byte[]> hashes, List<Map<Integer, byte[]>> legacyHashes, Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes) {
 		for (Integer sourceNodeId : streamGraph.getSourceIDs()) {
-			createChain(sourceNodeId, sourceNodeId, hashes, legacyHashes, 0);
+			createChain(sourceNodeId, sourceNodeId, hashes, legacyHashes, 0, chainedOperatorHashes);
 		}
 	}
 
@@ -201,7 +204,8 @@ private List<StreamEdge> createChain(
 			Integer currentNodeId,
 			Map<Integer, byte[]> hashes,
 			List<Map<Integer, byte[]>> legacyHashes,
-			int chainIndex) {
+			int chainIndex,
+			Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes) {
 
 		if (!builtVertices.contains(startNodeId)) {
 
@@ -220,20 +224,27 @@ private List<StreamEdge> createChain(
 
 			for (StreamEdge chainable : chainableOutputs) {
 				transitiveOutEdges.addAll(
-						createChain(startNodeId, chainable.getTargetId(), hashes, legacyHashes, chainIndex + 1));
+						createChain(startNodeId, chainable.getTargetId(), hashes, legacyHashes, chainIndex + 1, chainedOperatorHashes));
 			}
 
 			for (StreamEdge nonChainable : nonChainableOutputs) {
 				transitiveOutEdges.add(nonChainable);
-				createChain(nonChainable.getTargetId(), nonChainable.getTargetId(), hashes, legacyHashes, 0);
+				createChain(nonChainable.getTargetId(), nonChainable.getTargetId(), hashes, legacyHashes, 0, chainedOperatorHashes);
+			}
+
+			List<Tuple2<byte[], byte[]>> operatorHashes = chainedOperatorHashes.get(startNodeId);
+			if (operatorHashes == null) {
+				operatorHashes = new ArrayList<>();
+				chainedOperatorHashes.put(startNodeId, operatorHashes);
 			}
+			operatorHashes.add(new Tuple2<>(hashes.get(currentNodeId), legacyHashes.get(1).get(currentNodeId)));
 
 			chainedNames.put(currentNodeId, createChainedName(currentNodeId, chainableOutputs));
 			chainedMinResources.put(currentNodeId, createChainedMinResources(currentNodeId, chainableOutputs));
 			chainedPreferredResources.put(currentNodeId, createChainedPreferredResources(currentNodeId, chainableOutputs));
 
 			StreamConfig config = currentNodeId.equals(startNodeId)
-					? createJobVertex(startNodeId, hashes, legacyHashes)
+					? createJobVertex(startNodeId, hashes, legacyHashes, chainedOperatorHashes)
 					: new StreamConfig(new Configuration());
 
 			setVertexConfig(currentNodeId, config, chainableOutputs, nonChainableOutputs);
@@ -308,7 +319,8 @@ private ResourceSpec createChainedPreferredResources(Integer vertexID, List<Stre
 	private StreamConfig createJobVertex(
 			Integer streamNodeId,
 			Map<Integer, byte[]> hashes,
-			List<Map<Integer, byte[]>> legacyHashes) {
+			List<Map<Integer, byte[]>> legacyHashes,
+			Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes) {
 
 		JobVertex jobVertex;
 		StreamNode streamNode = streamGraph.getStreamNode(streamNodeId);
@@ -330,18 +342,32 @@ private StreamConfig createJobVertex(
 			}
 		}
 
+		List<Tuple2<byte[], byte[]>> chainedOperators = chainedOperatorHashes.get(streamNodeId);
+		List<OperatorID> chainedOperatorVertexIds = new ArrayList<>();
+		List<OperatorID> userDefinedChainedOperatorVertexIds = new ArrayList<>();
+		if (chainedOperators != null) {
+			for (Tuple2<byte[], byte[]> chainedOperator : chainedOperators) {
+				chainedOperatorVertexIds.add(new OperatorID(chainedOperator.f0));
+				userDefinedChainedOperatorVertexIds.add(chainedOperator.f1 != null ? new OperatorID(chainedOperator.f1) : null);
+			}
+		}
+
 		if (streamNode.getInputFormat() != null) {
 			jobVertex = new InputFormatVertex(
 					chainedNames.get(streamNodeId),
 					jobVertexId,
-					legacyJobVertexIds);
+					legacyJobVertexIds,
+					chainedOperatorVertexIds,
+					userDefinedChainedOperatorVertexIds);
 			TaskConfig taskConfig = new TaskConfig(jobVertex.getConfiguration());
 			taskConfig.setStubWrapper(new UserCodeObjectWrapper<Object>(streamNode.getInputFormat()));
 		} else {
 			jobVertex = new JobVertex(
 					chainedNames.get(streamNodeId),
 					jobVertexId,
-					legacyJobVertexIds);
+					legacyJobVertexIds,
+					chainedOperatorVertexIds,
+					userDefinedChainedOperatorVertexIds);
 		}
 
 		jobVertex.setResources(chainedMinResources.get(streamNodeId), chainedPreferredResources.get(streamNodeId));
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java
index 05aa694b538d9..a8976744d9dd7 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java
@@ -47,7 +47,6 @@
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  * Tests the {@link StreamNode} hash assignment during translation from {@link StreamGraph} to
@@ -392,10 +391,10 @@ public void testManualHashAssignmentCollisionThrowsException() throws Exception
 	}
 
 	/**
-	 * Tests that a manual hash for an intermediate chain node throws an Exception.
+	 * Tests that a manual hash for an intermediate chain node is accepted.
 	 */
-	@Test(expected = UnsupportedOperationException.class)
-	public void testManualHashAssignmentForIntermediateNodeInChainThrowsException() throws Exception {
+	@Test
+	public void testManualHashAssignmentForIntermediateNodeInChain() throws Exception {
 		StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
 		env.setParallelism(4);
 
@@ -409,9 +408,6 @@ public void testManualHashAssignmentForIntermediateNodeInChainThrowsException()
 
 	/**
 	 * Tests that a manual hash at the beginning of a chain is accepted.
-	 *
-	 * <p>This should work, because the ID is used at the beginning of a chain. This is currently
-	 * not allowed for intermediate nodes (see {@link #testManualHashAssignmentForIntermediateNodeInChainThrowsException()}).
 	 */
 	@Test
 	public void testManualHashAssignmentForStartNodeInInChain() throws Exception {
@@ -446,7 +442,7 @@ public void testUserProvidedHashing() {
 	}
 
 	@Test
-	public void testUserProvidedHashingOnChainNotSupported() {
+	public void testUserProvidedHashingOnChainSupported() {
 		StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
 
 		env.addSource(new NoOpSourceFunction(), "src").setUidHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
@@ -455,11 +451,7 @@ public void testUserProvidedHashingOnChainNotSupported() {
 				.keyBy(new NoOpKeySelector())
 				.reduce(new NoOpReduceFunction()).name("reduce").setUidHash("dddddddddddddddddddddddddddddddd");
 
-		try {
-			env.getStreamGraph().getJobGraph();
-			fail();
-		} catch (UnsupportedOperationException ignored) {
-		}
+		env.getStreamGraph().getJobGraph();
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
index e0de7d2f39bfd..72a1b63baec2c 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
@@ -32,28 +32,32 @@
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.testutils.OneShotLatch;
 import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.checkpoint.SubtaskState;
-import org.apache.flink.runtime.checkpoint.TaskState;
+import org.apache.flink.runtime.checkpoint.OperatorState;
+import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
 import org.apache.flink.runtime.checkpoint.savepoint.SavepointV2;
 import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.TaskInformation;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.messages.JobManagerMessages.CancelJob;
 import org.apache.flink.runtime.messages.JobManagerMessages.DisposeSavepoint;
 import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepoint;
 import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepointSuccess;
-import org.apache.flink.runtime.state.ChainedStateHandle;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
 import org.apache.flink.runtime.state.filesystem.FsStateBackend;
@@ -93,7 +97,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -102,7 +108,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -310,30 +315,43 @@ protected void run() {
 				};
 			}};
 
+			ExecutionGraph graph = (ExecutionGraph) ((JobManagerMessages.JobFound) Await.result(jobManager.ask(new JobManagerMessages.RequestJob(jobId), deadline.timeLeft()), deadline.timeLeft())).executionGraph();
+
 			// - Verification START -------------------------------------------
 
 			String errMsg = "Error during gathering of TaskDeploymentDescriptors";
-			assertNull(errMsg, error[0]);
+			if (error[0] != null) {
+				throw new RuntimeException(error[0]);
+			}
+
+			Map<OperatorID, Tuple2<Integer, ExecutionJobVertex>> operatorToJobVertexMapping = new HashMap<>();
+			for (ExecutionJobVertex task : graph.getVerticesTopologically()) {
+				List<OperatorID> operatorIDs = task.getOperatorIDs();
+				for (int x = 0; x < operatorIDs.size(); x++) {
+					operatorToJobVertexMapping.put(operatorIDs.get(x), new Tuple2<>(x, task));
+				}
+			}
 
 			// Verify that all tasks, which are part of the savepoint
 			// have a matching task deployment descriptor.
-			for (TaskState taskState : savepoint.getTaskStates()) {
-				Collection<TaskDeploymentDescriptor> taskTdds = tdds.get(taskState.getJobVertexID());
+			for (OperatorState operatorState : savepoint.getOperatorStates()) {
+				Tuple2<Integer, ExecutionJobVertex> chainIndexAndJobVertex = operatorToJobVertexMapping.get(operatorState.getOperatorID());
+				Collection<TaskDeploymentDescriptor> taskTdds = tdds.get(chainIndexAndJobVertex.f1.getJobVertexId());
 
 				errMsg = "Missing task for savepoint state for operator "
-					+ taskState.getJobVertexID() + ".";
+					+ operatorState.getOperatorID() + ".";
 				assertTrue(errMsg, taskTdds.size() > 0);
 
-				assertEquals(taskState.getNumberCollectedStates(), taskTdds.size());
+				assertEquals(operatorState.getNumberCollectedStates(), taskTdds.size());
 
 				for (TaskDeploymentDescriptor tdd : taskTdds) {
-					SubtaskState subtaskState = taskState.getState(tdd.getSubtaskIndex());
+					OperatorSubtaskState subtaskState = operatorState.getState(tdd.getSubtaskIndex());
 
 					assertNotNull(subtaskState);
 
 					errMsg = "Initial operator state mismatch.";
 					assertEquals(errMsg, subtaskState.getLegacyOperatorState(),
-						tdd.getTaskStateHandles().getLegacyOperatorState());
+						tdd.getTaskStateHandles().getLegacyOperatorState().get(chainIndexAndJobVertex.f0));
 				}
 			}
 
@@ -360,15 +378,13 @@ protected void run() {
 			// The checkpoint files
 			List<File> checkpointFiles = new ArrayList<>();
 
-			for (TaskState stateForTaskGroup : savepoint.getTaskStates()) {
-				for (SubtaskState subtaskState : stateForTaskGroup.getStates()) {
-					ChainedStateHandle<StreamStateHandle> streamTaskState = subtaskState.getLegacyOperatorState();
+			for (OperatorState stateForTaskGroup : savepoint.getOperatorStates()) {
+				for (OperatorSubtaskState subtaskState : stateForTaskGroup.getStates()) {
+					StreamStateHandle streamTaskState = subtaskState.getLegacyOperatorState();
 
-					for (int i = 0; i < streamTaskState.getLength(); i++) {
-						if (streamTaskState.get(i) != null) {
-							FileStateHandle fileStateHandle = (FileStateHandle) streamTaskState.get(i);
-							checkpointFiles.add(new File(fileStateHandle.getFilePath().toUri()));
-						}
+					if (streamTaskState != null) {
+						FileStateHandle fileStateHandle = (FileStateHandle) streamTaskState;
+						checkpointFiles.add(new File(fileStateHandle.getFilePath().toUri()));
 					}
 				}
 			}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java
new file mode 100644
index 0000000000000..2eecf499417bd
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.PoisonPill;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.akka.ListeningBehaviour;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.instance.AkkaActorGateway;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.runtime.jobmanager.JobManager;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
+import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.runtime.testingUtils.TestingJobManager;
+import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
+import org.apache.flink.runtime.testingUtils.TestingMemoryArchivist;
+import org.apache.flink.runtime.testingUtils.TestingTaskManager;
+import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import scala.Option;
+import scala.Tuple2;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.io.File;
+import java.net.URL;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Abstract class to verify that it is possible to migrate a 1.2 savepoint to 1.3 and that the topology can be modified
+ * from that point on.
+ * 
+ * The verification is done in 2 Steps:
+ * Step 1: Migrate the job to 1.3 by submitting the same job used for the 1.2 savepoint, and create a new savepoint.
+ * Step 2: Modify the job topology, and restore from the savepoint created in step 1.
+ */
+public abstract class AbstractOperatorRestoreTestBase {
+
+	@Rule
+	public final TemporaryFolder tmpFolder = new TemporaryFolder();
+
+	private static ActorSystem actorSystem = null;
+	private static ActorGateway jobManager = null;
+	private static ActorGateway archiver = null;
+	private static ActorGateway taskManager = null;
+
+	private static final FiniteDuration timeout = new FiniteDuration(30, TimeUnit.SECONDS);
+
+	@BeforeClass
+	public static void setupCluster() throws Exception {
+		FiniteDuration timeout = new FiniteDuration(30, TimeUnit.SECONDS);
+
+		actorSystem = AkkaUtils.createLocalActorSystem(new Configuration());
+
+		Tuple2<ActorRef, ActorRef> master = JobManager.startJobManagerActors(
+			new Configuration(),
+			actorSystem,
+			TestingUtils.defaultExecutor(),
+			TestingUtils.defaultExecutor(),
+			Option.apply("jm"),
+			Option.apply("arch"),
+			TestingJobManager.class,
+			TestingMemoryArchivist.class);
+
+		jobManager = new AkkaActorGateway(master._1(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+		archiver = new AkkaActorGateway(master._2(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+		Configuration tmConfig = new Configuration();
+		tmConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
+
+		ActorRef taskManagerRef = TaskManager.startTaskManagerComponentsAndActor(
+			tmConfig,
+			ResourceID.generate(),
+			actorSystem,
+			"localhost",
+			Option.apply("tm"),
+			Option.<LeaderRetrievalService>apply(new StandaloneLeaderRetrievalService(jobManager.path(), HighAvailabilityServices.DEFAULT_LEADER_ID)),
+			true,
+			TestingTaskManager.class);
+
+		taskManager = new AkkaActorGateway(taskManagerRef, HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+		// Wait until connected
+		Object msg = new TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager(jobManager.actor());
+		Await.ready(taskManager.ask(msg, timeout), timeout);
+	}
+
+	@AfterClass
+	public static void tearDownCluster() {
+		if (actorSystem != null) {
+			actorSystem.shutdown();
+		}
+
+		if (archiver != null) {
+			archiver.actor().tell(PoisonPill.getInstance(), ActorRef.noSender());
+		}
+
+		if (jobManager != null) {
+			jobManager.actor().tell(PoisonPill.getInstance(), ActorRef.noSender());
+		}
+
+		if (taskManager != null) {
+			taskManager.actor().tell(PoisonPill.getInstance(), ActorRef.noSender());
+		}
+	}
+
+	@Test
+	public void testMigrationAndRestore() throws Throwable {
+		// submit 1.2 job and create a migrated 1.3 savepoint
+		String savepointPath = migrateJob();
+		// restore from migrated 1.3 savepoint
+		restoreJob(savepointPath);
+	}
+
+	private String migrateJob() throws Throwable {
+		URL savepointResource = AbstractOperatorRestoreTestBase.class.getClassLoader().getResource("operatorstate/" + getMigrationSavepointName());
+		if (savepointResource == null) {
+			throw new IllegalArgumentException("Savepoint file does not exist.");
+		}
+		JobGraph jobToMigrate = createJobGraph(ExecutionMode.MIGRATE);
+		jobToMigrate.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointResource.getFile()));
+
+		Object msg;
+		Object result;
+
+		// Submit job graph
+		msg = new JobManagerMessages.SubmitJob(jobToMigrate, ListeningBehaviour.DETACHED);
+		result = Await.result(jobManager.ask(msg, timeout), timeout);
+
+		if (result instanceof JobManagerMessages.JobResultFailure) {
+			JobManagerMessages.JobResultFailure failure = (JobManagerMessages.JobResultFailure) result;
+			throw new Exception(failure.cause());
+		}
+		Assert.assertSame(JobManagerMessages.JobSubmitSuccess.class, result.getClass());
+
+		// Wait for all tasks to be running
+		msg = new TestingJobManagerMessages.WaitForAllVerticesToBeRunning(jobToMigrate.getJobID());
+		Await.result(jobManager.ask(msg, timeout), timeout);
+
+		// Trigger savepoint
+		File targetDirectory = tmpFolder.newFolder();
+		msg = new JobManagerMessages.CancelJobWithSavepoint(jobToMigrate.getJobID(), targetDirectory.getAbsolutePath());
+		Future<Object> future = jobManager.ask(msg, timeout);
+		result = Await.result(future, timeout);
+
+		if (result instanceof JobManagerMessages.CancellationFailure) {
+			JobManagerMessages.CancellationFailure failure = (JobManagerMessages.CancellationFailure) result;
+			throw new Exception(failure.cause());
+		}
+
+		String savepointPath = ((JobManagerMessages.CancellationSuccess) result).savepointPath();
+
+		// Wait until canceled
+		msg = new TestingJobManagerMessages.NotifyWhenJobStatus(jobToMigrate.getJobID(), JobStatus.CANCELED);
+		Await.ready(jobManager.ask(msg, timeout), timeout);
+
+		return savepointPath;
+	}
+
+	private void restoreJob(String savepointPath) throws Exception {
+		JobGraph jobToRestore = createJobGraph(ExecutionMode.RESTORE);
+		jobToRestore.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, true));
+
+		Object msg;
+		Object result;
+
+		// Submit job graph
+		msg = new JobManagerMessages.SubmitJob(jobToRestore, ListeningBehaviour.DETACHED);
+		result = Await.result(jobManager.ask(msg, timeout), timeout);
+
+		if (result instanceof JobManagerMessages.JobResultFailure) {
+			JobManagerMessages.JobResultFailure failure = (JobManagerMessages.JobResultFailure) result;
+			throw new Exception(failure.cause());
+		}
+		Assert.assertSame(JobManagerMessages.JobSubmitSuccess.class, result.getClass());
+
+		msg = new JobManagerMessages.RequestJobStatus(jobToRestore.getJobID());
+		JobStatus status = ((JobManagerMessages.CurrentJobStatus) Await.result(jobManager.ask(msg, timeout), timeout)).status();
+		while (!status.isTerminalState()) {
+			status = ((JobManagerMessages.CurrentJobStatus) Await.result(jobManager.ask(msg, timeout), timeout)).status();
+		}
+
+		Assert.assertEquals(JobStatus.FINISHED, status);
+	}
+
+	private JobGraph createJobGraph(ExecutionMode mode) {
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
+		env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
+		env.setRestartStrategy(RestartStrategies.noRestart());
+		env.setStateBackend(new MemoryStateBackend());
+
+		switch (mode) {
+			case MIGRATE:
+				createMigrationJob(env);
+				break;
+			case RESTORE:
+				createRestoredJob(env);
+				break;
+		}
+
+		return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
+	}
+
+	/**
+	 * Recreates the job used to create the 1.2 savepoint.
+	 *
+	 * @param env StreamExecutionEnvironment to use
+	 */
+	protected abstract void createMigrationJob(StreamExecutionEnvironment env);
+
+	/**
+	 * Creates a modified version of the job used to create the 1.2 savepoint.
+	 *
+	 * @param env StreamExecutionEnvironment to use
+	 */
+	protected abstract void createRestoredJob(StreamExecutionEnvironment env);
+
+	/**
+	 * Returns the name of the savepoint directory to use, relative to "resources/operatorstate".
+	 *
+	 * @return savepoint directory to use
+	 */
+	protected abstract String getMigrationSavepointName();
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/ExecutionMode.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/ExecutionMode.java
new file mode 100644
index 0000000000000..f333acadb6858
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/ExecutionMode.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore;
+
+/**
+ * Enum to control function behavior for the different test stages.
+ * 
+ * {@link ExecutionMode#GENERATE} should be used when creating the 1.2 savepoint.
+ * {@link ExecutionMode#MIGRATE} should be used when migrating the 1.2 savepoint to 1.3.
+ * {@link ExecutionMode#RESTORE} should be used when restoring from the migrated 1.3 savepoint.
+ */
+public enum ExecutionMode {
+	GENERATE,
+	MIGRATE,
+	RESTORE
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
new file mode 100644
index 0000000000000..28cd15a157234
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.keyed;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.state.operator.restore.AbstractOperatorRestoreTestBase;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+
+public class KeyedComplexChainTest extends AbstractOperatorRestoreTestBase {
+
+	@Override
+	public void createMigrationJob(StreamExecutionEnvironment env) {
+		/**
+		 * Source -> keyBy -> C(Window -> StatefulMap1 -> StatefulMap2)
+		 */
+		SingleOutputStreamOperator<Tuple2<Integer, Integer>> source = KeyedJob.createIntegerTupleSource(env, ExecutionMode.MIGRATE);
+
+		SingleOutputStreamOperator<Integer> window = KeyedJob.createWindowFunction(ExecutionMode.MIGRATE, source);
+
+		SingleOutputStreamOperator<Integer> first = KeyedJob.createFirstStatefulMap(ExecutionMode.MIGRATE, window);
+
+		SingleOutputStreamOperator<Integer> second = KeyedJob.createSecondStatefulMap(ExecutionMode.MIGRATE, first);
+	}
+
+	@Override
+	protected void createRestoredJob(StreamExecutionEnvironment env) {
+		/**
+		 * Source -> keyBy -> C(Window -> StatefulMap2) -> StatefulMap1
+		 */
+		SingleOutputStreamOperator<Tuple2<Integer, Integer>> source = KeyedJob.createIntegerTupleSource(env, ExecutionMode.RESTORE);
+
+		SingleOutputStreamOperator<Integer> window = KeyedJob.createWindowFunction(ExecutionMode.RESTORE, source);
+
+		SingleOutputStreamOperator<Integer> second = KeyedJob.createSecondStatefulMap(ExecutionMode.RESTORE, window);
+
+		SingleOutputStreamOperator<Integer> first = KeyedJob.createFirstStatefulMap(ExecutionMode.RESTORE, second);
+		first.startNewChain();
+	}
+
+	@Override
+	protected final String getMigrationSavepointName() {
+		return "complexKeyed";
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
new file mode 100644
index 0000000000000..6add7b2a76f8f
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.keyed;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
+import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+import org.apache.flink.util.Collector;
+import org.junit.Assert;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Savepoint generator to create the job used by the {@link KeyedComplexChainTest}.
+ *
+ * The job should be cancelled manually through the REST API using the cancel-with-savepoint operation.
+ */
+public class KeyedJob {
+
+	public static void main(String[] args) throws Exception {
+		ParameterTool pt = ParameterTool.fromArgs(args);
+
+		String savepointsPath = pt.getRequired("savepoint-path");
+
+		Configuration config = new Configuration();
+		config.setString(ConfigConstants.SAVEPOINT_DIRECTORY_KEY, savepointsPath);
+
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config);
+		env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
+		env.setRestartStrategy(RestartStrategies.noRestart());
+
+		env.setStateBackend(new MemoryStateBackend());
+
+		/**
+		 * Source -> keyBy -> C(Window -> StatefulMap1 -> StatefulMap2)
+		 */
+
+		SingleOutputStreamOperator<Tuple2<Integer, Integer>> source = createIntegerTupleSource(env, ExecutionMode.GENERATE);
+
+		SingleOutputStreamOperator<Integer> window = createWindowFunction(ExecutionMode.GENERATE, source);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, window);
+
+		SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first);
+
+		env.execute("job");
+	}
+
+	public static SingleOutputStreamOperator<Tuple2<Integer, Integer>> createIntegerTupleSource(StreamExecutionEnvironment env, ExecutionMode mode) {
+		return env.addSource(new IntegerTupleSource(mode));
+	}
+
+	public static SingleOutputStreamOperator<Integer> createWindowFunction(ExecutionMode mode, DataStream<Tuple2<Integer, Integer>> input) {
+		return input
+			.keyBy(0)
+			.countWindow(1)
+			.apply(new StatefulWindowFunction(mode))
+			.setParallelism(4)
+			.uid("window");
+	}
+
+	public static SingleOutputStreamOperator<Integer> createFirstStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
+		SingleOutputStreamOperator<Integer> map = input
+			.map(new StatefulStringStoringMap(mode, "first"))
+			.setParallelism(4);
+
+		// TODO: re-enable this when generating the actual 1.2 savepoint
+		//if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
+		map.uid("first");
+		//}
+
+		return map;
+	}
+
+	public static SingleOutputStreamOperator<Integer> createSecondStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
+		SingleOutputStreamOperator<Integer> map = input
+			.map(new StatefulStringStoringMap(mode, "second"))
+			.setParallelism(4);
+
+		// TODO: re-enable this when generating the actual 1.2 savepoint
+		//if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
+		map.uid("second");
+		//}
+
+		return map;
+	}
+
+	private static final class IntegerTupleSource extends RichSourceFunction<Tuple2<Integer, Integer>> {
+
+		private static final long serialVersionUID = 1912878510707871659L;
+		private final ExecutionMode mode;
+
+		private boolean running = true;
+
+		private IntegerTupleSource(ExecutionMode mode) {
+			this.mode = mode;
+		}
+
+		@Override
+		public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
+			for (int x = 0; x < 10; x++) {
+				ctx.collect(new Tuple2<>(x, x));
+			}
+
+			switch (mode) {
+				case GENERATE:
+				case MIGRATE:
+					synchronized (this) {
+						while (running) {
+							this.wait();
+						}
+					}
+			}
+		}
+
+		@Override
+		public void cancel() {
+			synchronized (this) {
+				running = false;
+				this.notifyAll();
+			}
+		}
+	}
+
+	private static final class StatefulWindowFunction extends RichWindowFunction<Tuple2<Integer, Integer>, Integer, Tuple, GlobalWindow> {
+
+		private static final long serialVersionUID = -7236313076792964055L;
+
+		private final ExecutionMode mode;
+		private transient ListState<Integer> state;
+
+		private boolean applyCalled = false;
+
+		private StatefulWindowFunction(ExecutionMode mode) {
+			this.mode = mode;
+		}
+
+		@Override
+		public void open(Configuration config) {
+			this.state = getRuntimeContext().getListState(new ListStateDescriptor<>("values", Integer.class));
+		}
+
+		@Override
+		public void apply(Tuple key, GlobalWindow window, Iterable<Tuple2<Integer, Integer>> values, Collector<Integer> out) throws Exception {
+			// fail-safe to make sure apply is actually called
+			applyCalled = true;
+			switch (mode) {
+				case GENERATE:
+					for (Tuple2<Integer, Integer> value : values) {
+						state.add(value.f1);
+					}
+					break;
+				case MIGRATE:
+				case RESTORE:
+					Iterator<Tuple2<Integer, Integer>> input = values.iterator();
+					Iterator<Integer> restored = state.get().iterator();
+					while (input.hasNext() && restored.hasNext()) {
+						Tuple2<Integer, Integer> value = input.next();
+						Integer rValue = restored.next();
+						Assert.assertEquals(rValue, value.f1);
+					}
+					Assert.assertEquals(restored.hasNext(), input.hasNext());
+			}
+		}
+
+		@Override
+		public void close() {
+			Assert.assertTrue("Apply was never called.", applyCalled);
+		}
+	}
+
+	private static class StatefulStringStoringMap extends RichMapFunction<Integer, Integer> implements ListCheckpointed<String> {
+
+		private static final long serialVersionUID = 6092985758425330235L;
+		private final ExecutionMode mode;
+		private final String valueToStore;
+
+		private StatefulStringStoringMap(ExecutionMode mode, String valueToStore) {
+			this.mode = mode;
+			this.valueToStore = valueToStore;
+		}
+
+		@Override
+		public Integer map(Integer value) throws Exception {
+			return value;
+		}
+
+		@Override
+		public List<String> snapshotState(long checkpointId, long timestamp) throws Exception {
+			return Arrays.asList(valueToStore + getRuntimeContext().getIndexOfThisSubtask());
+		}
+
+		@Override
+		public void restoreState(List<String> state) throws Exception {
+			switch (mode) {
+				case GENERATE:
+					break;
+				case MIGRATE:
+				case RESTORE:
+					Assert.assertEquals("Failed for " + valueToStore + getRuntimeContext().getIndexOfThisSubtask(), 1, state.size());
+					String value = state.get(0);
+					Assert.assertEquals(valueToStore + getRuntimeContext().getIndexOfThisSubtask(), value);
+			}
+		}
+	}
+
+
+	private KeyedJob() {
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
new file mode 100644
index 0000000000000..5b517651aa12c
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.unkeyed;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.state.operator.restore.AbstractOperatorRestoreTestBase;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSecondStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSource;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createStatelessMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createThirdStatefulMap;
+
+/**
+ * All classes extending this class will use the same savepoint and migration job.
+ */
+public abstract class AbstractNonKeyedOperatorRestoreTestBase extends AbstractOperatorRestoreTestBase {
+
+	@Override
+	public void createMigrationJob(StreamExecutionEnvironment env) {
+		/**
+		 * Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
+		 */
+		DataStream<Integer> source = createSource(env, ExecutionMode.MIGRATE);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.MIGRATE, source);
+		first.startNewChain();
+
+		SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.MIGRATE, first);
+		second.startNewChain();
+
+		SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);
+
+		SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.MIGRATE, stateless);
+	}
+
+	@Override
+	protected final String getMigrationSavepointName() {
+		return "nonKeyed";
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
new file mode 100644
index 0000000000000..68380702cbbac
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.unkeyed;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSecondStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSource;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createStatelessMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createThirdStatefulMap;
+
+/**
+ * Verifies that the state of all operators is restored if a topology change breaks up a chain.
+ */
+public class ChainBreakTest extends AbstractNonKeyedOperatorRestoreTestBase {
+
+	@Override
+	public void createRestoredJob(StreamExecutionEnvironment env) {
+		/**
+		 * Original job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
+		 * Modified job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map) -> StatefulMap3
+		 */
+		DataStream<Integer> source = createSource(env, ExecutionMode.RESTORE);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.RESTORE, source);
+		first.startNewChain();
+
+		SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.RESTORE, first);
+		second.startNewChain();
+
+		SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);
+
+		SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.RESTORE, stateless);
+		third.startNewChain();
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
new file mode 100644
index 0000000000000..e405e76e54652
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.unkeyed;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSource;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createStatelessMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createThirdStatefulMap;
+
+/**
+ * Verifies that the state of all operators is restored if a topology change removes an operator from a chain.
+ */
+public class ChainLengthDecreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
+
+	@Override
+	public void createRestoredJob(StreamExecutionEnvironment env) {
+		/**
+		 * Original job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
+		 * Modified job: Source -> StatefulMap1 -> CHAIN(Map -> StatefulMap3)
+		 */
+		DataStream<Integer> source = createSource(env, ExecutionMode.RESTORE);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.RESTORE, source);
+		first.startNewChain();
+
+		SingleOutputStreamOperator<Integer> stateless = createStatelessMap(first);
+		stateless.startNewChain();
+
+		SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.RESTORE, stateless);
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
new file mode 100644
index 0000000000000..b78aa1074bf7f
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.unkeyed;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSecondStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSource;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createStatelessMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createThirdStatefulMap;
+
+/**
+ * Verifies that the state of all operator is restored if a topology change adds an operator to a chain.
+ */
+public class ChainLengthIncreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
+
+	@Override
+	public void createRestoredJob(StreamExecutionEnvironment env) {
+		/**
+		 * Original job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
+		 * Modified job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3 -> StatefulMap4)
+		 */
+		DataStream<Integer> source = createSource(env, ExecutionMode.RESTORE);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.RESTORE, source);
+		first.startNewChain();
+
+		SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.RESTORE, first);
+		second.startNewChain();
+
+		SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);
+
+		SingleOutputStreamOperator<Integer> stateless2 = createStatelessMap(stateless);
+
+		SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.RESTORE, stateless2);
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
new file mode 100644
index 0000000000000..7c68b4e00ff1d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.unkeyed;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSecondStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSource;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createStatelessMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createThirdStatefulMap;
+
+/**
+ * Verifies that the state of all operators is restored if a topology change causes the ordering of a chain to change.
+ */
+public class ChainOrderTest extends AbstractNonKeyedOperatorRestoreTestBase {
+
+	@Override
+	public void createRestoredJob(StreamExecutionEnvironment env) {
+		/**
+		 * Original job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
+		 * Modified job: Source -> StatefulMap1 -> CHAIN(StatefulMap3 -> Map -> StatefulMap2)
+		 */
+		DataStream<Integer> source = createSource(env, ExecutionMode.RESTORE);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.RESTORE, source);
+		first.startNewChain();
+
+		SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.RESTORE, first);
+		third.startNewChain();
+
+		SingleOutputStreamOperator<Integer> stateless = createStatelessMap(third);
+
+		SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.RESTORE, stateless);
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
new file mode 100644
index 0000000000000..3f2fba45f33d7
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.unkeyed;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSecondStatefulMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createSource;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createStatelessMap;
+import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createThirdStatefulMap;
+
+/**
+ * Verifies that the state of all operators is restored if a topology change creates a new chain.
+ */
+public class ChainUnionTest extends AbstractNonKeyedOperatorRestoreTestBase {
+
+	@Override
+	public void createRestoredJob(StreamExecutionEnvironment env) {
+		/**
+		 * Original job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
+		 * Modified job: Source -> CHAIN(StatefulMap1 -> StatefulMap2 -> Map -> StatefulMap3)
+		 */
+		DataStream<Integer> source = createSource(env, ExecutionMode.RESTORE);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.RESTORE, source);
+		first.startNewChain();
+
+		SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.RESTORE, first);
+
+		SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);
+
+		SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.RESTORE, stateless);
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
new file mode 100644
index 0000000000000..32067b39a9608
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.state.operator.restore.unkeyed;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.test.state.operator.restore.ExecutionMode;
+import org.junit.Assert;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Savepoint generator to create the job used by the {@link AbstractNonKeyedOperatorRestoreTestBase}.
+ *
+ * The job should be cancelled manually through the REST API using the cancel-with-savepoint operation.
+ */
+public class NonKeyedJob {
+
+	public static void main(String[] args) throws Exception {
+		ParameterTool pt = ParameterTool.fromArgs(args);
+
+		String savepointsPath = pt.getRequired("savepoint-path");
+
+		Configuration config = new Configuration();
+		config.setString(ConfigConstants.SAVEPOINT_DIRECTORY_KEY, savepointsPath);
+
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config);
+		env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
+		env.setRestartStrategy(RestartStrategies.noRestart());
+
+		env.setStateBackend(new MemoryStateBackend());
+
+		/**
+		 * Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
+		 */
+		DataStream<Integer> source = createSource(env, ExecutionMode.GENERATE);
+
+		SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, source);
+		first.startNewChain();
+
+		SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first);
+		second.startNewChain();
+
+		SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);
+
+		SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.GENERATE, stateless);
+
+		env.execute("job");
+	}
+
+	public static SingleOutputStreamOperator<Integer> createSource(StreamExecutionEnvironment env, ExecutionMode mode) {
+		return env.addSource(new IntegerSource(mode))
+			.setParallelism(4);
+	}
+
+	public static SingleOutputStreamOperator<Integer> createFirstStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
+		return input
+			.map(new StatefulStringStoringMap(mode, "first"))
+			.setParallelism(4)
+			.uid("first");
+	}
+
+	public static SingleOutputStreamOperator<Integer> createSecondStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
+		return input
+			.map(new StatefulStringStoringMap(mode, "second"))
+			.setParallelism(4)
+			.uid("second");
+	}
+
+	public static SingleOutputStreamOperator<Integer> createThirdStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
+		SingleOutputStreamOperator<Integer> map = input
+			.map(new StatefulStringStoringMap(mode, "third"))
+			.setParallelism(4);
+
+		// we cannot set the uid on a chained operator in 1.2
+		if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
+			map.uid("third");
+		}
+
+		return map;
+	}
+
+	public static SingleOutputStreamOperator<Integer> createStatelessMap(DataStream<Integer> input) {
+		return input.map(new NoOpMapFunction())
+			.setParallelism(4);
+	}
+
+	private static class StatefulStringStoringMap extends RichMapFunction<Integer, Integer> implements ListCheckpointed<String> {
+
+		private static final long serialVersionUID = 6092985758425330235L;
+		private final ExecutionMode mode;
+		private final String valueToStore;
+
+		private StatefulStringStoringMap(ExecutionMode mode, String valueToStore) {
+			this.mode = mode;
+			this.valueToStore = valueToStore;
+		}
+
+		@Override
+		public Integer map(Integer value) throws Exception {
+			return value;
+		}
+
+		@Override
+		public List<String> snapshotState(long checkpointId, long timestamp) throws Exception {
+			return Arrays.asList(valueToStore + getRuntimeContext().getIndexOfThisSubtask());
+		}
+
+		@Override
+		public void restoreState(List<String> state) throws Exception {
+			switch (mode) {
+				case GENERATE:
+					break;
+				case MIGRATE:
+				case RESTORE:
+					Assert.assertEquals("Failed for " + valueToStore + getRuntimeContext().getIndexOfThisSubtask(), 1, state.size());
+					String value = state.get(0);
+					Assert.assertEquals(valueToStore + getRuntimeContext().getIndexOfThisSubtask(), value);
+			}
+		}
+	}
+
+	private static class NoOpMapFunction implements MapFunction<Integer, Integer> {
+
+		private static final long serialVersionUID = 6584823409744624276L;
+
+		@Override
+		public Integer map(Integer value) throws Exception {
+			return value;
+		}
+	}
+
+	private static final class IntegerSource extends RichParallelSourceFunction<Integer> {
+
+		private static final long serialVersionUID = 1912878510707871659L;
+		private final ExecutionMode mode;
+
+		private volatile boolean running = true;
+
+		private IntegerSource(ExecutionMode mode) {
+			this.mode = mode;
+		}
+
+		@Override
+		public void run(SourceContext<Integer> ctx) throws Exception {
+			ctx.collect(1);
+
+			switch (mode) {
+				case GENERATE:
+				case MIGRATE:
+					// keep the job running until cancel-with-savepoint was done
+					synchronized (this) {
+						while (running) {
+							this.wait();
+						}
+					}
+			}
+		}
+
+		@Override
+		public void cancel() {
+			synchronized (this) {
+				running = false;
+				this.notifyAll();
+			}
+		}
+	}
+
+	private NonKeyedJob() {
+	}
+}
diff --git a/flink-tests/src/test/resources/operatorstate/complexKeyed/_metadata b/flink-tests/src/test/resources/operatorstate/complexKeyed/_metadata
new file mode 100644
index 0000000000000000000000000000000000000000..9e03876cfce81f15b09d91532b38322e647c55a2
GIT binary patch
literal 137490
zcmeHQZ;V{kb-z37Ke2b$Yr8ez)@?V5F-_^4oj-5h%!_d`jsY8{HjSyF7zf^ab7yy8
zc4nD*V`7^U^PvKNgNP4Ep~97-7^ov56+WOQ68V%?Mv<DJno0-*;zKKuw5rrXi$r?Q
zyJ!C2v3K1G*>QeXn*Gf^^WNL{?z{J%Klj{o_CGjh2_a(YLnW(0jL#+hz25lr!T<S}
zzrT3#pTf7t6@8WZoKl}L{w{vWMb)W{(E(#(-2(+#^cv;xfrv<dRQb-B*q$sDf^;fy
zthDWVmYofJ%T1+|mYa2anQ@D$L?NdReMx=Bx-WF@=VEoeCah|0F77RQ!MuzY%caW0
z@mix2mKJ2Z9(tjS-{Bqe4#`@{E0<2l+HHsK{mFOj_1{*{_RX5G`_A<iOYxw(uu!eU
z!{dvx5thpJ_}%JT`x12Ls*KSk7U>J!NGR6UWl*gY648&*0n2R)yWl5nSxi_-StwX`
zrkJ&S$8)W;7vy}q;G{F@g1Yd?G)1!26tU$tMbXYCy^MNx-}4fdoi1c8&rSN4bOY)7
zUb2|;ilfvNsa8{rFSjXt+qHc^o3)Y&)irE4SFmz{U9b|4?aNfUkZ`3pN==d0O))m}
z)tCOp`q4+VZ;b!dt_%P0pNe|9xg<J~CLJm>(O)saoowQEt{-KrGoEIAf$=rQ^NhdH
zI7$CpmC4^><Z(QC8`tk=tT8^t__vHdV!XikzZ$1DGVWl!iSg@<^Nf!(KFjzr;~R|s
z%=nSU4KX!$+Bv6K^ljfN`Z?dS{gkx)T#&JnNtv@#Mc=WLju0EJQ>V~euIA|21)eo`
z8fU(T)Lhvy?=>gVyBmw;0TW<N-2DHz(jKoa_4Cklx@x~z?bS-*@%>^=Sytx4`6FUu
zu_Vid_T*a+MfONu{mWTB<<CY_zN(aF^|U?Pnzm=Vr|q42u|A{+`C8eltIKT4)6>qD
zy~^BdMA|tthM<P{XN?-)Xg6<K0)N@*KC$iW>(_kxT#LadE|)X5av!JiRfER&SHgCK
zwVRh1-kxP#rn~tLl`3;(8CEODUwZQ8AIBfPmG15l@wixZtR{9CFRObKHF-ShK}~+3
zUiDaB#EOYfZ0>jo2mOa+5Yk(?eo2?>e6mGB^FmClIp&oclE%L3?BXJKcBk)p&7Q4A
zgu~TR;efXw>q?3k*w2UDbi)!p)RQ*&F;ka$%v)+T?|8mc@8})Vr%%4~?)ht9iQdR2
zS(Xd3v-5<)xw5e~l%ceuyFE>)<6?DCRl+8Bw=?`6_VvAodRUX*f*L_;GAP9#EmaCt
z^=FR1)#JC9tG-vhC!%j3G<Jt0tg3tcfbY}oyHDfn6wN95YGrQFjjha)5fjt8p)Dnr
zhpJmQXN3`YP2(r7-eC>t!ytVSGo@LDl%-*c53!X!kJ4jLEm?+WDpE>@hz*t@=$9m2
zc=`&Z3H2JdkQ|}KXF^?TBE2mtP3%6lsi|~RS<m1rHP2kFFVsU_W2CN)sL@sFV)SKI
z&xv0!p196PU2mjL7^#y+>XebX!APBssL{>SP12Vy^xQmaOg+?1#>;JPUyd4!F1M!V
z#NRTW_}fP679;f<BlTJ%m2@pC(G#{h##T-2p|YykL;YRjvD=K)-!oEw-$?y~ermIa
z&hSqR<In3aYsv7Y%KEo7ZdlZ~nY+Q37j=E>UX9z{(>U{(#_ik~H@vFrH{PW&&N^)B
zDP4EAYjoexxa)q6yFbym_gRhm5*qJ(Q{%yTjfZ$l9X_M$_uZuN=-V2-P^09YHGfXm
z%lkA|KUBK3Ur0HAChJ&1CL36GF_pHwL@r~=oSO^WAeqS~T_NhH)G1?PiX{s&GuG<N
zr@_<=Ol??d&X9TH*ox+fD?_Co0c|E4ztL1nH%rqK#wIUxz}$5;o4YVJ^>SV`0jN>0
z<Y@J}VT$f-k7zms-;7LdpQkArV-sU@Nqz$un;4st*OS}sB#>#bg!>!0gbTh2z6rj$
zGTkdMHZe9aHZeBIz1rv`((NqPYleVK3z-%&ZRZ|^mE#eHv5B#Xv5B#XH56Vs?G;|e
zV5fzh7Is>Rtu|eoLVsho+Q1eBTM%qPum!;uge|J%cNO^=!Jn4QqP=cLU8)55Cio`!
zCio`!Cio`!W-D3~_$G@Dt#DJ#;1meEvb;E4u+zd$3p*|BwB5n`V5jYkj|ks`u5giU
z0pxNVcn`vr^@qbmxUwq<R|Z=UY(cOE!4?Et5NtuP1;G}i1D7vd675`ALfnIXr5`?f
zdY?w`g5<9<1TSP-$h44YA=5&pg-i>X7BVelTBMaQEX8#16kg$JB_Pv6riDxknHDlF
zWLn6ykZB>)LZ*dGtFl-9YO_=!CJADaASMZ7k{~7tVv?+w6r->O!4?Et5NtuP1;G{s
zTM%qPum!;u1Y6Lr!xp4A`bLGlU?ZShrZfCeFu|UV6Su1h)sHe#7I0N2PBXs1_!{GR
z#`hUN);P&YZYRIYb(IX1%H%;t$#|0S8OF1WKV<wF<9}<M;#{gzU*bCFT%9_=^#bDw
z#-|y-$M^%rpECZd%4M2Mrd->%vzAQDoMo%3l}q>uE1k=w-Bc=-bF%98r#@4sC`Ly4
zXJZ#wyFn6DBrz=ylyIZHv?JxfA!uHRi8aT(azoPCr);4YxwAXf-mS6C&<pj@3uXLp
zwNyCZEy%jE#K}XlW-!Lo#Gby3kQ;wc?TKBm)TSF&!d&&F4Svj2-yL>8<}J0FcRXLJ
zhwVq4K7I0?ch6t@N=%I9#U@#n3$oJjb10m=y6CLlN4t!`pejkD-FG*}_Ui6s>ZMVM
z?176bI@TCXsN-UFP;FEw?KE|_Gb3O*qWk=S@6%n{8;W{#zFL`UcN=vPDr~l0nIj`6
zra60eP_2ZjTQ_Hg5qVAHC$8RM4e7%meLSKp;54hml+62Cqb9OPXzo>^Uc8`73GZV$
z-BW1aKK<Cro=54ir;gRc4&%f2Zfr_L6Hhet#PT9mOoU={o(`HVdzHD_gZ@J@2vtt(
zSTRW#o>q+L3byU5e=^hjM-kF{q4Gry(R8ncY9f5eNM#M8=h*GWW3M+-cNnQR7^#0~
zq<+On{UamwMkDo)ja17>jT@=6Mry)HZS7l&er>jT>!o*5(@~8Wnpmek^krR7W83tj
zD$zmF;G$ZF{;1MTbu_Vunl(~cdFwfrRmC3aSB+QLX{6p{r0z0O|HMeWxu4pjG4lap
z{CWLtEkDfqw{(32#d%d_Gi$6{Uexuido^x*PvgvE8n<7k@rGA5-guYB_(vL3Pib_x
z=euv{`mXym?*2sM-e)!L<DPNno4S5*UgM!v8V{e*cpo>`(YJNo3pL7(8t2bxEbr4;
z{ZMJliEP$$Q&L(5nei+;Y3D3I$YiWkQfB;2E}bZ53qsUSsZ++p6w41`X+hTN%#Fdi
z46JJy(mpl$kU8JjispQ)mdx~8vtp#C4Pts%#_C2uXNksdG+EQl()5I}$tDUV>8uYM
zi)9&^=tF_Ac{LfeFgD?d0#6ipq71atcWeF;pH$3IsZCL`2JlUISHruy(|m@AsD0K%
zZD2!0)L~d!p>G-klXvkY$h44YA=5&pg-i?Y>ZYOsz6rkBOsaP|Nwia#KtxnTL|v-m
z0M<}w*+dnrq0Dxh&9H`|Rt|^k!$-b1Sl?{%vhEJo$--zgGb$LHkyePYSvt|p)G;=-
z8&0V*7uVnF@!QK)-z(o!s{A^3CBZL9Td=@4!8dtCVr+Wxu*L10yKZ{&x+1Gwh^%rw
zj_UB)gFh{N_TaPkC}i4t9DFn0jy1B}U0zs2!5RwIP=>JUTv_9=4+UA}daQ7;(^8xZ
z_|w9l7XGvx_TuJCY*>da2>!IREiWF~0O5N8--AVo_07*GvKD*~s+Eq_2>2f8Ksw70
zVurod@I8R<feLbnJz~R)#sxbq?6gZAQBBuC+9v^D_ineiC=3kZ1sMku^iAlS&^J5c
z7<W&k@O3A@S@`V1XKzRmW+BrW*>LBqvxv9{h<kvz2Z(!sxCe-PfVc-;0quKZyn}D5
zfYW;Acu>j>_#Sk15><iE9(?xTvo|l7k)#Rhn^@n(`X<&lhnj5&wjkJou)c|XDB9Mw
z)NGjW*@MqsTi30Jocs<xdjnI<?(fXg+NlgfriDxknHDl_s}Eow3i6@M>Tm+szlr^u
zqq2XK(jZ5Lg_{T{m+5ZxnNc5ACfL();&!edWu#o+s!W__e1Y*b#`BEtGk&aba*C02
z_fB$3*U5uSmy9PFpJ6=9_(R5@G5#ClFEvhWVdR{vQwO-Nl8#cDI>Gof<M$YU!1z<f
z|Ds%`e!3{VV$O03#e!waplJEoOv1|A884aiJzr*RAvR2?Q`CYZ1^<m*U@sH|T|&^M
zt10M`pm`xC)*SQ74M}64qUm1b&hGT>({2VDs)t@E<A<xI!U1nV*44}?+e)aq0u|ou
zJ!GHJVI|B}_t)UZOkL(NZ>iP1<M~p(!!36D^vQSLJ%8;hQCitevMd*5r&bvXhceJF
zUkO*673{vd@l2}hB{nCn=-j<Dp^l5yLA6n#wA0kx&WwPEAKm8%e4p;p-cV%0%U3IN
zqct*OVp^L4te{#6Rkv==3M2BG#!p<m!y3|uLHZyIIL#^~_I}nF-hRW_%AQB*v8R@t
z2tB15h>Bv$vAh_R2%UaO(uJq5EWE1v2fOLNs6rE_7M}@kp+Bm0Q}<|M5A|zC>Ru!D
zRwMN`Bb7Csp8vMbc<g>7^$sKTP9rsMr2eUq%DPO?ow4TBL%qv*g+DV=ziy-+GE(n0
zQvcjYJseS^cS(Qrr~B33V?6O*BlRzg)ccIoBSz}|Mk=ecJq`Gc=vaNX>Z|_Jee)0W
zU#>@e<^#m|^ZMIbR!i5vrRy6w>e=Q=UElJe#;x3qx4oz9GmmN9ex1e}Ue$ObzvlQy
zx}JJUqqAM3`-a9{{N%eo(e=I0YTTF5c;}lM56){mv`XXQGaB!^N#oJCHF}{&xl!Z%
zIgMo=Ce;s>`YclgH<wJOtzyPYShk%EEZ6aqmLna<Eu?ZqJDCxpeoCD>hLHaV`LDWx
z;?Ihs3|SS#(l?gA+qxB&zU8HYAV=CLmcG59wSfXD&5_caSrbWX*am@Z5ZDHRZ4eZ#
zxg!W5_-1cBTJTNqP4LYDeE=?Fa2bQkSh*et-vr+z+c84^m-@r^K;LYq>*|eRfe?&p
z(9p;sH-@OAU=0OpD6NSUYbXf8h_K>Abo0Tsdu+SMw)-Kt9h0y>WEVV)O^i*9O^i*1
z#OSvn3}dGVshL!uZz9DUQp~aY6xKHftZgARYp+Rv2|M$4x=(C7`}#GXK1VLFkvVTg
zZ!1}~<Q*lj(}HgzRZhpAU@b~D7rbDng`F05TG(m155O-7enIdHf?tr>*B3kj`X=;E
zUfzda5d4A$xQoF~ix7+m!8j@*7!mh?r?H(#m4mnk2rJ$SDUCH0tf9ab1X~bnL9hkE
z7Sz3DhSaQ`?Eza5Y(cOEY11%Lvmz!5$&4cqC>(ZL*lA&>g`F004+aP?*dvBGdx*1#
zID3e*7qJM)v<5y=6R-ur7PO<!RgU)CAYKsl3_W*nAHn`jiT#_{zlr^u;G41rz6rh=
zmelI<B{F1!Z>k{Yu+zd$3p*|Bv<Cefz6bC<7|<;u)`@sQ&^MuPj)1;7t^zVfr77_e
zQZCa6)n`V1RGHvNE)$#}cY^b#PEg`+`u#NH3yiNZo@acY@nemXQ;c6`bQw9J?WE-T
zNycXw&oW+M{JF*{x?feMwldB#a`N9P&Ivp92-BZr{A<R)WBf<PiyAk?R4!95TPXN$
zvS?+qwzO;~SFn7qs8X8-jttzS?>kAA+I7SA>J$~ghYW483+#o0WZZ^i+(U{=hehJg
ze~~zv7h+<~F|XW^H1>(hU*yj2^x5KTjY?Qrknwuxg))A)S}Gjy7Gzy%6tbO^Q(b`y
z3)w@Ci8HLertYu7kD0p6W8PA$dB^jmdPmrw>C-3QdH4Lauf)VyUTl(OxgaYYKZnA}
ztBcMq+mTV_%eDLNW(fhubc^!>!HSMGMic6|SRGVVPohquK4(V2!;kLs1HMmpX>TY>
zhLf*W=0<B|#Kg4Tx@QH|N~pSZb5<CU*ED|O>K)dQJ`B<aWdx^LC8lKF&xRO#Dz>uc
zQF`pDB?oRysRp8=m~t#HV#P!#Hs@(oeYWgX=4KE256K`@`?h1nBwct~F{0~@(_HUL
zyuNIEY5Kt?AwqgDR6GsQbgzYKBKSsXV5Am|RMskbu3I!7J7=WM8>uBD^&unmVI#F{
zq%Ih#6(hB3q>`163NfU1OKL4BYHd=S=*LKbTFaHSJ;%2e%+&AgtFZ3Zb1boGD&7CF
zp@}`zV@B$uM(Q_>)Z<3#2_uzt$)0<7%y{fy8mW&*)ac1ptFQEBb<gF#)jovUmkv=<
zUqo!R_k*fB6Omef#-G<;*D6ht%KEo7Zr~TVc~aN6yr^;OUX9z{(>TMAvHdz-zu{Gl
zH{PW&{*lJiQyQJ^8r?TE?z&&&?oTxCeOBYXgvLAH)Oe5wz@b&Te)x>W`)<;B^lgnE
zcLTXm*XPe^Ebr4;{ZJ{^g>1r0B%HuX7aiZSWzx2Cb~0xbGnup}bA@!yv4yCgQl}tT
z2HRQ@EJN{{AuEZ*!VuKB6?OuP;@KltrjH|#4aOz{*|Z#RTQf@24tO_(HEG*7_IUM{
zL>?2X_KVeCtrQ;LFV>W0WiFgQA~qIFvRr7NQ4a;iX1P?+mfxU?4HDYPc?3*?k=z?8
z0(7$^)<dR+OiSHB>4lJK<KUa%n?p6oA~7TqLpHa9T}~2>m~(3?#>T?V_J~B4AupC-
zY+`I;Y!1PBgXBkyP2*r<mqn7VmDkOnHOwnpxrDxHN;V6=3BHMqh1ggqYp~Nw_WU<`
zV@AGy`#W7gJ0}jTp}-adTM%qPtv-Nc07wRaWB^D8fMfvOViNSt&h~(v78~-hA%EF>
z=-{&leG~d-OV_yEN-2Ey4EDX9?i1V2zJAT8&yg!n#D`b(W-$1=w|Y5z-QiCQe_Apy
zb*U0q-^BW66u#;*5i;R>@cGAJ!Pvyu#MnIk(vvU$IR5CZBynT^W>@Mz$h44YA=5&p
z9kA=NQ_x2;03-vb3Ha<8Y<iu;giLx-!TwDIHAYb55eRCGz17%Tt!8ZOt&X<FV0{zo
zoA9S?^#S<P!k-rYwD6~eKW$U3)gfr$PutlZ@TY}8E&OTWPg{y3&K_&hi|{>w?*V)d
z;CnDI0x5hC;Cley1Na`m_h3MX=1WhI@TY}8?TLPu6!;#@hRrQIv)v;I`A`r$8nL4(
z%9KvE)*L4B0UQ|nHx0gC*uQxN+52D%f-MNPAlQOn3qr0U#66hr4ETA;thb}bH^2WM
zPs@kDIC0Io@2AG!`bGUKN*$&2pQqG^9O9!?pehJ?R8*bH7!k+Bx(5of=rzjW0}+w_
zsPY|^`!nr3w&UiKmh@9e%eDi@a)V;V%BXtA4YF<_O-YC;-RW5Og}D8!zI?+?iMJkl
zp^V?5vK6-zhuwDQ-k*HeUjJ?F=wSDqYYfxct!dJotD+N5(HFXrP^>AIRJhwj^jq}%
za$6!1xQ<GFnz8JJOsg!Y*^K2Cf})j66_dUbq@AqHs0)uuOEkl-tGu_%Z3(xS@_j$y
zT5i^LE!#<@tz6b~EE!}{UZR*v=aShGYKdkNZIxVkxh+vhCNf?wZCg&!aa6}hNz2V;
Ze5>fY2{)5-96v{if=8uKq-kIX@qhL6+;{*0

literal 0
HcmV?d00001

diff --git a/flink-tests/src/test/resources/operatorstate/nonKeyed/_metadata b/flink-tests/src/test/resources/operatorstate/nonKeyed/_metadata
new file mode 100644
index 0000000000000000000000000000000000000000..8fcd1ead4fb260fe34ad058c67ff816f61cf1eb3
GIT binary patch
literal 3212
zcmdUxxo#9e5Qb-M4jFQCBoYU;>TCLV0VGHvAhWA4d*t=*YIYU@5s!dKNeMv0Pry4M
zArV02C*VjC0aUNoW-WrX95F4`OwUYrsq3GwtNPyV9#mB|AS-Q~Y-5jopUr>m{`~U(
z`1oV@d6V8<A$v}C+OK{u+wxtxl`(^AYbU0R^HH}$k7s^g4XQiJN-HpEP#i6&!~{5I
zIb9@oo@J?2rYUb0)6O}1_qDn@Yb%(vdqbRJ*iXYeYQ_h{c0TT!!!(?A*rnlpe2foL
z+u*2qmfE`y9({PNZoJ&6swZuw{+Wx@W*8=ihm-NJdpb??t{KgS52^RpMTvmgptzWp
zPFYA-UF-InHnN2%=hj(<lnK;Iz??f{P4=KrA*fIr@XQK87F}%;*ES*zeM>etuiSqY
z@by^=lQBhn0s!4;#g`!<L}f5V;6i1>Kqjy}Cde#dz(@#aMXvio0AEXhrx=ZM8U)Y8
zeIOPr=K+kXBVrb@c8&t>4FQ}MTT;NHZCru?5d}J_9q2kSP%?RNQAh|nSd`hPs1rHw
z3jty+0a$ZgYl)Bz3Q$G{a9nu^HR>cx5?M$JxHkj{TI?ACo0lLUIn9+u1(KTx%Ekzm
zsT!QGl}RRIwndP>5FpnQ;5k!P#SB@5QU#0$&vS%Sc0%Arvs4mmZwM%V=Y)WxA1~6u
z?VGo@j>P7tZ?kRsaC%NCWw7*D&k2jtI9PW|&>;sebOI+lZ5YoZV0IjWNFmDX6bem#
zd%#C-+F4hA?ea{pS}`xCMfN&Nu6QXL3RLtG(4_#L(x@|z1z~$qFBe*r>ao;fwRT)g
z3uMVFrY%^T9i6D80PAR1X-qaIG+MJ#O8de^$)igxR<r2Ev{2I6Kv6@MC8gAnR8e$M
S-U-S!J{f6yq(#XnCGbD%r-&{9

literal 0
HcmV?d00001


From 93969b349a522dec4e153e7b4362f0605fcd1d93 Mon Sep 17 00:00:00 2001
From: "guowei.mgw" <guowei.mgw@gmail.com>
Date: Fri, 28 Apr 2017 19:40:58 +0200
Subject: [PATCH 093/253] [FLINK-5892] Add tests for topology modifications

This closes #3770.
---
 .../checkpoint/CheckpointCoordinatorTest.java | 292 ++++++++++++++++++
 1 file changed, 292 insertions(+)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index 0d2e9031d1042..41b0e35d84147 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -2370,6 +2370,22 @@ public void testRestoreLatestCheckpointedStateScaleOut() throws Exception {
 		testRestoreLatestCheckpointedStateWithChangingParallelism(false);
 	}
 
+	@Test
+	public void testStateRecoveryWhenTopologyChangeOut() throws Exception {
+		testStateRecoveryWithTopologyChange(0);
+	}
+
+	@Test
+	public void testStateRecoveryWhenTopologyChangeIn() throws Exception {
+		testStateRecoveryWithTopologyChange(1);
+	}
+
+	@Test
+	public void testStateRecoveryWhenTopologyChange() throws Exception {
+		testStateRecoveryWithTopologyChange(2);
+	}
+
+
 	/**
 	 * Tests the checkpoint restoration with changing parallelism of job vertex with partitioned
 	 * state.
@@ -2530,6 +2546,282 @@ private void testRestoreLatestCheckpointedStateWithChangingParallelism(boolean s
 		comparePartitionableState(expectedOpStatesRaw, actualOpStatesRaw);
 	}
 
+	private static Tuple2<JobVertexID, OperatorID> generateIDPair() {
+		JobVertexID jobVertexID = new JobVertexID();
+		OperatorID operatorID = OperatorID.fromJobVertexID(jobVertexID);
+		return new Tuple2<>(jobVertexID, operatorID);
+	}
+	
+	/**
+	 * old topology
+	 * [operator1,operator2] * parallelism1 -> [operator3,operator4] * parallelism2
+	 *
+	 *
+	 * new topology
+	 *
+	 * [operator5,operator1,operator3] * newParallelism1 -> [operator3, operator6] * newParallelism2
+	 *
+	 * scaleType:
+	 * 0  increase parallelism
+	 * 1  decrease parallelism
+	 * 2  same parallelism
+	 */
+	public void testStateRecoveryWithTopologyChange(int scaleType) throws Exception {
+
+		/**
+		 * Old topology
+		 * CHAIN(op1 -> op2) * parallelism1 -> CHAIN(op3 -> op4) * parallelism2
+		 */
+		Tuple2<JobVertexID, OperatorID> id1 = generateIDPair();
+		Tuple2<JobVertexID, OperatorID> id2 = generateIDPair();
+		int parallelism1 = 10;
+		int maxParallelism1 = 64;
+
+		Tuple2<JobVertexID, OperatorID> id3 = generateIDPair();
+		Tuple2<JobVertexID, OperatorID> id4 = generateIDPair();
+		int parallelism2 = 10;
+		int maxParallelism2 = 64;
+
+		List<KeyGroupRange> keyGroupPartitions2 =
+			StateAssignmentOperation.createKeyGroupPartitions(maxParallelism2, parallelism2);
+
+		Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
+
+		//prepare vertex1 state
+		for (Tuple2<JobVertexID, OperatorID> id : Lists.newArrayList(id1, id2)) {
+			OperatorState taskState = new OperatorState(id.f1, parallelism1, maxParallelism1);
+			operatorStates.put(id.f1, taskState);
+			for (int index = 0; index < taskState.getParallelism(); index++) {
+				StreamStateHandle subNonPartitionedState = 
+					generateStateForVertex(id.f0, index)
+						.get(0);
+				OperatorStateHandle subManagedOperatorState =
+					generateChainedPartitionableStateHandle(id.f0, index, 2, 8, false)
+						.get(0);
+				OperatorStateHandle subRawOperatorState =
+					generateChainedPartitionableStateHandle(id.f0, index, 2, 8, true)
+						.get(0);
+
+				OperatorSubtaskState subtaskState = new OperatorSubtaskState(subNonPartitionedState,
+					subManagedOperatorState,
+					subRawOperatorState,
+					null,
+					null);
+				taskState.putState(index, subtaskState);
+			}
+		}
+
+		List<List<ChainedStateHandle<OperatorStateHandle>>> expectedManagedOperatorStates = new ArrayList<>();
+		List<List<ChainedStateHandle<OperatorStateHandle>>> expectedRawOperatorStates = new ArrayList<>();
+		//prepare vertex2 state
+		for (Tuple2<JobVertexID, OperatorID> id : Lists.newArrayList(id3, id4)) {
+			OperatorState operatorState = new OperatorState(id.f1, parallelism2, maxParallelism2);
+			operatorStates.put(id.f1, operatorState);
+			List<ChainedStateHandle<OperatorStateHandle>> expectedManagedOperatorState = new ArrayList<>();
+			List<ChainedStateHandle<OperatorStateHandle>> expectedRawOperatorState = new ArrayList<>();
+			expectedManagedOperatorStates.add(expectedManagedOperatorState);
+			expectedRawOperatorStates.add(expectedRawOperatorState);
+
+			for (int index = 0; index < operatorState.getParallelism(); index++) {
+				OperatorStateHandle subManagedOperatorState =
+					generateChainedPartitionableStateHandle(id.f0, index, 2, 8, false)
+						.get(0);
+				OperatorStateHandle subRawOperatorState =
+					generateChainedPartitionableStateHandle(id.f0, index, 2, 8, true)
+						.get(0);
+				KeyGroupsStateHandle subManagedKeyedState = id.f0.equals(id3.f0)
+					? generateKeyGroupState(id.f0, keyGroupPartitions2.get(index), false)
+					: null;
+				KeyGroupsStateHandle subRawKeyedState = id.f0.equals(id3.f0)
+					? generateKeyGroupState(id.f0, keyGroupPartitions2.get(index), true)
+					: null;
+
+				expectedManagedOperatorState.add(ChainedStateHandle.wrapSingleHandle(subManagedOperatorState));
+				expectedRawOperatorState.add(ChainedStateHandle.wrapSingleHandle(subRawOperatorState));
+
+				OperatorSubtaskState subtaskState = new OperatorSubtaskState(
+					null,
+					subManagedOperatorState,
+					subRawOperatorState,
+					subManagedKeyedState,
+					subRawKeyedState);
+				operatorState.putState(index, subtaskState);
+			}
+		}
+
+		/**
+		 * New topology
+		 * CHAIN(op5 -> op1 -> op2) * newParallelism1 -> CHAIN(op3 -> op6) * newParallelism2
+		 */
+		Tuple2<JobVertexID, OperatorID> id5 = generateIDPair();
+		int newParallelism1 = 10;
+
+		Tuple2<JobVertexID, OperatorID> id6 = generateIDPair();
+		int newParallelism2 = parallelism2;
+
+		if (scaleType == 0) {
+			newParallelism2 = 20;
+		} else if (scaleType == 1) {
+			newParallelism2 = 8;
+		}
+
+		List<KeyGroupRange> newKeyGroupPartitions2 =
+			StateAssignmentOperation.createKeyGroupPartitions(maxParallelism2, newParallelism2);
+
+		final ExecutionJobVertex newJobVertex1 = mockExecutionJobVertex(
+			id5.f0,
+			Lists.newArrayList(id2.f1, id1.f1, id5.f1),
+			newParallelism1,
+			maxParallelism1);
+
+		final ExecutionJobVertex newJobVertex2 = mockExecutionJobVertex(
+			id3.f0,
+			Lists.newArrayList(id6.f1, id3.f1),
+			newParallelism2,
+			maxParallelism2);
+
+		Map<JobVertexID, ExecutionJobVertex> tasks = new HashMap<>();
+
+		tasks.put(id5.f0, newJobVertex1);
+		tasks.put(id3.f0, newJobVertex2);
+
+		JobID jobID = new JobID();
+		StandaloneCompletedCheckpointStore standaloneCompletedCheckpointStore =
+			spy(new StandaloneCompletedCheckpointStore(1));
+
+		CompletedCheckpoint completedCheckpoint = new CompletedCheckpoint(
+				jobID,
+				2,
+				System.currentTimeMillis(),
+				System.currentTimeMillis() + 3000,
+				operatorStates,
+				Collections.<MasterState>emptyList(),
+				CheckpointProperties.forStandardCheckpoint(),
+				null,
+				null);
+
+		when(standaloneCompletedCheckpointStore.getLatestCheckpoint()).thenReturn(completedCheckpoint);
+
+		// set up the coordinator and validate the initial state
+		CheckpointCoordinator coord = new CheckpointCoordinator(
+			new JobID(),
+			600000,
+			600000,
+			0,
+			Integer.MAX_VALUE,
+			ExternalizedCheckpointSettings.none(),
+			newJobVertex1.getTaskVertices(),
+			newJobVertex1.getTaskVertices(),
+			newJobVertex1.getTaskVertices(),
+			new StandaloneCheckpointIDCounter(),
+			standaloneCompletedCheckpointStore,
+			null,
+			Executors.directExecutor());
+
+		coord.restoreLatestCheckpointedState(tasks, false, true);
+
+		for (int i = 0; i < newJobVertex1.getParallelism(); i++) {
+
+			TaskStateHandles taskStateHandles = newJobVertex1.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskStateHandles();
+			ChainedStateHandle<StreamStateHandle> actualSubNonPartitionedState = taskStateHandles.getLegacyOperatorState();
+			List<Collection<OperatorStateHandle>> actualSubManagedOperatorState = taskStateHandles.getManagedOperatorState();
+			List<Collection<OperatorStateHandle>> actualSubRawOperatorState = taskStateHandles.getRawOperatorState();
+
+			assertNull(taskStateHandles.getManagedKeyedState());
+			assertNull(taskStateHandles.getRawKeyedState());
+
+			// operator5
+			{
+				int operatorIndexInChain = 2;
+				assertNull(actualSubNonPartitionedState.get(operatorIndexInChain));
+				assertNull(actualSubManagedOperatorState.get(operatorIndexInChain));
+				assertNull(actualSubRawOperatorState.get(operatorIndexInChain));
+			}
+			// operator1
+			{
+				int operatorIndexInChain = 1;
+				ChainedStateHandle<StreamStateHandle> expectSubNonPartitionedState = generateStateForVertex(id1.f0, i);
+				ChainedStateHandle<OperatorStateHandle> expectedManagedOpState = generateChainedPartitionableStateHandle(
+					id1.f0, i, 2, 8, false);
+				ChainedStateHandle<OperatorStateHandle> expectedRawOpState = generateChainedPartitionableStateHandle(
+					id1.f0, i, 2, 8, true);
+
+				assertTrue(CommonTestUtils.isSteamContentEqual(
+					expectSubNonPartitionedState.get(0).openInputStream(),
+					actualSubNonPartitionedState.get(operatorIndexInChain).openInputStream()));
+
+				assertTrue(CommonTestUtils.isSteamContentEqual(expectedManagedOpState.get(0).openInputStream(),
+					actualSubManagedOperatorState.get(operatorIndexInChain).iterator().next().openInputStream()));
+
+				assertTrue(CommonTestUtils.isSteamContentEqual(expectedRawOpState.get(0).openInputStream(),
+					actualSubRawOperatorState.get(operatorIndexInChain).iterator().next().openInputStream()));
+			}
+			// operator2
+			{
+				int operatorIndexInChain = 0;
+				ChainedStateHandle<StreamStateHandle> expectSubNonPartitionedState = generateStateForVertex(id2.f0, i);
+				ChainedStateHandle<OperatorStateHandle> expectedManagedOpState = generateChainedPartitionableStateHandle(
+					id2.f0, i, 2, 8, false);
+				ChainedStateHandle<OperatorStateHandle> expectedRawOpState = generateChainedPartitionableStateHandle(
+					id2.f0, i, 2, 8, true);
+
+				assertTrue(CommonTestUtils.isSteamContentEqual(expectSubNonPartitionedState.get(0).openInputStream(),
+					actualSubNonPartitionedState.get(operatorIndexInChain).openInputStream()));
+
+				assertTrue(CommonTestUtils.isSteamContentEqual(expectedManagedOpState.get(0).openInputStream(),
+					actualSubManagedOperatorState.get(operatorIndexInChain).iterator().next().openInputStream()));
+
+				assertTrue(CommonTestUtils.isSteamContentEqual(expectedRawOpState.get(0).openInputStream(),
+					actualSubRawOperatorState.get(operatorIndexInChain).iterator().next().openInputStream()));
+			}
+		}
+
+		List<List<Collection<OperatorStateHandle>>> actualManagedOperatorStates = new ArrayList<>(newJobVertex2.getParallelism());
+		List<List<Collection<OperatorStateHandle>>> actualRawOperatorStates = new ArrayList<>(newJobVertex2.getParallelism());
+
+		for (int i = 0; i < newJobVertex2.getParallelism(); i++) {
+			TaskStateHandles taskStateHandles = newJobVertex2.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskStateHandles();
+
+			// operator 3
+			{
+				int operatorIndexInChain = 1;
+				List<Collection<OperatorStateHandle>> actualSubManagedOperatorState = new ArrayList<>(1);
+				actualSubManagedOperatorState.add(taskStateHandles.getManagedOperatorState().get(operatorIndexInChain));
+
+				List<Collection<OperatorStateHandle>> actualSubRawOperatorState = new ArrayList<>(1);
+				actualSubRawOperatorState.add(taskStateHandles.getRawOperatorState().get(operatorIndexInChain));
+
+				actualManagedOperatorStates.add(actualSubManagedOperatorState);
+				actualRawOperatorStates.add(actualSubRawOperatorState);
+
+				assertNull(taskStateHandles.getLegacyOperatorState().get(operatorIndexInChain));
+			}
+
+			// operator 6
+			{
+				int operatorIndexInChain = 0;
+				assertNull(taskStateHandles.getManagedOperatorState().get(operatorIndexInChain));
+				assertNull(taskStateHandles.getRawOperatorState().get(operatorIndexInChain));
+				assertNull(taskStateHandles.getLegacyOperatorState().get(operatorIndexInChain));
+
+			}
+
+			KeyGroupsStateHandle originalKeyedStateBackend = generateKeyGroupState(id3.f0, newKeyGroupPartitions2.get(i), false);
+			KeyGroupsStateHandle originalKeyedStateRaw = generateKeyGroupState(id3.f0, newKeyGroupPartitions2.get(i), true);
+
+
+			Collection<KeyedStateHandle> keyedStateBackend = taskStateHandles.getManagedKeyedState();
+			Collection<KeyedStateHandle> keyGroupStateRaw = taskStateHandles.getRawKeyedState();
+
+
+			compareKeyedState(Collections.singletonList(originalKeyedStateBackend), keyedStateBackend);
+			compareKeyedState(Collections.singletonList(originalKeyedStateRaw), keyGroupStateRaw);
+		}
+
+		comparePartitionableState(expectedManagedOperatorStates.get(0), actualManagedOperatorStates);
+		comparePartitionableState(expectedRawOperatorStates.get(0), actualRawOperatorStates);
+	}
+
 	/**
 	 * Tests that the externalized checkpoint configuration is respected.
 	 */

From 72be3572eb9807bac7fe60f860d95c32b9c7f333 Mon Sep 17 00:00:00 2001
From: Fabian Hueske <fhueske@apache.org>
Date: Wed, 15 Mar 2017 13:24:42 +0100
Subject: [PATCH 094/253] [FLINK-6059] [table] Reject GenericType<Row> when
 converting DataSet or DataStream to Table.

This closes #3546.
---
 .../flink/table/api/TableEnvironment.scala    | 13 +++++++-
 .../java/batch/TableEnvironmentITCase.java    | 31 +++++++++++++++++++
 .../flink/table/TableEnvironmentTest.scala    | 18 +++++++++--
 .../scala/batch/TableEnvironmentITCase.scala  | 30 ++++++++++++++++++
 4 files changed, 88 insertions(+), 4 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 16c40fe3fc062..bd974b0eafa3a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -565,7 +565,14 @@ abstract class TableEnvironment(val config: TableConfig) {
     */
   protected[flink] def getFieldInfo[A](inputType: TypeInformation[A]):
   (Array[String], Array[Int]) = {
-    (TableEnvironment.getFieldNames(inputType), TableEnvironment.getFieldIndices(inputType))
+
+    if (inputType.isInstanceOf[GenericTypeInfo[A]] && inputType.getTypeClass == classOf[Row]) {
+      throw new TableException(
+        "An input of GenericTypeInfo<Row> cannot be converted to Table. " +
+          "Please specify the type of the input with a RowTypeInfo.")
+    } else {
+      (TableEnvironment.getFieldNames(inputType), TableEnvironment.getFieldIndices(inputType))
+    }
   }
 
   /**
@@ -584,6 +591,10 @@ abstract class TableEnvironment(val config: TableConfig) {
     TableEnvironment.validateType(inputType)
 
     val indexedNames: Array[(Int, String)] = inputType match {
+      case g: GenericTypeInfo[A] if g.getTypeClass == classOf[Row] =>
+        throw new TableException(
+          "An input of GenericTypeInfo<Row> cannot be converted to Table. " +
+            "Please specify the type of the input with a RowTypeInfo.")
       case a: AtomicType[A] =>
         if (exprs.length != 1) {
           throw new TableException("Table of atomic type can only have a single field.")
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
index e165983b2c908..cab3855af1c19 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
@@ -28,6 +28,7 @@
 import org.apache.calcite.tools.RuleSets;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
 import org.apache.flink.table.api.java.BatchTableEnvironment;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.tuple.Tuple4;
@@ -46,6 +47,8 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import static org.junit.Assert.assertTrue;
+
 @RunWith(Parameterized.class)
 public class TableEnvironmentITCase extends TableProgramsCollectionTestBase {
 
@@ -374,6 +377,34 @@ public void testAsWithPojoAndGenericTypes() throws Exception {
 		compareResultAsText(results, expected);
 	}
 
+	@Test(expected = TableException.class)
+	public void testGenericRow() throws Exception {
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+		// use null value the enforce GenericType
+		DataSet<Row> dataSet = env.fromElements(Row.of(1, 2L, "Hello", null));
+		assertTrue(dataSet.getType() instanceof GenericTypeInfo);
+		assertTrue(dataSet.getType().getTypeClass().equals(Row.class));
+
+		// Must fail. Cannot import DataSet<Row> with GenericTypeInfo.
+		tableEnv.fromDataSet(dataSet);
+	}
+
+	@Test(expected = TableException.class)
+	public void testGenericRowWithAlias() throws Exception {
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+		// use null value the enforce GenericType
+		DataSet<Row> dataSet = env.fromElements(Row.of((Integer)null));
+		assertTrue(dataSet.getType() instanceof GenericTypeInfo);
+		assertTrue(dataSet.getType().getTypeClass().equals(Row.class));
+
+		// Must fail. Cannot import DataSet<Row> with GenericTypeInfo.
+		tableEnv.fromDataSet(dataSet, "nullField");
+	}
+
 	@Test(expected = TableException.class)
 	public void testAsWithToFewFields() throws Exception {
 		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index 3d93f4531f9ca..9939a9c17db4c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -21,13 +21,13 @@ package org.apache.flink.table
 import org.apache.flink.api.scala._
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.{TupleTypeInfo, TypeExtractor}
 import org.apache.flink.table.api.scala._
+import org.apache.flink.api.java.typeutils.{GenericTypeInfo, TupleTypeInfo, TypeExtractor}
 import org.apache.flink.table.api.TableException
 import org.apache.flink.table.expressions.{Alias, UnresolvedFieldReference}
 import org.apache.flink.table.utils.{MockTableEnvironment, TableTestBase}
-import org.apache.flink.table.utils.TableTestUtil._
-
+import org.apache.flink.table.utils.TableTestUtil.{batchTableNode, binaryNode, streamTableNode, term, unaryNode}
+import org.apache.flink.types.Row
 import org.junit.Test
 import org.junit.Assert.assertEquals
 
@@ -46,6 +46,8 @@ class TableEnvironmentTest extends TableTestBase {
 
   val atomicType = INT_TYPE_INFO
 
+  val genericRowType = new GenericTypeInfo[Row](classOf[Row])
+
   @Test
   def testGetFieldInfoTuple(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(tupleType)
@@ -78,6 +80,11 @@ class TableEnvironmentTest extends TableTestBase {
     fieldInfo._2.zip(Array(0)).foreach(x => assertEquals(x._2, x._1))
   }
 
+  @Test(expected = classOf[TableException])
+  def testGetFieldInfoGenericRow(): Unit = {
+    tEnv.getFieldInfo(genericRowType)
+  }
+
   @Test
   def testGetFieldInfoTupleNames(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(
@@ -278,6 +285,11 @@ class TableEnvironmentTest extends TableTestBase {
       ))
   }
 
+  @Test(expected = classOf[TableException])
+  def testGetFieldInfoGenericRowAlias(): Unit = {
+    tEnv.getFieldInfo(genericRowType, Array(UnresolvedFieldReference("first")))
+  }
+
   @Test
   def testSqlWithoutRegisteringForBatchTables(): Unit = {
     val util = batchTestUtil()
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
index 6cbe83402aae5..e61e1907e76c4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.api.scala.batch
 
 import java.util
 
+import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api.scala._
 import org.apache.flink.api.scala.util.CollectionDataSets
@@ -31,6 +32,7 @@ import org.apache.flink.test.util.TestBaseUtils
 import org.junit._
 import org.junit.runner.RunWith
 import org.junit.runners.Parameterized
+import org.junit.Assert.assertTrue
 
 import scala.collection.JavaConverters._
 
@@ -254,6 +256,34 @@ class TableEnvironmentITCase(
     CollectionDataSets.get3TupleDataSet(env)
       .toTable(tEnv, 'a as 'foo, 'b, 'c)
   }
+
+  @Test(expected = classOf[TableException])
+  def testGenericRow() {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    // use null value the enforce GenericType
+    val dataSet = env.fromElements(Row.of(null))
+    assertTrue(dataSet.getType().isInstanceOf[GenericTypeInfo[_]])
+    assertTrue(dataSet.getType().getTypeClass == classOf[Row])
+
+    // Must fail. Cannot import DataSet<Row> with GenericTypeInfo.
+    tableEnv.fromDataSet(dataSet)
+  }
+
+  @Test(expected = classOf[TableException])
+  def testGenericRowWithAlias() {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    // use null value the enforce GenericType
+    val dataSet = env.fromElements(Row.of(null))
+    assertTrue(dataSet.getType().isInstanceOf[GenericTypeInfo[_]])
+    assertTrue(dataSet.getType().getTypeClass == classOf[Row])
+
+    // Must fail. Cannot import DataSet<Row> with GenericTypeInfo.
+    tableEnv.fromDataSet(dataSet, "nullField")
+  }
 }
 
 object TableEnvironmentITCase {

From 61f42277be287b6b5b49698943e947808f914cae Mon Sep 17 00:00:00 2001
From: sunjincheng121 <sunjincheng121@gmail.com>
Date: Fri, 28 Apr 2017 14:59:42 +0800
Subject: [PATCH 095/253] [FLINK-6406] [table] Remove unused imports

This closes #3795
---
 .../main/scala/org/apache/flink/table/api/scala/package.scala   | 1 -
 .../org/apache/flink/table/catalog/ExternalCatalogTable.scala   | 1 -
 .../scala/org/apache/flink/table/expressions/arithmetic.scala   | 1 -
 .../main/scala/org/apache/flink/table/expressions/array.scala   | 2 +-
 .../scala/org/apache/flink/table/plan/stats/ColumnStats.scala   | 2 +-
 5 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala
index cd341cbe5ba48..5b431ec5b376e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala
@@ -26,7 +26,6 @@ import org.apache.flink.table.api.scala.{StreamTableEnvironment => ScalaStreamTa
 import org.apache.flink.table.api.scala.{BatchTableEnvironment => ScalaBatchTableEnv}
 
 import _root_.scala.language.implicitConversions
-import _root_.scala.reflect.ClassTag
 
 /**
   * == Table API (Scala) ==
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
index 4fdab66145885..01eca6db5aa67 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
@@ -21,7 +21,6 @@ package org.apache.flink.table.catalog
 import java.util.{HashMap => JHashMap, Map => JMap}
 import java.lang.{Long => JLong}
 
-import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.TableSchema
 import org.apache.flink.table.plan.stats.TableStats
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/arithmetic.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/arithmetic.scala
index ad1af63140239..28049e8570e5a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/arithmetic.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/arithmetic.scala
@@ -18,7 +18,6 @@
 package org.apache.flink.table.expressions
 
 import org.apache.calcite.rex.RexNode
-import org.apache.calcite.sql.`type`.IntervalSqlType
 import org.apache.calcite.sql.SqlOperator
 import org.apache.calcite.sql.fun.SqlStdOperatorTable
 import org.apache.calcite.tools.RelBuilder
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/array.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/array.scala
index b087b617bad79..7446228408b8a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/array.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/array.scala
@@ -22,7 +22,7 @@ import org.apache.calcite.rex.RexNode
 import org.apache.calcite.sql.fun.SqlStdOperatorTable
 import org.apache.calcite.tools.RelBuilder
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo.INT_TYPE_INFO
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, PrimitiveArrayTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, PrimitiveArrayTypeInfo}
 import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo
 import org.apache.flink.table.calcite.FlinkRelBuilder
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/stats/ColumnStats.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/stats/ColumnStats.scala
index 152bf83d84f3b..8cf06b509464a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/stats/ColumnStats.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/stats/ColumnStats.scala
@@ -18,7 +18,7 @@
 
 package org.apache.flink.table.plan.stats
 
-import java.lang.{Double, Long, Integer, Number}
+import java.lang.{Double, Long}
 
 /**
   * column statistics

From 1196bacb8b08b641b11089c7dc644518a5dfd486 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 27 Apr 2017 15:59:12 +0200
Subject: [PATCH 096/253] [FLINK-6395] [tests] Mark test bases as abstract

This closes #3790
---
 .../runtime/operators/testutils/BinaryOperatorTestBase.java     | 2 +-
 .../flink/runtime/operators/testutils/DriverTestBase.java       | 2 +-
 .../runtime/operators/testutils/UnaryOperatorTestBase.java      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
index 790ac9e8ddc55..96c8b73cec0f2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
@@ -55,7 +55,7 @@
 import java.util.List;
 
 @RunWith(Parameterized.class)
-public class BinaryOperatorTestBase<S extends Function, IN, OUT> extends TestLogger implements TaskContext<S, OUT> {
+public abstract class BinaryOperatorTestBase<S extends Function, IN, OUT> extends TestLogger implements TaskContext<S, OUT> {
 	
 	protected static final int PAGE_SIZE = 32 * 1024;
 	
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
index 6f132c99ee2ba..d2cedb994d97a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
@@ -54,7 +54,7 @@
 import java.util.List;
 
 @RunWith(Parameterized.class)
-public class DriverTestBase<S extends Function> extends TestLogger implements TaskContext<S, Record> {
+public abstract class DriverTestBase<S extends Function> extends TestLogger implements TaskContext<S, Record> {
 	
 	protected static final long DEFAULT_PER_SORT_MEM = 16 * 1024 * 1024;
 	
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
index af8a29d36d435..141aec6882325 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
@@ -53,7 +53,7 @@
 import java.util.List;
 
 @RunWith(Parameterized.class)
-public class UnaryOperatorTestBase<S extends Function, IN, OUT> extends TestLogger implements TaskContext<S, OUT> {
+public abstract class UnaryOperatorTestBase<S extends Function, IN, OUT> extends TestLogger implements TaskContext<S, OUT> {
 	
 	protected static final long DEFAULT_PER_SORT_MEM = 16 * 1024 * 1024;
 	

From fcb595fc5bcd47351c545b4ceb4a07f3194ee11d Mon Sep 17 00:00:00 2001
From: mtunique <oatgnem@gmail.com>
Date: Fri, 7 Apr 2017 18:04:03 +0800
Subject: [PATCH 097/253] [FLINK-6112] [table] Support Calcite 1.12's new
 numerical functions

This closes #3718.
---
 docs/dev/table_api.md                         | 396 +++++++++++++++++
 .../flink/table/api/scala/expressionDsl.scala |  68 +++
 .../table/codegen/calls/BuiltInMethods.scala  |  46 ++
 .../table/codegen/calls/ConstantCallGen.scala |  38 ++
 .../codegen/calls/FunctionGenerator.scala     | 161 +++++++
 .../table/expressions/ExpressionParser.scala  |  11 +-
 .../table/expressions/mathExpressions.scala   | 158 +++++++
 .../table/validate/FunctionCatalog.scala      |  24 ++
 .../expressions/ScalarFunctionsTest.scala     | 402 +++++++++++++++++-
 .../table/expressions/SqlExpressionTest.scala |  12 +
 10 files changed, 1312 insertions(+), 4 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 4b9e5653eef70..6d52a788ef6df 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1977,6 +1977,127 @@ BOOLEAN.isNotFalse
       </td>
     </tr>
 
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.sin()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the sine of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.cos()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the cosine of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.tan()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.cot()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.asin()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.acos()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.atan()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.degrees()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Converts <i>numeric</i> from radians to degrees.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.radians()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Converts <i>numeric</i> from degrees to radians.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.sign()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the signum of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+NUMERIC.round(numeric2)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Rounds the given number to the other value places right to the decimal point.</p>
+      </td>
+    </tr>
+
   </tbody>
 </table>
 
@@ -2156,6 +2277,17 @@ NUMERIC.floor()
       </td>
     </tr>
 
+    <tr>
+      <td>
+        {% highlight java %}
+pi()
+{% endhighlight %}
+      </td>
+      <td>
+        <p> Returns pi.</p>
+      </td>
+    </tr>
+    
   </tbody>
 </table>
 
@@ -3175,6 +3307,138 @@ NUMERIC.floor()
         <p>Calculates the largest integer less than or equal to a given number.</p>
       </td>
     </tr>
+    
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.sin()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the sine of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.cos()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the cosine of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.tan()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.cot()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.asin()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.acos()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.atan()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.degrees()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Converts <i>numeric</i> from radians to degrees.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.radians()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Converts <i>numeric</i> from degrees to radians.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.sign()
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the signum of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+NUMERIC.round(numeric2)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Rounds the given number to the other value places right to the decimal point.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+pi()
+{% endhighlight %}
+      </td>
+      <td>
+        <p> Returns pi.</p>
+      </td>
+    </tr>
 
   </tbody>
 </table>
@@ -4172,6 +4436,138 @@ boolean IS NOT UNKNOWN
       </td>
     </tr>
 
+    <tr>
+      <td>
+        {% highlight text %}
+SIN(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the sine of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+COS(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the cosine of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+TAN(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+COT(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+ASIN(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+ACOS(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+ATAN(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the tangent of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+DEGREES(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Converts <i>numeric</i> from radians to degrees.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+RADIANS(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Converts <i>numeric</i> from degrees to radians.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+SIGN(numeric)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Calculates the signum of a given number.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+ROUND(numeric1, numeric2)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Rounds the given number to the other value places right to the decimal point.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+PI
+{% endhighlight %}
+      </td>
+      <td>
+        <p> Returns pi.</p>
+      </td>
+    </tr>
+
   </tbody>
 </table>
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
index fc6cb1016ff8c..169a72991255e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
@@ -275,6 +275,61 @@ trait ImplicitExpressionOperations {
     */
   def ceil() = Ceil(expr)
 
+  /**
+    * Calculates the sine of a given number.
+    */
+  def sin() = Sin(expr)
+
+  /**
+    * Calculates the cosine of a given number.
+    */
+  def cos() = Cos(expr)
+
+  /**
+    * Calculates the tangent of a given number.
+    */
+  def tan() = Tan(expr)
+
+  /**
+    * Calculates the tangent of a given number.
+    */
+  def cot() = Cot(expr)
+
+  /**
+    * Calculates the tangent of a given number.
+    */
+  def asin() = Asin(expr)
+
+  /**
+    * Calculates the tangent of a given number.
+    */
+  def acos() = Acos(expr)
+
+  /**
+    * Calculates the tangent of a given number.
+    */
+  def atan() = Atan(expr)
+
+  /**
+    * Converts numeric from radians to degrees.
+    */
+  def degrees() = Degrees(expr)
+
+  /**
+    * Converts numeric from degrees to radians.
+    */
+  def radians() = Radians(expr)
+
+  /**
+    * Calculates the signum of a given number.
+    */
+  def sign() = Sign(expr)
+
+  /**
+    * Rounds the given number to the other value places right to the decimal point.
+    */
+  def round(other: Expression) = Round(expr, other)
+
   // String operations
 
   /**
@@ -807,4 +862,17 @@ object array {
   }
 }
 
+/**
+  * Returns pi.
+  */
+object pi {
+
+  /**
+    * Returns pi.
+    */
+  def apply(): Expression = {
+    Pi()
+  }
+}
+
 // scalastyle:on object.name
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala
index 649d3b27ffba4..dd4be46e4d179 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala
@@ -17,6 +17,7 @@
  */
 package org.apache.flink.table.codegen.calls
 
+import java.lang.{Long => JLong}
 import java.math.{BigDecimal => JBigDecimal}
 
 import org.apache.calcite.linq4j.tree.Types
@@ -25,15 +26,60 @@ import org.apache.flink.table.functions.utils.MathFunctions
 
 object BuiltInMethods {
   val LOG10 = Types.lookupMethod(classOf[Math], "log10", classOf[Double])
+
   val EXP = Types.lookupMethod(classOf[Math], "exp", classOf[Double])
+
   val POWER = Types.lookupMethod(classOf[Math], "pow", classOf[Double], classOf[Double])
   val POWER_DEC = Types.lookupMethod(
     classOf[MathFunctions], "power", classOf[Double], classOf[JBigDecimal])
+
   val LN = Types.lookupMethod(classOf[Math], "log", classOf[Double])
+
   val ABS = Types.lookupMethod(classOf[SqlFunctions], "abs", classOf[Double])
   val ABS_DEC = Types.lookupMethod(classOf[SqlFunctions], "abs", classOf[JBigDecimal])
+
   val LIKE_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "like",
     classOf[String], classOf[String], classOf[String])
+
   val SIMILAR_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "similar",
     classOf[String], classOf[String], classOf[String])
+
+  val SIN = Types.lookupMethod(classOf[Math], "sin", classOf[Double])
+  val SIN_DEC = Types.lookupMethod(classOf[SqlFunctions], "sin", classOf[JBigDecimal])
+
+  val COS = Types.lookupMethod(classOf[Math], "cos", classOf[Double])
+  val COS_DEC = Types.lookupMethod(classOf[SqlFunctions], "cos", classOf[JBigDecimal])
+
+  val TAN = Types.lookupMethod(classOf[Math], "tan", classOf[Double])
+  val TAN_DEC = Types.lookupMethod(classOf[SqlFunctions], "tan", classOf[JBigDecimal])
+
+  val COT = Types.lookupMethod(classOf[SqlFunctions], "cot", classOf[Double])
+  val COT_DEC = Types.lookupMethod(classOf[SqlFunctions], "cot", classOf[JBigDecimal])
+
+  val ASIN = Types.lookupMethod(classOf[Math], "asin", classOf[Double])
+  val ASIN_DEC = Types.lookupMethod(classOf[SqlFunctions], "asin", classOf[JBigDecimal])
+
+  val ACOS = Types.lookupMethod(classOf[Math], "acos", classOf[Double])
+  val ACOS_DEC = Types.lookupMethod(classOf[SqlFunctions], "acos", classOf[JBigDecimal])
+
+  val ATAN = Types.lookupMethod(classOf[Math], "atan", classOf[Double])
+  val ATAN_DEC = Types.lookupMethod(classOf[SqlFunctions], "atan", classOf[JBigDecimal])
+
+  val DEGREES = Types.lookupMethod(classOf[Math], "toDegrees", classOf[Double])
+  val DEGREES_DEC = Types.lookupMethod(classOf[SqlFunctions], "degrees", classOf[JBigDecimal])
+
+  val RADIANS = Types.lookupMethod(classOf[Math], "toRadians", classOf[Double])
+  val RADIANS_DEC = Types.lookupMethod(classOf[SqlFunctions], "radians", classOf[JBigDecimal])
+
+  val SIGN_DOUBLE = Types.lookupMethod(classOf[Math], "signum", classOf[Double])
+  val SIGN_INT = Types.lookupMethod(classOf[Integer], "signum", classOf[Int])
+  val SIGN_LONG = Types.lookupMethod(classOf[JLong], "signum", classOf[Long])
+  val SIGN_DEC = Types.lookupMethod(classOf[SqlFunctions], "sign", classOf[JBigDecimal])
+
+  val ROUND_DOUBLE = Types.lookupMethod(classOf[SqlFunctions], "sround", classOf[Double],
+    classOf[Int])
+  val ROUND_INT = Types.lookupMethod(classOf[SqlFunctions], "sround", classOf[Int], classOf[Int])
+  val ROUND_LONG = Types.lookupMethod(classOf[SqlFunctions], "sround", classOf[Long], classOf[Int])
+  val ROUND_DEC = Types.lookupMethod(classOf[SqlFunctions], "sround", classOf[JBigDecimal],
+    classOf[Int])
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala
new file mode 100644
index 0000000000000..39fb13b751ab0
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.codegen.calls
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.table.codegen.{CodeGenerator, GeneratedExpression}
+
+/**
+  * Generates a function call which return a constant.
+  */
+class ConstantCallGen(
+    targetType: TypeInformation[_],
+    constantCode: String) extends CallGenerator {
+
+  override def generate(
+      codeGenerator: CodeGenerator,
+      operands: Seq[GeneratedExpression])
+    : GeneratedExpression = {
+    codeGenerator.generateNonNullLiteral(targetType, constantCode)
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
index 27e6dc6a449cb..b8b001f5bef3b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
@@ -179,6 +179,167 @@ object FunctionGenerator {
     DOUBLE_TYPE_INFO,
     BuiltInMethods.POWER_DEC)
 
+  addSqlFunctionMethod(
+    SIN,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.SIN)
+
+  addSqlFunctionMethod(
+    SIN,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.SIN_DEC)
+
+  addSqlFunctionMethod(
+    COS,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.COS)
+
+  addSqlFunctionMethod(
+    COS,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.COS_DEC)
+
+  addSqlFunctionMethod(
+    TAN,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.TAN)
+
+  addSqlFunctionMethod(
+    TAN,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.TAN_DEC)
+
+  addSqlFunctionMethod(
+    COT,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.COT)
+
+  addSqlFunctionMethod(
+    COT,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.COT_DEC)
+
+  addSqlFunctionMethod(
+    ASIN,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.ASIN)
+
+  addSqlFunctionMethod(
+    ASIN,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.ASIN_DEC)
+
+  addSqlFunctionMethod(
+    ACOS,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.ACOS)
+
+  addSqlFunctionMethod(
+    ACOS,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.ACOS_DEC)
+
+  addSqlFunctionMethod(
+    ATAN,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.ATAN)
+
+  addSqlFunctionMethod(
+    ATAN,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.ATAN_DEC)
+
+  addSqlFunctionMethod(
+    DEGREES,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.DEGREES)
+
+  addSqlFunctionMethod(
+    DEGREES,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.DEGREES_DEC)
+
+  addSqlFunctionMethod(
+    RADIANS,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.RADIANS)
+
+  addSqlFunctionMethod(
+    RADIANS,
+    Seq(BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.RADIANS_DEC)
+
+  addSqlFunction(
+    PI,
+    Seq(),
+    new ConstantCallGen(DOUBLE_TYPE_INFO, Math.PI.toString))
+
+  addSqlFunctionMethod(
+    SIGN,
+    Seq(DOUBLE_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.SIGN_DOUBLE)
+
+  addSqlFunctionMethod(
+    SIGN,
+    Seq(INT_TYPE_INFO),
+    INT_TYPE_INFO,
+    BuiltInMethods.SIGN_INT)
+
+  addSqlFunctionMethod(
+    SIGN,
+    Seq(LONG_TYPE_INFO),
+    LONG_TYPE_INFO,
+    BuiltInMethods.SIGN_LONG)
+
+  addSqlFunctionMethod(
+    SIGN,
+    Seq(BIG_DEC_TYPE_INFO),
+    BIG_DEC_TYPE_INFO,
+    BuiltInMethods.SIGN_DEC)
+
+  addSqlFunctionMethod(
+    ROUND,
+    Seq(LONG_TYPE_INFO, INT_TYPE_INFO),
+    LONG_TYPE_INFO,
+    BuiltInMethods.ROUND_LONG)
+
+  addSqlFunctionMethod(
+    ROUND,
+    Seq(INT_TYPE_INFO, INT_TYPE_INFO),
+    INT_TYPE_INFO,
+    BuiltInMethods.ROUND_INT)
+
+  addSqlFunctionMethod(
+    ROUND,
+    Seq(BIG_DEC_TYPE_INFO, INT_TYPE_INFO),
+    BIG_DEC_TYPE_INFO,
+    BuiltInMethods.ROUND_DEC)
+
+  addSqlFunctionMethod(
+    ROUND,
+    Seq(DOUBLE_TYPE_INFO, INT_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.ROUND_DOUBLE)
+
   addSqlFunction(
     ABS,
     Seq(DOUBLE_TYPE_INFO),
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
index 1985248a648d5..113b85ad41ae5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
@@ -92,6 +92,7 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val CURRENT_RANGE: Keyword = Keyword("current_range")
   lazy val UNBOUNDED_ROW: Keyword = Keyword("unbounded_row")
   lazy val UNBOUNDED_RANGE: Keyword = Keyword("unbounded_range")
+  lazy val ASIN: Keyword = Keyword("asin")
 
   def functionIdent: ExpressionParser.Parser[String] =
     not(ARRAY) ~ not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~
@@ -315,11 +316,14 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val suffixAgg: PackratParser[Expression] =
     suffixSum | suffixMin | suffixMax | suffixCount | suffixAvg
 
+  lazy val suffixAsin: PackratParser[Expression] =
+    composite <~ "." ~ ASIN ~ opt("()") ^^ { e => Asin(e) }
+
   lazy val suffixed: PackratParser[Expression] =
     suffixTimeInterval | suffixRowInterval | suffixStart | suffixEnd | suffixAgg |
       suffixCast | suffixAs | suffixTrim | suffixTrimWithoutArgs | suffixIf | suffixAsc |
       suffixDesc | suffixToDate | suffixToTimestamp | suffixToTime | suffixExtract |
-      suffixFloor | suffixCeil | suffixGet | suffixFlattening |
+      suffixFloor | suffixCeil | suffixGet | suffixFlattening | suffixAsin |
       suffixFunctionCall | suffixFunctionCallOneArg // function call must always be at the end
 
   // prefix operators
@@ -403,10 +407,13 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val prefixAgg: PackratParser[Expression] =
     prefixSum | prefixMin | prefixMax | prefixCount | prefixAvg
 
+  lazy val prefixAsin: PackratParser[Expression] =
+    ASIN ~ "(" ~> composite <~ ")" ^^ { e => Asin(e) }
+
   lazy val prefixed: PackratParser[Expression] =
     prefixArray | prefixAgg | prefixStart | prefixEnd | prefixCast | prefixAs | prefixTrim |
       prefixTrimWithoutArgs | prefixIf | prefixExtract | prefixFloor | prefixCeil | prefixGet |
-      prefixFlattening | prefixFunctionCall |
+      prefixFlattening | prefixAsin | prefixFunctionCall |
       prefixFunctionCallOneArg // function call must always be at the end
 
   // over
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala
index b31367c4fec90..16cb34a85e307 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala
@@ -127,3 +127,161 @@ case class Sqrt(child: Expression) extends UnaryExpression with InputTypeSpec {
     relBuilder.call(SqlStdOperatorTable.POWER, child.toRexNode, Literal(0.5).toRexNode)
   }
 }
+
+case class Sin(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Sin")
+
+  override def toString: String = s"sin($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.SIN, child.toRexNode)
+  }
+}
+
+case class Cos(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Cos")
+
+  override def toString: String = s"cos($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.COS, child.toRexNode)
+  }
+}
+
+case class Tan(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Tan")
+
+  override def toString: String = s"tan($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.TAN, child.toRexNode)
+  }
+}
+
+case class Cot(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Cot")
+
+  override def toString: String = s"cot($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.COT, child.toRexNode)
+  }
+}
+
+case class Asin(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Asin")
+
+  override def toString: String = s"asin($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.ASIN, child.toRexNode)
+  }
+}
+
+case class Acos(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Acos")
+
+  override def toString: String = s"acos($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.ACOS, child.toRexNode)
+  }
+}
+
+case class Atan(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Atan")
+
+  override def toString: String = s"atan($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.ATAN, child.toRexNode)
+  }
+}
+
+case class Degrees(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Degrees")
+
+  override def toString: String = s"degrees($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.DEGREES, child.toRexNode)
+  }
+}
+
+case class Radians(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "Radians")
+
+  override def toString: String = s"radians($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.DEGREES, child.toRexNode)
+  }
+}
+
+case class Pi() extends LeafExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override def toString: String = s"pi()"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.PI)
+  }
+}
+
+case class Sign(child: Expression) extends UnaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = child.resultType
+
+  override private[flink] def validateInput(): ValidationResult =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "sign")
+
+  override def toString: String = s"sign($child)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.SIGN, child.toRexNode)
+  }
+}
+
+case class Round(left: Expression, right: Expression) extends BinaryExpression {
+  override private[flink] def resultType: TypeInformation[_] = left.resultType
+
+  override private[flink] def validateInput(): ValidationResult = {
+    if (!TypeCheckUtils.isInteger(right.resultType)) {
+      ValidationFailure(s"round right requires int, get " +
+        s"$right : ${right.resultType}")
+    }
+    TypeCheckUtils.assertNumericExpr(left.resultType, s"round left :$left")
+  }
+
+  override def toString: String = s"round($left, $right)"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.ROUND, left.toRexNode, right.toRexNode)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
index 74b371aedd9fd..97a7e96b70086 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
@@ -201,6 +201,18 @@ object FunctionCatalog {
     "mod" -> classOf[Mod],
     "sqrt" -> classOf[Sqrt],
     "minusPrefix" -> classOf[UnaryMinus],
+    "sin" -> classOf[Sin],
+    "cos" -> classOf[Cos],
+    "tan" -> classOf[Tan],
+    "cot" -> classOf[Cot],
+    "asin" -> classOf[Asin],
+    "acos" -> classOf[Acos],
+    "atan" -> classOf[Atan],
+    "degrees" -> classOf[Degrees],
+    "radians" -> classOf[Radians],
+    "pi" -> classOf[Pi],
+    "sign" -> classOf[Sign],
+    "round" -> classOf[Round],
 
     // temporal functions
     "extract" -> classOf[Extract],
@@ -348,6 +360,18 @@ class BasicOperatorTable extends ReflectiveSqlOperatorTable {
     SqlStdOperatorTable.QUARTER,
     SqlStdOperatorTable.SCALAR_QUERY,
     SqlStdOperatorTable.EXISTS,
+    SqlStdOperatorTable.SIN,
+    SqlStdOperatorTable.COS,
+    SqlStdOperatorTable.TAN,
+    SqlStdOperatorTable.COT,
+    SqlStdOperatorTable.ASIN,
+    SqlStdOperatorTable.ACOS,
+    SqlStdOperatorTable.ATAN,
+    SqlStdOperatorTable.DEGREES,
+    SqlStdOperatorTable.RADIANS,
+    SqlStdOperatorTable.PI,
+    SqlStdOperatorTable.SIGN,
+    SqlStdOperatorTable.ROUND,
     // EXTENSIONS
     EventTimeExtractor,
     ProcTimeExtractor,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
index 474043a3d3a35..85804bc68b85a 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
@@ -696,6 +696,388 @@ class ScalarFunctionsTest extends ExpressionTestBase {
       "-1231")
   }
 
+  @Test
+  def testSin(): Unit = {
+    testAllApis(
+      'f2.sin(),
+      "f2.sin()",
+      "SIN(f2)",
+      math.sin(42.toByte).toString)
+
+    testAllApis(
+      'f3.sin(),
+      "f3.sin()",
+      "SIN(f3)",
+      math.sin(43.toShort).toString)
+
+    testAllApis(
+      'f4.sin(),
+      "f4.sin()",
+      "SIN(f4)",
+      math.sin(44.toLong).toString)
+
+    testAllApis(
+      'f5.sin(),
+      "f5.sin()",
+      "SIN(f5)",
+      math.sin(4.5.toFloat).toString)
+
+    testAllApis(
+      'f6.sin(),
+      "f6.sin()",
+      "SIN(f6)",
+      math.sin(4.6).toString)
+
+    testAllApis(
+      'f15.sin(),
+      "sin(f15)",
+      "SIN(f15)",
+      math.sin(-1231.1231231321321321111).toString)
+  }
+
+  @Test
+  def testCos(): Unit = {
+    testAllApis(
+      'f2.cos(),
+      "f2.cos()",
+      "COS(f2)",
+      math.cos(42.toByte).toString)
+
+    testAllApis(
+      'f3.cos(),
+      "f3.cos()",
+      "COS(f3)",
+      math.cos(43.toShort).toString)
+
+    testAllApis(
+      'f4.cos(),
+      "f4.cos()",
+      "COS(f4)",
+      math.cos(44.toLong).toString)
+
+    testAllApis(
+      'f5.cos(),
+      "f5.cos()",
+      "COS(f5)",
+      math.cos(4.5.toFloat).toString)
+
+    testAllApis(
+      'f6.cos(),
+      "f6.cos()",
+      "COS(f6)",
+      math.cos(4.6).toString)
+
+    testAllApis(
+      'f15.cos(),
+      "cos(f15)",
+      "COS(f15)",
+      math.cos(-1231.1231231321321321111).toString)
+  }
+
+  @Test
+  def testTan(): Unit = {
+    testAllApis(
+      'f2.tan(),
+      "f2.tan()",
+      "TAN(f2)",
+      math.tan(42.toByte).toString)
+
+    testAllApis(
+      'f3.tan(),
+      "f3.tan()",
+      "TAN(f3)",
+      math.tan(43.toShort).toString)
+
+    testAllApis(
+      'f4.tan(),
+      "f4.tan()",
+      "TAN(f4)",
+      math.tan(44.toLong).toString)
+
+    testAllApis(
+      'f5.tan(),
+      "f5.tan()",
+      "TAN(f5)",
+      math.tan(4.5.toFloat).toString)
+
+    testAllApis(
+      'f6.tan(),
+      "f6.tan()",
+      "TAN(f6)",
+      math.tan(4.6).toString)
+
+    testAllApis(
+      'f15.tan(),
+      "tan(f15)",
+      "TAN(f15)",
+      math.tan(-1231.1231231321321321111).toString)
+  }
+
+  @Test
+  def testCot(): Unit = {
+    testAllApis(
+      'f2.cot(),
+      "f2.cot()",
+      "COT(f2)",
+      (1.0d / math.tan(42.toByte)).toString)
+
+    testAllApis(
+      'f3.cot(),
+      "f3.cot()",
+      "COT(f3)",
+      (1.0d / math.tan(43.toShort)).toString)
+
+    testAllApis(
+      'f4.cot(),
+      "f4.cot()",
+      "COT(f4)",
+      (1.0d / math.tan(44.toLong)).toString)
+
+    testAllApis(
+      'f5.cot(),
+      "f5.cot()",
+      "COT(f5)",
+      (1.0d / math.tan(4.5.toFloat)).toString)
+
+    testAllApis(
+      'f6.cot(),
+      "f6.cot()",
+      "COT(f6)",
+      (1.0d / math.tan(4.6)).toString)
+
+    testAllApis(
+      'f15.cot(),
+      "cot(f15)",
+      "COT(f15)",
+      (1.0d / math.tan(-1231.1231231321321321111)).toString)
+  }
+
+  @Test
+  def testAsin(): Unit = {
+    testAllApis(
+      'f25.asin(),
+      "f25.asin()",
+      "ASIN(f25)",
+      math.asin(0.42.toByte).toString)
+
+    testAllApis(
+      'f26.asin(),
+      "f26.asin()",
+      "ASIN(f26)",
+      math.asin(0.toShort).toString)
+
+    testAllApis(
+      'f27.asin(),
+      "f27.asin()",
+      "ASIN(f27)",
+      math.asin(0.toLong).toString)
+
+    testAllApis(
+      'f28.asin(),
+      "f28.asin()",
+      "ASIN(f28)",
+      math.asin(0.45.toFloat).toString)
+
+    testAllApis(
+      'f29.asin(),
+      "f29.asin()",
+      "ASIN(f29)",
+      math.asin(0.46).toString)
+
+    testAllApis(
+      'f30.asin(),
+      "f30.asin()",
+      "ASIN(f30)",
+      math.asin(1).toString)
+
+    testAllApis(
+      'f31.asin(),
+      "f31.asin()",
+      "ASIN(f31)",
+      math.asin(-0.1231231321321321111).toString)
+  }
+
+  @Test
+  def testAcos(): Unit = {
+    testAllApis(
+      'f25.acos(),
+      "f25.acos()",
+      "ACOS(f25)",
+      math.acos(0.42.toByte).toString)
+
+    testAllApis(
+      'f26.acos(),
+      "f26.acos()",
+      "ACOS(f26)",
+      math.acos(0.toShort).toString)
+
+    testAllApis(
+      'f27.acos(),
+      "f27.acos()",
+      "ACOS(f27)",
+      math.acos(0.toLong).toString)
+
+    testAllApis(
+      'f28.acos(),
+      "f28.acos()",
+      "ACOS(f28)",
+      math.acos(0.45.toFloat).toString)
+
+    testAllApis(
+      'f29.acos(),
+      "f29.acos()",
+      "ACOS(f29)",
+      math.acos(0.46).toString)
+
+    testAllApis(
+      'f30.acos(),
+      "f30.acos()",
+      "ACOS(f30)",
+      math.acos(1).toString)
+
+    testAllApis(
+      'f31.acos(),
+      "f31.acos()",
+      "ACOS(f31)",
+      math.acos(-0.1231231321321321111).toString)
+  }
+
+  @Test
+  def testAtan(): Unit = {
+    testAllApis(
+      'f25.atan(),
+      "f25.atan()",
+      "ATAN(f25)",
+      math.atan(0.42.toByte).toString)
+
+    testAllApis(
+      'f26.atan(),
+      "f26.atan()",
+      "ATAN(f26)",
+      math.atan(0.toShort).toString)
+
+    testAllApis(
+      'f27.atan(),
+      "f27.atan()",
+      "ATAN(f27)",
+      math.atan(0.toLong).toString)
+
+    testAllApis(
+      'f28.atan(),
+      "f28.atan()",
+      "ATAN(f28)",
+      math.atan(0.45.toFloat).toString)
+
+    testAllApis(
+      'f29.atan(),
+      "f29.atan()",
+      "ATAN(f29)",
+      math.atan(0.46).toString)
+
+    testAllApis(
+      'f30.atan(),
+      "f30.atan()",
+      "ATAN(f30)",
+      math.atan(1).toString)
+
+    testAllApis(
+      'f31.atan(),
+      "f31.atan()",
+      "ATAN(f31)",
+      math.atan(-0.1231231321321321111).toString)
+  }
+
+  @Test
+  def testDegrees(): Unit = {
+    testAllApis(
+      'f2.degrees(),
+      "f2.degrees()",
+      "DEGREES(f2)",
+      math.toDegrees(42.toByte).toString)
+
+    testAllApis(
+      'f3.degrees(),
+      "f3.degrees()",
+      "DEGREES(f3)",
+      math.toDegrees(43.toShort).toString)
+
+    testAllApis(
+      'f4.degrees(),
+      "f4.degrees()",
+      "DEGREES(f4)",
+      math.toDegrees(44.toLong).toString)
+
+    testAllApis(
+      'f5.degrees(),
+      "f5.degrees()",
+      "DEGREES(f5)",
+      math.toDegrees(4.5.toFloat).toString)
+
+    testAllApis(
+      'f6.degrees(),
+      "f6.degrees()",
+      "DEGREES(f6)",
+      math.toDegrees(4.6).toString)
+
+    testAllApis(
+      'f15.degrees(),
+      "degrees(f15)",
+      "DEGREES(f15)",
+      math.toDegrees(-1231.1231231321321321111).toString)
+  }
+
+  @Test
+  def testPi(): Unit = {
+    testAllApis(
+      pi(),
+      "pi()",
+      "PI",
+      math.Pi.toString)
+  }
+
+  @Test
+  def testSign(): Unit = {
+    testAllApis(
+      'f4.sign(),
+      "f4.sign()",
+      "SIGN(f4)",
+      1.toString)
+
+    testAllApis(
+      'f6.sign(),
+      "f6.sign()",
+      "SIGN(f6)",
+      1.0.toString)
+
+    testAllApis(
+      'f15.sign(),
+      "sign(f15)",
+      "SIGN(f15)",
+      (-1).toString)
+  }
+
+  @Test
+  def testRound(): Unit = {
+    testAllApis(
+      'f29.round('f30),
+      "f29.round(f30)",
+      "ROUND(f29, f30)",
+      0.5.toString)
+
+    testAllApis(
+      'f31.round('f7),
+      "f31.round(f7)",
+      "ROUND(f31, f7)",
+      (-0.123).toString)
+
+    testAllApis(
+      'f4.round('f32),
+      "f4.round(f32)",
+      "ROUND(f4, f32)",
+      40.toString)
+
+  }
+
   // ----------------------------------------------------------------------------------------------
   // Temporal functions
   // ----------------------------------------------------------------------------------------------
@@ -1103,7 +1485,7 @@ class ScalarFunctionsTest extends ExpressionTestBase {
   // ----------------------------------------------------------------------------------------------
 
   def testData = {
-    val testData = new Row(25)
+    val testData = new Row(33)
     testData.setField(0, "This is a test String.")
     testData.setField(1, true)
     testData.setField(2, 42.toByte)
@@ -1129,6 +1511,14 @@ class ScalarFunctionsTest extends ExpressionTestBase {
     testData.setField(22, BigDecimal("2").bigDecimal)
     testData.setField(23, "%This is a test String.")
     testData.setField(24, "*_This is a test String.")
+    testData.setField(25, 0.42.toByte)
+    testData.setField(26, 0.toShort)
+    testData.setField(27, 0.toLong)
+    testData.setField(28, 0.45.toFloat)
+    testData.setField(29, 0.46)
+    testData.setField(30, 1)
+    testData.setField(31, BigDecimal("-0.1231231321321321111").bigDecimal)
+    testData.setField(32, -1)
     testData
   }
 
@@ -1158,7 +1548,15 @@ class ScalarFunctionsTest extends ExpressionTestBase {
       Types.BOOLEAN,
       Types.DECIMAL,
       Types.STRING,
-      Types.STRING).asInstanceOf[TypeInformation[Any]]
+      Types.STRING,
+      Types.BYTE,
+      Types.SHORT,
+      Types.LONG,
+      Types.FLOAT,
+      Types.DOUBLE,
+      Types.INT,
+      Types.DECIMAL,
+      Types.INT).asInstanceOf[TypeInformation[Any]]
 
   }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala
index 76e95ff4868ce..b11e291c2602c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala
@@ -100,7 +100,19 @@ class SqlExpressionTest extends ExpressionTestBase {
     testSqlApi("LOG10(1)", "0.0")
     testSqlApi("EXP(0)", "1.0")
     testSqlApi("CEIL(2.5)", "3")
+    testSqlApi("CEILING(2.5)", "3")
     testSqlApi("FLOOR(2.5)", "2")
+    testSqlApi("SIN(2.5)", "0.5984721441039564")
+    testSqlApi("COS(2.5)", "-0.8011436155469337")
+    testSqlApi("TAN(2.5)", "-0.7470222972386603")
+    testSqlApi("COT(2.5)", "-1.3386481283041514")
+    testSqlApi("ASIN(0.5)", "0.5235987755982989")
+    testSqlApi("ACOS(0.5)", "1.0471975511965979")
+    testSqlApi("ATAN(0.5)", "0.4636476090008061")
+    testSqlApi("DEGREES(0.5)", "28.64788975654116")
+    testSqlApi("PI", "3.141592653589793")
+    testSqlApi("SIGN(-1.1)", "-1")
+    testSqlApi("ROUND(-12.345, 2)", "-12.35")
   }
 
   @Test

From 331b18a96ea29d9d7aa80aa597afdcb0334a84a6 Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Sun, 30 Apr 2017 15:03:33 +0200
Subject: [PATCH 098/253] [FLINK-6112] [table] Improve documentation,
 consistency, and fix bugs

---
 docs/dev/table_api.md                         | 297 +++++++++---------
 .../flink/table/api/scala/expressionDsl.scala |  16 +-
 .../table/codegen/calls/ConstantCallGen.scala |   2 +-
 .../codegen/calls/FunctionGenerator.scala     |  66 ++--
 .../table/expressions/mathExpressions.scala   |  22 +-
 .../table/validate/FunctionCatalog.scala      |   4 +-
 .../expressions/ScalarFunctionsTest.scala     |  48 ++-
 .../table/expressions/SqlExpressionTest.scala |   3 +-
 8 files changed, 249 insertions(+), 209 deletions(-)

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 6d52a788ef6df..022a73d178c7d 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1977,303 +1977,303 @@ BOOLEAN.isNotFalse
       </td>
     </tr>
 
+  </tbody>
+</table>
+
+
+<table class="table table-bordered">
+  <thead>
     <tr>
+      <th class="text-left" style="width: 40%">Arithmetic functions</th>
+      <th class="text-center">Description</th>
+    </tr>
+  </thead>
+
+  <tbody>
+
+   <tr>
       <td>
         {% highlight java %}
-NUMERIC.sin()
++ numeric
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the sine of a given number.</p>
+        <p>Returns <i>numeric</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.cos()
+- numeric
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the cosine of a given number.</p>
+        <p>Returns negative <i>numeric</i>.</p>
       </td>
     </tr>
-
+    
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.tan()
+numeric1 + numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> plus <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.cot()
+numeric1 - numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> minus <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.asin()
+numeric1 * numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> multiplied by <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.acos()
+numeric1 / numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> divided by <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.atan()
+numeric1.power(numeric2)
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> raised to the power of <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.degrees()
+NUMERIC.abs()
 {% endhighlight %}
       </td>
       <td>
-        <p>Converts <i>numeric</i> from radians to degrees.</p>
+        <p>Calculates the absolute value of given value.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.radians()
+numeric1 % numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Converts <i>numeric</i> from degrees to radians.</p>
+        <p>Returns the remainder (modulus) of <i>numeric1</i> divided by <i>numeric2</i>. The result is negative only if <i>numeric1</i> is negative.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.sign()
+NUMERIC.sqrt()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the signum of a given number.</p>
+        <p>Calculates the square root of a given value.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.round(numeric2)
+NUMERIC.ln()
 {% endhighlight %}
       </td>
       <td>
-        <p>Rounds the given number to the other value places right to the decimal point.</p>
+        <p>Calculates the natural logarithm of given value.</p>
       </td>
     </tr>
 
-  </tbody>
-</table>
-
-
-<table class="table table-bordered">
-  <thead>
     <tr>
-      <th class="text-left" style="width: 40%">Arithmetic functions</th>
-      <th class="text-center">Description</th>
-    </tr>
-  </thead>
-
-  <tbody>
-
-   <tr>
       <td>
         {% highlight java %}
-+ numeric
+NUMERIC.log10()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric</i>.</p>
+        <p>Calculates the base 10 logarithm of given value.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-- numeric
+NUMERIC.exp()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns negative <i>numeric</i>.</p>
+        <p>Calculates the Euler's number raised to the given power.</p>
       </td>
     </tr>
-    
+
     <tr>
       <td>
         {% highlight java %}
-numeric1 + numeric2
+NUMERIC.ceil()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> plus <i>numeric2</i>.</p>
+        <p>Calculates the smallest integer greater than or equal to a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-numeric1 - numeric2
+NUMERIC.floor()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> minus <i>numeric2</i>.</p>
+        <p>Calculates the largest integer less than or equal to a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-numeric1 * numeric2
+NUMERIC.sin()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> multiplied by <i>numeric2</i>.</p>
+        <p>Calculates the sine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-numeric1 / numeric2
+NUMERIC.cos()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> divided by <i>numeric2</i>.</p>
+        <p>Calculates the cosine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-numeric1.power(numeric2)
+NUMERIC.tan()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> raised to the power of <i>numeric2</i>.</p>
+        <p>Calculates the tangent of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.abs()
+NUMERIC.cot()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the absolute value of given value.</p>
+        <p>Calculates the cotangent of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-numeric1 % numeric2
+NUMERIC.asin()
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the remainder (modulus) of <i>numeric1</i> divided by <i>numeric2</i>. The result is negative only if <i>numeric1</i> is negative.</p>
+        <p>Calculates the arc sine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.sqrt()
+NUMERIC.acos()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the square root of a given value.</p>
+        <p>Calculates the arc cosine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.ln()
+NUMERIC.atan()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the natural logarithm of given value.</p>
+        <p>Calculates the arc tangent of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.log10()
+NUMERIC.degrees()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the base 10 logarithm of given value.</p>
+        <p>Converts <i>numeric</i> from radians to degrees.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.exp()
+NUMERIC.radians()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the Euler's number raised to the given power.</p>
+        <p>Converts <i>numeric</i> from degrees to radians.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.ceil()
+NUMERIC.sign()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the smallest integer greater than or equal to a given number.</p>
+        <p>Calculates the signum of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight java %}
-NUMERIC.floor()
+NUMERIC.round(INT)
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the largest integer less than or equal to a given number.</p>
+        <p>Rounds the given number to <i>integer</i> places right to the decimal point.</p>
       </td>
     </tr>
 
@@ -2284,7 +2284,7 @@ pi()
 {% endhighlight %}
       </td>
       <td>
-        <p> Returns pi.</p>
+        <p>Returns a value that is closer than any other value to pi.</p>
       </td>
     </tr>
     
@@ -3337,7 +3337,7 @@ NUMERIC.tan()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Calculates the cotangent of a given number.</p>
       </td>
     </tr>
 
@@ -3348,7 +3348,7 @@ NUMERIC.cot()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Calculates the arc sine of a given number.</p>
       </td>
     </tr>
 
@@ -3359,7 +3359,7 @@ NUMERIC.asin()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Calculates the arc cosine of a given number.</p>
       </td>
     </tr>
 
@@ -3370,7 +3370,7 @@ NUMERIC.acos()
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Calculates the arc tangent of a given number.</p>
       </td>
     </tr>
 
@@ -3421,11 +3421,11 @@ NUMERIC.sign()
     <tr>
       <td>
         {% highlight scala %}
-NUMERIC.round(numeric2)
+NUMERIC.round(INT)
 {% endhighlight %}
       </td>
       <td>
-        <p>Rounds the given number to the other value places right to the decimal point.</p>
+        <p>Rounds the given number to <i>integer</i> places right to the decimal point.</p>
       </td>
     </tr>
 
@@ -3436,7 +3436,7 @@ pi()
 {% endhighlight %}
       </td>
       <td>
-        <p> Returns pi.</p>
+        <p>Returns a value that is closer than any other value to pi.</p>
       </td>
     </tr>
 
@@ -4436,312 +4436,313 @@ boolean IS NOT UNKNOWN
       </td>
     </tr>
 
+  </tbody>
+</table>
+
+<table class="table table-bordered">
+  <thead>
+    <tr>
+      <th class="text-left" style="width: 40%">Arithmetic functions</th>
+      <th class="text-center">Description</th>
+    </tr>
+  </thead>
+
+  <tbody>
     <tr>
       <td>
         {% highlight text %}
-SIN(numeric)
++ numeric
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the sine of a given number.</p>
+        <p>Returns <i>numeric</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-COS(numeric)
+- numeric
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the cosine of a given number.</p>
+        <p>Returns negative <i>numeric</i>.</p>
       </td>
     </tr>
-
+    
     <tr>
       <td>
         {% highlight text %}
-TAN(numeric)
+numeric1 + numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> plus <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-COT(numeric)
+numeric1 - numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> minus <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-ASIN(numeric)
+numeric1 * numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> multiplied by <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-ACOS(numeric)
+numeric1 / numeric2
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> divided by <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-ATAN(numeric)
+POWER(numeric1, numeric2)
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the tangent of a given number.</p>
+        <p>Returns <i>numeric1</i> raised to the power of <i>numeric2</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-DEGREES(numeric)
+ABS(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Converts <i>numeric</i> from radians to degrees.</p>
+        <p>Returns the absolute value of <i>numeric</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-RADIANS(numeric)
+MOD(numeric1, numeric2)
 {% endhighlight %}
       </td>
       <td>
-        <p>Converts <i>numeric</i> from degrees to radians.</p>
+        <p>Returns the remainder (modulus) of <i>numeric1</i> divided by <i>numeric2</i>. The result is negative only if <i>numeric1</i> is negative.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-SIGN(numeric)
+SQRT(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Calculates the signum of a given number.</p>
+        <p>Returns the square root of <i>numeric</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-ROUND(numeric1, numeric2)
+LN(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Rounds the given number to the other value places right to the decimal point.</p>
+        <p>Returns the natural logarithm (base e) of <i>numeric</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-PI
+LOG10(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p> Returns pi.</p>
+        <p>Returns the base 10 logarithm of <i>numeric</i>.</p>
       </td>
     </tr>
 
-  </tbody>
-</table>
-
-<table class="table table-bordered">
-  <thead>
-    <tr>
-      <th class="text-left" style="width: 40%">Arithmetic functions</th>
-      <th class="text-center">Description</th>
-    </tr>
-  </thead>
-
-  <tbody>
     <tr>
       <td>
         {% highlight text %}
-+ numeric
+EXP(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric</i>.</p>
+        <p>Returns e raised to the power of <i>numeric</i>.</p>
       </td>
-    </tr>
+    </tr>   
 
     <tr>
       <td>
         {% highlight text %}
-- numeric
+CEIL(numeric)
+CEILING(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns negative <i>numeric</i>.</p>
+        <p>Rounds <i>numeric</i> up, and returns the smallest number that is greater than or equal to <i>numeric</i>.</p>
       </td>
-    </tr>
-    
+    </tr>  
+
     <tr>
       <td>
         {% highlight text %}
-numeric1 + numeric2
+FLOOR(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> plus <i>numeric2</i>.</p>
+        <p>Rounds <i>numeric</i> down, and returns the largest number that is less than or equal to <i>numeric</i>.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-numeric1 - numeric2
+SIN(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> minus <i>numeric2</i>.</p>
+        <p>Calculates the sine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-numeric1 * numeric2
+COS(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> multiplied by <i>numeric2</i>.</p>
+        <p>Calculates the cosine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-numeric1 / numeric2
+TAN(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> divided by <i>numeric2</i>.</p>
+        <p>Calculates the tangent of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-POWER(numeric1, numeric2)
+COT(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns <i>numeric1</i> raised to the power of <i>numeric2</i>.</p>
+        <p>Calculates the cotangent of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-ABS(numeric)
+ASIN(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the absolute value of <i>numeric</i>.</p>
+        <p>Calculates the arc sine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-MOD(numeric1, numeric2)
+ACOS(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the remainder (modulus) of <i>numeric1</i> divided by <i>numeric2</i>. The result is negative only if <i>numeric1</i> is negative.</p>
+        <p>Calculates the arc cosine of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-SQRT(numeric)
+ATAN(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the square root of <i>numeric</i>.</p>
+        <p>Calculates the arc tangent of a given number.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-LN(numeric)
+DEGREES(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the natural logarithm (base e) of <i>numeric</i>.</p>
+        <p>Converts <i>numeric</i> from radians to degrees.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-LOG10(numeric)
+RADIANS(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the base 10 logarithm of <i>numeric</i>.</p>
+        <p>Converts <i>numeric</i> from degrees to radians.</p>
       </td>
     </tr>
 
     <tr>
       <td>
         {% highlight text %}
-EXP(numeric)
+SIGN(numeric)
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns e raised to the power of <i>numeric</i>.</p>
+        <p>Calculates the signum of a given number.</p>
       </td>
-    </tr>   
+    </tr>
 
     <tr>
       <td>
         {% highlight text %}
-CEIL(numeric)
+ROUND(numeric, int)
 {% endhighlight %}
       </td>
       <td>
-        <p>Rounds <i>numeric</i> up, and returns the smallest number that is greater than or equal to <i>numeric</i>.</p>
+        <p>Rounds the given number to <i>integer</i> places right to the decimal point.</p>
       </td>
-    </tr>  
+    </tr>
 
     <tr>
       <td>
         {% highlight text %}
-FLOOR(numeric)
+PI()
 {% endhighlight %}
       </td>
       <td>
-        <p>Rounds <i>numeric</i> down, and returns the largest number that is less than or equal to <i>numeric</i>.</p>
+        <p>Returns a value that is closer than any other value to pi.</p>
       </td>
     </tr>
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
index 169a72991255e..a512098177265 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
@@ -291,22 +291,22 @@ trait ImplicitExpressionOperations {
   def tan() = Tan(expr)
 
   /**
-    * Calculates the tangent of a given number.
+    * Calculates the cotangent of a given number.
     */
   def cot() = Cot(expr)
 
   /**
-    * Calculates the tangent of a given number.
+    * Calculates the arc sine of a given number.
     */
   def asin() = Asin(expr)
 
   /**
-    * Calculates the tangent of a given number.
+    * Calculates the arc cosine of a given number.
     */
   def acos() = Acos(expr)
 
   /**
-    * Calculates the tangent of a given number.
+    * Calculates the arc tangent of a given number.
     */
   def atan() = Atan(expr)
 
@@ -326,9 +326,9 @@ trait ImplicitExpressionOperations {
   def sign() = Sign(expr)
 
   /**
-    * Rounds the given number to the other value places right to the decimal point.
+    * Rounds the given number to integer places right to the decimal point.
     */
-  def round(other: Expression) = Round(expr, other)
+  def round(places: Expression) = Round(expr, places)
 
   // String operations
 
@@ -863,12 +863,12 @@ object array {
 }
 
 /**
-  * Returns pi.
+  * Returns a value that is closer than any other value to pi.
   */
 object pi {
 
   /**
-    * Returns pi.
+    * Returns a value that is closer than any other value to pi.
     */
   def apply(): Expression = {
     Pi()
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala
index 39fb13b751ab0..779d077f9d2cd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ConstantCallGen.scala
@@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.codegen.{CodeGenerator, GeneratedExpression}
 
 /**
-  * Generates a function call which return a constant.
+  * Generates a function call which returns a constant.
   */
 class ConstantCallGen(
     targetType: TypeInformation[_],
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
index b8b001f5bef3b..a0d132bf811ac 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
@@ -179,6 +179,36 @@ object FunctionGenerator {
     DOUBLE_TYPE_INFO,
     BuiltInMethods.POWER_DEC)
 
+  addSqlFunction(
+    ABS,
+    Seq(DOUBLE_TYPE_INFO),
+    new MultiTypeMethodCallGen(BuiltInMethods.ABS))
+
+  addSqlFunction(
+    ABS,
+    Seq(BIG_DEC_TYPE_INFO),
+    new MultiTypeMethodCallGen(BuiltInMethods.ABS_DEC))
+
+  addSqlFunction(
+    FLOOR,
+    Seq(DOUBLE_TYPE_INFO),
+    new FloorCeilCallGen(BuiltInMethod.FLOOR.method))
+
+  addSqlFunction(
+    FLOOR,
+    Seq(BIG_DEC_TYPE_INFO),
+    new FloorCeilCallGen(BuiltInMethod.FLOOR.method))
+
+  addSqlFunction(
+    CEIL,
+    Seq(DOUBLE_TYPE_INFO),
+    new FloorCeilCallGen(BuiltInMethod.CEIL.method))
+
+  addSqlFunction(
+    CEIL,
+    Seq(BIG_DEC_TYPE_INFO),
+    new FloorCeilCallGen(BuiltInMethod.CEIL.method))
+
   addSqlFunctionMethod(
     SIN,
     Seq(DOUBLE_TYPE_INFO),
@@ -287,11 +317,6 @@ object FunctionGenerator {
     DOUBLE_TYPE_INFO,
     BuiltInMethods.RADIANS_DEC)
 
-  addSqlFunction(
-    PI,
-    Seq(),
-    new ConstantCallGen(DOUBLE_TYPE_INFO, Math.PI.toString))
-
   addSqlFunctionMethod(
     SIGN,
     Seq(DOUBLE_TYPE_INFO),
@@ -341,34 +366,9 @@ object FunctionGenerator {
     BuiltInMethods.ROUND_DOUBLE)
 
   addSqlFunction(
-    ABS,
-    Seq(DOUBLE_TYPE_INFO),
-    new MultiTypeMethodCallGen(BuiltInMethods.ABS))
-
-  addSqlFunction(
-    ABS,
-    Seq(BIG_DEC_TYPE_INFO),
-    new MultiTypeMethodCallGen(BuiltInMethods.ABS_DEC))
-
-  addSqlFunction(
-    FLOOR,
-    Seq(DOUBLE_TYPE_INFO),
-    new FloorCeilCallGen(BuiltInMethod.FLOOR.method))
-
-  addSqlFunction(
-    FLOOR,
-    Seq(BIG_DEC_TYPE_INFO),
-    new FloorCeilCallGen(BuiltInMethod.FLOOR.method))
-
-  addSqlFunction(
-    CEIL,
-    Seq(DOUBLE_TYPE_INFO),
-    new FloorCeilCallGen(BuiltInMethod.CEIL.method))
-
-  addSqlFunction(
-    CEIL,
-    Seq(BIG_DEC_TYPE_INFO),
-    new FloorCeilCallGen(BuiltInMethod.CEIL.method))
+    PI,
+    Seq(),
+    new ConstantCallGen(DOUBLE_TYPE_INFO, Math.PI.toString))
 
   // ----------------------------------------------------------------------------------------------
   // Temporal functions
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala
index 16cb34a85e307..fae4470e2aaac 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/mathExpressions.scala
@@ -241,17 +241,7 @@ case class Radians(child: Expression) extends UnaryExpression {
   override def toString: String = s"radians($child)"
 
   override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
-    relBuilder.call(SqlStdOperatorTable.DEGREES, child.toRexNode)
-  }
-}
-
-case class Pi() extends LeafExpression {
-  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
-
-  override def toString: String = s"pi()"
-
-  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
-    relBuilder.call(SqlStdOperatorTable.PI)
+    relBuilder.call(SqlStdOperatorTable.RADIANS, child.toRexNode)
   }
 }
 
@@ -285,3 +275,13 @@ case class Round(left: Expression, right: Expression) extends BinaryExpression {
     relBuilder.call(SqlStdOperatorTable.ROUND, left.toRexNode, right.toRexNode)
   }
 }
+
+case class Pi() extends LeafExpression {
+  override private[flink] def resultType: TypeInformation[_] = DOUBLE_TYPE_INFO
+
+  override def toString: String = s"pi()"
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    relBuilder.call(SqlStdOperatorTable.PI)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
index 97a7e96b70086..729ad48d5cfa7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
@@ -210,9 +210,9 @@ object FunctionCatalog {
     "atan" -> classOf[Atan],
     "degrees" -> classOf[Degrees],
     "radians" -> classOf[Radians],
-    "pi" -> classOf[Pi],
     "sign" -> classOf[Sign],
     "round" -> classOf[Round],
+    "pi" -> classOf[Pi],
 
     // temporal functions
     "extract" -> classOf[Extract],
@@ -369,9 +369,9 @@ class BasicOperatorTable extends ReflectiveSqlOperatorTable {
     SqlStdOperatorTable.ATAN,
     SqlStdOperatorTable.DEGREES,
     SqlStdOperatorTable.RADIANS,
-    SqlStdOperatorTable.PI,
     SqlStdOperatorTable.SIGN,
     SqlStdOperatorTable.ROUND,
+    SqlStdOperatorTable.PI,
     // EXTENSIONS
     EventTimeExtractor,
     ProcTimeExtractor,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
index 85804bc68b85a..c1c25084970ed 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
@@ -1027,12 +1027,42 @@ class ScalarFunctionsTest extends ExpressionTestBase {
   }
 
   @Test
-  def testPi(): Unit = {
+  def testRadians(): Unit = {
     testAllApis(
-      pi(),
-      "pi()",
-      "PI",
-      math.Pi.toString)
+      'f2.radians(),
+      "f2.radians()",
+      "RADIANS(f2)",
+      math.toRadians(42.toByte).toString)
+
+    testAllApis(
+      'f3.radians(),
+      "f3.radians()",
+      "RADIANS(f3)",
+      math.toRadians(43.toShort).toString)
+
+    testAllApis(
+      'f4.radians(),
+      "f4.radians()",
+      "RADIANS(f4)",
+      math.toRadians(44.toLong).toString)
+
+    testAllApis(
+      'f5.radians(),
+      "f5.radians()",
+      "RADIANS(f5)",
+      math.toRadians(4.5.toFloat).toString)
+
+    testAllApis(
+      'f6.radians(),
+      "f6.radians()",
+      "RADIANS(f6)",
+      math.toRadians(4.6).toString)
+
+    testAllApis(
+      'f15.radians(),
+      "radians(f15)",
+      "RADIANS(f15)",
+      math.toRadians(-1231.1231231321321321111).toString)
   }
 
   @Test
@@ -1075,7 +1105,15 @@ class ScalarFunctionsTest extends ExpressionTestBase {
       "f4.round(f32)",
       "ROUND(f4, f32)",
       40.toString)
+  }
 
+  @Test
+  def testPi(): Unit = {
+    testAllApis(
+      pi(),
+      "pi()",
+      "PI",
+      math.Pi.toString)
   }
 
   // ----------------------------------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala
index b11e291c2602c..5cc8bd18d1f9b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala
@@ -110,9 +110,10 @@ class SqlExpressionTest extends ExpressionTestBase {
     testSqlApi("ACOS(0.5)", "1.0471975511965979")
     testSqlApi("ATAN(0.5)", "0.4636476090008061")
     testSqlApi("DEGREES(0.5)", "28.64788975654116")
-    testSqlApi("PI", "3.141592653589793")
+    testSqlApi("RADIANS(0.5)", "0.008726646259971648")
     testSqlApi("SIGN(-1.1)", "-1")
     testSqlApi("ROUND(-12.345, 2)", "-12.35")
+    testSqlApi("PI", "3.141592653589793")
   }
 
   @Test

From 2434bebbe990f66148c9319ec91b9113b885a6de Mon Sep 17 00:00:00 2001
From: Haohui Mai <wheat9@apache.org>
Date: Mon, 24 Apr 2017 23:12:29 -0700
Subject: [PATCH 099/253] [FLINK-6377] [table] Support map types in the Table /
 SQL API

This closes #3767.
---
 .../table/calcite/FlinkTypeFactory.scala      | 13 +++--
 .../flink/table/codegen/CodeGenerator.scala   | 22 ++++++---
 .../table/codegen/calls/ScalarOperators.scala | 34 ++++++++++++-
 .../flink/table/plan/nodes/FlinkRelNode.scala |  2 +-
 .../table/plan/schema/MapRelDataType.scala    | 49 +++++++++++++++++++
 .../table/api/java/batch/sql/SqlITCase.java   | 33 +++++++++++++
 6 files changed, 140 insertions(+), 13 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/MapRelDataType.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
index 22a5c9f74b99a..7762ff853482a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
@@ -28,13 +28,12 @@ import org.apache.calcite.sql.parser.SqlParserPos
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.{NothingTypeInfo, PrimitiveArrayTypeInfo, SqlTimeTypeInfo, TypeInformation}
 import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.api.java.typeutils.{ObjectArrayTypeInfo, RowTypeInfo}
+import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo, RowTypeInfo}
 import org.apache.flink.api.java.typeutils.ValueTypeInfo._
 import org.apache.flink.table.api.TableException
-import org.apache.flink.table.plan.schema.{CompositeRelDataType, GenericRelDataType}
+import org.apache.flink.table.plan.schema.{ArrayRelDataType, CompositeRelDataType, GenericRelDataType, MapRelDataType}
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
 import org.apache.flink.table.typeutils.TypeCheckUtils.isSimple
-import org.apache.flink.table.plan.schema.ArrayRelDataType
 import org.apache.flink.table.calcite.FlinkTypeFactory.typeInfoToSqlTypeName
 import org.apache.flink.types.Row
 
@@ -123,6 +122,10 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
     case oa: ObjectArrayTypeInfo[_, _] =>
       new ArrayRelDataType(oa, createTypeFromTypeInfo(oa.getComponentInfo), true)
 
+    case mp: MapTypeInfo[_, _] =>
+      new MapRelDataType(mp, createTypeFromTypeInfo(mp.getKeyTypeInfo),
+        createTypeFromTypeInfo(mp.getValueTypeInfo), true)
+
     case ti: TypeInformation[_] =>
       new GenericRelDataType(typeInfo, getTypeSystem.asInstanceOf[FlinkTypeSystem])
 
@@ -226,6 +229,10 @@ object FlinkTypeFactory {
       val arrayRelDataType = relDataType.asInstanceOf[ArrayRelDataType]
       arrayRelDataType.typeInfo
 
+    case MAP if relDataType.isInstanceOf[MapRelDataType] =>
+      val mapRelDataType = relDataType.asInstanceOf[MapRelDataType]
+      mapRelDataType.typeInfo
+
     case _@t =>
       throw TableException(s"Type is not supported: $t")
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 298fb70d742fd..648efe6af7d73 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -28,9 +28,9 @@ import org.apache.calcite.sql.`type`.SqlTypeName._
 import org.apache.calcite.sql.fun.SqlStdOperatorTable._
 import org.apache.flink.api.common.functions._
 import org.apache.flink.api.common.io.GenericInputFormat
-import org.apache.flink.api.common.typeinfo.{AtomicType, SqlTimeTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeinfo.{AtomicType, PrimitiveArrayTypeInfo, SqlTimeTypeInfo, TypeInformation}
 import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.api.java.typeutils.{GenericTypeInfo, PojoTypeInfo, RowTypeInfo, TupleTypeInfo}
+import org.apache.flink.api.java.typeutils._
 import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.table.api.TableConfig
@@ -1414,11 +1414,19 @@ class CodeGenerator(
         generateArray(this, resultType, operands)
 
       case ITEM =>
-        val array = operands.head
-        val index = operands(1)
-        requireArray(array)
-        requireInteger(index)
-        generateArrayElementAt(this, array, index)
+        operands.head.resultType match {
+          case _: ObjectArrayTypeInfo[_, _] | _: PrimitiveArrayTypeInfo[_] =>
+            val array = operands.head
+            val index = operands(1)
+            requireInteger(index)
+            generateArrayElementAt(this, array, index)
+
+          case map: MapTypeInfo[_, _] =>
+            val key = operands(1)
+            generateMapGet(this, operands.head, key)
+
+          case _ => throw new CodeGenException("Expect an array or a map.")
+        }
 
       case CARDINALITY =>
         val array = operands.head
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala
index 47a81ab0df048..0c5baa6343038 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala
@@ -22,9 +22,9 @@ import org.apache.calcite.avatica.util.{DateTimeUtils, TimeUnitRange}
 import org.apache.calcite.util.BuiltInMethod
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.{NumericTypeInfo, PrimitiveArrayTypeInfo, SqlTimeTypeInfo, TypeInformation}
-import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo
+import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo}
 import org.apache.flink.table.codegen.CodeGenUtils._
-import org.apache.flink.table.codegen.{CodeGenerator, CodeGenException, GeneratedExpression}
+import org.apache.flink.table.codegen.{CodeGenException, CodeGenerator, GeneratedExpression}
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
 import org.apache.flink.table.typeutils.TypeCheckUtils._
 
@@ -911,6 +911,36 @@ object ScalarOperators {
     }
   }
 
+  def generateMapGet(
+      codeGenerator: CodeGenerator,
+      map: GeneratedExpression,
+      key: GeneratedExpression)
+  : GeneratedExpression = {
+
+    val resultTerm = newName("result")
+    val nullTerm = newName("isNull")
+    val ty = map.resultType.asInstanceOf[MapTypeInfo[_,_]]
+    val resultType = ty.getValueTypeInfo
+    val resultTypeTerm = boxedTypeTermForTypeInfo(ty.getValueTypeInfo)
+    val accessCode = if (codeGenerator.nullCheck) {
+          s"""
+             |${map.code}
+             |${key.code}
+             |boolean $nullTerm = (${map.nullTerm} || ${key.nullTerm});
+             |$resultTypeTerm $resultTerm = $nullTerm ?
+             |  null : ($resultTypeTerm) ${map.resultTerm}.get(${key.resultTerm});
+             |""".stripMargin
+        } else {
+          s"""
+             |${map.code}
+             |${key.code}
+             |$resultTypeTerm $resultTerm = ($resultTypeTerm)
+             | ${map.resultTerm}.get(${key.resultTerm});
+             |""".stripMargin
+        }
+    GeneratedExpression(resultTerm, nullTerm, accessCode, resultType)
+  }
+
   // ----------------------------------------------------------------------------------------------
 
   private def generateUnaryOperatorIfNotNull(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
index ccdddef154a02..7554ea91c9d79 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
@@ -93,7 +93,7 @@ trait FlinkRelNode extends RelNode {
     case SqlTypeName.ARRAY =>
       // 16 is an arbitrary estimate
       estimateDataTypeSize(t.getComponentType) * 16
-    case SqlTypeName.ANY => 128 // 128 is an arbitrary estimate
+    case SqlTypeName.ANY | SqlTypeName.MAP => 128 // 128 is an arbitrary estimate
     case _ => throw TableException(s"Unsupported data type encountered: $t")
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/MapRelDataType.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/MapRelDataType.scala
new file mode 100644
index 0000000000000..b3ff99fc85d41
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/MapRelDataType.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.plan.schema
+
+import com.google.common.base.Objects
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.sql.`type`.MapSqlType
+import org.apache.flink.api.common.typeinfo.TypeInformation
+
+class MapRelDataType(
+   val typeInfo: TypeInformation[_],
+   val keyType: RelDataType,
+   val valueType: RelDataType,
+   isNullable: Boolean) extends MapSqlType(keyType, valueType, isNullable) {
+
+  override def toString: String = s"MAP($typeInfo)"
+
+  def canEqual(other: Any): Boolean = other.isInstanceOf[MapRelDataType]
+
+  override def equals(other: Any): Boolean = other match {
+    case that: MapRelDataType =>
+      super.equals(that) &&
+        (that canEqual this) &&
+        keyType == that.keyType &&
+        valueType == that.valueType &&
+        isNullable == that.isNullable
+    case _ => false
+  }
+
+  override def hashCode(): Int = {
+    Objects.hashCode(keyType, valueType)
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
index 5ba67dd355372..114226cb0d20e 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
@@ -18,9 +18,14 @@
 
 package org.apache.flink.table.api.java.batch.sql;
 
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.typeutils.MapTypeInfo;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.table.api.java.BatchTableEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.tuple.Tuple5;
 import org.apache.flink.types.Row;
@@ -32,7 +37,10 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 @RunWith(Parameterized.class)
 public class SqlITCase extends TableProgramsCollectionTestBase {
@@ -138,4 +146,29 @@ public void testJoin() throws Exception {
 		String expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n";
 		compareResultAsText(results, expected);
 	}
+
+	@Test
+	public void testMap() throws Exception {
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+		List<Tuple2<Integer, Map<String, String>>> rows = new ArrayList<>();
+		rows.add(new Tuple2<>(1, Collections.singletonMap("foo", "bar")));
+		rows.add(new Tuple2<>(2, Collections.singletonMap("foo", "spam")));
+
+		TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(
+			BasicTypeInfo.INT_TYPE_INFO,
+			new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));
+
+		DataSet<Tuple2<Integer, Map<String, String>>> ds1 = env.fromCollection(rows, ty);
+		tableEnv.registerDataSet("t1", ds1, "a, b");
+
+		String sqlQuery = "SELECT b['foo'] FROM t1";
+		Table result = tableEnv.sql(sqlQuery);
+
+		DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
+		List<Row> results = resultSet.collect();
+		String expected = "bar\n" + "spam\n";
+		compareResultAsText(results, expected);
+	}
 }

From bba27d0dbc04c0cbafa801006be4772f340ea852 Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Wed, 26 Apr 2017 18:04:00 +0200
Subject: [PATCH 100/253] [FLINK-6377] [table] Add additional map tests

---
 docs/dev/table_api.md                         |  1 +
 .../org/apache/flink/table/api/Types.scala    | 12 +++-
 .../table/codegen/ExpressionReducer.scala     |  7 +-
 .../flink/table/plan/nodes/FlinkRelNode.scala |  5 +-
 .../flink/table/expressions/MapTypeTest.scala | 72 +++++++++++++++++++
 5 files changed, 93 insertions(+), 4 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 022a73d178c7d..2b777c6841c2c 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1747,6 +1747,7 @@ The Table API is built on top of Flink's DataSet and DataStream API. Internally,
 | `Types.INTERVAL_MILLIS`| `INTERVAL DAY TO SECOND(3)` | `java.lang.Long`       |
 | `Types.PRIMITIVE_ARRAY`| `ARRAY`                     | e.g. `int[]`           |
 | `Types.OBJECT_ARRAY`   | `ARRAY`                     | e.g. `java.lang.Byte[]`|
+| `Types.MAP`            | `MAP`                       | `java.util.HashMap`    |
 
 
 Advanced types such as generic types, composite types (e.g. POJOs or Tuples), and array types (object or primitive arrays) can be fields of a row. 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
index f22fa3242dbc0..2152b727fff5f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
@@ -18,7 +18,7 @@
 package org.apache.flink.table.api
 
 import org.apache.flink.api.common.typeinfo.{PrimitiveArrayTypeInfo, TypeInformation, Types => JTypes}
-import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo
+import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo}
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
 import org.apache.flink.types.Row
 
@@ -100,4 +100,14 @@ object Types {
   def OBJECT_ARRAY(elementType: TypeInformation[_]): TypeInformation[_] = {
     ObjectArrayTypeInfo.getInfoFor(elementType)
   }
+
+  /**
+    * Generates type information for a Java HashMap.
+    *
+    * @param keyType type of the keys of the map e.g. Types.STRING
+    * @param valueType type of the values of the map e.g. Types.STRING
+    */
+  def MAP(keyType: TypeInformation[_], valueType: TypeInformation[_]): TypeInformation[_] = {
+    new MapTypeInfo(keyType, valueType)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala
index 3fcbdc1af6afc..b7e13352707ec 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala
@@ -65,7 +65,10 @@ class ExpressionReducer(config: TableConfig)
         )
 
       // we don't support object literals yet, we skip those constant expressions
-      case (SqlTypeName.ANY, _) | (SqlTypeName.ROW, _) | (SqlTypeName.ARRAY, _) => None
+      case (SqlTypeName.ANY, _) |
+           (SqlTypeName.ROW, _) |
+           (SqlTypeName.ARRAY, _) |
+           (SqlTypeName.MAP, _) => None
 
       case (_, e) => Some(e)
     }
@@ -103,7 +106,7 @@ class ExpressionReducer(config: TableConfig)
       val unreduced = constExprs.get(i)
       unreduced.getType.getSqlTypeName match {
         // we insert the original expression for object literals
-        case SqlTypeName.ANY | SqlTypeName.ROW | SqlTypeName.ARRAY =>
+        case SqlTypeName.ANY | SqlTypeName.ROW | SqlTypeName.ARRAY | SqlTypeName.MAP =>
           reducedValues.add(unreduced)
         case _ =>
           val reducedValue = reduced.getField(reducedIdx)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
index 7554ea91c9d79..0b244e9a77363 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
@@ -93,7 +93,10 @@ trait FlinkRelNode extends RelNode {
     case SqlTypeName.ARRAY =>
       // 16 is an arbitrary estimate
       estimateDataTypeSize(t.getComponentType) * 16
-    case SqlTypeName.ANY | SqlTypeName.MAP => 128 // 128 is an arbitrary estimate
+    case SqlTypeName.MAP =>
+      // 16 is an arbitrary estimate
+      (estimateDataTypeSize(t.getKeyType) + estimateDataTypeSize(t.getValueType)) * 16
+    case SqlTypeName.ANY => 128 // 128 is an arbitrary estimate
     case _ => throw TableException(s"Unsupported data type encountered: $t")
   }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala
new file mode 100644
index 0000000000000..ca807372eaa3f
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.expressions
+
+import java.util.{HashMap => JHashMap}
+
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.java.typeutils.{MapTypeInfo, RowTypeInfo}
+import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.expressions.utils.ExpressionTestBase
+import org.apache.flink.types.Row
+import org.junit.Test
+
+class MapTypeTest extends ExpressionTestBase {
+
+  @Test(expected = classOf[ValidationException])
+  def testWrongKeyType(): Unit = {
+    testSqlApi("f4[12]", "FAIL")
+  }
+
+  @Test
+  def testItem(): Unit = {
+    testSqlApi("f0['map is null']", "null")
+    testSqlApi("f1['map is empty']", "null")
+    testSqlApi("f2['b']", "13")
+    testSqlApi("f3[1]", "null")
+    testSqlApi("f3[12]", "a")
+  }
+
+  // ----------------------------------------------------------------------------------------------
+
+  override def testData: Any = {
+    val testData = new Row(4)
+    testData.setField(0, null)
+    testData.setField(1, new JHashMap[String, Int]())
+    val map = new JHashMap[String, Int]()
+    map.put("a", 12)
+    map.put("b", 13)
+    testData.setField(2, map)
+    val map2 = new JHashMap[Int, String]()
+    map2.put(12, "a")
+    map2.put(13, "b")
+    testData.setField(3, map2)
+    testData
+  }
+
+  override def typeInfo: TypeInformation[Any] = {
+    new RowTypeInfo(
+      new MapTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
+      new MapTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO),
+      new MapTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO),
+      new MapTypeInfo(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO)
+    ).asInstanceOf[TypeInformation[Any]]
+  }
+
+}

From 2dd610632917273e227b61ccf3dede5715b636d4 Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Tue, 25 Apr 2017 11:36:08 -0400
Subject: [PATCH 101/253] [FLINK-6382] [gelly] Support additional types for
 generated graphs in Gelly examples

The Gelly examples current support IntValue, LongValue, and StringValue
for RMatGraph. Allow transformations and tests for all generated graphs
for ByteValue, Byte, ShortValue, Short, CharValue, Character, Integer,
Long, and String.

This closes #3779
---
 .../flink/graph/drivers/GraphMetrics.java     |   3 +-
 .../flink/graph/drivers/JaccardIndex.java     |   4 +
 .../graph/drivers/input/CompleteGraph.java    |  13 +-
 .../flink/graph/drivers/input/CycleGraph.java |  13 +-
 .../flink/graph/drivers/input/EmptyGraph.java |  13 +-
 .../graph/drivers/input/GeneratedGraph.java   | 372 +++++++++++++
 .../drivers/input/GeneratedMultiGraph.java    |  59 ++
 .../flink/graph/drivers/input/GridGraph.java  |  33 +-
 .../graph/drivers/input/HypercubeGraph.java   |  16 +-
 .../flink/graph/drivers/input/PathGraph.java  |  13 +-
 .../flink/graph/drivers/input/RMatGraph.java  |  80 +--
 .../drivers/input/SingletonEdgeGraph.java     |  16 +-
 .../flink/graph/drivers/input/StarGraph.java  |  13 +-
 .../drivers/parameter/ChoiceParameter.java    |  23 +-
 .../org/apache/flink/graph/RunnerITCase.java  |   4 +-
 .../flink/graph/drivers/AdamicAdarITCase.java |  28 +-
 .../drivers/ClusteringCoefficientITCase.java  | 179 ++++--
 .../drivers/ConnectedComponentsITCase.java    | 123 ++++-
 .../CopyableValueDriverBaseITCase.java        |  52 ++
 .../flink/graph/drivers/DriverBaseITCase.java |  74 ++-
 .../flink/graph/drivers/EdgeListITCase.java   | 522 +++++++++++++-----
 .../graph/drivers/GraphMetricsITCase.java     | 141 +++--
 .../flink/graph/drivers/HITSITCase.java       |  34 +-
 .../graph/drivers/JaccardIndexITCase.java     |  91 ++-
 .../flink/graph/drivers/PageRankITCase.java   |  34 +-
 .../graph/drivers/TriangleListingITCase.java  | 276 +++++++--
 .../drivers/input/GeneratedGraphTest.java     | 203 +++++++
 .../LongValueToSignedIntValue.java            |   2 +
 .../LongValueToUnsignedIntValue.java          |   6 +-
 .../flink/graph/generator/CompleteGraph.java  |   7 +-
 .../flink/graph/generator/GridGraph.java      |   2 +-
 .../flink/graph/generator/RMatGraph.java      |   2 +-
 .../flink/graph/generator/StarGraph.java      |   2 +-
 .../graph/library/link_analysis/HITS.java     |   2 -
 .../library/similarity/JaccardIndex.java      |  50 +-
 .../apache/flink/graph/asm/AsmTestBase.java   |   2 +-
 .../LongValueToSignedIntValueTest.java        |   4 +-
 .../LongValueToUnsignedIntValueTest.java      |   6 +-
 38 files changed, 2054 insertions(+), 463 deletions(-)
 create mode 100644 flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedGraph.java
 create mode 100644 flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedMultiGraph.java
 create mode 100644 flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/CopyableValueDriverBaseITCase.java
 create mode 100644 flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/input/GeneratedGraphTest.java

diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
index cc5a8946e289e..aef8f9f3adcab 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
@@ -26,7 +26,6 @@
 import org.apache.flink.graph.drivers.output.Print;
 import org.apache.flink.graph.drivers.parameter.ChoiceParameter;
 import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
-import org.apache.flink.types.CopyableValue;
 
 /**
  * Driver for directed and undirected graph metrics analytics.
@@ -36,7 +35,7 @@
  * @see org.apache.flink.graph.library.metric.undirected.EdgeMetrics
  * @see org.apache.flink.graph.library.metric.undirected.VertexMetrics
  */
-public class GraphMetrics<K extends Comparable<K> & CopyableValue<K>, VV, EV>
+public class GraphMetrics<K extends Comparable<K>, VV, EV>
 extends ParameterizedBase
 implements Driver<K, VV, EV>, Hash, Print {
 
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java
index ae0d5f8ccd0ee..d5b2ae3c258cb 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java
@@ -25,6 +25,7 @@
 import org.apache.flink.graph.drivers.output.CSV;
 import org.apache.flink.graph.drivers.output.Hash;
 import org.apache.flink.graph.drivers.output.Print;
+import org.apache.flink.graph.drivers.parameter.BooleanParameter;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
 import org.apache.flink.graph.library.similarity.JaccardIndex.Result;
 import org.apache.flink.types.CopyableValue;
@@ -57,6 +58,8 @@ public class JaccardIndex<K extends CopyableValue<K>, VV, EV>
 	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
 		.setDefaultValue(PARALLELISM_DEFAULT);
 
+	private BooleanParameter mirrorResults = new BooleanParameter(this, "mirror_results");
+
 	@Override
 	public String getName() {
 		return this.getClass().getSimpleName();
@@ -88,6 +91,7 @@ protected DataSet<Result<K>> simplePlan(Graph<K, VV, EV> graph) throws Exception
 			.run(new org.apache.flink.graph.library.similarity.JaccardIndex<K, VV, EV>()
 				.setMinimumScore(minNumerator.getValue().intValue(), minDenominator.getValue().intValue())
 				.setMaximumScore(maxNumerator.getValue().intValue(), maxDenominator.getValue().intValue())
+				.setMirrorResults(mirrorResults.getValue())
 				.setLittleParallelism(lp));
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CompleteGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CompleteGraph.java
index c31c5a84d33dd..dc85df44b4fc8 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CompleteGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CompleteGraph.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
@@ -32,8 +31,7 @@
  * Generate a {@link org.apache.flink.graph.generator.CompleteGraph}.
  */
 public class CompleteGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private LongParameter vertexCount = new LongParameter(this, "vertex_count")
 		.setMinimumValue(MINIMUM_VERTEX_COUNT);
@@ -48,11 +46,16 @@ public String getName() {
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + vertexCount.getValue() + ")";
+		return getTypeName() + " " + getName() + " (" + vertexCount.getValue() + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		return vertexCount.getValue();
+	}
+
+	@Override
+	protected Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) throws Exception {
 		return new org.apache.flink.graph.generator.CompleteGraph(env, vertexCount.getValue())
 			.setParallelism(littleParallelism.getValue().intValue())
 			.generate();
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CycleGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CycleGraph.java
index df66dabb75a0e..9ef67c35a12c2 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CycleGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CycleGraph.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
@@ -32,8 +31,7 @@
  * Generate a {@link org.apache.flink.graph.generator.CycleGraph}.
  */
 public class CycleGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private LongParameter vertexCount = new LongParameter(this, "vertex_count")
 		.setMinimumValue(MINIMUM_VERTEX_COUNT);
@@ -48,11 +46,16 @@ public String getName() {
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + vertexCount + ")";
+		return getTypeName() + " " + getName() + " (" + vertexCount + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		return vertexCount.getValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
 		return new org.apache.flink.graph.generator.CycleGraph(env, vertexCount.getValue())
 			.setParallelism(littleParallelism.getValue().intValue())
 			.generate();
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EmptyGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EmptyGraph.java
index 794317b4e333a..e7b59421a2be9 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EmptyGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EmptyGraph.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
@@ -31,8 +30,7 @@
  * Generate an {@link org.apache.flink.graph.generator.EmptyGraph}.
  */
 public class EmptyGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private LongParameter vertexCount = new LongParameter(this, "vertex_count")
 		.setMinimumValue(MINIMUM_VERTEX_COUNT);
@@ -44,11 +42,16 @@ public String getName() {
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + vertexCount + ")";
+		return getTypeName() + " " + getName() + " (" + vertexCount + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		return vertexCount.getValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
 		return new org.apache.flink.graph.generator.EmptyGraph(env, vertexCount.getValue())
 			.generate();
 	}
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedGraph.java
new file mode 100644
index 0000000000000..610722c8e716a
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedGraph.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.drivers.input;
+
+import org.apache.commons.lang3.text.WordUtils;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.asm.translate.TranslateFunction;
+import org.apache.flink.graph.asm.translate.TranslateGraphIds;
+import org.apache.flink.graph.asm.translate.translators.LongValueToStringValue;
+import org.apache.flink.graph.asm.translate.translators.LongValueToUnsignedIntValue;
+import org.apache.flink.graph.drivers.parameter.ChoiceParameter;
+import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
+import org.apache.flink.types.ByteValue;
+import org.apache.flink.types.CharValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.types.ShortValue;
+
+/**
+ * Base class for generated graphs.
+ *
+ * @param <K> graph ID type
+ */
+public abstract class GeneratedGraph<K>
+extends ParameterizedBase
+implements Input<K, NullValue, NullValue> {
+
+	private static final String BYTE = "byte";
+	private static final String NATIVE_BYTE = "nativeByte";
+
+	private static final String SHORT = "short";
+	private static final String NATIVE_SHORT = "nativeShort";
+
+	private static final String CHAR = "char";
+	private static final String NATIVE_CHAR = "nativeChar";
+
+	private static final String INTEGER = "integer";
+	private static final String NATIVE_INTEGER = "nativeInteger";
+
+	private static final String LONG = "long";
+	private static final String NATIVE_LONG = "nativeLong";
+
+	private static final String STRING = "string";
+	private static final String NATIVE_STRING = "nativeString";
+
+	private ChoiceParameter type = new ChoiceParameter(this, "type")
+		.setDefaultValue(INTEGER)
+		.addChoices(LONG, STRING)
+		.addHiddenChoices(BYTE, NATIVE_BYTE, SHORT, NATIVE_SHORT, CHAR, NATIVE_CHAR, NATIVE_INTEGER, NATIVE_LONG, NATIVE_STRING);
+
+	/**
+	 * The vertex count is verified to be no greater than the capacity of the
+	 * selected data type. All vertices must be counted even if skipped or
+	 * unused when generating graph edges.
+	 *
+	 * @return number of vertices configured for the graph
+	 */
+	protected abstract long vertexCount();
+
+	/**
+	 * Generate the graph as configured.
+	 *
+	 * @param env Flink execution environment
+	 * @return generated graph
+	 * @throws Exception on error
+	 */
+	protected abstract Graph<K, NullValue, NullValue> generate(ExecutionEnvironment env) throws Exception;
+
+	/**
+	 * Get the name of the type.
+	 *
+	 * @return name of the type
+	 */
+	protected String getTypeName() {
+		return WordUtils.capitalize(type.getValue());
+	}
+
+	@Override
+	public Graph<K, NullValue, NullValue> create(ExecutionEnvironment env)
+			throws Exception {
+		long maxVertexCount = Long.MAX_VALUE;
+		TranslateFunction translator = null;
+
+		switch (type.getValue()) {
+			case BYTE:
+				maxVertexCount = LongValueToUnsignedByteValue.MAX_VERTEX_COUNT;
+				translator = new LongValueToUnsignedByteValue();
+				break;
+
+			case NATIVE_BYTE:
+				maxVertexCount = LongValueToUnsignedByte.MAX_VERTEX_COUNT;
+				translator = new LongValueToUnsignedByte();
+				break;
+
+			case SHORT:
+				maxVertexCount = LongValueToUnsignedShortValue.MAX_VERTEX_COUNT;
+				translator = new LongValueToUnsignedShortValue();
+				break;
+
+			case NATIVE_SHORT:
+				maxVertexCount = LongValueToUnsignedShort.MAX_VERTEX_COUNT;
+				translator = new LongValueToUnsignedShort();
+				break;
+
+			case CHAR:
+				maxVertexCount = LongValueToCharValue.MAX_VERTEX_COUNT;
+				translator = new LongValueToCharValue();
+				break;
+
+			case NATIVE_CHAR:
+				maxVertexCount = LongValueToChar.MAX_VERTEX_COUNT;
+				translator = new LongValueToChar();
+				break;
+
+			case INTEGER:
+				maxVertexCount = LongValueToUnsignedIntValue.MAX_VERTEX_COUNT;
+				translator = new LongValueToUnsignedIntValue();
+				break;
+
+			case NATIVE_INTEGER:
+				maxVertexCount = LongValueToUnsignedInt.MAX_VERTEX_COUNT;
+				translator = new LongValueToUnsignedInt();
+				break;
+
+			case LONG:
+				break;
+
+			case NATIVE_LONG:
+				translator = new LongValueToLong();
+				break;
+
+			case STRING:
+				translator = new LongValueToStringValue();
+				break;
+
+			case NATIVE_STRING:
+				translator = new LongValueToString();
+				break;
+
+			default:
+				throw new ProgramParametrizationException("Unknown type '" + type.getValue() + "'");
+		}
+
+		long vertexCount = vertexCount();
+		if (vertexCount > maxVertexCount) {
+			throw new ProgramParametrizationException("Vertex count '" + vertexCount +
+				"' must be no greater than " + maxVertexCount +  " for type '" + type.getValue() + "'.");
+		}
+
+		Graph<K, NullValue, NullValue> graph = generate(env);
+
+		if (translator != null) {
+			graph = (Graph<K, NullValue, NullValue>) graph.run(new TranslateGraphIds(translator));
+		}
+
+		return graph;
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link ByteValue}.
+	 *
+	 * Throws {@link RuntimeException} for byte overflow.
+	 */
+	static class LongValueToUnsignedByteValue
+	implements TranslateFunction<LongValue, ByteValue> {
+		public static final long MAX_VERTEX_COUNT = 1L << 8;
+
+		@Override
+		public ByteValue translate(LongValue value, ByteValue reuse)
+				throws Exception {
+			if (reuse == null) {
+				reuse = new ByteValue();
+			}
+
+			long l = value.getValue();
+
+			if (l < 0 || l >= MAX_VERTEX_COUNT) {
+				throw new IllegalArgumentException("Cannot cast long value " + value + " to byte.");
+			} else {
+				reuse.setValue((byte) (l & (MAX_VERTEX_COUNT - 1)));
+			}
+
+			return reuse;
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link Byte}.
+	 *
+	 * Throws {@link RuntimeException} for byte overflow.
+	 */
+	static class LongValueToUnsignedByte
+	implements TranslateFunction<LongValue, Byte> {
+		public static final long MAX_VERTEX_COUNT = 1L << 8;
+
+		@Override
+		public Byte translate(LongValue value, Byte reuse)
+				throws Exception {
+			long l = value.getValue();
+
+			if (l < 0 || l >= MAX_VERTEX_COUNT) {
+				throw new IllegalArgumentException("Cannot cast long value " + value + " to byte.");
+			}
+
+			return (byte) (l & (MAX_VERTEX_COUNT - 1));
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link ShortValue}.
+	 *
+	 * Throws {@link RuntimeException} for short overflow.
+	 */
+	static class LongValueToUnsignedShortValue
+	implements TranslateFunction<LongValue, ShortValue> {
+		public static final long MAX_VERTEX_COUNT = 1L << 16;
+
+		@Override
+		public ShortValue translate(LongValue value, ShortValue reuse)
+				throws Exception {
+			if (reuse == null) {
+				reuse = new ShortValue();
+			}
+
+			long l = value.getValue();
+
+			if (l < 0 || l >= MAX_VERTEX_COUNT) {
+				throw new IllegalArgumentException("Cannot cast long value " + value + " to short.");
+			} else {
+				reuse.setValue((short) (l & (MAX_VERTEX_COUNT - 1)));
+			}
+
+			return reuse;
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link Short}.
+	 *
+	 * Throws {@link RuntimeException} for short overflow.
+	 */
+	static class LongValueToUnsignedShort
+	implements TranslateFunction<LongValue, Short> {
+		public static final long MAX_VERTEX_COUNT = 1L << 16;
+
+		@Override
+		public Short translate(LongValue value, Short reuse)
+				throws Exception {
+			long l = value.getValue();
+
+			if (l < 0 || l >= MAX_VERTEX_COUNT) {
+				throw new IllegalArgumentException("Cannot cast long value " + value + " to short.");
+			}
+
+			return (short) (l & (MAX_VERTEX_COUNT - 1));
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link CharValue}.
+	 *
+	 * Throws {@link RuntimeException} for char overflow.
+	 */
+	static class LongValueToCharValue
+	implements TranslateFunction<LongValue, CharValue> {
+		public static final long MAX_VERTEX_COUNT = 1L << 16;
+
+		@Override
+		public CharValue translate(LongValue value, CharValue reuse)
+				throws Exception {
+			if (reuse == null) {
+				reuse = new CharValue();
+			}
+
+			long l = value.getValue();
+
+			if (l < 0 || l >= MAX_VERTEX_COUNT) {
+				throw new IllegalArgumentException("Cannot cast long value " + value + " to char.");
+			} else {
+				reuse.setValue((char) (l & (MAX_VERTEX_COUNT - 1)));
+			}
+
+			return reuse;
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@code Character}.
+	 *
+	 * Throws {@link RuntimeException} for char overflow.
+	 */
+	static class LongValueToChar
+	implements TranslateFunction<LongValue, Character> {
+		public static final long MAX_VERTEX_COUNT = 1L << 16;
+
+		@Override
+		public Character translate(LongValue value, Character reuse)
+				throws Exception {
+			long l = value.getValue();
+
+			if (l < 0 || l >= MAX_VERTEX_COUNT) {
+				throw new IllegalArgumentException("Cannot cast long value " + value + " to char.");
+			}
+
+			return (char) (l & (MAX_VERTEX_COUNT - 1));
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link Integer}.
+	 *
+	 * Throws {@link RuntimeException} for integer overflow.
+	 */
+	static class LongValueToUnsignedInt
+	implements TranslateFunction<LongValue, Integer> {
+		public static final long MAX_VERTEX_COUNT = 1L << 32;
+
+		@Override
+		public Integer translate(LongValue value, Integer reuse)
+				throws Exception {
+			long l = value.getValue();
+
+			if (l < 0 || l >= MAX_VERTEX_COUNT) {
+				throw new IllegalArgumentException("Cannot cast long value " + value + " to integer.");
+			}
+
+			return (int) (l & (MAX_VERTEX_COUNT - 1));
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link Long}.
+	 */
+	static class LongValueToLong
+	implements TranslateFunction<LongValue, Long> {
+		@Override
+		public Long translate(LongValue value, Long reuse)
+				throws Exception {
+			return value.getValue();
+		}
+	}
+
+	/**
+	 * Translate {@link LongValue} to {@link String}.
+	 */
+	static class LongValueToString
+	implements TranslateFunction<LongValue, String> {
+		@Override
+		public String translate(LongValue value, String reuse)
+				throws Exception {
+			return Long.toString(value.getValue());
+		}
+	}
+}
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedMultiGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedMultiGraph.java
new file mode 100644
index 0000000000000..c0a16a8d74919
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GeneratedMultiGraph.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.drivers.input;
+
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.drivers.parameter.Simplify;
+import org.apache.flink.types.NullValue;
+
+/**
+ * Base class for graph generators which may create duplicate edges.
+ *
+ * @param <K> graph ID type
+ */
+public abstract class GeneratedMultiGraph<K extends Comparable<K>>
+extends GeneratedGraph<K> {
+
+	private Simplify simplify = new Simplify(this);
+
+	/**
+	 * Get the short string representation of the simplify transformation.
+	 *
+	 * @return short string representation of the simplify transformation
+	 */
+	protected String getSimplifyShortString() {
+		return simplify.getShortString();
+	}
+
+	/**
+	 * Generate the graph as configured.
+	 *
+	 * @param env Flink execution environment
+	 * @return input graph
+	 */
+	public Graph<K, NullValue, NullValue> create(ExecutionEnvironment env)
+			throws Exception {
+		Graph<K, NullValue, NullValue> graph = super.create(env);
+
+		// simplify after the translation to improve the performance of the
+		// simplify operators by processing smaller data types
+		return simplify.simplify(graph);
+	}
+}
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java
index d502215001e80..b41b86e33f12e 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java
@@ -23,10 +23,10 @@
 import org.apache.flink.client.program.ProgramParametrizationException;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
+import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -38,8 +38,7 @@
  * Generate a {@link org.apache.flink.graph.generator.GridGraph}.
  */
 public class GridGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private static final String PREFIX = "dim";
 
@@ -67,10 +66,10 @@ public void configure(ParameterTool parameterTool) throws ProgramParametrization
 		Map<Integer, String> dimensionMap = new TreeMap<>();
 
 		// first parse all dimensions into a sorted map
-		for (Map.Entry<String, String> entry : parameterTool.toMap().entrySet()) {
-			if (entry.getKey().startsWith(PREFIX)) {
-				int dimensionId = Integer.parseInt(entry.getKey().substring(PREFIX.length()));
-				dimensionMap.put(dimensionId, entry.getValue());
+		for (String key : parameterTool.toMap().keySet()) {
+			if (key.startsWith(PREFIX)) {
+				int dimensionId = Integer.parseInt(key.substring(PREFIX.length()));
+				dimensionMap.put(dimensionId, parameterTool.get(key));
 			}
 		}
 
@@ -82,11 +81,27 @@ public void configure(ParameterTool parameterTool) throws ProgramParametrization
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + dimensions + ")";
+		return getTypeName() + " " + getName() + " (" + dimensions + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		// in Java 8 use Math.multiplyExact(long, long)
+		BigInteger vertexCount = BigInteger.ONE;
+		for (Dimension dimension : dimensions) {
+			vertexCount = vertexCount.multiply(BigInteger.valueOf(dimension.size));
+		}
+
+		if (vertexCount.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
+			throw new ProgramParametrizationException("Number of vertices in grid graph '" + vertexCount +
+				"' is greater than Long.MAX_VALUE.");
+		}
+
+		return vertexCount.longValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
 		org.apache.flink.graph.generator.GridGraph graph = new org.apache.flink.graph.generator.GridGraph(env);
 
 		for (Dimension dimension : dimensions) {
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/HypercubeGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/HypercubeGraph.java
index 3b87b0080b118..8d1e8b166a953 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/HypercubeGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/HypercubeGraph.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
@@ -32,11 +31,11 @@
  * Generate a {@link org.apache.flink.graph.generator.HypercubeGraph}.
  */
 public class HypercubeGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private LongParameter dimensions = new LongParameter(this, "dimensions")
-		.setMinimumValue(MINIMUM_DIMENSIONS);
+		.setMinimumValue(MINIMUM_DIMENSIONS)
+		.setMaximumValue(63);
 
 	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
 		.setDefaultValue(PARALLELISM_DEFAULT);
@@ -48,11 +47,16 @@ public String getName() {
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + dimensions + ")";
+		return getTypeName() + " " + getName() + " (" + dimensions + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		return 1L << dimensions.getValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
 		return new org.apache.flink.graph.generator.HypercubeGraph(env, dimensions.getValue())
 			.setParallelism(littleParallelism.getValue().intValue())
 			.generate();
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/PathGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/PathGraph.java
index 968628c712d4b..9e02056c7c405 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/PathGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/PathGraph.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
@@ -32,8 +31,7 @@
  * Generate a {@link org.apache.flink.graph.generator.PathGraph}.
  */
 public class PathGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private LongParameter vertexCount = new LongParameter(this, "vertex_count")
 		.setMinimumValue(MINIMUM_VERTEX_COUNT);
@@ -48,11 +46,16 @@ public String getName() {
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + vertexCount + ")";
+		return getTypeName() + " " + getName() + " (" + vertexCount + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		return vertexCount.getValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
 		return new org.apache.flink.graph.generator.PathGraph(env, vertexCount.getValue())
 			.setParallelism(littleParallelism.getValue().intValue())
 			.generate();
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java
index d64534bec3ac8..adee1ebd14f29 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/RMatGraph.java
@@ -18,20 +18,12 @@
 
 package org.apache.flink.graph.drivers.input;
 
-import org.apache.commons.lang3.text.WordUtils;
 import org.apache.commons.math3.random.JDKRandomGenerator;
 import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.client.program.ProgramParametrizationException;
 import org.apache.flink.graph.Graph;
-import org.apache.flink.graph.asm.translate.TranslateGraphIds;
-import org.apache.flink.graph.asm.translate.translators.LongValueToStringValue;
-import org.apache.flink.graph.asm.translate.translators.LongValueToUnsignedIntValue;
 import org.apache.flink.graph.drivers.parameter.BooleanParameter;
-import org.apache.flink.graph.drivers.parameter.ChoiceParameter;
 import org.apache.flink.graph.drivers.parameter.DoubleParameter;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
-import org.apache.flink.graph.drivers.parameter.Simplify;
 import org.apache.flink.graph.generator.random.JDKRandomGeneratorFactory;
 import org.apache.flink.graph.generator.random.RandomGenerableFactory;
 import org.apache.flink.types.IntValue;
@@ -46,22 +38,9 @@
  * or {@link StringValue} keys.
  *
  * @see org.apache.flink.graph.generator.RMatGraph
- *
- * @param <K> key type
  */
-public class RMatGraph<K extends Comparable<K>>
-extends ParameterizedBase
-implements Input<K, NullValue, NullValue> {
-
-	private static final String INTEGER = "integer";
-
-	private static final String LONG = "long";
-
-	private static final String STRING = "string";
-
-	private ChoiceParameter type = new ChoiceParameter(this, "type")
-		.setDefaultValue(INTEGER)
-		.addChoices(LONG, STRING);
+public class RMatGraph
+extends GeneratedMultiGraph<LongValue> {
 
 	// generate graph with 2^scale vertices
 	private LongParameter scale = new LongParameter(this, "scale")
@@ -99,20 +78,23 @@ public class RMatGraph<K extends Comparable<K>>
 	private LongParameter seed = new LongParameter(this, "seed")
 		.setDefaultValue(JDKRandomGeneratorFactory.DEFAULT_SEED);
 
-	private Simplify simplify = new Simplify(this);
-
 	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
 		.setDefaultValue(PARALLELISM_DEFAULT);
 
 	@Override
 	public String getName() {
-		return RMatGraph.class.getSimpleName();
+		return this.getClass().getSimpleName();
 	}
 
 	@Override
 	public String getIdentity() {
-		return getName() + WordUtils.capitalize(type.getValue()) +
-			" (s" + scale + "e" + edgeFactor + simplify.getShortString() + ")";
+		return getTypeName() + " " + getName() +
+			" (s" + scale + "e" + edgeFactor + getSimplifyShortString() + ")";
+	}
+
+	@Override
+	protected long vertexCount() {
+		return 1L << scale.getValue();
 	}
 
 	/**
@@ -121,7 +103,7 @@ public String getIdentity() {
 	 * @param env Flink execution environment
 	 * @return input graph
 	 */
-	public Graph<K, NullValue, NullValue> create(ExecutionEnvironment env) throws Exception {
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) throws Exception {
 		int lp = littleParallelism.getValue().intValue();
 
 		RandomGenerableFactory<JDKRandomGenerator> rnd = new JDKRandomGeneratorFactory();
@@ -129,49 +111,11 @@ public Graph<K, NullValue, NullValue> create(ExecutionEnvironment env) throws Ex
 		long vertexCount = 1L << scale.getValue();
 		long edgeCount = vertexCount * edgeFactor.getValue();
 
-		Graph<LongValue, NullValue, NullValue> rmatGraph = new org.apache.flink.graph.generator.RMatGraph<>(
+		return new org.apache.flink.graph.generator.RMatGraph<>(
 				env, rnd, vertexCount, edgeCount)
 			.setConstants(a.getValue().floatValue(), b.getValue().floatValue(), c.getValue().floatValue())
 			.setNoise(noiseEnabled.getValue(), noise.getValue().floatValue())
 			.setParallelism(lp)
 			.generate();
-
-		Graph<K, NullValue, NullValue> graph;
-
-		switch (type.getValue()) {
-			case INTEGER:
-				if (scale.getValue() > 32) {
-					throw new ProgramParametrizationException(
-						"Scale '" + scale.getValue() + "' must be no greater than 32 for type 'integer'");
-				}
-				graph = (Graph<K, NullValue, NullValue>) rmatGraph
-					.run(new TranslateGraphIds<LongValue, IntValue, NullValue, NullValue>(new LongValueToUnsignedIntValue()));
-				break;
-
-			case LONG:
-				if (scale.getValue() > 64) {
-					throw new ProgramParametrizationException(
-						"Scale '" + scale.getValue() + "' must be no greater than 64 for type 'long'");
-				}
-				graph = (Graph<K, NullValue, NullValue>) rmatGraph;
-				break;
-
-			case STRING:
-				// scale bound is same as LONG since keys are generated as LongValue
-				if (scale.getValue() > 64) {
-					throw new ProgramParametrizationException(
-						"Scale '" + scale.getValue() + "' must be no greater than 64 for type 'string'");
-				}
-				graph = (Graph<K, NullValue, NullValue>) rmatGraph
-					.run(new TranslateGraphIds<LongValue, StringValue, NullValue, NullValue>(new LongValueToStringValue()));
-				break;
-
-			default:
-				throw new ProgramParametrizationException("Unknown type '" + type.getValue() + "'");
-		}
-
-		// simplify *after* the translation from LongValue to IntValue or
-		// StringValue to improve the performance of the simplify operators
-		return simplify.simplify(graph);
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/SingletonEdgeGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/SingletonEdgeGraph.java
index 502826fedc313..65e0196ec0659 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/SingletonEdgeGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/SingletonEdgeGraph.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
@@ -32,11 +31,11 @@
  * Generate a {@link org.apache.flink.graph.generator.SingletonEdgeGraph}.
  */
 public class SingletonEdgeGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private LongParameter vertexPairCount = new LongParameter(this, "vertex_pair_count")
-		.setMinimumValue(MINIMUM_VERTEX_COUNT);
+		.setMinimumValue(MINIMUM_VERTEX_COUNT)
+		.setMaximumValue(1L << 62);
 
 	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
 		.setDefaultValue(PARALLELISM_DEFAULT);
@@ -48,11 +47,16 @@ public String getName() {
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + vertexPairCount + ")";
+		return getTypeName() + " " + getName() + " (" + vertexPairCount + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		return 2 * vertexPairCount.getValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
 		return new org.apache.flink.graph.generator.SingletonEdgeGraph(env, vertexPairCount.getValue())
 			.setParallelism(littleParallelism.getValue().intValue())
 			.generate();
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/StarGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/StarGraph.java
index b794f5c9db224..b37dc493a4163 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/StarGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/StarGraph.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.graph.Graph;
 import org.apache.flink.graph.drivers.parameter.LongParameter;
-import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
 
@@ -32,8 +31,7 @@
  * Generate a {@link org.apache.flink.graph.generator.StarGraph}.
  */
 public class StarGraph
-extends ParameterizedBase
-implements Input<LongValue, NullValue, NullValue> {
+extends GeneratedGraph<LongValue> {
 
 	private LongParameter vertexCount = new LongParameter(this, "vertex_count")
 		.setMinimumValue(MINIMUM_VERTEX_COUNT);
@@ -48,11 +46,16 @@ public String getName() {
 
 	@Override
 	public String getIdentity() {
-		return getName() + " (" + vertexCount + ")";
+		return getTypeName() + " " + getName() + " (" + vertexCount + ")";
 	}
 
 	@Override
-	public Graph<LongValue, NullValue, NullValue> create(ExecutionEnvironment env) {
+	protected long vertexCount() {
+		return vertexCount.getValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
 		return new org.apache.flink.graph.generator.StarGraph(env, vertexCount.getValue())
 			.setParallelism(littleParallelism.getValue().intValue())
 			.generate();
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/parameter/ChoiceParameter.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/parameter/ChoiceParameter.java
index b239b931582fa..f1b716d911950 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/parameter/ChoiceParameter.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/parameter/ChoiceParameter.java
@@ -18,8 +18,8 @@
 
 package org.apache.flink.graph.drivers.parameter;
 
-import org.apache.commons.lang3.text.StrBuilder;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.text.StrBuilder;
 import org.apache.flink.api.java.utils.ParameterTool;
 import org.apache.flink.client.program.ProgramParametrizationException;
 import org.apache.flink.util.Preconditions;
@@ -37,6 +37,8 @@ public class ChoiceParameter
 
 	private List<String> choices = new ArrayList<>();
 
+	private List<String> hiddenChoices = new ArrayList<>();
+
 	/**
 	 * Set the parameter name and add this parameter to the list of parameters
 	 * stored by owner.
@@ -71,6 +73,18 @@ public ChoiceParameter addChoices(String... choices) {
 		return this;
 	}
 
+	/**
+	 * Add additional hidden choices. This function can be called multiple
+	 * times. These choices will not be printed in the usage string.
+	 *
+	 * @param hiddenChoices additional hidden choices
+	 * @return this
+	 */
+	public ChoiceParameter addHiddenChoices(String... hiddenChoices) {
+		Collections.addAll(this.hiddenChoices, hiddenChoices);
+		return this;
+	}
+
 	@Override
 	public String getUsage() {
 		String option = new StrBuilder()
@@ -107,6 +121,13 @@ public void configure(ParameterTool parameterTool) {
 			}
 		}
 
+		for (String choice : hiddenChoices) {
+			if (choice.equals(selected)) {
+				this.value = selected;
+				return;
+			}
+		}
+
 		throw new ProgramParametrizationException(
 			"Selection '" + selected + "' for option '" + name + "' is not in choices '[" + StringUtils.join(choices, ", ") + "]'");
 	}
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/RunnerITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/RunnerITCase.java
index a48fdf10a9966..f93dc31711184 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/RunnerITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/RunnerITCase.java
@@ -33,8 +33,8 @@ public class RunnerITCase
 	@Rule
 	public ExpectedException thrown = ExpectedException.none();
 
-	public RunnerITCase(TestExecutionMode mode) {
-		super(mode);
+	public RunnerITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
 	}
 
 	@Test
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/AdamicAdarITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/AdamicAdarITCase.java
index 400c24114820b..25482634cce5b 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/AdamicAdarITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/AdamicAdarITCase.java
@@ -18,17 +18,28 @@
 
 package org.apache.flink.graph.drivers;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
 public class AdamicAdarITCase
-extends DriverBaseITCase {
+extends CopyableValueDriverBaseITCase {
 
-	public AdamicAdarITCase(TestExecutionMode mode) {
-		super(mode);
+	public AdamicAdarITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String output, String... additionalParameters) {
+		String[] parameters = new String[] {
+			"--algorithm", "AdamicAdar",
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", "undirected",
+			"--output", output};
+
+		return ArrayUtils.addAll(parameters, additionalParameters);
 	}
 
 	@Test
@@ -42,11 +53,12 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testPrintWithRMatIntegerGraph() throws Exception {
+	public void testPrintWithRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
 		expectedCount(
-			new String[]{"--algorithm", "AdamicAdar",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "print"},
-			221628);
+			parameters(7, "print"),
+			5694);
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ClusteringCoefficientITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ClusteringCoefficientITCase.java
index f215b914279fc..86eee019dcafb 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ClusteringCoefficientITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ClusteringCoefficientITCase.java
@@ -19,16 +19,25 @@
 package org.apache.flink.graph.drivers;
 
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
 public class ClusteringCoefficientITCase
-extends DriverBaseITCase {
+extends CopyableValueDriverBaseITCase {
 
-	public ClusteringCoefficientITCase(TestExecutionMode mode) {
-		super(mode);
+	public ClusteringCoefficientITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String order, String simplify, String output) {
+		return new String[] {
+			"--algorithm", "ClusteringCoefficient", "--order", order,
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", simplify,
+			"--output", output};
 	}
 
 	@Test
@@ -42,48 +51,148 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testDirectedHashWithRMatIntegerGraph() throws Exception {
+	public void testHashWithSmallDirectedRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x0000003621c62ca1L;
+				break;
+
+			case "long":
+				checksum = 0x0000003b74c6719bL;
+				break;
+
+			case "string":
+				checksum = 0x0000003ab67abea8L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
 		String expected = "\n" +
-			"ChecksumHashCode 0x000001c0409df6c0, count 902\n" +
-			"triplet count: 1003442, triangle count: 225147, global clustering coefficient: 0.22437470[0-9]+\n" +
-			"vertex count: 902, average clustering coefficient: 0.32943748[0-9]+\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "ClusteringCoefficient", "--order", "directed",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-				"--output", "hash"},
-			expected);
+			new Checksum(117, checksum) + "\n" +
+			"triplet count: 29286, triangle count: 11466, global clustering coefficient: 0.39151813[0-9]+\n" +
+			"vertex count: 117, average clustering coefficient: 0.45125697[0-9]+\n";
+
+		expectedOutput(parameters(7, "directed", "directed", "hash"), expected);
 	}
 
 	@Test
-	public void testDirectedPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "ClusteringCoefficient", "--order", "directed",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-				"--output", "print"},
-			904);
+	public void testHashWithSmallUndirectedRMatGraph() throws Exception {
+		long directed_checksum;
+		long undirected_checksum;
+		switch (idType) {
+			case "byte":
+			case "short":
+			case "char":
+			case "integer":
+				directed_checksum = 0x0000003875b38c43L;
+				undirected_checksum = 0x0000003c20344c75L;
+				break;
+
+			case "long":
+				directed_checksum = 0x0000003671970c59L;
+				undirected_checksum = 0x0000003939645d8cL;
+				break;
+
+			case "string":
+				directed_checksum = 0x0000003be109a770L;
+				undirected_checksum = 0x0000003b8c98d14aL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		String expected = "\n" +
+			"triplet count: 29286, triangle count: 11466, global clustering coefficient: 0.39151813[0-9]+\n" +
+			"vertex count: 117, average clustering coefficient: 0.57438679[0-9]+\n";
+
+		expectedOutput(parameters(7, "directed", "undirected", "hash"),
+			"\n" + new Checksum(117, directed_checksum) + expected);
+		expectedOutput(parameters(7, "undirected", "undirected", "hash"),
+			"\n" + new Checksum(117, undirected_checksum) + expected);
 	}
 
 	@Test
-	public void testUndirectedHashWithRMatIntegerGraph() throws Exception {
+	public void testHashWithLargeDirectedRMatGraph() throws Exception {
+		// computation is too large for collection mode
+		Assume.assumeFalse(mode == TestExecutionMode.COLLECTION);
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+				return;
+
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x0000067a9d18e7f3L;
+				break;
+
+			case "long":
+				checksum = 0x00000694a90ee6d4L;
+				break;
+
+			case "string":
+				checksum = 0x000006893e3b314fL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
 		String expected = "\n" +
-			"ChecksumHashCode 0x000001ccf8c45fdb, count 902\n" +
-			"triplet count: 1003442, triangle count: 225147, global clustering coefficient: 0.22437470[0-9]+\n" +
-			"vertex count: 902, average clustering coefficient: 0.42173070[0-9]+\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "ClusteringCoefficient", "--order", "undirected",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "hash"},
-			expected);
+			new Checksum(3349, checksum) + "\n" +
+			"triplet count: 9276207, triangle count: 1439454, global clustering coefficient: 0.15517700[0-9]+\n" +
+			"vertex count: 3349, average clustering coefficient: 0.24571815[0-9]+\n";
+
+		expectedOutput(parameters(12, "directed", "directed", "hash"), expected);
 	}
 
 	@Test
-	public void testUndirectedPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "ClusteringCoefficient", "--order", "undirected",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "print"},
-			904);
+	public void testHashWithLargeUndirectedRMatGraph() throws Exception {
+		// computation is too large for collection mode
+		Assume.assumeFalse(mode == TestExecutionMode.COLLECTION);
+
+		long directed_checksum;
+		long undirected_checksum;
+		switch (idType) {
+			case "byte":
+				return;
+
+			case "short":
+			case "char":
+			case "integer":
+				directed_checksum = 0x00000681fad1587eL;
+				undirected_checksum = 0x0000068713b3b7f1L;
+				break;
+
+			case "long":
+				directed_checksum = 0x000006928a6301b1L;
+				undirected_checksum = 0x000006a399edf0e6L;
+				break;
+
+			case "string":
+				directed_checksum = 0x000006749670a2f7L;
+				undirected_checksum = 0x0000067f19c6c4d5L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		String expected = "\n" +
+			"triplet count: 9276207, triangle count: 1439454, global clustering coefficient: 0.15517700[0-9]+\n" +
+			"vertex count: 3349, average clustering coefficient: 0.33029442[0-9]+\n";
+
+		expectedOutput(parameters(12, "directed", "undirected", "hash"),
+			"\n" + new Checksum(3349, directed_checksum) + expected);
+		expectedOutput(parameters(12, "undirected", "undirected", "hash"),
+			"\n" + new Checksum(3349, undirected_checksum) + expected);
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ConnectedComponentsITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ConnectedComponentsITCase.java
index b91abb33231f8..95f0c66b14b6a 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ConnectedComponentsITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/ConnectedComponentsITCase.java
@@ -19,6 +19,8 @@
 package org.apache.flink.graph.drivers;
 
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -27,8 +29,16 @@
 public class ConnectedComponentsITCase
 extends DriverBaseITCase {
 
-	public ConnectedComponentsITCase(TestExecutionMode mode) {
-		super(mode);
+	public ConnectedComponentsITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String output) {
+		return new String[] {
+			"--algorithm", "ConnectedComponents",
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", "undirected",
+				"--edge_factor", "1", "--a", "0.25", "--b", "0.25", "--c", "0.25", "--noise_enabled", "--noise", "1.0",
+			"--output", output};
 	}
 
 	@Test
@@ -42,24 +52,101 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testHashWithRMatIntegerGraph() throws Exception {
-		String expected = "\\nChecksumHashCode 0x0000000000cdc7e7, count 838\\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "ConnectedComponents",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected", "--edge_factor", "1",
-					"--a", "0.25", "--b", "0.25", "--c", "0.25", "--noise_enabled", "--noise", "1.0",
-				"--output", "hash"},
-			expected);
+	public void testHashWithSmallRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x0000000000033e88L;
+				break;
+
+			case "long":
+				checksum = 0x0000000000057848L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x000000000254a4c3L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(parameters(7, "hash"), 106, checksum);
 	}
 
 	@Test
-	public void testPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "ConnectedComponents",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected", "--edge_factor", "1",
-					"--a", "0.25", "--b", "0.25", "--c", "0.25", "--noise_enabled", "--noise", "1.0",
-				"--output", "print"},
-			838);
+	public void testHashWithLargeRMatGraph() throws Exception {
+		// computation is too large for collection mode
+		Assume.assumeFalse(mode == TestExecutionMode.COLLECTION);
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+				return;
+
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x00000003094ffba2L;
+				break;
+
+			case "long":
+				checksum = 0x000000030b68e522L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x00001839ad14edb1L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(parameters(15, "hash"), 25572, checksum);
+	}
+
+	@Test
+	public void testPrintWithSmallRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "integer":
+			case "nativeInteger":
+			case "long":
+			case "nativeLong":
+				checksum = 0x00000024edd0568dL;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x000000232d8bf58dL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedOutputChecksum(parameters(7, "print"), new Checksum(106, checksum));
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/CopyableValueDriverBaseITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/CopyableValueDriverBaseITCase.java
new file mode 100644
index 0000000000000..bc0982055aee2
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/CopyableValueDriverBaseITCase.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.drivers;
+
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Base class for drivers requiring the key ID to implement
+ * {@code CopyableValue}. This class overrides {@link DriverBaseITCase}
+ * to restrict the tested ID types.
+ */
+public abstract class CopyableValueDriverBaseITCase
+extends DriverBaseITCase {
+
+	protected CopyableValueDriverBaseITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	// limit tests to types supporting CopyableValue
+	@Parameterized.Parameters(name = "ID type = {0}, Execution mode = {1}")
+	public static Collection<Object[]> executionModes() {
+		List<Object[]> executionModes = new ArrayList<>();
+
+		for (String idType : new String[] {"byte", "short", "char", "integer", "long", "string"}) {
+			for (TestExecutionMode executionMode : TestExecutionMode.values()) {
+				executionModes.add(new Object[] {idType, executionMode});
+			}
+		}
+
+		return executionModes;
+	}
+}
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/DriverBaseITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/DriverBaseITCase.java
index d19ca97f99140..5b0e42e5ee197 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/DriverBaseITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/DriverBaseITCase.java
@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.flink.graph.Runner;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
 import org.apache.flink.test.util.MultipleProgramsTestBase;
 import org.apache.flink.util.FlinkRuntimeException;
 import org.hamcrest.Description;
@@ -31,27 +32,55 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.regex.Pattern;
 
+/**
+ *
+ */
 public abstract class DriverBaseITCase
 extends MultipleProgramsTestBase {
 
 	@Rule
 	public ExpectedException expectedException = ExpectedException.none();
 
-	protected DriverBaseITCase(TestExecutionMode mode) {
+	protected final String idType;
+
+	protected DriverBaseITCase(String idType, TestExecutionMode mode) {
 		super(mode);
+
+		this.idType = idType;
 	}
 
-	// extend MultipleProgramsTestBase default to include object reuse mode
-	@Parameterized.Parameters(name = "Execution mode = {0}")
+	@Parameterized.Parameters(name = "ID type = {0}, Execution mode = {1}")
 	public static Collection<Object[]> executionModes() {
-		return Arrays.asList(
-			new Object[] { TestExecutionMode.CLUSTER },
-			new Object[] { TestExecutionMode.CLUSTER_OBJECT_REUSE },
-			new Object[] { TestExecutionMode.COLLECTION });
+		List<Object[]> executionModes = new ArrayList<>();
+
+		for (String idType : new String[] {"byte", "nativeByte", "short", "nativeShort", "char", "nativeChar",
+								"integer", "nativeInteger", "long", "nativeLong", "string", "nativeString"}) {
+			for (TestExecutionMode executionMode : TestExecutionMode.values()) {
+				executionModes.add(new Object[] {idType, executionMode});
+			}
+		}
+
+		return executionModes;
+	}
+
+	/**
+	 * Simpler variant of {@link #expectedOutput(String[], String)}
+	 * that converts the {@link Checksum} to a string and ignores
+	 * leading and trailing newlines.
+	 *
+	 * @param parameters algorithm, input, and output arguments
+	 * @param expectedCount expected number of records
+	 * @param expectedChecksum expected checksum over records
+	 * @throws Exception on error
+	 */
+	protected void expectedChecksum(String[] parameters, long expectedCount, long expectedChecksum) throws Exception {
+		Checksum checksum = new Checksum(expectedCount, expectedChecksum);
+		expectedOutput(parameters, "\n*" + checksum.toString() + "\n*");
 	}
 
 	/**
@@ -86,6 +115,33 @@ protected void expectedOutput(String[] parameters, String expected) throws Excep
 		Assert.assertThat(output, RegexMatcher.matchesRegex(expected));
 	}
 
+	/**
+	 * Simpler variant of {@link #expectedOutput(String[], String)}
+	 * that sums the hashCode() of each line of output.
+	 *
+	 * @param parameters algorithm, input, and output arguments
+	 * @param expected expected checksum over lines of output
+	 * @throws Exception on error
+	 */
+	protected void expectedOutputChecksum(String[] parameters, Checksum expected) throws Exception {
+		String output = getSystemOutput(parameters);
+
+		long count = 0;
+		long checksum = 0;
+
+		for (String line : output.split(System.getProperty("line.separator"))) {
+			if (line.length() > 0) {
+				count++;
+
+				// convert 32-bit integer to non-negative long
+				checksum += line.hashCode() & 0xffffffffL;
+			}
+		}
+
+		Assert.assertEquals(expected.getCount(), count);
+		Assert.assertEquals(expected.getChecksum(), checksum);
+	}
+
 	/**
 	 * Executes the driver with the provided arguments and compares the
 	 * exception and exception method with the given class and regular
@@ -96,7 +152,7 @@ protected void expectedOutput(String[] parameters, String expected) throws Excep
 	 * @param exception expected exception
 	 * @throws Exception on error when not matching exception
 	 */
-	protected void expectedOutputFromException(String[] parameters, String expected,Class<? extends Throwable> exception) throws Exception {
+	protected void expectedOutputFromException(String[] parameters, String expected, Class<? extends Throwable> exception) throws Exception {
 		expectedException.expect(exception);
 		expectedException.expectMessage(RegexMatcher.matchesRegex(expected));
 
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java
index d9cac8b9cb1b1..f5662180a5d64 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java
@@ -18,7 +18,10 @@
 
 package org.apache.flink.graph.drivers;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -27,8 +30,17 @@
 public class EdgeListITCase
 extends DriverBaseITCase {
 
-	public EdgeListITCase(TestExecutionMode mode) {
-		super(mode);
+	public EdgeListITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(String input, String output, String... additionalParameters) {
+		String[] parameters = new String[] {
+			"--algorithm", "EdgeList",
+			"--input", input, "--type", idType,
+			"--output", output};
+
+		return ArrayUtils.addAll(parameters, additionalParameters);
 	}
 
 	@Test
@@ -43,198 +55,448 @@ public void testLongDescription() throws Exception {
 
 	@Test
 	public void testHashWithCompleteGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x0000000006788c22, count 1722\n";
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x000000000217bbe2L;
+				break;
+
+			case "long":
+				checksum = 0x0000000006788c22L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x000000007ddfd962L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("CompleteGraph", "hash", "--vertex_count", "42"),
+			1722, checksum);
+	}
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "CompleteGraph", "--vertex_count", "42",
-				"--output", "hash"},
-			expected);
+	@Test
+	public void testPrintWithCompleteGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		expectedOutputChecksum(
+			parameters("CompleteGraph", "print", "--vertex_count", "42"),
+			new Checksum(1722, 0x0000031109a0c398L));
 	}
 
 	@Test
 	public void testHashWithCycleGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x000000000050cea4, count 84\n";
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x00000000001a2224L;
+				break;
+
+			case "long":
+				checksum = 0x000000000050cea4L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x000000000623e524L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("CycleGraph", "hash", "--vertex_count", "42"),
+			84, checksum);
+	}
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "CycleGraph", "--vertex_count", "42",
-				"--output", "hash"},
-			expected);
+	@Test
+	public void testPrintWithCycleGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		expectedOutputChecksum(
+			parameters("CycleGraph", "print", "--vertex_count", "42"),
+			new Checksum(84, 0x000000272a136fcaL));
 	}
 
 	@Test
 	public void testHashWithEmptyGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x0000000000000000, count 0\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "EmptyGraph", "--vertex_count", "42",
-				"--output", "hash"},
-			expected);
+		expectedChecksum(
+			parameters("EmptyGraph", "hash", "--vertex_count", "42"),
+			0, 0x0000000000000000);
 	}
 
 	@Test
 	public void testHashWithGridGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x00000000357d33a6, count 2990\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "GridGraph", "--dim0", "5:true", "--dim1", "8:false", "--dim2", "13:true",
-				"--output", "hash"},
-			expected);
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x00000000001ca34aL;
+				break;
+
+			case "long":
+				checksum = 0x000000000071408aL;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x00000000081ee80aL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("GridGraph", "hash", "--dim0", "2:true", "--dim1", "3:false", "--dim2", "5:true"),
+			130, checksum);
 	}
 
 	@Test
-	public void testHashWithHypercubeGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x0000000014a72800, count 2048\n";
+	public void testPrintWithGridGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "HypercubeGraph", "--dimensions", "8",
-				"--output", "hash"},
-			expected);
+		expectedOutputChecksum(
+			parameters("GridGraph", "print", "--dim0", "2:true", "--dim1", "3:false", "--dim2", "5:true"),
+			new Checksum(130, 0x00000033237d24eeL));
 	}
 
 	@Test
-	public void testHashWithPathGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x00000000004ee21a, count 82\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "PathGraph", "--vertex_count", "42",
-				"--output", "hash"},
-			expected);
+	public void testHashWithHypercubeGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x00000000035df180L;
+				break;
+
+			case "long":
+				checksum = 0x0000000005a52180L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x0000000273474480L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("HypercubeGraph", "hash", "--dimensions", "7"),
+			896, checksum);
 	}
 
 	@Test
-	public void testHashWithRMatIntegerGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x00000000ed469103, count 16384\n";
+	public void testPrintWithHypercubeGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "integer",
-				"--output", "hash"},
-			expected);
+		expectedOutputChecksum(
+			parameters("HypercubeGraph", "print", "--dimensions", "7"),
+			new Checksum(896, 0x000001f243ee33b2L));
 	}
 
 	@Test
-	public void testHashWithRMatIntegerDirectedGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x00000000c53bfc9b, count 12009\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-				"--output", "hash"},
-			expected);
+	public void testHashWithPathGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x00000000001982daL;
+				break;
+
+			case "long":
+				checksum = 0x00000000004ee21aL;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x00000000060a065aL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("PathGraph", "hash", "--vertex_count", "42"),
+			82, checksum);
 	}
 
 	@Test
-	public void testHashWithRMatIntegerUndirectedGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x00000001664eb9e4, count 20884\n";
+	public void testPrintWithPathGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "hash"},
-			expected);
+		expectedOutputChecksum(
+			parameters("PathGraph", "print", "--vertex_count", "42"),
+			new Checksum(82, 0x000000269be2d4c2L));
 	}
 
 	@Test
-	public void testHashWithRMatLongGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x0000000116ee9103, count 16384\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "long",
-				"--output", "hash"},
-			expected);
+	public void testHashWithRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x0000000003bf67f7L;
+				break;
+
+			case "long":
+				checksum = 0x0000000008f467f7L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x00000001660861bdL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("RMatGraph", "hash", "--scale", "7"),
+			2048, checksum);
 	}
 
 	@Test
-	public void testPrintWithRMatLongGraph() throws Exception {
+	public void testPrintWithRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedCount(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "long",
-				"--output", "print"},
-			16384);
+		expectedOutputChecksum(
+			parameters("RMatGraph", "print", "--scale", "7"),
+			new Checksum(2048, 0x000002f737939f05L));
 	}
 
 	@Test
-	public void testHashWithRMatLongDirectedGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x00000000e3c4643b, count 12009\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "long", "--simplify", "directed",
-				"--output", "hash"},
-			expected);
+	public void testHashWithDirectedRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x00000000029aafb3L;
+				break;
+
+			case "long":
+				checksum = 0x000000000592e9b3L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x000000011b079691L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("RMatGraph", "hash", "--scale", "7", "--simplify", "directed"),
+			1168, checksum);
 	}
 
 	@Test
-	public void testHashWithRMatLongUndirectedGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x000000019b67ae64, count 20884\n";
+	public void testPrintWithDirectedRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "long", "--simplify", "undirected",
-				"--output", "hash"},
-			expected);
+		expectedOutputChecksum(
+			parameters("RMatGraph", "print", "--scale", "7", "--simplify", "directed"),
+			new Checksum(1168, 0x0000020e35b0f35dL));
 	}
 
 	@Test
-	public void testHashWithRMatStringGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x00000071dc80a623, count 16384\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "string",
-				"--output", "hash"},
-			expected);
+	public void testHashWithUndirectedRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x0000000004627ab6L;
+				break;
+
+			case "long":
+				checksum = 0x0000000009193576L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x00000001e9adcf56L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("RMatGraph", "hash", "--scale", "7", "--simplify", "undirected"),
+			1854, checksum);
 	}
 
 	@Test
-	public void testHashWithRMatStringDirectedGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x0000005d58b3fa7d, count 12009\n";
+	public void testPrintWithUndirectedRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "string", "--simplify", "directed",
-				"--output", "hash"},
-			expected);
+		expectedOutputChecksum(
+			parameters("RMatGraph", "print", "--scale", "7", "--simplify", "undirected"),
+			new Checksum(1854, 0x0000036fe5802162L));
 	}
 
 	@Test
-	public void testHashWithRMatStringUndirectedGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x000000aa54987304, count 20884\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "RMatGraph", "--type", "string", "--simplify", "undirected",
-				"--output", "hash"},
-			expected);
+	public void testHashWithSingletonEdgeGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x000000000034d5a4L;
+				break;
+
+			case "long":
+				checksum = 0x00000000006b8224L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x000000000757c6a4L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("SingletonEdgeGraph", "hash", "--vertex_pair_count", "42"),
+			84, checksum);
 	}
 
 	@Test
-	public void testHashWithSingletonEdgeGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x0000000001af8ee8, count 200\n";
+	public void testPrintWithSingletonEdgeGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "SingletonEdgeGraph", "--vertex_pair_count", "100",
-				"--output", "hash"},
-			expected);
+		expectedOutputChecksum(
+			parameters("SingletonEdgeGraph", "print", "--vertex_pair_count", "42"),
+			new Checksum(84, 0x0000002e59e10d9aL));
 	}
 
 	@Test
 	public void testHashWithStarGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x000000000042789a, count 82\n";
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x00000000000d195aL;
+				break;
+
+			case "long":
+				checksum = 0x000000000042789aL;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x00000000032f0adaL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("StarGraph", "hash", "--vertex_count", "42"),
+			82, checksum);
+	}
+
+	@Test
+	public void testPrintWithStarGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
 
-		expectedOutput(
-			new String[]{"--algorithm", "EdgeList",
-				"--input", "StarGraph", "--vertex_count", "42",
-				"--output", "hash"},
-			expected);
+		expectedOutputChecksum(
+			parameters("StarGraph", "print", "--vertex_count", "42"),
+			new Checksum(82, 0x00000011ec3faee8L));
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
index a5ea48628d6d4..8c5ed86c94854 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
@@ -18,8 +18,8 @@
 
 package org.apache.flink.graph.drivers;
 
-import org.apache.commons.lang3.ArrayUtils;
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -28,8 +28,15 @@
 public class GraphMetricsITCase
 extends DriverBaseITCase {
 
-	public GraphMetricsITCase(TestExecutionMode mode) {
-		super(mode);
+	public GraphMetricsITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String order, String output) {
+		return new String[] {
+			"--algorithm", "GraphMetrics", "--order", order,
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", order,
+			"--output", output};
 	}
 
 	@Test
@@ -43,58 +50,110 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testWithDirectedRMatIntegerGraph() throws Exception {
+	public void testWithSmallDirectedRMatIntegerGraph() throws Exception {
 		String expected = "\n" +
 			"Vertex metrics:\n" +
-			"  vertex count: 902\n" +
-			"  edge count: 12,009\n" +
-			"  unidirectional edge count: 8,875\n" +
-			"  bidirectional edge count: 1,567\n" +
-			"  average degree: 13.314\n" +
-			"  density: 0.01477663\n" +
-			"  triplet count: 1,003,442\n" +
-			"  maximum degree: 463\n" +
-			"  maximum out degree: 334\n" +
-			"  maximum in degree: 342\n" +
-			"  maximum triplets: 106,953\n" +
+			"  vertex count: 117\n" +
+			"  edge count: 1,168\n" +
+			"  unidirectional edge count: 686\n" +
+			"  bidirectional edge count: 241\n" +
+			"  average degree: 9.983\n" +
+			"  density: 0.08605953\n" +
+			"  triplet count: 29,286\n" +
+			"  maximum degree: 91\n" +
+			"  maximum out degree: 77\n" +
+			"  maximum in degree: 68\n" +
+			"  maximum triplets: 4,095\n" +
 			"\n" +
 			"Edge metrics:\n" +
-			"  triangle triplet count: 107,817\n" +
-			"  rectangle triplet count: 315,537\n" +
-			"  maximum triangle triplets: 820\n" +
-			"  maximum rectangle triplets: 3,822\n";
+			"  triangle triplet count: 4,575\n" +
+			"  rectangle triplet count: 11,756\n" +
+			"  maximum triangle triplets: 153\n" +
+			"  maximum rectangle triplets: 391\n";
+
+		expectedOutput(parameters(7, "directed", "hash"), expected);
+		expectedOutput(parameters(7, "directed", "print"), expected);
+	}
+
+	@Test
+	public void testWithLargeDirectedRMatIntegerGraph() throws Exception {
+		// skip 'byte' which cannot store vertex IDs for scale > 8
+		Assume.assumeFalse(idType.equals("byte") || idType.equals("nativeByte"));
+
+		// skip 'string' which does not compare numerically and generates a different triangle count
+		Assume.assumeFalse(idType.equals("string") || idType.equals("nativeString"));
 
-		String[] arguments = new String[]{"--algorithm", "GraphMetrics", "--order", "directed",
-			"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-			"--output"};
+		String expected = "\n" +
+			"Vertex metrics:\n" +
+			"  vertex count: 3,349\n" +
+			"  edge count: 53,368\n" +
+			"  unidirectional edge count: 43,602\n" +
+			"  bidirectional edge count: 4,883\n" +
+			"  average degree: 15.936\n" +
+			"  density: 0.00475971\n" +
+			"  triplet count: 9,276,207\n" +
+			"  maximum degree: 1,356\n" +
+			"  maximum out degree: 921\n" +
+			"  maximum in degree: 966\n" +
+			"  maximum triplets: 918,690\n" +
+			"\n" +
+			"Edge metrics:\n" +
+			"  triangle triplet count: 779,202\n" +
+			"  rectangle triplet count: 2,506,371\n" +
+			"  maximum triangle triplets: 3,160\n" +
+			"  maximum rectangle triplets: 16,835\n";
 
-		expectedOutput(ArrayUtils.addAll(arguments, "hash"), expected);
-		expectedOutput(ArrayUtils.addAll(arguments, "print"), expected);
+		expectedOutput(parameters(12, "directed", "hash"), expected);
+		expectedOutput(parameters(12, "directed", "print"), expected);
 	}
 
 	@Test
-	public void testWithUndirectedRMatIntegerGraph() throws Exception {
+	public void testWithSmallUndirectedRMatIntegerGraph() throws Exception {
 		String expected = "\n" +
 			"Vertex metrics:\n" +
-			"  vertex count: 902\n" +
-			"  edge count: 10,442\n" +
-			"  average degree: 23.153\n" +
-			"  density: 0.025697\n" +
-			"  triplet count: 1,003,442\n" +
-			"  maximum degree: 463\n" +
-			"  maximum triplets: 106,953\n" +
+			"  vertex count: 117\n" +
+			"  edge count: 927\n" +
+			"  average degree: 15.846\n" +
+			"  density: 0.13660477\n" +
+			"  triplet count: 29,286\n" +
+			"  maximum degree: 91\n" +
+			"  maximum triplets: 4,095\n" +
 			"\n" +
 			"Edge metrics:\n" +
-			"  triangle triplet count: 107,817\n" +
-			"  rectangle triplet count: 315,537\n" +
-			"  maximum triangle triplets: 820\n" +
-			"  maximum rectangle triplets: 3,822\n";
+			"  triangle triplet count: 4,575\n" +
+			"  rectangle triplet count: 11,756\n" +
+			"  maximum triangle triplets: 153\n" +
+			"  maximum rectangle triplets: 391\n";
+
+		expectedOutput(parameters(7, "undirected", "hash"), expected);
+		expectedOutput(parameters(7, "undirected", "print"), expected);
+	}
+
+	@Test
+	public void testWithLargelUndirectedRMatIntegerGraph() throws Exception {
+		// skip 'byte' which cannot store vertex IDs for scale > 8
+		Assume.assumeFalse(idType.equals("byte") || idType.equals("nativeByte"));
 
-		String[] arguments = new String[]{"--algorithm", "GraphMetrics", "--order", "undirected",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output"};
+		// skip 'string' which does not compare numerically and generates a different triangle count
+		Assume.assumeFalse(idType.equals("string") || idType.equals("nativeString"));
+
+		String expected = "\n" +
+			"Vertex metrics:\n" +
+			"  vertex count: 3,349\n" +
+			"  edge count: 48,485\n" +
+			"  average degree: 28.955\n" +
+			"  density: 0.00864842\n" +
+			"  triplet count: 9,276,207\n" +
+			"  maximum degree: 1,356\n" +
+			"  maximum triplets: 918,690\n" +
+			"\n" +
+			"Edge metrics:\n" +
+			"  triangle triplet count: 779,202\n" +
+			"  rectangle triplet count: 2,506,371\n" +
+			"  maximum triangle triplets: 3,160\n" +
+			"  maximum rectangle triplets: 16,835\n";
 
-		expectedOutput(ArrayUtils.addAll(arguments, "hash"), expected);
-		expectedOutput(ArrayUtils.addAll(arguments, "print"), expected);
+		expectedOutput(parameters(12, "undirected", "hash"), expected);
+		expectedOutput(parameters(12, "undirected", "print"), expected);
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/HITSITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/HITSITCase.java
index 5474d1b423fff..282d3d57fdda4 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/HITSITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/HITSITCase.java
@@ -19,6 +19,7 @@
 package org.apache.flink.graph.drivers;
 
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -27,8 +28,15 @@
 public class HITSITCase
 extends DriverBaseITCase {
 
-	public HITSITCase(TestExecutionMode mode) {
-		super(mode);
+	public HITSITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String output) {
+		return new String[] {
+			"--algorithm", "HITS",
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", "directed",
+			"--output", output};
 	}
 
 	@Test
@@ -42,11 +50,21 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "HITS",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-				"--output", "print"},
-			902);
+	public void testPrintWithSmallRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		expectedCount(parameters(8, "print"), 233);
+	}
+
+	@Test
+	public void testPrintWithLargeRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		// skip 'byte' which cannot store vertex IDs for scale > 8
+		Assume.assumeFalse(idType.equals("byte") || idType.equals("nativeByte"));
+
+		expectedCount(parameters(12, "print"), 3349);
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/JaccardIndexITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/JaccardIndexITCase.java
index 063285601c7a9..0391771174385 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/JaccardIndexITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/JaccardIndexITCase.java
@@ -18,17 +18,29 @@
 
 package org.apache.flink.graph.drivers;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
 public class JaccardIndexITCase
-extends DriverBaseITCase {
+extends CopyableValueDriverBaseITCase {
 
-	public JaccardIndexITCase(TestExecutionMode mode) {
-		super(mode);
+	public JaccardIndexITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String output, String... additionalParameters) {
+		String[] parameters = new String[] {
+			"--algorithm", "JaccardIndex", "--mirror_results",
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", "undirected",
+			"--output", output};
+
+		return ArrayUtils.addAll(parameters, additionalParameters);
 	}
 
 	@Test
@@ -42,22 +54,67 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testHashWithRMatIntegerGraph() throws Exception {
-		String expected = "\nChecksumHashCode 0x0001b188570b2572, count 221628\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "JaccardIndex",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "hash"},
-			expected);
+	public void testHashWithSmallRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x0000164757052eebL;
+				break;
+
+			case "long":
+				checksum = 0x000016337a6a7270L;
+				break;
+
+			case "string":
+				checksum = 0x00001622a522a290L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(parameters(7, "hash"), 11388, checksum);
 	}
 
 	@Test
-	public void testPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "JaccardIndex",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "print"},
-			221628);
+	public void testHashWithLargeRMatGraph() throws Exception {
+		// computation is too large for collection mode
+		Assume.assumeFalse(mode == TestExecutionMode.COLLECTION);
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+				return;
+
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x0021ce158d811c4eL;
+				break;
+
+			case "long":
+				checksum = 0x0021d20fb3904720L;
+				break;
+
+			case "string":
+				checksum = 0x0021cd8fafec1524L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(parameters(12, "hash"), 4432058, checksum);
+	}
+
+	@Test
+	public void testPrintWithSmallRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		expectedOutputChecksum(parameters(7, "print"), new Checksum(11388, 0x0000163b17088256L));
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/PageRankITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/PageRankITCase.java
index d7301d0aeec45..4ca0a85a88e65 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/PageRankITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/PageRankITCase.java
@@ -19,6 +19,7 @@
 package org.apache.flink.graph.drivers;
 
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -27,8 +28,15 @@
 public class PageRankITCase
 extends DriverBaseITCase {
 
-	public PageRankITCase(TestExecutionMode mode) {
-		super(mode);
+	public PageRankITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String output) {
+		return new String[] {
+			"--algorithm", "PageRank",
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", "directed",
+			"--output", output};
 	}
 
 	@Test
@@ -42,11 +50,21 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "PageRank",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-				"--output", "print"},
-			902);
+	public void testPrintWithSmallRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		expectedCount(parameters(8, "print"), 233);
+	}
+
+	@Test
+	public void testPrintWithLargeRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		// skip 'byte' which cannot store vertex IDs for scale > 8
+		Assume.assumeFalse(idType.equals("byte") || idType.equals("nativeByte"));
+
+		expectedCount(parameters(12, "print"), 3349);
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/TriangleListingITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/TriangleListingITCase.java
index 0d2897c5f0672..fabdae1a08928 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/TriangleListingITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/TriangleListingITCase.java
@@ -18,17 +18,33 @@
 
 package org.apache.flink.graph.drivers;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
 public class TriangleListingITCase
-extends DriverBaseITCase {
+extends CopyableValueDriverBaseITCase {
 
-	public TriangleListingITCase(TestExecutionMode mode) {
-		super(mode);
+	public TriangleListingITCase(String idType, TestExecutionMode mode) {
+		super(idType, mode);
+	}
+
+	private String[] parameters(int scale, String order, String output) {
+		String[] parameters =  new String[] {
+			"--algorithm", "TriangleListing", "--order", order,
+			"--input", "RMatGraph", "--scale", Integer.toString(scale), "--type", idType, "--simplify", order,
+			"--output", output};
+
+		if (output.equals("hash")) {
+			return ArrayUtils.addAll(parameters, "--sort_triangle_vertices", "--triadic_census");
+		} else {
+			return parameters;
+		}
 	}
 
 	@Test
@@ -42,66 +58,224 @@ public void testLongDescription() throws Exception {
 	}
 
 	@Test
-	public void testDirectedHashWithRMatIntegerGraph() throws Exception {
+	public void testHashWithSmallDirectedRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x000000003d2f0a9aL;
+				break;
+
+			case "long":
+				checksum = 0x000000016aba3720L;
+				break;
+
+			case "string":
+				checksum = 0x0000005bfef84facL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
 		String expected = "\n" +
-			"ChecksumHashCode 0x0000001beffe6edd, count 75049\n" +
+			new Checksum(3822, checksum) + "\n" +
 			"Triadic census:\n" +
-			"  003: 113,435,893\n" +
-			"  012: 6,632,528\n" +
-			"  102: 983,535\n" +
-			"  021d: 118,574\n" +
-			"  021u: 118,566\n" +
-			"  021c: 237,767\n" +
-			"  111d: 129,773\n" +
-			"  111u: 130,041\n" +
-			"  030t: 16,981\n" +
-			"  030c: 5,535\n" +
-			"  201: 43,574\n" +
-			"  120d: 7,449\n" +
-			"  120u: 7,587\n" +
-			"  120c: 15,178\n" +
-			"  210: 17,368\n" +
-			"  300: 4,951\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "TriangleListing", "--order", "directed", "--sort_triangle_vertices", "--triadic_census",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-				"--output", "hash"},
-			expected);
+			"  003: 178,989\n" +
+			"  012: 47,736\n" +
+			"  102: 11,763\n" +
+			"  021d: 2,258\n" +
+			"  021u: 2,064\n" +
+			"  021c: 4,426\n" +
+			"  111d: 3,359\n" +
+			"  111u: 3,747\n" +
+			"  030t: 624\n" +
+			"  030c: 220\n" +
+			"  201: 1,966\n" +
+			"  120d: 352\n" +
+			"  120u: 394\n" +
+			"  120c: 704\n" +
+			"  210: 1,120\n" +
+			"  300: 408\n";
+
+		expectedOutput(parameters(7, "directed", "hash"), expected);
 	}
 
 	@Test
-	public void testDirectedPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "TriangleListing", "--order", "directed",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "directed",
-				"--output", "print"},
-			75049);
+	public void testHashWithSmallUndirectedRMatGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x0000000001f92b0cL;
+				break;
+
+			case "long":
+				checksum = 0x000000000bb355c6L;
+				break;
+
+			case "string":
+				checksum = 0x00000002f7b5576aL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		String expected = "\n" +
+			new Checksum(3822, checksum) + "\n" +
+			"Triadic census:\n" +
+			"  03: 178,989\n" +
+			"  12: 59,499\n" +
+			"  21: 17,820\n" +
+			"  30: 3,822\n";
+
+		expectedOutput(parameters(7, "undirected", "hash"), expected);
 	}
 
 	@Test
-	public void testUndirectedHashWithRMatIntegerGraph() throws Exception {
+	public void testHashWithLargeDirectedRMatGraph() throws Exception {
+		// computation is too large for collection mode
+		Assume.assumeFalse(mode == TestExecutionMode.COLLECTION);
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+				return;
+
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x00000248fef26209L;
+				break;
+
+			case "long":
+				checksum = 0x000002dcdf0fbb1bL;
+				break;
+
+			case "string":
+				checksum = 0x00035b760ab9da74L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
 		String expected = "\n" +
-			"ChecksumHashCode 0x00000000e6b3f32c, count 75049\n" +
+			new Checksum(479818, checksum) + "\n" +
 			"Triadic census:\n" +
-			"  03: 113,435,893\n" +
-			"  12: 7,616,063\n" +
-			"  21: 778,295\n" +
-			"  30: 75,049\n";
-
-		expectedOutput(
-			new String[]{"--algorithm", "TriangleListing", "--order", "undirected", "--sort_triangle_vertices", "--triadic_census",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "hash"},
-			expected);
+			"  003: 6,101,196,568\n" +
+			"  012: 132,051,207\n" +
+			"  102: 13,115,128\n" +
+			"  021d: 1,330,423\n" +
+			"  021u: 1,336,897\n" +
+			"  021c: 2,669,285\n" +
+			"  111d: 1,112,144\n" +
+			"  111u: 1,097,452\n" +
+			"  030t: 132,048\n" +
+			"  030c: 44,127\n" +
+			"  201: 290,552\n" +
+			"  120d: 47,734\n" +
+			"  120u: 47,780\n" +
+			"  120c: 95,855\n" +
+			"  210: 90,618\n" +
+			"  300: 21,656\n";
+
+		expectedOutput(parameters(12, "directed", "hash"), expected);
 	}
 
 	@Test
-	public void testUndirectedPrintWithRMatIntegerGraph() throws Exception {
-		expectedCount(
-			new String[]{"--algorithm", "TriangleListing", "--order", "undirected",
-				"--input", "RMatGraph", "--type", "integer", "--simplify", "undirected",
-				"--output", "print"},
-			75049);
+	public void testHashWithLargeUndirectedRMatGraph() throws Exception {
+		// computation is too large for collection mode
+		Assume.assumeFalse(mode == TestExecutionMode.COLLECTION);
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+				return;
+
+			case "short":
+			case "char":
+			case "integer":
+				checksum = 0x00000012dee4bf2cL;
+				break;
+
+			case "long":
+				checksum = 0x00000017a40efbdaL;
+				break;
+
+			case "string":
+				checksum = 0x000159e8be3e370bL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		String expected = "\n" +
+			new Checksum(479818, checksum) + "\n" +
+			"Triadic census:\n" +
+			"  03: 6,101,196,568\n" +
+			"  12: 145,166,335\n" +
+			"  21: 7,836,753\n" +
+			"  30: 479,818\n";
+
+		expectedOutput(parameters(12, "undirected", "hash"), expected);
+	}
+
+
+	@Test
+	public void testPrintWithSmallDirectedRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "short":
+			case "integer":
+			case "long":
+				checksum = 0x00000764227995aaL;
+				break;
+
+			case "string":
+				checksum = 0x000007643d93c30aL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedOutputChecksum(parameters(7, "directed", "print"), new Checksum(3822, checksum));
+	}
+
+
+	@Test
+	public void testPrintWithSmallUndirectedRMatGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "short":
+			case "integer":
+			case "long":
+				checksum = 0x0000077d1582e206L;
+				break;
+
+			case "string":
+				checksum = 0x0000077a49cb5268L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedOutputChecksum(parameters(7, "undirected", "print"), new Checksum(3822, checksum));
 	}
 }
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/input/GeneratedGraphTest.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/input/GeneratedGraphTest.java
new file mode 100644
index 0000000000000..da77b0d4f87c0
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/input/GeneratedGraphTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.drivers.input;
+
+import org.apache.flink.graph.asm.translate.TranslateFunction;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToChar;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToCharValue;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToString;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToUnsignedByte;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToUnsignedByteValue;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToUnsignedInt;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToLong;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToUnsignedShort;
+import org.apache.flink.graph.drivers.input.GeneratedGraph.LongValueToUnsignedShortValue;
+import org.apache.flink.types.ByteValue;
+import org.apache.flink.types.CharValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.ShortValue;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class GeneratedGraphTest {
+
+	private TranslateFunction<LongValue, ByteValue> byteValueTranslator = new LongValueToUnsignedByteValue();
+	private TranslateFunction<LongValue, Byte> byteTranslator = new LongValueToUnsignedByte();
+	private TranslateFunction<LongValue, ShortValue> shortValueTranslator = new LongValueToUnsignedShortValue();
+	private TranslateFunction<LongValue, Short> shortTranslator = new LongValueToUnsignedShort();
+	private TranslateFunction<LongValue, CharValue> charValueTranslator = new LongValueToCharValue();
+	private TranslateFunction<LongValue, Character> charTranslator = new LongValueToChar();
+	private TranslateFunction<LongValue, Integer> intTranslator = new LongValueToUnsignedInt();
+	private TranslateFunction<LongValue, Long> longTranslator = new LongValueToLong();
+	private TranslateFunction<LongValue, String> stringTranslator = new LongValueToString();
+
+	private ByteValue byteValue = new ByteValue();
+	private ShortValue shortValue = new ShortValue();
+	private CharValue charValue = new CharValue();
+
+	// ByteValue
+
+	@Test
+	public void testByteValueTranslation() throws Exception {
+		assertEquals(new ByteValue((byte) 0), byteValueTranslator.translate(new LongValue(0L), byteValue));
+		assertEquals(new ByteValue(Byte.MIN_VALUE), byteValueTranslator.translate(new LongValue((long) Byte.MAX_VALUE + 1), byteValue));
+		assertEquals(new ByteValue((byte) -1), byteValueTranslator.translate(new LongValue((1L << 8) - 1), byteValue));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testByteValueTranslationUpperOutOfRange() throws Exception {
+		byteValueTranslator.translate(new LongValue(1L << 8), byteValue);
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testByteValueTranslationLowerOutOfRange() throws Exception {
+		byteValueTranslator.translate(new LongValue(-1), byteValue);
+	}
+
+	// Byte
+
+	@Test
+	public void testByteTranslation() throws Exception {
+		assertEquals(Byte.valueOf((byte) 0), byteTranslator.translate(new LongValue(0L), null));
+		assertEquals(Byte.valueOf(Byte.MIN_VALUE), byteTranslator.translate(new LongValue((long) Byte.MAX_VALUE + 1), null));
+		assertEquals(Byte.valueOf((byte) -1), byteTranslator.translate(new LongValue((1L << 8) - 1), null));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testByteTranslationUpperOutOfRange() throws Exception {
+		byteTranslator.translate(new LongValue(1L << 8), null);
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testByteTranslationLowerOutOfRange() throws Exception {
+		byteTranslator.translate(new LongValue(-1), null);
+	}
+
+	// ShortValue
+
+	@Test
+	public void testShortValueTranslation() throws Exception {
+		assertEquals(new ShortValue((short) 0), shortValueTranslator.translate(new LongValue(0L), shortValue));
+		assertEquals(new ShortValue(Short.MIN_VALUE), shortValueTranslator.translate(new LongValue((long) Short.MAX_VALUE + 1), shortValue));
+		assertEquals(new ShortValue((short) -1), shortValueTranslator.translate(new LongValue((1L << 16) - 1), shortValue));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testShortValueTranslationUpperOutOfRange() throws Exception {
+		shortValueTranslator.translate(new LongValue(1L << 16), shortValue);
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testShortValueTranslationLowerOutOfRange() throws Exception {
+		shortValueTranslator.translate(new LongValue(-1), shortValue);
+	}
+
+	// Short
+
+	@Test
+	public void testShortTranslation() throws Exception {
+		assertEquals(Short.valueOf((short) 0), shortTranslator.translate(new LongValue(0L), null));
+		assertEquals(Short.valueOf(Short.MIN_VALUE), shortTranslator.translate(new LongValue((long) Short.MAX_VALUE + 1), null));
+		assertEquals(Short.valueOf((short) -1), shortTranslator.translate(new LongValue((1L << 16) - 1), null));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testShortTranslationUpperOutOfRange() throws Exception {
+		shortTranslator.translate(new LongValue(1L << 16), null);
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testShortTranslationLowerOutOfRange() throws Exception {
+		shortTranslator.translate(new LongValue(-1), null);
+	}
+
+	// CharValue
+
+	@Test
+	public void testCharValueTranslation() throws Exception {
+		assertEquals(new CharValue((char) 0), charValueTranslator.translate(new LongValue(0L), charValue));
+		assertEquals(new CharValue(Character.MAX_VALUE), charValueTranslator.translate(new LongValue((long) Character.MAX_VALUE), charValue));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testCharValueTranslationUpperOutOfRange() throws Exception {
+		charValueTranslator.translate(new LongValue(1L << 16), charValue);
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testCharValueTranslationLowerOutOfRange() throws Exception {
+		charValueTranslator.translate(new LongValue(-1), charValue);
+	}
+
+	// Character
+
+	@Test
+	public void testCharacterTranslation() throws Exception {
+		assertEquals(Character.valueOf((char) 0), charTranslator.translate(new LongValue(0L), null));
+		assertEquals(Character.valueOf(Character.MAX_VALUE), charTranslator.translate(new LongValue((long) Character.MAX_VALUE), null));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testCharacterTranslationUpperOutOfRange() throws Exception {
+		charTranslator.translate(new LongValue(1L << 16), null);
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testCharacterTranslationLowerOutOfRange() throws Exception {
+		charTranslator.translate(new LongValue(-1), null);
+	}
+
+	// Integer
+
+	@Test
+	public void testIntegerTranslation() throws Exception {
+		assertEquals(Integer.valueOf(0), intTranslator.translate(new LongValue(0L), null));
+		assertEquals(Integer.valueOf(Integer.MIN_VALUE), intTranslator.translate(new LongValue((long) Integer.MAX_VALUE + 1), null));
+		assertEquals(Integer.valueOf(-1), intTranslator.translate(new LongValue((1L << 32) - 1), null));
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testIntegerTranslationUpperOutOfRange() throws Exception {
+		intTranslator.translate(new LongValue(1L << 32), null);
+	}
+
+	@Test(expected=IllegalArgumentException.class)
+	public void testIntegerTranslationLowerOutOfRange() throws Exception {
+		intTranslator.translate(new LongValue(-1), null);
+	}
+
+	// Long
+
+	@Test
+	public void testLongTranslation() throws Exception {
+		assertEquals(Long.valueOf(0L), longTranslator.translate(new LongValue(0L), null));
+		assertEquals(Long.valueOf(Long.MIN_VALUE), longTranslator.translate(new LongValue(Long.MIN_VALUE), null));
+		assertEquals(Long.valueOf(Long.MAX_VALUE), longTranslator.translate(new LongValue(Long.MAX_VALUE), null));
+	}
+
+	// String
+
+	@Test
+	public void testStringTranslation() throws Exception {
+		assertEquals("0", stringTranslator.translate(new LongValue(0L), null));
+		assertEquals("-9223372036854775808", stringTranslator.translate(new LongValue(Long.MIN_VALUE), null));
+		assertEquals("9223372036854775807", stringTranslator.translate(new LongValue(Long.MAX_VALUE), null));
+	}
+}
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValue.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValue.java
index 038a2e48691ed..30a74df0823cd 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValue.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValue.java
@@ -31,6 +31,8 @@
 public class LongValueToSignedIntValue
 implements TranslateFunction<LongValue, IntValue> {
 
+	public static final long MAX_VERTEX_COUNT = 1L << 32;
+
 	@Override
 	public IntValue translate(LongValue value, IntValue reuse)
 			throws Exception {
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValue.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValue.java
index 8fe665c90066f..741bd6272ba77 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValue.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValue.java
@@ -30,6 +30,8 @@
 public class LongValueToUnsignedIntValue
 implements TranslateFunction<LongValue, IntValue> {
 
+	public static final long MAX_VERTEX_COUNT = 1L << 32;
+
 	@Override
 	public IntValue translate(LongValue value, IntValue reuse)
 			throws Exception {
@@ -39,10 +41,10 @@ public IntValue translate(LongValue value, IntValue reuse)
 
 		long l = value.getValue();
 
-		if (l < 0 || l >= (1L << 32)) {
+		if (l < 0 || l >= MAX_VERTEX_COUNT) {
 			throw new IllegalArgumentException("Cannot cast long value " + value + " to integer.");
 		} else {
-			reuse.setValue((int)(l & 0xffffffffL));
+			reuse.setValue((int) (l & (MAX_VERTEX_COUNT - 1)));
 		}
 
 		return reuse;
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
index dfa7eb267aef5..84cb791957dac 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
@@ -80,17 +80,16 @@ public Graph<LongValue, NullValue, NullValue> generate() {
 	}
 
 	@ForwardedFields("*->f0")
-	public class LinkVertexToAll
+	private static class LinkVertexToAll
 	implements FlatMapFunction<LongValue, Edge<LongValue, NullValue>> {
-
 		private final long vertexCount;
 
 		private LongValue target = new LongValue();
 
 		private Edge<LongValue, NullValue> edge = new Edge<>(null, target, NullValue.getInstance());
 
-		public LinkVertexToAll(long vertex_count) {
-			this.vertexCount = vertex_count;
+		public LinkVertexToAll(long vertexCount) {
+			this.vertexCount = vertexCount;
 		}
 
 		@Override
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
index 0ca804e02d814..16a9ab94ead02 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
@@ -107,7 +107,7 @@ public Graph<LongValue, NullValue, NullValue> generate() {
 	}
 
 	@ForwardedFields("*->f0")
-	public class LinkVertexToNeighbors
+	private static class LinkVertexToNeighbors
 	implements FlatMapFunction<LongValue, Edge<LongValue, NullValue>> {
 
 		private long vertexCount;
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java
index 95a4f85ef4a31..0fa4127d22efe 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java
@@ -166,7 +166,7 @@ public Graph<LongValue, NullValue, NullValue> generate() {
 		return Graph.fromDataSet(vertices, edges, env);
 	}
 
-	private static final class GenerateEdges<T extends RandomGenerator>
+	private static class GenerateEdges<T extends RandomGenerator>
 	implements FlatMapFunction<BlockInfo<T>, Edge<LongValue, NullValue>> {
 
 		// Configuration
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java
index 6c7c433d83234..316d749ee9671 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java
@@ -80,7 +80,7 @@ public Graph<LongValue, NullValue, NullValue> generate() {
 	}
 
 	@ForwardedFields("*->f0")
-	public class LinkVertexToCenter
+	private static class LinkVertexToCenter
 	implements FlatMapFunction<LongValue, Edge<LongValue, NullValue>> {
 
 		private LongValue center = new LongValue(0);
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/HITS.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/HITS.java
index 4857add235166..ba1ab210bb7f8 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/HITS.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/link_analysis/HITS.java
@@ -60,8 +60,6 @@
  * <p>
  * http://www.cs.cornell.edu/home/kleinber/auth.pdf
  *
- * http://www.cs.cornell.edu/home/kleinber/auth.pdf
- *
  * @param <K> graph ID type
  * @param <VV> vertex value type
  * @param <EV> edge value type
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java
index bc3cb86c20658..b3e69f1b5f04f 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/similarity/JaccardIndex.java
@@ -80,6 +80,8 @@ public class JaccardIndex<K extends CopyableValue<K>, VV, EV>
 
 	private int maximumScoreDenominator = 1;
 
+	private boolean mirrorResults;
+
 	private int littleParallelism = PARALLELISM_DEFAULT;
 
 	/**
@@ -140,6 +142,20 @@ public JaccardIndex<K, VV, EV> setMaximumScore(int numerator, int denominator) {
 		return this;
 	}
 
+	/**
+	 * By default only one result is output for each pair of vertices. When
+	 * mirroring a second result with the vertex order flipped is output for
+	 * each pair of vertices.
+	 *
+	 * @param mirrorResults whether output results should be mirrored
+	 * @return this
+	 */
+	public JaccardIndex<K, VV, EV> setMirrorResults(boolean mirrorResults) {
+		this.mirrorResults = mirrorResults;
+
+		return this;
+	}
+
 	/**
 	 * Override the parallelism of operators processing small amounts of data.
 	 *
@@ -176,7 +192,8 @@ protected boolean mergeConfiguration(GraphAlgorithmWrappingDataSet other) {
 			minimumScoreNumerator != rhs.minimumScoreNumerator ||
 			minimumScoreDenominator != rhs.minimumScoreDenominator ||
 			maximumScoreNumerator != rhs.maximumScoreNumerator ||
-			maximumScoreDenominator != rhs.maximumScoreDenominator) {
+			maximumScoreDenominator != rhs.maximumScoreDenominator ||
+			mirrorResults != rhs.mirrorResults) {
 			return false;
 		}
 
@@ -230,12 +247,20 @@ public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
 				.name("Generate group pairs");
 
 		// t, u, intersection, union
-		return twoPaths
+		DataSet<Result<K>> scores = twoPaths
 			.groupBy(0, 1)
 			.reduceGroup(new ComputeScores<K>(unboundedScores,
 					minimumScoreNumerator, minimumScoreDenominator,
 					maximumScoreNumerator, maximumScoreDenominator))
 				.name("Compute scores");
+
+		if (mirrorResults) {
+			scores = scores
+				.flatMap(new MirrorResult<K, Result<K>>())
+					.name("Mirror results");
+		}
+
+		return scores;
 	}
 
 	/**
@@ -450,6 +475,27 @@ public void reduce(Iterable<Tuple3<T, T, IntValue>> values, Collector<Result<T>>
 		}
 	}
 
+	/**
+	 * Output each input and a second result with the vertex order flipped.
+	 *
+	 * @param <T> ID type
+	 * @param <RT> result type
+	 */
+	private static class MirrorResult<T, RT extends BinaryResult<T>>
+	implements FlatMapFunction<RT, RT> {
+		@Override
+		public void flatMap(RT value, Collector<RT> out)
+				throws Exception {
+			out.collect(value);
+
+			T tmp = value.getVertexId0();
+			value.setVertexId0(value.getVertexId1());
+			value.setVertexId1(tmp);
+
+			out.collect(value);
+		}
+	}
+
 	/**
 	 * Wraps the vertex type to encapsulate results from the jaccard index algorithm.
 	 *
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/AsmTestBase.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/AsmTestBase.java
index 18110384ebb16..469a23f79ef69 100644
--- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/AsmTestBase.java
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/AsmTestBase.java
@@ -60,7 +60,7 @@ public void setup()
 		env = ExecutionEnvironment.createCollectionsEnvironment();
 		env.getConfig().enableObjectReuse();
 
-		// the "fish" graph
+		// a "fish" graph
 		Object[][] edges = new Object[][]{
 			new Object[]{0, 1},
 			new Object[]{0, 2},
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValueTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValueTest.java
index 976cc34317d49..f730adf561525 100644
--- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValueTest.java
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToSignedIntValueTest.java
@@ -40,11 +40,11 @@ public void testTranslation() throws Exception {
 
 	@Test(expected=IllegalArgumentException.class)
 	public void testUpperOutOfRange() throws Exception {
-		assertEquals(new IntValue(), translator.translate(new LongValue((long)Integer.MAX_VALUE + 1), reuse));
+		translator.translate(new LongValue((long)Integer.MAX_VALUE + 1), reuse);
 	}
 
 	@Test(expected=IllegalArgumentException.class)
 	public void testLowerOutOfRange() throws Exception {
-		assertEquals(new IntValue(), translator.translate(new LongValue((long)Integer.MIN_VALUE - 1), reuse));
+		translator.translate(new LongValue((long)Integer.MIN_VALUE - 1), reuse);
 	}
 }
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValueTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValueTest.java
index 6a2ae83500b0c..ca70162d3032e 100644
--- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValueTest.java
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/asm/translate/translators/LongValueToUnsignedIntValueTest.java
@@ -34,17 +34,17 @@ public class LongValueToUnsignedIntValueTest {
 	@Test
 	public void testTranslation() throws Exception {
 		assertEquals(new IntValue(0), translator.translate(new LongValue(0L), reuse));
-		assertEquals(new IntValue(Integer.MIN_VALUE), translator.translate(new LongValue((long)Integer.MAX_VALUE + 1), reuse));
+		assertEquals(new IntValue(Integer.MIN_VALUE), translator.translate(new LongValue((long) Integer.MAX_VALUE + 1), reuse));
 		assertEquals(new IntValue(-1), translator.translate(new LongValue((1L << 32) - 1), reuse));
 	}
 
 	@Test(expected=IllegalArgumentException.class)
 	public void testUpperOutOfRange() throws Exception {
-		assertEquals(new IntValue(), translator.translate(new LongValue(1L << 32), reuse));
+		translator.translate(new LongValue(1L << 32), reuse);
 	}
 
 	@Test(expected=IllegalArgumentException.class)
 	public void testLowerOutOfRange() throws Exception {
-		assertEquals(new IntValue(), translator.translate(new LongValue(-1), reuse));
+		translator.translate(new LongValue(-1), reuse);
 	}
 }

From 441be5484ec9c204d922f99ec12de019f9ac164e Mon Sep 17 00:00:00 2001
From: Zhijiang <wangzhijiang999@aliyun.com>
Date: Wed, 26 Apr 2017 16:18:54 +0800
Subject: [PATCH 102/253] [FLINK-6337][network] Remove the buffer provider from
 PartitionRequestServerHandler

The buffer provider is not needed and most likely a left over from prior
refactorings.

This closes #3785.
---
 .../runtime/io/network/ConnectionManager.java |  4 +--
 .../io/network/LocalConnectionManager.java    |  5 +--
 .../io/network/NetworkEnvironment.java        |  2 +-
 .../network/netty/NettyConnectionManager.java |  5 ++-
 .../netty/PartitionRequestProtocol.java       |  7 ++---
 .../netty/PartitionRequestServerHandler.java  | 19 ++----------
 .../netty/SequenceNumberingViewReader.java    |  5 +--
 .../partition/PipelinedSubpartition.java      |  3 +-
 .../io/network/partition/ResultPartition.java |  4 +--
 .../partition/ResultPartitionManager.java     |  4 +--
 .../partition/ResultPartitionProvider.java    |  3 --
 .../network/partition/ResultSubpartition.java |  3 +-
 .../partition/SpillableSubpartition.java      |  7 ++---
 .../partition/consumer/LocalInputChannel.java |  2 +-
 .../netty/CancelPartitionRequestTest.java     | 13 ++++----
 .../ClientTransportErrorHandlingTest.java     | 10 ++----
 .../netty/NettyConnectionManagerTest.java     |  7 ++---
 .../netty/PartitionRequestQueueTest.java      |  5 +--
 .../ServerTransportErrorHandlingTest.java     |  9 ++----
 .../partition/InputChannelTestUtils.java      |  7 ++---
 .../partition/PipelinedSubpartitionTest.java  |  8 ++---
 .../partition/SpillableSubpartitionTest.java  | 31 ++++++++-----------
 .../partition/SubpartitionTestBase.java       |  5 +--
 .../consumer/LocalInputChannelTest.java       |  8 ++---
 .../consumer/SingleInputGateTest.java         |  8 ++---
 ...ccessAfterNetworkBuffersFailureITCase.java |  2 +-
 26 files changed, 62 insertions(+), 124 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ConnectionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ConnectionManager.java
index 02deb9d2ff8c2..1225230f25433 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ConnectionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ConnectionManager.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.runtime.io.network;
 
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.netty.PartitionRequestClient;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionProvider;
 
@@ -31,8 +30,7 @@
 public interface ConnectionManager {
 
 	void start(ResultPartitionProvider partitionProvider,
-				TaskEventDispatcher taskEventDispatcher,
-				NetworkBufferPool networkbufferPool) throws IOException;
+				TaskEventDispatcher taskEventDispatcher) throws IOException;
 
 	/**
 	 * Creates a {@link PartitionRequestClient} instance for the given {@link ConnectionID}.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java
index 4f51a56ae2ee2..bece6a0044d7d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.runtime.io.network;
 
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.netty.PartitionRequestClient;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionProvider;
 
@@ -29,9 +28,7 @@
 public class LocalConnectionManager implements ConnectionManager {
 
 	@Override
-	public void start(ResultPartitionProvider partitionProvider,
-						TaskEventDispatcher taskEventDispatcher,
-						NetworkBufferPool networkbufferPool) {
+	public void start(ResultPartitionProvider partitionProvider, TaskEventDispatcher taskEventDispatcher) {
 	}
 
 	@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
index 4d4b3056e2740..cc4cb77b0be98 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
@@ -287,7 +287,7 @@ public void start() throws IOException {
 
 			try {
 				LOG.debug("Starting network connection manager");
-				connectionManager.start(resultPartitionManager, taskEventDispatcher, networkBufferPool);
+				connectionManager.start(resultPartitionManager, taskEventDispatcher);
 			}
 			catch (IOException t) {
 				throw new IOException("Failed to instantiate network connection manager.", t);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java
index abee2a8fb1b02..fcf618a1982a1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java
@@ -21,7 +21,6 @@
 import org.apache.flink.runtime.io.network.ConnectionID;
 import org.apache.flink.runtime.io.network.ConnectionManager;
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionProvider;
 
 import java.io.IOException;
@@ -45,10 +44,10 @@ public NettyConnectionManager(NettyConfig nettyConfig) {
 	}
 
 	@Override
-	public void start(ResultPartitionProvider partitionProvider, TaskEventDispatcher taskEventDispatcher, NetworkBufferPool networkbufferPool)
+	public void start(ResultPartitionProvider partitionProvider, TaskEventDispatcher taskEventDispatcher)
 			throws IOException {
 		PartitionRequestProtocol partitionRequestProtocol =
-				new PartitionRequestProtocol(partitionProvider, taskEventDispatcher, networkbufferPool);
+				new PartitionRequestProtocol(partitionProvider, taskEventDispatcher);
 
 		client.init(partitionRequestProtocol, bufferPool);
 		server.init(partitionRequestProtocol, bufferPool);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestProtocol.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestProtocol.java
index a39f08510c34e..d06a0187e17cc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestProtocol.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestProtocol.java
@@ -20,7 +20,6 @@
 
 import io.netty.channel.ChannelHandler;
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionProvider;
 
 import static org.apache.flink.runtime.io.network.netty.NettyMessage.NettyMessageEncoder;
@@ -34,12 +33,10 @@ class PartitionRequestProtocol implements NettyProtocol {
 
 	private final ResultPartitionProvider partitionProvider;
 	private final TaskEventDispatcher taskEventDispatcher;
-	private final NetworkBufferPool networkbufferPool;
 
-	PartitionRequestProtocol(ResultPartitionProvider partitionProvider, TaskEventDispatcher taskEventDispatcher, NetworkBufferPool networkbufferPool) {
+	PartitionRequestProtocol(ResultPartitionProvider partitionProvider, TaskEventDispatcher taskEventDispatcher) {
 		this.partitionProvider = partitionProvider;
 		this.taskEventDispatcher = taskEventDispatcher;
-		this.networkbufferPool = networkbufferPool;
 	}
 
 	// +-------------------------------------------------------------------+
@@ -77,7 +74,7 @@ class PartitionRequestProtocol implements NettyProtocol {
 	public ChannelHandler[] getServerChannelHandlers() {
 		PartitionRequestQueue queueOfPartitionQueues = new PartitionRequestQueue();
 		PartitionRequestServerHandler serverHandler = new PartitionRequestServerHandler(
-				partitionProvider, taskEventDispatcher, queueOfPartitionQueues, networkbufferPool);
+				partitionProvider, taskEventDispatcher, queueOfPartitionQueues);
 
 		return new ChannelHandler[] {
 				messageEncoder,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
index 6f568773d9ce2..1bd05f6f0d22e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
@@ -21,8 +21,6 @@
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.SimpleChannelInboundHandler;
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.BufferPool;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.netty.NettyMessage.CancelPartitionRequest;
 import org.apache.flink.runtime.io.network.netty.NettyMessage.CloseRequest;
 import org.apache.flink.runtime.io.network.partition.PartitionNotFoundException;
@@ -47,36 +45,24 @@ class PartitionRequestServerHandler extends SimpleChannelInboundHandler<NettyMes
 
 	private final PartitionRequestQueue outboundQueue;
 
-	private final NetworkBufferPool networkBufferPool;
-
-	private BufferPool bufferPool;
-
 	PartitionRequestServerHandler(
 		ResultPartitionProvider partitionProvider,
 		TaskEventDispatcher taskEventDispatcher,
-		PartitionRequestQueue outboundQueue,
-		NetworkBufferPool networkBufferPool) {
+		PartitionRequestQueue outboundQueue) {
 
 		this.partitionProvider = partitionProvider;
 		this.taskEventDispatcher = taskEventDispatcher;
 		this.outboundQueue = outboundQueue;
-		this.networkBufferPool = networkBufferPool;
 	}
 
 	@Override
 	public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
 		super.channelRegistered(ctx);
-
-		bufferPool = networkBufferPool.createBufferPool(1, Integer.MAX_VALUE);
 	}
 
 	@Override
 	public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
 		super.channelUnregistered(ctx);
-
-		if (bufferPool != null) {
-			bufferPool.lazyDestroy();
-		}
 	}
 
 	@Override
@@ -100,8 +86,7 @@ protected void channelRead0(ChannelHandlerContext ctx, NettyMessage msg) throws
 					reader.requestSubpartitionView(
 						partitionProvider,
 						request.partitionId,
-						request.queueIndex,
-						bufferPool);
+						request.queueIndex);
 				} catch (PartitionNotFoundException notFound) {
 					respondWithError(ctx, notFound, request.receiverId);
 				}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java
index 5036bb77ca54c..6d95ca5040364 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java
@@ -19,7 +19,6 @@
 package org.apache.flink.runtime.io.network.netty;
 
 import org.apache.flink.runtime.io.network.buffer.Buffer;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 import org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionProvider;
@@ -60,8 +59,7 @@ class SequenceNumberingViewReader implements BufferAvailabilityListener {
 	void requestSubpartitionView(
 		ResultPartitionProvider partitionProvider,
 		ResultPartitionID resultPartitionId,
-		int subPartitionIndex,
-		BufferProvider bufferProvider) throws IOException {
+		int subPartitionIndex) throws IOException {
 
 		synchronized (requestLock) {
 			if (subpartitionView == null) {
@@ -72,7 +70,6 @@ void requestSubpartitionView(
 				this.subpartitionView = partitionProvider.createSubpartitionView(
 					resultPartitionId,
 					subPartitionIndex,
-					bufferProvider,
 					this);
 			} else {
 				throw new IllegalStateException("Subpartition already requested");
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java
index 4fd74e2ea6ef8..ed72b51ad5292 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java
@@ -21,7 +21,6 @@
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
 import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -163,7 +162,7 @@ public boolean isReleased() {
 	}
 
 	@Override
-	public PipelinedSubpartitionView createReadView(BufferProvider bufferProvider, BufferAvailabilityListener availabilityListener) throws IOException {
+	public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException {
 		final int queueSize;
 
 		synchronized (buffers) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
index d207f60137a61..9b02e4d603c30 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
@@ -362,7 +362,7 @@ public void destroyBufferPool() {
 	/**
 	 * Returns the requested subpartition.
 	 */
-	public ResultSubpartitionView createSubpartitionView(int index, BufferProvider bufferProvider, BufferAvailabilityListener availabilityListener) throws IOException {
+	public ResultSubpartitionView createSubpartitionView(int index, BufferAvailabilityListener availabilityListener) throws IOException {
 		int refCnt = pendingReferences.get();
 
 		checkState(refCnt != -1, "Partition released.");
@@ -370,7 +370,7 @@ public ResultSubpartitionView createSubpartitionView(int index, BufferProvider b
 
 		checkElementIndex(index, subpartitions.length, "Subpartition not found.");
 
-		ResultSubpartitionView readView = subpartitions[index].createReadView(bufferProvider, availabilityListener);
+		ResultSubpartitionView readView = subpartitions[index].createReadView(availabilityListener);
 
 		LOG.debug("Created {}", readView);
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionManager.java
index 8ad3e342e0f36..f681548263491 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionManager.java
@@ -22,7 +22,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Table;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,7 +65,6 @@ public void registerResultPartition(ResultPartition partition) throws IOExceptio
 	public ResultSubpartitionView createSubpartitionView(
 			ResultPartitionID partitionId,
 			int subpartitionIndex,
-			BufferProvider bufferProvider,
 			BufferAvailabilityListener availabilityListener) throws IOException {
 
 		synchronized (registeredPartitions) {
@@ -79,7 +77,7 @@ public ResultSubpartitionView createSubpartitionView(
 
 			LOG.debug("Requesting subpartition {} of {}.", subpartitionIndex, partition);
 
-			return partition.createSubpartitionView(subpartitionIndex, bufferProvider, availabilityListener);
+			return partition.createSubpartitionView(subpartitionIndex, availabilityListener);
 		}
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionProvider.java
index 3fbfd49f5cc49..db72d63cf8eed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionProvider.java
@@ -18,8 +18,6 @@
 
 package org.apache.flink.runtime.io.network.partition;
 
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
-
 import java.io.IOException;
 
 public interface ResultPartitionProvider {
@@ -30,7 +28,6 @@ public interface ResultPartitionProvider {
 	ResultSubpartitionView createSubpartitionView(
 			ResultPartitionID partitionId,
 			int index,
-			BufferProvider bufferProvider,
 			BufferAvailabilityListener availabilityListener) throws IOException;
 
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java
index d3cd887ca559b..3b4e3c9b1a39b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartition.java
@@ -19,7 +19,6 @@
 package org.apache.flink.runtime.io.network.partition;
 
 import org.apache.flink.runtime.io.network.buffer.Buffer;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 
 import java.io.IOException;
 
@@ -77,7 +76,7 @@ protected Throwable getFailureCause() {
 
 	abstract public void release() throws IOException;
 
-	abstract public ResultSubpartitionView createReadView(BufferProvider bufferProvider, BufferAvailabilityListener availabilityListener) throws IOException;
+	abstract public ResultSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException;
 
 	abstract int releaseMemory() throws IOException;
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
index a5781880b51ef..11c6d1647ddc6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
@@ -25,7 +25,6 @@
 import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.buffer.BufferPool;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -164,7 +163,7 @@ public void release() throws IOException {
 	}
 
 	@Override
-	public ResultSubpartitionView createReadView(BufferProvider bufferProvider, BufferAvailabilityListener availabilityListener) throws IOException {
+	public ResultSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException {
 		synchronized (buffers) {
 			if (!isFinished) {
 				throw new IllegalStateException("Subpartition has not been finished yet, " +
@@ -180,7 +179,7 @@ public ResultSubpartitionView createReadView(BufferProvider bufferProvider, Buff
 			if (spillWriter != null) {
 				readView = new SpilledSubpartitionView(
 					this,
-					bufferProvider.getMemorySegmentSize(),
+					parent.getBufferProvider().getMemorySegmentSize(),
 					spillWriter,
 					getTotalNumberOfBuffers(),
 					availabilityListener);
@@ -189,7 +188,7 @@ public ResultSubpartitionView createReadView(BufferProvider bufferProvider, Buff
 					this,
 					buffers,
 					ioManager,
-					bufferProvider.getMemorySegmentSize(),
+					parent.getBufferProvider().getMemorySegmentSize(),
 					availabilityListener);
 			}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java
index 4e14e93d46c12..3ade2f8326953 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java
@@ -113,7 +113,7 @@ void requestSubpartition(int subpartitionIndex) throws IOException, InterruptedE
 
 				try {
 					ResultSubpartitionView subpartitionView = partitionManager.createSubpartitionView(
-						partitionId, subpartitionIndex, inputGate.getBufferProvider(), this);
+						partitionId, subpartitionIndex, this);
 
 					if (subpartitionView == null) {
 						throw new IOException("Error requesting subpartition.");
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java
index a2f866a191671..8b4fc0e72566a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java
@@ -22,7 +22,6 @@
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.buffer.BufferProvider;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.netty.NettyTestUtil.NettyServerAndClient;
 import org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
@@ -78,18 +77,18 @@ public void testCancelPartitionRequest() throws Exception {
 			final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
 
 			// Return infinite subpartition
-			when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferProvider.class), any(BufferAvailabilityListener.class)))
+			when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferAvailabilityListener.class)))
 				.thenAnswer(new Answer<ResultSubpartitionView>() {
 					@Override
 					public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
-						BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[3];
+						BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
 						listener.notifyBuffersAvailable(Long.MAX_VALUE);
 						return view;
 					}
 				});
 
 			PartitionRequestProtocol protocol = new PartitionRequestProtocol(
-					partitions, mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class));
+					partitions, mock(TaskEventDispatcher.class));
 
 			serverAndClient = initServerAndClient(protocol);
 
@@ -129,18 +128,18 @@ public void testDuplicateCancel() throws Exception {
 			final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
 
 			// Return infinite subpartition
-			when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferProvider.class), any(BufferAvailabilityListener.class)))
+			when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferAvailabilityListener.class)))
 					.thenAnswer(new Answer<ResultSubpartitionView>() {
 						@Override
 						public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
-							BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[3];
+							BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
 							listener.notifyBuffersAvailable(Long.MAX_VALUE);
 							return view;
 						}
 					});
 
 			PartitionRequestProtocol protocol = new PartitionRequestProtocol(
-					partitions, mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class));
+					partitions, mock(TaskEventDispatcher.class));
 
 			serverAndClient = initServerAndClient(protocol);
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ClientTransportErrorHandlingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ClientTransportErrorHandlingTest.java
index 22e7754fcb0b0..b4fc46fbdbe3e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ClientTransportErrorHandlingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ClientTransportErrorHandlingTest.java
@@ -29,7 +29,6 @@
 import io.netty.channel.embedded.EmbeddedChannel;
 import org.apache.flink.runtime.io.network.ConnectionID;
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.netty.NettyTestUtil.NettyServerAndClient;
 import org.apache.flink.runtime.io.network.netty.exception.LocalTransportException;
 import org.apache.flink.runtime.io.network.netty.exception.RemoteTransportException;
@@ -85,8 +84,7 @@ public ChannelHandler[] getServerChannelHandlers() {
 			public ChannelHandler[] getClientChannelHandlers() {
 				return new PartitionRequestProtocol(
 						mock(ResultPartitionProvider.class),
-						mock(TaskEventDispatcher.class),
-						mock(NetworkBufferPool.class)).getClientChannelHandlers();
+						mock(TaskEventDispatcher.class)).getClientChannelHandlers();
 			}
 		};
 
@@ -235,8 +233,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg)
 			public ChannelHandler[] getClientChannelHandlers() {
 				return new PartitionRequestProtocol(
 						mock(ResultPartitionProvider.class),
-						mock(TaskEventDispatcher.class),
-						mock(NetworkBufferPool.class)).getClientChannelHandlers();
+						mock(TaskEventDispatcher.class)).getClientChannelHandlers();
 			}
 		};
 
@@ -383,8 +380,7 @@ public void testChannelClosedOnExceptionDuringErrorNotification() throws Excepti
 	private EmbeddedChannel createEmbeddedChannel() {
 		PartitionRequestProtocol protocol = new PartitionRequestProtocol(
 				mock(ResultPartitionProvider.class),
-				mock(TaskEventDispatcher.class),
-				mock(NetworkBufferPool.class));
+				mock(TaskEventDispatcher.class));
 
 		return new EmbeddedChannel(protocol.getClientChannelHandlers());
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
index 8ab572eabe175..77de6bf34b895 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
@@ -23,7 +23,6 @@
 import io.netty.channel.EventLoopGroup;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionProvider;
 import org.apache.flink.util.NetUtils;
 import org.junit.Test;
@@ -60,8 +59,7 @@ public void testMatchingNumberOfArenasAndThreadsAsDefault() throws Exception {
 
 		connectionManager.start(
 				mock(ResultPartitionProvider.class),
-				mock(TaskEventDispatcher.class),
-				mock(NetworkBufferPool.class));
+				mock(TaskEventDispatcher.class));
 
 		assertEquals(numberOfSlots, connectionManager.getBufferPool().getNumberOfArenas());
 
@@ -129,8 +127,7 @@ public void testManualConfiguration() throws Exception {
 
 		connectionManager.start(
 				mock(ResultPartitionProvider.class),
-				mock(TaskEventDispatcher.class),
-				mock(NetworkBufferPool.class));
+				mock(TaskEventDispatcher.class));
 
 		assertEquals(numberOfArenas, connectionManager.getBufferPool().getNumberOfArenas());
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java
index 7224e96e373cf..b969b1c51e308 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java
@@ -20,7 +20,6 @@
 
 import io.netty.channel.embedded.EmbeddedChannel;
 import org.apache.flink.runtime.execution.CancelTaskException;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 import org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionProvider;
@@ -43,7 +42,6 @@ public void testProducerFailedException() throws Exception {
 
 		ResultPartitionProvider partitionProvider = mock(ResultPartitionProvider.class);
 		ResultPartitionID rpid = new ResultPartitionID();
-		BufferProvider bufferProvider = mock(BufferProvider.class);
 
 		ResultSubpartitionView view = mock(ResultSubpartitionView.class);
 		when(view.isReleased()).thenReturn(true);
@@ -52,13 +50,12 @@ public void testProducerFailedException() throws Exception {
 		when(partitionProvider.createSubpartitionView(
 			eq(rpid),
 			eq(0),
-			eq(bufferProvider),
 			any(BufferAvailabilityListener.class))).thenReturn(view);
 
 		EmbeddedChannel ch = new EmbeddedChannel(queue);
 
 		SequenceNumberingViewReader seqView = new SequenceNumberingViewReader(new InputChannelID(), queue);
-		seqView.requestSubpartitionView(partitionProvider, rpid, 0, bufferProvider);
+		seqView.requestSubpartitionView(partitionProvider, rpid, 0);
 
 		// Enqueue the erroneous view
 		queue.notifyReaderNonEmpty(seqView);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java
index 1c3557ea10a9d..3c4ebb3359cc6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java
@@ -23,8 +23,6 @@
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
 import org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
@@ -63,11 +61,11 @@ public void testRemoteClose() throws Exception {
 		final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
 
 		when(partitionManager
-			.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class)))
+			.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class)))
 			.thenAnswer(new Answer<ResultSubpartitionView>() {
 				@Override
 				public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
-					BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[3];
+					BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
 					listener.notifyBuffersAvailable(Long.MAX_VALUE);
 					return new CancelPartitionRequestTest.InfiniteSubpartitionView(outboundBuffers, sync);
 				}
@@ -78,8 +76,7 @@ public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws T
 			public ChannelHandler[] getServerChannelHandlers() {
 				return new PartitionRequestProtocol(
 					partitionManager,
-					mock(TaskEventDispatcher.class),
-					mock(NetworkBufferPool.class)).getServerChannelHandlers();
+					mock(TaskEventDispatcher.class)).getServerChannelHandlers();
 			}
 
 			@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputChannelTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputChannelTestUtils.java
index e292576111c1d..a32783899ec78 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputChannelTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputChannelTestUtils.java
@@ -21,7 +21,6 @@
 import org.apache.flink.runtime.io.network.ConnectionID;
 import org.apache.flink.runtime.io.network.ConnectionManager;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 import org.apache.flink.runtime.io.network.netty.PartitionRequestClient;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
@@ -60,14 +59,14 @@ public static ResultPartitionManager createResultPartitionManager(final ResultSu
 
 			@Override
 			public ResultSubpartitionView answer(InvocationOnMock invocation) throws Throwable {
-				BufferAvailabilityListener channel = (BufferAvailabilityListener) invocation.getArguments()[3];
-				return sources[num++].createReadView(null, channel);
+				BufferAvailabilityListener channel = (BufferAvailabilityListener) invocation.getArguments()[2];
+				return sources[num++].createReadView(channel);
 			}
 		};
 
 		ResultPartitionManager manager = mock(ResultPartitionManager.class);
 		when(manager.createSubpartitionView(
-				any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class)))
+				any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class)))
 				.thenAnswer(viewCreator);
 
 		return manager;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java
index e3200d1c3307b..de1e8a07109d1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java
@@ -70,14 +70,14 @@ public void testIllegalReadViewRequest() throws Exception {
 		final PipelinedSubpartition subpartition = createSubpartition();
 
 		// Successful request
-		assertNotNull(subpartition.createReadView(null, new BufferAvailabilityListener() {
+		assertNotNull(subpartition.createReadView(new BufferAvailabilityListener() {
 			@Override
 			public void notifyBuffersAvailable(long numBuffers) {
 			}
 		}));
 
 		try {
-			subpartition.createReadView(null, new BufferAvailabilityListener() {
+			subpartition.createReadView(new BufferAvailabilityListener() {
 				@Override
 				public void notifyBuffersAvailable(long numBuffers) {
 				}
@@ -94,7 +94,7 @@ public void testBasicPipelinedProduceConsumeLogic() throws Exception {
 
 		BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
 
-		ResultSubpartitionView view = subpartition.createReadView(null, listener);
+		ResultSubpartitionView view = subpartition.createReadView(listener);
 
 		// Empty => should return null
 		assertNull(view.getNextBuffer());
@@ -221,7 +221,7 @@ public void onEvent(AbstractEvent event) {
 		final PipelinedSubpartition subpartition = createSubpartition();
 
 		TestSubpartitionConsumer consumer = new TestSubpartitionConsumer(isSlowConsumer, consumerCallback);
-		final PipelinedSubpartitionView view = subpartition.createReadView(null, consumer);
+		final PipelinedSubpartitionView view = subpartition.createReadView(consumer);
 		consumer.setSubpartitionView(view);
 
 		Future<Boolean> producerResult = executorService.submit(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java
index b53ef683161c0..2b356a8f49986 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java
@@ -26,8 +26,8 @@
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
 import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
+import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler;
-import org.apache.flink.runtime.io.network.util.TestInfiniteBufferProvider;
 import org.junit.AfterClass;
 import org.junit.Test;
 import org.mockito.invocation.InvocationOnMock;
@@ -67,8 +67,12 @@ public static void shutdown() {
 	}
 
 	@Override
-	ResultSubpartition createSubpartition() {
-		return new SpillableSubpartition(0, mock(ResultPartition.class), ioManager);
+	SpillableSubpartition createSubpartition() {
+		ResultPartition parent = mock(ResultPartition.class);
+		BufferProvider bufferProvider = mock(BufferProvider.class);
+		when(parent.getBufferProvider()).thenReturn(bufferProvider);
+		when(bufferProvider.getMemorySegmentSize()).thenReturn(32 * 1024);
+		return new SpillableSubpartition(0, parent, ioManager);
 	}
 
 	/**
@@ -138,14 +142,13 @@ public Void call() throws Exception {
 	@Test
 	public void testReleasePartitionAndGetNext() throws Exception {
 		// Create partition and add some buffers
-		SpillableSubpartition partition = new SpillableSubpartition(
-			0, mock(ResultPartition.class), ioManager);
+		SpillableSubpartition partition = createSubpartition();
 
 		partition.finish();
 
 		// Create the read view
 		ResultSubpartitionView readView = spy(partition
-			.createReadView(new TestInfiniteBufferProvider(), new BufferAvailabilityListener() {
+			.createReadView(new BufferAvailabilityListener() {
 				@Override
 				public void notifyBuffersAvailable(long numBuffers) {
 
@@ -168,11 +171,7 @@ public void notifyBuffersAvailable(long numBuffers) {
 	 */
 	@Test
 	public void testConsumeSpilledPartition() throws Exception {
-		ResultPartition parent = mock(ResultPartition.class);
-		SpillableSubpartition partition = new SpillableSubpartition(
-			0,
-			parent,
-			ioManager);
+		SpillableSubpartition partition = createSubpartition();
 
 		Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(4096), FreeingBufferRecycler.INSTANCE);
 		buffer.retain();
@@ -187,7 +186,7 @@ public void testConsumeSpilledPartition() throws Exception {
 		partition.finish();
 
 		BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
-		SpilledSubpartitionView reader = (SpilledSubpartitionView) partition.createReadView(new TestInfiniteBufferProvider(), listener);
+		SpilledSubpartitionView reader = (SpilledSubpartitionView) partition.createReadView(listener);
 
 		verify(listener, times(1)).notifyBuffersAvailable(eq(4L));
 
@@ -216,11 +215,7 @@ public void testConsumeSpilledPartition() throws Exception {
 	 */
 	@Test
 	public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception {
-		ResultPartition parent = mock(ResultPartition.class);
-		SpillableSubpartition partition = new SpillableSubpartition(
-			0,
-			parent,
-			ioManager);
+		SpillableSubpartition partition = createSubpartition();
 
 		Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(4096), FreeingBufferRecycler.INSTANCE);
 		buffer.retain();
@@ -232,7 +227,7 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception
 		partition.finish();
 
 		AwaitableBufferAvailablityListener listener = new AwaitableBufferAvailablityListener();
-		SpillableSubpartitionView reader = (SpillableSubpartitionView) partition.createReadView(new TestInfiniteBufferProvider(), listener);
+		SpillableSubpartitionView reader = (SpillableSubpartitionView) partition.createReadView(listener);
 
 		// Initial notification
 		assertEquals(1, listener.getNumNotifiedBuffers());
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
index 14942bc484102..800542e08633e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
@@ -20,7 +20,6 @@
 
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.util.TestBufferFactory;
-import org.apache.flink.runtime.io.network.util.TestInfiniteBufferProvider;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
@@ -91,11 +90,9 @@ private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition)
 		partition.add(buffer);
 		partition.finish();
 
-		TestInfiniteBufferProvider buffers = new TestInfiniteBufferProvider();
-
 		// Create the view
 		BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
-		ResultSubpartitionView view = partition.createReadView(buffers, listener);
+		ResultSubpartitionView view = partition.createReadView(listener);
 
 		// The added buffer and end-of-partition event
 		assertNotNull(view.getNextBuffer());
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
index 18c3038792a20..fe819a419b749 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
@@ -198,7 +198,7 @@ public void testPartitionRequestExponentialBackoff() throws Exception {
 		LocalInputChannel ch = createLocalInputChannel(inputGate, partitionManager, backoff);
 
 		when(partitionManager
-				.createSubpartitionView(eq(ch.partitionId), eq(0), eq(bufferProvider), any(BufferAvailabilityListener.class)))
+				.createSubpartitionView(eq(ch.partitionId), eq(0), any(BufferAvailabilityListener.class)))
 				.thenThrow(new PartitionNotFoundException(ch.partitionId));
 
 		Timer timer = mock(Timer.class);
@@ -214,7 +214,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
 		// Initial request
 		ch.requestSubpartition(0);
 		verify(partitionManager)
-				.createSubpartitionView(eq(ch.partitionId), eq(0), eq(bufferProvider), any(BufferAvailabilityListener.class));
+				.createSubpartitionView(eq(ch.partitionId), eq(0), any(BufferAvailabilityListener.class));
 
 		// Request subpartition and verify that the actual requests are delayed.
 		for (long expected : expectedDelays) {
@@ -241,7 +241,7 @@ public void testProducerFailedException() throws Exception {
 
 		ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
 		when(partitionManager
-				.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class)))
+				.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class)))
 				.thenReturn(view);
 
 		SingleInputGate inputGate = mock(SingleInputGate.class);
@@ -296,7 +296,6 @@ public void testConcurrentReleaseAndRetriggerPartitionRequest() throws Exception
 			.createSubpartitionView(
 				any(ResultPartitionID.class),
 				anyInt(),
-				any(BufferProvider.class),
 				any(BufferAvailabilityListener.class)))
 			.thenAnswer(new Answer<ResultSubpartitionView>() {
 				@Override
@@ -360,7 +359,6 @@ public void testGetNextAfterPartitionReleased() throws Exception {
 		when(partitionManager.createSubpartitionView(
 				any(ResultPartitionID.class),
 				anyInt(),
-				any(BufferProvider.class),
 				any(BufferAvailabilityListener.class))).thenReturn(reader);
 
 		LocalInputChannel channel = new LocalInputChannel(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
index 2d1b4b244d1c5..737f17b8ca34c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
@@ -32,7 +32,6 @@
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.buffer.BufferPool;
-import org.apache.flink.runtime.io.network.buffer.BufferProvider;
 import org.apache.flink.runtime.io.network.buffer.BufferRecycler;
 import org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
@@ -128,7 +127,6 @@ public void testBackwardsEventWithUninitializedChannel() throws Exception {
 		when(partitionManager.createSubpartitionView(
 			any(ResultPartitionID.class),
 			anyInt(),
-			any(BufferProvider.class),
 			any(BufferAvailabilityListener.class))).thenReturn(iterator);
 
 		// Setup reader with one local and one unknown input channel
@@ -163,7 +161,7 @@ public void testBackwardsEventWithUninitializedChannel() throws Exception {
 		inputGate.requestPartitions();
 
 		// Only the local channel can request
-		verify(partitionManager, times(1)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class));
+		verify(partitionManager, times(1)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class));
 
 		// Send event backwards and initialize unknown channel afterwards
 		final TaskEvent event = new TestTaskEvent();
@@ -175,7 +173,7 @@ public void testBackwardsEventWithUninitializedChannel() throws Exception {
 		// After the update, the pending event should be send to local channel
 		inputGate.updateInputChannel(new InputChannelDeploymentDescriptor(new ResultPartitionID(unknownPartitionId.getPartitionId(), unknownPartitionId.getProducerId()), ResultPartitionLocation.createLocal()));
 
-		verify(partitionManager, times(2)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class));
+		verify(partitionManager, times(2)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class));
 		verify(taskEventDispatcher, times(2)).publish(any(ResultPartitionID.class), any(TaskEvent.class));
 	}
 
@@ -215,7 +213,7 @@ public void testUpdateChannelBeforeRequest() throws Exception {
 			ResultPartitionLocation.createLocal()));
 
 		verify(partitionManager, never()).createSubpartitionView(
-			any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class));
+			any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class));
 	}
 
 	/**
diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
index 9729b0905db7c..710e7df4dd5bc 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
@@ -52,7 +52,7 @@ public void testSuccessfulProgramAfterFailure() {
 			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
 			config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 80L);
 			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
-			config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 840);
+			config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 640);
 			
 			cluster = new LocalFlinkMiniCluster(config, false);
 

From 97d6af8549208531de399e04b623acc82ec277dc Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Fri, 28 Apr 2017 17:17:30 +0200
Subject: [PATCH 103/253] [FLINK-6409] [table] TUMBLE/HOP/SESSION_START/END do
 not resolve time field correctly

This closes #3799.
---
 .../calcite/sql/fun/SqlGroupFunction.java     |  103 +
 .../calcite/sql/fun/SqlStdOperatorTable.java  | 2133 +++++++
 .../calcite/sql/validate/AggChecker.java      |  225 +
 .../calcite/sql2rel/AuxiliaryConverter.java   |   79 +
 .../calcite/sql2rel/SqlToRelConverter.java    | 5356 +++++++++++++++++
 .../scala/batch/sql/WindowAggregateTest.scala |   31 +
 tools/maven/suppressions.xml                  |    2 +
 7 files changed, 7929 insertions(+)
 create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java
 create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
 create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
 create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/AuxiliaryConverter.java
 create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java

diff --git a/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java
new file mode 100644
index 0000000000000..a57cf104c17f2
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.fun;
+
+/*
+ * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT UNTIL CALCITE-1761 IS FIXED.
+ */
+
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlOperandTypeChecker;
+import org.apache.calcite.sql.validate.SqlMonotonicity;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * SQL function that computes keys by which rows can be partitioned and
+ * aggregated.
+ *
+ * <p>Grouped window functions always occur in the GROUP BY clause. They often
+ * have auxiliary functions that access information about the group. For
+ * example, {@code HOP} is a group function, and its auxiliary functions are
+ * {@code HOP_START} and {@code HOP_END}. Here they are used in a streaming
+ * query:
+ *
+ * <blockquote><pre>
+ * SELECT STREAM HOP_START(rowtime, INTERVAL '1' HOUR),
+ *   HOP_END(rowtime, INTERVAL '1' HOUR),
+ *   MIN(unitPrice)
+ * FROM Orders
+ * GROUP BY HOP(rowtime, INTERVAL '1' HOUR), productId
+ * </pre></blockquote>
+ */
+class SqlGroupFunction extends SqlFunction {
+	/** The grouped function, if this an auxiliary function; null otherwise. */
+	final SqlGroupFunction groupFunction;
+
+	/** Creates a SqlGroupFunction.
+	 *
+	 * @param kind Kind; also determines function name
+	 * @param groupFunction Group function, if this is an auxiliary;
+	 *                      null, if this is a group function
+	 * @param operandTypeChecker Operand type checker
+	 */
+	SqlGroupFunction(SqlKind kind, SqlGroupFunction groupFunction,
+		SqlOperandTypeChecker operandTypeChecker) {
+		super(kind.name(), kind, ReturnTypes.ARG0, null,
+			operandTypeChecker, SqlFunctionCategory.SYSTEM);
+		this.groupFunction = groupFunction;
+		if (groupFunction != null) {
+			assert groupFunction.groupFunction == null;
+		}
+	}
+
+	/** Creates an auxiliary function from this grouped window function. */
+	SqlGroupFunction auxiliary(SqlKind kind) {
+		return new SqlGroupFunction(kind, this, getOperandTypeChecker());
+	}
+
+	/** Returns a list of this grouped window function's auxiliary functions. */
+	List<SqlGroupFunction> getAuxiliaryFunctions() {
+		return ImmutableList.of();
+	}
+
+	@Override public boolean isGroup() {
+		// Auxiliary functions are not group functions
+		return groupFunction == null;
+	}
+
+	@Override public boolean isGroupAuxiliary() {
+		return groupFunction != null;
+	}
+
+	@Override public SqlMonotonicity getMonotonicity(SqlOperatorBinding call) {
+		// Monotonic iff its first argument is, but not strict.
+		//
+		// Note: This strategy happens to works for all current group functions
+		// (HOP, TUMBLE, SESSION). When there are exceptions to this rule, we'll
+		// make the method abstract.
+		return call.getOperandMonotonicity(0).unstrict();
+	}
+}
+
+// End SqlGroupFunction.java
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
new file mode 100644
index 0000000000000..6faf8e84d3668
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
@@ -0,0 +1,2133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.fun;
+
+/*
+ * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT UNTIL CALCITE-1761 IS FIXED.
+ */
+
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlAsOperator;
+import org.apache.calcite.sql.SqlBasicCall;
+import org.apache.calcite.sql.SqlBinaryOperator;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlFilterOperator;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlInternalOperator;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlLateralOperator;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNumericLiteral;
+import org.apache.calcite.sql.SqlOperandCountRange;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.SqlOverOperator;
+import org.apache.calcite.sql.SqlPostfixOperator;
+import org.apache.calcite.sql.SqlPrefixOperator;
+import org.apache.calcite.sql.SqlProcedureCallOperator;
+import org.apache.calcite.sql.SqlRankFunction;
+import org.apache.calcite.sql.SqlSampleSpec;
+import org.apache.calcite.sql.SqlSetOperator;
+import org.apache.calcite.sql.SqlSpecialOperator;
+import org.apache.calcite.sql.SqlUnnestOperator;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlValuesOperator;
+import org.apache.calcite.sql.SqlWindow;
+import org.apache.calcite.sql.SqlWriter;
+import org.apache.calcite.sql.type.InferTypes;
+import org.apache.calcite.sql.type.IntervalSqlType;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlOperandCountRanges;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable;
+import org.apache.calcite.sql.validate.SqlModality;
+import org.apache.calcite.sql2rel.AuxiliaryConverter;
+import org.apache.calcite.util.Litmus;
+import org.apache.calcite.util.Pair;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * Implementation of {@link org.apache.calcite.sql.SqlOperatorTable} containing
+ * the standard operators and functions.
+ */
+public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable {
+	//~ Static fields/initializers ---------------------------------------------
+
+	/**
+	 * The standard operator table.
+	 */
+	private static SqlStdOperatorTable instance;
+
+	//-------------------------------------------------------------
+	//                   SET OPERATORS
+	//-------------------------------------------------------------
+	// The set operators can be compared to the arithmetic operators
+	// UNION -> +
+	// EXCEPT -> -
+	// INTERSECT -> *
+	// which explains the different precedence values
+	public static final SqlSetOperator UNION =
+		new SqlSetOperator("UNION", SqlKind.UNION, 14, false);
+
+	public static final SqlSetOperator UNION_ALL =
+		new SqlSetOperator("UNION ALL", SqlKind.UNION, 14, true);
+
+	public static final SqlSetOperator EXCEPT =
+		new SqlSetOperator("EXCEPT", SqlKind.EXCEPT, 14, false);
+
+	public static final SqlSetOperator EXCEPT_ALL =
+		new SqlSetOperator("EXCEPT ALL", SqlKind.EXCEPT, 14, true);
+
+	public static final SqlSetOperator INTERSECT =
+		new SqlSetOperator("INTERSECT", SqlKind.INTERSECT, 18, false);
+
+	public static final SqlSetOperator INTERSECT_ALL =
+		new SqlSetOperator("INTERSECT ALL", SqlKind.INTERSECT, 18, true);
+
+	/**
+	 * The "MULTISET UNION" operator.
+	 */
+	public static final SqlMultisetSetOperator MULTISET_UNION =
+		new SqlMultisetSetOperator("MULTISET UNION", 14, false);
+
+	/**
+	 * The "MULTISET UNION ALL" operator.
+	 */
+	public static final SqlMultisetSetOperator MULTISET_UNION_ALL =
+		new SqlMultisetSetOperator("MULTISET UNION ALL", 14, true);
+
+	/**
+	 * The "MULTISET EXCEPT" operator.
+	 */
+	public static final SqlMultisetSetOperator MULTISET_EXCEPT =
+		new SqlMultisetSetOperator("MULTISET EXCEPT", 14, false);
+
+	/**
+	 * The "MULTISET EXCEPT ALL" operator.
+	 */
+	public static final SqlMultisetSetOperator MULTISET_EXCEPT_ALL =
+		new SqlMultisetSetOperator("MULTISET EXCEPT ALL", 14, true);
+
+	/**
+	 * The "MULTISET INTERSECT" operator.
+	 */
+	public static final SqlMultisetSetOperator MULTISET_INTERSECT =
+		new SqlMultisetSetOperator("MULTISET INTERSECT", 18, false);
+
+	/**
+	 * The "MULTISET INTERSECT ALL" operator.
+	 */
+	public static final SqlMultisetSetOperator MULTISET_INTERSECT_ALL =
+		new SqlMultisetSetOperator("MULTISET INTERSECT ALL", 18, true);
+
+	//-------------------------------------------------------------
+	//                   BINARY OPERATORS
+	//-------------------------------------------------------------
+
+	/**
+	 * Logical <code>AND</code> operator.
+	 */
+	public static final SqlBinaryOperator AND =
+		new SqlBinaryOperator(
+			"AND",
+			SqlKind.AND,
+			24,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE_OPTIMIZED,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN_BOOLEAN);
+
+	/**
+	 * <code>AS</code> operator associates an expression in the SELECT clause
+	 * with an alias.
+	 */
+	public static final SqlAsOperator AS = new SqlAsOperator();
+
+	/**
+	 * <code>ARGUMENT_ASSIGNMENT</code> operator (<code>=&lt;</code>)
+	 * assigns an argument to a function call to a particular named parameter.
+	 */
+	public static final SqlSpecialOperator ARGUMENT_ASSIGNMENT =
+		new SqlArgumentAssignmentOperator();
+
+	/**
+	 * <code>DEFAULT</code> operator indicates that an argument to a function call
+	 * is to take its default value..
+	 */
+	public static final SqlSpecialOperator DEFAULT = new SqlDefaultOperator();
+
+	/** <code>FILTER</code> operator filters which rows are included in an
+	 *  aggregate function. */
+	public static final SqlFilterOperator FILTER = new SqlFilterOperator();
+
+	/** {@code CUBE} operator, occurs within {@code GROUP BY} clause
+	 * or nested within a {@code GROUPING SETS}. */
+	public static final SqlInternalOperator CUBE =
+		new SqlRollupOperator("CUBE", SqlKind.CUBE);
+
+	/** {@code ROLLUP} operator, occurs within {@code GROUP BY} clause
+	 * or nested within a {@code GROUPING SETS}. */
+	public static final SqlInternalOperator ROLLUP =
+		new SqlRollupOperator("ROLLUP", SqlKind.ROLLUP);
+
+	/** {@code GROUPING SETS} operator, occurs within {@code GROUP BY} clause
+	 * or nested within a {@code GROUPING SETS}. */
+	public static final SqlInternalOperator GROUPING_SETS =
+		new SqlRollupOperator("GROUPING SETS", SqlKind.GROUPING_SETS);
+
+	/** {@code GROUPING} function. Occurs in similar places to an aggregate
+	 * function ({@code SELECT}, {@code HAVING} clause, etc. of an aggregate
+	 * query), but not technically an aggregate function. */
+	public static final SqlGroupingFunction GROUPING =
+		new SqlGroupingFunction("GROUPING");
+
+	/** {@code GROUP_ID} function. */
+	public static final SqlGroupIdFunction GROUP_ID =
+		new SqlGroupIdFunction();
+
+	/** {@code GROUP_ID} function is a synonym for {@code GROUPING}.
+	 *
+	 * <p>Some history. The {@code GROUPING} function is in the SQL standard,
+	 * and originally supported only one argument. The {@code GROUP_ID} is not
+	 * standard (though supported in Oracle and SQL Server) and supports zero or
+	 * more arguments.
+	 *
+	 * <p>The SQL standard has changed to allow {@code GROUPING} to have multiple
+	 * arguments. It is now equivalent to {@code GROUP_ID}, so we made
+	 * {@code GROUP_ID} a synonym for {@code GROUPING}. */
+	public static final SqlGroupingFunction GROUPING_ID =
+		new SqlGroupingFunction("GROUPING_ID");
+
+	/** {@code EXTEND} operator. */
+	public static final SqlInternalOperator EXTEND = new SqlExtendOperator();
+
+	/**
+	 * String concatenation operator, '<code>||</code>'.
+	 */
+	public static final SqlBinaryOperator CONCAT =
+		new SqlBinaryOperator(
+			"||",
+			SqlKind.OTHER,
+			60,
+			true,
+			ReturnTypes.DYADIC_STRING_SUM_PRECISION_NULLABLE,
+			null,
+			OperandTypes.STRING_SAME_SAME);
+
+	/**
+	 * Arithmetic division operator, '<code>/</code>'.
+	 */
+	public static final SqlBinaryOperator DIVIDE =
+		new SqlBinaryOperator(
+			"/",
+			SqlKind.DIVIDE,
+			60,
+			true,
+			ReturnTypes.QUOTIENT_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.DIVISION_OPERATOR);
+
+	/** The {@code RAND_INTEGER([seed, ] bound)} function, which yields a random
+	 * integer, optionally with seed. */
+	public static final SqlRandIntegerFunction RAND_INTEGER =
+		new SqlRandIntegerFunction();
+
+	/** The {@code RAND([seed])} function, which yields a random double,
+	 * optionally with seed. */
+	public static final SqlRandFunction RAND = new SqlRandFunction();
+
+	/**
+	 * Internal integer arithmetic division operator, '<code>/INT</code>'. This
+	 * is only used to adjust scale for numerics. We distinguish it from
+	 * user-requested division since some personalities want a floating-point
+	 * computation, whereas for the internal scaling use of division, we always
+	 * want integer division.
+	 */
+	public static final SqlBinaryOperator DIVIDE_INTEGER =
+		new SqlBinaryOperator(
+			"/INT",
+			SqlKind.DIVIDE,
+			60,
+			true,
+			ReturnTypes.INTEGER_QUOTIENT_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.DIVISION_OPERATOR);
+
+	/**
+	 * Dot operator, '<code>.</code>', used for referencing fields of records.
+	 */
+	public static final SqlBinaryOperator DOT =
+		new SqlBinaryOperator(
+			".",
+			SqlKind.DOT,
+			80,
+			true,
+			null,
+			null,
+			OperandTypes.ANY_ANY);
+
+	/**
+	 * Logical equals operator, '<code>=</code>'.
+	 */
+	public static final SqlBinaryOperator EQUALS =
+		new SqlBinaryOperator(
+			"=",
+			SqlKind.EQUALS,
+			30,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
+
+	/**
+	 * Logical greater-than operator, '<code>&gt;</code>'.
+	 */
+	public static final SqlBinaryOperator GREATER_THAN =
+		new SqlBinaryOperator(
+			">",
+			SqlKind.GREATER_THAN,
+			30,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
+
+	/**
+	 * <code>IS DISTINCT FROM</code> operator.
+	 */
+	public static final SqlBinaryOperator IS_DISTINCT_FROM =
+		new SqlBinaryOperator(
+			"IS DISTINCT FROM",
+			SqlKind.IS_DISTINCT_FROM,
+			30,
+			true,
+			ReturnTypes.BOOLEAN,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
+
+	/**
+	 * <code>IS NOT DISTINCT FROM</code> operator. Is equivalent to <code>NOT(x
+	 * IS DISTINCT FROM y)</code>
+	 */
+	public static final SqlBinaryOperator IS_NOT_DISTINCT_FROM =
+		new SqlBinaryOperator(
+			"IS NOT DISTINCT FROM",
+			SqlKind.IS_NOT_DISTINCT_FROM,
+			30,
+			true,
+			ReturnTypes.BOOLEAN,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
+
+	/**
+	 * The internal <code>$IS_DIFFERENT_FROM</code> operator is the same as the
+	 * user-level {@link #IS_DISTINCT_FROM} in all respects except that
+	 * the test for equality on character datatypes treats trailing spaces as
+	 * significant.
+	 */
+	public static final SqlBinaryOperator IS_DIFFERENT_FROM =
+		new SqlBinaryOperator(
+			"$IS_DIFFERENT_FROM",
+			SqlKind.OTHER,
+			30,
+			true,
+			ReturnTypes.BOOLEAN,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
+
+	/**
+	 * Logical greater-than-or-equal operator, '<code>&gt;=</code>'.
+	 */
+	public static final SqlBinaryOperator GREATER_THAN_OR_EQUAL =
+		new SqlBinaryOperator(
+			">=",
+			SqlKind.GREATER_THAN_OR_EQUAL,
+			30,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
+
+	/**
+	 * <code>IN</code> operator tests for a value's membership in a sub-query or
+	 * a list of values.
+	 */
+	public static final SqlBinaryOperator IN = new SqlInOperator(false);
+
+	/**
+	 * <code>NOT IN</code> operator tests for a value's membership in a sub-query
+	 * or a list of values.
+	 */
+	public static final SqlBinaryOperator NOT_IN =
+		new SqlInOperator(true);
+
+	/**
+	 * Logical less-than operator, '<code>&lt;</code>'.
+	 */
+	public static final SqlBinaryOperator LESS_THAN =
+		new SqlBinaryOperator(
+			"<",
+			SqlKind.LESS_THAN,
+			30,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
+
+	/**
+	 * Logical less-than-or-equal operator, '<code>&lt;=</code>'.
+	 */
+	public static final SqlBinaryOperator LESS_THAN_OR_EQUAL =
+		new SqlBinaryOperator(
+			"<=",
+			SqlKind.LESS_THAN_OR_EQUAL,
+			30,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
+
+	/**
+	 * Infix arithmetic minus operator, '<code>-</code>'.
+	 *
+	 * <p>Its precedence is less than the prefix {@link #UNARY_PLUS +}
+	 * and {@link #UNARY_MINUS -} operators.
+	 */
+	public static final SqlBinaryOperator MINUS =
+		new SqlMonotonicBinaryOperator(
+			"-",
+			SqlKind.MINUS,
+			40,
+			true,
+
+			// Same type inference strategy as sum
+			ReturnTypes.NULLABLE_SUM,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.MINUS_OPERATOR);
+
+	/**
+	 * Arithmetic multiplication operator, '<code>*</code>'.
+	 */
+	public static final SqlBinaryOperator MULTIPLY =
+		new SqlMonotonicBinaryOperator(
+			"*",
+			SqlKind.TIMES,
+			60,
+			true,
+			ReturnTypes.PRODUCT_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.MULTIPLY_OPERATOR);
+
+	/**
+	 * Logical not-equals operator, '<code>&lt;&gt;</code>'.
+	 */
+	public static final SqlBinaryOperator NOT_EQUALS =
+		new SqlBinaryOperator(
+			"<>",
+			SqlKind.NOT_EQUALS,
+			30,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
+
+	/**
+	 * Logical <code>OR</code> operator.
+	 */
+	public static final SqlBinaryOperator OR =
+		new SqlBinaryOperator(
+			"OR",
+			SqlKind.OR,
+			22,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE_OPTIMIZED,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN_BOOLEAN);
+
+	/**
+	 * Infix arithmetic plus operator, '<code>+</code>'.
+	 */
+	public static final SqlBinaryOperator PLUS =
+		new SqlMonotonicBinaryOperator(
+			"+",
+			SqlKind.PLUS,
+			40,
+			true,
+			ReturnTypes.NULLABLE_SUM,
+			InferTypes.FIRST_KNOWN,
+			OperandTypes.PLUS_OPERATOR);
+
+	/**
+	 * Infix datetime plus operator, '<code>DATETIME + INTERVAL</code>'.
+	 */
+	public static final SqlSpecialOperator DATETIME_PLUS =
+		new SqlSpecialOperator("DATETIME_PLUS", SqlKind.PLUS, 40, true, null,
+			InferTypes.FIRST_KNOWN, OperandTypes.PLUS_OPERATOR) {
+			@Override public RelDataType
+			inferReturnType(SqlOperatorBinding opBinding) {
+				final RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
+				final RelDataType leftType = opBinding.getOperandType(0);
+				final IntervalSqlType unitType =
+					(IntervalSqlType) opBinding.getOperandType(1);
+				switch (unitType.getIntervalQualifier().getStartUnit()) {
+					case HOUR:
+					case MINUTE:
+					case SECOND:
+					case MILLISECOND:
+					case MICROSECOND:
+						return typeFactory.createTypeWithNullability(
+							typeFactory.createSqlType(SqlTypeName.TIMESTAMP),
+							leftType.isNullable() || unitType.isNullable());
+					default:
+						return leftType;
+				}
+			}
+		};
+
+	/**
+	 * Multiset {@code MEMBER OF}, which returns whether a element belongs to a
+	 * multiset.
+	 *
+	 * <p>For example, the following returns <code>false</code>:
+	 *
+	 * <blockquote>
+	 * <code>'green' MEMBER OF MULTISET ['red','almost green','blue']</code>
+	 * </blockquote>
+	 */
+	public static final SqlBinaryOperator MEMBER_OF =
+		new SqlMultisetMemberOfOperator();
+
+	/**
+	 * Submultiset. Checks to see if an multiset is a sub-set of another
+	 * multiset.
+	 *
+	 * <p>For example, the following returns <code>false</code>:
+	 *
+	 * <blockquote>
+	 * <code>MULTISET ['green'] SUBMULTISET OF
+	 * MULTISET['red', 'almost green', 'blue']</code>
+	 * </blockquote>
+	 *
+	 * <p>The following returns <code>true</code>, in part because multisets are
+	 * order-independent:
+	 *
+	 * <blockquote>
+	 * <code>MULTISET ['blue', 'red'] SUBMULTISET OF
+	 * MULTISET ['red', 'almost green', 'blue']</code>
+	 * </blockquote>
+	 */
+	public static final SqlBinaryOperator SUBMULTISET_OF =
+
+		// TODO: check if precedence is correct
+		new SqlBinaryOperator(
+			"SUBMULTISET OF",
+			SqlKind.OTHER,
+			30,
+			true,
+			ReturnTypes.BOOLEAN_NULLABLE,
+			null,
+			OperandTypes.MULTISET_MULTISET);
+
+	//-------------------------------------------------------------
+	//                   POSTFIX OPERATORS
+	//-------------------------------------------------------------
+	public static final SqlPostfixOperator DESC =
+		new SqlPostfixOperator(
+			"DESC",
+			SqlKind.DESCENDING,
+			20,
+			ReturnTypes.ARG0,
+			InferTypes.RETURN_TYPE,
+			OperandTypes.ANY);
+
+	public static final SqlPostfixOperator NULLS_FIRST =
+		new SqlPostfixOperator(
+			"NULLS FIRST",
+			SqlKind.NULLS_FIRST,
+			18,
+			ReturnTypes.ARG0,
+			InferTypes.RETURN_TYPE,
+			OperandTypes.ANY);
+
+	public static final SqlPostfixOperator NULLS_LAST =
+		new SqlPostfixOperator(
+			"NULLS LAST",
+			SqlKind.NULLS_LAST,
+			18,
+			ReturnTypes.ARG0,
+			InferTypes.RETURN_TYPE,
+			OperandTypes.ANY);
+
+	public static final SqlPostfixOperator IS_NOT_NULL =
+		new SqlPostfixOperator(
+			"IS NOT NULL",
+			SqlKind.IS_NOT_NULL,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.VARCHAR_1024,
+			OperandTypes.ANY);
+
+	public static final SqlPostfixOperator IS_NULL =
+		new SqlPostfixOperator(
+			"IS NULL",
+			SqlKind.IS_NULL,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.VARCHAR_1024,
+			OperandTypes.ANY);
+
+	public static final SqlPostfixOperator IS_NOT_TRUE =
+		new SqlPostfixOperator(
+			"IS NOT TRUE",
+			SqlKind.IS_NOT_TRUE,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN);
+
+	public static final SqlPostfixOperator IS_TRUE =
+		new SqlPostfixOperator(
+			"IS TRUE",
+			SqlKind.IS_TRUE,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN);
+
+	public static final SqlPostfixOperator IS_NOT_FALSE =
+		new SqlPostfixOperator(
+			"IS NOT FALSE",
+			SqlKind.IS_NOT_FALSE,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN);
+
+	public static final SqlPostfixOperator IS_FALSE =
+		new SqlPostfixOperator(
+			"IS FALSE",
+			SqlKind.IS_FALSE,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN);
+
+	public static final SqlPostfixOperator IS_NOT_UNKNOWN =
+		new SqlPostfixOperator(
+			"IS NOT UNKNOWN",
+			SqlKind.IS_NOT_NULL,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN);
+
+	public static final SqlPostfixOperator IS_UNKNOWN =
+		new SqlPostfixOperator(
+			"IS UNKNOWN",
+			SqlKind.IS_NULL,
+			28,
+			ReturnTypes.BOOLEAN_NOT_NULL,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN);
+
+	public static final SqlPostfixOperator IS_A_SET =
+		new SqlPostfixOperator(
+			"IS A SET",
+			SqlKind.OTHER,
+			28,
+			ReturnTypes.BOOLEAN,
+			null,
+			OperandTypes.MULTISET);
+
+	//-------------------------------------------------------------
+	//                   PREFIX OPERATORS
+	//-------------------------------------------------------------
+	public static final SqlPrefixOperator EXISTS =
+		new SqlPrefixOperator(
+			"EXISTS",
+			SqlKind.EXISTS,
+			40,
+			ReturnTypes.BOOLEAN,
+			null,
+			OperandTypes.ANY) {
+			public boolean argumentMustBeScalar(int ordinal) {
+				return false;
+			}
+
+			@Override public boolean validRexOperands(int count, Litmus litmus) {
+				if (count != 0) {
+					return litmus.fail("wrong operand count {} for {}", count, this);
+				}
+				return litmus.succeed();
+			}
+		};
+
+	public static final SqlPrefixOperator NOT =
+		new SqlPrefixOperator(
+			"NOT",
+			SqlKind.NOT,
+			26,
+			ReturnTypes.ARG0,
+			InferTypes.BOOLEAN,
+			OperandTypes.BOOLEAN);
+
+	/**
+	 * Prefix arithmetic minus operator, '<code>-</code>'.
+	 *
+	 * <p>Its precedence is greater than the infix '{@link #PLUS +}' and
+	 * '{@link #MINUS -}' operators.
+	 */
+	public static final SqlPrefixOperator UNARY_MINUS =
+		new SqlPrefixOperator(
+			"-",
+			SqlKind.MINUS_PREFIX,
+			80,
+			ReturnTypes.ARG0,
+			InferTypes.RETURN_TYPE,
+			OperandTypes.NUMERIC_OR_INTERVAL);
+
+	/**
+	 * Prefix arithmetic plus operator, '<code>+</code>'.
+	 *
+	 * <p>Its precedence is greater than the infix '{@link #PLUS +}' and
+	 * '{@link #MINUS -}' operators.
+	 */
+	public static final SqlPrefixOperator UNARY_PLUS =
+		new SqlPrefixOperator(
+			"+",
+			SqlKind.PLUS_PREFIX,
+			80,
+			ReturnTypes.ARG0,
+			InferTypes.RETURN_TYPE,
+			OperandTypes.NUMERIC_OR_INTERVAL);
+
+	/**
+	 * Keyword which allows an identifier to be explicitly flagged as a table.
+	 * For example, <code>select * from (TABLE t)</code> or <code>TABLE
+	 * t</code>. See also {@link #COLLECTION_TABLE}.
+	 */
+	public static final SqlPrefixOperator EXPLICIT_TABLE =
+		new SqlPrefixOperator(
+			"TABLE",
+			SqlKind.EXPLICIT_TABLE,
+			2,
+			null,
+			null,
+			null);
+
+	//-------------------------------------------------------------
+	// AGGREGATE OPERATORS
+	//-------------------------------------------------------------
+	/**
+	 * <code>SUM</code> aggregate function.
+	 */
+	public static final SqlAggFunction SUM = new SqlSumAggFunction(null);
+
+	/**
+	 * <code>COUNT</code> aggregate function.
+	 */
+	public static final SqlAggFunction COUNT = new SqlCountAggFunction();
+
+	/**
+	 * <code>MIN</code> aggregate function.
+	 */
+	public static final SqlAggFunction MIN =
+		new SqlMinMaxAggFunction(SqlKind.MIN);
+
+	/**
+	 * <code>MAX</code> aggregate function.
+	 */
+	public static final SqlAggFunction MAX =
+		new SqlMinMaxAggFunction(SqlKind.MAX);
+
+	/**
+	 * <code>LAST_VALUE</code> aggregate function.
+	 */
+	public static final SqlAggFunction LAST_VALUE =
+		new SqlFirstLastValueAggFunction(SqlKind.LAST_VALUE);
+
+	/**
+	 * <code>FIRST_VALUE</code> aggregate function.
+	 */
+	public static final SqlAggFunction FIRST_VALUE =
+		new SqlFirstLastValueAggFunction(SqlKind.FIRST_VALUE);
+
+	/**
+	 * <code>LEAD</code> aggregate function.
+	 */
+	public static final SqlAggFunction LEAD =
+		new SqlLeadLagAggFunction(SqlKind.LEAD);
+
+	/**
+	 * <code>LAG</code> aggregate function.
+	 */
+	public static final SqlAggFunction LAG =
+		new SqlLeadLagAggFunction(SqlKind.LAG);
+
+	/**
+	 * <code>NTILE</code> aggregate function.
+	 */
+	public static final SqlAggFunction NTILE =
+		new SqlNtileAggFunction();
+
+	/**
+	 * <code>SINGLE_VALUE</code> aggregate function.
+	 */
+	public static final SqlAggFunction SINGLE_VALUE =
+		new SqlSingleValueAggFunction(null);
+
+	/**
+	 * <code>AVG</code> aggregate function.
+	 */
+	public static final SqlAggFunction AVG =
+		new SqlAvgAggFunction(SqlKind.AVG);
+
+	/**
+	 * <code>STDDEV_POP</code> aggregate function.
+	 */
+	public static final SqlAggFunction STDDEV_POP =
+		new SqlAvgAggFunction(SqlKind.STDDEV_POP);
+
+	/**
+	 * <code>REGR_SXX</code> aggregate function.
+	 */
+	public static final SqlAggFunction REGR_SXX =
+		new SqlCovarAggFunction(SqlKind.REGR_SXX);
+
+	/**
+	 * <code>REGR_SYY</code> aggregate function.
+	 */
+	public static final SqlAggFunction REGR_SYY =
+		new SqlCovarAggFunction(SqlKind.REGR_SYY);
+
+	/**
+	 * <code>COVAR_POP</code> aggregate function.
+	 */
+	public static final SqlAggFunction COVAR_POP =
+		new SqlCovarAggFunction(SqlKind.COVAR_POP);
+
+	/**
+	 * <code>COVAR_SAMP</code> aggregate function.
+	 */
+	public static final SqlAggFunction COVAR_SAMP =
+		new SqlCovarAggFunction(SqlKind.COVAR_SAMP);
+
+	/**
+	 * <code>STDDEV_SAMP</code> aggregate function.
+	 */
+	public static final SqlAggFunction STDDEV_SAMP =
+		new SqlAvgAggFunction(SqlKind.STDDEV_SAMP);
+
+	/**
+	 * <code>VAR_POP</code> aggregate function.
+	 */
+	public static final SqlAggFunction VAR_POP =
+		new SqlAvgAggFunction(SqlKind.VAR_POP);
+
+	/**
+	 * <code>VAR_SAMP</code> aggregate function.
+	 */
+	public static final SqlAggFunction VAR_SAMP =
+		new SqlAvgAggFunction(SqlKind.VAR_SAMP);
+
+	//-------------------------------------------------------------
+	// WINDOW Aggregate Functions
+	//-------------------------------------------------------------
+	/**
+	 * <code>HISTOGRAM</code> aggregate function support. Used by window
+	 * aggregate versions of MIN/MAX
+	 */
+	public static final SqlAggFunction HISTOGRAM_AGG =
+		new SqlHistogramAggFunction(null);
+
+	/**
+	 * <code>HISTOGRAM_MIN</code> window aggregate function.
+	 */
+	public static final SqlFunction HISTOGRAM_MIN =
+		new SqlFunction(
+			"$HISTOGRAM_MIN",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE,
+			null,
+			OperandTypes.NUMERIC_OR_STRING,
+			SqlFunctionCategory.NUMERIC);
+
+	/**
+	 * <code>HISTOGRAM_MAX</code> window aggregate function.
+	 */
+	public static final SqlFunction HISTOGRAM_MAX =
+		new SqlFunction(
+			"$HISTOGRAM_MAX",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE,
+			null,
+			OperandTypes.NUMERIC_OR_STRING,
+			SqlFunctionCategory.NUMERIC);
+
+	/**
+	 * <code>HISTOGRAM_FIRST_VALUE</code> window aggregate function.
+	 */
+	public static final SqlFunction HISTOGRAM_FIRST_VALUE =
+		new SqlFunction(
+			"$HISTOGRAM_FIRST_VALUE",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE,
+			null,
+			OperandTypes.NUMERIC_OR_STRING,
+			SqlFunctionCategory.NUMERIC);
+
+	/**
+	 * <code>HISTOGRAM_LAST_VALUE</code> window aggregate function.
+	 */
+	public static final SqlFunction HISTOGRAM_LAST_VALUE =
+		new SqlFunction(
+			"$HISTOGRAM_LAST_VALUE",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE,
+			null,
+			OperandTypes.NUMERIC_OR_STRING,
+			SqlFunctionCategory.NUMERIC);
+
+	/**
+	 * <code>SUM0</code> aggregate function.
+	 */
+	public static final SqlAggFunction SUM0 =
+		new SqlSumEmptyIsZeroAggFunction();
+
+	//-------------------------------------------------------------
+	// WINDOW Rank Functions
+	//-------------------------------------------------------------
+	/**
+	 * <code>CUME_DIST</code> window function.
+	 */
+	public static final SqlRankFunction CUME_DIST =
+		new SqlRankFunction(true, SqlKind.CUME_DIST);
+
+	/**
+	 * <code>DENSE_RANK</code> window function.
+	 */
+	public static final SqlRankFunction DENSE_RANK =
+		new SqlRankFunction(true, SqlKind.DENSE_RANK);
+
+	/**
+	 * <code>PERCENT_RANK</code> window function.
+	 */
+	public static final SqlRankFunction PERCENT_RANK =
+		new SqlRankFunction(true, SqlKind.PERCENT_RANK);
+
+	/**
+	 * <code>RANK</code> window function.
+	 */
+	public static final SqlRankFunction RANK =
+		new SqlRankFunction(true, SqlKind.RANK);
+
+	/**
+	 * <code>ROW_NUMBER</code> window function.
+	 */
+	public static final SqlRankFunction ROW_NUMBER =
+		new SqlRankFunction(false, SqlKind.ROW_NUMBER);
+
+	//-------------------------------------------------------------
+	//                   SPECIAL OPERATORS
+	//-------------------------------------------------------------
+	public static final SqlRowOperator ROW = new SqlRowOperator("ROW");
+
+	/**
+	 * A special operator for the subtraction of two DATETIMEs. The format of
+	 * DATETIME subtraction is:
+	 *
+	 * <blockquote><code>"(" &lt;datetime&gt; "-" &lt;datetime&gt; ")"
+	 * &lt;interval qualifier&gt;</code></blockquote>
+	 *
+	 * <p>This operator is special since it needs to hold the
+	 * additional interval qualifier specification.</p>
+	 */
+	public static final SqlDatetimeSubtractionOperator MINUS_DATE =
+		new SqlDatetimeSubtractionOperator();
+
+	/**
+	 * The MULTISET Value Constructor. e.g. "<code>MULTISET[1,2,3]</code>".
+	 */
+	public static final SqlMultisetValueConstructor MULTISET_VALUE =
+		new SqlMultisetValueConstructor();
+
+	/**
+	 * The MULTISET Query Constructor. e.g. "<code>SELECT dname, MULTISET(SELECT
+	 * FROM emp WHERE deptno = dept.deptno) FROM dept</code>".
+	 */
+	public static final SqlMultisetQueryConstructor MULTISET_QUERY =
+		new SqlMultisetQueryConstructor();
+
+	/**
+	 * The ARRAY Query Constructor. e.g. "<code>SELECT dname, ARRAY(SELECT
+	 * FROM emp WHERE deptno = dept.deptno) FROM dept</code>".
+	 */
+	public static final SqlMultisetQueryConstructor ARRAY_QUERY =
+		new SqlArrayQueryConstructor();
+
+	/**
+	 * The MAP Query Constructor. e.g. "<code>MAP(SELECT empno, deptno
+	 * FROM emp)</code>".
+	 */
+	public static final SqlMultisetQueryConstructor MAP_QUERY =
+		new SqlMapQueryConstructor();
+
+	/**
+	 * The CURSOR constructor. e.g. "<code>SELECT * FROM
+	 * TABLE(DEDUP(CURSOR(SELECT * FROM EMPS), 'name'))</code>".
+	 */
+	public static final SqlCursorConstructor CURSOR =
+		new SqlCursorConstructor();
+
+	/**
+	 * The COLUMN_LIST constructor. e.g. the ROW() call in "<code>SELECT * FROM
+	 * TABLE(DEDUP(CURSOR(SELECT * FROM EMPS), ROW(name, empno)))</code>".
+	 */
+	public static final SqlColumnListConstructor COLUMN_LIST =
+		new SqlColumnListConstructor();
+
+	/**
+	 * The <code>UNNEST</code> operator.
+	 */
+	public static final SqlUnnestOperator UNNEST =
+		new SqlUnnestOperator(false);
+
+	/**
+	 * The <code>UNNEST WITH ORDINALITY</code> operator.
+	 */
+	public static final SqlUnnestOperator UNNEST_WITH_ORDINALITY =
+		new SqlUnnestOperator(true);
+
+	/**
+	 * The <code>LATERAL</code> operator.
+	 */
+	public static final SqlSpecialOperator LATERAL =
+		new SqlLateralOperator(SqlKind.LATERAL);
+
+	/**
+	 * The "table function derived table" operator, which a table-valued
+	 * function into a relation, e.g. "<code>SELECT * FROM
+	 * TABLE(ramp(5))</code>".
+	 *
+	 * <p>This operator has function syntax (with one argument), whereas
+	 * {@link #EXPLICIT_TABLE} is a prefix operator.
+	 */
+	public static final SqlSpecialOperator COLLECTION_TABLE =
+		new SqlCollectionTableOperator("TABLE", SqlModality.RELATION);
+
+	public static final SqlOverlapsOperator OVERLAPS =
+		new SqlOverlapsOperator();
+
+	public static final SqlSpecialOperator VALUES =
+		new SqlValuesOperator();
+
+	public static final SqlLiteralChainOperator LITERAL_CHAIN =
+		new SqlLiteralChainOperator();
+
+	public static final SqlThrowOperator THROW = new SqlThrowOperator();
+
+	public static final SqlBetweenOperator BETWEEN =
+		new SqlBetweenOperator(
+			SqlBetweenOperator.Flag.ASYMMETRIC,
+			false);
+
+	public static final SqlBetweenOperator SYMMETRIC_BETWEEN =
+		new SqlBetweenOperator(
+			SqlBetweenOperator.Flag.SYMMETRIC,
+			false);
+
+	public static final SqlBetweenOperator NOT_BETWEEN =
+		new SqlBetweenOperator(
+			SqlBetweenOperator.Flag.ASYMMETRIC,
+			true);
+
+	public static final SqlBetweenOperator SYMMETRIC_NOT_BETWEEN =
+		new SqlBetweenOperator(
+			SqlBetweenOperator.Flag.SYMMETRIC,
+			true);
+
+	public static final SqlSpecialOperator NOT_LIKE =
+		new SqlLikeOperator("NOT LIKE", SqlKind.LIKE, true);
+
+	public static final SqlSpecialOperator LIKE =
+		new SqlLikeOperator("LIKE", SqlKind.LIKE, false);
+
+	public static final SqlSpecialOperator NOT_SIMILAR_TO =
+		new SqlLikeOperator("NOT SIMILAR TO", SqlKind.SIMILAR, true);
+
+	public static final SqlSpecialOperator SIMILAR_TO =
+		new SqlLikeOperator("SIMILAR TO", SqlKind.SIMILAR, false);
+
+	/**
+	 * Internal operator used to represent the ESCAPE clause of a LIKE or
+	 * SIMILAR TO expression.
+	 */
+	public static final SqlSpecialOperator ESCAPE =
+		new SqlSpecialOperator("ESCAPE", SqlKind.ESCAPE, 0);
+
+	public static final SqlCaseOperator CASE = SqlCaseOperator.INSTANCE;
+
+	public static final SqlOperator PROCEDURE_CALL =
+		new SqlProcedureCallOperator();
+
+	public static final SqlOperator NEW = new SqlNewOperator();
+
+	/**
+	 * The <code>OVER</code> operator, which applies an aggregate functions to a
+	 * {@link SqlWindow window}.
+	 *
+	 * <p>Operands are as follows:
+	 *
+	 * <ol>
+	 * <li>name of window function ({@link org.apache.calcite.sql.SqlCall})</li>
+	 * <li>window name ({@link org.apache.calcite.sql.SqlLiteral}) or window
+	 * in-line specification (@link SqlWindowOperator})</li>
+	 * </ol>
+	 */
+	public static final SqlBinaryOperator OVER = new SqlOverOperator();
+
+	/**
+	 * An <code>REINTERPRET</code> operator is internal to the planner. When the
+	 * physical storage of two types is the same, this operator may be used to
+	 * reinterpret values of one type as the other. This operator is similar to
+	 * a cast, except that it does not alter the data value. Like a regular cast
+	 * it accepts one operand and stores the target type as the return type. It
+	 * performs an overflow check if it has <i>any</i> second operand, whether
+	 * true or not.
+	 */
+	public static final SqlSpecialOperator REINTERPRET =
+		new SqlSpecialOperator("Reinterpret", SqlKind.REINTERPRET) {
+			public SqlOperandCountRange getOperandCountRange() {
+				return SqlOperandCountRanges.between(1, 2);
+			}
+		};
+
+	/** Internal operator that extracts time periods (year, month, date) from a
+	 * date in internal format (number of days since epoch). */
+	public static final SqlSpecialOperator EXTRACT_DATE =
+		new SqlSpecialOperator("EXTRACT_DATE", SqlKind.EXTRACT);
+
+	//-------------------------------------------------------------
+	//                   FUNCTIONS
+	//-------------------------------------------------------------
+
+	/**
+	 * The character substring function: <code>SUBSTRING(string FROM start [FOR
+	 * length])</code>.
+	 *
+	 * <p>If the length parameter is a constant, the length of the result is the
+	 * minimum of the length of the input and that length. Otherwise it is the
+	 * length of the input.
+	 */
+	public static final SqlFunction SUBSTRING = new SqlSubstringFunction();
+
+	/** The {@code REPLACE(string, search, replace)} function. Not standard SQL,
+	 * but in Oracle and Postgres. */
+	public static final SqlFunction REPLACE =
+		new SqlFunction("REPLACE", SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE_VARYING, null,
+			OperandTypes.STRING_STRING_STRING, SqlFunctionCategory.STRING);
+
+	public static final SqlFunction CONVERT =
+		new SqlConvertFunction("CONVERT");
+
+	/**
+	 * The <code>TRANSLATE(<i>char_value</i> USING <i>translation_name</i>)</code> function
+	 * alters the character set of a string value from one base character set to another.
+	 *
+	 * <p>It is defined in the SQL standard. See also non-standard
+	 * {@link OracleSqlOperatorTable#TRANSLATE3}.
+	 */
+	public static final SqlFunction TRANSLATE =
+		new SqlConvertFunction("TRANSLATE");
+
+	public static final SqlFunction OVERLAY = new SqlOverlayFunction();
+
+	/** The "TRIM" function. */
+	public static final SqlFunction TRIM = SqlTrimFunction.INSTANCE;
+
+	public static final SqlFunction POSITION = new SqlPositionFunction();
+
+	public static final SqlFunction CHAR_LENGTH =
+		new SqlFunction(
+			"CHAR_LENGTH",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.INTEGER_NULLABLE,
+			null,
+			OperandTypes.CHARACTER,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction CHARACTER_LENGTH =
+		new SqlFunction(
+			"CHARACTER_LENGTH",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.INTEGER_NULLABLE,
+			null,
+			OperandTypes.CHARACTER,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction UPPER =
+		new SqlFunction(
+			"UPPER",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE,
+			null,
+			OperandTypes.CHARACTER,
+			SqlFunctionCategory.STRING);
+
+	public static final SqlFunction LOWER =
+		new SqlFunction(
+			"LOWER",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE,
+			null,
+			OperandTypes.CHARACTER,
+			SqlFunctionCategory.STRING);
+
+	public static final SqlFunction INITCAP =
+		new SqlFunction(
+			"INITCAP",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0_NULLABLE,
+			null,
+			OperandTypes.CHARACTER,
+			SqlFunctionCategory.STRING);
+
+	/**
+	 * Uses SqlOperatorTable.useDouble for its return type since we don't know
+	 * what the result type will be by just looking at the operand types. For
+	 * example POW(int, int) can return a non integer if the second operand is
+	 * negative.
+	 */
+	public static final SqlFunction POWER =
+		new SqlFunction(
+			"POWER",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC_NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction SQRT =
+		new SqlFunction(
+			"SQRT",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction MOD =
+		// Return type is same as divisor (2nd operand)
+		// SQL2003 Part2 Section 6.27, Syntax Rules 9
+		new SqlFunction(
+			"MOD",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG1_NULLABLE,
+			null,
+			OperandTypes.EXACT_NUMERIC_EXACT_NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction LN =
+		new SqlFunction(
+			"LN",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction LOG10 =
+		new SqlFunction(
+			"LOG10",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction ABS =
+		new SqlFunction(
+			"ABS",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0,
+			null,
+			OperandTypes.NUMERIC_OR_INTERVAL,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction ACOS =
+		new SqlFunction(
+			"ACOS",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction ASIN =
+		new SqlFunction(
+			"ASIN",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction ATAN =
+		new SqlFunction(
+			"ATAN",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction ATAN2 =
+		new SqlFunction(
+			"ATAN2",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC_NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction COS =
+		new SqlFunction(
+			"COS",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction COT =
+		new SqlFunction(
+			"COT",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction DEGREES =
+		new SqlFunction(
+			"DEGREES",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction EXP =
+		new SqlFunction(
+			"EXP",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction RADIANS =
+		new SqlFunction(
+			"RADIANS",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction ROUND =
+		new SqlFunction(
+			"ROUND",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0,
+			null,
+			OperandTypes.NUMERIC_INTEGER,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction SIGN =
+		new SqlFunction(
+			"SIGN",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction SIN =
+		new SqlFunction(
+			"SIN",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+
+	public static final SqlFunction TAN =
+		new SqlFunction(
+			"TAN",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.DOUBLE_NULLABLE,
+			null,
+			OperandTypes.NUMERIC,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction TRUNCATE =
+		new SqlFunction(
+			"TRUNCATE",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0,
+			null,
+			OperandTypes.NUMERIC_INTEGER,
+			SqlFunctionCategory.NUMERIC);
+
+	public static final SqlFunction PI =
+		new SqlBaseContextVariable("PI", ReturnTypes.DOUBLE,
+			SqlFunctionCategory.NUMERIC);
+
+	/** {@code FINAL} function to be used within {@code MATCH_RECOGNIZE}. */
+	public static final SqlFunction FINAL =
+		new SqlFunction("FINAL", SqlKind.FINAL, ReturnTypes.ARG0_NULLABLE, null,
+			OperandTypes.ANY, SqlFunctionCategory.MATCH_RECOGNIZE);
+
+	/** {@code RUNNING} function to be used within {@code MATCH_RECOGNIZE}. */
+	public static final SqlFunction RUNNING =
+		new SqlFunction("RUNNING", SqlKind.RUNNING, ReturnTypes.ARG0_NULLABLE,
+			null, OperandTypes.ANY, SqlFunctionCategory.MATCH_RECOGNIZE);
+
+	/** {@code FIRST} function to be used within {@code MATCH_RECOGNIZE}. */
+	public static final SqlFunction FIRST =
+		new SqlFunction("FIRST", SqlKind.FIRST, ReturnTypes.ARG0_NULLABLE,
+			null, OperandTypes.ANY_NUMERIC, SqlFunctionCategory.MATCH_RECOGNIZE);
+
+	/** {@code LAST} function to be used within {@code MATCH_RECOGNIZE}. */
+	public static final SqlFunction LAST =
+		new SqlFunction("LAST", SqlKind.LAST, ReturnTypes.ARG0_NULLABLE,
+			null, OperandTypes.ANY_NUMERIC, SqlFunctionCategory.MATCH_RECOGNIZE);
+
+	/** {@code PREV} function to be used within {@code MATCH_RECOGNIZE}. */
+	public static final SqlFunction PREV =
+		new SqlFunction("PREV", SqlKind.PREV, ReturnTypes.ARG0_NULLABLE,
+			null, OperandTypes.ANY_NUMERIC, SqlFunctionCategory.MATCH_RECOGNIZE);
+
+	/** {@code NEXT} function to be used within {@code MATCH_RECOGNIZE}. */
+	public static final SqlFunction NEXT =
+		new SqlFunction("NEXT", SqlKind.NEXT, ReturnTypes.ARG0_NULLABLE, null,
+			OperandTypes.ANY_NUMERIC, SqlFunctionCategory.MATCH_RECOGNIZE);
+
+	public static final SqlFunction NULLIF = new SqlNullifFunction();
+
+	/**
+	 * The COALESCE builtin function.
+	 */
+	public static final SqlFunction COALESCE = new SqlCoalesceFunction();
+
+	/**
+	 * The <code>FLOOR</code> function.
+	 */
+	public static final SqlFunction FLOOR = new SqlFloorFunction(SqlKind.FLOOR);
+
+	/**
+	 * The <code>CEIL</code> function.
+	 */
+	public static final SqlFunction CEIL = new SqlFloorFunction(SqlKind.CEIL);
+
+	/**
+	 * The <code>USER</code> function.
+	 */
+	public static final SqlFunction USER =
+		new SqlStringContextVariable("USER");
+
+	/**
+	 * The <code>CURRENT_USER</code> function.
+	 */
+	public static final SqlFunction CURRENT_USER =
+		new SqlStringContextVariable("CURRENT_USER");
+
+	/**
+	 * The <code>SESSION_USER</code> function.
+	 */
+	public static final SqlFunction SESSION_USER =
+		new SqlStringContextVariable("SESSION_USER");
+
+	/**
+	 * The <code>SYSTEM_USER</code> function.
+	 */
+	public static final SqlFunction SYSTEM_USER =
+		new SqlStringContextVariable("SYSTEM_USER");
+
+	/**
+	 * The <code>CURRENT_PATH</code> function.
+	 */
+	public static final SqlFunction CURRENT_PATH =
+		new SqlStringContextVariable("CURRENT_PATH");
+
+	/**
+	 * The <code>CURRENT_ROLE</code> function.
+	 */
+	public static final SqlFunction CURRENT_ROLE =
+		new SqlStringContextVariable("CURRENT_ROLE");
+
+	/**
+	 * The <code>CURRENT_CATALOG</code> function.
+	 */
+	public static final SqlFunction CURRENT_CATALOG =
+		new SqlStringContextVariable("CURRENT_CATALOG");
+
+	/**
+	 * The <code>CURRENT_SCHEMA</code> function.
+	 */
+	public static final SqlFunction CURRENT_SCHEMA =
+		new SqlStringContextVariable("CURRENT_SCHEMA");
+
+	/**
+	 * The <code>LOCALTIME [(<i>precision</i>)]</code> function.
+	 */
+	public static final SqlFunction LOCALTIME =
+		new SqlAbstractTimeFunction("LOCALTIME", SqlTypeName.TIME);
+
+	/**
+	 * The <code>LOCALTIMESTAMP [(<i>precision</i>)]</code> function.
+	 */
+	public static final SqlFunction LOCALTIMESTAMP =
+		new SqlAbstractTimeFunction("LOCALTIMESTAMP", SqlTypeName.TIMESTAMP);
+
+	/**
+	 * The <code>CURRENT_TIME [(<i>precision</i>)]</code> function.
+	 */
+	public static final SqlFunction CURRENT_TIME =
+		new SqlAbstractTimeFunction("CURRENT_TIME", SqlTypeName.TIME);
+
+	/**
+	 * The <code>CURRENT_TIMESTAMP [(<i>precision</i>)]</code> function.
+	 */
+	public static final SqlFunction CURRENT_TIMESTAMP =
+		new SqlAbstractTimeFunction("CURRENT_TIMESTAMP", SqlTypeName.TIMESTAMP);
+
+	/**
+	 * The <code>CURRENT_DATE</code> function.
+	 */
+	public static final SqlFunction CURRENT_DATE =
+		new SqlCurrentDateFunction();
+
+	/** The <code>TIMESTAMPADD</code> function. */
+	public static final SqlFunction TIMESTAMP_ADD = new SqlTimestampAddFunction();
+
+	/** The <code>TIMESTAMPDIFF</code> function. */
+	public static final SqlFunction TIMESTAMP_DIFF = new SqlTimestampDiffFunction();
+
+	/**
+	 * Use of the <code>IN_FENNEL</code> operator forces the argument to be
+	 * evaluated in Fennel. Otherwise acts as identity function.
+	 */
+	public static final SqlFunction IN_FENNEL =
+		new SqlMonotonicUnaryFunction(
+			"IN_FENNEL",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.ARG0,
+			null,
+			OperandTypes.ANY,
+			SqlFunctionCategory.SYSTEM);
+
+	/**
+	 * The SQL <code>CAST</code> operator.
+	 *
+	 * <p>The SQL syntax is
+	 *
+	 * <blockquote><code>CAST(<i>expression</i> AS <i>type</i>)</code>
+	 * </blockquote>
+	 *
+	 * <p>When the CAST operator is applies as a {@link SqlCall}, it has two
+	 * arguments: the expression and the type. The type must not include a
+	 * constraint, so <code>CAST(x AS INTEGER NOT NULL)</code>, for instance, is
+	 * invalid.</p>
+	 *
+	 * <p>When the CAST operator is applied as a <code>RexCall</code>, the
+	 * target type is simply stored as the return type, not an explicit operand.
+	 * For example, the expression <code>CAST(1 + 2 AS DOUBLE)</code> will
+	 * become a call to <code>CAST</code> with the expression <code>1 + 2</code>
+	 * as its only operand.</p>
+	 *
+	 * <p>The <code>RexCall</code> form can also have a type which contains a
+	 * <code>NOT NULL</code> constraint. When this expression is implemented, if
+	 * the value is NULL, an exception will be thrown.</p>
+	 */
+	public static final SqlFunction CAST = new SqlCastFunction();
+
+	/**
+	 * The SQL <code>EXTRACT</code> operator. Extracts a specified field value
+	 * from a DATETIME or an INTERVAL. E.g.<br>
+	 * <code>EXTRACT(HOUR FROM INTERVAL '364 23:59:59')</code> returns <code>
+	 * 23</code>
+	 */
+	public static final SqlFunction EXTRACT = new SqlExtractFunction();
+
+	/**
+	 * The SQL <code>YEAR</code> operator. Returns the Year
+	 * from a DATETIME  E.g.<br>
+	 * <code>YEAR(date '2008-9-23')</code> returns <code>
+	 * 2008</code>
+	 */
+	public static final SqlDatePartFunction YEAR =
+		new SqlDatePartFunction("YEAR", TimeUnit.YEAR);
+
+	/**
+	 * The SQL <code>QUARTER</code> operator. Returns the Quarter
+	 * from a DATETIME  E.g.<br>
+	 * <code>QUARTER(date '2008-9-23')</code> returns <code>
+	 * 3</code>
+	 */
+	public static final SqlDatePartFunction QUARTER =
+		new SqlDatePartFunction("QUARTER", TimeUnit.QUARTER);
+
+	/**
+	 * The SQL <code>MONTH</code> operator. Returns the Month
+	 * from a DATETIME  E.g.<br>
+	 * <code>MONTH(date '2008-9-23')</code> returns <code>
+	 * 9</code>
+	 */
+	public static final SqlDatePartFunction MONTH =
+		new SqlDatePartFunction("MONTH", TimeUnit.MONTH);
+
+	/**
+	 * The SQL <code>WEEK</code> operator. Returns the Week
+	 * from a DATETIME  E.g.<br>
+	 * <code>WEEK(date '2008-9-23')</code> returns <code>
+	 * 39</code>
+	 */
+	public static final SqlDatePartFunction WEEK =
+		new SqlDatePartFunction("WEEK", TimeUnit.WEEK);
+
+	/**
+	 * The SQL <code>DAYOFYEAR</code> operator. Returns the DOY
+	 * from a DATETIME  E.g.<br>
+	 * <code>DAYOFYEAR(date '2008-9-23')</code> returns <code>
+	 * 267</code>
+	 */
+	public static final SqlDatePartFunction DAYOFYEAR =
+		new SqlDatePartFunction("DAYOFYEAR", TimeUnit.DOY);
+
+	/**
+	 * The SQL <code>DAYOFMONTH</code> operator. Returns the Day
+	 * from a DATETIME  E.g.<br>
+	 * <code>DAYOFMONTH(date '2008-9-23')</code> returns <code>
+	 * 23</code>
+	 */
+	public static final SqlDatePartFunction DAYOFMONTH =
+		new SqlDatePartFunction("DAYOFMONTH", TimeUnit.DAY);
+
+	/**
+	 * The SQL <code>DAYOFWEEK</code> operator. Returns the DOW
+	 * from a DATETIME  E.g.<br>
+	 * <code>DAYOFWEEK(date '2008-9-23')</code> returns <code>
+	 * 2</code>
+	 */
+	public static final SqlDatePartFunction DAYOFWEEK =
+		new SqlDatePartFunction("DAYOFWEEK", TimeUnit.DOW);
+
+	/**
+	 * The SQL <code>HOUR</code> operator. Returns the Hour
+	 * from a DATETIME  E.g.<br>
+	 * <code>HOUR(timestamp '2008-9-23 01:23:45')</code> returns <code>
+	 * 1</code>
+	 */
+	public static final SqlDatePartFunction HOUR =
+		new SqlDatePartFunction("HOUR", TimeUnit.HOUR);
+
+	/**
+	 * The SQL <code>MINUTE</code> operator. Returns the Minute
+	 * from a DATETIME  E.g.<br>
+	 * <code>MINUTE(timestamp '2008-9-23 01:23:45')</code> returns <code>
+	 * 23</code>
+	 */
+	public static final SqlDatePartFunction MINUTE =
+		new SqlDatePartFunction("MINUTE", TimeUnit.MINUTE);
+
+	/**
+	 * The SQL <code>SECOND</code> operator. Returns the Second
+	 * from a DATETIME  E.g.<br>
+	 * <code>SECOND(timestamp '2008-9-23 01:23:45')</code> returns <code>
+	 * 45</code>
+	 */
+	public static final SqlDatePartFunction SECOND =
+		new SqlDatePartFunction("SECOND", TimeUnit.SECOND);
+
+	/**
+	 * The ELEMENT operator, used to convert a multiset with only one item to a
+	 * "regular" type. Example ... log(ELEMENT(MULTISET[1])) ...
+	 */
+	public static final SqlFunction ELEMENT =
+		new SqlFunction(
+			"ELEMENT",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.MULTISET_ELEMENT_NULLABLE,
+			null,
+			OperandTypes.COLLECTION,
+			SqlFunctionCategory.SYSTEM);
+
+	/**
+	 * The item operator {@code [ ... ]}, used to access a given element of an
+	 * array or map. For example, {@code myArray[3]} or {@code "myMap['foo']"}.
+	 *
+	 * <p>The SQL standard calls the ARRAY variant a
+	 * &lt;array element reference&gt;. Index is 1-based. The standard says
+	 * to raise "data exception - array element error" but we currently return
+	 * null.</p>
+	 *
+	 * <p>MAP is not standard SQL.</p>
+	 */
+	public static final SqlOperator ITEM = new SqlItemOperator();
+
+	/**
+	 * The ARRAY Value Constructor. e.g. "<code>ARRAY[1, 2, 3]</code>".
+	 */
+	public static final SqlArrayValueConstructor ARRAY_VALUE_CONSTRUCTOR =
+		new SqlArrayValueConstructor();
+
+	/**
+	 * The MAP Value Constructor,
+	 * e.g. "<code>MAP['washington', 1, 'obama', 44]</code>".
+	 */
+	public static final SqlMapValueConstructor MAP_VALUE_CONSTRUCTOR =
+		new SqlMapValueConstructor();
+
+	/**
+	 * The internal "$SLICE" operator takes a multiset of records and returns a
+	 * multiset of the first column of those records.
+	 *
+	 * <p>It is introduced when multisets of scalar types are created, in order
+	 * to keep types consistent. For example, <code>MULTISET [5]</code> has type
+	 * <code>INTEGER MULTISET</code> but is translated to an expression of type
+	 * <code>RECORD(INTEGER EXPR$0) MULTISET</code> because in our internal
+	 * representation of multisets, every element must be a record. Applying the
+	 * "$SLICE" operator to this result converts the type back to an <code>
+	 * INTEGER MULTISET</code> multiset value.
+	 *
+	 * <p><code>$SLICE</code> is often translated away when the multiset type is
+	 * converted back to scalar values.
+	 */
+	public static final SqlInternalOperator SLICE =
+		new SqlInternalOperator(
+			"$SLICE",
+			SqlKind.OTHER,
+			0,
+			false,
+			ReturnTypes.MULTISET_PROJECT0,
+			null,
+			OperandTypes.RECORD_COLLECTION) {
+		};
+
+	/**
+	 * The internal "$ELEMENT_SLICE" operator returns the first field of the
+	 * only element of a multiset.
+	 *
+	 * <p>It is introduced when multisets of scalar types are created, in order
+	 * to keep types consistent. For example, <code>ELEMENT(MULTISET [5])</code>
+	 * is translated to <code>$ELEMENT_SLICE(MULTISET (VALUES ROW (5
+	 * EXPR$0))</code> It is translated away when the multiset type is converted
+	 * back to scalar values.</p>
+	 *
+	 * <p>NOTE: jhyde, 2006/1/9: Usages of this operator are commented out, but
+	 * I'm not deleting the operator, because some multiset tests are disabled,
+	 * and we may need this operator to get them working!</p>
+	 */
+	public static final SqlInternalOperator ELEMENT_SLICE =
+		new SqlInternalOperator(
+			"$ELEMENT_SLICE",
+			SqlKind.OTHER,
+			0,
+			false,
+			ReturnTypes.MULTISET_RECORD,
+			null,
+			OperandTypes.MULTISET) {
+			public void unparse(
+				SqlWriter writer,
+				SqlCall call,
+				int leftPrec,
+				int rightPrec) {
+				SqlUtil.unparseFunctionSyntax(
+					this,
+					writer, call);
+			}
+		};
+
+	/**
+	 * The internal "$SCALAR_QUERY" operator returns a scalar value from a
+	 * record type. It assumes the record type only has one field, and returns
+	 * that field as the output.
+	 */
+	public static final SqlInternalOperator SCALAR_QUERY =
+		new SqlInternalOperator(
+			"$SCALAR_QUERY",
+			SqlKind.SCALAR_QUERY,
+			0,
+			false,
+			ReturnTypes.RECORD_TO_SCALAR,
+			null,
+			OperandTypes.RECORD_TO_SCALAR) {
+			public void unparse(
+				SqlWriter writer,
+				SqlCall call,
+				int leftPrec,
+				int rightPrec) {
+				final SqlWriter.Frame frame = writer.startList("(", ")");
+				call.operand(0).unparse(writer, 0, 0);
+				writer.endList(frame);
+			}
+
+			public boolean argumentMustBeScalar(int ordinal) {
+				// Obvious, really.
+				return false;
+			}
+		};
+
+	/**
+	 * The CARDINALITY operator, used to retrieve the number of elements in a
+	 * MULTISET, ARRAY or MAP.
+	 */
+	public static final SqlFunction CARDINALITY =
+		new SqlFunction(
+			"CARDINALITY",
+			SqlKind.OTHER_FUNCTION,
+			ReturnTypes.INTEGER_NULLABLE,
+			null,
+			OperandTypes.COLLECTION_OR_MAP,
+			SqlFunctionCategory.SYSTEM);
+
+	/**
+	 * The COLLECT operator. Multiset aggregator function.
+	 */
+	public static final SqlAggFunction COLLECT =
+		new SqlAggFunction("COLLECT",
+			null,
+			SqlKind.COLLECT,
+			ReturnTypes.TO_MULTISET,
+			null,
+			OperandTypes.ANY,
+			SqlFunctionCategory.SYSTEM, false, false) {
+		};
+
+	/**
+	 * The FUSION operator. Multiset aggregator function.
+	 */
+	public static final SqlFunction FUSION =
+		new SqlAggFunction("FUSION", null,
+			SqlKind.FUSION,
+			ReturnTypes.ARG0,
+			null,
+			OperandTypes.MULTISET,
+			SqlFunctionCategory.SYSTEM, false, false) {
+		};
+
+	/**
+	 * The sequence next value function: <code>NEXT VALUE FOR sequence</code>
+	 */
+	public static final SqlOperator NEXT_VALUE =
+		new SqlSequenceValueOperator(SqlKind.NEXT_VALUE);
+
+	/**
+	 * The sequence current value function: <code>CURRENT VALUE FOR
+	 * sequence</code>
+	 */
+	public static final SqlOperator CURRENT_VALUE =
+		new SqlSequenceValueOperator(SqlKind.CURRENT_VALUE);
+
+	/**
+	 * The <code>TABLESAMPLE</code> operator.
+	 *
+	 * <p>Examples:
+	 *
+	 * <ul>
+	 * <li><code>&lt;query&gt; TABLESAMPLE SUBSTITUTE('sampleName')</code>
+	 * (non-standard)
+	 * <li><code>&lt;query&gt; TABLESAMPLE BERNOULLI(&lt;percent&gt;)
+	 * [REPEATABLE(&lt;seed&gt;)]</code> (standard, but not implemented for FTRS
+	 * yet)
+	 * <li><code>&lt;query&gt; TABLESAMPLE SYSTEM(&lt;percent&gt;)
+	 * [REPEATABLE(&lt;seed&gt;)]</code> (standard, but not implemented for FTRS
+	 * yet)
+	 * </ul>
+	 *
+	 * <p>Operand #0 is a query or table; Operand #1 is a {@link SqlSampleSpec}
+	 * wrapped in a {@link SqlLiteral}.
+	 */
+	public static final SqlSpecialOperator TABLESAMPLE =
+		new SqlSpecialOperator(
+			"TABLESAMPLE",
+			SqlKind.TABLESAMPLE,
+			20,
+			true,
+			ReturnTypes.ARG0,
+			null,
+			OperandTypes.VARIADIC) {
+			public void unparse(
+				SqlWriter writer,
+				SqlCall call,
+				int leftPrec,
+				int rightPrec) {
+				call.operand(0).unparse(writer, leftPrec, 0);
+				writer.keyword("TABLESAMPLE");
+				call.operand(1).unparse(writer, 0, rightPrec);
+			}
+		};
+
+	/** The {@code TUMBLE} group function. */
+	public static final SqlGroupFunction TUMBLE =
+		new SqlGroupFunction(SqlKind.TUMBLE, null,
+			OperandTypes.or(OperandTypes.DATETIME_INTERVAL,
+				OperandTypes.DATETIME_INTERVAL_TIME)) {
+			@Override List<SqlGroupFunction> getAuxiliaryFunctions() {
+				return ImmutableList.of(TUMBLE_START, TUMBLE_END);
+			}
+		};
+
+	/** The {@code TUMBLE_START} auxiliary function of
+	 * the {@code TUMBLE} group function. */
+	public static final SqlGroupFunction TUMBLE_START =
+		TUMBLE.auxiliary(SqlKind.TUMBLE_START);
+
+	/** The {@code TUMBLE_END} auxiliary function of
+	 * the {@code TUMBLE} group function. */
+	public static final SqlGroupFunction TUMBLE_END =
+		TUMBLE.auxiliary(SqlKind.TUMBLE_END);
+
+	/** The {@code HOP} group function. */
+	public static final SqlGroupFunction HOP =
+		new SqlGroupFunction(SqlKind.HOP, null,
+			OperandTypes.or(OperandTypes.DATETIME_INTERVAL_INTERVAL,
+				OperandTypes.DATETIME_INTERVAL_INTERVAL_TIME)) {
+			@Override List<SqlGroupFunction> getAuxiliaryFunctions() {
+				return ImmutableList.of(HOP_START, HOP_END);
+			}
+		};
+
+	/** The {@code HOP_START} auxiliary function of
+	 * the {@code HOP} group function. */
+	public static final SqlGroupFunction HOP_START =
+		HOP.auxiliary(SqlKind.HOP_START);
+
+	/** The {@code HOP_END} auxiliary function of
+	 * the {@code HOP} group function. */
+	public static final SqlGroupFunction HOP_END =
+		HOP.auxiliary(SqlKind.HOP_END);
+
+	/** The {@code SESSION} group function. */
+	public static final SqlGroupFunction SESSION =
+		new SqlGroupFunction(SqlKind.SESSION, null,
+			OperandTypes.or(OperandTypes.DATETIME_INTERVAL,
+				OperandTypes.DATETIME_INTERVAL_TIME)) {
+			@Override List<SqlGroupFunction> getAuxiliaryFunctions() {
+				return ImmutableList.of(SESSION_START, SESSION_END);
+			}
+		};
+
+	/** The {@code SESSION_START} auxiliary function of
+	 * the {@code SESSION} group function. */
+	public static final SqlGroupFunction SESSION_START =
+		SESSION.auxiliary(SqlKind.SESSION_START);
+
+	/** The {@code SESSION_END} auxiliary function of
+	 * the {@code SESSION} group function. */
+	public static final SqlGroupFunction SESSION_END =
+		SESSION.auxiliary(SqlKind.SESSION_END);
+
+	/** {@code |} operator to create alternate patterns
+	 * within {@code MATCH_RECOGNIZE}.
+	 *
+	 * <p>If {@code p1} and {@code p2} are patterns then {@code p1 | p2} is a
+	 * pattern that matches {@code p1} or {@code p2}. */
+	public static final SqlBinaryOperator PATTERN_ALTER =
+		new SqlBinaryOperator("|", SqlKind.PATTERN_ALTER, 70, true, null, null, null);
+
+	/** Operator to concatenate patterns within {@code MATCH_RECOGNIZE}.
+	 *
+	 * <p>If {@code p1} and {@code p2} are patterns then {@code p1 p2} is a
+	 * pattern that matches {@code p1} followed by {@code p2}. */
+	public static final SqlBinaryOperator PATTERN_CONCAT =
+		new SqlBinaryOperator("", SqlKind.PATTERN_CONCAT, 80, true, null, null, null);
+
+	/** Operator to quantify patterns within {@code MATCH_RECOGNIZE}.
+	 *
+	 * <p>If {@code p} is a pattern then {@code p{3, 5}} is a
+	 * pattern that matches between 3 and 5 occurrences of {@code p}. */
+	public static final SqlSpecialOperator PATTERN_QUANTIFIER =
+		new SqlSpecialOperator("PATTERN_QUANTIFIER", SqlKind.PATTERN_QUANTIFIER,
+			90) {
+			@Override public void unparse(SqlWriter writer, SqlCall call,
+				int leftPrec, int rightPrec) {
+				call.operand(0).unparse(writer, this.getLeftPrec(), this.getRightPrec());
+				int startNum = ((SqlNumericLiteral) call.operand(1)).intValue(true);
+				SqlNumericLiteral endRepNum = call.operand(2);
+				boolean isReluctant = ((SqlLiteral) call.operand(3)).booleanValue();
+				int endNum = endRepNum.intValue(true);
+				if (startNum == endNum) {
+					writer.keyword("{ " + startNum + " }");
+				} else {
+					if (endNum == -1) {
+						if (startNum == 0) {
+							writer.keyword("*");
+						} else if (startNum == 1) {
+							writer.keyword("+");
+						} else {
+							writer.keyword("{ " + startNum + ", }");
+						}
+					} else {
+						if (startNum == 0 && endNum == 1) {
+							writer.keyword("?");
+						} else if (startNum == -1) {
+							writer.keyword("{ , " + endNum + " }");
+						} else {
+							writer.keyword("{ " + startNum + ", " + endNum + " }");
+						}
+					}
+					if (isReluctant) {
+						writer.keyword("?");
+					}
+				}
+			}
+		};
+
+	/** {@code PERMUTE} operator to combine patterns within
+	 * {@code MATCH_RECOGNIZE}.
+	 *
+	 * <p>If {@code p1} and {@code p2} are patterns then {@code PERMUTE (p1, p2)}
+	 * is a pattern that matches all permutations of {@code p1} and
+	 * {@code p2}. */
+	public static final SqlSpecialOperator PATTERN_PERMUTE =
+		new SqlSpecialOperator("PATTERN_PERMUTE", SqlKind.PATTERN_PERMUTE, 100) {
+			@Override public void unparse(SqlWriter writer, SqlCall call,
+				int leftPrec, int rightPrec) {
+				writer.keyword("PERMUTE");
+				SqlWriter.Frame frame = writer.startList("(", ")");
+				for (int i = 0; i < call.getOperandList().size(); i++) {
+					SqlNode pattern = call.getOperandList().get(i);
+					pattern.unparse(writer, 0, 0);
+					if (i != call.getOperandList().size() - 1) {
+						writer.print(",");
+					}
+				}
+				writer.endList(frame);
+			}
+		};
+
+	/** {@code EXCLUDE} operator within {@code MATCH_RECOGNIZE}.
+	 *
+	 * <p>If {@code p} is a pattern then {@code {- p -} }} is a
+	 * pattern that excludes {@code p} from the output. */
+	public static final SqlSpecialOperator PATTERN_EXCLUDE =
+		new SqlSpecialOperator("PATTERN_EXCLUDE", SqlKind.PATTERN_EXCLUDED,
+			100) {
+			@Override public void unparse(SqlWriter writer, SqlCall call,
+				int leftPrec, int rightPrec) {
+				SqlWriter.Frame frame = writer.startList("{-", "-}");
+				SqlNode node = call.getOperandList().get(0);
+				node.unparse(writer, 0, 0);
+				writer.endList(frame);
+			}
+		};
+
+	//~ Methods ----------------------------------------------------------------
+
+	/**
+	 * Returns the standard operator table, creating it if necessary.
+	 */
+	public static synchronized SqlStdOperatorTable instance() {
+		if (instance == null) {
+			// Creates and initializes the standard operator table.
+			// Uses two-phase construction, because we can't initialize the
+			// table until the constructor of the sub-class has completed.
+			instance = new SqlStdOperatorTable();
+			instance.init();
+		}
+		return instance;
+	}
+
+	/** Returns the group function for which a given kind is an auxiliary
+	 * function, or null if it is not an auxiliary function. */
+	public static SqlGroupFunction auxiliaryToGroup(SqlKind kind) {
+		switch (kind) {
+			case TUMBLE_START:
+			case TUMBLE_END:
+				return TUMBLE;
+			case HOP_START:
+			case HOP_END:
+				return HOP;
+			case SESSION_START:
+			case SESSION_END:
+				return SESSION;
+			default:
+				return null;
+		}
+	}
+
+	/** Converts a call to a grouped auxiliary function
+	 * to a call to the grouped window function. For other calls returns null.
+	 *
+	 * <p>For example, converts {@code TUMBLE_START(rowtime, INTERVAL '1' HOUR))}
+	 * to {@code TUMBLE(rowtime, INTERVAL '1' HOUR))}. */
+	public static SqlCall convertAuxiliaryToGroupCall(SqlCall call) {
+		final SqlOperator op = call.getOperator();
+		if (op instanceof SqlGroupFunction
+			&& op.isGroupAuxiliary()) {
+			return copy(call, ((SqlGroupFunction) op).groupFunction);
+		}
+		return null;
+	}
+
+	/** Converts a call to a grouped window function to a call to its auxiliary
+	 * window function(s). For other calls returns null.
+	 *
+	 * <p>For example, converts {@code TUMBLE_START(rowtime, INTERVAL '1' HOUR))}
+	 * to {@code TUMBLE(rowtime, INTERVAL '1' HOUR))}. */
+	public static List<Pair<SqlNode, AuxiliaryConverter>>
+	convertGroupToAuxiliaryCalls(SqlCall call) {
+		final SqlOperator op = call.getOperator();
+		if (op instanceof SqlGroupFunction
+			&& op.isGroup()) {
+			ImmutableList.Builder<Pair<SqlNode, AuxiliaryConverter>> builder =
+				ImmutableList.builder();
+			for (final SqlGroupFunction f
+				: ((SqlGroupFunction) op).getAuxiliaryFunctions()) {
+				builder.add(
+					Pair.<SqlNode, AuxiliaryConverter>of(copy(call, f),
+						new AuxiliaryConverter.Impl(f)));
+			}
+			return builder.build();
+		}
+		return ImmutableList.of();
+	}
+
+	/** Creates a copy of a call with a new operator. */
+	private static SqlCall copy(SqlCall call, SqlOperator operator) {
+		final List<SqlNode> list = call.getOperandList();
+		return new SqlBasicCall(operator, list.toArray(new SqlNode[list.size()]),
+			call.getParserPosition());
+	}
+
+}
+
+// End SqlStdOperatorTable.java
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/validate/AggChecker.java b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
new file mode 100644
index 0000000000000..56a364fd545b9
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.validate;
+
+/*
+ * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT UNTIL CALCITE-1761 IS FIXED.
+ */
+
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlWindow;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.util.Litmus;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+
+import static org.apache.calcite.util.Static.RESOURCE;
+
+/**
+ * Visitor which throws an exception if any component of the expression is not a
+ * group expression.
+ */
+class AggChecker extends SqlBasicVisitor<Void> {
+	//~ Instance fields --------------------------------------------------------
+
+	private final Deque<SqlValidatorScope> scopes = new ArrayDeque<>();
+	private final List<SqlNode> extraExprs;
+	private final List<SqlNode> groupExprs;
+	private boolean distinct;
+	private SqlValidatorImpl validator;
+
+	//~ Constructors -----------------------------------------------------------
+
+	/**
+	 * Creates an AggChecker.
+	 *
+	 * @param validator  Validator
+	 * @param scope      Scope
+	 * @param groupExprs Expressions in GROUP BY (or SELECT DISTINCT) clause,
+	 *                   that are therefore available
+	 * @param distinct   Whether aggregation checking is because of a SELECT
+	 *                   DISTINCT clause
+	 */
+	AggChecker(
+		SqlValidatorImpl validator,
+		AggregatingScope scope,
+		List<SqlNode> extraExprs,
+		List<SqlNode> groupExprs,
+		boolean distinct) {
+		this.validator = validator;
+		this.extraExprs = extraExprs;
+		this.groupExprs = groupExprs;
+		this.distinct = distinct;
+		this.scopes.push(scope);
+	}
+
+	//~ Methods ----------------------------------------------------------------
+
+	boolean isGroupExpr(SqlNode expr) {
+		for (SqlNode groupExpr : groupExprs) {
+			if (groupExpr.equalsDeep(expr, Litmus.IGNORE)) {
+				return true;
+			}
+		}
+
+		for (SqlNode extraExpr : extraExprs) {
+			if (extraExpr.equalsDeep(expr, Litmus.IGNORE)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	public Void visit(SqlIdentifier id) {
+		if (isGroupExpr(id) || id.isStar()) {
+			// Star may validly occur in "SELECT COUNT(*) OVER w"
+			return null;
+		}
+
+		// Is it a call to a parentheses-free function?
+		SqlCall call =
+			SqlUtil.makeCall(
+				validator.getOperatorTable(),
+				id);
+		if (call != null) {
+			return call.accept(this);
+		}
+
+		// Didn't find the identifier in the group-by list as is, now find
+		// it fully-qualified.
+		// TODO: It would be better if we always compared fully-qualified
+		// to fully-qualified.
+		final SqlQualified fqId = scopes.peek().fullyQualify(id);
+		if (isGroupExpr(fqId.identifier)) {
+			return null;
+		}
+		SqlNode originalExpr = validator.getOriginal(id);
+		final String exprString = originalExpr.toString();
+		throw validator.newValidationError(originalExpr,
+			distinct
+				? RESOURCE.notSelectDistinctExpr(exprString)
+				: RESOURCE.notGroupExpr(exprString));
+	}
+
+	public Void visit(SqlCall call) {
+		final SqlValidatorScope scope = scopes.peek();
+		if (call.getOperator().isAggregator()) {
+			if (distinct) {
+				if (scope instanceof AggregatingSelectScope) {
+					SqlNodeList selectList =
+						((SqlSelect) scope.getNode()).getSelectList();
+
+					// Check if this aggregation function is just an element in the select
+					for (SqlNode sqlNode : selectList) {
+						if (sqlNode.getKind() == SqlKind.AS) {
+							sqlNode = ((SqlCall) sqlNode).operand(0);
+						}
+
+						if (validator.expand(sqlNode, scope)
+							.equalsDeep(call, Litmus.IGNORE)) {
+							return null;
+						}
+					}
+				}
+
+				// Cannot use agg fun in ORDER BY clause if have SELECT DISTINCT.
+				SqlNode originalExpr = validator.getOriginal(call);
+				final String exprString = originalExpr.toString();
+				throw validator.newValidationError(call,
+					RESOURCE.notSelectDistinctExpr(exprString));
+			}
+
+			// For example, 'sum(sal)' in 'SELECT sum(sal) FROM emp GROUP
+			// BY deptno'
+			return null;
+		}
+		if (call.getKind() == SqlKind.FILTER) {
+			call.operand(0).accept(this);
+			return null;
+		}
+		// Visit the operand in window function
+		if (call.getKind() == SqlKind.OVER) {
+			for (SqlNode operand : call.<SqlCall>operand(0).getOperandList()) {
+				operand.accept(this);
+			}
+			// Check the OVER clause
+			final SqlNode over = call.operand(1);
+			if (over instanceof SqlCall) {
+				over.accept(this);
+			} else if (over instanceof SqlIdentifier) {
+				// Check the corresponding SqlWindow in WINDOW clause
+				final SqlWindow window =
+					scope.lookupWindow(((SqlIdentifier) over).getSimple());
+				window.getPartitionList().accept(this);
+				window.getOrderList().accept(this);
+			}
+		}
+		if (isGroupExpr(call)) {
+			// This call matches an expression in the GROUP BY clause.
+			return null;
+		}
+
+		final SqlCall groupCall =
+			SqlStdOperatorTable.convertAuxiliaryToGroupCall(call);
+		if (groupCall != null) {
+			if (isGroupExpr(groupCall)) {
+				// This call is an auxiliary function that matches a group call in the
+				// GROUP BY clause.
+				//
+				// For example TUMBLE_START is an auxiliary of the TUMBLE
+				// group function, and
+				//   TUMBLE_START(rowtime, INTERVAL '1' HOUR)
+				// matches
+				//   TUMBLE(rowtime, INTERVAL '1' HOUR')
+				return null;
+			}
+			throw validator.newValidationError(groupCall,
+				RESOURCE.auxiliaryWithoutMatchingGroupCall(
+					call.getOperator().getName(), groupCall.getOperator().getName()));
+		}
+
+		if (call.isA(SqlKind.QUERY)) {
+			// Allow queries for now, even though they may contain
+			// references to forbidden columns.
+			return null;
+		}
+
+		// Switch to new scope.
+		SqlValidatorScope newScope = scope.getOperandScope(call);
+		scopes.push(newScope);
+
+		// Visit the operands (only expressions).
+		call.getOperator()
+			.acceptCall(this, call, true, ArgHandlerImpl.<Void>instance());
+
+		// Restore scope.
+		scopes.pop();
+		return null;
+	}
+
+}
+
+// End AggChecker.java
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/AuxiliaryConverter.java b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/AuxiliaryConverter.java
new file mode 100644
index 0000000000000..98e119947bd4a
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/AuxiliaryConverter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql2rel;
+
+/*
+ * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT UNTIL CALCITE-1761 IS FIXED.
+ */
+
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+
+/** Converts an expression for a group window function (e.g. TUMBLE)
+ * into an expression for an auxiliary group function (e.g. TUMBLE_START).
+ *
+ * @see SqlStdOperatorTable#TUMBLE
+ */
+public interface AuxiliaryConverter {
+	/** Converts an expression.
+	 *
+	 * @param rexBuilder Rex  builder
+	 * @param groupCall Call to the group function, e.g. "TUMBLE($2, 36000)"
+	 * @param e Expression holding result of the group function, e.g. "$0"
+	 *
+	 * @return Expression for auxiliary function, e.g. "$0 + 36000" converts
+	 * the result of TUMBLE to the result of TUMBLE_END
+	 */
+	RexNode convert(RexBuilder rexBuilder, RexNode groupCall, RexNode e);
+
+	/** Simple implementation of {@link AuxiliaryConverter}. */
+	class Impl implements AuxiliaryConverter {
+		private final SqlFunction f;
+
+		public Impl(SqlFunction f) {
+			this.f = f;
+		}
+
+		public RexNode convert(RexBuilder rexBuilder, RexNode groupCall,
+			RexNode e) {
+			return rexBuilder.makeCall(this.f, e);
+			// FLINK QUICK FIX
+			// we do not use this logic right now
+//      switch (f.getKind()) {
+//      case TUMBLE_START:
+//      case HOP_START:
+//      case SESSION_START:
+//      case SESSION_END: // TODO: ?
+//        return e;
+//      case TUMBLE_END:
+//        return rexBuilder.makeCall(
+//            SqlStdOperatorTable.PLUS, e,
+//            ((RexCall) groupCall).operands.get(1));
+//      case HOP_END:
+//        return rexBuilder.makeCall(
+//            SqlStdOperatorTable.PLUS, e,
+//            ((RexCall) groupCall).operands.get(2));
+//      default:
+//        throw new AssertionError("unknown: " + f);
+//      }
+		}
+	}
+}
+
+// End AuxiliaryConverter.java
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
new file mode 100644
index 0000000000000..b7f9ce78b4fb7
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -0,0 +1,5356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql2rel;
+
+/*
+ * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT UNTIL CALCITE-1761 IS FIXED.
+ */
+
+import org.apache.calcite.avatica.util.Spaces;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptSamplingParameters;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.prepare.RelOptTableImpl;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.SingleRel;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Collect;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.core.Sample;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.Uncollect;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalCorrelate;
+import org.apache.calcite.rel.logical.LogicalFilter;
+import org.apache.calcite.rel.logical.LogicalIntersect;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rel.logical.LogicalMinus;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
+import org.apache.calcite.rel.logical.LogicalTableModify;
+import org.apache.calcite.rel.logical.LogicalTableScan;
+import org.apache.calcite.rel.logical.LogicalUnion;
+import org.apache.calcite.rel.logical.LogicalValues;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
+import org.apache.calcite.rel.metadata.RelColumnMapping;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.stream.Delta;
+import org.apache.calcite.rel.stream.LogicalDelta;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCallBinding;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexFieldCollation;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexWindowBound;
+import org.apache.calcite.schema.ModifiableTable;
+import org.apache.calcite.schema.ModifiableView;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.schema.Wrapper;
+import org.apache.calcite.sql.JoinConditionType;
+import org.apache.calcite.sql.JoinType;
+import org.apache.calcite.sql.SemiJoinType;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlBasicCall;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlDataTypeSpec;
+import org.apache.calcite.sql.SqlDelete;
+import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlInsert;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlJoin;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlMatchRecognize;
+import org.apache.calcite.sql.SqlMerge;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlNumericLiteral;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlOrderBy;
+import org.apache.calcite.sql.SqlSampleSpec;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlSelectKeyword;
+import org.apache.calcite.sql.SqlSetOperator;
+import org.apache.calcite.sql.SqlUnnestOperator;
+import org.apache.calcite.sql.SqlUpdate;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlValuesOperator;
+import org.apache.calcite.sql.SqlWindow;
+import org.apache.calcite.sql.SqlWith;
+import org.apache.calcite.sql.SqlWithItem;
+import org.apache.calcite.sql.fun.SqlCountAggFunction;
+import org.apache.calcite.sql.fun.SqlInOperator;
+import org.apache.calcite.sql.fun.SqlRowOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.sql.type.TableFunctionReturnTypeInference;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.calcite.sql.validate.AggregatingSelectScope;
+import org.apache.calcite.sql.validate.CollectNamespace;
+import org.apache.calcite.sql.validate.DelegatingScope;
+import org.apache.calcite.sql.validate.ListScope;
+import org.apache.calcite.sql.validate.ParameterScope;
+import org.apache.calcite.sql.validate.SelectScope;
+import org.apache.calcite.sql.validate.SqlMonotonicity;
+import org.apache.calcite.sql.validate.SqlNameMatcher;
+import org.apache.calcite.sql.validate.SqlQualified;
+import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction;
+import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.sql.validate.SqlValidatorImpl;
+import org.apache.calcite.sql.validate.SqlValidatorNamespace;
+import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.calcite.sql.validate.SqlValidatorTable;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.Litmus;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.NumberUtil;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+import org.apache.calcite.util.trace.CalciteTrace;
+
+import com.google.common.base.Function;
+import org.apache.flink.util.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
+
+/**
+ * Converts a SQL parse tree (consisting of
+ * {@link org.apache.calcite.sql.SqlNode} objects) into a relational algebra
+ * expression (consisting of {@link org.apache.calcite.rel.RelNode} objects).
+ *
+ * <p>The public entry points are: {@link #convertQuery},
+ * {@link #convertExpression(SqlNode)}.
+ */
+public class SqlToRelConverter {
+	//~ Static fields/initializers ---------------------------------------------
+
+	protected static final Logger SQL2REL_LOGGER =
+		CalciteTrace.getSqlToRelTracer();
+
+	private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
+	/** Size of the smallest IN list that will be converted to a semijoin to a
+	 * static table. */
+	public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+	@Deprecated // to be removed before 2.0
+	public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+		DEFAULT_IN_SUB_QUERY_THRESHOLD;
+
+	//~ Instance fields --------------------------------------------------------
+
+	protected final SqlValidator validator;
+	protected final RexBuilder rexBuilder;
+	protected final Prepare.CatalogReader catalogReader;
+	protected final RelOptCluster cluster;
+	private SubQueryConverter subQueryConverter;
+	protected final List<RelNode> leaves = new ArrayList<>();
+	private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
+	private final SqlOperatorTable opTab;
+	protected final RelDataTypeFactory typeFactory;
+	private final SqlNodeToRexConverter exprConverter;
+	private int explainParamCount;
+	public final SqlToRelConverter.Config config;
+
+	/**
+	 * Fields used in name resolution for correlated sub-queries.
+	 */
+	private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred =
+		new HashMap<>();
+
+	/**
+	 * Stack of names of datasets requested by the <code>
+	 * TABLE(SAMPLE(&lt;datasetName&gt;, &lt;query&gt;))</code> construct.
+	 */
+	private final Deque<String> datasetStack = new ArrayDeque<>();
+
+	/**
+	 * Mapping of non-correlated sub-queries that have been converted to their
+	 * equivalent constants. Used to avoid re-evaluating the sub-query if it's
+	 * already been evaluated.
+	 */
+	private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
+		new HashMap<>();
+
+	public final RelOptTable.ViewExpander viewExpander;
+
+	//~ Constructors -----------------------------------------------------------
+	/**
+	 * Creates a converter.
+	 *
+	 * @param viewExpander    Preparing statement
+	 * @param validator       Validator
+	 * @param catalogReader   Schema
+	 * @param planner         Planner
+	 * @param rexBuilder      Rex builder
+	 * @param convertletTable Expression converter
+	 */
+	@Deprecated // to be removed before 2.0
+	public SqlToRelConverter(
+		RelOptTable.ViewExpander viewExpander,
+		SqlValidator validator,
+		Prepare.CatalogReader catalogReader,
+		RelOptPlanner planner,
+		RexBuilder rexBuilder,
+		SqlRexConvertletTable convertletTable) {
+		this(viewExpander, validator, catalogReader,
+			RelOptCluster.create(planner, rexBuilder), convertletTable,
+			Config.DEFAULT);
+	}
+
+	@Deprecated // to be removed before 2.0
+	public SqlToRelConverter(
+		RelOptTable.ViewExpander viewExpander,
+		SqlValidator validator,
+		Prepare.CatalogReader catalogReader,
+		RelOptCluster cluster,
+		SqlRexConvertletTable convertletTable) {
+		this(viewExpander, validator, catalogReader, cluster, convertletTable,
+			Config.DEFAULT);
+	}
+
+	/* Creates a converter. */
+	public SqlToRelConverter(
+		RelOptTable.ViewExpander viewExpander,
+		SqlValidator validator,
+		Prepare.CatalogReader catalogReader,
+		RelOptCluster cluster,
+		SqlRexConvertletTable convertletTable,
+		Config config) {
+		this.viewExpander = viewExpander;
+		this.opTab =
+			(validator
+				== null) ? SqlStdOperatorTable.instance()
+				: validator.getOperatorTable();
+		this.validator = validator;
+		this.catalogReader = catalogReader;
+		this.subQueryConverter = new NoOpSubQueryConverter();
+		this.rexBuilder = cluster.getRexBuilder();
+		this.typeFactory = rexBuilder.getTypeFactory();
+		this.cluster = Preconditions.checkNotNull(cluster);
+		this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable);
+		this.explainParamCount = 0;
+		this.config = new ConfigBuilder().withConfig(config).build();
+	}
+
+	//~ Methods ----------------------------------------------------------------
+
+	/**
+	 * @return the RelOptCluster in use.
+	 */
+	public RelOptCluster getCluster() {
+		return cluster;
+	}
+
+	/**
+	 * Returns the row-expression builder.
+	 */
+	public RexBuilder getRexBuilder() {
+		return rexBuilder;
+	}
+
+	/**
+	 * Returns the number of dynamic parameters encountered during translation;
+	 * this must only be called after {@link #convertQuery}.
+	 *
+	 * @return number of dynamic parameters
+	 */
+	public int getDynamicParamCount() {
+		return dynamicParamSqlNodes.size();
+	}
+
+	/**
+	 * Returns the type inferred for a dynamic parameter.
+	 *
+	 * @param index 0-based index of dynamic parameter
+	 * @return inferred type, never null
+	 */
+	public RelDataType getDynamicParamType(int index) {
+		SqlNode sqlNode = dynamicParamSqlNodes.get(index);
+		if (sqlNode == null) {
+			throw Util.needToImplement("dynamic param type inference");
+		}
+		return validator.getValidatedNodeType(sqlNode);
+	}
+
+	/**
+	 * Returns the current count of the number of dynamic parameters in an
+	 * EXPLAIN PLAN statement.
+	 *
+	 * @param increment if true, increment the count
+	 * @return the current count before the optional increment
+	 */
+	public int getDynamicParamCountInExplain(boolean increment) {
+		int retVal = explainParamCount;
+		if (increment) {
+			++explainParamCount;
+		}
+		return retVal;
+	}
+
+	/**
+	 * @return mapping of non-correlated sub-queries that have been converted to
+	 * the constants that they evaluate to
+	 */
+	public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
+		return mapConvertedNonCorrSubqs;
+	}
+
+	/**
+	 * Adds to the current map of non-correlated converted sub-queries the
+	 * elements from another map that contains non-correlated sub-queries that
+	 * have been converted by another SqlToRelConverter.
+	 *
+	 * @param alreadyConvertedNonCorrSubqs the other map
+	 */
+	public void addConvertedNonCorrSubqs(
+		Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
+		mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
+	}
+
+	/**
+	 * Sets a new SubQueryConverter. To have any effect, this must be called
+	 * before any convert method.
+	 *
+	 * @param converter new SubQueryConverter
+	 */
+	public void setSubQueryConverter(SubQueryConverter converter) {
+		subQueryConverter = converter;
+	}
+
+	/**
+	 * Sets the number of dynamic parameters in the current EXPLAIN PLAN
+	 * statement.
+	 *
+	 * @param explainParamCount number of dynamic parameters in the statement
+	 */
+	public void setDynamicParamCountInExplain(int explainParamCount) {
+		assert config.isExplain();
+		this.explainParamCount = explainParamCount;
+	}
+
+	private void checkConvertedType(SqlNode query, RelNode result) {
+		if (query.isA(SqlKind.DML)) {
+			return;
+		}
+		// Verify that conversion from SQL to relational algebra did
+		// not perturb any type information.  (We can't do this if the
+		// SQL statement is something like an INSERT which has no
+		// validator type information associated with its result,
+		// hence the namespace check above.)
+		final List<RelDataTypeField> validatedFields =
+			validator.getValidatedNodeType(query).getFieldList();
+		final RelDataType validatedRowType =
+			validator.getTypeFactory().createStructType(
+				Pair.right(validatedFields),
+				SqlValidatorUtil.uniquify(Pair.left(validatedFields),
+					catalogReader.nameMatcher().isCaseSensitive()));
+
+		final List<RelDataTypeField> convertedFields =
+			result.getRowType().getFieldList().subList(0, validatedFields.size());
+		final RelDataType convertedRowType =
+			validator.getTypeFactory().createStructType(convertedFields);
+
+		if (!RelOptUtil.equal("validated row type", validatedRowType,
+			"converted row type", convertedRowType, Litmus.IGNORE)) {
+			throw new AssertionError("Conversion to relational algebra failed to "
+				+ "preserve datatypes:\n"
+				+ "validated type:\n"
+				+ validatedRowType.getFullTypeString()
+				+ "\nconverted type:\n"
+				+ convertedRowType.getFullTypeString()
+				+ "\nrel:\n"
+				+ RelOptUtil.toString(result));
+		}
+	}
+
+	public RelNode flattenTypes(
+		RelNode rootRel,
+		boolean restructure) {
+		RelStructuredTypeFlattener typeFlattener =
+			new RelStructuredTypeFlattener(rexBuilder, createToRelContext(), restructure);
+		return typeFlattener.rewrite(rootRel);
+	}
+
+	/**
+	 * If sub-query is correlated and decorrelation is enabled, performs
+	 * decorrelation.
+	 *
+	 * @param query   Query
+	 * @param rootRel Root relational expression
+	 * @return New root relational expression after decorrelation
+	 */
+	public RelNode decorrelate(SqlNode query, RelNode rootRel) {
+		if (!enableDecorrelation()) {
+			return rootRel;
+		}
+		final RelNode result = decorrelateQuery(rootRel);
+		if (result != rootRel) {
+			checkConvertedType(query, result);
+		}
+		return result;
+	}
+
+	/**
+	 * Walks over a tree of relational expressions, replacing each
+	 * {@link RelNode} with a 'slimmed down' relational expression that projects
+	 * only the fields required by its consumer.
+	 *
+	 * <p>This may make things easier for the optimizer, by removing crud that
+	 * would expand the search space, but is difficult for the optimizer itself
+	 * to do it, because optimizer rules must preserve the number and type of
+	 * fields. Hence, this transform that operates on the entire tree, similar
+	 * to the {@link RelStructuredTypeFlattener type-flattening transform}.
+	 *
+	 * <p>Currently this functionality is disabled in farrago/luciddb; the
+	 * default implementation of this method does nothing.
+	 *
+	 * @param ordered Whether the relational expression must produce results in
+	 * a particular order (typically because it has an ORDER BY at top level)
+	 * @param rootRel Relational expression that is at the root of the tree
+	 * @return Trimmed relational expression
+	 */
+	public RelNode trimUnusedFields(boolean ordered, RelNode rootRel) {
+		// Trim fields that are not used by their consumer.
+		if (isTrimUnusedFields()) {
+			final RelFieldTrimmer trimmer = newFieldTrimmer();
+			final List<RelCollation> collations =
+				rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
+			rootRel = trimmer.trim(rootRel);
+			if (!ordered
+				&& collations != null
+				&& !collations.isEmpty()
+				&& !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
+				final RelTraitSet traitSet = rootRel.getTraitSet()
+					.replace(RelCollationTraitDef.INSTANCE, collations);
+				rootRel = rootRel.copy(traitSet, rootRel.getInputs());
+			}
+			if (SQL2REL_LOGGER.isDebugEnabled()) {
+				SQL2REL_LOGGER.debug(
+					RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel,
+						SqlExplainFormat.TEXT, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+			}
+		}
+		return rootRel;
+	}
+
+	/**
+	 * Creates a RelFieldTrimmer.
+	 *
+	 * @return Field trimmer
+	 */
+	protected RelFieldTrimmer newFieldTrimmer() {
+		final RelBuilder relBuilder =
+			RelFactories.LOGICAL_BUILDER.create(cluster, null);
+		return new RelFieldTrimmer(validator, relBuilder);
+	}
+
+	/**
+	 * Converts an unvalidated query's parse tree into a relational expression.
+	 *
+	 * @param query           Query to convert
+	 * @param needsValidation Whether to validate the query before converting;
+	 *                        <code>false</code> if the query has already been
+	 *                        validated.
+	 * @param top             Whether the query is top-level, say if its result
+	 *                        will become a JDBC result set; <code>false</code> if
+	 *                        the query will be part of a view.
+	 */
+	public RelRoot convertQuery(
+		SqlNode query,
+		final boolean needsValidation,
+		final boolean top) {
+		if (needsValidation) {
+			query = validator.validate(query);
+		}
+
+		RelMetadataQuery.THREAD_PROVIDERS.set(
+			JaninoRelMetadataProvider.of(cluster.getMetadataProvider()));
+		RelNode result = convertQueryRecursive(query, top, null).rel;
+		if (top) {
+			if (isStream(query)) {
+				result = new LogicalDelta(cluster, result.getTraitSet(), result);
+			}
+		}
+		RelCollation collation = RelCollations.EMPTY;
+		if (!query.isA(SqlKind.DML)) {
+			if (isOrdered(query)) {
+				collation = requiredCollation(result);
+			}
+		}
+		checkConvertedType(query, result);
+
+		if (SQL2REL_LOGGER.isDebugEnabled()) {
+			SQL2REL_LOGGER.debug(
+				RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode",
+					result, SqlExplainFormat.TEXT,
+					SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+		}
+
+		final RelDataType validatedRowType = validator.getValidatedNodeType(query);
+		return RelRoot.of(result, validatedRowType, query.getKind())
+			.withCollation(collation);
+	}
+
+	private static boolean isStream(SqlNode query) {
+		return query instanceof SqlSelect
+			&& ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+	}
+
+	public static boolean isOrdered(SqlNode query) {
+		switch (query.getKind()) {
+			case SELECT:
+				return ((SqlSelect) query).getOrderList() != null
+					&& ((SqlSelect) query).getOrderList().size() > 0;
+			case WITH:
+				return isOrdered(((SqlWith) query).body);
+			case ORDER_BY:
+				return ((SqlOrderBy) query).orderList.size() > 0;
+			default:
+				return false;
+		}
+	}
+
+	private RelCollation requiredCollation(RelNode r) {
+		if (r instanceof Sort) {
+			return ((Sort) r).collation;
+		}
+		if (r instanceof Project) {
+			return requiredCollation(((Project) r).getInput());
+		}
+		if (r instanceof Delta) {
+			return requiredCollation(((Delta) r).getInput());
+		}
+		throw new AssertionError();
+	}
+
+	/**
+	 * Converts a SELECT statement's parse tree into a relational expression.
+	 */
+	public RelNode convertSelect(SqlSelect select, boolean top) {
+		final SqlValidatorScope selectScope = validator.getWhereScope(select);
+		final Blackboard bb = createBlackboard(selectScope, null, top);
+		convertSelectImpl(bb, select);
+		return bb.root;
+	}
+
+	/**
+	 * Factory method for creating translation workspace.
+	 */
+	protected Blackboard createBlackboard(SqlValidatorScope scope,
+		Map<String, RexNode> nameToNodeMap, boolean top) {
+		return new Blackboard(scope, nameToNodeMap, top);
+	}
+
+	/**
+	 * Implementation of {@link #convertSelect(SqlSelect, boolean)};
+	 * derived class may override.
+	 */
+	protected void convertSelectImpl(
+		final Blackboard bb,
+		SqlSelect select) {
+		convertFrom(
+			bb,
+			select.getFrom());
+		convertWhere(
+			bb,
+			select.getWhere());
+
+		final List<SqlNode> orderExprList = new ArrayList<>();
+		final List<RelFieldCollation> collationList = new ArrayList<>();
+		gatherOrderExprs(
+			bb,
+			select,
+			select.getOrderList(),
+			orderExprList,
+			collationList);
+		final RelCollation collation =
+			cluster.traitSet().canonize(RelCollations.of(collationList));
+
+		if (validator.isAggregate(select)) {
+			convertAgg(
+				bb,
+				select,
+				orderExprList);
+		} else {
+			convertSelectList(
+				bb,
+				select,
+				orderExprList);
+		}
+
+		if (select.isDistinct()) {
+			distinctify(bb, true);
+		}
+		convertOrder(
+			select, bb, collation, orderExprList, select.getOffset(),
+			select.getFetch());
+		bb.setRoot(bb.root, true);
+	}
+
+	/**
+	 * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds
+	 * a relational expression to make the results unique.
+	 *
+	 * <p>If the SELECT clause contains duplicate expressions, adds
+	 * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are
+	 * grouping on the minimal set of keys. The performance gain isn't huge, but
+	 * it is difficult to detect these duplicate expressions later.
+	 *
+	 * @param bb               Blackboard
+	 * @param checkForDupExprs Check for duplicate expressions
+	 */
+	private void distinctify(
+		Blackboard bb,
+		boolean checkForDupExprs) {
+		// Look for duplicate expressions in the project.
+		// Say we have 'select x, y, x, z'.
+		// Then dups will be {[2, 0]}
+		// and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
+		RelNode rel = bb.root;
+		if (checkForDupExprs && (rel instanceof LogicalProject)) {
+			LogicalProject project = (LogicalProject) rel;
+			final List<RexNode> projectExprs = project.getProjects();
+			final List<Integer> origins = new ArrayList<>();
+			int dupCount = 0;
+			for (int i = 0; i < projectExprs.size(); i++) {
+				int x = findExpr(projectExprs.get(i), projectExprs, i);
+				if (x >= 0) {
+					origins.add(x);
+					++dupCount;
+				} else {
+					origins.add(i);
+				}
+			}
+			if (dupCount == 0) {
+				distinctify(bb, false);
+				return;
+			}
+
+			final Map<Integer, Integer> squished = Maps.newHashMap();
+			final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
+			final List<Pair<RexNode, String>> newProjects = Lists.newArrayList();
+			for (int i = 0; i < fields.size(); i++) {
+				if (origins.get(i) == i) {
+					squished.put(i, newProjects.size());
+					newProjects.add(RexInputRef.of2(i, fields));
+				}
+			}
+			rel =
+				LogicalProject.create(rel, Pair.left(newProjects),
+					Pair.right(newProjects));
+			bb.root = rel;
+			distinctify(bb, false);
+			rel = bb.root;
+
+			// Create the expressions to reverse the mapping.
+			// Project($0, $1, $0, $2).
+			final List<Pair<RexNode, String>> undoProjects = Lists.newArrayList();
+			for (int i = 0; i < fields.size(); i++) {
+				final int origin = origins.get(i);
+				RelDataTypeField field = fields.get(i);
+				undoProjects.add(
+					Pair.of(
+						(RexNode) new RexInputRef(
+							squished.get(origin), field.getType()),
+						field.getName()));
+			}
+
+			rel =
+				LogicalProject.create(rel, Pair.left(undoProjects),
+					Pair.right(undoProjects));
+			bb.setRoot(
+				rel,
+				false);
+
+			return;
+		}
+
+		// Usual case: all of the expressions in the SELECT clause are
+		// different.
+		final ImmutableBitSet groupSet =
+			ImmutableBitSet.range(rel.getRowType().getFieldCount());
+		rel =
+			createAggregate(bb, false, groupSet, ImmutableList.of(groupSet),
+				ImmutableList.<AggregateCall>of());
+
+		bb.setRoot(
+			rel,
+			false);
+	}
+
+	private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
+		for (int i = 0; i < count; i++) {
+			RexNode expr = exprs.get(i);
+			if (expr.toString().equals(seek.toString())) {
+				return i;
+			}
+		}
+		return -1;
+	}
+
+	/**
+	 * Converts a query's ORDER BY clause, if any.
+	 *
+	 * @param select        Query
+	 * @param bb            Blackboard
+	 * @param collation     Collation list
+	 * @param orderExprList Method populates this list with orderBy expressions
+	 *                      not present in selectList
+	 * @param offset        Expression for number of rows to discard before
+	 *                      returning first row
+	 * @param fetch         Expression for number of rows to fetch
+	 */
+	protected void convertOrder(
+		SqlSelect select,
+		Blackboard bb,
+		RelCollation collation,
+		List<SqlNode> orderExprList,
+		SqlNode offset,
+		SqlNode fetch) {
+		if (select.getOrderList() == null
+			|| select.getOrderList().getList().isEmpty()) {
+			assert collation.getFieldCollations().isEmpty();
+			if ((offset == null
+				|| ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO))
+				&& fetch == null) {
+				return;
+			}
+		}
+
+		// Create a sorter using the previously constructed collations.
+		bb.setRoot(
+			LogicalSort.create(bb.root, collation,
+				offset == null ? null : convertExpression(offset),
+				fetch == null ? null : convertExpression(fetch)),
+			false);
+
+		// If extra expressions were added to the project list for sorting,
+		// add another project to remove them. But make the collation empty, because
+		// we can't represent the real collation.
+		//
+		// If it is the top node, use the real collation, but don't trim fields.
+		if (orderExprList.size() > 0 && !bb.top) {
+			final List<RexNode> exprs = new ArrayList<>();
+			final RelDataType rowType = bb.root.getRowType();
+			final int fieldCount =
+				rowType.getFieldCount() - orderExprList.size();
+			for (int i = 0; i < fieldCount; i++) {
+				exprs.add(rexBuilder.makeInputRef(bb.root, i));
+			}
+			bb.setRoot(
+				LogicalProject.create(bb.root, exprs,
+					rowType.getFieldNames().subList(0, fieldCount)),
+				false);
+		}
+	}
+
+	/**
+	 * Returns whether a given node contains a {@link SqlInOperator}.
+	 *
+	 * @param node a RexNode tree
+	 */
+	private static boolean containsInOperator(
+		SqlNode node) {
+		try {
+			SqlVisitor<Void> visitor =
+				new SqlBasicVisitor<Void>() {
+					public Void visit(SqlCall call) {
+						if (call.getOperator() instanceof SqlInOperator) {
+							throw new Util.FoundOne(call);
+						}
+						return super.visit(call);
+					}
+				};
+			node.accept(visitor);
+			return false;
+		} catch (Util.FoundOne e) {
+			Util.swallow(e, null);
+			return true;
+		}
+	}
+
+	/**
+	 * Push down all the NOT logical operators into any IN/NOT IN operators.
+	 *
+	 * @param scope Scope where {@code sqlNode} occurs
+	 * @param sqlNode the root node from which to look for NOT operators
+	 * @return the transformed SqlNode representation with NOT pushed down.
+	 */
+	private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+		SqlNode sqlNode) {
+		if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
+			SqlCall sqlCall = (SqlCall) sqlNode;
+			if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
+				|| (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
+				SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
+				for (int i = 0; i < sqlOperands.length; i++) {
+					sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
+				}
+				return reg(scope, sqlNode);
+			} else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+				SqlNode childNode = sqlCall.operand(0);
+				assert childNode instanceof SqlCall;
+				SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
+				if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
+					SqlNode[] andOperands = childSqlCall.getOperands();
+					SqlNode[] orOperands = new SqlNode[andOperands.length];
+					for (int i = 0; i < orOperands.length; i++) {
+						orOperands[i] = reg(scope,
+							SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO,
+								andOperands[i]));
+					}
+					for (int i = 0; i < orOperands.length; i++) {
+						orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
+					}
+					return reg(scope,
+						SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
+							orOperands[0], orOperands[1]));
+				} else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
+					SqlNode[] orOperands = childSqlCall.getOperands();
+					SqlNode[] andOperands = new SqlNode[orOperands.length];
+					for (int i = 0; i < andOperands.length; i++) {
+						andOperands[i] = reg(scope,
+							SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO,
+								orOperands[i]));
+					}
+					for (int i = 0; i < andOperands.length; i++) {
+						andOperands[i] = pushDownNotForIn(scope, andOperands[i]);
+					}
+					return reg(scope,
+						SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
+							andOperands[0], andOperands[1]));
+				} else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+					SqlNode[] notOperands = childSqlCall.getOperands();
+					assert notOperands.length == 1;
+					return pushDownNotForIn(scope, notOperands[0]);
+				} else if (childSqlCall.getOperator() instanceof SqlInOperator) {
+					SqlNode[] inOperands = childSqlCall.getOperands();
+					SqlInOperator inOp =
+						(SqlInOperator) childSqlCall.getOperator();
+					if (inOp.isNotIn()) {
+						return reg(scope,
+							SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO,
+								inOperands[0], inOperands[1]));
+					} else {
+						return reg(scope,
+							SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO,
+								inOperands[0], inOperands[1]));
+					}
+				} else {
+					// childSqlCall is "leaf" node in a logical expression tree
+					// (only considering AND, OR, NOT)
+					return sqlNode;
+				}
+			} else {
+				// sqlNode is "leaf" node in a logical expression tree
+				// (only considering AND, OR, NOT)
+				return sqlNode;
+			}
+		} else {
+			// tree rooted at sqlNode does not contain inOperator
+			return sqlNode;
+		}
+	}
+
+	/** Registers with the validator a {@link SqlNode} that has been created
+	 * during the Sql-to-Rel process. */
+	private static SqlNode reg(SqlValidatorScope scope, SqlNode e) {
+		scope.getValidator().deriveType(scope, e);
+		return e;
+	}
+
+	/**
+	 * Converts a WHERE clause.
+	 *
+	 * @param bb    Blackboard
+	 * @param where WHERE clause, may be null
+	 */
+	private void convertWhere(
+		final Blackboard bb,
+		final SqlNode where) {
+		if (where == null) {
+			return;
+		}
+		SqlNode newWhere = pushDownNotForIn(bb.scope, where);
+		replaceSubQueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+		final RexNode convertedWhere = bb.convertExpression(newWhere);
+
+		// only allocate filter if the condition is not TRUE
+		if (convertedWhere.isAlwaysTrue()) {
+			return;
+		}
+
+		final RelFactories.FilterFactory factory =
+			RelFactories.DEFAULT_FILTER_FACTORY;
+		final RelNode filter = factory.createFilter(bb.root, convertedWhere);
+		final RelNode r;
+		final CorrelationUse p = getCorrelationUse(bb, filter);
+		if (p != null) {
+			assert p.r instanceof Filter;
+			Filter f = (Filter) p.r;
+			r = LogicalFilter.create(f.getInput(), f.getCondition(),
+				ImmutableSet.of(p.id));
+		} else {
+			r = filter;
+		}
+
+		bb.setRoot(r, false);
+	}
+
+	private void replaceSubQueries(
+		final Blackboard bb,
+		final SqlNode expr,
+		RelOptUtil.Logic logic) {
+		findSubQueries(bb, expr, logic, false);
+		for (SubQuery node : bb.subQueryList) {
+			substituteSubQuery(bb, node);
+		}
+	}
+
+	private void substituteSubQuery(Blackboard bb, SubQuery subQuery) {
+		final RexNode expr = subQuery.expr;
+		if (expr != null) {
+			// Already done.
+			return;
+		}
+
+		final SqlBasicCall call;
+		final RelNode rel;
+		final SqlNode query;
+		final RelOptUtil.Exists converted;
+		switch (subQuery.node.getKind()) {
+			case CURSOR:
+				convertCursor(bb, subQuery);
+				return;
+
+			case MULTISET_QUERY_CONSTRUCTOR:
+			case MULTISET_VALUE_CONSTRUCTOR:
+			case ARRAY_QUERY_CONSTRUCTOR:
+				rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
+				subQuery.expr = bb.register(rel, JoinRelType.INNER);
+				return;
+
+			case IN:
+				call = (SqlBasicCall) subQuery.node;
+				query = call.operand(1);
+				if (!config.isExpand() && !(query instanceof SqlNodeList)) {
+					return;
+				}
+				final SqlNode leftKeyNode = call.operand(0);
+
+				final List<RexNode> leftKeys;
+				switch (leftKeyNode.getKind()) {
+					case ROW:
+						leftKeys = Lists.newArrayList();
+						for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
+							leftKeys.add(bb.convertExpression(sqlExpr));
+						}
+						break;
+					default:
+						leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
+				}
+
+				final boolean notIn = ((SqlInOperator) call.getOperator()).isNotIn();
+				if (query instanceof SqlNodeList) {
+					SqlNodeList valueList = (SqlNodeList) query;
+					if (!containsNullLiteral(valueList)
+						&& valueList.size() < config.getInSubQueryThreshold()) {
+						// We're under the threshold, so convert to OR.
+						subQuery.expr =
+							convertInToOr(
+								bb,
+								leftKeys,
+								valueList,
+								notIn);
+						return;
+					}
+
+					// Otherwise, let convertExists translate
+					// values list into an inline table for the
+					// reference to Q below.
+				}
+
+				// Project out the search columns from the left side
+
+				// Q1:
+				// "select from emp where emp.deptno in (select col1 from T)"
+				//
+				// is converted to
+				//
+				// "select from
+				//   emp inner join (select distinct col1 from T)) q
+				//   on emp.deptno = q.col1
+				//
+				// Q2:
+				// "select from emp where emp.deptno not in (Q)"
+				//
+				// is converted to
+				//
+				// "select from
+				//   emp left outer join (select distinct col1, TRUE from T) q
+				//   on emp.deptno = q.col1
+				//   where emp.deptno <> null
+				//         and q.indicator <> TRUE"
+				//
+				final RelDataType targetRowType =
+					SqlTypeUtil.promoteToRowType(typeFactory,
+						validator.getValidatedNodeType(leftKeyNode), null);
+				converted =
+					convertExists(query, RelOptUtil.SubQueryType.IN, subQuery.logic,
+						notIn, targetRowType);
+				if (converted.indicator) {
+					// Generate
+					//    emp CROSS JOIN (SELECT COUNT(*) AS c,
+					//                       COUNT(deptno) AS ck FROM dept)
+					final RelDataType longType =
+						typeFactory.createSqlType(SqlTypeName.BIGINT);
+					final RelNode seek = converted.r.getInput(0); // fragile
+					final int keyCount = leftKeys.size();
+					final List<Integer> args = ImmutableIntList.range(0, keyCount);
+					LogicalAggregate aggregate =
+						LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null,
+							ImmutableList.of(
+								AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+									ImmutableList.<Integer>of(), -1, longType, null),
+								AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+									args, -1, longType, null)));
+					LogicalJoin join =
+						LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true),
+							ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
+					bb.setRoot(join, false);
+				}
+				final RexNode rex =
+					bb.register(converted.r,
+						converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER,
+						leftKeys);
+
+				RelOptUtil.Logic logic = subQuery.logic;
+				switch (logic) {
+					case TRUE_FALSE_UNKNOWN:
+					case UNKNOWN_AS_TRUE:
+						if (!converted.indicator) {
+							logic = RelOptUtil.Logic.TRUE_FALSE;
+						}
+				}
+				subQuery.expr = translateIn(logic, bb.root, rex);
+				if (notIn) {
+					subQuery.expr =
+						rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
+				}
+				return;
+
+			case EXISTS:
+				// "select from emp where exists (select a from T)"
+				//
+				// is converted to the following if the sub-query is correlated:
+				//
+				// "select from emp left outer join (select AGG_TRUE() as indicator
+				// from T group by corr_var) q where q.indicator is true"
+				//
+				// If there is no correlation, the expression is replaced with a
+				// boolean indicating whether the sub-query returned 0 or >= 1 row.
+				call = (SqlBasicCall) subQuery.node;
+				query = call.operand(0);
+				if (!config.isExpand()) {
+					return;
+				}
+				converted = convertExists(query, RelOptUtil.SubQueryType.EXISTS,
+					subQuery.logic, true, null);
+				assert !converted.indicator;
+				if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, true)) {
+					return;
+				}
+				subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
+				return;
+
+			case SCALAR_QUERY:
+				// Convert the sub-query.  If it's non-correlated, convert it
+				// to a constant expression.
+				if (!config.isExpand()) {
+					return;
+				}
+				call = (SqlBasicCall) subQuery.node;
+				query = call.operand(0);
+				converted = convertExists(query, RelOptUtil.SubQueryType.SCALAR,
+					subQuery.logic, true, null);
+				assert !converted.indicator;
+				if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, false)) {
+					return;
+				}
+				rel = convertToSingleValueSubq(query, converted.r);
+				subQuery.expr = bb.register(rel, JoinRelType.LEFT);
+				return;
+
+			case SELECT:
+				// This is used when converting multiset queries:
+				//
+				// select * from unnest(select multiset[deptno] from emps);
+				//
+				converted = convertExists(subQuery.node, RelOptUtil.SubQueryType.SCALAR,
+					subQuery.logic, true, null);
+				assert !converted.indicator;
+				subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
+				return;
+
+			default:
+				throw new AssertionError("unexpected kind of sub-query: "
+					+ subQuery.node);
+		}
+	}
+
+	private RexNode translateIn(RelOptUtil.Logic logic, RelNode root,
+		final RexNode rex) {
+		switch (logic) {
+			case TRUE:
+				return rexBuilder.makeLiteral(true);
+
+			case TRUE_FALSE:
+			case UNKNOWN_AS_FALSE:
+				assert rex instanceof RexRangeRef;
+				final int fieldCount = rex.getType().getFieldCount();
+				RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
+				rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
+
+				// Then append the IS NOT NULL(leftKeysForIn).
+				//
+				// RexRangeRef contains the following fields:
+				//   leftKeysForIn,
+				//   rightKeysForIn (the original sub-query select list),
+				//   nullIndicator
+				//
+				// The first two lists contain the same number of fields.
+				final int k = (fieldCount - 1) / 2;
+				for (int i = 0; i < k; i++) {
+					rexNode =
+						rexBuilder.makeCall(
+							SqlStdOperatorTable.AND,
+							rexNode,
+							rexBuilder.makeCall(
+								SqlStdOperatorTable.IS_NOT_NULL,
+								rexBuilder.makeFieldAccess(rex, i)));
+				}
+				return rexNode;
+
+			case TRUE_FALSE_UNKNOWN:
+			case UNKNOWN_AS_TRUE:
+				// select e.deptno,
+				//   case
+				//   when ct.c = 0 then false
+				//   when dt.i is not null then true
+				//   when e.deptno is null then null
+				//   when ct.ck < ct.c then null
+				//   else false
+				//   end
+				// from e
+				// cross join (select count(*) as c, count(deptno) as ck from v) as ct
+				// left join (select distinct deptno, true as i from v) as dt
+				//   on e.deptno = dt.deptno
+				final Join join = (Join) root;
+				final Project left = (Project) join.getLeft();
+				final RelNode leftLeft = ((Join) left.getInput()).getLeft();
+				final int leftLeftCount = leftLeft.getRowType().getFieldCount();
+				final RelDataType longType =
+					typeFactory.createSqlType(SqlTypeName.BIGINT);
+				final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
+				final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
+				final RexNode iRef =
+					rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
+
+				final RexLiteral zero =
+					rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
+				final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
+				final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
+				final RexNode unknownLiteral =
+					rexBuilder.makeNullLiteral(trueLiteral.getType());
+
+				final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
+				args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
+					falseLiteral,
+					rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
+					trueLiteral);
+				final JoinInfo joinInfo = join.analyzeCondition();
+				for (int leftKey : joinInfo.leftKeys) {
+					final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
+					args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
+						unknownLiteral);
+				}
+				args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
+					unknownLiteral,
+					falseLiteral);
+
+				return rexBuilder.makeCall(SqlStdOperatorTable.CASE, args.build());
+
+			default:
+				throw new AssertionError(logic);
+		}
+	}
+
+	private static boolean containsNullLiteral(SqlNodeList valueList) {
+		for (SqlNode node : valueList.getList()) {
+			if (node instanceof SqlLiteral) {
+				SqlLiteral lit = (SqlLiteral) node;
+				if (lit.getValue() == null) {
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Determines if a sub-query is non-correlated and if so, converts it to a
+	 * constant.
+	 *
+	 * @param subQuery  the call that references the sub-query
+	 * @param bb        blackboard used to convert the sub-query
+	 * @param converted RelNode tree corresponding to the sub-query
+	 * @param isExists  true if the sub-query is part of an EXISTS expression
+	 * @return Whether the sub-query can be converted to a constant
+	 */
+	private boolean convertNonCorrelatedSubQuery(
+		SubQuery subQuery,
+		Blackboard bb,
+		RelNode converted,
+		boolean isExists) {
+		SqlCall call = (SqlBasicCall) subQuery.node;
+		if (subQueryConverter.canConvertSubQuery()
+			&& isSubQueryNonCorrelated(converted, bb)) {
+			// First check if the sub-query has already been converted
+			// because it's a nested sub-query.  If so, don't re-evaluate
+			// it again.
+			RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
+			if (constExpr == null) {
+				constExpr =
+					subQueryConverter.convertSubQuery(
+						call,
+						this,
+						isExists,
+						config.isExplain());
+			}
+			if (constExpr != null) {
+				subQuery.expr = constExpr;
+				mapConvertedNonCorrSubqs.put(call, constExpr);
+				return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Converts the RelNode tree for a select statement to a select that
+	 * produces a single value.
+	 *
+	 * @param query the query
+	 * @param plan   the original RelNode tree corresponding to the statement
+	 * @return the converted RelNode tree
+	 */
+	public RelNode convertToSingleValueSubq(
+		SqlNode query,
+		RelNode plan) {
+		// Check whether query is guaranteed to produce a single value.
+		if (query instanceof SqlSelect) {
+			SqlSelect select = (SqlSelect) query;
+			SqlNodeList selectList = select.getSelectList();
+			SqlNodeList groupList = select.getGroup();
+
+			if ((selectList.size() == 1)
+				&& ((groupList == null) || (groupList.size() == 0))) {
+				SqlNode selectExpr = selectList.get(0);
+				if (selectExpr instanceof SqlCall) {
+					SqlCall selectExprCall = (SqlCall) selectExpr;
+					if (Util.isSingleValue(selectExprCall)) {
+						return plan;
+					}
+				}
+
+				// If there is a limit with 0 or 1,
+				// it is ensured to produce a single value
+				if (select.getFetch() != null
+					&& select.getFetch() instanceof SqlNumericLiteral) {
+					SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
+					if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
+						return plan;
+					}
+				}
+			}
+		} else if (query instanceof SqlCall) {
+			// If the query is (values ...),
+			// it is necessary to look into the operands to determine
+			// whether SingleValueAgg is necessary
+			SqlCall exprCall = (SqlCall) query;
+			if (exprCall.getOperator()
+				instanceof SqlValuesOperator
+				&& Util.isSingleValue(exprCall)) {
+				return plan;
+			}
+		}
+
+		// If not, project SingleValueAgg
+		return RelOptUtil.createSingleValueAggRel(
+			cluster,
+			plan);
+	}
+
+	/**
+	 * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...".
+	 *
+	 * @param leftKeys   LHS
+	 * @param valuesList RHS
+	 * @param isNotIn    is this a NOT IN operator
+	 * @return converted expression
+	 */
+	private RexNode convertInToOr(
+		final Blackboard bb,
+		final List<RexNode> leftKeys,
+		SqlNodeList valuesList,
+		boolean isNotIn) {
+		final List<RexNode> comparisons = new ArrayList<>();
+		for (SqlNode rightVals : valuesList) {
+			RexNode rexComparison;
+			if (leftKeys.size() == 1) {
+				rexComparison =
+					rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+						leftKeys.get(0),
+						ensureSqlType(leftKeys.get(0).getType(),
+							bb.convertExpression(rightVals)));
+			} else {
+				assert rightVals instanceof SqlCall;
+				final SqlBasicCall call = (SqlBasicCall) rightVals;
+				assert (call.getOperator() instanceof SqlRowOperator)
+					&& call.operandCount() == leftKeys.size();
+				rexComparison =
+					RexUtil.composeConjunction(
+						rexBuilder,
+						Iterables.transform(
+							Pair.zip(leftKeys, call.getOperandList()),
+							new Function<Pair<RexNode, SqlNode>, RexNode>() {
+								public RexNode apply(Pair<RexNode, SqlNode> pair) {
+									return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+										pair.left,
+										ensureSqlType(pair.left.getType(),
+											bb.convertExpression(pair.right)));
+								}
+							}),
+						false);
+			}
+			comparisons.add(rexComparison);
+		}
+
+		RexNode result =
+			RexUtil.composeDisjunction(rexBuilder, comparisons, true);
+		assert result != null;
+
+		if (isNotIn) {
+			result =
+				rexBuilder.makeCall(
+					SqlStdOperatorTable.NOT,
+					result);
+		}
+
+		return result;
+	}
+
+	/** Ensures that an expression has a given {@link SqlTypeName}, applying a
+	 * cast if necessary. If the expression already has the right type family,
+	 * returns the expression unchanged. */
+	private RexNode ensureSqlType(RelDataType type, RexNode node) {
+		if (type.getSqlTypeName() == node.getType().getSqlTypeName()
+			|| (type.getSqlTypeName() == SqlTypeName.VARCHAR
+			&& node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
+			return node;
+		}
+		return rexBuilder.ensureType(type, node, true);
+	}
+
+	/**
+	 * Gets the list size threshold under which {@link #convertInToOr} is used.
+	 * Lists of this size or greater will instead be converted to use a join
+	 * against an inline table
+	 * ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than a
+	 * predicate. A threshold of 0 forces usage of an inline table in all cases; a
+	 * threshold of Integer.MAX_VALUE forces usage of OR in all cases
+	 *
+	 * @return threshold, default {@link #DEFAULT_IN_SUB_QUERY_THRESHOLD}
+	 */
+	@Deprecated // to be removed before 2.0
+	protected int getInSubqueryThreshold() {
+		return config.getInSubQueryThreshold();
+	}
+
+	/**
+	 * Converts an EXISTS or IN predicate into a join. For EXISTS, the sub-query
+	 * produces an indicator variable, and the result is a relational expression
+	 * which outer joins that indicator to the original query. After performing
+	 * the outer join, the condition will be TRUE if the EXISTS condition holds,
+	 * NULL otherwise.
+	 *
+	 * @param seek           A query, for example 'select * from emp' or
+	 *                       'values (1,2,3)' or '('Foo', 34)'.
+	 * @param subQueryType   Whether sub-query is IN, EXISTS or scalar
+	 * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+	 *     FALSE, UNKNOWN) will be required, or whether we can accept an
+	 *     approximation (say representing UNKNOWN as FALSE)
+	 * @param notIn Whether the operation is NOT IN
+	 * @return join expression
+	 */
+	private RelOptUtil.Exists convertExists(
+		SqlNode seek,
+		RelOptUtil.SubQueryType subQueryType,
+		RelOptUtil.Logic logic,
+		boolean notIn,
+		RelDataType targetDataType) {
+		final SqlValidatorScope seekScope =
+			(seek instanceof SqlSelect)
+				? validator.getSelectScope((SqlSelect) seek)
+				: null;
+		final Blackboard seekBb = createBlackboard(seekScope, null, false);
+		RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType);
+
+		return RelOptUtil.createExistsPlan(seekRel, subQueryType, logic, notIn);
+	}
+
+	private RelNode convertQueryOrInList(
+		Blackboard bb,
+		SqlNode seek,
+		RelDataType targetRowType) {
+		// NOTE: Once we start accepting single-row queries as row constructors,
+		// there will be an ambiguity here for a case like X IN ((SELECT Y FROM
+		// Z)).  The SQL standard resolves the ambiguity by saying that a lone
+		// select should be interpreted as a table expression, not a row
+		// expression.  The semantic difference is that a table expression can
+		// return multiple rows.
+		if (seek instanceof SqlNodeList) {
+			return convertRowValues(
+				bb,
+				seek,
+				((SqlNodeList) seek).getList(),
+				false,
+				targetRowType);
+		} else {
+			return convertQueryRecursive(seek, false, null).project();
+		}
+	}
+
+	private RelNode convertRowValues(
+		Blackboard bb,
+		SqlNode rowList,
+		Collection<SqlNode> rows,
+		boolean allowLiteralsOnly,
+		RelDataType targetRowType) {
+		// NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
+		// literals into a single LogicalValues; this gives the optimizer a smaller
+		// input tree.  For everything else (computed expressions, row
+		// sub-queries), we union each row in as a projection on top of a
+		// LogicalOneRow.
+
+		final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
+			ImmutableList.builder();
+		final RelDataType rowType;
+		if (targetRowType != null) {
+			rowType = targetRowType;
+		} else {
+			rowType =
+				SqlTypeUtil.promoteToRowType(
+					typeFactory,
+					validator.getValidatedNodeType(rowList),
+					null);
+		}
+
+		final List<RelNode> unionInputs = new ArrayList<>();
+		for (SqlNode node : rows) {
+			SqlBasicCall call;
+			if (isRowConstructor(node)) {
+				call = (SqlBasicCall) node;
+				ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
+				for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
+					RexLiteral rexLiteral =
+						convertLiteralInValuesList(
+							operand.e,
+							bb,
+							rowType,
+							operand.i);
+					if ((rexLiteral == null) && allowLiteralsOnly) {
+						return null;
+					}
+					if ((rexLiteral == null) || !config.isCreateValuesRel()) {
+						// fallback to convertRowConstructor
+						tuple = null;
+						break;
+					}
+					tuple.add(rexLiteral);
+				}
+				if (tuple != null) {
+					tupleList.add(tuple.build());
+					continue;
+				}
+			} else {
+				RexLiteral rexLiteral =
+					convertLiteralInValuesList(
+						node,
+						bb,
+						rowType,
+						0);
+				if ((rexLiteral != null) && config.isCreateValuesRel()) {
+					tupleList.add(ImmutableList.of(rexLiteral));
+					continue;
+				} else {
+					if ((rexLiteral == null) && allowLiteralsOnly) {
+						return null;
+					}
+				}
+
+				// convert "1" to "row(1)"
+				call =
+					(SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
+						SqlParserPos.ZERO,
+						node);
+			}
+			unionInputs.add(convertRowConstructor(bb, call));
+		}
+		LogicalValues values =
+			LogicalValues.create(cluster, rowType, tupleList.build());
+		RelNode resultRel;
+		if (unionInputs.isEmpty()) {
+			resultRel = values;
+		} else {
+			if (!values.getTuples().isEmpty()) {
+				unionInputs.add(values);
+			}
+			resultRel = LogicalUnion.create(unionInputs, true);
+		}
+		leaves.add(resultRel);
+		return resultRel;
+	}
+
+	private RexLiteral convertLiteralInValuesList(
+		SqlNode sqlNode,
+		Blackboard bb,
+		RelDataType rowType,
+		int iField) {
+		if (!(sqlNode instanceof SqlLiteral)) {
+			return null;
+		}
+		RelDataTypeField field = rowType.getFieldList().get(iField);
+		RelDataType type = field.getType();
+		if (type.isStruct()) {
+			// null literals for weird stuff like UDT's need
+			// special handling during type flattening, so
+			// don't use LogicalValues for those
+			return null;
+		}
+
+		RexNode literalExpr =
+			exprConverter.convertLiteral(
+				bb,
+				(SqlLiteral) sqlNode);
+
+		if (!(literalExpr instanceof RexLiteral)) {
+			assert literalExpr.isA(SqlKind.CAST);
+			RexNode child = ((RexCall) literalExpr).getOperands().get(0);
+			assert RexLiteral.isNullLiteral(child);
+
+			// NOTE jvs 22-Nov-2006:  we preserve type info
+			// in LogicalValues digest, so it's OK to lose it here
+			return (RexLiteral) child;
+		}
+
+		RexLiteral literal = (RexLiteral) literalExpr;
+
+		Comparable value = literal.getValue();
+
+		if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) {
+			BigDecimal roundedValue =
+				NumberUtil.rescaleBigDecimal(
+					(BigDecimal) value,
+					type.getScale());
+			return rexBuilder.makeExactLiteral(
+				roundedValue,
+				type);
+		}
+
+		if ((value instanceof NlsString)
+			&& (type.getSqlTypeName() == SqlTypeName.CHAR)) {
+			// pad fixed character type
+			NlsString unpadded = (NlsString) value;
+			return rexBuilder.makeCharLiteral(
+				new NlsString(
+					Spaces.padRight(unpadded.getValue(), type.getPrecision()),
+					unpadded.getCharsetName(),
+					unpadded.getCollation()));
+		}
+		return literal;
+	}
+
+	private boolean isRowConstructor(SqlNode node) {
+		if (!(node.getKind() == SqlKind.ROW)) {
+			return false;
+		}
+		SqlCall call = (SqlCall) node;
+		return call.getOperator().getName().equalsIgnoreCase("row");
+	}
+
+	/**
+	 * Builds a list of all <code>IN</code> or <code>EXISTS</code> operators
+	 * inside SQL parse tree. Does not traverse inside queries.
+	 *
+	 * @param bb                           blackboard
+	 * @param node                         the SQL parse tree
+	 * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+	 *              FALSE, UNKNOWN) will be required, or whether we can accept
+	 *              an approximation (say representing UNKNOWN as FALSE)
+	 * @param registerOnlyScalarSubQueries if set to true and the parse tree
+	 *                                     corresponds to a variation of a select
+	 *                                     node, only register it if it's a scalar
+	 *                                     sub-query
+	 */
+	private void findSubQueries(
+		Blackboard bb,
+		SqlNode node,
+		RelOptUtil.Logic logic,
+		boolean registerOnlyScalarSubQueries) {
+		final SqlKind kind = node.getKind();
+		switch (kind) {
+			case EXISTS:
+			case SELECT:
+			case MULTISET_QUERY_CONSTRUCTOR:
+			case MULTISET_VALUE_CONSTRUCTOR:
+			case ARRAY_QUERY_CONSTRUCTOR:
+			case CURSOR:
+			case SCALAR_QUERY:
+				if (!registerOnlyScalarSubQueries
+					|| (kind == SqlKind.SCALAR_QUERY)) {
+					bb.registerSubQuery(node, RelOptUtil.Logic.TRUE_FALSE);
+				}
+				return;
+			case IN:
+				if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) {
+					logic = logic.negate();
+				}
+				break;
+			case NOT:
+				logic = logic.negate();
+				break;
+		}
+		if (node instanceof SqlCall) {
+			for (SqlNode operand : ((SqlCall) node).getOperandList()) {
+				if (operand != null) {
+					// In the case of an IN expression, locate scalar
+					// sub-queries so we can convert them to constants
+					findSubQueries(
+						bb,
+						operand,
+						logic,
+						kind == SqlKind.IN || registerOnlyScalarSubQueries);
+				}
+			}
+		} else if (node instanceof SqlNodeList) {
+			for (SqlNode child : (SqlNodeList) node) {
+				findSubQueries(
+					bb,
+					child,
+					logic,
+					kind == SqlKind.IN || registerOnlyScalarSubQueries);
+			}
+		}
+
+		// Now that we've located any scalar sub-queries inside the IN
+		// expression, register the IN expression itself.  We need to
+		// register the scalar sub-queries first so they can be converted
+		// before the IN expression is converted.
+		if (kind == SqlKind.IN) {
+			switch (logic) {
+				case TRUE_FALSE_UNKNOWN:
+					if (validator.getValidatedNodeType(node).isNullable()) {
+						break;
+					} else if (true) {
+						break;
+					}
+					// fall through
+				case UNKNOWN_AS_FALSE:
+					logic = RelOptUtil.Logic.TRUE;
+			}
+			bb.registerSubQuery(node, logic);
+		}
+	}
+
+	/**
+	 * Converts an expression from {@link SqlNode} to {@link RexNode} format.
+	 *
+	 * @param node Expression to translate
+	 * @return Converted expression
+	 */
+	public RexNode convertExpression(
+		SqlNode node) {
+		Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
+		final ParameterScope scope =
+			new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
+		final Blackboard bb = createBlackboard(scope, null, false);
+		return bb.convertExpression(node);
+	}
+
+	/**
+	 * Converts an expression from {@link SqlNode} to {@link RexNode} format,
+	 * mapping identifier references to predefined expressions.
+	 *
+	 * @param node          Expression to translate
+	 * @param nameToNodeMap map from String to {@link RexNode}; when an
+	 *                      {@link SqlIdentifier} is encountered, it is used as a
+	 *                      key and translated to the corresponding value from
+	 *                      this map
+	 * @return Converted expression
+	 */
+	public RexNode convertExpression(
+		SqlNode node,
+		Map<String, RexNode> nameToNodeMap) {
+		final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
+		for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
+			nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
+		}
+		final ParameterScope scope =
+			new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
+		final Blackboard bb = createBlackboard(scope, nameToNodeMap, false);
+		return bb.convertExpression(node);
+	}
+
+	/**
+	 * Converts a non-standard expression.
+	 *
+	 * <p>This method is an extension-point that derived classes can override. If
+	 * this method returns a null result, the normal expression translation
+	 * process will proceed. The default implementation always returns null.
+	 *
+	 * @param node Expression
+	 * @param bb   Blackboard
+	 * @return null to proceed with the usual expression translation process
+	 */
+	protected RexNode convertExtendedExpression(
+		SqlNode node,
+		Blackboard bb) {
+		return null;
+	}
+
+	private RexNode convertOver(Blackboard bb, SqlNode node) {
+		SqlCall call = (SqlCall) node;
+		SqlCall aggCall = call.operand(0);
+		SqlNode windowOrRef = call.operand(1);
+		final SqlWindow window =
+			validator.resolveWindow(windowOrRef, bb.scope, true);
+
+		// ROW_NUMBER() expects specific kind of framing.
+		if (aggCall.getKind() == SqlKind.ROW_NUMBER) {
+			window.setLowerBound(SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO));
+			window.setUpperBound(SqlWindow.createCurrentRow(SqlParserPos.ZERO));
+			window.setRows(SqlLiteral.createBoolean(true, SqlParserPos.ZERO));
+		}
+		final SqlNodeList partitionList = window.getPartitionList();
+		final ImmutableList.Builder<RexNode> partitionKeys =
+			ImmutableList.builder();
+		for (SqlNode partition : partitionList) {
+			partitionKeys.add(bb.convertExpression(partition));
+		}
+		RexNode lowerBound = bb.convertExpression(window.getLowerBound());
+		RexNode upperBound = bb.convertExpression(window.getUpperBound());
+		SqlNodeList orderList = window.getOrderList();
+		if ((orderList.size() == 0) && !window.isRows()) {
+			// A logical range requires an ORDER BY clause. Use the implicit
+			// ordering of this relation. There must be one, otherwise it would
+			// have failed validation.
+			orderList = bb.scope.getOrderList();
+			if (orderList == null) {
+				throw new AssertionError(
+					"Relation should have sort key for implicit ORDER BY");
+			}
+		}
+		final ImmutableList.Builder<RexFieldCollation> orderKeys =
+			ImmutableList.builder();
+		final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
+		for (SqlNode order : orderList) {
+			flags.clear();
+			RexNode e = bb.convertSortExpression(order, flags);
+			orderKeys.add(new RexFieldCollation(e, flags));
+		}
+		try {
+			Preconditions.checkArgument(bb.window == null,
+				"already in window agg mode");
+			bb.window = window;
+			RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
+			rexAgg =
+				rexBuilder.ensureType(
+					validator.getValidatedNodeType(call), rexAgg, false);
+
+			// Walk over the tree and apply 'over' to all agg functions. This is
+			// necessary because the returned expression is not necessarily a call
+			// to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
+			final RexShuttle visitor =
+				new HistogramShuttle(
+					partitionKeys.build(), orderKeys.build(),
+					RexWindowBound.create(window.getLowerBound(), lowerBound),
+					RexWindowBound.create(window.getUpperBound(), upperBound),
+					window);
+			return rexAgg.accept(visitor);
+		} finally {
+			bb.window = null;
+		}
+	}
+
+	/**
+	 * Converts a FROM clause into a relational expression.
+	 *
+	 * @param bb   Scope within which to resolve identifiers
+	 * @param from FROM clause of a query. Examples include:
+	 *
+	 *             <ul>
+	 *             <li>a single table ("SALES.EMP"),
+	 *             <li>an aliased table ("EMP AS E"),
+	 *             <li>a list of tables ("EMP, DEPT"),
+	 *             <li>an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO =
+	 *             DEPT.DEPTNO"),
+	 *             <li>a VALUES clause ("VALUES ('Fred', 20)"),
+	 *             <li>a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"),
+	 *             <li>or any combination of the above.
+	 *             </ul>
+	 */
+	protected void convertFrom(
+		Blackboard bb,
+		SqlNode from) {
+		if (from == null) {
+			bb.setRoot(LogicalValues.createOneRow(cluster), false);
+			return;
+		}
+
+		final SqlCall call;
+		final SqlNode[] operands;
+		switch (from.getKind()) {
+			case MATCH_RECOGNIZE:
+				convertMatchRecognize(bb, (SqlCall) from);
+				return;
+
+			case AS:
+				convertFrom(bb, ((SqlCall) from).operand(0));
+				return;
+
+			case WITH_ITEM:
+				convertFrom(bb, ((SqlWithItem) from).query);
+				return;
+
+			case WITH:
+				convertFrom(bb, ((SqlWith) from).body);
+				return;
+
+			case TABLESAMPLE:
+				operands = ((SqlBasicCall) from).getOperands();
+				SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
+				if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
+					String sampleName =
+						((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
+							.getName();
+					datasetStack.push(sampleName);
+					convertFrom(bb, operands[0]);
+					datasetStack.pop();
+				} else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
+					SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
+						(SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
+					convertFrom(bb, operands[0]);
+					RelOptSamplingParameters params =
+						new RelOptSamplingParameters(
+							tableSampleSpec.isBernoulli(),
+							tableSampleSpec.getSamplePercentage(),
+							tableSampleSpec.isRepeatable(),
+							tableSampleSpec.getRepeatableSeed());
+					bb.setRoot(new Sample(cluster, bb.root, params), false);
+				} else {
+					throw new AssertionError("unknown TABLESAMPLE type: " + sampleSpec);
+				}
+				return;
+
+			case IDENTIFIER:
+				convertIdentifier(bb, (SqlIdentifier) from, null);
+				return;
+
+			case EXTEND:
+				call = (SqlCall) from;
+				SqlIdentifier id = (SqlIdentifier) call.getOperandList().get(0);
+				SqlNodeList extendedColumns = (SqlNodeList) call.getOperandList().get(1);
+				convertIdentifier(bb, id, extendedColumns);
+				return;
+
+			case JOIN:
+				final SqlJoin join = (SqlJoin) from;
+				final SqlValidatorScope scope = validator.getJoinScope(from);
+				final Blackboard fromBlackboard = createBlackboard(scope, null, false);
+				SqlNode left = join.getLeft();
+				SqlNode right = join.getRight();
+				final boolean isNatural = join.isNatural();
+				final JoinType joinType = join.getJoinType();
+				final SqlValidatorScope leftScope =
+					Util.first(validator.getJoinScope(left),
+						((DelegatingScope) bb.scope).getParent());
+				final Blackboard leftBlackboard =
+					createBlackboard(leftScope, null, false);
+				final SqlValidatorScope rightScope =
+					Util.first(validator.getJoinScope(right),
+						((DelegatingScope) bb.scope).getParent());
+				final Blackboard rightBlackboard =
+					createBlackboard(rightScope, null, false);
+				convertFrom(leftBlackboard, left);
+				RelNode leftRel = leftBlackboard.root;
+				convertFrom(rightBlackboard, right);
+				RelNode rightRel = rightBlackboard.root;
+				JoinRelType convertedJoinType = convertJoinType(joinType);
+				RexNode conditionExp;
+				final SqlValidatorNamespace leftNamespace = validator.getNamespace(left);
+				final SqlValidatorNamespace rightNamespace = validator.getNamespace(right);
+				if (isNatural) {
+					final RelDataType leftRowType = leftNamespace.getRowType();
+					final RelDataType rightRowType = rightNamespace.getRowType();
+					final List<String> columnList =
+						SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType,
+							rightRowType);
+					conditionExp = convertUsing(leftNamespace, rightNamespace,
+						columnList);
+				} else {
+					conditionExp =
+						convertJoinCondition(
+							fromBlackboard,
+							leftNamespace,
+							rightNamespace,
+							join.getCondition(),
+							join.getConditionType(),
+							leftRel,
+							rightRel);
+				}
+
+				final RelNode joinRel =
+					createJoin(
+						fromBlackboard,
+						leftRel,
+						rightRel,
+						conditionExp,
+						convertedJoinType);
+				bb.setRoot(joinRel, false);
+				return;
+
+			case SELECT:
+			case INTERSECT:
+			case EXCEPT:
+			case UNION:
+				final RelNode rel = convertQueryRecursive(from, false, null).project();
+				bb.setRoot(rel, true);
+				return;
+
+			case VALUES:
+				convertValuesImpl(bb, (SqlCall) from, null);
+				return;
+
+			case UNNEST:
+				call = (SqlCall) from;
+				final List<SqlNode> nodes = call.getOperandList();
+				final SqlUnnestOperator operator = (SqlUnnestOperator) call.getOperator();
+				for (SqlNode node : nodes) {
+					replaceSubQueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+				}
+				final List<RexNode> exprs = new ArrayList<>();
+				final List<String> fieldNames = new ArrayList<>();
+				for (Ord<SqlNode> node : Ord.zip(nodes)) {
+					exprs.add(bb.convertExpression(node.e));
+					fieldNames.add(validator.deriveAlias(node.e, node.i));
+				}
+				final RelNode input =
+					RelOptUtil.createProject(
+						(null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster),
+						exprs, fieldNames, true);
+
+				Uncollect uncollect =
+					new Uncollect(cluster, cluster.traitSetOf(Convention.NONE),
+						input, operator.withOrdinality);
+				bb.setRoot(uncollect, true);
+				return;
+
+			case COLLECTION_TABLE:
+				call = (SqlCall) from;
+
+				// Dig out real call; TABLE() wrapper is just syntactic.
+				assert call.getOperandList().size() == 1;
+				final SqlCall call2 = call.operand(0);
+				convertCollectionTable(bb, call2);
+				return;
+
+			default:
+				throw new AssertionError("not a join operator " + from);
+		}
+	}
+
+	protected void convertMatchRecognize(Blackboard bb, SqlCall call) {
+		final SqlMatchRecognize matchRecognize = (SqlMatchRecognize) call;
+		final SqlValidatorNamespace ns = validator.getNamespace(matchRecognize);
+		final SqlValidatorScope scope = validator.getMatchRecognizeScope(matchRecognize);
+
+		final Blackboard mrBlackBoard = createBlackboard(scope, null, false);
+		final RelDataType rowType = ns.getRowType();
+		// convert inner query, could be a table name or a derived table
+		SqlNode expr = matchRecognize.getTableRef();
+		convertFrom(mrBlackBoard, expr);
+		final RelNode input = mrBlackBoard.root;
+
+		// convert pattern
+		final Set<String> patternVarsSet = new HashSet<>();
+		SqlNode pattern = matchRecognize.getPattern();
+		final SqlBasicVisitor<RexNode> patternVarVisitor =
+			new SqlBasicVisitor<RexNode>() {
+				@Override public RexNode visit(SqlCall call) {
+					List<SqlNode> operands = call.getOperandList();
+					List<RexNode> newOperands = Lists.newArrayList();
+					for (SqlNode node : operands) {
+						newOperands.add(node.accept(this));
+					}
+					return rexBuilder.makeCall(
+						validator.getUnknownType(), call.getOperator(), newOperands);
+				}
+
+				@Override public RexNode visit(SqlIdentifier id) {
+					assert id.isSimple();
+					patternVarsSet.add(id.getSimple());
+					return rexBuilder.makeLiteral(id.getSimple());
+				}
+
+				@Override public RexNode visit(SqlLiteral literal) {
+					if (literal instanceof SqlNumericLiteral) {
+						return rexBuilder.makeExactLiteral(BigDecimal.valueOf(literal.intValue(true)));
+					} else {
+						return rexBuilder.makeLiteral(literal.booleanValue());
+					}
+				}
+			};
+		final RexNode patternNode = pattern.accept(patternVarVisitor);
+
+		mrBlackBoard.setPatternVarRef(true);
+
+		// convert definitions
+		final ImmutableMap.Builder<String, RexNode> definitionNodes =
+			ImmutableMap.builder();
+		for (SqlNode def : matchRecognize.getPatternDefList()) {
+			List<SqlNode> operands = ((SqlCall) def).getOperandList();
+			String alias = ((SqlIdentifier) operands.get(1)).getSimple();
+			RexNode rex = mrBlackBoard.convertExpression(operands.get(0));
+			definitionNodes.put(alias, rex);
+		}
+
+		mrBlackBoard.setPatternVarRef(false);
+
+		final RelFactories.MatchFactory factory =
+			RelFactories.DEFAULT_MATCH_FACTORY;
+		final RelNode rel =
+			factory.createMatchRecognize(input, patternNode,
+				matchRecognize.getStrictStart().booleanValue(),
+				matchRecognize.getStrictEnd().booleanValue(),
+				definitionNodes.build(),
+				rowType);
+		bb.setRoot(rel, false);
+	}
+
+	private void convertIdentifier(Blackboard bb, SqlIdentifier id,
+		SqlNodeList extendedColumns) {
+		final SqlValidatorNamespace fromNamespace =
+			validator.getNamespace(id).resolve();
+		if (fromNamespace.getNode() != null) {
+			convertFrom(bb, fromNamespace.getNode());
+			return;
+		}
+		final String datasetName =
+			datasetStack.isEmpty() ? null : datasetStack.peek();
+		final boolean[] usedDataset = {false};
+		RelOptTable table =
+			SqlValidatorUtil.getRelOptTable(fromNamespace, catalogReader,
+				datasetName, usedDataset);
+		if (extendedColumns != null && extendedColumns.size() > 0) {
+			assert table != null;
+			final SqlValidatorTable validatorTable =
+				table.unwrap(SqlValidatorTable.class);
+			final List<RelDataTypeField> extendedFields =
+				SqlValidatorUtil.getExtendedColumns(validator, validatorTable,
+					extendedColumns);
+			table = table.extend(extendedFields);
+		}
+		final RelNode tableRel;
+		if (config.isConvertTableAccess()) {
+			tableRel = toRel(table);
+		} else {
+			tableRel = LogicalTableScan.create(cluster, table);
+		}
+		bb.setRoot(tableRel, true);
+		if (usedDataset[0]) {
+			bb.setDataset(datasetName);
+		}
+	}
+
+	protected void convertCollectionTable(
+		Blackboard bb,
+		SqlCall call) {
+		final SqlOperator operator = call.getOperator();
+		if (operator == SqlStdOperatorTable.TABLESAMPLE) {
+			final String sampleName = (String) SqlLiteral.value(call.operand(0));
+			datasetStack.push(sampleName);
+			SqlCall cursorCall = call.operand(1);
+			SqlNode query = cursorCall.operand(0);
+			RelNode converted = convertQuery(query, false, false).rel;
+			bb.setRoot(converted, false);
+			datasetStack.pop();
+			return;
+		}
+		replaceSubQueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+		// Expand table macro if possible. It's more efficient than
+		// LogicalTableFunctionScan.
+		final SqlCallBinding callBinding =
+			new SqlCallBinding(bb.scope.getValidator(), bb.scope, call);
+		if (operator instanceof SqlUserDefinedTableMacro) {
+			final SqlUserDefinedTableMacro udf =
+				(SqlUserDefinedTableMacro) operator;
+			final TranslatableTable table =
+				udf.getTable(typeFactory, callBinding.operands());
+			final RelDataType rowType = table.getRowType(typeFactory);
+			RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table,
+				udf.getNameAsId().names);
+			RelNode converted = toRel(relOptTable);
+			bb.setRoot(converted, true);
+			return;
+		}
+
+		Type elementType;
+		if (operator instanceof SqlUserDefinedTableFunction) {
+			SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
+			elementType = udtf.getElementType(typeFactory, callBinding.operands());
+		} else {
+			elementType = null;
+		}
+
+		RexNode rexCall = bb.convertExpression(call);
+		final List<RelNode> inputs = bb.retrieveCursors();
+		Set<RelColumnMapping> columnMappings =
+			getColumnMappings(operator);
+		LogicalTableFunctionScan callRel =
+			LogicalTableFunctionScan.create(
+				cluster,
+				inputs,
+				rexCall,
+				elementType,
+				validator.getValidatedNodeType(call),
+				columnMappings);
+		bb.setRoot(callRel, true);
+		afterTableFunction(bb, call, callRel);
+	}
+
+	protected void afterTableFunction(
+		SqlToRelConverter.Blackboard bb,
+		SqlCall call,
+		LogicalTableFunctionScan callRel) {
+	}
+
+	private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
+		SqlReturnTypeInference rti = op.getReturnTypeInference();
+		if (rti == null) {
+			return null;
+		}
+		if (rti instanceof TableFunctionReturnTypeInference) {
+			TableFunctionReturnTypeInference tfrti =
+				(TableFunctionReturnTypeInference) rti;
+			return tfrti.getColumnMappings();
+		} else {
+			return null;
+		}
+	}
+
+	protected RelNode createJoin(
+		Blackboard bb,
+		RelNode leftRel,
+		RelNode rightRel,
+		RexNode joinCond,
+		JoinRelType joinType) {
+		assert joinCond != null;
+
+		final CorrelationUse p = getCorrelationUse(bb, rightRel);
+		if (p != null) {
+			LogicalCorrelate corr = LogicalCorrelate.create(leftRel, p.r,
+				p.id, p.requiredColumns, SemiJoinType.of(joinType));
+			if (!joinCond.isAlwaysTrue()) {
+				final RelFactories.FilterFactory factory =
+					RelFactories.DEFAULT_FILTER_FACTORY;
+				return factory.createFilter(corr, joinCond);
+			}
+			return corr;
+		}
+
+		final Join originalJoin =
+			(Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel,
+				joinCond, ImmutableSet.<CorrelationId>of(), joinType, false);
+
+		return RelOptUtil.pushDownJoinConditions(originalJoin);
+	}
+
+	private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) {
+		final Set<CorrelationId> correlatedVariables =
+			RelOptUtil.getVariablesUsed(r0);
+		if (correlatedVariables.isEmpty()) {
+			return null;
+		}
+		final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder();
+		final List<CorrelationId> correlNames = Lists.newArrayList();
+
+		// All correlations must refer the same namespace since correlation
+		// produces exactly one correlation source.
+		// The same source might be referenced by different variables since
+		// DeferredLookups are not de-duplicated at create time.
+		SqlValidatorNamespace prevNs = null;
+
+		for (CorrelationId correlName : correlatedVariables) {
+			DeferredLookup lookup =
+				mapCorrelToDeferred.get(correlName);
+			RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
+			String originalRelName = lookup.getOriginalRelName();
+			String originalFieldName = fieldAccess.getField().getName();
+
+			final SqlNameMatcher nameMatcher =
+				lookup.bb.scope.getValidator().getCatalogReader().nameMatcher();
+			final SqlValidatorScope.ResolvedImpl resolved =
+				new SqlValidatorScope.ResolvedImpl();
+			lookup.bb.scope.resolve(ImmutableList.of(originalRelName),
+				nameMatcher, false, resolved);
+			assert resolved.count() == 1;
+			final SqlValidatorScope.Resolve resolve = resolved.only();
+			final SqlValidatorNamespace foundNs = resolve.namespace;
+			final RelDataType rowType = resolve.rowType();
+			final int childNamespaceIndex = resolve.path.steps().get(0).i;
+			final SqlValidatorScope ancestorScope = resolve.scope;
+			boolean correlInCurrentScope = ancestorScope == bb.scope;
+
+			if (!correlInCurrentScope) {
+				continue;
+			}
+
+			if (prevNs == null) {
+				prevNs = foundNs;
+			} else {
+				assert prevNs == foundNs : "All correlation variables should resolve"
+					+ " to the same namespace."
+					+ " Prev ns=" + prevNs
+					+ ", new ns=" + foundNs;
+			}
+
+			int namespaceOffset = 0;
+			if (childNamespaceIndex > 0) {
+				// If not the first child, need to figure out the width
+				// of output types from all the preceding namespaces
+				assert ancestorScope instanceof ListScope;
+				List<SqlValidatorNamespace> children =
+					((ListScope) ancestorScope).getChildren();
+
+				for (int i = 0; i < childNamespaceIndex; i++) {
+					SqlValidatorNamespace child = children.get(i);
+					namespaceOffset +=
+						child.getRowType().getFieldCount();
+				}
+			}
+
+			RexFieldAccess topLevelFieldAccess = fieldAccess;
+			while (topLevelFieldAccess.getReferenceExpr() instanceof RexFieldAccess) {
+				topLevelFieldAccess = (RexFieldAccess) topLevelFieldAccess.getReferenceExpr();
+			}
+			final RelDataTypeField field = rowType.getFieldList()
+				.get(topLevelFieldAccess.getField().getIndex() - namespaceOffset);
+			int pos = namespaceOffset + field.getIndex();
+
+			assert field.getType()
+				== topLevelFieldAccess.getField().getType();
+
+			assert pos != -1;
+
+			if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
+				// bb.root is an aggregate and only projects group by
+				// keys.
+				Map<Integer, Integer> exprProjection =
+					bb.mapRootRelToFieldProjection.get(bb.root);
+
+				// sub-query can reference group by keys projected from
+				// the root of the outer relation.
+				if (exprProjection.containsKey(pos)) {
+					pos = exprProjection.get(pos);
+				} else {
+					// correl not grouped
+					throw new AssertionError("Identifier '" + originalRelName + "."
+						+ originalFieldName + "' is not a group expr");
+				}
+			}
+
+			requiredColumns.set(pos);
+			correlNames.add(correlName);
+		}
+
+		if (correlNames.isEmpty()) {
+			// None of the correlating variables originated in this scope.
+			return null;
+		}
+
+		RelNode r = r0;
+		if (correlNames.size() > 1) {
+			// The same table was referenced more than once.
+			// So we deduplicate
+			r = DeduplicateCorrelateVariables.go(rexBuilder, correlNames.get(0),
+				Util.skip(correlNames), r0);
+		}
+		return new CorrelationUse(correlNames.get(0), requiredColumns.build(), r);
+	}
+
+	/**
+	 * Determines whether a sub-query is non-correlated. Note that a
+	 * non-correlated sub-query can contain correlated references, provided those
+	 * references do not reference select statements that are parents of the
+	 * sub-query.
+	 *
+	 * @param subq the sub-query
+	 * @param bb   blackboard used while converting the sub-query, i.e., the
+	 *             blackboard of the parent query of this sub-query
+	 * @return true if the sub-query is non-correlated
+	 */
+	private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) {
+		Set<CorrelationId> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
+		for (CorrelationId correlName : correlatedVariables) {
+			DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
+			String originalRelName = lookup.getOriginalRelName();
+
+			final SqlNameMatcher nameMatcher =
+				lookup.bb.scope.getValidator().getCatalogReader().nameMatcher();
+			final SqlValidatorScope.ResolvedImpl resolved =
+				new SqlValidatorScope.ResolvedImpl();
+			lookup.bb.scope.resolve(ImmutableList.of(originalRelName), nameMatcher,
+				false, resolved);
+
+			SqlValidatorScope ancestorScope = resolved.only().scope;
+
+			// If the correlated reference is in a scope that's "above" the
+			// sub-query, then this is a correlated sub-query.
+			SqlValidatorScope parentScope = bb.scope;
+			do {
+				if (ancestorScope == parentScope) {
+					return false;
+				}
+				if (parentScope instanceof DelegatingScope) {
+					parentScope = ((DelegatingScope) parentScope).getParent();
+				} else {
+					break;
+				}
+			} while (parentScope != null);
+		}
+		return true;
+	}
+
+	/**
+	 * Returns a list of fields to be prefixed to each relational expression.
+	 *
+	 * @return List of system fields
+	 */
+	protected List<RelDataTypeField> getSystemFields() {
+		return Collections.emptyList();
+	}
+
+	private RexNode convertJoinCondition(Blackboard bb,
+		SqlValidatorNamespace leftNamespace,
+		SqlValidatorNamespace rightNamespace,
+		SqlNode condition,
+		JoinConditionType conditionType,
+		RelNode leftRel,
+		RelNode rightRel) {
+		if (condition == null) {
+			return rexBuilder.makeLiteral(true);
+		}
+		bb.setRoot(ImmutableList.of(leftRel, rightRel));
+		replaceSubQueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+		switch (conditionType) {
+			case ON:
+				bb.setRoot(ImmutableList.of(leftRel, rightRel));
+				return bb.convertExpression(condition);
+			case USING:
+				final SqlNodeList list = (SqlNodeList) condition;
+				final List<String> nameList = new ArrayList<>();
+				for (SqlNode columnName : list) {
+					final SqlIdentifier id = (SqlIdentifier) columnName;
+					String name = id.getSimple();
+					nameList.add(name);
+				}
+				return convertUsing(leftNamespace, rightNamespace, nameList);
+			default:
+				throw Util.unexpected(conditionType);
+		}
+	}
+
+	/**
+	 * Returns an expression for matching columns of a USING clause or inferred
+	 * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y =
+	 * b.y". Returns null if the column list is empty.
+	 *
+	 * @param leftNamespace Namespace of left input to join
+	 * @param rightNamespace Namespace of right input to join
+	 * @param nameList List of column names to join on
+	 * @return Expression to match columns from name list, or true if name list
+	 * is empty
+	 */
+	private RexNode convertUsing(SqlValidatorNamespace leftNamespace,
+		SqlValidatorNamespace rightNamespace,
+		List<String> nameList) {
+		final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+		final List<RexNode> list = Lists.newArrayList();
+		for (String name : nameList) {
+			List<RexNode> operands = new ArrayList<>();
+			int offset = 0;
+			for (SqlValidatorNamespace n : ImmutableList.of(leftNamespace,
+				rightNamespace)) {
+				final RelDataType rowType = n.getRowType();
+				final RelDataTypeField field = nameMatcher.field(rowType, name);
+				operands.add(
+					rexBuilder.makeInputRef(field.getType(),
+						offset + field.getIndex()));
+				offset += rowType.getFieldList().size();
+			}
+			list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands));
+		}
+		return RexUtil.composeConjunction(rexBuilder, list, false);
+	}
+
+	private static JoinRelType convertJoinType(JoinType joinType) {
+		switch (joinType) {
+			case COMMA:
+			case INNER:
+			case CROSS:
+				return JoinRelType.INNER;
+			case FULL:
+				return JoinRelType.FULL;
+			case LEFT:
+				return JoinRelType.LEFT;
+			case RIGHT:
+				return JoinRelType.RIGHT;
+			default:
+				throw Util.unexpected(joinType);
+		}
+	}
+
+	/**
+	 * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query.
+	 *
+	 * <p>This method extracts SELECT, GROUP BY and HAVING clauses, and creates
+	 * an {@link AggConverter}, then delegates to {@link #createAggImpl}.
+	 * Derived class may override this method to change any of those clauses or
+	 * specify a different {@link AggConverter}.
+	 *
+	 * @param bb            Scope within which to resolve identifiers
+	 * @param select        Query
+	 * @param orderExprList Additional expressions needed to implement ORDER BY
+	 */
+	protected void convertAgg(
+		Blackboard bb,
+		SqlSelect select,
+		List<SqlNode> orderExprList) {
+		assert bb.root != null : "precondition: child != null";
+		SqlNodeList groupList = select.getGroup();
+		SqlNodeList selectList = select.getSelectList();
+		SqlNode having = select.getHaving();
+
+		final AggConverter aggConverter = new AggConverter(bb, select);
+		createAggImpl(
+			bb,
+			aggConverter,
+			selectList,
+			groupList,
+			having,
+			orderExprList);
+	}
+
+	protected final void createAggImpl(
+		Blackboard bb,
+		final AggConverter aggConverter,
+		SqlNodeList selectList,
+		SqlNodeList groupList,
+		SqlNode having,
+		List<SqlNode> orderExprList) {
+		// Find aggregate functions in SELECT and HAVING clause
+		final AggregateFinder aggregateFinder = new AggregateFinder();
+		selectList.accept(aggregateFinder);
+		if (having != null) {
+			having.accept(aggregateFinder);
+		}
+
+		// first replace the sub-queries inside the aggregates
+		// because they will provide input rows to the aggregates.
+		replaceSubQueries(bb, aggregateFinder.list,
+			RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+		// If group-by clause is missing, pretend that it has zero elements.
+		if (groupList == null) {
+			groupList = SqlNodeList.EMPTY;
+		}
+
+		replaceSubQueries(bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+		// register the group exprs
+
+		// build a map to remember the projections from the top scope to the
+		// output of the current root.
+		//
+		// Calcite allows expressions, not just column references in
+		// group by list. This is not SQL 2003 compliant, but hey.
+
+		final AggregatingSelectScope scope = aggConverter.aggregatingSelectScope;
+		final AggregatingSelectScope.Resolved r = scope.resolved.get();
+		for (SqlNode groupExpr : r.groupExprList) {
+			aggConverter.addGroupExpr(groupExpr);
+		}
+
+		RexNode havingExpr = null;
+		final List<Pair<RexNode, String>> projects = Lists.newArrayList();
+
+		try {
+			Preconditions.checkArgument(bb.agg == null, "already in agg mode");
+			bb.agg = aggConverter;
+
+			// convert the select and having expressions, so that the
+			// agg converter knows which aggregations are required
+
+			selectList.accept(aggConverter);
+			// Assert we don't have dangling items left in the stack
+			assert !aggConverter.inOver;
+			for (SqlNode expr : orderExprList) {
+				expr.accept(aggConverter);
+				assert !aggConverter.inOver;
+			}
+			if (having != null) {
+				having.accept(aggConverter);
+				assert !aggConverter.inOver;
+			}
+
+			// compute inputs to the aggregator
+			List<Pair<RexNode, String>> preExprs = aggConverter.getPreExprs();
+
+			if (preExprs.size() == 0) {
+				// Special case for COUNT(*), where we can end up with no inputs
+				// at all.  The rest of the system doesn't like 0-tuples, so we
+				// select a dummy constant here.
+				final RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO);
+				preExprs = ImmutableList.of(Pair.of(zero, (String) null));
+			}
+
+			final RelNode inputRel = bb.root;
+
+			// Project the expressions required by agg and having.
+			bb.setRoot(
+				RelOptUtil.createProject(
+					inputRel,
+					preExprs,
+					true),
+				false);
+			bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection);
+
+			// REVIEW jvs 31-Oct-2007:  doesn't the declaration of
+			// monotonicity here assume sort-based aggregation at
+			// the physical level?
+
+			// Tell bb which of group columns are sorted.
+			bb.columnMonotonicities.clear();
+			for (SqlNode groupItem : groupList) {
+				bb.columnMonotonicities.add(
+					bb.scope.getMonotonicity(groupItem));
+			}
+
+			// Add the aggregator
+			bb.setRoot(
+				createAggregate(bb, r.indicator, r.groupSet, r.groupSets,
+					aggConverter.getAggCalls()),
+				false);
+
+			// Generate NULL values for rolled-up not-null fields.
+			final Aggregate aggregate = (Aggregate) bb.root;
+			if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
+				assert aggregate.indicator;
+				List<Pair<RexNode, String>> projects2 = Lists.newArrayList();
+				int converted = 0;
+				final int groupCount = aggregate.getGroupSet().cardinality();
+				for (RelDataTypeField field : aggregate.getRowType().getFieldList()) {
+					final int i = field.getIndex();
+					final RexNode rex;
+					if (i < groupCount && r.isNullable(i)) {
+						++converted;
+
+						rex = rexBuilder.makeCall(SqlStdOperatorTable.CASE,
+							rexBuilder.makeInputRef(aggregate, groupCount + i),
+							rexBuilder.makeCast(
+								typeFactory.createTypeWithNullability(
+									field.getType(), true),
+								rexBuilder.constantNull()),
+							rexBuilder.makeInputRef(aggregate, i));
+					} else {
+						rex = rexBuilder.makeInputRef(aggregate, i);
+					}
+					projects2.add(Pair.of(rex, field.getName()));
+				}
+				if (converted > 0) {
+					bb.setRoot(
+						RelOptUtil.createProject(bb.root, projects2, true),
+						false);
+				}
+			}
+
+			bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection);
+
+			// Replace sub-queries in having here and modify having to use
+			// the replaced expressions
+			if (having != null) {
+				SqlNode newHaving = pushDownNotForIn(bb.scope, having);
+				replaceSubQueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+				havingExpr = bb.convertExpression(newHaving);
+				if (havingExpr.isAlwaysTrue()) {
+					havingExpr = null;
+				}
+			}
+
+			// Now convert the other sub-queries in the select list.
+			// This needs to be done separately from the sub-query inside
+			// any aggregate in the select list, and after the aggregate rel
+			// is allocated.
+			replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+			// Now sub-queries in the entire select list have been converted.
+			// Convert the select expressions to get the final list to be
+			// projected.
+			int k = 0;
+
+			// For select expressions, use the field names previously assigned
+			// by the validator. If we derive afresh, we might generate names
+			// like "EXPR$2" that don't match the names generated by the
+			// validator. This is especially the case when there are system
+			// fields; system fields appear in the relnode's rowtype but do not
+			// (yet) appear in the validator type.
+			final SelectScope selectScope =
+				SqlValidatorUtil.getEnclosingSelectScope(bb.scope);
+			assert selectScope != null;
+			final SqlValidatorNamespace selectNamespace =
+				validator.getNamespace(selectScope.getNode());
+			final List<String> names =
+				selectNamespace.getRowType().getFieldNames();
+			int sysFieldCount = selectList.size() - names.size();
+			for (SqlNode expr : selectList) {
+				projects.add(
+					Pair.of(bb.convertExpression(expr),
+						k < sysFieldCount
+							? validator.deriveAlias(expr, k++)
+							: names.get(k++ - sysFieldCount)));
+			}
+
+			for (SqlNode expr : orderExprList) {
+				projects.add(
+					Pair.of(bb.convertExpression(expr),
+						validator.deriveAlias(expr, k++)));
+			}
+		} finally {
+			bb.agg = null;
+		}
+
+		// implement HAVING (we have already checked that it is non-trivial)
+		if (havingExpr != null) {
+			final RelFactories.FilterFactory factory =
+				RelFactories.DEFAULT_FILTER_FACTORY;
+			bb.setRoot(factory.createFilter(bb.root, havingExpr), false);
+		}
+
+		// implement the SELECT list
+		bb.setRoot(
+			RelOptUtil.createProject(
+				bb.root,
+				projects,
+				true),
+			false);
+
+		// Tell bb which of group columns are sorted.
+		bb.columnMonotonicities.clear();
+		for (SqlNode selectItem : selectList) {
+			bb.columnMonotonicities.add(
+				bb.scope.getMonotonicity(selectItem));
+		}
+	}
+
+	/**
+	 * Creates an Aggregate.
+	 *
+	 * <p>In case the aggregate rel changes the order in which it projects
+	 * fields, the <code>groupExprProjection</code> parameter is provided, and
+	 * the implementation of this method may modify it.
+	 *
+	 * <p>The <code>sortedCount</code> parameter is the number of expressions
+	 * known to be monotonic. These expressions must be on the leading edge of
+	 * the grouping keys. The default implementation of this method ignores this
+	 * parameter.
+	 *
+	 * @param bb       Blackboard
+	 * @param indicator Whether to output fields indicating grouping sets
+	 * @param groupSet Bit set of ordinals of grouping columns
+	 * @param groupSets Grouping sets
+	 * @param aggCalls Array of calls to aggregate functions
+	 * @return LogicalAggregate
+	 */
+	protected RelNode createAggregate(Blackboard bb, boolean indicator,
+		ImmutableBitSet groupSet, ImmutableList<ImmutableBitSet> groupSets,
+		List<AggregateCall> aggCalls) {
+		return LogicalAggregate.create(
+			bb.root, indicator, groupSet, groupSets, aggCalls);
+	}
+
+	public RexDynamicParam convertDynamicParam(
+		final SqlDynamicParam dynamicParam) {
+		// REVIEW jvs 8-Jan-2005:  dynamic params may be encountered out of
+		// order.  Should probably cross-check with the count from the parser
+		// at the end and make sure they all got filled in.  Why doesn't List
+		// have a resize() method?!?  Make this a utility.
+		while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) {
+			dynamicParamSqlNodes.add(null);
+		}
+
+		dynamicParamSqlNodes.set(
+			dynamicParam.getIndex(),
+			dynamicParam);
+		return rexBuilder.makeDynamicParam(
+			getDynamicParamType(dynamicParam.getIndex()),
+			dynamicParam.getIndex());
+	}
+
+	/**
+	 * Creates a list of collations required to implement the ORDER BY clause,
+	 * if there is one. Populates <code>extraOrderExprs</code> with any sort
+	 * expressions which are not in the select clause.
+	 *
+	 * @param bb              Scope within which to resolve identifiers
+	 * @param select          Select clause. Never null, because we invent a
+	 *                        dummy SELECT if ORDER BY is applied to a set
+	 *                        operation (UNION etc.)
+	 * @param orderList       Order by clause, may be null
+	 * @param extraOrderExprs Sort expressions which are not in the select
+	 *                        clause (output)
+	 * @param collationList   List of collations (output)
+	 */
+	protected void gatherOrderExprs(
+		Blackboard bb,
+		SqlSelect select,
+		SqlNodeList orderList,
+		List<SqlNode> extraOrderExprs,
+		List<RelFieldCollation> collationList) {
+		// TODO:  add validation rules to SqlValidator also
+		assert bb.root != null : "precondition: child != null";
+		assert select != null;
+		if (orderList == null) {
+			return;
+		}
+		for (SqlNode orderItem : orderList) {
+			collationList.add(
+				convertOrderItem(
+					select,
+					orderItem,
+					extraOrderExprs,
+					RelFieldCollation.Direction.ASCENDING,
+					RelFieldCollation.NullDirection.UNSPECIFIED));
+		}
+	}
+
+	protected RelFieldCollation convertOrderItem(
+		SqlSelect select,
+		SqlNode orderItem, List<SqlNode> extraExprs,
+		RelFieldCollation.Direction direction,
+		RelFieldCollation.NullDirection nullDirection) {
+		assert select != null;
+		// Handle DESC keyword, e.g. 'select a, b from t order by a desc'.
+		switch (orderItem.getKind()) {
+			case DESCENDING:
+				return convertOrderItem(
+					select,
+					((SqlCall) orderItem).operand(0),
+					extraExprs,
+					RelFieldCollation.Direction.DESCENDING,
+					nullDirection);
+			case NULLS_FIRST:
+				return convertOrderItem(
+					select,
+					((SqlCall) orderItem).operand(0),
+					extraExprs,
+					direction,
+					RelFieldCollation.NullDirection.FIRST);
+			case NULLS_LAST:
+				return convertOrderItem(
+					select,
+					((SqlCall) orderItem).operand(0),
+					extraExprs,
+					direction,
+					RelFieldCollation.NullDirection.LAST);
+		}
+
+		SqlNode converted = validator.expandOrderExpr(select, orderItem);
+
+		switch (nullDirection) {
+			case UNSPECIFIED:
+				nullDirection = validator.getDefaultNullCollation().last(desc(direction))
+					? RelFieldCollation.NullDirection.LAST
+					: RelFieldCollation.NullDirection.FIRST;
+		}
+
+		// Scan the select list and order exprs for an identical expression.
+		final SelectScope selectScope = validator.getRawSelectScope(select);
+		int ordinal = -1;
+		for (SqlNode selectItem : selectScope.getExpandedSelectList()) {
+			++ordinal;
+			if (converted.equalsDeep(stripAs(selectItem), Litmus.IGNORE)) {
+				return new RelFieldCollation(ordinal, direction, nullDirection);
+			}
+		}
+
+		for (SqlNode extraExpr : extraExprs) {
+			++ordinal;
+			if (converted.equalsDeep(extraExpr, Litmus.IGNORE)) {
+				return new RelFieldCollation(ordinal, direction, nullDirection);
+			}
+		}
+
+		// TODO:  handle collation sequence
+		// TODO: flag expressions as non-standard
+
+		extraExprs.add(converted);
+		return new RelFieldCollation(ordinal + 1, direction, nullDirection);
+	}
+
+	private static boolean desc(RelFieldCollation.Direction direction) {
+		switch (direction) {
+			case DESCENDING:
+			case STRICTLY_DESCENDING:
+				return true;
+			default:
+				return false;
+		}
+	}
+
+	@Deprecated // to be removed before 2.0
+	protected boolean enableDecorrelation() {
+		// disable sub-query decorrelation when needed.
+		// e.g. if outer joins are not supported.
+		return config.isDecorrelationEnabled();
+	}
+
+	protected RelNode decorrelateQuery(RelNode rootRel) {
+		return RelDecorrelator.decorrelateQuery(rootRel);
+	}
+
+	/**
+	 * Returns whether to trim unused fields as part of the conversion process.
+	 *
+	 * @return Whether to trim unused fields
+	 */
+	@Deprecated // to be removed before 2.0
+	public boolean isTrimUnusedFields() {
+		return config.isTrimUnusedFields();
+	}
+
+	/**
+	 * Recursively converts a query to a relational expression.
+	 *
+	 * @param query         Query
+	 * @param top           Whether this query is the top-level query of the
+	 *                      statement
+	 * @param targetRowType Target row type, or null
+	 * @return Relational expression
+	 */
+	protected RelRoot convertQueryRecursive(SqlNode query, boolean top,
+		RelDataType targetRowType) {
+		final SqlKind kind = query.getKind();
+		switch (kind) {
+			case SELECT:
+				return RelRoot.of(convertSelect((SqlSelect) query, top), kind);
+			case INSERT:
+				return RelRoot.of(convertInsert((SqlInsert) query), kind);
+			case DELETE:
+				return RelRoot.of(convertDelete((SqlDelete) query), kind);
+			case UPDATE:
+				return RelRoot.of(convertUpdate((SqlUpdate) query), kind);
+			case MERGE:
+				return RelRoot.of(convertMerge((SqlMerge) query), kind);
+			case UNION:
+			case INTERSECT:
+			case EXCEPT:
+				return RelRoot.of(convertSetOp((SqlCall) query), kind);
+			case WITH:
+				return convertWith((SqlWith) query, top);
+			case VALUES:
+				return RelRoot.of(convertValues((SqlCall) query, targetRowType), kind);
+			default:
+				throw new AssertionError("not a query: " + query);
+		}
+	}
+
+	/**
+	 * Converts a set operation (UNION, INTERSECT, MINUS) into relational
+	 * expressions.
+	 *
+	 * @param call Call to set operator
+	 * @return Relational expression
+	 */
+	protected RelNode convertSetOp(SqlCall call) {
+		final RelNode left =
+			convertQueryRecursive(call.operand(0), false, null).project();
+		final RelNode right =
+			convertQueryRecursive(call.operand(1), false, null).project();
+		switch (call.getKind()) {
+			case UNION:
+				return LogicalUnion.create(ImmutableList.of(left, right), all(call));
+
+			case INTERSECT:
+				return LogicalIntersect.create(ImmutableList.of(left, right), all(call));
+
+			case EXCEPT:
+				return LogicalMinus.create(ImmutableList.of(left, right), all(call));
+
+			default:
+				throw Util.unexpected(call.getKind());
+		}
+	}
+
+	private boolean all(SqlCall call) {
+		return ((SqlSetOperator) call.getOperator()).isAll();
+	}
+
+	protected RelNode convertInsert(SqlInsert call) {
+		RelOptTable targetTable = getTargetTable(call);
+
+		final RelDataType targetRowType =
+			validator.getValidatedNodeType(call);
+		assert targetRowType != null;
+		RelNode sourceRel =
+			convertQueryRecursive(call.getSource(), false, targetRowType).project();
+		RelNode massagedRel = convertColumnList(call, sourceRel);
+
+		return createModify(targetTable, massagedRel);
+	}
+
+	/** Creates a relational expression to modify a table or modifiable view. */
+	private RelNode createModify(RelOptTable targetTable, RelNode source) {
+		final ModifiableTable modifiableTable =
+			targetTable.unwrap(ModifiableTable.class);
+		if (modifiableTable != null) {
+			return modifiableTable.toModificationRel(cluster, targetTable,
+				catalogReader, source, LogicalTableModify.Operation.INSERT, null,
+				null, false);
+		}
+		final ModifiableView modifiableView =
+			targetTable.unwrap(ModifiableView.class);
+		if (modifiableView != null) {
+			final Table delegateTable = modifiableView.getTable();
+			final RelDataType delegateRowType = delegateTable.getRowType(typeFactory);
+			final RelOptTable delegateRelOptTable =
+				RelOptTableImpl.create(null, delegateRowType, delegateTable,
+					modifiableView.getTablePath());
+			final RelNode newSource =
+				createSource(targetTable, source, modifiableView, delegateRowType);
+			return createModify(delegateRelOptTable, newSource);
+		}
+		return LogicalTableModify.create(targetTable, catalogReader, source,
+			LogicalTableModify.Operation.INSERT, null, null, false);
+	}
+
+	/** Wraps a relational expression in the projects and filters implied by
+	 * a {@link ModifiableView}.
+	 *
+	 * <p>The input relational expression is suitable for inserting into the view,
+	 * and the returned relational expression is suitable for inserting into its
+	 * delegate table.
+	 *
+	 * <p>In principle, the delegate table of a view might be another modifiable
+	 * view, and if so, the process can be repeated. */
+	private RelNode createSource(RelOptTable targetTable, RelNode source,
+		ModifiableView modifiableView, RelDataType delegateRowType) {
+		final ImmutableIntList mapping = modifiableView.getColumnMapping();
+		assert mapping.size() == targetTable.getRowType().getFieldCount();
+
+		// For columns represented in the mapping, the expression is just a field
+		// reference.
+		final Map<Integer, RexNode> projectMap = new HashMap<>();
+		final List<RexNode> filters = new ArrayList<>();
+		for (int i = 0; i < mapping.size(); i++) {
+			int target = mapping.get(i);
+			if (target >= 0) {
+				projectMap.put(target, RexInputRef.of(i, source.getRowType()));
+			}
+		}
+
+		// For columns that are not in the mapping, and have a constraint of the
+		// form "column = value", the expression is the literal "value".
+		//
+		// If a column has multiple constraints, the extra ones will become a
+		// filter.
+		final RexNode constraint =
+			modifiableView.getConstraint(rexBuilder, delegateRowType);
+		RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
+		final List<Pair<RexNode, String>> projects = new ArrayList<>();
+		for (RelDataTypeField field : delegateRowType.getFieldList()) {
+			RexNode node = projectMap.get(field.getIndex());
+			if (node == null) {
+				node = rexBuilder.makeNullLiteral(field.getType());
+			}
+			projects.add(
+				Pair.of(rexBuilder.ensureType(field.getType(), node, false),
+					field.getName()));
+		}
+
+		source = RelOptUtil.createProject(source, projects, true);
+		if (filters.size() > 0) {
+			source = RelOptUtil.createFilter(source, filters);
+		}
+		return source;
+	}
+
+	private RelOptTable.ToRelContext createToRelContext() {
+		return new RelOptTable.ToRelContext() {
+			public RelOptCluster getCluster() {
+				return cluster;
+			}
+
+			@Override public RelRoot expandView(
+				RelDataType rowType,
+				String queryString,
+				List<String> schemaPath,
+				List<String> viewPath) {
+				return viewExpander.expandView(rowType, queryString, schemaPath, viewPath);
+			}
+
+		};
+	}
+
+	public RelNode toRel(RelOptTable table) {
+		return table.toRel(createToRelContext());
+	}
+
+	protected RelOptTable getTargetTable(SqlNode call) {
+		SqlValidatorNamespace targetNs = validator.getNamespace(call).resolve();
+		return SqlValidatorUtil.getRelOptTable(targetNs, catalogReader, null, null);
+	}
+
+	/**
+	 * Creates a source for an INSERT statement.
+	 *
+	 * <p>If the column list is not specified, source expressions match target
+	 * columns in order.
+	 *
+	 * <p>If the column list is specified, Source expressions are mapped to
+	 * target columns by name via targetColumnList, and may not cover the entire
+	 * target table. So, we'll make up a full row, using a combination of
+	 * default values and the source expressions provided.
+	 *
+	 * @param call      Insert expression
+	 * @param sourceRel Source relational expression
+	 * @return Converted INSERT statement
+	 */
+	protected RelNode convertColumnList(
+		SqlInsert call,
+		RelNode sourceRel) {
+		RelDataType sourceRowType = sourceRel.getRowType();
+		final RexNode sourceRef =
+			rexBuilder.makeRangeReference(sourceRowType, 0, false);
+		final List<String> targetColumnNames = new ArrayList<>();
+		final List<RexNode> columnExprs = new ArrayList<>();
+		collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs);
+
+		final RelOptTable targetTable = getTargetTable(call);
+		final RelDataType targetRowType = targetTable.getRowType();
+		final List<RelDataTypeField> targetFields =
+			targetRowType.getFieldList();
+		final List<RexNode> sourceExps =
+			new ArrayList<>(
+				Collections.<RexNode>nCopies(targetFields.size(), null));
+		final List<String> fieldNames =
+			new ArrayList<>(
+				Collections.<String>nCopies(targetFields.size(), null));
+
+		final InitializerExpressionFactory initializerFactory =
+			getInitializerFactory(validator.getNamespace(call).getTable());
+
+		// Walk the name list and place the associated value in the
+		// expression list according to the ordinal value returned from
+		// the table construct, leaving nulls in the list for columns
+		// that are not referenced.
+		final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+		for (Pair<String, RexNode> p : Pair.zip(targetColumnNames, columnExprs)) {
+			RelDataTypeField field = nameMatcher.field(targetRowType, p.left);
+			assert field != null : "column " + p.left + " not found";
+			sourceExps.set(field.getIndex(), p.right);
+		}
+
+		// Walk the expression list and get default values for any columns
+		// that were not supplied in the statement. Get field names too.
+		for (int i = 0; i < targetFields.size(); ++i) {
+			final RelDataTypeField field = targetFields.get(i);
+			final String fieldName = field.getName();
+			fieldNames.set(i, fieldName);
+			if (sourceExps.get(i) != null) {
+				if (initializerFactory.isGeneratedAlways(targetTable, i)) {
+					throw RESOURCE.insertIntoAlwaysGenerated(fieldName).ex();
+				}
+				continue;
+			}
+			sourceExps.set(i,
+				initializerFactory.newColumnDefaultValue(targetTable, i));
+
+			// bare nulls are dangerous in the wrong hands
+			sourceExps.set(i,
+				castNullLiteralIfNeeded(sourceExps.get(i), field.getType()));
+		}
+
+		return RelOptUtil.createProject(sourceRel, sourceExps, fieldNames, true);
+	}
+
+	private InitializerExpressionFactory getInitializerFactory(
+		SqlValidatorTable validatorTable) {
+		// We might unwrap a null instead of a InitializerExpressionFactory.
+		final Table table = unwrap(validatorTable, Table.class);
+		if (table != null) {
+			InitializerExpressionFactory f =
+				unwrap(table, InitializerExpressionFactory.class);
+			if (f != null) {
+				return f;
+			}
+		}
+		return new NullInitializerExpressionFactory(typeFactory);
+	}
+
+	private static <T> T unwrap(Object o, Class<T> clazz) {
+		if (o instanceof Wrapper) {
+			return ((Wrapper) o).unwrap(clazz);
+		}
+		return null;
+	}
+
+	private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) {
+		if (!RexLiteral.isNullLiteral(node)) {
+			return node;
+		}
+		return rexBuilder.makeCast(type, node);
+	}
+
+	/**
+	 * Given an INSERT statement, collects the list of names to be populated and
+	 * the expressions to put in them.
+	 *
+	 * @param call              Insert statement
+	 * @param sourceRef         Expression representing a row from the source
+	 *                          relational expression
+	 * @param targetColumnNames List of target column names, to be populated
+	 * @param columnExprs       List of expressions, to be populated
+	 */
+	protected void collectInsertTargets(
+		SqlInsert call,
+		final RexNode sourceRef,
+		final List<String> targetColumnNames,
+		List<RexNode> columnExprs) {
+		final RelOptTable targetTable = getTargetTable(call);
+		final RelDataType tableRowType = targetTable.getRowType();
+		SqlNodeList targetColumnList = call.getTargetColumnList();
+		if (targetColumnList == null) {
+			if (validator.getConformance().isInsertSubsetColumnsAllowed()) {
+				final RelDataType targetRowType =
+					typeFactory.createStructType(
+						tableRowType.getFieldList()
+							.subList(0, sourceRef.getType().getFieldCount()));
+				targetColumnNames.addAll(targetRowType.getFieldNames());
+			} else {
+				targetColumnNames.addAll(tableRowType.getFieldNames());
+			}
+		} else {
+			for (int i = 0; i < targetColumnList.size(); i++) {
+				SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i);
+				RelDataTypeField field =
+					SqlValidatorUtil.getTargetField(
+						tableRowType, typeFactory, id, catalogReader, targetTable);
+				assert field != null : "column " + id.toString() + " not found";
+				targetColumnNames.add(field.getName());
+			}
+		}
+
+		for (int i = 0; i < targetColumnNames.size(); i++) {
+			final RexNode expr = rexBuilder.makeFieldAccess(sourceRef, i);
+			columnExprs.add(expr);
+		}
+	}
+
+	private RelNode convertDelete(SqlDelete call) {
+		RelOptTable targetTable = getTargetTable(call);
+		RelNode sourceRel = convertSelect(call.getSourceSelect(), false);
+		return LogicalTableModify.create(targetTable, catalogReader, sourceRel,
+			LogicalTableModify.Operation.DELETE, null, null, false);
+	}
+
+	private RelNode convertUpdate(SqlUpdate call) {
+		final SqlValidatorScope scope = validator.getWhereScope(call.getSourceSelect());
+		Blackboard bb = createBlackboard(scope, null, false);
+
+		Builder<RexNode> rexNodeSourceExpressionListBuilder = ImmutableList.builder();
+		for (SqlNode n : call.getSourceExpressionList()) {
+			RexNode rn = bb.convertExpression(n);
+			rexNodeSourceExpressionListBuilder.add(rn);
+		}
+
+		RelOptTable targetTable = getTargetTable(call);
+
+		// convert update column list from SqlIdentifier to String
+		final List<String> targetColumnNameList = new ArrayList<>();
+		final RelDataType targetRowType = targetTable.getRowType();
+		for (SqlNode node : call.getTargetColumnList()) {
+			SqlIdentifier id = (SqlIdentifier) node;
+			RelDataTypeField field =
+				SqlValidatorUtil.getTargetField(
+					targetRowType, typeFactory, id, catalogReader, targetTable);
+			assert field != null : "column " + id.toString() + " not found";
+			targetColumnNameList.add(field.getName());
+		}
+
+		RelNode sourceRel = convertSelect(call.getSourceSelect(), false);
+
+		return LogicalTableModify.create(targetTable, catalogReader, sourceRel,
+			LogicalTableModify.Operation.UPDATE, targetColumnNameList,
+			rexNodeSourceExpressionListBuilder.build(), false);
+	}
+
+	private RelNode convertMerge(SqlMerge call) {
+		RelOptTable targetTable = getTargetTable(call);
+
+		// convert update column list from SqlIdentifier to String
+		final List<String> targetColumnNameList = new ArrayList<>();
+		final RelDataType targetRowType = targetTable.getRowType();
+		SqlUpdate updateCall = call.getUpdateCall();
+		if (updateCall != null) {
+			for (SqlNode targetColumn : updateCall.getTargetColumnList()) {
+				SqlIdentifier id = (SqlIdentifier) targetColumn;
+				RelDataTypeField field =
+					SqlValidatorUtil.getTargetField(
+						targetRowType, typeFactory, id, catalogReader, targetTable);
+				assert field != null : "column " + id.toString() + " not found";
+				targetColumnNameList.add(field.getName());
+			}
+		}
+
+		// replace the projection of the source select with a
+		// projection that contains the following:
+		// 1) the expressions corresponding to the new insert row (if there is
+		//    an insert)
+		// 2) all columns from the target table (if there is an update)
+		// 3) the set expressions in the update call (if there is an update)
+
+		// first, convert the merge's source select to construct the columns
+		// from the target table and the set expressions in the update call
+		RelNode mergeSourceRel = convertSelect(call.getSourceSelect(), false);
+
+		// then, convert the insert statement so we can get the insert
+		// values expressions
+		SqlInsert insertCall = call.getInsertCall();
+		int nLevel1Exprs = 0;
+		List<RexNode> level1InsertExprs = null;
+		List<RexNode> level2InsertExprs = null;
+		if (insertCall != null) {
+			RelNode insertRel = convertInsert(insertCall);
+
+			// if there are 2 level of projections in the insert source, combine
+			// them into a single project; level1 refers to the topmost project;
+			// the level1 projection contains references to the level2
+			// expressions, except in the case where no target expression was
+			// provided, in which case, the expression is the default value for
+			// the column; or if the expressions directly map to the source
+			// table
+			level1InsertExprs =
+				((LogicalProject) insertRel.getInput(0)).getProjects();
+			if (insertRel.getInput(0).getInput(0) instanceof LogicalProject) {
+				level2InsertExprs =
+					((LogicalProject) insertRel.getInput(0).getInput(0))
+						.getProjects();
+			}
+			nLevel1Exprs = level1InsertExprs.size();
+		}
+
+		LogicalJoin join = (LogicalJoin) mergeSourceRel.getInput(0);
+		int nSourceFields = join.getLeft().getRowType().getFieldCount();
+		final List<RexNode> projects = new ArrayList<>();
+		for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) {
+			if ((level2InsertExprs != null)
+				&& (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) {
+				int level2Idx =
+					((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex();
+				projects.add(level2InsertExprs.get(level2Idx));
+			} else {
+				projects.add(level1InsertExprs.get(level1Idx));
+			}
+		}
+		if (updateCall != null) {
+			final LogicalProject project = (LogicalProject) mergeSourceRel;
+			projects.addAll(
+				Util.skip(project.getProjects(), nSourceFields));
+		}
+
+		RelNode massagedRel =
+			RelOptUtil.createProject(join, projects, null, true);
+
+		return LogicalTableModify.create(targetTable, catalogReader, massagedRel,
+			LogicalTableModify.Operation.MERGE, targetColumnNameList, null, false);
+	}
+
+	/**
+	 * Converts an identifier into an expression in a given scope. For example,
+	 * the "empno" in "select empno from emp join dept" becomes "emp.empno".
+	 */
+	private RexNode convertIdentifier(
+		Blackboard bb,
+		SqlIdentifier identifier) {
+		// first check for reserved identifiers like CURRENT_USER
+		final SqlCall call = SqlUtil.makeCall(opTab, identifier);
+		if (call != null) {
+			return bb.convertExpression(call);
+		}
+
+		String pv = null;
+		if (bb.isPatternVarRef && identifier.names.size() > 1) {
+			pv = identifier.names.get(0);
+		}
+
+		final SqlQualified qualified;
+		if (bb.scope != null) {
+			qualified = bb.scope.fullyQualify(identifier);
+		} else {
+			qualified = SqlQualified.create(null, 1, null, identifier);
+		}
+		final Pair<RexNode, Map<String, Integer>> e0 = bb.lookupExp(qualified);
+		RexNode e = e0.left;
+		for (String name : qualified.suffixTranslated()) {
+			if (e == e0.left && e0.right != null) {
+				int i = e0.right.get(name);
+				e = rexBuilder.makeFieldAccess(e, i);
+			} else {
+				final boolean caseSensitive = true; // name already fully-qualified
+				e = rexBuilder.makeFieldAccess(e, name, caseSensitive);
+			}
+		}
+		if (e instanceof RexInputRef) {
+			// adjust the type to account for nulls introduced by outer joins
+			e = adjustInputRef(bb, (RexInputRef) e);
+		}
+
+		if (e0.left instanceof RexCorrelVariable) {
+			assert e instanceof RexFieldAccess;
+			final RexNode prev =
+				bb.mapCorrelateToRex.put(((RexCorrelVariable) e0.left).id,
+					(RexFieldAccess) e);
+			assert prev == null;
+		}
+		return e;
+	}
+
+	/**
+	 * Adjusts the type of a reference to an input field to account for nulls
+	 * introduced by outer joins; and adjusts the offset to match the physical
+	 * implementation.
+	 *
+	 * @param bb       Blackboard
+	 * @param inputRef Input ref
+	 * @return Adjusted input ref
+	 */
+	protected RexNode adjustInputRef(
+		Blackboard bb,
+		RexInputRef inputRef) {
+		RelDataTypeField field = bb.getRootField(inputRef);
+		if (field != null) {
+			return rexBuilder.makeInputRef(
+				field.getType(),
+				inputRef.getIndex());
+		}
+		return inputRef;
+	}
+
+	/**
+	 * Converts a row constructor into a relational expression.
+	 *
+	 * @param bb             Blackboard
+	 * @param rowConstructor Row constructor expression
+	 * @return Relational expression which returns a single row.
+	 */
+	private RelNode convertRowConstructor(
+		Blackboard bb,
+		SqlCall rowConstructor) {
+		Preconditions.checkArgument(isRowConstructor(rowConstructor));
+		final List<SqlNode> operands = rowConstructor.getOperandList();
+		return convertMultisets(operands, bb);
+	}
+
+	private RelNode convertCursor(Blackboard bb, SubQuery subQuery) {
+		final SqlCall cursorCall = (SqlCall) subQuery.node;
+		assert cursorCall.operandCount() == 1;
+		SqlNode query = cursorCall.operand(0);
+		RelNode converted = convertQuery(query, false, false).rel;
+		int iCursor = bb.cursors.size();
+		bb.cursors.add(converted);
+		subQuery.expr =
+			new RexInputRef(
+				iCursor,
+				converted.getRowType());
+		return converted;
+	}
+
+	private RelNode convertMultisets(final List<SqlNode> operands,
+		Blackboard bb) {
+		// NOTE: Wael 2/04/05: this implementation is not the most efficient in
+		// terms of planning since it generates XOs that can be reduced.
+		final List<Object> joinList = new ArrayList<>();
+		List<SqlNode> lastList = new ArrayList<>();
+		for (int i = 0; i < operands.size(); i++) {
+			SqlNode operand = operands.get(i);
+			if (!(operand instanceof SqlCall)) {
+				lastList.add(operand);
+				continue;
+			}
+
+			final SqlCall call = (SqlCall) operand;
+			final RelNode input;
+			switch (call.getKind()) {
+				case MULTISET_VALUE_CONSTRUCTOR:
+				case ARRAY_VALUE_CONSTRUCTOR:
+					final SqlNodeList list =
+						new SqlNodeList(call.getOperandList(), call.getParserPosition());
+					CollectNamespace nss =
+						(CollectNamespace) validator.getNamespace(call);
+					Blackboard usedBb;
+					if (null != nss) {
+						usedBb = createBlackboard(nss.getScope(), null, false);
+					} else {
+						usedBb =
+							createBlackboard(new ListScope(bb.scope) {
+								public SqlNode getNode() {
+									return call;
+								}
+							}, null, false);
+					}
+					RelDataType multisetType = validator.getValidatedNodeType(call);
+					((SqlValidatorImpl) validator).setValidatedNodeType(list,
+						multisetType.getComponentType());
+					input = convertQueryOrInList(usedBb, list, null);
+					break;
+				case MULTISET_QUERY_CONSTRUCTOR:
+				case ARRAY_QUERY_CONSTRUCTOR:
+					final RelRoot root = convertQuery(call.operand(0), false, true);
+					input = root.rel;
+					break;
+				default:
+					lastList.add(operand);
+					continue;
+			}
+
+			if (lastList.size() > 0) {
+				joinList.add(lastList);
+			}
+			lastList = new ArrayList<>();
+			Collect collect =
+				new Collect(
+					cluster,
+					cluster.traitSetOf(Convention.NONE),
+					input,
+					validator.deriveAlias(call, i));
+			joinList.add(collect);
+		}
+
+		if (joinList.size() == 0) {
+			joinList.add(lastList);
+		}
+
+		for (int i = 0; i < joinList.size(); i++) {
+			Object o = joinList.get(i);
+			if (o instanceof List) {
+				@SuppressWarnings("unchecked")
+				List<SqlNode> projectList = (List<SqlNode>) o;
+				final List<RexNode> selectList = new ArrayList<>();
+				final List<String> fieldNameList = new ArrayList<>();
+				for (int j = 0; j < projectList.size(); j++) {
+					SqlNode operand = projectList.get(j);
+					selectList.add(bb.convertExpression(operand));
+
+					// REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal
+					// instead of deriveAlias to match field names from
+					// SqlRowOperator. Otherwise, get error   Type
+					// 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when
+					// doing   select * from unnest(     select multiset[empno]
+					// from sales.emps);
+
+					fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j));
+				}
+
+				RelNode projRel =
+					RelOptUtil.createProject(
+						LogicalValues.createOneRow(cluster),
+						selectList,
+						fieldNameList);
+
+				joinList.set(i, projRel);
+			}
+		}
+
+		RelNode ret = (RelNode) joinList.get(0);
+		for (int i = 1; i < joinList.size(); i++) {
+			RelNode relNode = (RelNode) joinList.get(i);
+			ret =
+				RelFactories.DEFAULT_JOIN_FACTORY.createJoin(
+					ret,
+					relNode,
+					rexBuilder.makeLiteral(true),
+					ImmutableSet.<CorrelationId>of(),
+					JoinRelType.INNER,
+					false);
+		}
+		return ret;
+	}
+
+	private void convertSelectList(
+		Blackboard bb,
+		SqlSelect select,
+		List<SqlNode> orderList) {
+		SqlNodeList selectList = select.getSelectList();
+		selectList = validator.expandStar(selectList, select, false);
+
+		replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+		List<String> fieldNames = new ArrayList<>();
+		final List<RexNode> exprs = new ArrayList<>();
+		final Collection<String> aliases = new TreeSet<>();
+
+		// Project any system fields. (Must be done before regular select items,
+		// because offsets may be affected.)
+		final List<SqlMonotonicity> columnMonotonicityList = new ArrayList<>();
+		extraSelectItems(
+			bb,
+			select,
+			exprs,
+			fieldNames,
+			aliases,
+			columnMonotonicityList);
+
+		// Project select clause.
+		int i = -1;
+		for (SqlNode expr : selectList) {
+			++i;
+			exprs.add(bb.convertExpression(expr));
+			fieldNames.add(deriveAlias(expr, aliases, i));
+		}
+
+		// Project extra fields for sorting.
+		for (SqlNode expr : orderList) {
+			++i;
+			SqlNode expr2 = validator.expandOrderExpr(select, expr);
+			exprs.add(bb.convertExpression(expr2));
+			fieldNames.add(deriveAlias(expr, aliases, i));
+		}
+
+		fieldNames = SqlValidatorUtil.uniquify(fieldNames,
+			catalogReader.nameMatcher().isCaseSensitive());
+
+		bb.setRoot(
+			RelOptUtil.createProject(bb.root, exprs, fieldNames),
+			false);
+
+		assert bb.columnMonotonicities.isEmpty();
+		bb.columnMonotonicities.addAll(columnMonotonicityList);
+		for (SqlNode selectItem : selectList) {
+			bb.columnMonotonicities.add(
+				selectItem.getMonotonicity(bb.scope));
+		}
+	}
+
+	/**
+	 * Adds extra select items. The default implementation adds nothing; derived
+	 * classes may add columns to exprList, nameList, aliasList and
+	 * columnMonotonicityList.
+	 *
+	 * @param bb                     Blackboard
+	 * @param select                 Select statement being translated
+	 * @param exprList               List of expressions in select clause
+	 * @param nameList               List of names, one per column
+	 * @param aliasList              Collection of aliases that have been used
+	 *                               already
+	 * @param columnMonotonicityList List of monotonicity, one per column
+	 */
+	protected void extraSelectItems(
+		Blackboard bb,
+		SqlSelect select,
+		List<RexNode> exprList,
+		List<String> nameList,
+		Collection<String> aliasList,
+		List<SqlMonotonicity> columnMonotonicityList) {
+	}
+
+	private String deriveAlias(
+		final SqlNode node,
+		Collection<String> aliases,
+		final int ordinal) {
+		String alias = validator.deriveAlias(node, ordinal);
+		if ((alias == null) || aliases.contains(alias)) {
+			String aliasBase = (alias == null) ? "EXPR$" : alias;
+			for (int j = 0;; j++) {
+				alias = aliasBase + j;
+				if (!aliases.contains(alias)) {
+					break;
+				}
+			}
+		}
+		aliases.add(alias);
+		return alias;
+	}
+
+	/**
+	 * Converts a WITH sub-query into a relational expression.
+	 */
+	public RelRoot convertWith(SqlWith with, boolean top) {
+		return convertQuery(with.body, false, top);
+	}
+
+	/**
+	 * Converts a SELECT statement's parse tree into a relational expression.
+	 */
+	public RelNode convertValues(
+		SqlCall values,
+		RelDataType targetRowType) {
+		final SqlValidatorScope scope = validator.getOverScope(values);
+		assert scope != null;
+		final Blackboard bb = createBlackboard(scope, null, false);
+		convertValuesImpl(bb, values, targetRowType);
+		return bb.root;
+	}
+
+	/**
+	 * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a
+	 * relational expression.
+	 *
+	 * @param bb            Blackboard
+	 * @param values        Call to SQL VALUES operator
+	 * @param targetRowType Target row type
+	 */
+	private void convertValuesImpl(
+		Blackboard bb,
+		SqlCall values,
+		RelDataType targetRowType) {
+		// Attempt direct conversion to LogicalValues; if that fails, deal with
+		// fancy stuff like sub-queries below.
+		RelNode valuesRel =
+			convertRowValues(
+				bb,
+				values,
+				values.getOperandList(),
+				true,
+				targetRowType);
+		if (valuesRel != null) {
+			bb.setRoot(valuesRel, true);
+			return;
+		}
+
+		final List<RelNode> unionRels = new ArrayList<>();
+		for (SqlNode rowConstructor1 : values.getOperandList()) {
+			SqlCall rowConstructor = (SqlCall) rowConstructor1;
+			Blackboard tmpBb = createBlackboard(bb.scope, null, false);
+			replaceSubQueries(tmpBb, rowConstructor,
+				RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+			final List<Pair<RexNode, String>> exps = new ArrayList<>();
+			for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
+				exps.add(
+					Pair.of(
+						tmpBb.convertExpression(operand.e),
+						validator.deriveAlias(operand.e, operand.i)));
+			}
+			RelNode in =
+				(null == tmpBb.root)
+					? LogicalValues.createOneRow(cluster)
+					: tmpBb.root;
+			unionRels.add(
+				RelOptUtil.createProject(
+					in,
+					Pair.left(exps),
+					Pair.right(exps),
+					true));
+		}
+
+		if (unionRels.size() == 0) {
+			throw new AssertionError("empty values clause");
+		} else if (unionRels.size() == 1) {
+			bb.setRoot(
+				unionRels.get(0),
+				true);
+		} else {
+			bb.setRoot(
+				LogicalUnion.create(unionRels, true),
+				true);
+		}
+
+		// REVIEW jvs 22-Jan-2004:  should I add
+		// mapScopeToLux.put(validator.getScope(values),bb.root);
+		// ?
+	}
+
+	//~ Inner Classes ----------------------------------------------------------
+
+	/**
+	 * Workspace for translating an individual SELECT statement (or sub-SELECT).
+	 */
+	protected class Blackboard implements SqlRexContext, SqlVisitor<RexNode> {
+		/**
+		 * Collection of {@link RelNode} objects which correspond to a SELECT
+		 * statement.
+		 */
+		public final SqlValidatorScope scope;
+		private final Map<String, RexNode> nameToNodeMap;
+		public RelNode root;
+		private List<RelNode> inputs;
+		private final Map<CorrelationId, RexFieldAccess> mapCorrelateToRex =
+			new HashMap<>();
+
+		private boolean isPatternVarRef = false;
+
+		final List<RelNode> cursors = new ArrayList<>();
+
+		/**
+		 * List of <code>IN</code> and <code>EXISTS</code> nodes inside this
+		 * <code>SELECT</code> statement (but not inside sub-queries).
+		 */
+		private final Set<SubQuery> subQueryList = new LinkedHashSet<>();
+
+		/**
+		 * Workspace for building aggregates.
+		 */
+		AggConverter agg;
+
+		/**
+		 * When converting window aggregate, we need to know if the window is
+		 * guaranteed to be non-empty.
+		 */
+		SqlWindow window;
+
+		/**
+		 * Project the groupby expressions out of the root of this sub-select.
+		 * Sub-queries can reference group by expressions projected from the
+		 * "right" to the sub-query.
+		 */
+		private final Map<RelNode, Map<Integer, Integer>>
+			mapRootRelToFieldProjection = new HashMap<>();
+
+		private final List<SqlMonotonicity> columnMonotonicities =
+			new ArrayList<>();
+
+		private final List<RelDataTypeField> systemFieldList = new ArrayList<>();
+		final boolean top;
+
+		private final InitializerExpressionFactory initializerExpressionFactory =
+			new NullInitializerExpressionFactory(typeFactory);
+
+		/**
+		 * Creates a Blackboard.
+		 *
+		 * @param scope         Name-resolution scope for expressions validated
+		 *                      within this query. Can be null if this Blackboard is
+		 *                      for a leaf node, say
+		 * @param nameToNodeMap Map which translates the expression to map a
+		 *                      given parameter into, if translating expressions;
+		 *                      null otherwise
+		 * @param top           Whether this is the root of the query
+		 */
+		protected Blackboard(SqlValidatorScope scope,
+			Map<String, RexNode> nameToNodeMap, boolean top) {
+			this.scope = scope;
+			this.nameToNodeMap = nameToNodeMap;
+			this.top = top;
+		}
+
+		public void setPatternVarRef(boolean isVarRef) {
+			this.isPatternVarRef = isVarRef;
+		}
+
+		public RexNode register(
+			RelNode rel,
+			JoinRelType joinType) {
+			return register(rel, joinType, null);
+		}
+
+		/**
+		 * Registers a relational expression.
+		 *
+		 * @param rel               Relational expression
+		 * @param joinType          Join type
+		 * @param leftKeys LHS of IN clause, or null for expressions
+		 *                          other than IN
+		 * @return Expression with which to refer to the row (or partial row)
+		 * coming from this relational expression's side of the join
+		 */
+		public RexNode register(
+			RelNode rel,
+			JoinRelType joinType,
+			List<RexNode> leftKeys) {
+			assert joinType != null;
+			if (root == null) {
+				assert leftKeys == null;
+				setRoot(rel, false);
+				return rexBuilder.makeRangeReference(
+					root.getRowType(),
+					0,
+					false);
+			}
+
+			final RexNode joinCond;
+			final int origLeftInputCount = root.getRowType().getFieldCount();
+			if (leftKeys != null) {
+				List<RexNode> newLeftInputExpr = Lists.newArrayList();
+				for (int i = 0; i < origLeftInputCount; i++) {
+					newLeftInputExpr.add(rexBuilder.makeInputRef(root, i));
+				}
+
+				final List<Integer> leftJoinKeys = Lists.newArrayList();
+				for (RexNode leftKey : leftKeys) {
+					int index = newLeftInputExpr.indexOf(leftKey);
+					if (index < 0 || joinType == JoinRelType.LEFT) {
+						index = newLeftInputExpr.size();
+						newLeftInputExpr.add(leftKey);
+					}
+					leftJoinKeys.add(index);
+				}
+
+				RelNode newLeftInput =
+					RelOptUtil.createProject(
+						root,
+						newLeftInputExpr,
+						null,
+						true);
+
+				// maintain the group by mapping in the new LogicalProject
+				if (mapRootRelToFieldProjection.containsKey(root)) {
+					mapRootRelToFieldProjection.put(
+						newLeftInput,
+						mapRootRelToFieldProjection.get(root));
+				}
+
+				setRoot(newLeftInput, false);
+
+				// right fields appear after the LHS fields.
+				final int rightOffset = root.getRowType().getFieldCount()
+					- newLeftInput.getRowType().getFieldCount();
+				final List<Integer> rightKeys =
+					Util.range(rightOffset, rightOffset + leftKeys.size());
+
+				joinCond =
+					RelOptUtil.createEquiJoinCondition(newLeftInput, leftJoinKeys,
+						rel, rightKeys, rexBuilder);
+			} else {
+				joinCond = rexBuilder.makeLiteral(true);
+			}
+
+			int leftFieldCount = root.getRowType().getFieldCount();
+			final RelNode join =
+				createJoin(
+					this,
+					root,
+					rel,
+					joinCond,
+					joinType);
+
+			setRoot(join, false);
+
+			if (leftKeys != null
+				&& joinType == JoinRelType.LEFT) {
+				final int leftKeyCount = leftKeys.size();
+				int rightFieldLength = rel.getRowType().getFieldCount();
+				assert leftKeyCount == rightFieldLength - 1;
+
+				final int rexRangeRefLength = leftKeyCount + rightFieldLength;
+				RelDataType returnType =
+					typeFactory.createStructType(
+						new AbstractList<Map.Entry<String, RelDataType>>() {
+							public Map.Entry<String, RelDataType> get(
+								int index) {
+								return join.getRowType().getFieldList()
+									.get(origLeftInputCount + index);
+							}
+
+							public int size() {
+								return rexRangeRefLength;
+							}
+						});
+
+				return rexBuilder.makeRangeReference(
+					returnType,
+					origLeftInputCount,
+					false);
+			} else {
+				return rexBuilder.makeRangeReference(
+					rel.getRowType(),
+					leftFieldCount,
+					joinType.generatesNullsOnRight());
+			}
+		}
+
+		/**
+		 * Sets a new root relational expression, as the translation process
+		 * backs its way further up the tree.
+		 *
+		 * @param root New root relational expression
+		 * @param leaf Whether the relational expression is a leaf, that is,
+		 *             derived from an atomic relational expression such as a table
+		 *             name in the from clause, or the projection on top of a
+		 *             select-sub-query. In particular, relational expressions
+		 *             derived from JOIN operators are not leaves, but set
+		 *             expressions are.
+		 */
+		public void setRoot(RelNode root, boolean leaf) {
+			setRoot(
+				Collections.singletonList(root), root, root instanceof LogicalJoin);
+			if (leaf) {
+				leaves.add(root);
+			}
+			this.columnMonotonicities.clear();
+		}
+
+		private void setRoot(
+			List<RelNode> inputs,
+			RelNode root,
+			boolean hasSystemFields) {
+			this.inputs = inputs;
+			this.root = root;
+			this.systemFieldList.clear();
+			if (hasSystemFields) {
+				this.systemFieldList.addAll(getSystemFields());
+			}
+		}
+
+		/**
+		 * Notifies this Blackboard that the root just set using
+		 * {@link #setRoot(RelNode, boolean)} was derived using dataset
+		 * substitution.
+		 *
+		 * <p>The default implementation is not interested in such
+		 * notifications, and does nothing.
+		 *
+		 * @param datasetName Dataset name
+		 */
+		public void setDataset(String datasetName) {
+		}
+
+		void setRoot(List<RelNode> inputs) {
+			setRoot(inputs, null, false);
+		}
+
+		/**
+		 * Returns an expression with which to reference a from-list item.
+		 *
+		 * @param qualified the alias of the from item
+		 * @return a {@link RexFieldAccess} or {@link RexRangeRef}, or null if
+		 * not found
+		 */
+		Pair<RexNode, Map<String, Integer>> lookupExp(SqlQualified qualified) {
+			if (nameToNodeMap != null && qualified.prefixLength == 1) {
+				RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0));
+				if (node == null) {
+					throw new AssertionError("Unknown identifier '" + qualified.identifier
+						+ "' encountered while expanding expression");
+				}
+				return Pair.of(node, null);
+			}
+			final SqlNameMatcher nameMatcher =
+				scope.getValidator().getCatalogReader().nameMatcher();
+			final SqlValidatorScope.ResolvedImpl resolved =
+				new SqlValidatorScope.ResolvedImpl();
+			scope.resolve(qualified.prefix(), nameMatcher, false, resolved);
+			if (!(resolved.count() == 1)) {
+				return null;
+			}
+			final SqlValidatorScope.Resolve resolve = resolved.only();
+			final RelDataType rowType = resolve.rowType();
+
+			// Found in current query's from list.  Find which from item.
+			// We assume that the order of the from clause items has been
+			// preserved.
+			final SqlValidatorScope ancestorScope = resolve.scope;
+			boolean isParent = ancestorScope != scope;
+			if ((inputs != null) && !isParent) {
+				final LookupContext rels =
+					new LookupContext(this, inputs, systemFieldList.size());
+				final RexNode node = lookup(resolve.path.steps().get(0).i, rels);
+				if (node == null) {
+					return null;
+				} else {
+					return Pair.of(node, null);
+				}
+			} else {
+				// We're referencing a relational expression which has not been
+				// converted yet. This occurs when from items are correlated,
+				// e.g. "select from emp as emp join emp.getDepts() as dept".
+				// Create a temporary expression.
+				DeferredLookup lookup =
+					new DeferredLookup(this, qualified.identifier.names.get(0));
+				final CorrelationId correlId = cluster.createCorrel();
+				mapCorrelToDeferred.put(correlId, lookup);
+				if (resolve.path.steps().get(0).i < 0) {
+					return Pair.of(rexBuilder.makeCorrel(rowType, correlId), null);
+				} else {
+					final RelDataTypeFactory.FieldInfoBuilder builder =
+						typeFactory.builder();
+					final ListScope ancestorScope1 = (ListScope) resolve.scope;
+					final ImmutableMap.Builder<String, Integer> fields =
+						ImmutableMap.builder();
+					int i = 0;
+					int offset = 0;
+					for (SqlValidatorNamespace c : ancestorScope1.getChildren()) {
+						builder.addAll(c.getRowType().getFieldList());
+						if (i == resolve.path.steps().get(0).i) {
+							for (RelDataTypeField field : c.getRowType().getFieldList()) {
+								fields.put(c.translate(field.getName()),
+									field.getIndex() + offset);
+							}
+						}
+						++i;
+						offset += c.getRowType().getFieldCount();
+					}
+					final RexNode c =
+						rexBuilder.makeCorrel(builder.uniquify().build(), correlId);
+					return Pair.<RexNode, Map<String, Integer>>of(c, fields.build());
+				}
+			}
+		}
+
+		/**
+		 * Creates an expression with which to reference the expression whose
+		 * offset in its from-list is {@code offset}.
+		 */
+		RexNode lookup(
+			int offset,
+			LookupContext lookupContext) {
+			Pair<RelNode, Integer> pair = lookupContext.findRel(offset);
+			return rexBuilder.makeRangeReference(
+				pair.left.getRowType(),
+				pair.right,
+				false);
+		}
+
+		RelDataTypeField getRootField(RexInputRef inputRef) {
+			int fieldOffset = inputRef.getIndex();
+			for (RelNode input : inputs) {
+				RelDataType rowType = input.getRowType();
+				if (rowType == null) {
+					// TODO:  remove this once leastRestrictive
+					// is correctly implemented
+					return null;
+				}
+				if (fieldOffset < rowType.getFieldCount()) {
+					return rowType.getFieldList().get(fieldOffset);
+				}
+				fieldOffset -= rowType.getFieldCount();
+			}
+			throw new AssertionError();
+		}
+
+		public void flatten(
+			List<RelNode> rels,
+			int systemFieldCount,
+			int[] start,
+			List<Pair<RelNode, Integer>> relOffsetList) {
+			for (RelNode rel : rels) {
+				if (leaves.contains(rel)) {
+					relOffsetList.add(
+						Pair.of(rel, start[0]));
+					start[0] += rel.getRowType().getFieldCount();
+				} else {
+					if (rel instanceof LogicalJoin
+						|| rel instanceof LogicalAggregate) {
+						start[0] += systemFieldCount;
+					}
+					flatten(
+						rel.getInputs(),
+						systemFieldCount,
+						start,
+						relOffsetList);
+				}
+			}
+		}
+
+		void registerSubQuery(SqlNode node, RelOptUtil.Logic logic) {
+			for (SubQuery subQuery : subQueryList) {
+				if (node.equalsDeep(subQuery.node, Litmus.IGNORE)) {
+					return;
+				}
+			}
+			subQueryList.add(new SubQuery(node, logic));
+		}
+
+		SubQuery getSubQuery(SqlNode expr) {
+			for (SubQuery subQuery : subQueryList) {
+				if (expr.equalsDeep(subQuery.node, Litmus.IGNORE)) {
+					return subQuery;
+				}
+			}
+
+			return null;
+		}
+
+		ImmutableList<RelNode> retrieveCursors() {
+			try {
+				return ImmutableList.copyOf(cursors);
+			} finally {
+				cursors.clear();
+			}
+		}
+
+		public RexNode convertExpression(SqlNode expr) {
+			// If we're in aggregation mode and this is an expression in the
+			// GROUP BY clause, return a reference to the field.
+			if (agg != null) {
+				final SqlNode expandedGroupExpr = validator.expand(expr, scope);
+				final int ref = agg.lookupGroupExpr(expandedGroupExpr);
+				if (ref >= 0) {
+					return rexBuilder.makeInputRef(root, ref);
+				}
+				if (expr instanceof SqlCall) {
+					final RexNode rex = agg.lookupAggregates((SqlCall) expr);
+					if (rex != null) {
+						return rex;
+					}
+				}
+			}
+
+			// Allow the derived class chance to override the standard
+			// behavior for special kinds of expressions.
+			RexNode rex = convertExtendedExpression(expr, this);
+			if (rex != null) {
+				return rex;
+			}
+
+			// Sub-queries and OVER expressions are not like ordinary
+			// expressions.
+			final SqlKind kind = expr.getKind();
+			final SubQuery subQuery;
+			if (!config.isExpand()) {
+				final SqlCall call;
+				final SqlNode query;
+				final RelRoot root;
+				switch (kind) {
+					case IN:
+						call = (SqlCall) expr;
+						query = call.operand(1);
+						if (!(query instanceof SqlNodeList)) {
+							final SqlInOperator op = (SqlInOperator) call.getOperator();
+							root = convertQueryRecursive(query, false, null);
+							final SqlNode operand = call.operand(0);
+							List<SqlNode> nodes;
+							switch (operand.getKind()) {
+								case ROW:
+									nodes = ((SqlCall) operand).getOperandList();
+									break;
+								default:
+									nodes = ImmutableList.of(operand);
+							}
+							final ImmutableList.Builder<RexNode> builder =
+								ImmutableList.builder();
+							for (SqlNode node : nodes) {
+								builder.add(convertExpression(node));
+							}
+							final RexSubQuery in = RexSubQuery.in(root.rel, builder.build());
+							return op.isNotIn()
+								? rexBuilder.makeCall(SqlStdOperatorTable.NOT, in)
+								: in;
+						}
+						break;
+
+					case EXISTS:
+						call = (SqlCall) expr;
+						query = Iterables.getOnlyElement(call.getOperandList());
+						root = convertQueryRecursive(query, false, null);
+						RelNode rel = root.rel;
+						while (rel instanceof Project
+							|| rel instanceof Sort
+							&& ((Sort) rel).fetch == null
+							&& ((Sort) rel).offset == null) {
+							rel = ((SingleRel) rel).getInput();
+						}
+						return RexSubQuery.exists(rel);
+
+					case SCALAR_QUERY:
+						call = (SqlCall) expr;
+						query = Iterables.getOnlyElement(call.getOperandList());
+						root = convertQueryRecursive(query, false, null);
+						return RexSubQuery.scalar(root.rel);
+				}
+			}
+
+			switch (kind) {
+				case CURSOR:
+				case IN:
+					subQuery = Preconditions.checkNotNull(getSubQuery(expr));
+					rex = Preconditions.checkNotNull(subQuery.expr);
+					return StandardConvertletTable.castToValidatedType(expr, rex,
+						validator, rexBuilder);
+
+				case SELECT:
+				case EXISTS:
+				case SCALAR_QUERY:
+					subQuery = getSubQuery(expr);
+					assert subQuery != null;
+					rex = subQuery.expr;
+					assert rex != null : "rex != null";
+
+					if (((kind == SqlKind.SCALAR_QUERY)
+						|| (kind == SqlKind.EXISTS))
+						&& isConvertedSubq(rex)) {
+						// scalar sub-query or EXISTS has been converted to a
+						// constant
+						return rex;
+					}
+
+					// The indicator column is the last field of the sub-query.
+					RexNode fieldAccess =
+						rexBuilder.makeFieldAccess(
+							rex,
+							rex.getType().getFieldCount() - 1);
+
+					// The indicator column will be nullable if it comes from
+					// the null-generating side of the join. For EXISTS, add an
+					// "IS TRUE" check so that the result is "BOOLEAN NOT NULL".
+					if (fieldAccess.getType().isNullable()
+						&& kind == SqlKind.EXISTS) {
+						fieldAccess =
+							rexBuilder.makeCall(
+								SqlStdOperatorTable.IS_NOT_NULL,
+								fieldAccess);
+					}
+					return fieldAccess;
+
+				case OVER:
+					return convertOver(this, expr);
+
+				default:
+					// fall through
+			}
+
+			// Apply standard conversions.
+			rex = expr.accept(this);
+			return Preconditions.checkNotNull(rex);
+		}
+
+		/**
+		 * Converts an item in an ORDER BY clause, extracting DESC, NULLS LAST
+		 * and NULLS FIRST flags first.
+		 */
+		public RexNode convertSortExpression(SqlNode expr, Set<SqlKind> flags) {
+			switch (expr.getKind()) {
+				case DESCENDING:
+				case NULLS_LAST:
+				case NULLS_FIRST:
+					flags.add(expr.getKind());
+					final SqlNode operand = ((SqlCall) expr).operand(0);
+					return convertSortExpression(operand, flags);
+				default:
+					return convertExpression(expr);
+			}
+		}
+
+		/**
+		 * Determines whether a RexNode corresponds to a sub-query that's been
+		 * converted to a constant.
+		 *
+		 * @param rex the expression to be examined
+		 * @return true if the expression is a dynamic parameter, a literal, or
+		 * a literal that is being cast
+		 */
+		private boolean isConvertedSubq(RexNode rex) {
+			if ((rex instanceof RexLiteral)
+				|| (rex instanceof RexDynamicParam)) {
+				return true;
+			}
+			if (rex instanceof RexCall) {
+				RexCall call = (RexCall) rex;
+				if (call.getOperator() == SqlStdOperatorTable.CAST) {
+					RexNode operand = call.getOperands().get(0);
+					if (operand instanceof RexLiteral) {
+						return true;
+					}
+				}
+			}
+			return false;
+		}
+
+		public int getGroupCount() {
+			if (agg != null) {
+				return agg.groupExprs.size();
+			}
+			if (window != null) {
+				return window.isAlwaysNonEmpty() ? 1 : 0;
+			}
+			return -1;
+		}
+
+		public RexBuilder getRexBuilder() {
+			return rexBuilder;
+		}
+
+		public RexRangeRef getSubQueryExpr(SqlCall call) {
+			final SubQuery subQuery = getSubQuery(call);
+			assert subQuery != null;
+			return (RexRangeRef) subQuery.expr;
+		}
+
+		public RelDataTypeFactory getTypeFactory() {
+			return typeFactory;
+		}
+
+		public InitializerExpressionFactory getInitializerExpressionFactory() {
+			return initializerExpressionFactory;
+		}
+
+		public SqlValidator getValidator() {
+			return validator;
+		}
+
+		public RexNode convertLiteral(SqlLiteral literal) {
+			return exprConverter.convertLiteral(this, literal);
+		}
+
+		public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) {
+			return exprConverter.convertInterval(this, intervalQualifier);
+		}
+
+		public RexNode visit(SqlLiteral literal) {
+			return exprConverter.convertLiteral(this, literal);
+		}
+
+		public RexNode visit(SqlCall call) {
+			if (agg != null) {
+				final SqlOperator op = call.getOperator();
+				if (window == null
+					&& (op.isAggregator() || op.getKind() == SqlKind.FILTER)) {
+					return agg.lookupAggregates(call);
+				}
+			}
+			return exprConverter.convertCall(this,
+				new SqlCallBinding(validator, scope, call).permutedCall());
+		}
+
+		public RexNode visit(SqlNodeList nodeList) {
+			throw new UnsupportedOperationException();
+		}
+
+		public RexNode visit(SqlIdentifier id) {
+			return convertIdentifier(this, id);
+		}
+
+		public RexNode visit(SqlDataTypeSpec type) {
+			throw new UnsupportedOperationException();
+		}
+
+		public RexNode visit(SqlDynamicParam param) {
+			return convertDynamicParam(param);
+		}
+
+		public RexNode visit(SqlIntervalQualifier intervalQualifier) {
+			return convertInterval(intervalQualifier);
+		}
+
+		public List<SqlMonotonicity> getColumnMonotonicities() {
+			return columnMonotonicities;
+		}
+
+	}
+
+	/** Deferred lookup. */
+	private static class DeferredLookup {
+		Blackboard bb;
+		String originalRelName;
+
+		DeferredLookup(
+			Blackboard bb,
+			String originalRelName) {
+			this.bb = bb;
+			this.originalRelName = originalRelName;
+		}
+
+		public RexFieldAccess getFieldAccess(CorrelationId name) {
+			return (RexFieldAccess) bb.mapCorrelateToRex.get(name);
+		}
+
+		public String getOriginalRelName() {
+			return originalRelName;
+		}
+	}
+
+	/**
+	 * A default implementation of SubQueryConverter that does no conversion.
+	 */
+	private class NoOpSubQueryConverter implements SubQueryConverter {
+		public boolean canConvertSubQuery() {
+			return false;
+		}
+
+		public RexNode convertSubQuery(
+			SqlCall subQuery,
+			SqlToRelConverter parentConverter,
+			boolean isExists,
+			boolean isExplain) {
+			throw new IllegalArgumentException();
+		}
+	}
+
+	/**
+	 * Converts expressions to aggregates.
+	 *
+	 * <p>Consider the expression
+	 *
+	 * <blockquote>
+	 * {@code SELECT deptno, SUM(2 * sal) FROM emp GROUP BY deptno}
+	 * </blockquote>
+	 *
+	 * <p>Then:
+	 *
+	 * <ul>
+	 * <li>groupExprs = {SqlIdentifier(deptno)}</li>
+	 * <li>convertedInputExprs = {RexInputRef(deptno), 2 *
+	 * RefInputRef(sal)}</li>
+	 * <li>inputRefs = {RefInputRef(#0), RexInputRef(#1)}</li>
+	 * <li>aggCalls = {AggCall(SUM, {1})}</li>
+	 * </ul>
+	 */
+	protected class AggConverter implements SqlVisitor<Void> {
+		private final Blackboard bb;
+		public final AggregatingSelectScope aggregatingSelectScope;
+
+		private final Map<String, String> nameMap = Maps.newHashMap();
+
+		/**
+		 * The group-by expressions, in {@link SqlNode} format.
+		 */
+		private final SqlNodeList groupExprs =
+			new SqlNodeList(SqlParserPos.ZERO);
+
+		/**
+		 * The auxiliary group-by expressions.
+		 */
+		private final Map<SqlNode, Ord<AuxiliaryConverter>> auxiliaryGroupExprs =
+			new HashMap<>();
+
+		/**
+		 * Input expressions for the group columns and aggregates, in
+		 * {@link RexNode} format. The first elements of the list correspond to the
+		 * elements in {@link #groupExprs}; the remaining elements are for
+		 * aggregates. The right field of each pair is the name of the expression,
+		 * where the expressions are simple mappings to input fields.
+		 */
+		private final List<Pair<RexNode, String>> convertedInputExprs =
+			new ArrayList<>();
+
+		/** Expressions to be evaluated as rows are being placed into the
+		 * aggregate's hash table. This is when group functions such as TUMBLE
+		 * cause rows to be expanded. */
+		private final List<RexNode> midExprs = new ArrayList<>();
+
+		private final List<AggregateCall> aggCalls = new ArrayList<>();
+		private final Map<SqlNode, RexNode> aggMapping = new HashMap<>();
+		private final Map<AggregateCall, RexNode> aggCallMapping =
+			new HashMap<>();
+
+		/** Are we directly inside a windowed aggregate? */
+		private boolean inOver = false;
+
+		/**
+		 * Creates an AggConverter.
+		 *
+		 * <p>The <code>select</code> parameter provides enough context to name
+		 * aggregate calls which are top-level select list items.
+		 *
+		 * @param bb     Blackboard
+		 * @param select Query being translated; provides context to give
+		 */
+		public AggConverter(Blackboard bb, SqlSelect select) {
+			this.bb = bb;
+			this.aggregatingSelectScope =
+				(AggregatingSelectScope) bb.getValidator().getSelectScope(select);
+
+			// Collect all expressions used in the select list so that aggregate
+			// calls can be named correctly.
+			final SqlNodeList selectList = select.getSelectList();
+			for (int i = 0; i < selectList.size(); i++) {
+				SqlNode selectItem = selectList.get(i);
+				String name = null;
+				if (SqlUtil.isCallTo(
+					selectItem,
+					SqlStdOperatorTable.AS)) {
+					final SqlCall call = (SqlCall) selectItem;
+					selectItem = call.operand(0);
+					name = call.operand(1).toString();
+				}
+				if (name == null) {
+					name = validator.deriveAlias(selectItem, i);
+				}
+				nameMap.put(selectItem.toString(), name);
+			}
+		}
+
+		public int addGroupExpr(SqlNode expr) {
+			int ref = lookupGroupExpr(expr);
+			if (ref >= 0) {
+				return ref;
+			}
+			final int index = groupExprs.size();
+			groupExprs.add(expr);
+			String name = nameMap.get(expr.toString());
+			RexNode convExpr = bb.convertExpression(expr);
+			addExpr(convExpr, name);
+
+			if (expr instanceof SqlCall) {
+				SqlCall call = (SqlCall) expr;
+				for (Pair<SqlNode, AuxiliaryConverter> p
+					: SqlStdOperatorTable.convertGroupToAuxiliaryCalls(call)) {
+					addAuxiliaryGroupExpr(p.left, index, p.right);
+				}
+			}
+
+			return index;
+		}
+
+		void addAuxiliaryGroupExpr(SqlNode node, int index,
+			AuxiliaryConverter converter) {
+			for (SqlNode node2 : auxiliaryGroupExprs.keySet()) {
+				if (node2.equalsDeep(node, Litmus.IGNORE)) {
+					return;
+				}
+			}
+			auxiliaryGroupExprs.put(node, Ord.of(index, converter));
+		}
+
+		/**
+		 * Adds an expression, deducing an appropriate name if possible.
+		 *
+		 * @param expr Expression
+		 * @param name Suggested name
+		 */
+		private void addExpr(RexNode expr, String name) {
+			if ((name == null) && (expr instanceof RexInputRef)) {
+				final int i = ((RexInputRef) expr).getIndex();
+				name = bb.root.getRowType().getFieldList().get(i).getName();
+			}
+			if (Pair.right(convertedInputExprs).contains(name)) {
+				// In case like 'SELECT ... GROUP BY x, y, x', don't add
+				// name 'x' twice.
+				name = null;
+			}
+			convertedInputExprs.add(Pair.of(expr, name));
+		}
+
+		public Void visit(SqlIdentifier id) {
+			return null;
+		}
+
+		public Void visit(SqlNodeList nodeList) {
+			for (int i = 0; i < nodeList.size(); i++) {
+				nodeList.get(i).accept(this);
+			}
+			return null;
+		}
+
+		public Void visit(SqlLiteral lit) {
+			return null;
+		}
+
+		public Void visit(SqlDataTypeSpec type) {
+			return null;
+		}
+
+		public Void visit(SqlDynamicParam param) {
+			return null;
+		}
+
+		public Void visit(SqlIntervalQualifier intervalQualifier) {
+			return null;
+		}
+
+		public Void visit(SqlCall call) {
+			switch (call.getKind()) {
+				case FILTER:
+					translateAgg((SqlCall) call.operand(0), call.operand(1), call);
+					return null;
+				case SELECT:
+					// rchen 2006-10-17:
+					// for now do not detect aggregates in sub-queries.
+					return null;
+			}
+			final boolean prevInOver = inOver;
+			// Ignore window aggregates and ranking functions (associated with OVER
+			// operator). However, do not ignore nested window aggregates.
+			if (call.getOperator().getKind() == SqlKind.OVER) {
+				// Track aggregate nesting levels only within an OVER operator.
+				inOver = true;
+			}
+
+			// Do not translate the top level window aggregate. Only do so for
+			// nested aggregates, if present
+			if (call.getOperator().isAggregator()) {
+				if (inOver) {
+					// Add the parent aggregate level before visiting its children
+					inOver = false;
+				} else {
+					// We're beyond the one ignored level
+					translateAgg(call, null, call);
+					return null;
+				}
+			}
+			for (SqlNode operand : call.getOperandList()) {
+				// Operands are occasionally null, e.g. switched CASE arg 0.
+				if (operand != null) {
+					operand.accept(this);
+				}
+			}
+			// Remove the parent aggregate level after visiting its children
+			inOver = prevInOver;
+			return null;
+		}
+
+		private void translateAgg(SqlCall call, SqlNode filter, SqlCall outerCall) {
+			assert bb.agg == this;
+			final List<Integer> args = new ArrayList<>();
+			int filterArg = -1;
+			final List<RelDataType> argTypes =
+				call.getOperator() instanceof SqlCountAggFunction
+					? new ArrayList<RelDataType>(call.getOperandList().size())
+					: null;
+			try {
+				// switch out of agg mode
+				bb.agg = null;
+				for (SqlNode operand : call.getOperandList()) {
+
+					// special case for COUNT(*):  delete the *
+					if (operand instanceof SqlIdentifier) {
+						SqlIdentifier id = (SqlIdentifier) operand;
+						if (id.isStar()) {
+							assert call.operandCount() == 1;
+							assert args.isEmpty();
+							break;
+						}
+					}
+					RexNode convertedExpr = bb.convertExpression(operand);
+					assert convertedExpr != null;
+					if (argTypes != null) {
+						argTypes.add(convertedExpr.getType());
+					}
+					args.add(lookupOrCreateGroupExpr(convertedExpr));
+				}
+
+				if (filter != null) {
+					RexNode convertedExpr = bb.convertExpression(filter);
+					assert convertedExpr != null;
+					if (convertedExpr.getType().isNullable()) {
+						convertedExpr =
+							rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, convertedExpr);
+					}
+					filterArg = lookupOrCreateGroupExpr(convertedExpr);
+				}
+			} finally {
+				// switch back into agg mode
+				bb.agg = this;
+			}
+
+			final SqlAggFunction aggFunction =
+				(SqlAggFunction) call.getOperator();
+			RelDataType type = validator.deriveType(bb.scope, call);
+			boolean distinct = false;
+			SqlLiteral quantifier = call.getFunctionQuantifier();
+			if ((null != quantifier)
+				&& (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) {
+				distinct = true;
+			}
+			final AggregateCall aggCall =
+				AggregateCall.create(
+					aggFunction,
+					distinct,
+					args,
+					filterArg,
+					type,
+					nameMap.get(outerCall.toString()));
+			final AggregatingSelectScope.Resolved r =
+				aggregatingSelectScope.resolved.get();
+			RexNode rex =
+				rexBuilder.addAggCall(
+					aggCall,
+					groupExprs.size(),
+					r.indicator,
+					aggCalls,
+					aggCallMapping,
+					argTypes);
+			aggMapping.put(outerCall, rex);
+		}
+
+		private int lookupOrCreateGroupExpr(RexNode expr) {
+			int index = 0;
+			for (RexNode convertedInputExpr : Pair.left(convertedInputExprs)) {
+				if (expr.toString().equals(convertedInputExpr.toString())) {
+					return index;
+				}
+				++index;
+			}
+
+			// not found -- add it
+			addExpr(expr, null);
+			return index;
+		}
+
+		/**
+		 * If an expression is structurally identical to one of the group-by
+		 * expressions, returns a reference to the expression, otherwise returns
+		 * null.
+		 */
+		public int lookupGroupExpr(SqlNode expr) {
+			for (int i = 0; i < groupExprs.size(); i++) {
+				SqlNode groupExpr = groupExprs.get(i);
+				if (expr.equalsDeep(groupExpr, Litmus.IGNORE)) {
+					return i;
+				}
+			}
+			return -1;
+		}
+
+		public RexNode lookupAggregates(SqlCall call) {
+			// assert call.getOperator().isAggregator();
+			assert bb.agg == this;
+
+			switch (call.getKind()) {
+				case GROUPING:
+				case GROUP_ID:
+					final RelDataType type = validator.getValidatedNodeType(call);
+					if (!aggregatingSelectScope.resolved.get().indicator) {
+						return rexBuilder.makeExactLiteral(
+							TWO.pow(effectiveArgCount(call)).subtract(BigDecimal.ONE), type);
+					} else {
+						final List<Integer> operands;
+						switch (call.getKind()) {
+							case GROUP_ID:
+								operands = ImmutableIntList.range(0, groupExprs.size());
+								break;
+							default:
+								operands = Lists.newArrayList();
+								for (SqlNode operand : call.getOperandList()) {
+									final int x = lookupGroupExpr(operand);
+									assert x >= 0;
+									operands.add(x);
+								}
+						}
+						RexNode node = null;
+						int shift = operands.size();
+						for (int operand : operands) {
+							node = bitValue(node, type, operand, --shift);
+						}
+						return node;
+					}
+			}
+
+			for (Map.Entry<SqlNode, Ord<AuxiliaryConverter>> e
+				: auxiliaryGroupExprs.entrySet()) {
+				if (call.equalsDeep(e.getKey(), Litmus.IGNORE)) {
+					AuxiliaryConverter converter = e.getValue().e;
+					final int groupOrdinal = e.getValue().i;
+					return converter.convert(rexBuilder,
+						convertedInputExprs.get(groupOrdinal).left,
+						rexBuilder.makeInputRef(bb.root, groupOrdinal));
+				}
+			}
+
+			return aggMapping.get(call);
+		}
+
+		private int effectiveArgCount(SqlCall call) {
+			switch (call.getKind()) {
+				case GROUPING:
+					return call.operandCount();
+				case GROUP_ID:
+					return groupExprs.size();
+				default:
+					throw new AssertionError(call.getKind());
+			}
+		}
+
+		private RexNode bitValue(RexNode previous, RelDataType type, int x,
+			int shift) {
+			final AggregatingSelectScope.Resolved r =
+				aggregatingSelectScope.resolved.get();
+			RexNode node = rexBuilder.makeCall(SqlStdOperatorTable.CASE,
+				rexBuilder.makeInputRef(bb.root, r.groupExprList.size() + x),
+				rexBuilder.makeExactLiteral(BigDecimal.ONE, type),
+				rexBuilder.makeExactLiteral(BigDecimal.ZERO, type));
+			if (shift > 0) {
+				node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, node,
+					rexBuilder.makeExactLiteral(TWO.pow(shift), type));
+			}
+			if (previous != null) {
+				node = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, previous, node);
+			}
+			return node;
+		}
+
+		public List<Pair<RexNode, String>> getPreExprs() {
+			return convertedInputExprs;
+		}
+
+		public List<AggregateCall> getAggCalls() {
+			return aggCalls;
+		}
+
+		public RelDataTypeFactory getTypeFactory() {
+			return typeFactory;
+		}
+	}
+
+	/**
+	 * Context to find a relational expression to a field offset.
+	 */
+	private static class LookupContext {
+		private final List<Pair<RelNode, Integer>> relOffsetList =
+			new ArrayList<>();
+
+		/**
+		 * Creates a LookupContext with multiple input relational expressions.
+		 *
+		 * @param bb               Context for translating this sub-query
+		 * @param rels             Relational expressions
+		 * @param systemFieldCount Number of system fields
+		 */
+		LookupContext(Blackboard bb, List<RelNode> rels, int systemFieldCount) {
+			bb.flatten(rels, systemFieldCount, new int[]{0}, relOffsetList);
+		}
+
+		/**
+		 * Returns the relational expression with a given offset, and the
+		 * ordinal in the combined row of its first field.
+		 *
+		 * <p>For example, in {@code Emp JOIN Dept}, findRel(1) returns the
+		 * relational expression for {@code Dept} and offset 6 (because
+		 * {@code Emp} has 6 fields, therefore the first field of {@code Dept}
+		 * is field 6.
+		 *
+		 * @param offset Offset of relational expression in FROM clause
+		 * @return Relational expression and the ordinal of its first field
+		 */
+		Pair<RelNode, Integer> findRel(int offset) {
+			return relOffsetList.get(offset);
+		}
+	}
+
+	/**
+	 * Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to
+	 * all agg functions.
+	 *
+	 * <p>This is necessary because the returned expression is not necessarily a
+	 * call to an agg function. For example,
+	 *
+	 * <blockquote><code>AVG(x)</code></blockquote>
+	 *
+	 * becomes
+	 *
+	 * <blockquote><code>SUM(x) / COUNT(x)</code></blockquote>
+	 *
+	 * <p>Any aggregate functions are converted to calls to the internal <code>
+	 * $Histogram</code> aggregation function and accessors such as <code>
+	 * $HistogramMin</code>; for example,
+	 *
+	 * <blockquote><code>MIN(x), MAX(x)</code></blockquote>
+	 *
+	 * are converted to
+	 *
+	 * <blockquote><code>$HistogramMin($Histogram(x)),
+	 * $HistogramMax($Histogram(x))</code></blockquote>
+	 *
+	 * Common sub-expression elmination will ensure that only one histogram is
+	 * computed.
+	 */
+	private class HistogramShuttle extends RexShuttle {
+		/**
+		 * Whether to convert calls to MIN(x) to HISTOGRAM_MIN(HISTOGRAM(x)).
+		 * Histograms allow rolling computation, but require more space.
+		 */
+		static final boolean ENABLE_HISTOGRAM_AGG = false;
+
+		private final List<RexNode> partitionKeys;
+		private final ImmutableList<RexFieldCollation> orderKeys;
+		private final RexWindowBound lowerBound;
+		private final RexWindowBound upperBound;
+		private final SqlWindow window;
+
+		HistogramShuttle(
+			List<RexNode> partitionKeys,
+			ImmutableList<RexFieldCollation> orderKeys,
+			RexWindowBound lowerBound, RexWindowBound upperBound,
+			SqlWindow window) {
+			this.partitionKeys = partitionKeys;
+			this.orderKeys = orderKeys;
+			this.lowerBound = lowerBound;
+			this.upperBound = upperBound;
+			this.window = window;
+		}
+
+		public RexNode visitCall(RexCall call) {
+			final SqlOperator op = call.getOperator();
+			if (!(op instanceof SqlAggFunction)) {
+				return super.visitCall(call);
+			}
+			final SqlAggFunction aggOp = (SqlAggFunction) op;
+			final RelDataType type = call.getType();
+			List<RexNode> exprs = call.getOperands();
+
+			SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG
+				? null
+				: getHistogramOp(aggOp);
+
+			if (histogramOp != null) {
+				final RelDataType histogramType = computeHistogramType(type);
+
+				// For DECIMAL, since it's already represented as a bigint we
+				// want to do a reinterpretCast instead of a cast to avoid
+				// losing any precision.
+				boolean reinterpretCast =
+					type.getSqlTypeName() == SqlTypeName.DECIMAL;
+
+				// Replace original expression with CAST of not one
+				// of the supported types
+				if (histogramType != type) {
+					exprs = new ArrayList<>(exprs);
+					exprs.set(
+						0,
+						reinterpretCast
+							? rexBuilder.makeReinterpretCast(histogramType, exprs.get(0),
+							rexBuilder.makeLiteral(false))
+							: rexBuilder.makeCast(histogramType, exprs.get(0)));
+				}
+
+				RexCallBinding bind =
+					new RexCallBinding(
+						rexBuilder.getTypeFactory(),
+						SqlStdOperatorTable.HISTOGRAM_AGG,
+						exprs,
+						ImmutableList.<RelCollation>of());
+
+				RexNode over =
+					rexBuilder.makeOver(
+						SqlStdOperatorTable.HISTOGRAM_AGG
+							.inferReturnType(bind),
+						SqlStdOperatorTable.HISTOGRAM_AGG,
+						exprs,
+						partitionKeys,
+						orderKeys,
+						lowerBound,
+						upperBound,
+						window.isRows(),
+						window.isAllowPartial(),
+						false);
+
+				RexNode histogramCall =
+					rexBuilder.makeCall(
+						histogramType,
+						histogramOp,
+						ImmutableList.of(over));
+
+				// If needed, post Cast result back to original
+				// type.
+				if (histogramType != type) {
+					if (reinterpretCast) {
+						histogramCall =
+							rexBuilder.makeReinterpretCast(
+								type,
+								histogramCall,
+								rexBuilder.makeLiteral(false));
+					} else {
+						histogramCall =
+							rexBuilder.makeCast(type, histogramCall);
+					}
+				}
+
+				return histogramCall;
+			} else {
+				boolean needSum0 = aggOp == SqlStdOperatorTable.SUM
+					&& type.isNullable();
+				SqlAggFunction aggOpToUse =
+					needSum0 ? SqlStdOperatorTable.SUM0
+						: aggOp;
+				return rexBuilder.makeOver(
+					type,
+					aggOpToUse,
+					exprs,
+					partitionKeys,
+					orderKeys,
+					lowerBound,
+					upperBound,
+					window.isRows(),
+					window.isAllowPartial(),
+					needSum0);
+			}
+		}
+
+		/**
+		 * Returns the histogram operator corresponding to a given aggregate
+		 * function.
+		 *
+		 * <p>For example, <code>getHistogramOp
+		 *({@link SqlStdOperatorTable#MIN}}</code> returns
+		 * {@link SqlStdOperatorTable#HISTOGRAM_MIN}.
+		 *
+		 * @param aggFunction An aggregate function
+		 * @return Its histogram function, or null
+		 */
+		SqlFunction getHistogramOp(SqlAggFunction aggFunction) {
+			if (aggFunction == SqlStdOperatorTable.MIN) {
+				return SqlStdOperatorTable.HISTOGRAM_MIN;
+			} else if (aggFunction == SqlStdOperatorTable.MAX) {
+				return SqlStdOperatorTable.HISTOGRAM_MAX;
+			} else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) {
+				return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE;
+			} else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) {
+				return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE;
+			} else {
+				return null;
+			}
+		}
+
+		/**
+		 * Returns the type for a histogram function. It is either the actual
+		 * type or an an approximation to it.
+		 */
+		private RelDataType computeHistogramType(RelDataType type) {
+			if (SqlTypeUtil.isExactNumeric(type)
+				&& type.getSqlTypeName() != SqlTypeName.BIGINT) {
+				return typeFactory.createSqlType(SqlTypeName.BIGINT);
+			} else if (SqlTypeUtil.isApproximateNumeric(type)
+				&& type.getSqlTypeName() != SqlTypeName.DOUBLE) {
+				return typeFactory.createSqlType(SqlTypeName.DOUBLE);
+			} else {
+				return type;
+			}
+		}
+	}
+
+	/** A sub-query, whether it needs to be translated using 2- or 3-valued
+	 * logic. */
+	private static class SubQuery {
+		final SqlNode node;
+		final RelOptUtil.Logic logic;
+		RexNode expr;
+
+		private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
+			this.node = node;
+			this.logic = logic;
+		}
+	}
+
+	/**
+	 * Visitor that collects all aggregate functions in a {@link SqlNode} tree.
+	 */
+	private static class AggregateFinder extends SqlBasicVisitor<Void> {
+		final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO);
+
+		@Override public Void visit(SqlCall call) {
+			// ignore window aggregates and ranking functions (associated with OVER operator)
+			if (call.getOperator().getKind() == SqlKind.OVER) {
+				return null;
+			}
+			if (call.getOperator().isAggregator()) {
+				list.add(call);
+				return null;
+			}
+
+			// Don't traverse into sub-queries, even if they contain aggregate
+			// functions.
+			if (call instanceof SqlSelect) {
+				return null;
+			}
+
+			return call.getOperator().acceptCall(this, call);
+		}
+	}
+
+	/** Use of a row as a correlating variable by a given relational
+	 * expression. */
+	private static class CorrelationUse {
+		private final CorrelationId id;
+		private final ImmutableBitSet requiredColumns;
+		private final RelNode r;
+
+		CorrelationUse(CorrelationId id, ImmutableBitSet requiredColumns,
+			RelNode r) {
+			this.id = id;
+			this.requiredColumns = requiredColumns;
+			this.r = r;
+		}
+	}
+
+	/** Creates a builder for a {@link Config}. */
+	public static ConfigBuilder configBuilder() {
+		return new ConfigBuilder();
+	}
+
+	/**
+	 * Interface to define the configuration for a SqlToRelConverter.
+	 * Provides methods to set each configuration option.
+	 *
+	 * @see ConfigBuilder
+	 * @see SqlToRelConverter#configBuilder()
+	 */
+	public interface Config {
+		/** Default configuration. */
+		Config DEFAULT = configBuilder().build();
+
+		/** Returns the {@code convertTableAccess} option. Controls whether table
+		 * access references are converted to physical rels immediately. The
+		 * optimizer doesn't like leaf rels to have {@link Convention#NONE}.
+		 * However, if we are doing further conversion passes (e.g.
+		 * {@link RelStructuredTypeFlattener}), then we may need to defer
+		 * conversion. */
+		boolean isConvertTableAccess();
+
+		/** Returns the {@code decorrelationEnabled} option. Controls whether to
+		 * disable sub-query decorrelation when needed. e.g. if outer joins are not
+		 * supported. */
+		boolean isDecorrelationEnabled();
+
+		/** Returns the {@code trimUnusedFields} option. Controls whether to trim
+		 * unused fields as part of the conversion process. */
+		boolean isTrimUnusedFields();
+
+		/** Returns the {@code createValuesRel} option. Controls whether instances
+		 * of {@link org.apache.calcite.rel.logical.LogicalValues} are generated.
+		 * These may not be supported by all physical implementations. */
+		boolean isCreateValuesRel();
+
+		/** Returns the {@code explain} option. Describes whether the current
+		 * statement is part of an EXPLAIN PLAN statement. */
+		boolean isExplain();
+
+		/** Returns the {@code expand} option. Controls whether to expand
+		 * sub-queries. If false, each sub-query becomes a
+		 * {@link org.apache.calcite.rex.RexSubQuery}. */
+		boolean isExpand();
+
+		/** Returns the {@code inSubQueryThreshold} option,
+		 * default {@link #DEFAULT_IN_SUB_QUERY_THRESHOLD}. Controls the list size
+		 * threshold under which {@link #convertInToOr} is used. Lists of this size
+		 * or greater will instead be converted to use a join against an inline
+		 * table ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than
+		 * a predicate. A threshold of 0 forces usage of an inline table in all
+		 * cases; a threshold of {@link Integer#MAX_VALUE} forces usage of OR in all
+		 * cases. */
+		int getInSubQueryThreshold();
+	}
+
+	/** Builder for a {@link Config}. */
+	public static class ConfigBuilder {
+		private boolean convertTableAccess = true;
+		private boolean decorrelationEnabled = true;
+		private boolean trimUnusedFields = false;
+		private boolean createValuesRel = true;
+		private boolean explain;
+		private boolean expand = true;
+		private int inSubQueryThreshold = DEFAULT_IN_SUB_QUERY_THRESHOLD;
+
+		private ConfigBuilder() {}
+
+		/** Sets configuration identical to a given {@link Config}. */
+		public ConfigBuilder withConfig(Config config) {
+			this.convertTableAccess = config.isConvertTableAccess();
+			this.decorrelationEnabled = config.isDecorrelationEnabled();
+			this.trimUnusedFields = config.isTrimUnusedFields();
+			this.createValuesRel = config.isCreateValuesRel();
+			this.explain = config.isExplain();
+			this.expand = config.isExpand();
+			this.inSubQueryThreshold = config.getInSubQueryThreshold();
+			return this;
+		}
+
+		public ConfigBuilder withConvertTableAccess(boolean convertTableAccess) {
+			this.convertTableAccess = convertTableAccess;
+			return this;
+		}
+
+		public ConfigBuilder withDecorrelationEnabled(boolean enabled) {
+			this.decorrelationEnabled = enabled;
+			return this;
+		}
+
+		public ConfigBuilder withTrimUnusedFields(boolean trimUnusedFields) {
+			this.trimUnusedFields = trimUnusedFields;
+			return this;
+		}
+
+		public ConfigBuilder withCreateValuesRel(boolean createValuesRel) {
+			this.createValuesRel = createValuesRel;
+			return this;
+		}
+
+		public ConfigBuilder withExplain(boolean explain) {
+			this.explain = explain;
+			return this;
+		}
+
+		public ConfigBuilder withExpand(boolean expand) {
+			this.expand = expand;
+			return this;
+		}
+
+		@Deprecated // to be removed before 2.0
+		public ConfigBuilder withInSubqueryThreshold(int inSubQueryThreshold) {
+			return withInSubQueryThreshold(inSubQueryThreshold);
+		}
+
+		public ConfigBuilder withInSubQueryThreshold(int inSubQueryThreshold) {
+			this.inSubQueryThreshold = inSubQueryThreshold;
+			return this;
+		}
+
+		/** Builds a {@link Config}. */
+		public Config build() {
+			return new ConfigImpl(convertTableAccess, decorrelationEnabled,
+				trimUnusedFields, createValuesRel, explain, expand,
+				inSubQueryThreshold);
+		}
+	}
+
+	/** Implementation of {@link Config}.
+	 * Called by builder; all values are in private final fields. */
+	private static class ConfigImpl implements Config {
+		private final boolean convertTableAccess;
+		private final boolean decorrelationEnabled;
+		private final boolean trimUnusedFields;
+		private final boolean createValuesRel;
+		private final boolean explain;
+		private final int inSubQueryThreshold;
+		private final boolean expand;
+
+		private ConfigImpl(boolean convertTableAccess, boolean decorrelationEnabled,
+			boolean trimUnusedFields, boolean createValuesRel, boolean explain,
+			boolean expand, int inSubQueryThreshold) {
+			this.convertTableAccess = convertTableAccess;
+			this.decorrelationEnabled = decorrelationEnabled;
+			this.trimUnusedFields = trimUnusedFields;
+			this.createValuesRel = createValuesRel;
+			this.explain = explain;
+			this.expand = expand;
+			this.inSubQueryThreshold = inSubQueryThreshold;
+		}
+
+		public boolean isConvertTableAccess() {
+			return convertTableAccess;
+		}
+
+		public boolean isDecorrelationEnabled() {
+			return decorrelationEnabled;
+		}
+
+		public boolean isTrimUnusedFields() {
+			return trimUnusedFields;
+		}
+
+		public boolean isCreateValuesRel() {
+			return createValuesRel;
+		}
+
+		public boolean isExplain() {
+			return explain;
+		}
+
+		public boolean isExpand() {
+			return expand;
+		}
+
+		public int getInSubQueryThreshold() {
+			return inSubQueryThreshold;
+		}
+	}
+}
+
+// End SqlToRelConverter.java
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
index 9d165365bac9b..4f46a73f90a3d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
@@ -199,6 +199,37 @@ class WindowAggregateTest extends TableTestBase {
     util.verifySql(sqlQuery, expected)
   }
 
+  @Test
+  def testWindowEndOnly(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Int, Long, String, Timestamp)]("T", 'a, 'b, 'c, 'ts)
+
+    val sqlQuery =
+      "SELECT " +
+        "  TUMBLE_END(ts, INTERVAL '4' MINUTE)" +
+        "FROM T " +
+        "GROUP BY TUMBLE(ts, INTERVAL '4' MINUTE), c"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        unaryNode(
+          "DataSetWindowAggregate",
+          unaryNode(
+            "DataSetCalc",
+            batchTableNode(0),
+            term("select", "ts, c")
+          ),
+          term("groupBy", "c"),
+          term("window", EventTimeTumblingGroupWindow('w$, 'ts, 240000.millis)),
+          term("select", "c, start('w$) AS w$start, end('w$) AS w$end")
+        ),
+        term("select", "CAST(w$end) AS w$end")
+      )
+
+    util.verifySql(sqlQuery, expected)
+  }
+
   @Test(expected = classOf[TableException])
   def testTumbleWindowNoOffset(): Unit = {
     val util = batchTestUtil()
diff --git a/tools/maven/suppressions.xml b/tools/maven/suppressions.xml
index 2c290541d6a72..8a80341289dca 100644
--- a/tools/maven/suppressions.xml
+++ b/tools/maven/suppressions.xml
@@ -25,4 +25,6 @@ under the License.
 <suppressions>
 		<suppress files="org[\\/]apache[\\/]flink[\\/]api[\\/]io[\\/]avro[\\/]example[\\/]User.java" checks="[a-zA-Z0-9]*"/>
 		<suppress files="org[\\/]apache[\\/]flink[\\/]api[\\/]io[\\/]avro[\\/]generated[\\/].*.java" checks="[a-zA-Z0-9]*"/>
+		<!-- Sometimes we have to temporarily fix very long, different formatted Calcite files. -->
+		<suppress files="org[\\/]apache[\\/]calcite.*" checks="[a-zA-Z0-9]*"/>
 </suppressions>

From edd5a952435081680d1849da6cced14e4697e91b Mon Sep 17 00:00:00 2001
From: Ufuk Celebi <uce@apache.org>
Date: Tue, 2 May 2017 15:43:32 +0200
Subject: [PATCH 104/253] [FLINK-6337] [network] Fix instability in
 SuccessAfterNetworkBuffersFailureITCase

The reduced number of network buffers as part of this issue was too low and
lead to instable tests.
---
 .../test/misc/SuccessAfterNetworkBuffersFailureITCase.java     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
index 710e7df4dd5bc..4f24452305dc6 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
@@ -42,7 +42,6 @@
 
 public class SuccessAfterNetworkBuffersFailureITCase extends TestLogger {
 	
-	
 	@Test
 	public void testSuccessfulProgramAfterFailure() {
 		LocalFlinkMiniCluster cluster = null;
@@ -52,7 +51,7 @@ public void testSuccessfulProgramAfterFailure() {
 			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
 			config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 80L);
 			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
-			config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 640);
+			config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 800);
 			
 			cluster = new LocalFlinkMiniCluster(config, false);
 

From ec7a46ed5beb0749f9f01cacbbf05911771eb6f6 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 2 May 2017 22:48:47 +0200
Subject: [PATCH 105/253] [hotfix] [tests] Fix test log level in flink-runtime

---
 flink-runtime/src/test/resources/log4j-test.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-runtime/src/test/resources/log4j-test.properties b/flink-runtime/src/test/resources/log4j-test.properties
index 812a256d5ca2a..7ba163366a631 100644
--- a/flink-runtime/src/test/resources/log4j-test.properties
+++ b/flink-runtime/src/test/resources/log4j-test.properties
@@ -16,7 +16,7 @@
 # limitations under the License.
 ################################################################################
 
-log4j.rootLogger=DEBUG, console
+log4j.rootLogger=OFF, console
 
 # -----------------------------------------------------------------------------
 # Console (use 'console')

From a1877c5651f7a586c824648fb33f384b7cd95652 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 28 Apr 2017 16:52:03 +0200
Subject: [PATCH 106/253] [FLINK-6415] [build] Make sure core Flink artifacts
 have no specific logger dependency

---
 flink-core/pom.xml                            | 20 -------------------
 flink-dist/pom.xml                            |  6 +++---
 .../resources/archetype-resources/pom.xml     |  2 +-
 .../resources/archetype-resources/pom.xml     |  2 +-
 .../flink-test-utils/pom.xml                  | 10 ++++++++++
 pom.xml                                       | 10 ++++++++++
 6 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index 0a0d06e086af1..1efcb290ddf22 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -95,26 +95,6 @@ under the License.
 			<version>${asm.version}</version>
 		</dependency>
 
-		<!--
-			Because there are no logger implementation dependency in the root pom, we
-			add them here so that they are available during execution of code (core 
-			and example) in the IDE
-
-			NOTE: Once we are confident that users will use the newer quickstart templates,
-			we can drop these dependencies and only add them to 'flink-dist' and as test
-			dependencies
-		-->
-
-		<dependency>
-			<groupId>org.slf4j</groupId>
-			<artifactId>slf4j-log4j12</artifactId>
-		</dependency>
-
-		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-		</dependency>
-
 		<!-- ================== test dependencies ================== -->
 
 		<dependency>
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index a700870624d41..507e78d31716e 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -139,9 +139,9 @@ under the License.
 			<version>${project.version}</version>
 		</dependency>
 
-		<!-- Concrete logging framework - we only add this here to not tie
-			the projects to one specific framework and make it easier for
-			users to swap logging frameworks -->
+		<!-- Concrete logging framework - we add this only here (and not in the 
+			root POM to not tie the projects to one specific framework and make
+			it easier for users to swap logging frameworks -->
 
 		<dependency>
 			<groupId>org.slf4j</groupId>
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 6ae8f166c1d1d..1e525fd8ca61d 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -91,7 +91,7 @@ under the License.
 			<version>${flink.version}</version>
 		</dependency>
 
-		<!-- explicitly add a standard loggin framework, as Flink does not (in the future) have
+		<!-- explicitly add a standard loggin framework, as Flink does not have
 			a hard dependency on one specific framework by default -->
 		<dependency>
 			<groupId>org.slf4j</groupId>
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 699a739189187..5abe496b54b73 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -92,7 +92,7 @@ under the License.
 			<version>${flink.version}</version>
 		</dependency>
 		
-		<!-- explicitly add a standard loggin framework, as Flink does not (in the future) have
+		<!-- explicitly add a standard loggin framework, as Flink does not have
 			a hard dependency on one specific framework by default -->
 		<dependency>
 			<groupId>org.slf4j</groupId>
diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml
index 069f1da60235b..14fca07da43b0 100644
--- a/flink-test-utils-parent/flink-test-utils/pom.xml
+++ b/flink-test-utils-parent/flink-test-utils/pom.xml
@@ -90,6 +90,16 @@ under the License.
 			on its own as well as include maven-bundle-plugin.
 			-->
 			<optional>true</optional>
+			<exclusions>
+				<exclusion>
+					<groupId>log4j</groupId>
+					<artifactId>log4j</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-log4j12</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 
 	</dependencies>
diff --git a/pom.xml b/pom.xml
index 874bead4d93fa..f822198b903a2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -414,6 +414,16 @@ under the License.
 				<groupId>org.apache.zookeeper</groupId>
 				<artifactId>zookeeper</artifactId>
 				<version>${zookeeper.version}</version>
+				<exclusions>
+					<exclusion>
+						<groupId>log4j</groupId>
+						<artifactId>log4j</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.slf4j</groupId>
+						<artifactId>slf4j-log4j12</artifactId>
+					</exclusion>
+				</exclusions>
 			</dependency>
 
 			<dependency>

From df0646bf93ff462e8b8913ed8f6e17e6f699f422 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 28 Apr 2017 17:04:10 +0200
Subject: [PATCH 107/253] [hotfix] [build] Remove unneeded logback-test.xml
 files

---
 .../src/test/resources/logback-test.xml       | 29 -------------
 .../src/test/resources/logback-test.xml       | 29 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 29 -------------
 .../src/test/resources/logback-test.xml       | 29 -------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 34 ---------------
 .../src/test/resources/log4j.properties       | 31 --------------
 .../src/test/resources/logback-test.xml       | 35 ----------------
 .../src/test/resources/logback-test.xml       | 34 ---------------
 .../src/test/resources/logback-test.xml       | 37 ----------------
 .../src/test/resources/logback-test.xml       | 34 ---------------
 .../src/test/resources/logback-test.xml       | 34 ---------------
 .../src/test/resources/logback-test.xml       | 34 ---------------
 .../src/test/resources/logback-test.xml       | 29 -------------
 .../src/test/resources/logback-test.xml       | 42 -------------------
 .../src/test/resources/logback-test.xml       | 42 -------------------
 .../src/test/resources/logback-test.xml       | 29 -------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 30 -------------
 .../src/test/resources/logback-test.xml       | 34 ---------------
 .../src/test/resources/log4j.properties       | 27 ------------
 .../src/test/resources/logback-test.xml       | 37 ----------------
 .../src/test/resources/logback-test.xml       | 34 ---------------
 43 files changed, 1332 deletions(-)
 delete mode 100644 flink-clients/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-avro/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-elasticsearch-base/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-elasticsearch/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-elasticsearch2/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-elasticsearch5/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-filesystem/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-kafka-0.10/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-kafka-0.8/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-kafka-0.9/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-connector-kafka-base/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-hadoop-compatibility/src/test/resources/logback-test.xml
 delete mode 100644 flink-connectors/flink-jdbc/src/test/resources/logback-test.xml
 delete mode 100644 flink-contrib/flink-connector-wikiedits/src/test/resources/log4j.properties
 delete mode 100644 flink-contrib/flink-connector-wikiedits/src/test/resources/logback-test.xml
 delete mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties
 delete mode 100644 flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml
 delete mode 100644 flink-contrib/flink-storm-examples/src/test/resources/log4j.properties
 delete mode 100644 flink-contrib/flink-storm-examples/src/test/resources/logback-test.xml
 delete mode 100644 flink-contrib/flink-storm/src/test/resources/log4j.properties
 delete mode 100644 flink-contrib/flink-storm/src/test/resources/logback-test.xml
 delete mode 100644 flink-contrib/flink-streaming-contrib/src/test/resources/log4j.properties
 delete mode 100644 flink-contrib/flink-streaming-contrib/src/test/resources/logback-test.xml
 delete mode 100644 flink-contrib/flink-tweet-inputformat/src/test/resources/log4j.properties
 delete mode 100644 flink-contrib/flink-tweet-inputformat/src/test/resources/logback-test.xml
 delete mode 100644 flink-core/src/test/resources/logback-test.xml
 delete mode 100644 flink-fs-tests/src/test/resources/log4j.properties
 delete mode 100644 flink-java/src/test/resources/logback-test.xml
 delete mode 100644 flink-libraries/flink-cep/src/test/resources/logback-test.xml
 delete mode 100644 flink-mesos/src/test/resources/logback-test.xml
 delete mode 100644 flink-metrics/flink-metrics-dropwizard/src/test/resources/logback-test.xml
 delete mode 100644 flink-metrics/flink-metrics-jmx/src/test/resources/logback-test.xml
 delete mode 100644 flink-metrics/flink-metrics-statsd/src/test/resources/logback-test.xml
 delete mode 100644 flink-optimizer/src/test/resources/logback-test.xml
 delete mode 100644 flink-runtime-web/src/test/resources/logback-test.xml
 delete mode 100644 flink-runtime/src/test/resources/logback-test.xml
 delete mode 100644 flink-scala-shell/src/test/resources/logback-test.xml
 delete mode 100644 flink-streaming-java/src/test/resources/log4j.properties
 delete mode 100644 flink-streaming-java/src/test/resources/logback-test.xml
 delete mode 100644 flink-test-utils-parent/flink-test-utils-junit/src/test/resources/logback-test.xml
 delete mode 100644 flink-tests/src/test/resources/log4j.properties
 delete mode 100644 flink-tests/src/test/resources/logback-test.xml
 delete mode 100644 flink-yarn/src/test/resources/logback-test.xml

diff --git a/flink-clients/src/test/resources/logback-test.xml b/flink-clients/src/test/resources/logback-test.xml
deleted file mode 100644
index 045f07e0627af..0000000000000
--- a/flink-clients/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-		<encoder>
-			<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-		</encoder>
-	</appender>
-
-	<root level="WARN">
-		<appender-ref ref="STDOUT"/>
-	</root>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-avro/src/test/resources/logback-test.xml b/flink-connectors/flink-avro/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27802b18..0000000000000
--- a/flink-connectors/flink-avro/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-elasticsearch-base/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92f0cf65..0000000000000
--- a/flink-connectors/flink-connector-elasticsearch-base/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-elasticsearch/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-elasticsearch/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92f0cf65..0000000000000
--- a/flink-connectors/flink-connector-elasticsearch/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-elasticsearch2/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-elasticsearch2/src/test/resources/logback-test.xml
deleted file mode 100644
index 7a077c26905cb..0000000000000
--- a/flink-connectors/flink-connector-elasticsearch2/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming.connectors.elasticsearch2" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-elasticsearch5/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-elasticsearch5/src/test/resources/logback-test.xml
deleted file mode 100644
index 8893f7c1f216c..0000000000000
--- a/flink-connectors/flink-connector-elasticsearch5/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming.connectors.elasticsearch5" level="WARN"/>
-</configuration>
diff --git a/flink-connectors/flink-connector-filesystem/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-filesystem/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92f0cf65..0000000000000
--- a/flink-connectors/flink-connector-filesystem/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-kafka-0.10/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92f0cf65..0000000000000
--- a/flink-connectors/flink-connector-kafka-0.10/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-kafka-0.8/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92f0cf65..0000000000000
--- a/flink-connectors/flink-connector-kafka-0.8/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-kafka-0.9/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92f0cf65..0000000000000
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/logback-test.xml b/flink-connectors/flink-connector-kafka-base/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92f0cf65..0000000000000
--- a/flink-connectors/flink-connector-kafka-base/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/resources/logback-test.xml b/flink-connectors/flink-hadoop-compatibility/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27802b18..0000000000000
--- a/flink-connectors/flink-hadoop-compatibility/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-</configuration>
\ No newline at end of file
diff --git a/flink-connectors/flink-jdbc/src/test/resources/logback-test.xml b/flink-connectors/flink-jdbc/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27802b18..0000000000000
--- a/flink-connectors/flink-jdbc/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-</configuration>
\ No newline at end of file
diff --git a/flink-contrib/flink-connector-wikiedits/src/test/resources/log4j.properties b/flink-contrib/flink-connector-wikiedits/src/test/resources/log4j.properties
deleted file mode 100644
index ed2bbcbbd8167..0000000000000
--- a/flink-contrib/flink-connector-wikiedits/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=OFF, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
diff --git a/flink-contrib/flink-connector-wikiedits/src/test/resources/logback-test.xml b/flink-contrib/flink-connector-wikiedits/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748368989..0000000000000
--- a/flink-contrib/flink-connector-wikiedits/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties
deleted file mode 100644
index 4a30c6f18449c..0000000000000
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=OFF, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml b/flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748368989..0000000000000
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-contrib/flink-storm-examples/src/test/resources/log4j.properties b/flink-contrib/flink-storm-examples/src/test/resources/log4j.properties
deleted file mode 100644
index ed2bbcbbd8167..0000000000000
--- a/flink-contrib/flink-storm-examples/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=OFF, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
diff --git a/flink-contrib/flink-storm-examples/src/test/resources/logback-test.xml b/flink-contrib/flink-storm-examples/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748368989..0000000000000
--- a/flink-contrib/flink-storm-examples/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-contrib/flink-storm/src/test/resources/log4j.properties b/flink-contrib/flink-storm/src/test/resources/log4j.properties
deleted file mode 100644
index ed2bbcbbd8167..0000000000000
--- a/flink-contrib/flink-storm/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=OFF, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
diff --git a/flink-contrib/flink-storm/src/test/resources/logback-test.xml b/flink-contrib/flink-storm/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748368989..0000000000000
--- a/flink-contrib/flink-storm/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-contrib/flink-streaming-contrib/src/test/resources/log4j.properties b/flink-contrib/flink-streaming-contrib/src/test/resources/log4j.properties
deleted file mode 100644
index ed2bbcbbd8167..0000000000000
--- a/flink-contrib/flink-streaming-contrib/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=OFF, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
diff --git a/flink-contrib/flink-streaming-contrib/src/test/resources/logback-test.xml b/flink-contrib/flink-streaming-contrib/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748368989..0000000000000
--- a/flink-contrib/flink-streaming-contrib/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-contrib/flink-tweet-inputformat/src/test/resources/log4j.properties b/flink-contrib/flink-tweet-inputformat/src/test/resources/log4j.properties
deleted file mode 100644
index ed2bbcbbd8167..0000000000000
--- a/flink-contrib/flink-tweet-inputformat/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=OFF, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
diff --git a/flink-contrib/flink-tweet-inputformat/src/test/resources/logback-test.xml b/flink-contrib/flink-tweet-inputformat/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748368989..0000000000000
--- a/flink-contrib/flink-tweet-inputformat/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-core/src/test/resources/logback-test.xml b/flink-core/src/test/resources/logback-test.xml
deleted file mode 100644
index 1c4ea08c44a20..0000000000000
--- a/flink-core/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-fs-tests/src/test/resources/log4j.properties b/flink-fs-tests/src/test/resources/log4j.properties
deleted file mode 100644
index f533ba2203516..0000000000000
--- a/flink-fs-tests/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,31 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# Tachyon's test-jar dependency adds a log4j.properties file to classpath.
-# Until the issue is resolved (see https://github.com/amplab/tachyon/pull/571)
-# we provide a log4j.properties file ourselves.
-
-log4j.rootLogger=OFF, testlogger
-
-log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
-log4j.appender.testlogger.target = System.err
-log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
-log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
-
-# suppress the irrelevant (wrong) warnings from the netty channel handler
-log4j.logger.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, testlogger
\ No newline at end of file
diff --git a/flink-java/src/test/resources/logback-test.xml b/flink-java/src/test/resources/logback-test.xml
deleted file mode 100644
index 159f76c4cc710..0000000000000
--- a/flink-java/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
-
diff --git a/flink-libraries/flink-cep/src/test/resources/logback-test.xml b/flink-libraries/flink-cep/src/test/resources/logback-test.xml
deleted file mode 100644
index 1c4ea08c44a20..0000000000000
--- a/flink-libraries/flink-cep/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-mesos/src/test/resources/logback-test.xml b/flink-mesos/src/test/resources/logback-test.xml
deleted file mode 100644
index 48e4374b1cd14..0000000000000
--- a/flink-mesos/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} %-5level [%thread] %logger{60} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-
-    <!--<logger name="org.apache.flink.runtime.operators.BatchTask" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.taskmanager.Task" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.jobmanager.JobManager" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.taskmanager.TaskManager" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.executiongraph.ExecutionGraph" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.jobmanager.EventCollector" level="OFF"/>-->
-</configuration>
\ No newline at end of file
diff --git a/flink-metrics/flink-metrics-dropwizard/src/test/resources/logback-test.xml b/flink-metrics/flink-metrics-dropwizard/src/test/resources/logback-test.xml
deleted file mode 100644
index 1c4ea08c44a20..0000000000000
--- a/flink-metrics/flink-metrics-dropwizard/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-metrics/flink-metrics-jmx/src/test/resources/logback-test.xml b/flink-metrics/flink-metrics-jmx/src/test/resources/logback-test.xml
deleted file mode 100644
index 1c4ea08c44a20..0000000000000
--- a/flink-metrics/flink-metrics-jmx/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-metrics/flink-metrics-statsd/src/test/resources/logback-test.xml b/flink-metrics/flink-metrics-statsd/src/test/resources/logback-test.xml
deleted file mode 100644
index 1c4ea08c44a20..0000000000000
--- a/flink-metrics/flink-metrics-statsd/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-optimizer/src/test/resources/logback-test.xml b/flink-optimizer/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27802b18..0000000000000
--- a/flink-optimizer/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-</configuration>
\ No newline at end of file
diff --git a/flink-runtime-web/src/test/resources/logback-test.xml b/flink-runtime-web/src/test/resources/logback-test.xml
deleted file mode 100644
index 22352512443f8..0000000000000
--- a/flink-runtime-web/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] [%X{sourceThread} - %X{akkaSource}] %-5level %logger{60} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-
-    <!-- The following loggers are disabled during tests, because many tests deliberately
-         throw error to test failing scenarios. Logging those would overflow the log. -->
-         <!---->
-    <logger name="org.apache.flink.runtime.operators.DataSinkTask" level="OFF"/>
-    <logger name="org.apache.flink.runtime.operators.BatchTask" level="OFF"/>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-    <logger name="org.apache.flink.runtime.taskmanager.Task" level="OFF"/>
-    <logger name="org.apache.flink.runtime.jobmanager.JobManager" level="OFF"/>
-    <logger name="org.apache.flink.runtime.testingUtils" level="OFF"/>
-    <logger name="org.apache.flink.runtime.executiongraph.ExecutionGraph" level="OFF"/>
-    <logger name="org.apache.flink.runtime.jobmanager.EventCollector" level="OFF"/>
-    <logger name="org.apache.flink.runtime.instance.InstanceManager" level="OFF"/>
-</configuration>
diff --git a/flink-runtime/src/test/resources/logback-test.xml b/flink-runtime/src/test/resources/logback-test.xml
deleted file mode 100644
index 1d64d46284302..0000000000000
--- a/flink-runtime/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] [%X{sourceThread} - %X{akkaSource}] %-5level %logger{60} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-
-    <!-- The following loggers are disabled during tests, because many tests deliberately
-         throw error to test failing scenarios. Logging those would overflow the log. -->
-         <!---->
-    <logger name="org.apache.flink.runtime.operators.DataSinkTask" level="OFF"/>
-    <logger name="org.apache.flink.runtime.operators.BatchTask" level="OFF"/>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-    <logger name="org.apache.flink.runtime.taskmanager.Task" level="OFF"/>
-    <logger name="org.apache.flink.runtime.jobmanager.JobManager" level="OFF"/>
-    <logger name="org.apache.flink.runtime.testingUtils" level="OFF"/>
-    <logger name="org.apache.flink.runtime.executiongraph.ExecutionGraph" level="OFF"/>
-    <logger name="org.apache.flink.runtime.jobmanager.EventCollector" level="OFF"/>
-    <logger name="org.apache.flink.runtime.instance.InstanceManager" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-scala-shell/src/test/resources/logback-test.xml b/flink-scala-shell/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27802b18..0000000000000
--- a/flink-scala-shell/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-</configuration>
\ No newline at end of file
diff --git a/flink-streaming-java/src/test/resources/log4j.properties b/flink-streaming-java/src/test/resources/log4j.properties
deleted file mode 100644
index 4a30c6f18449c..0000000000000
--- a/flink-streaming-java/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=OFF, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
diff --git a/flink-streaming-java/src/test/resources/logback-test.xml b/flink-streaming-java/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748368989..0000000000000
--- a/flink-streaming-java/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/test/resources/logback-test.xml b/flink-test-utils-parent/flink-test-utils-junit/src/test/resources/logback-test.xml
deleted file mode 100644
index 1c4ea08c44a20..0000000000000
--- a/flink-test-utils-parent/flink-test-utils-junit/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
\ No newline at end of file
diff --git a/flink-tests/src/test/resources/log4j.properties b/flink-tests/src/test/resources/log4j.properties
deleted file mode 100644
index 9fbcecdc33d79..0000000000000
--- a/flink-tests/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=INFO, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
diff --git a/flink-tests/src/test/resources/logback-test.xml b/flink-tests/src/test/resources/logback-test.xml
deleted file mode 100644
index 48e4374b1cd14..0000000000000
--- a/flink-tests/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} %-5level [%thread] %logger{60} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-
-    <!--<logger name="org.apache.flink.runtime.operators.BatchTask" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.taskmanager.Task" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.jobmanager.JobManager" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.taskmanager.TaskManager" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.executiongraph.ExecutionGraph" level="OFF"/>-->
-    <!--<logger name="org.apache.flink.runtime.jobmanager.EventCollector" level="OFF"/>-->
-</configuration>
\ No newline at end of file
diff --git a/flink-yarn/src/test/resources/logback-test.xml b/flink-yarn/src/test/resources/logback-test.xml
deleted file mode 100644
index 1c4ea08c44a20..0000000000000
--- a/flink-yarn/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="WARN">
-        <appender-ref ref="STDOUT"/>
-    </root>
-    
-    <logger name="org.apache.flink.api.common.io.DelimitedInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.api.common.io.FileInputFormat" level="OFF"/>
-    <logger name="org.apache.flink.configuration.GlobalConfiguration" level="OFF"/>
-    <logger name="org.apache.flink.configuration.Configuration" level="OFF"/>
-</configuration>
\ No newline at end of file

From 55a8fe718810d868d5c6c504bf78d29100696d37 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Sat, 29 Apr 2017 17:40:30 +0200
Subject: [PATCH 108/253] [hotfix] [security] Reduce logging verbosity for
 SSLUtils

---
 .../apache/flink/runtime/net/SSLUtils.java    | 26 ++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
index 5bafeb866e1ac..2267eacc8b13b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
@@ -18,10 +18,10 @@
 
 package org.apache.flink.runtime.net;
 
-
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.util.Preconditions;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,6 +35,7 @@
 import java.io.FileInputStream;
 import java.net.ServerSocket;
 import java.security.KeyStore;
+import java.util.Arrays;
 
 /**
  * Common utilities to manage SSL transport settings
@@ -66,14 +67,21 @@ public static boolean getSSLEnabled(Configuration sslConfig) {
 	 */
 	public static void setSSLVerAndCipherSuites(ServerSocket socket, Configuration config) {
 		if (socket instanceof SSLServerSocket) {
-			((SSLServerSocket) socket).setEnabledProtocols(config.getString(
-				ConfigConstants.SECURITY_SSL_PROTOCOL,
-				ConfigConstants.DEFAULT_SECURITY_SSL_PROTOCOL).split(","));
-			((SSLServerSocket) socket).setEnabledCipherSuites(config.getString(
-				ConfigConstants.SECURITY_SSL_ALGORITHMS,
-				ConfigConstants.DEFAULT_SECURITY_SSL_ALGORITHMS).split(","));
-		} else {
-			LOG.warn("Not a SSL socket, will skip setting tls version and cipher suites.");
+			final String[] protocols = config.getString(
+					ConfigConstants.SECURITY_SSL_PROTOCOL,
+					ConfigConstants.DEFAULT_SECURITY_SSL_PROTOCOL).split(",");
+
+			final String[] cipherSuites = config.getString(
+					ConfigConstants.SECURITY_SSL_ALGORITHMS,
+					ConfigConstants.DEFAULT_SECURITY_SSL_ALGORITHMS).split(",");
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("Configuring TLS version and cipher suites on SSL socket {} / {}",
+						Arrays.toString(protocols), Arrays.toString(cipherSuites));
+			}
+
+			((SSLServerSocket) socket).setEnabledProtocols(protocols);
+			((SSLServerSocket) socket).setEnabledCipherSuites(cipherSuites);
 		}
 	}
 

From 1d7013403208bfc107dfcd9f14a9cdb0bf441820 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 2 May 2017 19:33:28 +0200
Subject: [PATCH 109/253] [hotfix] [runtime] SerializedThrowable copy
 constructor preserves cause

---
 .../flink/runtime/util/SerializedThrowable.java    |  2 ++
 .../runtime/util/SerializedThrowableTest.java      | 14 ++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
index 4dea59c3737e0..ed485d0213b44 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
@@ -108,6 +108,8 @@ private SerializedThrowable(Throwable exception, Set<Throwable> alreadySeen) {
 			this.originalErrorClassName = other.originalErrorClassName;
 			this.fullStingifiedStackTrace = other.fullStingifiedStackTrace;
 			this.cachedException = other.cachedException;
+			this.setStackTrace(other.getStackTrace());
+			this.initCause(other.getCause());
 		}
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/SerializedThrowableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/SerializedThrowableTest.java
index 4d57892869373..45988f5a47a5a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/SerializedThrowableTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/SerializedThrowableTest.java
@@ -175,4 +175,18 @@ public void testCyclicCauseChaining() {
 		assertArrayEquals(root.getStackTrace(), st.getStackTrace());
 		assertEquals(ExceptionUtils.stringifyException(root), ExceptionUtils.stringifyException(st));
 	}
+
+	@Test
+	public void testCopyPreservesCause() {
+		Exception original = new Exception("original message");
+		Exception parent = new Exception("parent message", original);
+
+		SerializedThrowable serialized = new SerializedThrowable(parent);
+		assertNotNull(serialized.getCause());
+
+		SerializedThrowable copy = new SerializedThrowable(serialized);
+		assertEquals("parent message", copy.getMessage());
+		assertNotNull(copy.getCause());
+		assertEquals("original message", copy.getCause().getMessage());
+	}
 }

From 3ae7cb7d5d83d3fcc2f00ce40fc07d873fb519c0 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 2 May 2017 21:24:10 +0200
Subject: [PATCH 110/253] [hotfix] [runtime] Minor improvement to logging in
 CheckpointCoordinator

---
 .../apache/flink/runtime/checkpoint/CheckpointCoordinator.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 96add0681fd0d..0bbf6b5f66dfa 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -646,7 +646,8 @@ else if (!props.forceCheckpoint()) {
 				}
 
 				int numUnsuccessful = numUnsuccessfulCheckpointsTriggers.incrementAndGet();
-				LOG.warn("Failed to trigger checkpoint (" + numUnsuccessful + " consecutive failed attempts so far)", t);
+				LOG.warn("Failed to trigger checkpoint {}. ({} consecutive failed attempts so far)",
+						checkpointID, numUnsuccessful, t);
 
 				if (!checkpoint.isDiscarded()) {
 					checkpoint.abortError(new Exception("Failed to trigger checkpoint", t));

From 4faa96c8787f72403da735082a1f5a0c99313fb3 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 2 May 2017 22:10:40 +0200
Subject: [PATCH 111/253] [hotfix] [client] Unwrap SerializedThrowable in
 client log statements when possible

---
 .../java/org/apache/flink/runtime/client/JobClientActor.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
index 368a2b6d3bda5..a0bf97dbfccbd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
@@ -34,6 +34,7 @@
 import org.apache.flink.runtime.messages.JobClientMessages.JobManagerActorRef;
 import org.apache.flink.runtime.messages.JobClientMessages.JobManagerLeaderAddress;
 import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.util.SerializedThrowable;
 import org.apache.flink.util.Preconditions;
 import scala.concurrent.duration.FiniteDuration;
 
@@ -287,7 +288,8 @@ private void logAndPrintMessage(ExecutionGraphMessages.JobStatusChanged message)
 				System.out.println(message.toString());
 			}
 		} else {
-			LOG.info(message.toString(), message.error());
+			Throwable error = SerializedThrowable.get(message.error(), getClass().getClassLoader());
+			LOG.info(message.toString(), error);
 			if (sysoutUpdates) {
 				System.out.println(message.toString());
 				message.error().printStackTrace(System.out);

From 96429c2a2d9da87053e9afcca63718a1430ce6f7 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 2 May 2017 22:48:08 +0200
Subject: [PATCH 112/253] [FLINK-2067] [runtime] Unwrap the
 ExceptionInChainedOperatorException exceptions to clean up stack traces

---
 .../flink/util/WrappingRuntimeException.java  | 54 ++++++++++++++++++
 .../flink/runtime/taskmanager/Task.java       |  7 +++
 .../flink/runtime/taskmanager/TaskTest.java   | 57 ++++++++++++++++++-
 .../ExceptionInChainedOperatorException.java  | 11 +---
 .../runtime/tasks/OperatorChain.java          |  4 +-
 5 files changed, 120 insertions(+), 13 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/util/WrappingRuntimeException.java

diff --git a/flink-core/src/main/java/org/apache/flink/util/WrappingRuntimeException.java b/flink-core/src/main/java/org/apache/flink/util/WrappingRuntimeException.java
new file mode 100644
index 0000000000000..f9306dfa366fc
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/WrappingRuntimeException.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import javax.annotation.Nonnull;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A runtime exception that is explicitly used to wrap non-runtime exceptions.
+ * 
+ * <p>The exception is recognized (for example by the Task when reporting exceptions as
+ * failure causes) and unwrapped to avoid including the wrapper's stack trace in the reports.
+ * That way, exception traces are keeping to the important parts.
+ */
+public class WrappingRuntimeException extends FlinkRuntimeException {
+
+	private static final long serialVersionUID = 1L;
+
+	public WrappingRuntimeException(@Nonnull Throwable cause) {
+		super(checkNotNull(cause));
+	}
+
+	public WrappingRuntimeException(String message, @Nonnull Throwable cause) {
+		super(message, checkNotNull(cause));
+	}
+
+	/**
+	 * Recursively unwraps this WrappingRuntimeException and its causes, getting the first
+	 * non wrapping exception.
+	 * 
+	 * @return The first cause that is not a wrapping exception.
+	 */
+	public Throwable unwrap() {
+		Throwable cause = getCause();
+		return (cause instanceof WrappingRuntimeException) ? ((WrappingRuntimeException) cause).unwrap() : cause;
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index dab0f9505a1cd..e626dae1c247b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -71,6 +71,8 @@
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.SerializedValue;
+
+import org.apache.flink.util.WrappingRuntimeException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -727,6 +729,11 @@ else if (current == ExecutionState.CANCELING) {
 		}
 		catch (Throwable t) {
 
+			// unwrap wrapped exceptions to make stack traces more compact
+			if (t instanceof WrappingRuntimeException) {
+				t = ((WrappingRuntimeException) t).unwrap();
+			}
+
 			// ----------------------------------------------------------------
 			// the execution failed. either the invokable code properly failed, or
 			// an exception was thrown as a side effect of cancelling
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index 252228778895f..56a3b076d4df8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -59,11 +59,13 @@
 import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
 import org.apache.flink.util.SerializedValue;
 import org.apache.flink.util.TestLogger;
+import org.apache.flink.util.WrappingRuntimeException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import scala.concurrent.duration.FiniteDuration;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.net.URL;
@@ -117,9 +119,9 @@ public class TaskTest extends TestLogger {
 	
 	@Before
 	public void createQueuesAndActors() {
-		taskManagerMessages = new LinkedBlockingQueue<Object>();
-		jobManagerMessages = new LinkedBlockingQueue<Object>();
-		listenerMessages = new LinkedBlockingQueue<Object>();
+		taskManagerMessages = new LinkedBlockingQueue<>();
+		jobManagerMessages = new LinkedBlockingQueue<>();
+		listenerMessages = new LinkedBlockingQueue<>();
 		taskManagerGateway = new ForwardingActorGateway(taskManagerMessages);
 		jobManagerGateway = new ForwardingActorGateway(jobManagerMessages);
 		listenerGateway = new ForwardingActorGateway(listenerMessages);
@@ -335,6 +337,32 @@ public void testExecutionFailsInInvoke() {
 			fail(e.getMessage());
 		}
 	}
+
+	@Test
+	public void testFailWithWrappedException() {
+		try {
+			Task task = createTask(FailingInvokableWithChainedException.class);
+			task.registerExecutionListener(listener);
+
+			task.run();
+
+			assertEquals(ExecutionState.FAILED, task.getExecutionState());
+			assertTrue(task.isCanceledOrFailed());
+
+			Throwable cause = task.getFailureCause();
+			assertTrue(cause instanceof IOException);
+
+			validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+			validateUnregisterTask(task.getExecutionId());
+
+			validateListenerMessage(ExecutionState.RUNNING, task, false);
+			validateListenerMessage(ExecutionState.FAILED, task, true);
+		}
+		catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
 	
 	@Test
 	public void testCancelDuringInvoke() {
@@ -1232,4 +1260,27 @@ public void invoke() throws Exception {
 		public void cancel() throws Exception {
 		}
 	}
+
+	public static final class FailingInvokableWithChainedException extends AbstractInvokable {
+
+		@Override
+		public void invoke() throws Exception {
+			throw new TestWrappedException(new IOException("test"));
+		}
+
+		@Override
+		public void cancel() {}
+	}
+
+	// ------------------------------------------------------------------------
+	//  test exceptions
+	// ------------------------------------------------------------------------
+
+	private static class TestWrappedException extends WrappingRuntimeException {
+		private static final long serialVersionUID = 1L;
+
+		public TestWrappedException(@Nonnull Throwable cause) {
+			super(cause);
+		}
+	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
index 77c80c965d196..d4027bfd0859a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/ExceptionInChainedOperatorException.java
@@ -21,12 +21,13 @@
 import static java.util.Objects.requireNonNull;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.WrappingRuntimeException;
 
 /**
  * A special exception that signifies that the cause exception came from a chained operator.
  */
 @Internal
-public class ExceptionInChainedOperatorException extends RuntimeException {
+public class ExceptionInChainedOperatorException extends WrappingRuntimeException {
 
 	private static final long serialVersionUID = 1L;
 
@@ -37,12 +38,4 @@ public ExceptionInChainedOperatorException(Throwable cause) {
 	public ExceptionInChainedOperatorException(String message, Throwable cause) {
 		super(message, requireNonNull(cause));
 	}
-
-	public Throwable getOriginalCause() {
-		Throwable ex = this;
-		do {
-			ex = ex.getCause();
-		} while (ex instanceof ExceptionInChainedOperatorException);
-		return ex;
-	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index b85461d956c6a..870c2edc0ecc7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple2;
@@ -54,6 +55,7 @@
 import org.apache.flink.streaming.runtime.streamstatus.StreamStatusProvider;
 import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.XORShiftRandom;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -525,7 +527,7 @@ protected <X> void pushToOperator(StreamRecord<X> record) {
 				operator.setKeyContextElement1(copy);
 				operator.processElement(copy);
 			} catch (Exception e) {
-				throw new RuntimeException("Could not forward element to next operator", e);
+				throw new ExceptionInChainedOperatorException(e);
 			}
 
 		}

From 3302f32942f08c907bd2266c482c0b09ee88d196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BCrgen=20Thomann?= <juergen.thomann@innogames.com>
Date: Wed, 3 May 2017 13:06:04 +0200
Subject: [PATCH 113/253] [FLINK-6427] Ensure file length is flushed in
 StreamWriterBase

---
 .../flink/streaming/connectors/fs/StreamWriterBase.java     | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/StreamWriterBase.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/StreamWriterBase.java
index 140246f69da7e..a04e4b58c81d9 100644
--- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/StreamWriterBase.java
+++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/StreamWriterBase.java
@@ -21,12 +21,14 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.EnumSet;
 
 /**
  * Base class for {@link Writer Writers} that write to a {@link FSDataOutputStream}.
@@ -70,6 +72,10 @@ protected void hflushOrSync(FSDataOutputStream os) throws IOException {
 			// At this point the refHflushOrSync cannot be null,
 			// since register method would have thrown if it was.
 			this.refHflushOrSync.invoke(os);
+
+			if (os instanceof HdfsDataOutputStream) {
+				((HdfsDataOutputStream) os).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+			}
 		} catch (InvocationTargetException e) {
 			String msg = "Error while trying to hflushOrSync!";
 			LOG.error(msg + " " + e.getCause());

From fcef95c9e35112129b4defd0010325b8f3b60f43 Mon Sep 17 00:00:00 2001
From: Anton Mushin <anton_mushin@epam.com>
Date: Fri, 3 Feb 2017 14:06:49 +0400
Subject: [PATCH 114/253] [FLINK-4604] [table] Add support for standard
 deviation/variance

This closes #3260.

Old PR: This closes #2762.
---
 docs/dev/table_api.md                         |  47 +++-
 .../flink/table/api/scala/expressionDsl.scala |  28 +++
 .../table/expressions/ExpressionParser.scala  |  43 +++-
 .../table/expressions/aggregations.scala      |  65 ++++++
 .../aggfunctions/Sum0AggFunction.scala        |  91 ++++++++
 .../table/plan/rules/FlinkRuleSets.scala      |   3 +
 .../rules/dataSet/DataSetAggregateRule.scala  |   8 +-
 .../DataSetAggregateWithNullValuesRule.scala  |   9 +-
 .../table/validate/FunctionCatalog.scala      |  10 +
 .../table/api/java/batch/sql/SqlITCase.java   |  53 ++++-
 .../scala/batch/sql/AggregationsITCase.scala  | 200 ++++++++++++++++++
 .../batch/table/AggregationsITCase.scala      |  74 +++++++
 .../batch/utils/TableProgramsTestBase.scala   |   5 +
 13 files changed, 625 insertions(+), 11 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 2b777c6841c2c..ec32e9b287883 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1066,7 +1066,7 @@ dataType = "BYTE" | "SHORT" | "INT" | "LONG" | "FLOAT" | "DOUBLE" | "BOOLEAN" |
 
 as = composite , ".as(" , fieldReference , ")" ;
 
-aggregation = composite , ( ".sum" | ".min" | ".max" | ".count" | ".avg" | ".start" | ".end" ) , [ "()" ] ;
+aggregation = composite , ( ".sum" | ".min" | ".max" | ".count" | ".avg" | ".start" | ".end" | ".stddev_pop" | ".stddev_samp" | ".var_pop" | ".var_samp" ) , [ "()" ] ;
 
 if = composite , ".?(" , expression , "," , expression , ")" ;
 
@@ -5233,7 +5233,7 @@ AVG(numeric)
         <p>Returns the average (arithmetic mean) of <i>numeric</i> across all input values.</p>
       </td>
     </tr>
-
+    
     <tr>
       <td>
         {% highlight text %}
@@ -5376,6 +5376,49 @@ ELEMENT(ARRAY)
         <p>Returns the sole element of an array with a single element. Returns <code>null</code> if the array is empty. Throws an exception if the array has more than one element.</p>
       </td>
     </tr>
+<tr>
+      <td>
+        {% highlight text %}
+STDDEV_POP(value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the standard deviation of numeric <i>value</i></p>
+      </td>
+    </tr>
+    
+<tr>
+      <td>
+        {% highlight text %}
+STDDEV_SAMP(value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample standard deviation of numeric <i>value</i></p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+VAR_POP(value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the variance of numeric <i>value</i></p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+VAR_SAMP (value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample variance of numeric <i>value</i></p>
+      </td>
+    </tr>
   </tbody>
 </table>
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
index a512098177265..b1d16b3ea6623 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
@@ -164,6 +164,12 @@ trait ImplicitExpressionOperations {
     */
   def sum = Sum(expr)
 
+  /**
+    * Returns the sum of the values which go into it like [[Sum]].
+    * It differs in that when no non null values are applied zero is returned instead of null.
+    */
+  def sum0 = Sum0(expr)
+  
   /**
     * Returns the minimum value of field across all input values.
     */
@@ -184,6 +190,28 @@ trait ImplicitExpressionOperations {
     */
   def avg = Avg(expr)
 
+  /**
+    * Returns the population standard deviation of an expression.
+    * (the square root of [[VarPop]])
+    */
+  def stddev_pop = StddevPop(expr)
+
+  /**
+    * Returns the sample standard deviation of an expression.
+    * (the square root of [[VarSamp]]).
+    */
+  def stddev_samp = StddevSamp(expr)
+
+  /**
+    * Returns the population standard variance of an expression.
+    */
+  def var_pop = VarPop(expr)
+
+  /**
+    *  Returns the sample variance of a given expression.
+    */
+  def var_samp = VarSamp(expr)
+
   /**
     * Converts a value to a given type.
     *
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
index 113b85ad41ae5..1356416bbd4f0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
@@ -57,6 +57,11 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val SUM: Keyword = Keyword("sum")
   lazy val START: Keyword = Keyword("start")
   lazy val END: Keyword = Keyword("end")
+  lazy val SUM0: Keyword = Keyword("sum0")
+  lazy val STDDEV_POP: Keyword = Keyword("stddev_pop")
+  lazy val STDDEV_SAMP: Keyword = Keyword("stddev_samp")
+  lazy val VAR_POP: Keyword = Keyword("var_pop")
+  lazy val VAR_SAMP: Keyword = Keyword("var_samp")
   lazy val CAST: Keyword = Keyword("cast")
   lazy val NULL: Keyword = Keyword("Null")
   lazy val IF: Keyword = Keyword("?")
@@ -95,7 +100,8 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val ASIN: Keyword = Keyword("asin")
 
   def functionIdent: ExpressionParser.Parser[String] =
-    not(ARRAY) ~ not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~
+    not(ARRAY) ~ not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~ not(STDDEV_POP) ~
+      not(STDDEV_SAMP) ~ not(VAR_SAMP) ~ not(VAR_POP) ~
       not(SUM) ~ not(START) ~ not(END)~ not(CAST) ~ not(NULL) ~ not(IF) ~
       not(CURRENT_ROW) ~ not(UNBOUNDED_ROW) ~ not(CURRENT_RANGE) ~ not(UNBOUNDED_RANGE) ~>
       super.ident
@@ -205,6 +211,9 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val suffixSum: PackratParser[Expression] =
     composite <~ "." ~ SUM ~ opt("()") ^^ { e => Sum(e) }
 
+  lazy val suffixSum0: PackratParser[Expression] =
+    composite <~ "." ~ SUM0 ~ opt("()") ^^ { e => Sum0(e) }
+
   lazy val suffixMin: PackratParser[Expression] =
     composite <~ "." ~ MIN ~ opt("()") ^^ { e => Min(e) }
 
@@ -223,6 +232,17 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val suffixEnd: PackratParser[Expression] =
     composite <~ "." ~ END ~ opt("()") ^^ { e => WindowEnd(e) }
 
+  lazy val suffixStddevPop: PackratParser[Expression] =
+    composite <~ "." ~ STDDEV_POP ~ opt("()") ^^ { e => StddevPop(e) }
+
+  lazy val suffixStddevSamp: PackratParser[Expression] =
+    composite <~ "." ~ STDDEV_SAMP ~ opt("()") ^^ { e => StddevSamp(e) }
+
+  lazy val suffixVarSamp: PackratParser[Expression] =
+    composite <~ "." ~ VAR_SAMP ~ opt("()") ^^ { e => VarSamp(e) }
+
+  lazy val suffixVarPop: PackratParser[Expression] =
+    composite <~ "." ~ VAR_POP ~ opt("()") ^^ { e => VarPop(e) }
   lazy val suffixCast: PackratParser[Expression] =
     composite ~ "." ~ CAST ~ "(" ~ dataType ~ ")" ^^ {
     case e ~ _ ~ _ ~ _ ~ dt ~ _ => Cast(e, dt)
@@ -320,8 +340,9 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
     composite <~ "." ~ ASIN ~ opt("()") ^^ { e => Asin(e) }
 
   lazy val suffixed: PackratParser[Expression] =
-    suffixTimeInterval | suffixRowInterval | suffixStart | suffixEnd | suffixAgg |
+    suffixTimeInterval | suffixRowInterval | suffixStart | suffixSum0 | suffixEnd | suffixAgg |
       suffixCast | suffixAs | suffixTrim | suffixTrimWithoutArgs | suffixIf | suffixAsc |
+      suffixStddevPop | suffixStddevSamp | suffixVarPop | suffixVarSamp |
       suffixDesc | suffixToDate | suffixToTimestamp | suffixToTime | suffixExtract |
       suffixFloor | suffixCeil | suffixGet | suffixFlattening | suffixAsin |
       suffixFunctionCall | suffixFunctionCallOneArg // function call must always be at the end
@@ -334,6 +355,9 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val prefixSum: PackratParser[Expression] =
     SUM ~ "(" ~> expression <~ ")" ^^ { e => Sum(e) }
 
+  lazy val prefixSum0: PackratParser[Expression] =
+    SUM0 ~ "(" ~> expression <~ ")" ^^ { e => Sum0(e) }
+
   lazy val prefixMin: PackratParser[Expression] =
     MIN ~ "(" ~> expression <~ ")" ^^ { e => Min(e) }
 
@@ -352,6 +376,17 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val prefixEnd: PackratParser[Expression] =
     END ~ "(" ~> expression <~ ")" ^^ { e => WindowEnd(e) }
 
+  lazy val prefixStddevPop: PackratParser[Expression] =
+    STDDEV_POP ~ "(" ~> expression <~ ")" ^^ { e => StddevPop(e) }
+
+  lazy val prefixStddevSamp: PackratParser[Expression] =
+    STDDEV_SAMP ~ "(" ~> expression <~ ")" ^^ { e => StddevSamp(e) }
+
+  lazy val prefixVarSamp: PackratParser[Expression] =
+    VAR_SAMP ~ "(" ~> expression <~ ")" ^^ { e => VarSamp(e) }
+
+  lazy val prefixVarPop: PackratParser[Expression] =
+    VAR_POP ~ "(" ~> expression <~ ")" ^^ { e => VarPop(e) }
   lazy val prefixCast: PackratParser[Expression] =
     CAST ~ "(" ~ expression ~ "," ~ dataType ~ ")" ^^ {
     case _ ~ _ ~ e ~ _ ~ dt ~ _ => Cast(e, dt)
@@ -411,7 +446,9 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
     ASIN ~ "(" ~> composite <~ ")" ^^ { e => Asin(e) }
 
   lazy val prefixed: PackratParser[Expression] =
-    prefixArray | prefixAgg | prefixStart | prefixEnd | prefixCast | prefixAs | prefixTrim |
+    prefixArray | prefixSum | prefixSum0 | prefixAgg | prefixStart | prefixEnd |
+      prefixCast | prefixAs | prefixTrim | prefixStddevPop | prefixStddevSamp | prefixVarSamp |
+      prefixVarPop |
       prefixTrimWithoutArgs | prefixIf | prefixExtract | prefixFloor | prefixCeil | prefixGet |
       prefixFlattening | prefixAsin | prefixFunctionCall |
       prefixFunctionCallOneArg // function call must always be at the end
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
index 4ef52099d6c1a..82f4428507fb3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
@@ -66,6 +66,19 @@ case class Sum(child: Expression) extends Aggregation {
   }
 }
 
+case class Sum0(child: Expression) extends Aggregation {
+  override def toString = s"sum0($child)"
+
+  override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
+    relBuilder.aggregateCall(SqlStdOperatorTable.SUM0, false, null, name, child.toRexNode)
+  }
+
+  override private[flink] def resultType = child.resultType
+
+  override private[flink] def validateInput =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "sum0")
+}
+
 case class Min(child: Expression) extends Aggregation {
   override def toString = s"min($child)"
 
@@ -130,3 +143,55 @@ case class Avg(child: Expression) extends Aggregation {
     new SqlAvgAggFunction(AVG)
   }
 }
+
+case class StddevPop(child: Expression) extends Aggregation {
+  override def toString = s"stddev_pop($child)"
+
+  override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
+    relBuilder.aggregateCall(SqlStdOperatorTable.STDDEV_POP, false, null, name, child.toRexNode)
+  }
+
+  override private[flink] def resultType = child.resultType
+
+  override private[flink] def validateInput =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "stddev_pop")
+}
+
+case class StddevSamp(child: Expression) extends Aggregation {
+  override def toString = s"stddev_samp($child)"
+
+  override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
+    relBuilder.aggregateCall(SqlStdOperatorTable.STDDEV_SAMP, false, null, name, child.toRexNode)
+  }
+
+  override private[flink] def resultType = child.resultType
+
+  override private[flink] def validateInput =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "stddev_samp")
+}
+
+case class VarPop(child: Expression) extends Aggregation {
+  override def toString = s"var_pop($child)"
+
+  override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
+    relBuilder.aggregateCall(SqlStdOperatorTable.VAR_POP, false, null, name, child.toRexNode)
+  }
+
+  override private[flink] def resultType = child.resultType
+
+  override private[flink] def validateInput =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "var_pop")
+}
+
+case class VarSamp(child: Expression) extends Aggregation {
+  override def toString = s"var_samp($child)"
+
+  override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
+    relBuilder.aggregateCall(SqlStdOperatorTable.VAR_SAMP, false, null, name, child.toRexNode)
+  }
+
+  override private[flink] def resultType = child.resultType
+
+  override private[flink] def validateInput =
+    TypeCheckUtils.assertNumericExpr(child.resultType, "var_samp")
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala
new file mode 100644
index 0000000000000..6a24fbe821a75
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.functions.aggfunctions
+
+import java.math.BigDecimal
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.table.functions.Accumulator
+
+abstract class Sum0AggFunction[T: Numeric] extends SumAggFunction[T] {
+
+  override def getValue(accumulator: Accumulator): T = {
+    val a = accumulator.asInstanceOf[SumAccumulator[T]]
+    if (a.f1) {
+      a.f0
+    } else {
+      0.asInstanceOf[T]
+    }
+  }
+}
+
+/**
+  * Built-in Byte Sum0 aggregate function
+  */
+class ByteSum0AggFunction extends Sum0AggFunction[Byte] {
+  override def getValueTypeInfo = BasicTypeInfo.BYTE_TYPE_INFO
+}
+
+/**
+  * Built-in Short Sum0 aggregate function
+  */
+class ShortSum0AggFunction extends Sum0AggFunction[Short] {
+  override def getValueTypeInfo = BasicTypeInfo.SHORT_TYPE_INFO
+}
+
+/**
+  * Built-in Int Sum0 aggregate function
+  */
+class IntSum0AggFunction extends Sum0AggFunction[Int] {
+  override def getValueTypeInfo = BasicTypeInfo.INT_TYPE_INFO
+}
+
+/**
+  * Built-in Long Sum0 aggregate function
+  */
+class LongSum0AggFunction extends Sum0AggFunction[Long] {
+  override def getValueTypeInfo = BasicTypeInfo.LONG_TYPE_INFO
+}
+
+/**
+  * Built-in Float Sum0 aggregate function
+  */
+class FloatSum0AggFunction extends Sum0AggFunction[Float] {
+  override def getValueTypeInfo = BasicTypeInfo.FLOAT_TYPE_INFO
+}
+
+/**
+  * Built-in Double Sum0 aggregate function
+  */
+class DoubleSum0AggFunction extends Sum0AggFunction[Double] {
+  override def getValueTypeInfo = BasicTypeInfo.DOUBLE_TYPE_INFO
+}
+
+/**
+  * Built-in Big Decimal Sum0 aggregate function
+  */
+class DecimalSum0AggFunction extends DecimalSumAggFunction {
+
+  override def getValue(accumulator: Accumulator): BigDecimal = {
+    if (!accumulator.asInstanceOf[DecimalSumAccumulator].f1) {
+      0.asInstanceOf[BigDecimal]
+    } else {
+      accumulator.asInstanceOf[DecimalSumAccumulator].f0
+    }
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
index 0bee4e5b03011..6ebbae491c693 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
@@ -74,6 +74,9 @@ object FlinkRuleSets {
     // expand distinct aggregate to normal aggregate with groupby
     AggregateExpandDistinctAggregatesRule.JOIN,
 
+    //aggregate reduce rule (deviation/variance functions)
+    AggregateReduceFunctionsRule.INSTANCE,
+
     // remove unnecessary sort rule
     SortRemoveRule.INSTANCE,
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala
index b4d5bc9cc61c9..faaeb97286570 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala
@@ -21,6 +21,7 @@ package org.apache.flink.table.plan.rules.dataSet
 import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet}
 import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.convert.ConverterRule
+import org.apache.calcite.sql.SqlKind
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.{DataSetAggregate, DataSetUnion}
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalAggregate
@@ -54,7 +55,12 @@ class DataSetAggregateRule
     // check if we have distinct aggregates
     val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
 
-    !distinctAggs
+    val supported = agg.getAggCallList.map(_.getAggregation.getKind).forall {
+      case SqlKind.STDDEV_POP | SqlKind.STDDEV_SAMP | SqlKind.VAR_POP | SqlKind.VAR_SAMP => false
+      case _ => true
+    }
+
+    !distinctAggs && supported
   }
 
   override def convert(rel: RelNode): RelNode = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala
index d183e605f8961..9636980465068 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala
@@ -17,11 +17,13 @@
  */
 package org.apache.flink.table.plan.rules.dataSet
 
+
 import com.google.common.collect.ImmutableList
 import org.apache.calcite.plan._
 import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.calcite.rex.RexLiteral
+import org.apache.calcite.sql.SqlKind
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.DataSetAggregate
 import org.apache.flink.table.plan.nodes.logical.{FlinkLogicalAggregate, FlinkLogicalUnion, FlinkLogicalValues}
@@ -51,7 +53,12 @@ class DataSetAggregateWithNullValuesRule
     // check if we have distinct aggregates
     val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
 
-    !distinctAggs
+    val supported = agg.getAggCallList.map(_.getAggregation.getKind).forall {
+      case SqlKind.STDDEV_POP | SqlKind.STDDEV_SAMP | SqlKind.VAR_POP | SqlKind.VAR_SAMP => false
+      case _ => true
+    }
+
+    !distinctAggs && supported
   }
 
   override def convert(rel: RelNode): RelNode = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
index 729ad48d5cfa7..cb37ce483ce58 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
@@ -169,6 +169,11 @@ object FunctionCatalog {
     "max" -> classOf[Max],
     "min" -> classOf[Min],
     "sum" -> classOf[Sum],
+    "sum0" -> classOf[Sum0],
+    "stddev_pop" -> classOf[StddevPop],
+    "stddev_samp" -> classOf[StddevSamp],
+    "var_pop" -> classOf[VarPop],
+    "var_samp" -> classOf[VarSamp],
 
     // string functions
     "charLength" -> classOf[CharLength],
@@ -305,10 +310,15 @@ class BasicOperatorTable extends ReflectiveSqlOperatorTable {
     SqlStdOperatorTable.GROUPING_ID,
     // AGGREGATE OPERATORS
     SqlStdOperatorTable.SUM,
+    SqlStdOperatorTable.SUM0,
     SqlStdOperatorTable.COUNT,
     SqlStdOperatorTable.MIN,
     SqlStdOperatorTable.MAX,
     SqlStdOperatorTable.AVG,
+    SqlStdOperatorTable.STDDEV_POP,
+    SqlStdOperatorTable.STDDEV_SAMP,
+    SqlStdOperatorTable.VAR_POP,
+    SqlStdOperatorTable.VAR_SAMP,
     // ARRAY OPERATORS
     SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
     SqlStdOperatorTable.ITEM,
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
index 114226cb0d20e..a5d202122934b 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.typeutils.MapTypeInfo;
@@ -36,9 +37,12 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
+import scala.collection.JavaConversions;
+import scala.collection.mutable.Buffer;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -136,7 +140,7 @@ public void testJoin() throws Exception {
 		DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
 
 		tableEnv.registerDataSet("t1", ds1, "a, b, c");
-		tableEnv.registerDataSet("t2",ds2, "d, e, f, g, h");
+		tableEnv.registerDataSet("t2", ds2, "d, e, f, g, h");
 
 		String sqlQuery = "SELECT c, g FROM t1, t2 WHERE b = e";
 		Table result = tableEnv.sql(sqlQuery);
@@ -156,9 +160,7 @@ public void testMap() throws Exception {
 		rows.add(new Tuple2<>(1, Collections.singletonMap("foo", "bar")));
 		rows.add(new Tuple2<>(2, Collections.singletonMap("foo", "spam")));
 
-		TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(
-			BasicTypeInfo.INT_TYPE_INFO,
-			new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));
+		TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));
 
 		DataSet<Tuple2<Integer, Map<String, String>>> ds1 = env.fromCollection(rows, ty);
 		tableEnv.registerDataSet("t1", ds1, "a, b");
@@ -171,4 +173,47 @@ public void testMap() throws Exception {
 		String expected = "bar\n" + "spam\n";
 		compareResultAsText(results, expected);
 	}
+
+	@Test
+	public void testDeviationAggregation() throws Exception {
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+		DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+		tableEnv.registerDataSet("AggTable", ds, "x, y, z");
+
+		Buffer<String> columnForAgg = JavaConversions.asScalaBuffer(Arrays.asList("x, y".split(",")));
+
+		String sqlQuery = getSelectQueryFromTemplate("AVG(?),STDDEV_POP(?),STDDEV_SAMP(?),VAR_POP(?),VAR_SAMP(?)", columnForAgg, "AggTable");
+		Table result = tableEnv.sql(sqlQuery);
+
+		String sqlQuery1 = getSelectQueryFromTemplate("SUM(?)/COUNT(?), " +
+			"SQRT( (SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / COUNT(?)), " +
+			"SQRT( (SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / CASE COUNT(?) WHEN 1 THEN NULL ELSE COUNT(?) - 1 END), " +
+			"(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / COUNT(?), " +
+			"(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / CASE COUNT(?) WHEN 1 THEN NULL ELSE COUNT(?) - 1 END", columnForAgg, "AggTable");
+
+		Table expected = tableEnv.sql(sqlQuery1);
+
+		DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
+		List<Row> results = resultSet.collect();
+
+		DataSet<Row> expectedResultSet = tableEnv.toDataSet(expected, Row.class);
+		String expectedResults = expectedResultSet.map(new MapFunction<Row, Object>() {
+			@Override
+			public Object map(Row value) throws Exception {
+				StringBuilder stringBuffer = new StringBuilder();
+
+				int arityCount = value.getArity();
+
+				for (int i = 0; i < arityCount; i++) {
+					Object product = value.getField(i);
+					stringBuffer.append(Double.valueOf(product.toString()).intValue()).append(",");
+				}
+				return stringBuffer.substring(0, stringBuffer.length() - 1);
+			}
+		}).collect().get(0).toString();
+
+		compareResultAsText(results, expectedResults);
+	}
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
index 600c15be7ba5c..4fbd73475c5bf 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
@@ -295,6 +295,206 @@ class AggregationsITCase(
     TestBaseUtils.compareResultAsText(results3.asJava, expected3)
   }
 
+
+
+  @Test
+  def testSqrtOfAggregatedSet(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val ds = env.fromElements((1.0f, 1), (2.0f, 2)).toTable(tEnv)
+
+    tEnv.registerTable("MyTable", ds)
+
+    val sqlQuery = "SELECT " +
+      "SQRT((SUM(a * a) - SUM(a) * SUM(a) / COUNT(a)) / COUNT(a)) " +
+      "from (select _1 as a from MyTable)"
+
+    val expected = "0.5"
+    val results = tEnv.sql(sqlQuery).toDataSet[Row].collect()
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testStddevPopAggregate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val ds = env.fromElements(
+      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
+      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
+    tEnv.registerTable("myTable", ds)
+    val columns = Array("_1","_2","_3","_4","_5","_6")
+
+    val sqlQuery = getSelectQueryFromTemplate("STDDEV_POP(?)")(columns,"myTable")
+
+    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
+    val expectedResult = "0,0,0,0,0.5,0.5"
+    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
+  }
+
+  @Test
+  def testStddevPopAggregateWithOtherAggreagteSUM0(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val ds = env.fromElements(
+      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
+      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
+    tEnv.registerTable("myTable", ds)
+    val columns = Array("_1","_2","_3","_4","_5","_6")
+
+    val sqlQuery = getSelectQueryFromTemplate("STDDEV_POP(?), " +
+                                  "$sum0(?), " +
+                                  "avg(?), " +
+                                  "max(?), " +
+                                  "min(?), " +
+                                  "count(?)" ) (columns,"myTable")
+
+    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
+
+    val expectedResult =
+        "0,3,1,2,1,2," +
+        "0,3,1,2,1,2," +
+        "0,3,1,2,1,2," +
+        "0,3,1,2,1,2," +
+        "0.5,3.0,1.5,2.0,1.0,2," +
+        "0.5,3.0,1.5,2.0,1.0,2"
+    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
+  }
+
+  @Test
+  def testStddevPopAggregateWithOtherAggreagte(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val ds = env.fromElements(
+      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
+      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
+    tEnv.registerTable("myTable", ds)
+    val columns = Array("_1","_2","_3","_4","_5","_6")
+
+    val sqlQuery = getSelectQueryFromTemplate("STDDEV_POP(?), " +
+                                  "sum(?), " +
+                                  "avg(?), " +
+                                  "max(?), " +
+                                  "min(?), " +
+                                  "count(?)" )(columns,"myTable")
+
+    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
+
+    val expectedResult =
+      "0,3,1,2,1,2," +
+        "0,3,1,2,1,2," +
+        "0,3,1,2,1,2," +
+        "0,3,1,2,1,2," +
+        "0.5,3.0,1.5,2.0,1.0,2," +
+        "0.5,3.0,1.5,2.0,1.0,2"
+    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
+  }
+
+  @Test
+  def testStddevSampAggregate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val ds1 = env.fromElements(
+      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
+      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
+    tEnv.registerTable("myTable", ds1)
+    val columns = Seq("_1","_2","_3","_4","_5","_6")
+
+    val sqlQuery = getSelectQueryFromTemplate("STDDEV_SAMP(?)")(columns,"myTable")
+    val sqlExpectedQuery = getSelectQueryFromTemplate(
+      "SQRT((SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / " +
+        "CASE " +
+        "COUNT(?) WHEN 1 THEN NULL " +
+        "ELSE COUNT(?) - 1 " +
+        "END)")(columns,"myTable")
+
+    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
+      .head
+      .toString
+      .split(",").map(x=>"%.5f".format(x.toFloat))
+
+    val expectedResult = tEnv.sql(sqlExpectedQuery).toDataSet[Row].collect()
+      .head
+      .toString
+      .split(",").map(x=>"%.5f".format(x.toFloat))
+
+    Assert.assertEquals(expectedResult.mkString(","), actualResult.mkString(","))
+  }
+
+  @Test
+  def testVarPopAggregate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val ds = env.fromElements(
+      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
+      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
+    tEnv.registerTable("myTable", ds)
+    val columns = Seq("_1","_2","_3","_4","_5","_6")
+
+    val sqlQuery = getSelectQueryFromTemplate("var_pop(?)")(columns,"myTable")
+    val sqlExpectedQuery = getSelectQueryFromTemplate(
+      "(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / COUNT(?)")(columns,"myTable")
+
+    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
+    val expectedResult = tEnv.sql(sqlExpectedQuery)
+      .toDataSet[Row]
+      .collect().head
+      .toString
+    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
+  }
+
+  @Test
+  def testVarSampAggregate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val ds = env.fromElements(
+      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
+      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
+    tEnv.registerTable("myTable", ds)
+    val columns = Seq("_1","_2","_3","_4","_5","_6")
+
+    val sqlQuery = getSelectQueryFromTemplate("var_samp(?)")(columns,"myTable")
+    val sqlExpectedQuery = getSelectQueryFromTemplate(
+      "(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / CASE COUNT(?) WHEN 1 THEN NULL " +
+        "ELSE COUNT(?) - 1 END")(columns,"myTable")
+
+    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
+    val expectedResult = tEnv.sql(sqlExpectedQuery)
+      .toDataSet[Row]
+      .collect().head
+      .toString
+    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
+  }
+
+  @Test
+  def testSumNullElements(): Unit = {
+
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val sqlQuery = getSelectQueryFromTemplate("$sum0(?)")(
+      Seq("_1","_2","_3","_4","_5","_6"),
+      "(select * from MyTable where _1 = 4)"
+    )
+
+    val ds = env.fromElements(
+      (1: Byte, 2L,1D,1F,1,1:Short ),
+      (2: Byte, 2L,1D,1F,1,1:Short ))
+    tEnv.registerDataSet("MyTable", ds)
+
+    val result = tEnv.sql(sqlQuery)
+
+    val expected = "null,null,null,null,null,null"
+    val results = result.toDataSet[Row].collect()
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
   @Test
   def testTumbleWindowAggregate(): Unit = {
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
index 48387474817b2..050f5f4e9f131 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
@@ -339,6 +339,80 @@ class AggregationsITCase(
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
+  @Test
+  def testAnalyticAggregation(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
+      .select('_1.stddev_pop, '_1.stddev_samp, '_1.var_pop, '_1.var_samp)
+    val results = t.toDataSet[Row].collect()
+    val expected = "6,6,36,38"
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testSQLStyleAnalyticAggregation(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+      .select(
+        """stddev_pop(a) as a1, a.stddev_pop as a2,
+          |stddev_samp (a) as b1, a.stddev_samp as b2,
+          |var_pop (a) as c1, a.var_pop as c2,
+          |var_samp (a) as d1, a.var_samp as d2
+        """.stripMargin)
+    val expected = "6,6,6,6,36,36,38,38"
+    val results = t.toDataSet[Row].collect()
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testWorkingAnalyticAggregationDataTypes(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    val ds = env.fromElements(
+      (1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d),
+      (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d)).toTable(tEnv)
+    val res = ds.select('_1.stddev_pop, '_2.stddev_pop, '_3.stddev_pop,
+                        '_4.stddev_pop, '_5.stddev_pop, '_6.stddev_pop,
+                        '_1.stddev_samp, '_2.stddev_samp, '_3.stddev_samp,
+                        '_4.stddev_samp, '_5.stddev_samp, '_6.stddev_samp,
+                        '_1.var_pop, '_2.var_pop, '_3.var_pop,
+                        '_4.var_pop, '_5.var_pop, '_6.var_pop,
+                        '_1.var_samp, '_2.var_samp, '_3.var_samp,
+                        '_4.var_samp, '_5.var_samp, '_6.var_samp)
+    val expected =
+        "0,0,0," +
+        "0,0.5,0.5," +
+        "1,1,1," +
+        "1,0.70710677,0.7071067811865476," +
+        "0,0,0," +
+        "0,0.25,0.25," +
+        "1,1,1," +
+        "1,0.5,0.5"
+    val results = res.toDataSet[Row].collect()
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testPojoAnalyticAggregation(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    val input = env.fromElements(
+      MyWC("hello", 1),
+      MyWC("hello", 8),
+      MyWC("ciao", 3),
+      MyWC("hola", 1),
+      MyWC("hola", 8))
+    val expr = input.toTable(tEnv)
+    val result = expr
+      .groupBy('word)
+      .select('word, 'frequency.stddev_pop)
+      .toDataSet[MyWC]
+    val mappedResult = result.map(w => (w.word, w.frequency)).collect()
+    val expected = "(hola,3)\n(ciao,0)\n(hello,3)"
+    TestBaseUtils.compareResultAsText(mappedResult.asJava, expected)
+  }
 }
 
 case class WC(word: String, frequency: Long)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala
index cf9d947f100c7..ee8e1f547b92e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala
@@ -37,6 +37,11 @@ class TableProgramsTestBase(
     }
     conf
   }
+
+  def getSelectQueryFromTemplate(selectBlock: String)
+                                (columnsName: Seq[String], table :String): String = {
+    s"SELECT ${columnsName.map(x=>selectBlock.replace("?",x)).mkString(",")} FROM $table"
+  }
 }
 
 object TableProgramsTestBase {

From 5398e546ef7a081c9ad12fb4d3d96c088cc2035a Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Wed, 3 May 2017 11:41:00 +0200
Subject: [PATCH 115/253] [FLINK-4604] [table] Clean-ups, improved Sum0
 aggregation, simplified tests and bug fixing

---
 docs/dev/table_api.md                         | 202 ++++++++++++++----
 .../flink/table/api/scala/expressionDsl.scala |  19 +-
 .../table/codegen/calls/BuiltInMethods.scala  |   2 +
 .../codegen/calls/FunctionGenerator.scala     |   6 +
 .../table/expressions/ExpressionParser.scala  |  71 +++---
 .../table/expressions/aggregations.scala      |  29 ++-
 .../aggfunctions/Sum0AggFunction.scala        |  21 +-
 .../Sum0WithRetractAggFunction.scala          |  95 ++++++++
 .../aggfunctions/SumAggFunction.scala         |  10 +-
 .../flink/table/plan/logical/operators.scala  |   4 +-
 .../nodes/logical/FlinkLogicalAggregate.scala |  15 +-
 .../table/plan/rules/FlinkRuleSets.scala      |   2 +-
 .../rules/dataSet/DataSetAggregateRule.scala  |   8 +-
 .../DataSetAggregateWithNullValuesRule.scala  |   8 +-
 .../runtime/aggregate/AggregateUtil.scala     |  43 +++-
 .../table/validate/FunctionCatalog.scala      |   8 +-
 .../table/api/java/batch/sql/SqlITCase.java   |  53 +----
 .../scala/batch/sql/AggregationsITCase.scala  | 200 -----------------
 .../batch/sql/QueryDecorrelationTest.scala    |   2 +-
 .../batch/table/AggregationsITCase.scala      |  90 ++------
 .../AggregationsStringExpressionTest.scala    | 123 ++++-------
 .../batch/utils/LogicalPlanFormatUtils.scala  |   9 +-
 .../batch/utils/TableProgramsTestBase.scala   |   5 -
 .../aggfunctions/Sum0AggFunctionTest.scala    | 156 ++++++++++++++
 .../Sum0WithRetractAggFunctionTest.scala      | 160 ++++++++++++++
 .../flink/table/utils/TableTestBase.scala     |   5 +-
 26 files changed, 798 insertions(+), 548 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunction.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunctionTest.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunctionTest.scala

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index ec32e9b287883..a77d9948916b4 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1066,7 +1066,7 @@ dataType = "BYTE" | "SHORT" | "INT" | "LONG" | "FLOAT" | "DOUBLE" | "BOOLEAN" |
 
 as = composite , ".as(" , fieldReference , ")" ;
 
-aggregation = composite , ( ".sum" | ".min" | ".max" | ".count" | ".avg" | ".start" | ".end" | ".stddev_pop" | ".stddev_samp" | ".var_pop" | ".var_samp" ) , [ "()" ] ;
+aggregation = composite , ( ".sum" | ".sum0" | ".min" | ".max" | ".count" | ".avg" | ".start" | ".end" | ".stddev_pop" | ".stddev_samp" | ".var_pop" | ".var_samp" ) , [ "()" ] ;
 
 if = composite , ".?(" , expression , "," , expression , ")" ;
 
@@ -2794,7 +2794,18 @@ FIELD.sum
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the sum of the numeric field across all input values.</p>
+        <p>Returns the sum of the numeric field across all input values. If all values are null, null is returned.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+FIELD.sum0
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sum of the numeric field across all input values. If all values are null, 0 is returned.</p>
       </td>
     </tr>
 
@@ -2820,6 +2831,51 @@ FIELD.min
       </td>
     </tr>
 
+
+    <tr>
+      <td>
+        {% highlight java %}
+FIELD.stddevPop
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the population standard deviation of the numeric field across all input values.</p>
+      </td>
+    </tr>
+    
+    <tr>
+      <td>
+        {% highlight java %}
+FIELD.stddevSamp
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample standard deviation of the numeric field across all input values.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+FIELD.varPop
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the population variance (square of the population standard deviation) of the numeric field across all input values.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight java %}
+FIELD.varSamp
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample variance (square of the sample standard deviation) of the numeric field across all input values.</p>
+      </td>
+    </tr>
+
     </tbody>
 </table>
 
@@ -3945,7 +4001,18 @@ FIELD.sum
 {% endhighlight %}
       </td>
       <td>
-        <p>Returns the sum of the numeric field across all input values.</p>
+        <p>Returns the sum of the numeric field across all input values. If all values are null, null is returned.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+FIELD.sum0
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sum of the numeric field across all input values. If all values are null, 0 is returned.</p>
       </td>
     </tr>
 
@@ -3971,6 +4038,49 @@ FIELD.min
       </td>
     </tr>
 
+    <tr>
+      <td>
+        {% highlight scala %}
+FIELD.stddevPop
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the population standard deviation of the numeric field across all input values.</p>
+      </td>
+    </tr>
+    
+    <tr>
+      <td>
+        {% highlight scala %}
+FIELD.stddevSamp
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample standard deviation of the numeric field across all input values.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+FIELD.varPop
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the population variance (square of the population standard deviation) of the numeric field across all input values.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight scala %}
+FIELD.varSamp
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample variance (square of the sample standard deviation) of the numeric field across all input values.</p>
+      </td>
+    </tr>
   </tbody>
 </table>
 
@@ -5266,6 +5376,49 @@ MIN(value)
         <p>Returns the minimum value of <i>value</i> across all input values.</p>
       </td>
     </tr>
+    <tr>
+      <td>
+        {% highlight text %}
+STDDEV_POP(value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the population standard deviation of the numeric field across all input values.</p>
+      </td>
+    </tr>
+    
+<tr>
+      <td>
+        {% highlight text %}
+STDDEV_SAMP(value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample standard deviation of the numeric field across all input values.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+VAR_POP(value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the population variance (square of the population standard deviation) of the numeric field across all input values.</p>
+      </td>
+    </tr>
+
+    <tr>
+      <td>
+        {% highlight text %}
+VAR_SAMP(value)
+{% endhighlight %}
+      </td>
+      <td>
+        <p>Returns the sample variance (square of the sample standard deviation) of the numeric field across all input values.</p>
+      </td>
+    </tr>
   </tbody>
 </table>
 
@@ -5376,49 +5529,6 @@ ELEMENT(ARRAY)
         <p>Returns the sole element of an array with a single element. Returns <code>null</code> if the array is empty. Throws an exception if the array has more than one element.</p>
       </td>
     </tr>
-<tr>
-      <td>
-        {% highlight text %}
-STDDEV_POP(value)
-{% endhighlight %}
-      </td>
-      <td>
-        <p>Returns the standard deviation of numeric <i>value</i></p>
-      </td>
-    </tr>
-    
-<tr>
-      <td>
-        {% highlight text %}
-STDDEV_SAMP(value)
-{% endhighlight %}
-      </td>
-      <td>
-        <p>Returns the sample standard deviation of numeric <i>value</i></p>
-      </td>
-    </tr>
-
-    <tr>
-      <td>
-        {% highlight text %}
-VAR_POP(value)
-{% endhighlight %}
-      </td>
-      <td>
-        <p>Returns the variance of numeric <i>value</i></p>
-      </td>
-    </tr>
-
-    <tr>
-      <td>
-        {% highlight text %}
-VAR_SAMP (value)
-{% endhighlight %}
-      </td>
-      <td>
-        <p>Returns the sample variance of numeric <i>value</i></p>
-      </td>
-    </tr>
   </tbody>
 </table>
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
index b1d16b3ea6623..b3de4a4b91fab 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
@@ -161,12 +161,13 @@ trait ImplicitExpressionOperations {
 
   /**
     * Returns the sum of the numeric field across all input values.
+    * If all values are null, null is returned.
     */
   def sum = Sum(expr)
 
   /**
-    * Returns the sum of the values which go into it like [[Sum]].
-    * It differs in that when no non null values are applied zero is returned instead of null.
+    * Returns the sum of the numeric field across all input values.
+    * If all values are null, 0 is returned.
     */
   def sum0 = Sum0(expr)
   
@@ -191,26 +192,24 @@ trait ImplicitExpressionOperations {
   def avg = Avg(expr)
 
   /**
-    * Returns the population standard deviation of an expression.
-    * (the square root of [[VarPop]])
+    * Returns the population standard deviation of an expression (the square root of varPop()).
     */
-  def stddev_pop = StddevPop(expr)
+  def stddevPop = StddevPop(expr)
 
   /**
-    * Returns the sample standard deviation of an expression.
-    * (the square root of [[VarSamp]]).
+    * Returns the sample standard deviation of an expression (the square root of varSamp()).
     */
-  def stddev_samp = StddevSamp(expr)
+  def stddevSamp = StddevSamp(expr)
 
   /**
     * Returns the population standard variance of an expression.
     */
-  def var_pop = VarPop(expr)
+  def varPop = VarPop(expr)
 
   /**
     *  Returns the sample variance of a given expression.
     */
-  def var_samp = VarSamp(expr)
+  def varSamp = VarSamp(expr)
 
   /**
     * Converts a value to a given type.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala
index dd4be46e4d179..40b1a592c314f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/BuiltInMethods.scala
@@ -32,6 +32,8 @@ object BuiltInMethods {
   val POWER = Types.lookupMethod(classOf[Math], "pow", classOf[Double], classOf[Double])
   val POWER_DEC = Types.lookupMethod(
     classOf[MathFunctions], "power", classOf[Double], classOf[JBigDecimal])
+  val POWER_DEC_DEC = Types.lookupMethod(
+    classOf[SqlFunctions], "power", classOf[JBigDecimal], classOf[JBigDecimal])
 
   val LN = Types.lookupMethod(classOf[Math], "log", classOf[Double])
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
index a0d132bf811ac..d9f394bc2bfd3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
@@ -179,6 +179,12 @@ object FunctionGenerator {
     DOUBLE_TYPE_INFO,
     BuiltInMethods.POWER_DEC)
 
+  addSqlFunctionMethod(
+    POWER,
+    Seq(BIG_DEC_TYPE_INFO, BIG_DEC_TYPE_INFO),
+    DOUBLE_TYPE_INFO,
+    BuiltInMethods.POWER_DEC_DEC)
+
   addSqlFunction(
     ABS,
     Seq(DOUBLE_TYPE_INFO),
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
index 1356416bbd4f0..c33f8fc456059 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
@@ -58,10 +58,10 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val START: Keyword = Keyword("start")
   lazy val END: Keyword = Keyword("end")
   lazy val SUM0: Keyword = Keyword("sum0")
-  lazy val STDDEV_POP: Keyword = Keyword("stddev_pop")
-  lazy val STDDEV_SAMP: Keyword = Keyword("stddev_samp")
-  lazy val VAR_POP: Keyword = Keyword("var_pop")
-  lazy val VAR_SAMP: Keyword = Keyword("var_samp")
+  lazy val STDDEV_POP: Keyword = Keyword("stddevPop")
+  lazy val STDDEV_SAMP: Keyword = Keyword("stddevSamp")
+  lazy val VAR_POP: Keyword = Keyword("varPop")
+  lazy val VAR_SAMP: Keyword = Keyword("varSamp")
   lazy val CAST: Keyword = Keyword("cast")
   lazy val NULL: Keyword = Keyword("Null")
   lazy val IF: Keyword = Keyword("?")
@@ -100,8 +100,8 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val ASIN: Keyword = Keyword("asin")
 
   def functionIdent: ExpressionParser.Parser[String] =
-    not(ARRAY) ~ not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~ not(STDDEV_POP) ~
-      not(STDDEV_SAMP) ~ not(VAR_SAMP) ~ not(VAR_POP) ~
+    not(ARRAY) ~ not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~
+      not(SUM0) ~ not(STDDEV_POP) ~ not(STDDEV_SAMP) ~ not(VAR_POP) ~ not(VAR_SAMP) ~
       not(SUM) ~ not(START) ~ not(END)~ not(CAST) ~ not(NULL) ~ not(IF) ~
       not(CURRENT_ROW) ~ not(UNBOUNDED_ROW) ~ not(CURRENT_RANGE) ~ not(UNBOUNDED_RANGE) ~>
       super.ident
@@ -214,6 +214,18 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val suffixSum0: PackratParser[Expression] =
     composite <~ "." ~ SUM0 ~ opt("()") ^^ { e => Sum0(e) }
 
+  lazy val suffixStddevPop: PackratParser[Expression] =
+    composite <~ "." ~ STDDEV_POP ~ opt("()") ^^ { e => StddevPop(e) }
+
+  lazy val suffixStddevSamp: PackratParser[Expression] =
+    composite <~ "." ~ STDDEV_SAMP ~ opt("()") ^^ { e => StddevSamp(e) }
+
+  lazy val suffixVarSamp: PackratParser[Expression] =
+    composite <~ "." ~ VAR_SAMP ~ opt("()") ^^ { e => VarSamp(e) }
+
+  lazy val suffixVarPop: PackratParser[Expression] =
+    composite <~ "." ~ VAR_POP ~ opt("()") ^^ { e => VarPop(e) }
+
   lazy val suffixMin: PackratParser[Expression] =
     composite <~ "." ~ MIN ~ opt("()") ^^ { e => Min(e) }
 
@@ -232,17 +244,6 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val suffixEnd: PackratParser[Expression] =
     composite <~ "." ~ END ~ opt("()") ^^ { e => WindowEnd(e) }
 
-  lazy val suffixStddevPop: PackratParser[Expression] =
-    composite <~ "." ~ STDDEV_POP ~ opt("()") ^^ { e => StddevPop(e) }
-
-  lazy val suffixStddevSamp: PackratParser[Expression] =
-    composite <~ "." ~ STDDEV_SAMP ~ opt("()") ^^ { e => StddevSamp(e) }
-
-  lazy val suffixVarSamp: PackratParser[Expression] =
-    composite <~ "." ~ VAR_SAMP ~ opt("()") ^^ { e => VarSamp(e) }
-
-  lazy val suffixVarPop: PackratParser[Expression] =
-    composite <~ "." ~ VAR_POP ~ opt("()") ^^ { e => VarPop(e) }
   lazy val suffixCast: PackratParser[Expression] =
     composite ~ "." ~ CAST ~ "(" ~ dataType ~ ")" ^^ {
     case e ~ _ ~ _ ~ _ ~ dt ~ _ => Cast(e, dt)
@@ -334,15 +335,15 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
     composite <~ "." ~ FLATTEN ~ opt("()") ^^ { e => Flattening(e) }
 
   lazy val suffixAgg: PackratParser[Expression] =
-    suffixSum | suffixMin | suffixMax | suffixCount | suffixAvg
+    suffixSum0 | suffixStddevPop | suffixStddevSamp | suffixVarPop | suffixVarSamp |
+      suffixSum | suffixMin | suffixMax | suffixCount | suffixAvg
 
   lazy val suffixAsin: PackratParser[Expression] =
     composite <~ "." ~ ASIN ~ opt("()") ^^ { e => Asin(e) }
 
   lazy val suffixed: PackratParser[Expression] =
-    suffixTimeInterval | suffixRowInterval | suffixStart | suffixSum0 | suffixEnd | suffixAgg |
+    suffixTimeInterval | suffixRowInterval | suffixStart | suffixEnd | suffixAgg |
       suffixCast | suffixAs | suffixTrim | suffixTrimWithoutArgs | suffixIf | suffixAsc |
-      suffixStddevPop | suffixStddevSamp | suffixVarPop | suffixVarSamp |
       suffixDesc | suffixToDate | suffixToTimestamp | suffixToTime | suffixExtract |
       suffixFloor | suffixCeil | suffixGet | suffixFlattening | suffixAsin |
       suffixFunctionCall | suffixFunctionCallOneArg // function call must always be at the end
@@ -358,6 +359,18 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val prefixSum0: PackratParser[Expression] =
     SUM0 ~ "(" ~> expression <~ ")" ^^ { e => Sum0(e) }
 
+  lazy val prefixStddevPop: PackratParser[Expression] =
+    STDDEV_POP ~ "(" ~> expression <~ ")" ^^ { e => StddevPop(e) }
+
+  lazy val prefixStddevSamp: PackratParser[Expression] =
+    STDDEV_SAMP ~ "(" ~> expression <~ ")" ^^ { e => StddevSamp(e) }
+
+  lazy val prefixVarPop: PackratParser[Expression] =
+    VAR_POP ~ "(" ~> expression <~ ")" ^^ { e => VarPop(e) }
+
+  lazy val prefixVarSamp: PackratParser[Expression] =
+    VAR_SAMP ~ "(" ~> expression <~ ")" ^^ { e => VarSamp(e) }
+
   lazy val prefixMin: PackratParser[Expression] =
     MIN ~ "(" ~> expression <~ ")" ^^ { e => Min(e) }
 
@@ -376,17 +389,6 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val prefixEnd: PackratParser[Expression] =
     END ~ "(" ~> expression <~ ")" ^^ { e => WindowEnd(e) }
 
-  lazy val prefixStddevPop: PackratParser[Expression] =
-    STDDEV_POP ~ "(" ~> expression <~ ")" ^^ { e => StddevPop(e) }
-
-  lazy val prefixStddevSamp: PackratParser[Expression] =
-    STDDEV_SAMP ~ "(" ~> expression <~ ")" ^^ { e => StddevSamp(e) }
-
-  lazy val prefixVarSamp: PackratParser[Expression] =
-    VAR_SAMP ~ "(" ~> expression <~ ")" ^^ { e => VarSamp(e) }
-
-  lazy val prefixVarPop: PackratParser[Expression] =
-    VAR_POP ~ "(" ~> expression <~ ")" ^^ { e => VarPop(e) }
   lazy val prefixCast: PackratParser[Expression] =
     CAST ~ "(" ~ expression ~ "," ~ dataType ~ ")" ^^ {
     case _ ~ _ ~ e ~ _ ~ dt ~ _ => Cast(e, dt)
@@ -440,15 +442,14 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
     FLATTEN ~ "(" ~> composite <~ ")" ^^ { e => Flattening(e) }
 
   lazy val prefixAgg: PackratParser[Expression] =
-    prefixSum | prefixMin | prefixMax | prefixCount | prefixAvg
+    prefixSum0 | prefixStddevPop | prefixStddevSamp | prefixVarPop | prefixVarSamp |
+      prefixSum | prefixMin | prefixMax | prefixCount | prefixAvg
 
   lazy val prefixAsin: PackratParser[Expression] =
     ASIN ~ "(" ~> composite <~ ")" ^^ { e => Asin(e) }
 
   lazy val prefixed: PackratParser[Expression] =
-    prefixArray | prefixSum | prefixSum0 | prefixAgg | prefixStart | prefixEnd |
-      prefixCast | prefixAs | prefixTrim | prefixStddevPop | prefixStddevSamp | prefixVarSamp |
-      prefixVarPop |
+    prefixArray | prefixAgg | prefixStart | prefixEnd | prefixCast | prefixAs | prefixTrim |
       prefixTrimWithoutArgs | prefixIf | prefixExtract | prefixFloor | prefixCeil | prefixGet |
       prefixFlattening | prefixAsin | prefixFunctionCall |
       prefixFunctionCallOneArg // function call must always be at the end
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
index 82f4428507fb3..72e7e4b515a75 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
@@ -19,13 +19,13 @@ package org.apache.flink.table.expressions
 
 import org.apache.calcite.rex.RexNode
 import org.apache.calcite.sql.SqlAggFunction
-import org.apache.calcite.sql.fun._
 import org.apache.calcite.sql.SqlKind._
+import org.apache.calcite.sql.fun._
 import org.apache.calcite.tools.RelBuilder
 import org.apache.calcite.tools.RelBuilder.AggCall
-import org.apache.flink.table.typeutils.TypeCheckUtils
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
 import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.typeutils.TypeCheckUtils
 
 abstract sealed class Aggregation extends UnaryExpression {
 
@@ -75,8 +75,11 @@ case class Sum0(child: Expression) extends Aggregation {
 
   override private[flink] def resultType = child.resultType
 
-  override private[flink] def validateInput =
+  override private[flink] def validateInput() =
     TypeCheckUtils.assertNumericExpr(child.resultType, "sum0")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) =
+    new SqlSumEmptyIsZeroAggFunction()
 }
 
 case class Min(child: Expression) extends Aggregation {
@@ -153,8 +156,11 @@ case class StddevPop(child: Expression) extends Aggregation {
 
   override private[flink] def resultType = child.resultType
 
-  override private[flink] def validateInput =
+  override private[flink] def validateInput() =
     TypeCheckUtils.assertNumericExpr(child.resultType, "stddev_pop")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) =
+    new SqlAvgAggFunction(STDDEV_POP)
 }
 
 case class StddevSamp(child: Expression) extends Aggregation {
@@ -166,8 +172,11 @@ case class StddevSamp(child: Expression) extends Aggregation {
 
   override private[flink] def resultType = child.resultType
 
-  override private[flink] def validateInput =
+  override private[flink] def validateInput() =
     TypeCheckUtils.assertNumericExpr(child.resultType, "stddev_samp")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) =
+    new SqlAvgAggFunction(STDDEV_SAMP)
 }
 
 case class VarPop(child: Expression) extends Aggregation {
@@ -179,8 +188,11 @@ case class VarPop(child: Expression) extends Aggregation {
 
   override private[flink] def resultType = child.resultType
 
-  override private[flink] def validateInput =
+  override private[flink] def validateInput() =
     TypeCheckUtils.assertNumericExpr(child.resultType, "var_pop")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) =
+    new SqlAvgAggFunction(VAR_POP)
 }
 
 case class VarSamp(child: Expression) extends Aggregation {
@@ -192,6 +204,9 @@ case class VarSamp(child: Expression) extends Aggregation {
 
   override private[flink] def resultType = child.resultType
 
-  override private[flink] def validateInput =
+  override private[flink] def validateInput() =
     TypeCheckUtils.assertNumericExpr(child.resultType, "var_samp")
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) =
+    new SqlAvgAggFunction(VAR_SAMP)
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala
index 6a24fbe821a75..b7f1744868d08 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunction.scala
@@ -20,14 +20,17 @@ package org.apache.flink.table.functions.aggfunctions
 import java.math.BigDecimal
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.table.functions.Accumulator
 
+/**
+  * Base class for built-in Sum0 aggregate function. If all values are null, 0 is returned.
+  *
+  * @tparam T the type for the aggregation result
+  */
 abstract class Sum0AggFunction[T: Numeric] extends SumAggFunction[T] {
 
-  override def getValue(accumulator: Accumulator): T = {
-    val a = accumulator.asInstanceOf[SumAccumulator[T]]
-    if (a.f1) {
-      a.f0
+  override def getValue(acc: SumAccumulator[T]): T = {
+    if (acc.f1) {
+      acc.f0
     } else {
       0.asInstanceOf[T]
     }
@@ -81,11 +84,11 @@ class DoubleSum0AggFunction extends Sum0AggFunction[Double] {
   */
 class DecimalSum0AggFunction extends DecimalSumAggFunction {
 
-  override def getValue(accumulator: Accumulator): BigDecimal = {
-    if (!accumulator.asInstanceOf[DecimalSumAccumulator].f1) {
-      0.asInstanceOf[BigDecimal]
+  override def getValue(acc: DecimalSumAccumulator): BigDecimal = {
+    if (acc.f1) {
+      acc.f0
     } else {
-      accumulator.asInstanceOf[DecimalSumAccumulator].f0
+      BigDecimal.ZERO
     }
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunction.scala
new file mode 100644
index 0000000000000..0550c198b09c6
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunction.scala
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.functions.aggfunctions
+
+import java.math.BigDecimal
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+
+/**
+  * Base class for built-in Sum0 with retract aggregate function.
+  * If all values are null, 0 is returned.
+  *
+  * @tparam T the type for the aggregation result
+  */
+abstract class Sum0WithRetractAggFunction[T: Numeric] extends SumWithRetractAggFunction[T] {
+
+  override def getValue(acc: SumWithRetractAccumulator[T]): T = {
+    if (acc.f1 > 0) {
+      acc.f0
+    } else {
+      0.asInstanceOf[T]
+    }
+  }
+}
+
+/**
+  * Built-in Byte Sum0 with retract aggregate function
+  */
+class ByteSum0WithRetractAggFunction extends Sum0WithRetractAggFunction[Byte] {
+  override def getValueTypeInfo = BasicTypeInfo.BYTE_TYPE_INFO
+}
+
+/**
+  * Built-in Short Sum0 with retract aggregate function
+  */
+class ShortSum0WithRetractAggFunction extends Sum0WithRetractAggFunction[Short] {
+  override def getValueTypeInfo = BasicTypeInfo.SHORT_TYPE_INFO
+}
+
+/**
+  * Built-in Int Sum0 with retract aggregate function
+  */
+class IntSum0WithRetractAggFunction extends Sum0WithRetractAggFunction[Int] {
+  override def getValueTypeInfo = BasicTypeInfo.INT_TYPE_INFO
+}
+
+/**
+  * Built-in Long Sum0 with retract aggregate function
+  */
+class LongSum0WithRetractAggFunction extends Sum0WithRetractAggFunction[Long] {
+  override def getValueTypeInfo = BasicTypeInfo.LONG_TYPE_INFO
+}
+
+/**
+  * Built-in Float Sum0 with retract aggregate function
+  */
+class FloatSum0WithRetractAggFunction extends Sum0WithRetractAggFunction[Float] {
+  override def getValueTypeInfo = BasicTypeInfo.FLOAT_TYPE_INFO
+}
+
+/**
+  * Built-in Double Sum0 with retract aggregate function
+  */
+class DoubleSum0WithRetractAggFunction extends Sum0WithRetractAggFunction[Double] {
+  override def getValueTypeInfo = BasicTypeInfo.DOUBLE_TYPE_INFO
+}
+
+/**
+  * Built-in Big Decimal Sum0 with retract aggregate function
+  */
+class DecimalSum0WithRetractAggFunction extends DecimalSumWithRetractAggFunction {
+
+  override def getValue(acc: DecimalSumWithRetractAccumulator): BigDecimal = {
+    if (acc.f1 > 0) {
+      acc.f0
+    } else {
+      BigDecimal.ZERO
+    }
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala
index 55996acaddb63..43fc7ff8dec30 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/SumAggFunction.scala
@@ -76,7 +76,7 @@ abstract class SumAggFunction[T: Numeric] extends AggregateFunction[T, SumAccumu
     acc.f1 = false
   }
 
-  def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType: TypeInformation[_] = {
     new TupleTypeInfo(
       (new SumAccumulator).getClass,
       getValueTypeInfo,
@@ -152,10 +152,10 @@ class DecimalSumAggFunction extends AggregateFunction[BigDecimal, DecimalSumAccu
   }
 
   override def getValue(acc: DecimalSumAccumulator): BigDecimal = {
-    if (!acc.f1) {
-      null.asInstanceOf[BigDecimal]
-    } else {
+    if (acc.f1) {
       acc.f0
+    } else {
+      null.asInstanceOf[BigDecimal]
     }
   }
 
@@ -175,7 +175,7 @@ class DecimalSumAggFunction extends AggregateFunction[BigDecimal, DecimalSumAccu
     acc.f1 = false
   }
 
-  def getAccumulatorType(): TypeInformation[_] = {
+  def getAccumulatorType: TypeInformation[_] = {
     new TupleTypeInfo(
       (new DecimalSumAccumulator).getClass,
       BasicTypeInfo.BIG_DEC_TYPE_INFO,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index f1bb644c720d1..dbfd3ced562fc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -511,7 +511,7 @@ case class Join(
 }
 
 case class CatalogNode(
-    tablePath: Array[String],
+    tablePath: Seq[String],
     rowType: RelDataType) extends LeafNode {
 
   val output: Seq[Attribute] = rowType.getFieldList.asScala.map { field =>
@@ -519,7 +519,7 @@ case class CatalogNode(
   }
 
   override protected[logical] def construct(relBuilder: RelBuilder): RelBuilder = {
-    relBuilder.scan(tablePath.toIterable.asJava)
+    relBuilder.scan(tablePath.asJava)
   }
 
   override def validate(tableEnv: TableEnvironment): LogicalNode = this
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala
index 0d5fdd5992e02..e1e93c7c583b9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala
@@ -26,9 +26,12 @@ import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.calcite.rel.core.{Aggregate, AggregateCall}
 import org.apache.calcite.rel.logical.LogicalAggregate
 import org.apache.calcite.rel.metadata.RelMetadataQuery
+import org.apache.calcite.sql.SqlKind
 import org.apache.calcite.util.ImmutableBitSet
 import org.apache.flink.table.plan.nodes.FlinkConventions
 
+import scala.collection.JavaConversions._
+
 class FlinkLogicalAggregate(
     cluster: RelOptCluster,
     traitSet: RelTraitSet,
@@ -54,7 +57,7 @@ class FlinkLogicalAggregate(
     val child = this.getInput
     val rowCnt = metadata.getRowCount(child)
     val rowSize = this.estimateRowSize(child.getRowType)
-    val aggCnt = this.aggCalls.size
+    val aggCnt = this.getAggCallList.size
     planner.getCostFactory.makeCost(rowCnt, rowCnt * aggCnt, rowCnt * rowSize)
   }
 }
@@ -68,7 +71,15 @@ private class FlinkLogicalAggregateConverter
 
   override def matches(call: RelOptRuleCall): Boolean = {
     val agg = call.rel(0).asInstanceOf[LogicalAggregate]
-    !agg.containsDistinctCall()
+
+    // we do not support these functions natively
+    // they have to be converted using the AggregateReduceFunctionsRule
+    val supported = agg.getAggCallList.map(_.getAggregation.getKind).forall {
+      case SqlKind.STDDEV_POP | SqlKind.STDDEV_SAMP | SqlKind.VAR_POP | SqlKind.VAR_SAMP => false
+      case _ => true
+    }
+
+    !agg.containsDistinctCall() && supported
   }
 
   override def convert(rel: RelNode): RelNode = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
index 6ebbae491c693..838cb22b6d6a2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
@@ -74,7 +74,7 @@ object FlinkRuleSets {
     // expand distinct aggregate to normal aggregate with groupby
     AggregateExpandDistinctAggregatesRule.JOIN,
 
-    //aggregate reduce rule (deviation/variance functions)
+    // reduce aggregate functions like AVG, STDDEV_POP etc.
     AggregateReduceFunctionsRule.INSTANCE,
 
     // remove unnecessary sort rule
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala
index faaeb97286570..b4d5bc9cc61c9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateRule.scala
@@ -21,7 +21,6 @@ package org.apache.flink.table.plan.rules.dataSet
 import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet}
 import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.convert.ConverterRule
-import org.apache.calcite.sql.SqlKind
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.{DataSetAggregate, DataSetUnion}
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalAggregate
@@ -55,12 +54,7 @@ class DataSetAggregateRule
     // check if we have distinct aggregates
     val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
 
-    val supported = agg.getAggCallList.map(_.getAggregation.getKind).forall {
-      case SqlKind.STDDEV_POP | SqlKind.STDDEV_SAMP | SqlKind.VAR_POP | SqlKind.VAR_SAMP => false
-      case _ => true
-    }
-
-    !distinctAggs && supported
+    !distinctAggs
   }
 
   override def convert(rel: RelNode): RelNode = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala
index 9636980465068..f8b7fd781a1bd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetAggregateWithNullValuesRule.scala
@@ -23,7 +23,6 @@ import org.apache.calcite.plan._
 import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.calcite.rex.RexLiteral
-import org.apache.calcite.sql.SqlKind
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.DataSetAggregate
 import org.apache.flink.table.plan.nodes.logical.{FlinkLogicalAggregate, FlinkLogicalUnion, FlinkLogicalValues}
@@ -53,12 +52,7 @@ class DataSetAggregateWithNullValuesRule
     // check if we have distinct aggregates
     val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
 
-    val supported = agg.getAggCallList.map(_.getAggregation.getKind).forall {
-      case SqlKind.STDDEV_POP | SqlKind.STDDEV_SAMP | SqlKind.VAR_POP | SqlKind.VAR_SAMP => false
-      case _ => true
-    }
-
-    !distinctAggs && supported
+    !distinctAggs
   }
 
   override def convert(rel: RelNode): RelNode = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index a82f38312ce16..2950a78a8f933 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -1092,7 +1092,7 @@ object AggregateUtil {
         .getSqlTypeName
       aggregateCall.getAggregation match {
 
-        case _: SqlSumAggFunction | _: SqlSumEmptyIsZeroAggFunction =>
+        case _: SqlSumAggFunction =>
           if (needRetraction) {
             aggregates(index) = sqlTypeName match {
               case TINYINT =>
@@ -1133,6 +1133,47 @@ object AggregateUtil {
             }
           }
 
+        case _: SqlSumEmptyIsZeroAggFunction =>
+          if (needRetraction) {
+            aggregates(index) = sqlTypeName match {
+              case TINYINT =>
+                new ByteSum0WithRetractAggFunction
+              case SMALLINT =>
+                new ShortSum0WithRetractAggFunction
+              case INTEGER =>
+                new IntSum0WithRetractAggFunction
+              case BIGINT =>
+                new LongSum0WithRetractAggFunction
+              case FLOAT =>
+                new FloatSum0WithRetractAggFunction
+              case DOUBLE =>
+                new DoubleSum0WithRetractAggFunction
+              case DECIMAL =>
+                new DecimalSum0WithRetractAggFunction
+              case sqlType: SqlTypeName =>
+                throw new TableException("Sum0 aggregate does no support type:" + sqlType)
+            }
+          } else {
+            aggregates(index) = sqlTypeName match {
+              case TINYINT =>
+                new ByteSum0AggFunction
+              case SMALLINT =>
+                new ShortSum0AggFunction
+              case INTEGER =>
+                new IntSum0AggFunction
+              case BIGINT =>
+                new LongSum0AggFunction
+              case FLOAT =>
+                new FloatSum0AggFunction
+              case DOUBLE =>
+                new DoubleSum0AggFunction
+              case DECIMAL =>
+                new DecimalSum0AggFunction
+              case sqlType: SqlTypeName =>
+                throw new TableException("Sum0 aggregate does no support type:" + sqlType)
+            }
+          }
+
         case _: SqlAvgAggFunction =>
           aggregates(index) = sqlTypeName match {
             case TINYINT =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
index cb37ce483ce58..1022e4d59eb6b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
@@ -170,10 +170,10 @@ object FunctionCatalog {
     "min" -> classOf[Min],
     "sum" -> classOf[Sum],
     "sum0" -> classOf[Sum0],
-    "stddev_pop" -> classOf[StddevPop],
-    "stddev_samp" -> classOf[StddevSamp],
-    "var_pop" -> classOf[VarPop],
-    "var_samp" -> classOf[VarSamp],
+    "stddevPop" -> classOf[StddevPop],
+    "stddevSamp" -> classOf[StddevSamp],
+    "varPop" -> classOf[VarPop],
+    "varSamp" -> classOf[VarSamp],
 
     // string functions
     "charLength" -> classOf[CharLength],
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
index a5d202122934b..114226cb0d20e 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/sql/SqlITCase.java
@@ -20,7 +20,6 @@
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.typeutils.MapTypeInfo;
@@ -37,12 +36,9 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
-import scala.collection.JavaConversions;
-import scala.collection.mutable.Buffer;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -140,7 +136,7 @@ public void testJoin() throws Exception {
 		DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
 
 		tableEnv.registerDataSet("t1", ds1, "a, b, c");
-		tableEnv.registerDataSet("t2", ds2, "d, e, f, g, h");
+		tableEnv.registerDataSet("t2",ds2, "d, e, f, g, h");
 
 		String sqlQuery = "SELECT c, g FROM t1, t2 WHERE b = e";
 		Table result = tableEnv.sql(sqlQuery);
@@ -160,7 +156,9 @@ public void testMap() throws Exception {
 		rows.add(new Tuple2<>(1, Collections.singletonMap("foo", "bar")));
 		rows.add(new Tuple2<>(2, Collections.singletonMap("foo", "spam")));
 
-		TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));
+		TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(
+			BasicTypeInfo.INT_TYPE_INFO,
+			new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));
 
 		DataSet<Tuple2<Integer, Map<String, String>>> ds1 = env.fromCollection(rows, ty);
 		tableEnv.registerDataSet("t1", ds1, "a, b");
@@ -173,47 +171,4 @@ public void testMap() throws Exception {
 		String expected = "bar\n" + "spam\n";
 		compareResultAsText(results, expected);
 	}
-
-	@Test
-	public void testDeviationAggregation() throws Exception {
-		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-		BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
-
-		DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
-		tableEnv.registerDataSet("AggTable", ds, "x, y, z");
-
-		Buffer<String> columnForAgg = JavaConversions.asScalaBuffer(Arrays.asList("x, y".split(",")));
-
-		String sqlQuery = getSelectQueryFromTemplate("AVG(?),STDDEV_POP(?),STDDEV_SAMP(?),VAR_POP(?),VAR_SAMP(?)", columnForAgg, "AggTable");
-		Table result = tableEnv.sql(sqlQuery);
-
-		String sqlQuery1 = getSelectQueryFromTemplate("SUM(?)/COUNT(?), " +
-			"SQRT( (SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / COUNT(?)), " +
-			"SQRT( (SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / CASE COUNT(?) WHEN 1 THEN NULL ELSE COUNT(?) - 1 END), " +
-			"(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / COUNT(?), " +
-			"(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / CASE COUNT(?) WHEN 1 THEN NULL ELSE COUNT(?) - 1 END", columnForAgg, "AggTable");
-
-		Table expected = tableEnv.sql(sqlQuery1);
-
-		DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
-		List<Row> results = resultSet.collect();
-
-		DataSet<Row> expectedResultSet = tableEnv.toDataSet(expected, Row.class);
-		String expectedResults = expectedResultSet.map(new MapFunction<Row, Object>() {
-			@Override
-			public Object map(Row value) throws Exception {
-				StringBuilder stringBuffer = new StringBuilder();
-
-				int arityCount = value.getArity();
-
-				for (int i = 0; i < arityCount; i++) {
-					Object product = value.getField(i);
-					stringBuffer.append(Double.valueOf(product.toString()).intValue()).append(",");
-				}
-				return stringBuffer.substring(0, stringBuffer.length() - 1);
-			}
-		}).collect().get(0).toString();
-
-		compareResultAsText(results, expectedResults);
-	}
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
index 4fbd73475c5bf..600c15be7ba5c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
@@ -295,206 +295,6 @@ class AggregationsITCase(
     TestBaseUtils.compareResultAsText(results3.asJava, expected3)
   }
 
-
-
-  @Test
-  def testSqrtOfAggregatedSet(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val ds = env.fromElements((1.0f, 1), (2.0f, 2)).toTable(tEnv)
-
-    tEnv.registerTable("MyTable", ds)
-
-    val sqlQuery = "SELECT " +
-      "SQRT((SUM(a * a) - SUM(a) * SUM(a) / COUNT(a)) / COUNT(a)) " +
-      "from (select _1 as a from MyTable)"
-
-    val expected = "0.5"
-    val results = tEnv.sql(sqlQuery).toDataSet[Row].collect()
-    TestBaseUtils.compareResultAsText(results.asJava, expected)
-  }
-
-  @Test
-  def testStddevPopAggregate(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val ds = env.fromElements(
-      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
-      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
-    tEnv.registerTable("myTable", ds)
-    val columns = Array("_1","_2","_3","_4","_5","_6")
-
-    val sqlQuery = getSelectQueryFromTemplate("STDDEV_POP(?)")(columns,"myTable")
-
-    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
-    val expectedResult = "0,0,0,0,0.5,0.5"
-    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
-  }
-
-  @Test
-  def testStddevPopAggregateWithOtherAggreagteSUM0(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val ds = env.fromElements(
-      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
-      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
-    tEnv.registerTable("myTable", ds)
-    val columns = Array("_1","_2","_3","_4","_5","_6")
-
-    val sqlQuery = getSelectQueryFromTemplate("STDDEV_POP(?), " +
-                                  "$sum0(?), " +
-                                  "avg(?), " +
-                                  "max(?), " +
-                                  "min(?), " +
-                                  "count(?)" ) (columns,"myTable")
-
-    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
-
-    val expectedResult =
-        "0,3,1,2,1,2," +
-        "0,3,1,2,1,2," +
-        "0,3,1,2,1,2," +
-        "0,3,1,2,1,2," +
-        "0.5,3.0,1.5,2.0,1.0,2," +
-        "0.5,3.0,1.5,2.0,1.0,2"
-    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
-  }
-
-  @Test
-  def testStddevPopAggregateWithOtherAggreagte(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val ds = env.fromElements(
-      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
-      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
-    tEnv.registerTable("myTable", ds)
-    val columns = Array("_1","_2","_3","_4","_5","_6")
-
-    val sqlQuery = getSelectQueryFromTemplate("STDDEV_POP(?), " +
-                                  "sum(?), " +
-                                  "avg(?), " +
-                                  "max(?), " +
-                                  "min(?), " +
-                                  "count(?)" )(columns,"myTable")
-
-    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
-
-    val expectedResult =
-      "0,3,1,2,1,2," +
-        "0,3,1,2,1,2," +
-        "0,3,1,2,1,2," +
-        "0,3,1,2,1,2," +
-        "0.5,3.0,1.5,2.0,1.0,2," +
-        "0.5,3.0,1.5,2.0,1.0,2"
-    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
-  }
-
-  @Test
-  def testStddevSampAggregate(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val ds1 = env.fromElements(
-      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
-      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
-    tEnv.registerTable("myTable", ds1)
-    val columns = Seq("_1","_2","_3","_4","_5","_6")
-
-    val sqlQuery = getSelectQueryFromTemplate("STDDEV_SAMP(?)")(columns,"myTable")
-    val sqlExpectedQuery = getSelectQueryFromTemplate(
-      "SQRT((SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / " +
-        "CASE " +
-        "COUNT(?) WHEN 1 THEN NULL " +
-        "ELSE COUNT(?) - 1 " +
-        "END)")(columns,"myTable")
-
-    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
-      .head
-      .toString
-      .split(",").map(x=>"%.5f".format(x.toFloat))
-
-    val expectedResult = tEnv.sql(sqlExpectedQuery).toDataSet[Row].collect()
-      .head
-      .toString
-      .split(",").map(x=>"%.5f".format(x.toFloat))
-
-    Assert.assertEquals(expectedResult.mkString(","), actualResult.mkString(","))
-  }
-
-  @Test
-  def testVarPopAggregate(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val ds = env.fromElements(
-      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
-      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
-    tEnv.registerTable("myTable", ds)
-    val columns = Seq("_1","_2","_3","_4","_5","_6")
-
-    val sqlQuery = getSelectQueryFromTemplate("var_pop(?)")(columns,"myTable")
-    val sqlExpectedQuery = getSelectQueryFromTemplate(
-      "(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / COUNT(?)")(columns,"myTable")
-
-    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
-    val expectedResult = tEnv.sql(sqlExpectedQuery)
-      .toDataSet[Row]
-      .collect().head
-      .toString
-    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
-  }
-
-  @Test
-  def testVarSampAggregate(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val ds = env.fromElements(
-      (1: Byte, 1 : Short, 1, 1L, 1F, 1D),
-      (2: Byte, 2 : Short, 2, 2L, 2F, 2D)).toTable(tEnv)
-    tEnv.registerTable("myTable", ds)
-    val columns = Seq("_1","_2","_3","_4","_5","_6")
-
-    val sqlQuery = getSelectQueryFromTemplate("var_samp(?)")(columns,"myTable")
-    val sqlExpectedQuery = getSelectQueryFromTemplate(
-      "(SUM(? * ?) - SUM(?) * SUM(?) / COUNT(?)) / CASE COUNT(?) WHEN 1 THEN NULL " +
-        "ELSE COUNT(?) - 1 END")(columns,"myTable")
-
-    val actualResult = tEnv.sql(sqlQuery).toDataSet[Row].collect()
-    val expectedResult = tEnv.sql(sqlExpectedQuery)
-      .toDataSet[Row]
-      .collect().head
-      .toString
-    TestBaseUtils.compareOrderedResultAsText(actualResult.asJava, expectedResult)
-  }
-
-  @Test
-  def testSumNullElements(): Unit = {
-
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val sqlQuery = getSelectQueryFromTemplate("$sum0(?)")(
-      Seq("_1","_2","_3","_4","_5","_6"),
-      "(select * from MyTable where _1 = 4)"
-    )
-
-    val ds = env.fromElements(
-      (1: Byte, 2L,1D,1F,1,1:Short ),
-      (2: Byte, 2L,1D,1F,1,1:Short ))
-    tEnv.registerDataSet("MyTable", ds)
-
-    val result = tEnv.sql(sqlQuery)
-
-    val expected = "null,null,null,null,null,null"
-    val results = result.toDataSet[Row].collect()
-    TestBaseUtils.compareResultAsText(results.asJava, expected)
-  }
-
   @Test
   def testTumbleWindowAggregate(): Unit = {
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
index 34ae346e3415e..de313ff7f7571 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
@@ -133,7 +133,7 @@ class QueryDecorrelationTest extends TableTestBase {
               unaryNode(
                 "DataSetCalc",
                 batchTableNode(0),
-                term("select", "deptno", "salary"),
+                term("select", "salary", "deptno"),
                 term("where", "IS NOT NULL(deptno)")
               ),
               term("groupBy", "deptno"),
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
index 050f5f4e9f131..ba6809dad1535 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
@@ -18,14 +18,16 @@
 
 package org.apache.flink.table.api.scala.batch.table
 
+import java.math.BigDecimal
+
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.scala.batch.utils.TableProgramsCollectionTestBase
-import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
-import org.apache.flink.table.api.scala._
 import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.types.Row
 import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.scala.batch.utils.TableProgramsCollectionTestBase
+import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
 import org.apache.flink.test.util.TestBaseUtils
+import org.apache.flink.types.Row
 import org.junit._
 import org.junit.runner.RunWith
 import org.junit.runners.Parameterized
@@ -44,10 +46,10 @@ class AggregationsITCase(
     val tEnv = TableEnvironment.getTableEnvironment(env, config)
 
     val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
-      .select('_1.sum, '_1.min, '_1.max, '_1.count, '_1.avg)
+      .select('_1.sum, '_1.sum0, '_1.min, '_1.max, '_1.count, '_1.avg)
 
     val results = t.toDataSet[Row].collect()
-    val expected = "231,1,21,21,11"
+    val expected = "231,231,1,21,21,11"
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
@@ -341,78 +343,32 @@ class AggregationsITCase(
 
   @Test
   def testAnalyticAggregation(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
-      .select('_1.stddev_pop, '_1.stddev_samp, '_1.var_pop, '_1.var_samp)
-    val results = t.toDataSet[Row].collect()
-    val expected = "6,6,36,38"
-    TestBaseUtils.compareResultAsText(results.asJava, expected)
-  }
-
-  @Test
-  def testSQLStyleAnalyticAggregation(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
-      .select(
-        """stddev_pop(a) as a1, a.stddev_pop as a2,
-          |stddev_samp (a) as b1, a.stddev_samp as b2,
-          |var_pop (a) as c1, a.var_pop as c2,
-          |var_samp (a) as d1, a.var_samp as d2
-        """.stripMargin)
-    val expected = "6,6,6,6,36,36,38,38"
-    val results = t.toDataSet[Row].collect()
-    TestBaseUtils.compareResultAsText(results.asJava, expected)
-  }
-
-  @Test
-  def testWorkingAnalyticAggregationDataTypes(): Unit = {
     val env = ExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
     val ds = env.fromElements(
-      (1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d),
-      (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d)).toTable(tEnv)
-    val res = ds.select('_1.stddev_pop, '_2.stddev_pop, '_3.stddev_pop,
-                        '_4.stddev_pop, '_5.stddev_pop, '_6.stddev_pop,
-                        '_1.stddev_samp, '_2.stddev_samp, '_3.stddev_samp,
-                        '_4.stddev_samp, '_5.stddev_samp, '_6.stddev_samp,
-                        '_1.var_pop, '_2.var_pop, '_3.var_pop,
-                        '_4.var_pop, '_5.var_pop, '_6.var_pop,
-                        '_1.var_samp, '_2.var_samp, '_3.var_samp,
-                        '_4.var_samp, '_5.var_samp, '_6.var_samp)
+      (1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, BigDecimal.ONE),
+      (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, new BigDecimal(2))).toTable(tEnv)
+    val res = ds.select(
+      '_1.stddevPop, '_2.stddevPop, '_3.stddevPop, '_4.stddevPop, '_5.stddevPop,
+      '_6.stddevPop, '_7.stddevPop,
+      '_1.stddevSamp, '_2.stddevSamp, '_3.stddevSamp, '_4.stddevSamp, '_5.stddevSamp,
+      '_6.stddevSamp, '_7.stddevSamp,
+      '_1.varPop, '_2.varPop, '_3.varPop, '_4.varPop, '_5.varPop,
+      '_6.varPop, '_7.varPop,
+      '_1.varSamp, '_2.varSamp, '_3.varSamp, '_4.varSamp, '_5.varSamp,
+      '_6.varSamp, '_7.varSamp)
     val expected =
         "0,0,0," +
-        "0,0.5,0.5," +
+        "0,0.5,0.5,0.5," +
         "1,1,1," +
-        "1,0.70710677,0.7071067811865476," +
+        "1,0.70710677,0.7071067811865476,0.7071067811865476," +
         "0,0,0," +
-        "0,0.25,0.25," +
+        "0,0.25,0.25,0.25," +
         "1,1,1," +
-        "1,0.5,0.5"
+        "1,0.5,0.5,0.5"
     val results = res.toDataSet[Row].collect()
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
-
-  @Test
-  def testPojoAnalyticAggregation(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    val input = env.fromElements(
-      MyWC("hello", 1),
-      MyWC("hello", 8),
-      MyWC("ciao", 3),
-      MyWC("hola", 1),
-      MyWC("hola", 8))
-    val expr = input.toTable(tEnv)
-    val result = expr
-      .groupBy('word)
-      .select('word, 'frequency.stddev_pop)
-      .toDataSet[MyWC]
-    val mappedResult = result.map(w => (w.word, w.frequency)).collect()
-    val expected = "(hola,3)\n(ciao,0)\n(hello,3)"
-    TestBaseUtils.compareResultAsText(mappedResult.asJava, expected)
-  }
 }
 
 case class WC(word: String, frequency: Long)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala
index ace53d9c0d314..528ffd8abc44e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala
@@ -20,12 +20,12 @@ package org.apache.flink.table.api.scala.batch.table.stringexpr
 
 import org.apache.flink.api.scala._
 import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.TableEnvironment
-import org.apache.flink.table.api.scala.batch.utils.LogicalPlanFormatUtils
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.utils.TableTestBase
 import org.junit._
 
-class AggregationsStringExpressionTest {
+class AggregationsStringExpressionTest extends TableTestBase {
 
   @Test
   def testAggregationTypes(): Unit = {
@@ -35,15 +35,10 @@ class AggregationsStringExpressionTest {
 
     val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
 
-    val t1 = t.select('_1.sum, '_1.min, '_1.max, '_1.count, '_1.avg)
-    val t2 = t.select("_1.sum, _1.min, _1.max, _1.count, _1.avg")
-
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
+    val t1 = t.select('_1.sum, '_1.sum0, '_1.min, '_1.max, '_1.count, '_1.avg)
+    val t2 = t.select("_1.sum, _1.sum0, _1.min, _1.max, _1.count, _1.avg")
 
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -59,12 +54,7 @@ class AggregationsStringExpressionTest {
     val t1 = t.select('_1.avg, '_2.avg, '_3.avg, '_4.avg, '_5.avg, '_6.avg, '_7.count)
     val t2 = t.select("_1.avg, _2.avg, _3.avg, _4.avg, _5.avg, _6.avg, _7.count")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -80,12 +70,7 @@ class AggregationsStringExpressionTest {
     val t1 = t.select('_1.avg, '_1.sum, '_1.count, '_2.avg, '_2.sum)
     val t2 = t.select("_1.avg, _1.sum, _1.count, _2.avg, _2.sum")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -99,12 +84,7 @@ class AggregationsStringExpressionTest {
     val t1 = t.select(('_1 + 2).avg + 2, '_2.count + 5)
     val t2 = t.select("(_1 + 2).avg + 2, _2.count + 5")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -118,12 +98,7 @@ class AggregationsStringExpressionTest {
     val t1 = t.select('_1.count, '_2.count)
     val t2 = t.select("_1.count, _2.count")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -142,12 +117,7 @@ class AggregationsStringExpressionTest {
     val t2 = t.select("_1, _2, _3")
       .select("_1.avg, _2.sum, _3.count")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -160,10 +130,7 @@ class AggregationsStringExpressionTest {
     val distinct = ds.select('b).distinct()
     val distinct2 = ds.select("b").distinct()
 
-    val lPlan1 = distinct.logicalPlan
-    val lPlan2 = distinct2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match", lPlan1, lPlan2)
+    verifyTableEquals(distinct, distinct2)
   }
 
   @Test
@@ -193,12 +160,7 @@ class AggregationsStringExpressionTest {
     val t1 = t.groupBy('b).select('b, 'a.sum)
     val t2 = t.groupBy("b").select("b, a.sum")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -212,12 +174,7 @@ class AggregationsStringExpressionTest {
     val t1 = t.groupBy('b).select('a.sum)
     val t2 = t.groupBy("b").select("a.sum")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -240,12 +197,7 @@ class AggregationsStringExpressionTest {
       .groupBy("b, d")
       .select("b")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -264,12 +216,7 @@ class AggregationsStringExpressionTest {
       .groupBy("four, a")
       .select("four, b.sum")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -287,12 +234,7 @@ class AggregationsStringExpressionTest {
       .groupBy("b, four")
       .select("four, a.sum")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -308,12 +250,7 @@ class AggregationsStringExpressionTest {
     val t2 = t.groupBy("e, b % 3")
       .select("c.min, e, a.avg, d.count")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
-
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+    verifyTableEquals(t1, t2)
   }
 
   @Test
@@ -331,12 +268,26 @@ class AggregationsStringExpressionTest {
       .select("b, a.sum")
       .where("b = 2")
 
-    val lPlan1 = t1.logicalPlan
-    val lPlan2 = t2.logicalPlan
+    verifyTableEquals(t1, t2)
+  }
 
-    Assert.assertEquals("Logical Plans do not match",
-      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
-      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+  @Test
+  def testAnalyticAggregation(): Unit = {
+    val util = batchTestUtil()
+    val t = util.addTable[(Int, Long, Float, Double)]('_1, '_2, '_3, '_4)
+
+    val resScala = t.select(
+      '_1.stddevPop, '_2.stddevPop, '_3.stddevPop, '_4.stddevPop,
+      '_1.stddevSamp, '_2.stddevSamp, '_3.stddevSamp, '_4.stddevSamp,
+      '_1.varPop, '_2.varPop, '_3.varPop, '_4.varPop,
+      '_1.varSamp, '_2.varSamp, '_3.varSamp, '_4.varSamp)
+    val resJava = t.select("""
+      _1.stddevPop, _2.stddevPop, _3.stddevPop, _4.stddevPop,
+      _1.stddevSamp, _2.stddevSamp, _3.stddevSamp, _4.stddevSamp,
+      _1.varPop, _2.varPop, _3.varPop, _4.varPop,
+      _1.varSamp, _2.varSamp, _3.varSamp, _4.varSamp""")
+
+    verifyTableEquals(resScala, resJava)
   }
 
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/LogicalPlanFormatUtils.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/LogicalPlanFormatUtils.scala
index 435d6b9c89f64..4a69271bc57b9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/LogicalPlanFormatUtils.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/LogicalPlanFormatUtils.scala
@@ -28,8 +28,13 @@ object LogicalPlanFormatUtils {
   }
 
   private def getMinTempTableId(logicalStr: String): Long = {
-    tempPattern.findAllIn(logicalStr).map(s => {
+    val found = tempPattern.findAllIn(logicalStr).map(s => {
       s.substring(4).toInt
-    }).min
+    })
+    if (found.isEmpty) {
+      0L
+    } else {
+      found.min
+    }
   }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala
index ee8e1f547b92e..cf9d947f100c7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/utils/TableProgramsTestBase.scala
@@ -37,11 +37,6 @@ class TableProgramsTestBase(
     }
     conf
   }
-
-  def getSelectQueryFromTemplate(selectBlock: String)
-                                (columnsName: Seq[String], table :String): String = {
-    s"SELECT ${columnsName.map(x=>selectBlock.replace("?",x)).mkString(",")} FROM $table"
-  }
 }
 
 object TableProgramsTestBase {
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunctionTest.scala
new file mode 100644
index 0000000000000..ba98fa2896a40
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0AggFunctionTest.scala
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.aggfunctions
+
+import java.math.BigDecimal
+import org.apache.flink.table.functions.AggregateFunction
+
+/**
+  * Test case for built-in sum0 aggregate function
+  *
+  * @tparam T the type for the aggregation result
+  */
+abstract class Sum0AggFunctionTestBase[T: Numeric]
+  extends AggFunctionTestBase[T, SumAccumulator[T]] {
+
+  private val numeric: Numeric[T] = implicitly[Numeric[T]]
+
+  def maxVal: T
+
+  private val minVal = numeric.negate(maxVal)
+
+  override def inputValueSets: Seq[Seq[T]] = Seq(
+    Seq(
+      minVal,
+      numeric.fromInt(1),
+      null.asInstanceOf[T],
+      numeric.fromInt(2),
+      numeric.fromInt(3),
+      numeric.fromInt(4),
+      numeric.fromInt(5),
+      numeric.fromInt(-10),
+      numeric.fromInt(-20),
+      numeric.fromInt(17),
+      null.asInstanceOf[T],
+      maxVal
+    ),
+    Seq(
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T]
+    )
+  )
+
+  override def expectedResults: Seq[T] = Seq(
+    numeric.fromInt(2),
+    0.asInstanceOf[T]
+  )
+}
+
+class ByteSum0AggFunctionTest extends Sum0AggFunctionTestBase[Byte] {
+
+  override def maxVal = (Byte.MaxValue / 2).toByte
+
+  override def aggregator: AggregateFunction[Byte, SumAccumulator[Byte]] =
+    new ByteSum0AggFunction
+}
+
+class ShortSum0AggFunctionTest extends Sum0AggFunctionTestBase[Short] {
+
+  override def maxVal = (Short.MaxValue / 2).toShort
+
+  override def aggregator: AggregateFunction[Short, SumAccumulator[Short]] =
+    new ShortSum0AggFunction
+}
+
+class IntSum0AggFunctionTest extends Sum0AggFunctionTestBase[Int] {
+
+  override def maxVal = Int.MaxValue / 2
+
+  override def aggregator: AggregateFunction[Int, SumAccumulator[Int]] =
+    new IntSum0AggFunction
+}
+
+class LongSum0AggFunctionTest extends Sum0AggFunctionTestBase[Long] {
+
+  override def maxVal = Long.MaxValue / 2
+
+  override def aggregator: AggregateFunction[Long, SumAccumulator[Long]] =
+    new LongSum0AggFunction
+}
+
+class FloatSum0AggFunctionTest extends Sum0AggFunctionTestBase[Float] {
+
+  override def maxVal = 12345.6789f
+
+  override def aggregator: AggregateFunction[Float, SumAccumulator[Float]] =
+    new FloatSum0AggFunction
+}
+
+class DoubleSum0AggFunctionTest extends Sum0AggFunctionTestBase[Double] {
+
+  override def maxVal = 12345.6789d
+
+  override def aggregator: AggregateFunction[Double, SumAccumulator[Double]] =
+    new DoubleSum0AggFunction
+}
+
+
+class DecimalSum0AggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, DecimalSumAccumulator] {
+
+  override def inputValueSets: Seq[Seq[_]] = Seq(
+    Seq(
+      new BigDecimal("1"),
+      new BigDecimal("2"),
+      new BigDecimal("3"),
+      null,
+      new BigDecimal("0"),
+      new BigDecimal("-1000"),
+      new BigDecimal("0.000000000002"),
+      new BigDecimal("1000"),
+      new BigDecimal("-0.000000000001"),
+      new BigDecimal("999.999"),
+      null,
+      new BigDecimal("4"),
+      new BigDecimal("-999.999"),
+      null
+    ),
+    Seq(
+      null,
+      null,
+      null,
+      null,
+      null
+    )
+  )
+
+  override def expectedResults: Seq[BigDecimal] = Seq(
+    new BigDecimal("10.000000000001"),
+    BigDecimal.ZERO
+  )
+
+  override def aggregator: AggregateFunction[BigDecimal, DecimalSumAccumulator] =
+    new DecimalSum0AggFunction()
+}
+
+
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunctionTest.scala
new file mode 100644
index 0000000000000..0a8f6337d5259
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/Sum0WithRetractAggFunctionTest.scala
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.aggfunctions
+
+import java.math.BigDecimal
+import org.apache.flink.table.functions.AggregateFunction
+
+/**
+  * Test case for built-in sum0 with retract aggregate function
+  *
+  * @tparam T the type for the aggregation result
+  */
+abstract class Sum0WithRetractAggFunctionTestBase[T: Numeric]
+  extends AggFunctionTestBase[T, SumWithRetractAccumulator[T]] {
+
+  private val numeric: Numeric[T] = implicitly[Numeric[T]]
+
+  def maxVal: T
+
+  private val minVal = numeric.negate(maxVal)
+
+  override def inputValueSets: Seq[Seq[T]] = Seq(
+    Seq(
+      minVal,
+      numeric.fromInt(1),
+      null.asInstanceOf[T],
+      numeric.fromInt(2),
+      numeric.fromInt(3),
+      numeric.fromInt(4),
+      numeric.fromInt(5),
+      numeric.fromInt(-10),
+      numeric.fromInt(-20),
+      numeric.fromInt(17),
+      null.asInstanceOf[T],
+      maxVal
+    ),
+    Seq(
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T],
+      null.asInstanceOf[T]
+    )
+  )
+
+  override def expectedResults: Seq[T] = Seq(
+    numeric.fromInt(2),
+    0.asInstanceOf[T]
+  )
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
+}
+
+class ByteSum0WithRetractAggFunctionTest extends Sum0WithRetractAggFunctionTestBase[Byte] {
+
+  override def maxVal = (Byte.MaxValue / 2).toByte
+
+  override def aggregator: AggregateFunction[Byte, SumWithRetractAccumulator[Byte]] =
+    new ByteSum0WithRetractAggFunction
+}
+
+class ShortSum0WithRetractAggFunctionTest extends Sum0WithRetractAggFunctionTestBase[Short] {
+
+  override def maxVal = (Short.MaxValue / 2).toShort
+
+  override def aggregator: AggregateFunction[Short, SumWithRetractAccumulator[Short]] =
+    new ShortSum0WithRetractAggFunction
+}
+
+class IntSum0WithRetractAggFunctionTest extends Sum0WithRetractAggFunctionTestBase[Int] {
+
+  override def maxVal = Int.MaxValue / 2
+
+  override def aggregator: AggregateFunction[Int, SumWithRetractAccumulator[Int]] =
+    new IntSum0WithRetractAggFunction
+}
+
+class LongSum0WithRetractAggFunctionTest extends Sum0WithRetractAggFunctionTestBase[Long] {
+
+  override def maxVal = Long.MaxValue / 2
+
+  override def aggregator: AggregateFunction[Long, SumWithRetractAccumulator[Long]] =
+    new LongSum0WithRetractAggFunction
+}
+
+class FloatSum0WithRetractAggFunctionTest extends Sum0WithRetractAggFunctionTestBase[Float] {
+
+  override def maxVal = 12345.6789f
+
+  override def aggregator: AggregateFunction[Float, SumWithRetractAccumulator[Float]] =
+    new FloatSum0WithRetractAggFunction
+}
+
+class DoubleSum0WithRetractAggFunctionTest extends Sum0WithRetractAggFunctionTestBase[Double] {
+
+  override def maxVal = 12345.6789d
+
+  override def aggregator: AggregateFunction[Double, SumWithRetractAccumulator[Double]] =
+    new DoubleSum0WithRetractAggFunction
+}
+
+
+class DecimalSum0WithRetractAggFunctionTest
+  extends AggFunctionTestBase[BigDecimal, DecimalSumWithRetractAccumulator] {
+
+  override def inputValueSets: Seq[Seq[_]] = Seq(
+    Seq(
+      new BigDecimal("1"),
+      new BigDecimal("2"),
+      new BigDecimal("3"),
+      null,
+      new BigDecimal("0"),
+      new BigDecimal("-1000"),
+      new BigDecimal("0.000000000002"),
+      new BigDecimal("1000"),
+      new BigDecimal("-0.000000000001"),
+      new BigDecimal("999.999"),
+      null,
+      new BigDecimal("4"),
+      new BigDecimal("-999.999"),
+      null
+    ),
+    Seq(
+      null,
+      null,
+      null,
+      null,
+      null
+    )
+  )
+
+  override def expectedResults: Seq[BigDecimal] = Seq(
+    new BigDecimal("10.000000000001"),
+    BigDecimal.ZERO
+  )
+
+  override def aggregator: AggregateFunction[BigDecimal, DecimalSumWithRetractAccumulator] =
+    new DecimalSum0WithRetractAggFunction()
+
+  override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any])
+}
+
+
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
index 15ef2b1d436ac..73bc2f8ab07ee 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
@@ -28,6 +28,7 @@ import org.apache.flink.table.expressions.Expression
 import org.apache.flink.table.functions.{ScalarFunction, TableFunction}
 import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream}
 import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
+import org.apache.flink.table.api.scala.batch.utils.LogicalPlanFormatUtils
 import org.junit.Assert.assertEquals
 import org.mockito.Mockito.{mock, when}
 
@@ -47,8 +48,8 @@ class TableTestBase {
   def verifyTableEquals(expected: Table, actual: Table): Unit = {
     assertEquals(
       "Logical plans do not match",
-      RelOptUtil.toString(expected.getRelNode),
-      RelOptUtil.toString(actual.getRelNode))
+      LogicalPlanFormatUtils.formatTempTableId(RelOptUtil.toString(expected.getRelNode)),
+      LogicalPlanFormatUtils.formatTempTableId(RelOptUtil.toString(actual.getRelNode)))
   }
 
 }

From 21428db95318cd15fa45ae10b9cf79360dacf8fa Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Fri, 21 Apr 2017 11:43:53 +0200
Subject: [PATCH 116/253] [FLINK-6353] Fix legacy user-state restore from 1.2

State that was checkpointed using Checkpointed (on a user function)
could be restored using CheckpointedRestoring when the savepoint was
done on Flink 1.2. The reason was an overzealous check in
AbstractUdfStreamOperator that only restores from "legacy" operator
state using CheckpointedRestoring when the stream is a Migration stream.

This removes that check but we still need to make sure to read away the
byte that indicates whether there is legacy state, which is written when
we're restoring from a Flink 1.1 savepoint.

After this fix, the procedure for a user to migrate a user function away
from the Checkpointed interface is this:

 - Perform savepoint with user function still implementing Checkpointed,
   shutdown job
 - Change user function to implement CheckpointedRestoring
 - Restore from previous savepoint, user function has to somehow move
   the state that is restored using CheckpointedRestoring to another
   type of state, .e.g operator state, using the OperatorStateStore.
 - Perform another savepoint, shutdown job
 - Remove CheckpointedRestoring interface from user function
 - Restore from the second savepoint
 - Done.

If the CheckpointedRestoring interface is not removed as prescribed in
the last steps then a future restore of a new savepoint will fail
because Flink will try to read legacy operator state that is not there
anymore.  The above steps also apply to Flink 1.3, when a user want's to
move away from the Checkpointed interface.
---
 .../api/operators/AbstractUdfStreamOperator.java          | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
index 2fa1e382b97d1..14857de0b06b6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
@@ -154,12 +154,14 @@ public void snapshotState(FSDataOutputStream out, long checkpointId, long timest
 
 	@Override
 	public void restoreState(FSDataInputStream in) throws Exception {
+		boolean haveReadUdfStateFlag = false;
 		if (userFunction instanceof Checkpointed ||
-				(userFunction instanceof CheckpointedRestoring && in instanceof Migration)) {
+				(userFunction instanceof CheckpointedRestoring)) {
 			@SuppressWarnings("unchecked")
 			CheckpointedRestoring<Serializable> chkFunction = (CheckpointedRestoring<Serializable>) userFunction;
 
 			int hasUdfState = in.read();
+			haveReadUdfStateFlag = true;
 
 			if (hasUdfState == 1) {
 				Serializable functionState = InstantiationUtil.deserializeObject(in, getUserCodeClassloader());
@@ -171,7 +173,9 @@ public void restoreState(FSDataInputStream in) throws Exception {
 					}
 				}
 			}
-		} else if (in instanceof Migration) {
+		}
+
+		if (in instanceof Migration && !haveReadUdfStateFlag) {
 			// absorb the introduced byte from the migration stream without too much further consequences
 			int hasUdfState = in.read();
 			if (hasUdfState == 1) {

From 76601fa01a0e0fa88d0e26b591ec72ba1147d3e4 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Tue, 25 Apr 2017 14:56:14 +0200
Subject: [PATCH 117/253] [FLINK-5969] Remove watermark callback service

In Flink 1.2 we only wrote timers to key-grouped state streams. With the
addition of the watermark callback service we started to also write the
watermark callbacks to the key-grouped streams. This breaks backwards
compatibility with saveoints taken on Flink 1.2

This replaces usage of the watermark callback service by setting a
regular timer for "current watermark + 1". These timers will fire as
soon as the watermark advances, thus simulating the watermark callback
service.
---
 .../AbstractKeyedCEPPatternOperator.java      | 134 ++++---
 .../flink/cep/operator/CEPOperatorTest.java   |  10 +-
 .../api/operators/AbstractStreamOperator.java |  20 -
 .../operators/InternalTimeServiceManager.java |  35 +-
 .../InternalWatermarkCallbackService.java     | 296 ---------------
 .../operators/AbstractStreamOperatorTest.java | 346 ------------------
 .../AbstractStreamOperatorTestHarness.java    |   9 -
 7 files changed, 85 insertions(+), 765 deletions(-)
 delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java

diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
index b232dbb14f4a0..5f2487590a7df 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
@@ -30,13 +30,15 @@
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.migration.streaming.runtime.streamrecord.MultiplexingStreamRecordSerializer;
-import org.apache.flink.streaming.api.operators.InternalWatermarkCallbackService;
+import org.apache.flink.runtime.state.VoidNamespace;
+import org.apache.flink.runtime.state.VoidNamespaceSerializer;
+import org.apache.flink.streaming.api.operators.InternalTimer;
+import org.apache.flink.streaming.api.operators.InternalTimerService;
 import org.apache.flink.runtime.state.StateInitializationContext;
 import org.apache.flink.streaming.api.operators.CheckpointedRestoringOperator;
 import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
-import org.apache.flink.streaming.api.operators.OnWatermarkCallback;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.api.operators.Triggerable;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -62,7 +64,7 @@
  */
 public abstract class AbstractKeyedCEPPatternOperator<IN, KEY, OUT>
 	extends AbstractStreamOperator<OUT>
-	implements OneInputStreamOperator<IN, OUT>, CheckpointedRestoringOperator {
+	implements OneInputStreamOperator<IN, OUT>, Triggerable<KEY, VoidNamespace>, CheckpointedRestoringOperator {
 
 	private static final long serialVersionUID = -4166778210774160757L;
 
@@ -89,6 +91,8 @@ public abstract class AbstractKeyedCEPPatternOperator<IN, KEY, OUT>
 	private final PriorityQueueFactory<StreamRecord<IN>> priorityQueueFactory = new PriorityQueueStreamRecordFactory<>();
 	private final NFACompiler.NFAFactory<IN> nfaFactory;
 
+	private transient InternalTimerService<VoidNamespace> timerService;
+
 	/**
 	 * {@link OutputTag} to use for late arriving events. Elements for which
 	 * {@code window.maxTimestamp + allowedLateness} is smaller than the current watermark will
@@ -159,53 +163,14 @@ public void initializeState(StateInitializationContext context) throws Exception
 	public void open() throws Exception {
 		super.open();
 
-		final InternalWatermarkCallbackService<KEY> watermarkCallbackService = getInternalWatermarkCallbackService();
-
-		watermarkCallbackService.setWatermarkCallback(
-			new OnWatermarkCallback<KEY>() {
-
-				@Override
-				public void onWatermark(KEY key, Watermark watermark) throws IOException {
-
-					// 1) get the queue of pending elements for the key and the corresponding NFA,
-					// 2) process the pending elements in event time order by feeding them in the NFA
-					// 3) advance the time to the current watermark, so that expired patterns are discarded.
-					// 4) update the stored state for the key, by only storing the new NFA and priority queue iff they
-					//		have state to be used later.
-					// 5) update the last seen watermark.
-
-					// STEP 1
-					PriorityQueue<StreamRecord<IN>> priorityQueue = getPriorityQueue();
-					NFA<IN> nfa = getNFA();
-
-					// STEP 2
-					while (!priorityQueue.isEmpty() && priorityQueue.peek().getTimestamp() <= watermark.getTimestamp()) {
-						StreamRecord<IN> streamRecord = priorityQueue.poll();
-						processEvent(nfa, streamRecord.getValue(), streamRecord.getTimestamp());
-					}
-
-					// STEP 3
-					advanceTime(nfa, watermark.getTimestamp());
-
-					// STEP 4
-					updatePriorityQueue(priorityQueue);
-					updateNFA(nfa);
-
-					if (priorityQueue.isEmpty() && nfa.isEmpty()) {
-						watermarkCallbackService.unregisterKeyFromWatermarkCallback(key);
-					}
-
-					// STEP 5
-					updateLastSeenWatermark(watermark);
-				}
-			},
-			keySerializer
-		);
+		timerService = getInternalTimerService(
+				"watermark-callbacks",
+				VoidNamespaceSerializer.INSTANCE,
+				this);
 	}
 
 	@Override
 	public void processElement(StreamRecord<IN> element) throws Exception {
-
 		if (isProcessingTime) {
 			// there can be no out of order elements in processing time
 			NFA<IN> nfa = getNFA();
@@ -223,7 +188,7 @@ public void processElement(StreamRecord<IN> element) throws Exception {
 				// we have an event with a valid timestamp, so
 				// we buffer it until we receive the proper watermark.
 
-				getInternalWatermarkCallbackService().registerKeyForWatermarkCallback(keySelector.getKey(element.getValue()));
+				saveRegisterWatermarkTimer();
 
 				PriorityQueue<StreamRecord<IN>> priorityQueue = getPriorityQueue();
 				if (getExecutionConfig().isObjectReuseEnabled()) {
@@ -239,8 +204,62 @@ public void processElement(StreamRecord<IN> element) throws Exception {
 		}
 	}
 
-	private void updateLastSeenWatermark(Watermark watermark) {
-		this.lastWatermark = watermark.getTimestamp();
+	/**
+	 * Registers a timer for {@code current watermark + 1}, this means that we get triggered
+	 * whenever the watermark advances, which is what we want for working off the queue of
+	 * buffered elements.
+	 */
+	private void saveRegisterWatermarkTimer() {
+		long currentWatermark = timerService.currentWatermark();
+		// protect against overflow
+		if (currentWatermark + 1 > currentWatermark) {
+			timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, currentWatermark + 1);
+		}
+	}
+
+	long count = 0;
+	@Override
+	public void onEventTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {
+
+		// 1) get the queue of pending elements for the key and the corresponding NFA,
+		// 2) process the pending elements in event time order by feeding them in the NFA
+		// 3) advance the time to the current watermark, so that expired patterns are discarded.
+		// 4) update the stored state for the key, by only storing the new NFA and priority queue iff they
+		//		have state to be used later.
+		// 5) update the last seen watermark.
+
+		// STEP 1
+		PriorityQueue<StreamRecord<IN>> priorityQueue = getPriorityQueue();
+		NFA<IN> nfa = getNFA();
+
+		// STEP 2
+		while (!priorityQueue.isEmpty() && priorityQueue.peek().getTimestamp() <= timerService.currentWatermark()) {
+			StreamRecord<IN> streamRecord = priorityQueue.poll();
+			processEvent(nfa, streamRecord.getValue(), streamRecord.getTimestamp());
+		}
+
+		// STEP 3
+		advanceTime(nfa, timerService.currentWatermark());
+
+		// STEP 4
+		updatePriorityQueue(priorityQueue);
+		updateNFA(nfa);
+
+		if (!priorityQueue.isEmpty() || !nfa.isEmpty()) {
+			saveRegisterWatermarkTimer();
+		}
+
+		// STEP 5
+		updateLastSeenWatermark(timerService.currentWatermark());
+	}
+
+	@Override
+	public void onProcessingTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {
+		// not used
+	}
+
+	private void updateLastSeenWatermark(long timestamp) {
+		this.lastWatermark = timestamp;
 	}
 
 	/**
@@ -309,12 +328,17 @@ public void restoreState(FSDataInputStream in) throws Exception {
 		in.read();
 
 		DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(in);
-		InternalWatermarkCallbackService<KEY> watermarkCallbackService = getInternalWatermarkCallbackService();
+		timerService = getInternalTimerService(
+				"watermark-callbacks",
+				VoidNamespaceSerializer.INSTANCE,
+				this);
 
 		if (migratingFromOldKeyedOperator) {
 			int numberEntries = inputView.readInt();
 			for (int i = 0; i <numberEntries; i++) {
-				watermarkCallbackService.registerKeyForWatermarkCallback(keySerializer.deserialize(inputView));
+				KEY key = keySerializer.deserialize(inputView);
+				setCurrentKey(key);
+				saveRegisterWatermarkTimer();
 			}
 		} else {
 
@@ -342,10 +366,8 @@ public void restoreState(FSDataInputStream in) throws Exception {
 				// this is relevant only for event/ingestion time
 
 				// need to work around type restrictions
-				InternalWatermarkCallbackService rawWatermarkCallbackService =
-					(InternalWatermarkCallbackService) watermarkCallbackService;
-
-				rawWatermarkCallbackService.registerKeyForWatermarkCallback((byte) 0);
+				setCurrentKey((byte) 0);
+				saveRegisterWatermarkTimer();
 			}
 			ois.close();
 		}
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java
index 8465bc34832ec..4048bc2639878 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java
@@ -307,7 +307,7 @@ public void testCEPOperatorCleanupEventTime() throws Exception {
 		// there must be 2 keys 42, 43 registered for the watermark callback
 		// all the seen elements must be in the priority queues but no NFA yet.
 
-		assertEquals(2L, harness.numKeysForWatermarkCallback());
+		assertEquals(2L, harness.numEventTimeTimers());
 		assertEquals(4L, operator.getPQSize(42));
 		assertEquals(1L, operator.getPQSize(43));
 		assertTrue(!operator.hasNonEmptyNFA(42));
@@ -322,7 +322,7 @@ public void testCEPOperatorCleanupEventTime() throws Exception {
 		// one element in PQ for 42 (the barfoo) as it arrived early
 		// for 43 the element entered the NFA and the PQ is empty
 
-		assertEquals(2L, harness.numKeysForWatermarkCallback());
+		assertEquals(2L, harness.numEventTimeTimers());
 		assertTrue(operator.hasNonEmptyNFA(42));
 		assertEquals(1L, operator.getPQSize(42));
 		assertTrue(operator.hasNonEmptyNFA(43));
@@ -336,7 +336,7 @@ public void testCEPOperatorCleanupEventTime() throws Exception {
 
 		// now we have 1 key because the 43 expired and was removed.
 		// 42 is still there due to startEvent2
-		assertEquals(1L, harness.numKeysForWatermarkCallback());
+		assertEquals(1L, harness.numEventTimeTimers());
 		assertTrue(operator.hasNonEmptyNFA(42));
 		assertTrue(!operator.hasNonEmptyPQ(42));
 		assertTrue(!operator.hasNonEmptyNFA(43));
@@ -355,7 +355,7 @@ public void testCEPOperatorCleanupEventTime() throws Exception {
 
 		assertTrue(!operator.hasNonEmptyNFA(42));
 		assertTrue(!operator.hasNonEmptyPQ(42));
-		assertEquals(0L, harness.numKeysForWatermarkCallback());
+		assertEquals(0L, harness.numEventTimeTimers());
 
 		verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent2, endEvent2);
 		verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent3, endEvent2);
@@ -430,7 +430,7 @@ public void testCEPOperatorCleanupProcessingTime() throws Exception {
 		// the pattern expired
 		assertTrue(!operator.hasNonEmptyNFA(42));
 
-		assertEquals(0L, harness.numKeysForWatermarkCallback());
+		assertEquals(0L, harness.numEventTimeTimers());
 		assertTrue(!operator.hasNonEmptyPQ(42));
 		assertTrue(!operator.hasNonEmptyPQ(43));
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 1ef8614c5bf54..18500073d52fd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -906,20 +906,6 @@ public void close() {
 	//  Watermark handling
 	// ------------------------------------------------------------------------
 
-	/**
-	 * Returns an {@link InternalWatermarkCallbackService} which  allows to register a
-	 * {@link OnWatermarkCallback} and multiple keys, for which
-	 * the callback will be invoked every time a new {@link Watermark} is received.
-	 *
-	 * <p><b>NOTE: </b> This service is only available to <b>keyed</b> operators.
-	 */
-	public <K> InternalWatermarkCallbackService<K> getInternalWatermarkCallbackService() {
-		checkTimerServiceInitialization();
-
-		InternalTimeServiceManager<K, ?> keyedTimeServiceHandler = (InternalTimeServiceManager<K, ?>) timeServiceManager;
-		return keyedTimeServiceHandler.getWatermarkCallbackService();
-	}
-
 	/**
 	 * Returns a {@link InternalTimerService} that can be used to query current processing time
 	 * and event time and to set timers. An operator can have several timer services, where
@@ -998,10 +984,4 @@ public int numEventTimeTimers() {
 		return timeServiceManager == null ? 0 :
 			timeServiceManager.numEventTimeTimers();
 	}
-
-	@VisibleForTesting
-	public int numKeysForWatermarkCallback() {
-		return timeServiceManager == null ? 0 :
-			timeServiceManager.numKeysForWatermarkCallback();
-	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
index 5886468051ec9..ff5164d092f1f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
@@ -34,8 +34,8 @@
 
 /**
  * An entity keeping all the time-related services available to all operators extending the
- * {@link AbstractStreamOperator}. These are the different {@link HeapInternalTimerService timer services}
- * and the {@link InternalWatermarkCallbackService}.
+ * {@link AbstractStreamOperator}. Right now, this is only a
+ * {@link HeapInternalTimerService timer services}.
  *
  * <b>NOTE:</b> These services are only available to keyed operators.
  *
@@ -52,7 +52,6 @@ class InternalTimeServiceManager<K, N> {
 	private final ProcessingTimeService processingTimeService;
 
 	private final Map<String, HeapInternalTimerService<K, N>> timerServices;
-	private final InternalWatermarkCallbackService<K> watermarkCallbackService;
 
 	InternalTimeServiceManager(
 			int totalKeyGroups,
@@ -68,16 +67,6 @@ class InternalTimeServiceManager<K, N> {
 		this.processingTimeService = Preconditions.checkNotNull(processingTimeService);
 
 		this.timerServices = new HashMap<>();
-		this.watermarkCallbackService = new InternalWatermarkCallbackService<>(totalKeyGroups, localKeyGroupRange, keyContext);
-	}
-
-	/**
-	 * Returns an {@link InternalWatermarkCallbackService} which  allows to register a
-	 * {@link OnWatermarkCallback} and multiple keys, for which
-	 * the callback will be invoked every time a new {@link Watermark} is received.
-	 */
-	public InternalWatermarkCallbackService<K> getWatermarkCallbackService() {
-		return watermarkCallbackService;
 	}
 
 	/**
@@ -117,7 +106,6 @@ public void advanceWatermark(Watermark watermark) throws Exception {
 		for (HeapInternalTimerService<?, ?> service : timerServices.values()) {
 			service.advanceWatermark(watermark.getTimestamp());
 		}
-		watermarkCallbackService.invokeOnWatermarkCallback(watermark);
 	}
 
 	//////////////////				Fault Tolerance Methods				///////////////////
@@ -132,15 +120,6 @@ public void snapshotStateForKeyGroup(DataOutputViewStreamWrapper stream, int key
 			stream.writeUTF(serviceName);
 			timerService.snapshotTimersForKeyGroup(stream, keyGroupIdx);
 		}
-
-		// write a byte indicating if there was a key
-		// registry service instantiated (1) or not (0).
-		if (watermarkCallbackService != null) {
-			stream.writeByte(1);
-			watermarkCallbackService.snapshotKeysForKeyGroup(stream, keyGroupIdx);
-		} else {
-			stream.writeByte(0);
-		}
 	}
 
 	public void restoreStateForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
@@ -161,11 +140,6 @@ public void restoreStateForKeyGroup(DataInputViewStreamWrapper stream, int keyGr
 			}
 			timerService.restoreTimersForKeyGroup(stream, keyGroupIdx, userCodeClassLoader);
 		}
-
-		byte hadKeyRegistry = stream.readByte();
-		if (hadKeyRegistry == 1) {
-			watermarkCallbackService.restoreKeysForKeyGroup(stream, keyGroupIdx, userCodeClassLoader);
-		}
 	}
 
 	////////////////////			Methods used ONLY IN TESTS				////////////////////
@@ -187,9 +161,4 @@ public int numEventTimeTimers() {
 		}
 		return count;
 	}
-
-	@VisibleForTesting
-	public int numKeysForWatermarkCallback() {
-		return watermarkCallbackService.numKeysForWatermarkCallback();
-	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java
deleted file mode 100644
index 441835e51a67d..0000000000000
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalWatermarkCallbackService.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.operators;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
-import org.apache.flink.runtime.state.KeyGroupsList;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.util.InstantiationUtil;
-
-/**
- * The watermark callback service allows to register a {@link OnWatermarkCallback OnWatermarkCallback}
- * and multiple keys, for which the callback will be invoked every time a new {@link Watermark} is received
- * (after the registration of the key).
- *
- * <p><b>NOTE: </b> This service is only available to <b>keyed</b> operators.
- *
- *  @param <K> The type of key returned by the {@code KeySelector}.
- */
-@Internal
-public class InternalWatermarkCallbackService<K> {
-
-	////////////			Information about the keyed state				//////////
-
-	private final KeyGroupsList localKeyGroupRange;
-	private final int totalKeyGroups;
-	private final int localKeyGroupRangeStartIdx;
-
-	private final KeyContext keyContext;
-
-	/**
-	 * An array of sets of keys keeping the registered keys split
-	 * by the key-group they belong to. Each key-group has one set.
-	 */
-	private final Set<K>[] registeredKeysByKeyGroup;
-
-	/**
-	 * An array of sets of keys keeping the keys "to delete" split
-	 * by the key-group they belong to. Each key-group has one set.
-	 *
-	 * <p>This is used to avoid potential concurrent modification
-	 * exception when deleting keys from inside the
-	 * {@link #invokeOnWatermarkCallback(Watermark)}.
-	 */
-	private final Set<K>[] deletedKeysByKeyGroup;
-
-	/** A serializer for the registered keys. */
-	private TypeSerializer<K> keySerializer;
-
-	/**
-	 * The {@link OnWatermarkCallback} to be invoked for each
-	 * registered key upon reception of the watermark.
-	 */
-	private OnWatermarkCallback<K> callback;
-
-	public InternalWatermarkCallbackService(int totalKeyGroups, KeyGroupsList localKeyGroupRange, KeyContext keyContext) {
-
-		this.totalKeyGroups = totalKeyGroups;
-		this.localKeyGroupRange = checkNotNull(localKeyGroupRange);
-		this.keyContext = checkNotNull(keyContext);
-
-		// find the starting index of the local key-group range
-		int startIdx = Integer.MAX_VALUE;
-		for (Integer keyGroupIdx : localKeyGroupRange) {
-			startIdx = Math.min(keyGroupIdx, startIdx);
-		}
-		this.localKeyGroupRangeStartIdx = startIdx;
-
-		// the list of ids of the key-groups this task is responsible for
-		int localKeyGroups = this.localKeyGroupRange.getNumberOfKeyGroups();
-		this.registeredKeysByKeyGroup = new Set[localKeyGroups];
-		this.deletedKeysByKeyGroup = new Set[localKeyGroups];
-	}
-
-	/**
-	 * Registers a {@link OnWatermarkCallback} with the current {@link InternalWatermarkCallbackService} service.
-	 * Before this method is called and the callback is set, the service is unusable.
-	 *
-	 * @param watermarkCallback The callback to be registered.
-	 * @param keySerializer A serializer for the registered keys.
-	 */
-	public void setWatermarkCallback(OnWatermarkCallback<K> watermarkCallback, TypeSerializer<K> keySerializer) {
-		if (callback == null) {
-			this.keySerializer = keySerializer;
-			this.callback = watermarkCallback;
-		} else {
-			throw new RuntimeException("The watermark callback has already been initialized.");
-		}
-	}
-
-	/**
-	 * Registers a key with the service. This will lead to the {@link OnWatermarkCallback}
-	 * being invoked for this key upon reception of each subsequent watermark.
-	 *
-	 * @param key The key to be registered.
-	 */
-	public boolean registerKeyForWatermarkCallback(K key) {
-		return getRegisteredKeysForKeyGroup(key).add(key);
-	}
-
-	/**
-	 * Unregisters the provided key from the service.
-	 *
-	 * @param key The key to be unregistered.
-	 */
-	public boolean unregisterKeyFromWatermarkCallback(K key) {
-		return getDeletedKeysForKeyGroup(key).add(key);
-	}
-
-	/**
-	 * Invokes the registered callback for all the registered keys.
-	 *
-	 * @param watermark The watermark that triggered the invocation.
-	 */
-	public void invokeOnWatermarkCallback(Watermark watermark) throws IOException {
-		// clean up any keys registered for deletion before calling the callback
-		cleanupRegisteredKeys();
-
-		if (callback != null) {
-			for (Set<K> keySet : registeredKeysByKeyGroup) {
-				if (keySet != null) {
-					for (K key : keySet) {
-						keyContext.setCurrentKey(key);
-						callback.onWatermark(key, watermark);
-					}
-				}
-			}
-		}
-	}
-
-	/**
-	 * Does the actual deletion of any keys registered for deletion using the
-	 * {@link #unregisterKeyFromWatermarkCallback(Object)}.
-	 */
-	private void cleanupRegisteredKeys() {
-		for (int keyGroupIdx = 0; keyGroupIdx < registeredKeysByKeyGroup.length; keyGroupIdx++) {
-
-			Set<K> deletedKeys = deletedKeysByKeyGroup[keyGroupIdx];
-			if (deletedKeys != null) {
-
-				Set<K> registeredKeys = registeredKeysByKeyGroup[keyGroupIdx];
-				if (registeredKeys != null) {
-
-					registeredKeys.removeAll(deletedKeys);
-					if (registeredKeys.isEmpty()) {
-						registeredKeysByKeyGroup[keyGroupIdx] = null;
-					}
-				}
-				deletedKeysByKeyGroup[keyGroupIdx] = null;
-			}
-		}
-	}
-
-	/**
-	 * Retrieve the set of keys for the key-group this key belongs to.
-	 *
-	 * @param key the key whose key-group we are searching.
-	 * @return the set of registered keys for the key-group.
-	 */
-	private Set<K> getRegisteredKeysForKeyGroup(K key) {
-		checkArgument(localKeyGroupRange != null, "The operator has not been initialized.");
-		int keyGroupIdx = KeyGroupRangeAssignment.assignToKeyGroup(key, totalKeyGroups);
-		return getRegisteredKeysForKeyGroup(keyGroupIdx);
-	}
-
-	/**
-	 * Retrieve the set of keys for the requested key-group.
-	 *
-	 * @param keyGroupIdx the index of the key group we are interested in.
-	 * @return the set of keys for the key-group.
-	 */
-	private Set<K> getRegisteredKeysForKeyGroup(int keyGroupIdx) {
-		int localIdx = getIndexForKeyGroup(keyGroupIdx);
-		Set<K> keys = registeredKeysByKeyGroup[localIdx];
-		if (keys == null) {
-			keys = new HashSet<>();
-			registeredKeysByKeyGroup[localIdx] = keys;
-		}
-		return keys;
-	}
-
-	private Set<K> getDeletedKeysForKeyGroup(K key) {
-		checkArgument(localKeyGroupRange != null, "The operator has not been initialized.");
-		int keyGroupIdx = KeyGroupRangeAssignment.assignToKeyGroup(key, totalKeyGroups);
-		return getDeletedKeysForKeyGroup(keyGroupIdx);
-	}
-
-	private Set<K> getDeletedKeysForKeyGroup(int keyGroupIdx) {
-		int localIdx = getIndexForKeyGroup(keyGroupIdx);
-		Set<K> keys = deletedKeysByKeyGroup[localIdx];
-		if (keys == null) {
-			keys = new HashSet<>();
-			deletedKeysByKeyGroup[localIdx] = keys;
-		}
-		return keys;
-	}
-
-	/**
-	 * Computes the index of the requested key-group in the local datastructures.
-	 *
-	 * <p>Currently we assume that each task is assigned a continuous range of key-groups,
-	 * e.g. 1,2,3,4, and not 1,3,5. We leverage this to keep the different states
-	 * key-grouped in arrays instead of maps, where the offset for each key-group is
-	 * the key-group id (an int) minus the id of the first key-group in the local range.
-	 * This is for performance reasons.
-	 */
-	private int getIndexForKeyGroup(int keyGroupIdx) {
-		checkArgument(localKeyGroupRange.contains(keyGroupIdx),
-			"Key Group " + keyGroupIdx + " does not belong to the local range.");
-		return keyGroupIdx - localKeyGroupRangeStartIdx;
-	}
-
-	//////////////////				Fault Tolerance Methods				///////////////////
-
-	public void snapshotKeysForKeyGroup(DataOutputViewStreamWrapper stream, int keyGroupIdx) throws Exception {
-
-		// we cleanup also here to avoid checkpointing the deletion set
-		cleanupRegisteredKeys();
-
-		Set<K> keySet = getRegisteredKeysForKeyGroup(keyGroupIdx);
-		if (keySet != null) {
-			stream.writeInt(keySet.size());
-
-			InstantiationUtil.serializeObject(stream, keySerializer);
-			for (K key : keySet) {
-				keySerializer.serialize(key, stream);
-			}
-		} else {
-			stream.writeInt(0);
-		}
-	}
-
-	public void restoreKeysForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
-									ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException {
-
-		checkArgument(localKeyGroupRange.contains(keyGroupIdx),
-			"Key Group " + keyGroupIdx + " does not belong to the local range.");
-
-		int numKeys = stream.readInt();
-		if (numKeys > 0) {
-
-			TypeSerializer<K> tmpKeyDeserializer = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
-
-			if (keySerializer != null && !keySerializer.equals(tmpKeyDeserializer)) {
-				throw new IllegalArgumentException("Tried to restore keys " +
-					"for the watermark callback service with mismatching serializers.");
-			}
-
-			this.keySerializer = tmpKeyDeserializer;
-
-			Set<K> keys = getRegisteredKeysForKeyGroup(keyGroupIdx);
-			for (int i = 0; i < numKeys; i++) {
-				keys.add(keySerializer.deserialize(stream));
-			}
-		}
-	}
-
-	//////////////////				Testing Methods				///////////////////
-
-	@VisibleForTesting
-	public int numKeysForWatermarkCallback() {
-		int count = 0;
-		for (Set<K> keyGroup: registeredKeysByKeyGroup) {
-			if (keyGroup != null) {
-				count += keyGroup.size();
-			}
-		}
-		return count;
-	}
-}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
index 1f68af9009bf1..774ab4a893cd5 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
@@ -614,280 +614,6 @@ public void testFailingBackendSnapshotMethod() throws Exception {
 		verify(futureKeyedStateHandle).cancel(anyBoolean());
 	}
 
-	@Test
-	public void testWatermarkCallbackServiceScalingUp() throws Exception {
-		final int MAX_PARALLELISM = 10;
-
-		KeySelector<Tuple2<Integer, String>, Integer> keySelector = new TestKeySelector();
-
-		Tuple2<Integer, String> element1 = new Tuple2<>(7, "first");
-		Tuple2<Integer, String> element2 = new Tuple2<>(10, "start");
-
-		int keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element1), MAX_PARALLELISM);
-		assertEquals(1, keygroup);
-		assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
-
-		keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element2), MAX_PARALLELISM);
-		assertEquals(9, keygroup);
-		assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
-
-		// now we start the test, we go from parallelism 1 to 2.
-
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness1 =
-			getTestHarness(MAX_PARALLELISM, 1, 0);
-		testHarness1.open();
-
-		testHarness1.processElement(new StreamRecord<>(element1));
-		testHarness1.processElement(new StreamRecord<>(element2));
-
-		assertEquals(0, testHarness1.getOutput().size());
-
-		// take a snapshot with some elements in internal sorting queue
-		OperatorStateHandles snapshot = testHarness1.snapshot(0, 0);
-		testHarness1.close();
-
-		// initialize two sub-tasks with the previously snapshotted state to simulate scaling up
-
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness2 =
-			getTestHarness(MAX_PARALLELISM, 2, 0);
-
-		testHarness2.setup();
-		testHarness2.initializeState(snapshot);
-		testHarness2.open();
-
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness3 =
-			getTestHarness(MAX_PARALLELISM, 2, 1);
-
-		testHarness3.setup();
-		testHarness3.initializeState(snapshot);
-		testHarness3.open();
-
-		testHarness2.processWatermark(new Watermark(10));
-		testHarness3.processWatermark(new Watermark(10));
-
-		assertEquals(2, testHarness2.getOutput().size());
-		verifyElement(testHarness2.getOutput().poll(), 7);
-		verifyWatermark(testHarness2.getOutput().poll(), 10);
-
-		assertEquals(2, testHarness3.getOutput().size());
-		verifyElement(testHarness3.getOutput().poll(), 10);
-		verifyWatermark(testHarness3.getOutput().poll(), 10);
-
-		testHarness1.close();
-		testHarness2.close();
-		testHarness3.close();
-	}
-
-	@Test
-	public void testWatermarkCallbackServiceScalingDown() throws Exception {
-		final int MAX_PARALLELISM = 10;
-
-		KeySelector<Tuple2<Integer, String>, Integer> keySelector = new TestKeySelector();
-
-		Tuple2<Integer, String> element1 = new Tuple2<>(7, "first");
-		Tuple2<Integer, String> element2 = new Tuple2<>(45, "start");
-		Tuple2<Integer, String> element3 = new Tuple2<>(90, "start");
-		Tuple2<Integer, String> element4 = new Tuple2<>(10, "start");
-
-		int keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element1), MAX_PARALLELISM);
-		assertEquals(1, keygroup);
-		assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
-		assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
-
-		keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element2), MAX_PARALLELISM);
-		assertEquals(6, keygroup);
-		assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
-		assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
-
-		keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element3), MAX_PARALLELISM);
-		assertEquals(2, keygroup);
-		assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
-		assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
-
-		keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(element4), MAX_PARALLELISM);
-		assertEquals(9, keygroup);
-		assertEquals(2, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 3, keygroup));
-		assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(MAX_PARALLELISM, 2, keygroup));
-
-		// starting the test, we will go from parallelism of 3 to parallelism of 2
-
-		// first operator
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness1 =
-			getTestHarness(MAX_PARALLELISM, 3, 0);
-		testHarness1.open();
-
-		// second operator
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness2 =
-			getTestHarness(MAX_PARALLELISM, 3, 1);
-		testHarness2.open();
-
-		// third operator
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness3 =
-			getTestHarness(MAX_PARALLELISM, 3, 2);
-		testHarness3.open();
-
-		testHarness1.processWatermark(Long.MIN_VALUE);
-		testHarness2.processWatermark(Long.MIN_VALUE);
-		testHarness3.processWatermark(Long.MIN_VALUE);
-
-		testHarness1.processElement(new StreamRecord<>(element1));
-		testHarness1.processElement(new StreamRecord<>(element3));
-
-		testHarness2.processElement(new StreamRecord<>(element2));
-		testHarness3.processElement(new StreamRecord<>(element4));
-
-		// so far we only have the initial watermark
-		assertEquals(1, testHarness1.getOutput().size());
-		verifyWatermark(testHarness1.getOutput().poll(), Long.MIN_VALUE);
-
-		assertEquals(1, testHarness2.getOutput().size());
-		verifyWatermark(testHarness2.getOutput().poll(), Long.MIN_VALUE);
-
-		assertEquals(1, testHarness3.getOutput().size());
-		verifyWatermark(testHarness3.getOutput().poll(), Long.MIN_VALUE);
-
-		// we take a snapshot and make it look as a single operator
-		// this will be the initial state of all downstream tasks.
-		OperatorStateHandles snapshot = AbstractStreamOperatorTestHarness.repackageState(
-			testHarness2.snapshot(0, 0),
-			testHarness1.snapshot(0, 0),
-			testHarness3.snapshot(0, 0)
-		);
-
-		// first new operator
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness4 =
-			getTestHarness(MAX_PARALLELISM, 2, 0);
-		testHarness4.setup();
-		testHarness4.initializeState(snapshot);
-		testHarness4.open();
-
-		// second new operator
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness5 =
-			getTestHarness(MAX_PARALLELISM, 2, 1);
-		testHarness5.setup();
-		testHarness5.initializeState(snapshot);
-		testHarness5.open();
-
-		testHarness4.processWatermark(10);
-		testHarness5.processWatermark(10);
-
-		assertEquals(3, testHarness4.getOutput().size());
-		verifyElement(testHarness4.getOutput().poll(), 7);
-		verifyElement(testHarness4.getOutput().poll(), 90);
-		verifyWatermark(testHarness4.getOutput().poll(), 10);
-
-		assertEquals(3, testHarness5.getOutput().size());
-		verifyElement(testHarness5.getOutput().poll(), 45);
-		verifyElement(testHarness5.getOutput().poll(), 10);
-		verifyWatermark(testHarness5.getOutput().poll(), 10);
-
-		testHarness1.close();
-		testHarness2.close();
-		testHarness3.close();
-		testHarness4.close();
-		testHarness5.close();
-	}
-
-	@Test
-	public void testWatermarkCallbackServiceKeyDeletion() throws Exception {
-		final int MAX_PARALLELISM = 10;
-
-		Tuple2<Integer, String> element1 = new Tuple2<>(7, "start");
-		Tuple2<Integer, String> element2 = new Tuple2<>(45, "start");
-		Tuple2<Integer, String> element3 = new Tuple2<>(90, "start");
-
-		TestOperatorWithDeletingKeyCallback op = new TestOperatorWithDeletingKeyCallback(45);
-
-		KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> testHarness1 =
-			new KeyedOneInputStreamOperatorTestHarness<>(
-				op,
-				new TestKeySelector(),
-				BasicTypeInfo.INT_TYPE_INFO,
-				MAX_PARALLELISM,
-				1,
-				0);
-		testHarness1.open();
-
-		testHarness1.processElement(new StreamRecord<>(element1));
-		testHarness1.processElement(new StreamRecord<>(element2));
-
-		testHarness1.processWatermark(10L);
-
-		assertEquals(3L, testHarness1.getOutput().size());
-		verifyElement(testHarness1.getOutput().poll(), 7);
-		verifyElement(testHarness1.getOutput().poll(), 45);
-		verifyWatermark(testHarness1.getOutput().poll(), 10);
-
-		testHarness1.processElement(new StreamRecord<>(element3));
-		testHarness1.processWatermark(20L);
-
-		// because at the first watermark the operator removed key 45
-		assertEquals(3L, testHarness1.getOutput().size());
-		verifyElement(testHarness1.getOutput().poll(), 7);
-		verifyElement(testHarness1.getOutput().poll(), 90);
-		verifyWatermark(testHarness1.getOutput().poll(), 20);
-
-		testHarness1.processWatermark(25L);
-
-		verifyElement(testHarness1.getOutput().poll(), 7);
-		verifyElement(testHarness1.getOutput().poll(), 90);
-		verifyWatermark(testHarness1.getOutput().poll(), 25);
-
-		// unregister key and then fail
-		op.unregisterKey(90);
-
-		// take a snapshot with some elements in internal sorting queue
-		OperatorStateHandles snapshot = testHarness1.snapshot(0, 0);
-		testHarness1.close();
-
-		testHarness1 = new KeyedOneInputStreamOperatorTestHarness<>(
-				new TestOperatorWithDeletingKeyCallback(45),
-				new TestKeySelector(),
-				BasicTypeInfo.INT_TYPE_INFO,
-				MAX_PARALLELISM,
-				1,
-				0);
-		testHarness1.setup();
-		testHarness1.initializeState(snapshot);
-		testHarness1.open();
-
-		testHarness1.processWatermark(30L);
-
-		assertEquals(2L, testHarness1.getOutput().size());
-		verifyElement(testHarness1.getOutput().poll(), 7);
-		verifyWatermark(testHarness1.getOutput().poll(), 30);
-
-		testHarness1.close();
-	}
-
-	private KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, Integer> getTestHarness(
-			int maxParallelism, int noOfTasks, int taskIdx) throws Exception {
-
-		return new KeyedOneInputStreamOperatorTestHarness<>(
-			new TestOperatorWithCallback(),
-			new TestKeySelector(),
-			BasicTypeInfo.INT_TYPE_INFO,
-			maxParallelism,
-			noOfTasks, /* num subtasks */
-			taskIdx /* subtask index */);
-	}
-
-	private void verifyWatermark(Object outputObject, long timestamp) {
-		Assert.assertTrue(outputObject instanceof Watermark);
-		assertEquals(timestamp, ((Watermark) outputObject).getTimestamp());
-	}
-
-	private void verifyElement(Object outputObject, int expected) {
-		Assert.assertTrue(outputObject instanceof StreamRecord);
-
-		StreamRecord<?> resultRecord = (StreamRecord<?>) outputObject;
-		Assert.assertTrue(resultRecord.getValue() instanceof Integer);
-
-		@SuppressWarnings("unchecked")
-		int actual = (Integer) resultRecord.getValue();
-		assertEquals(expected, actual);
-	}
-
 	/**
 	 * Extracts the result values form the test harness and clear the output queue.
 	 */
@@ -913,78 +639,6 @@ public Integer getKey(Tuple2<Integer, String> value) throws Exception {
 		}
 	}
 
-	private static class TestOperatorWithCallback
-			extends AbstractStreamOperator<Integer>
-			implements OneInputStreamOperator<Tuple2<Integer, String>, Integer> {
-
-		private static final long serialVersionUID = 9215057823264582305L;
-
-		@Override
-		public void open() throws Exception {
-			super.open();
-
-			InternalWatermarkCallbackService<Integer> callbackService = getInternalWatermarkCallbackService();
-
-			callbackService.setWatermarkCallback(new OnWatermarkCallback<Integer>() {
-
-				@Override
-				public void onWatermark(Integer integer, Watermark watermark) throws IOException {
-					output.collect(new StreamRecord<>(integer));
-				}
-			}, IntSerializer.INSTANCE);
-		}
-
-		@Override
-		public void processElement(StreamRecord<Tuple2<Integer, String>> element) throws Exception {
-			getInternalWatermarkCallbackService().registerKeyForWatermarkCallback(element.getValue().f0);
-		}
-	}
-
-	private static class TestOperatorWithDeletingKeyCallback
-			extends AbstractStreamOperator<Integer>
-			implements OneInputStreamOperator<Tuple2<Integer, String>, Integer> {
-
-		private static final long serialVersionUID = 9215057823264582305L;
-
-		private final int keyToDelete;
-
-		public TestOperatorWithDeletingKeyCallback(int keyToDelete) {
-			this.keyToDelete = keyToDelete;
-		}
-
-		@Override
-		public void open() throws Exception {
-			super.open();
-
-			InternalWatermarkCallbackService<Integer> callbackService = getInternalWatermarkCallbackService();
-
-			callbackService.setWatermarkCallback(new OnWatermarkCallback<Integer>() {
-
-				@Override
-				public void onWatermark(Integer integer, Watermark watermark) throws IOException {
-
-					// this is to simulate the case where we may have a concurrent modification
-					// exception as we iterate over the list of registered keys and we concurrently
-					// delete the key.
-
-					if (integer.equals(keyToDelete)) {
-						getInternalWatermarkCallbackService().unregisterKeyFromWatermarkCallback(integer);
-					}
-					output.collect(new StreamRecord<>(integer));
-				}
-			}, IntSerializer.INSTANCE);
-		}
-
-		@Override
-		public void processElement(StreamRecord<Tuple2<Integer, String>> element) throws Exception {
-			getInternalWatermarkCallbackService().registerKeyForWatermarkCallback(element.getValue().f0);
-		}
-
-		public void unregisterKey(int key) {
-			getInternalWatermarkCallbackService().unregisterKeyFromWatermarkCallback(key);
-		}
-	}
-
 	/**
 	 * Testing operator that can respond to commands by either setting/deleting state, emitting
 	 * state or setting timers.
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
index a2a4f79e88bfa..4a9463a1593c1 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
@@ -610,15 +610,6 @@ public int numEventTimeTimers() {
 		}
 	}
 
-	@VisibleForTesting
-	public int numKeysForWatermarkCallback() {
-		if (operator instanceof AbstractStreamOperator) {
-			return ((AbstractStreamOperator) operator).numKeysForWatermarkCallback();
-		} else {
-			throw new UnsupportedOperationException();
-		}
-	}
-
 	private class MockOutput implements Output<StreamRecord<OUT>> {
 
 		private TypeSerializer<OUT> outputSerializer;

From 282788bff312ce2e114e4b3c1f0901e8dc949350 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Thu, 20 Apr 2017 14:48:22 +0200
Subject: [PATCH 118/253] [FLINK-5969] Augment SavepointMigrationTestBase to
 catch failed jobs

---
 .../flink/client/program/ClusterClient.java   | 32 ++++++++++++++++++
 .../utils/SavepointMigrationTestBase.java     | 33 ++++++++++++++++++-
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
index 8d0e84192ad02..ab4daa95155c6 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
@@ -488,6 +488,38 @@ public JobExecutionResult retrieveJob(JobID jobID) throws JobExecutionException
 		return JobClient.awaitJobResult(listeningContext);
 	}
 
+	/**
+	 * Reattaches to a running job with the given job id.
+	 *
+	 * @param jobID The job id of the job to attach to
+	 * @return The JobExecutionResult for the jobID
+	 * @throws JobExecutionException if an error occurs during monitoring the job execution
+	 */
+	public JobListeningContext connectToJob(JobID jobID) throws JobExecutionException {
+		final LeaderRetrievalService leaderRetrievalService;
+		try {
+			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig);
+		} catch (Exception e) {
+			throw new JobRetrievalException(jobID, "Could not create the leader retrieval service", e);
+		}
+
+		ActorGateway jobManagerGateway;
+		try {
+			jobManagerGateway = getJobManagerGateway();
+		} catch (Exception e) {
+			throw new JobRetrievalException(jobID, "Could not retrieve the JobManager Gateway", e);
+		}
+
+		return JobClient.attachToRunningJob(
+				jobID,
+				jobManagerGateway,
+				flinkConfig,
+				actorSystemLoader.get(),
+				leaderRetrievalService,
+				timeout,
+				printStatusDuringExecution);
+	}
+
 	/**
 	 * Cancels a job identified by the job id.
 	 * @param jobId the job id
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java
index fced68c816a10..301fc72a59c33 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java
@@ -25,14 +25,17 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobSubmissionResult;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.client.program.StandaloneClusterClient;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobListeningContext;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
@@ -49,6 +52,7 @@
 import scala.concurrent.Await;
 import scala.concurrent.Future;
 import scala.concurrent.duration.Deadline;
+import scala.concurrent.duration.Duration;
 import scala.concurrent.duration.FiniteDuration;
 
 import static junit.framework.Assert.fail;
@@ -207,11 +211,38 @@ protected final void restoreAndExecute(
 		JobSubmissionResult jobSubmissionResult = cluster.submitJobDetached(jobGraph);
 
 		StandaloneClusterClient clusterClient = new StandaloneClusterClient(cluster.configuration());
+		JobListeningContext jobListeningContext = clusterClient.connectToJob(jobSubmissionResult.getJobID());
 
 		boolean done = false;
 		while (DEADLINE.hasTimeLeft()) {
+
+			// try and get a job result, this will fail if the job already failed. Use this
+			// to get out of this loop
+			JobID jobId = jobSubmissionResult.getJobID();
+			FiniteDuration timeout = FiniteDuration.apply(5, TimeUnit.SECONDS);
+
+			try {
+
+				Future<Object> future = clusterClient
+						.getJobManagerGateway()
+						.ask(JobManagerMessages.getRequestJobStatus(jobSubmissionResult.getJobID()), timeout);
+
+				Object result = Await.result(future, timeout);
+
+				if (result instanceof JobManagerMessages.CurrentJobStatus) {
+					if (((JobManagerMessages.CurrentJobStatus) result).status() == JobStatus.FAILED) {
+						Object jobResult = Await.result(
+								jobListeningContext.getJobResultFuture(),
+								Duration.apply(5, TimeUnit.SECONDS));
+						fail("Job failed: " + jobResult);
+					}
+				}
+			} catch (Exception e) {
+				fail("Could not connect to job: " + e);
+			}
+
 			Thread.sleep(100);
-			Map<String, Object> accumulators = clusterClient.getAccumulators(jobSubmissionResult.getJobID());
+			Map<String, Object> accumulators = clusterClient.getAccumulators(jobId);
 
 			boolean allDone = true;
 			for (Tuple2<String, Integer> acc : expectedAccumulators) {

From da641d54af525f3d1c99e70f16d3e4191c76cbf5 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Thu, 20 Apr 2017 10:46:10 +0200
Subject: [PATCH 119/253] [FLINK-5969] Add savepoint IT case that checks
 restore from 1.2

The binary savepoints in this were created on the Flink 1.2.0 release
commit.
---
 .../utils/SavepointMigrationTestBase.java     |  24 +-
 ...fulJobSavepointFrom11MigrationITCase.java} |   2 +-
 ...efulJobSavepointFrom12MigrationITCase.java | 769 ++++++++++++++++++
 ...ul-udf-migration-itcase-flink1.2-savepoint | Bin 0 -> 25245 bytes
 ...igration-itcase-flink1.2-savepoint-rocksdb | Bin 0 -> 25256 bytes
 5 files changed, 771 insertions(+), 24 deletions(-)
 rename flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/{StatefulUDFSavepointMigrationITCase.java => StatefulJobSavepointFrom11MigrationITCase.java} (99%)
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
 create mode 100644 flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint
 create mode 100644 flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint-rocksdb

diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java
index 301fc72a59c33..c5672a21a974a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java
@@ -155,15 +155,10 @@ protected final void executeAndSavepoint(
 		}
 
 		LOG.info("Triggering savepoint.");
-		// Flink 1.2
+
 		final Future<Object> savepointResultFuture =
 				jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobSubmissionResult.getJobID(), Option.<String>empty()), DEADLINE.timeLeft());
 
-		// Flink 1.1
-//		final Future<Object> savepointResultFuture =
-//				jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobSubmissionResult.getJobID()), DEADLINE.timeLeft());
-
-
 		Object savepointResult = Await.result(savepointResultFuture, DEADLINE.timeLeft());
 
 		if (savepointResult instanceof JobManagerMessages.TriggerSavepointFailure) {
@@ -174,24 +169,7 @@ protected final void executeAndSavepoint(
 		final String jobmanagerSavepointPath = ((JobManagerMessages.TriggerSavepointSuccess) savepointResult).savepointPath();
 		LOG.info("Saved savepoint: " + jobmanagerSavepointPath);
 
-		// Flink 1.2
 		FileUtils.moveFile(new File(new URI(jobmanagerSavepointPath).getPath()), new File(savepointPath));
-
-		// Flink 1.1
-		// Retrieve the savepoint from the testing job manager
-//		LOG.info("Requesting the savepoint.");
-//		Future<Object> savepointFuture = jobManager.ask(new TestingJobManagerMessages.RequestSavepoint(jobmanagerSavepointPath), DEADLINE.timeLeft());
-//
-//		Savepoint savepoint = ((TestingJobManagerMessages.ResponseSavepoint) Await.result(savepointFuture, DEADLINE.timeLeft())).savepoint();
-//		LOG.info("Retrieved savepoint: " + jobmanagerSavepointPath + ".");
-//
-//		LOG.info("Storing savepoint to file.");
-//		Configuration config = new Configuration();
-//		config.setString(org.apache.flink.runtime.checkpoint.savepoint.SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
-//		config.setString(org.apache.flink.runtime.checkpoint.savepoint.SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, "file:///Users/aljoscha/Downloads");
-//		String path = org.apache.flink.runtime.checkpoint.savepoint.SavepointStoreFactory.createFromConfig(config).storeSavepoint(savepoint);
-//
-//		FileUtils.moveFile(new File(new URI(path).getPath()), new File(savepointPath));
 	}
 
 	@SafeVarargs
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulUDFSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom11MigrationITCase.java
similarity index 99%
rename from flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulUDFSavepointMigrationITCase.java
rename to flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom11MigrationITCase.java
index 10a89989f3f5a..4d94d25c869f3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulUDFSavepointMigrationITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom11MigrationITCase.java
@@ -54,7 +54,7 @@
  *
  * <p>The test pipeline contains both "Checkpointed" state and keyed user state.
  */
-public class StatefulUDFSavepointMigrationITCase extends SavepointMigrationTestBase {
+public class StatefulJobSavepointFrom11MigrationITCase extends SavepointMigrationTestBase {
 	private static final int NUM_SOURCE_ELEMENTS = 4;
 	private static final String EXPECTED_ELEMENTS_ACCUMULATOR = "NUM_EXPECTED_ELEMENTS";
 	private static final String SUCCESSFUL_CHECK_ACCUMULATOR = "SUCCESSFUL_CHECKS";
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
new file mode 100644
index 0000000000000..e60cb5d9e5026
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
@@ -0,0 +1,769 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.checkpointing.utils;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.flink.api.common.accumulators.IntCounter;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.api.common.typeutils.base.LongSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FSDataOutputStream;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.checkpoint.Checkpointed;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedRestoring;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
+import org.apache.flink.streaming.api.operators.InternalTimer;
+import org.apache.flink.streaming.api.operators.InternalTimerService;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.TimestampedCollector;
+import org.apache.flink.streaming.api.operators.Triggerable;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.util.Collector;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * This verifies that we can restore a complete job from a Flink 1.2 savepoint.
+ *
+ * <p>The test pipeline contains both "Checkpointed" state and keyed user state.
+ *
+ * <p>The tests will time out if they don't see the required number of successful checks within
+ * a time limit.
+ */
+public class StatefulJobSavepointFrom12MigrationITCase extends SavepointMigrationTestBase {
+	private static final int NUM_SOURCE_ELEMENTS = 4;
+
+	/**
+	 * This has to be manually executed to create the savepoint on Flink 1.2.
+	 */
+	@Test
+	@Ignore
+	public void testCreateSavepointOnFlink12() throws Exception {
+
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+		// we only test memory state backend yet
+		env.setStateBackend(new MemoryStateBackend());
+		env.enableCheckpointing(500);
+		env.setParallelism(4);
+		env.setMaxParallelism(4);
+
+		env
+				.addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+				.flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+				.keyBy(0)
+				.flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+				.keyBy(0)
+				.flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+				.keyBy(0)
+				.transform(
+						"custom_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
+				.keyBy(0)
+				.transform(
+						"timely_stateful_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
+				.addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+		executeAndSavepoint(
+				env,
+				"src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint",
+				new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+	}
+
+	/**
+	 * This has to be manually executed to create the savepoint on Flink 1.2.
+	 */
+	@Test
+	@Ignore
+	public void testCreateSavepointOnFlink12WithRocksDB() throws Exception {
+
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+		RocksDBStateBackend rocksBackend =
+				new RocksDBStateBackend(new MemoryStateBackend());
+		env.setStateBackend(rocksBackend);
+		env.enableCheckpointing(500);
+		env.setParallelism(4);
+		env.setMaxParallelism(4);
+
+		env
+				.addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+				.flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+				.keyBy(0)
+				.flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+				.keyBy(0)
+				.flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+				.keyBy(0)
+				.transform(
+						"custom_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
+				.keyBy(0)
+				.transform(
+						"timely_stateful_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
+				.addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+		executeAndSavepoint(
+				env,
+				"src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint-rocksdb",
+				new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+	}
+
+
+	@Test
+	public void testSavepointRestoreFromFlink12() throws Exception {
+
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.setRestartStrategy(RestartStrategies.noRestart());
+		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+		// we only test memory state backend yet
+		env.setStateBackend(new MemoryStateBackend());
+		env.enableCheckpointing(500);
+		env.setParallelism(4);
+		env.setMaxParallelism(4);
+
+		env
+				.addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+				.flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+				.keyBy(0)
+				.flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+				.keyBy(0)
+				.flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+				.keyBy(0)
+				.transform(
+						"custom_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
+				.keyBy(0)
+				.transform(
+						"timely_stateful_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
+				.addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+		restoreAndExecute(
+				env,
+				getResourceFilename("stateful-udf-migration-itcase-flink1.2-savepoint"),
+				new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
+				new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+	}
+
+	@Test
+	public void testSavepointRestoreFromFlink12FromRocksDB() throws Exception {
+
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.setRestartStrategy(RestartStrategies.noRestart());
+		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+		// we only test memory state backend yet
+		env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
+		env.enableCheckpointing(500);
+		env.setParallelism(4);
+		env.setMaxParallelism(4);
+
+		env
+				.addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+				.flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+				.keyBy(0)
+				.flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+				.keyBy(0)
+				.flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+				.keyBy(0)
+				.transform(
+						"custom_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
+				.keyBy(0)
+				.transform(
+						"timely_stateful_operator",
+						new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+						new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
+				.addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+		restoreAndExecute(
+				env,
+				getResourceFilename("stateful-udf-migration-itcase-flink1.2-savepoint-rocksdb"),
+				new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
+				new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+				new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+	}
+
+	private static class LegacyCheckpointedSource
+			implements SourceFunction<Tuple2<Long, Long>>, Checkpointed<String> {
+
+		public static String CHECKPOINTED_STRING = "Here be dragons!";
+
+		private static final long serialVersionUID = 1L;
+
+		private volatile boolean isRunning = true;
+
+		private final int numElements;
+
+		public LegacyCheckpointedSource(int numElements) {
+			this.numElements = numElements;
+		}
+
+		@Override
+		public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
+
+			ctx.emitWatermark(new Watermark(0));
+
+			synchronized (ctx.getCheckpointLock()) {
+				for (long i = 0; i < numElements; i++) {
+					ctx.collect(new Tuple2<>(i, i));
+				}
+			}
+
+			// don't emit a final watermark so that we don't trigger the registered event-time
+			// timers
+			while (isRunning) {
+				Thread.sleep(20);
+			}
+		}
+
+		@Override
+		public void cancel() {
+			isRunning = false;
+		}
+
+		@Override
+		public void restoreState(String state) throws Exception {
+			assertEquals(CHECKPOINTED_STRING, state);
+		}
+
+		@Override
+		public String snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+			return CHECKPOINTED_STRING;
+		}
+	}
+
+	private static class CheckingRestoringSource
+			extends RichSourceFunction<Tuple2<Long, Long>>
+			implements CheckpointedRestoring<String> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringSource.class + "_RESTORE_CHECK";
+
+		private volatile boolean isRunning = true;
+
+		private final int numElements;
+
+		private String restoredState;
+
+		public CheckingRestoringSource(int numElements) {
+			this.numElements = numElements;
+		}
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+
+			getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
+			assertEquals(LegacyCheckpointedSource.CHECKPOINTED_STRING, restoredState);
+			getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+
+			// immediately trigger any set timers
+			ctx.emitWatermark(new Watermark(1000));
+
+			synchronized (ctx.getCheckpointLock()) {
+				for (long i = 0; i < numElements; i++) {
+					ctx.collect(new Tuple2<>(i, i));
+				}
+			}
+
+			while (isRunning) {
+				Thread.sleep(20);
+			}
+		}
+
+		@Override
+		public void cancel() {
+			isRunning = false;
+		}
+
+		@Override
+		public void restoreState(String state) throws Exception {
+			restoredState = state;
+		}
+	}
+
+	public static class LegacyCheckpointedFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>
+			implements Checkpointed<Tuple2<String, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static Tuple2<String, Long> CHECKPOINTED_TUPLE =
+				new Tuple2<>("hello", 42L);
+
+		@Override
+		public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+			out.collect(value);
+		}
+
+		@Override
+		public void restoreState(Tuple2<String, Long> state) throws Exception {
+		}
+
+		@Override
+		public Tuple2<String, Long> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+			return CHECKPOINTED_TUPLE;
+		}
+	}
+
+	public static class CheckingRestoringFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>
+			implements CheckpointedRestoring<Tuple2<String, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMap.class + "_RESTORE_CHECK";
+
+		private transient Tuple2<String, Long> restoredState;
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+
+			getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+			out.collect(value);
+
+			assertEquals(LegacyCheckpointedFlatMap.CHECKPOINTED_TUPLE, restoredState);
+			getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+
+		}
+
+		@Override
+		public void restoreState(Tuple2<String, Long> state) throws Exception {
+			restoredState = state;
+		}
+	}
+
+	public static class LegacyCheckpointedFlatMapWithKeyedState
+			extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>
+			implements Checkpointed<Tuple2<String, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static Tuple2<String, Long> CHECKPOINTED_TUPLE =
+				new Tuple2<>("hello", 42L);
+
+		private final ValueStateDescriptor<Long> stateDescriptor =
+				new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+		@Override
+		public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+			out.collect(value);
+
+			getRuntimeContext().getState(stateDescriptor).update(value.f1);
+
+			assertEquals(value.f1, getRuntimeContext().getState(stateDescriptor).value());
+		}
+
+		@Override
+		public void restoreState(Tuple2<String, Long> state) throws Exception {
+		}
+
+		@Override
+		public Tuple2<String, Long> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+			return CHECKPOINTED_TUPLE;
+		}
+	}
+
+	public static class CheckingRestoringFlatMapWithKeyedState extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>
+			implements CheckpointedRestoring<Tuple2<String, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedState.class + "_RESTORE_CHECK";
+
+		private transient Tuple2<String, Long> restoredState;
+
+		private final ValueStateDescriptor<Long> stateDescriptor =
+				new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+
+			getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+			out.collect(value);
+
+			ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+			if (state == null) {
+				throw new RuntimeException("Missing key value state for " + value);
+			}
+
+			assertEquals(value.f1, state.value());
+			assertEquals(LegacyCheckpointedFlatMap.CHECKPOINTED_TUPLE, restoredState);
+			getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+		}
+
+		@Override
+		public void restoreState(Tuple2<String, Long> state) throws Exception {
+			restoredState = state;
+		}
+	}
+
+	public static class CheckingRestoringFlatMapWithKeyedStateInOperator extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>
+			implements CheckpointedRestoring<Tuple2<String, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedStateInOperator.class + "_RESTORE_CHECK";
+
+		private transient Tuple2<String, Long> restoredState;
+
+		private final ValueStateDescriptor<Long> stateDescriptor =
+				new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+
+			getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+			out.collect(value);
+
+			ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+			if (state == null) {
+				throw new RuntimeException("Missing key value state for " + value);
+			}
+
+			assertEquals(value.f1, state.value());
+			assertEquals(LegacyCheckpointedFlatMap.CHECKPOINTED_TUPLE, restoredState);
+			getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+		}
+
+		@Override
+		public void restoreState(Tuple2<String, Long> state) throws Exception {
+			restoredState = state;
+		}
+	}
+
+	public static class KeyedStateSettingFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		private final ValueStateDescriptor<Long> stateDescriptor =
+				new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+		@Override
+		public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+			out.collect(value);
+
+			getRuntimeContext().getState(stateDescriptor).update(value.f1);
+		}
+	}
+
+	public static class CheckingKeyedStateFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingKeyedStateFlatMap.class + "_RESTORE_CHECK";
+
+		private final ValueStateDescriptor<Long> stateDescriptor =
+				new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+
+			getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+			out.collect(value);
+
+			ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+			if (state == null) {
+				throw new RuntimeException("Missing key value state for " + value);
+			}
+
+			assertEquals(value.f1, state.value());
+			getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+		}
+	}
+
+	public static class CheckpointedUdfOperator
+			extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
+			implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+		private static final long serialVersionUID = 1L;
+
+		private static final String CHECKPOINTED_STRING = "Oh my, that's nice!";
+
+		public CheckpointedUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
+			super(userFunction);
+		}
+
+		@Override
+		public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+			userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
+		}
+
+		@Override
+		public void processWatermark(Watermark mark) throws Exception {
+			output.emitWatermark(mark);
+		}
+
+		@Override
+		public void snapshotState(
+				FSDataOutputStream out, long checkpointId, long timestamp) throws Exception {
+			super.snapshotState(out, checkpointId, timestamp);
+
+			DataOutputViewStreamWrapper streamWrapper = new DataOutputViewStreamWrapper(out);
+
+			streamWrapper.writeUTF(CHECKPOINTED_STRING);
+			streamWrapper.flush();
+		}
+	}
+
+	public static class CheckingRestoringUdfOperator
+			extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
+			implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+		private static final long serialVersionUID = 1L;
+
+		public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringUdfOperator.class + "_RESTORE_CHECK";
+
+		private String restoredState;
+
+		public CheckingRestoringUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
+			super(userFunction);
+		}
+
+		@Override
+		public void open() throws Exception {
+			super.open();
+
+			getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+			userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
+
+			assertEquals(CheckpointedUdfOperator.CHECKPOINTED_STRING, restoredState);
+			getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+		}
+
+		@Override
+		public void processWatermark(Watermark mark) throws Exception {
+			output.emitWatermark(mark);
+		}
+
+		@Override
+		public void restoreState(FSDataInputStream in) throws Exception {
+			super.restoreState(in);
+
+			DataInputViewStreamWrapper streamWrapper = new DataInputViewStreamWrapper(in);
+
+			restoredState = streamWrapper.readUTF();
+		}
+	}
+
+	public static class TimelyStatefulOperator
+			extends AbstractStreamOperator<Tuple2<Long, Long>>
+			implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
+		private static final long serialVersionUID = 1L;
+
+		private final ValueStateDescriptor<Long> stateDescriptor =
+				new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+		private transient InternalTimerService<Long> timerService;
+
+		@Override
+		public void open() throws Exception {
+			super.open();
+
+			timerService = getInternalTimerService(
+					"timer",
+					LongSerializer.INSTANCE,
+					this);
+
+		}
+
+		@Override
+		public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+			ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+					element.getValue().f0,
+					LongSerializer.INSTANCE,
+					stateDescriptor);
+
+			state.update(element.getValue().f1);
+
+			timerService.registerEventTimeTimer(element.getValue().f0, timerService.currentWatermark() + 10);
+			timerService.registerProcessingTimeTimer(element.getValue().f0, timerService.currentProcessingTime() + 30_000);
+
+			output.collect(element);
+		}
+
+		@Override
+		public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
+
+		}
+
+		@Override
+		public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
+
+		}
+
+		@Override
+		public void processWatermark(Watermark mark) throws Exception {
+			output.emitWatermark(mark);
+		}
+	}
+
+	public static class CheckingTimelyStatefulOperator
+			extends AbstractStreamOperator<Tuple2<Long, Long>>
+			implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
+		private static final long serialVersionUID = 1L;
+
+		public static final String SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PROCESS_CHECKS";
+		public static final String SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_ET_CHECKS";
+		public static final String SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PT_CHECKS";
+
+		private final ValueStateDescriptor<Long> stateDescriptor =
+				new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+		private transient InternalTimerService<Long> timerService;
+
+		@Override
+		public void open() throws Exception {
+			super.open();
+
+			timerService = getInternalTimerService(
+					"timer",
+					LongSerializer.INSTANCE,
+					this);
+
+			getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, new IntCounter());
+			getRuntimeContext().addAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, new IntCounter());
+			getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+			ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+					element.getValue().f0,
+					LongSerializer.INSTANCE,
+					stateDescriptor);
+
+			assertEquals(state.value(), element.getValue().f1);
+			getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR).add(1);
+
+			output.collect(element);
+		}
+
+		@Override
+		public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
+			ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+					timer.getNamespace(),
+					LongSerializer.INSTANCE,
+					stateDescriptor);
+
+			assertEquals(state.value(), timer.getNamespace());
+			getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
+		}
+
+		@Override
+		public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
+			ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+					timer.getNamespace(),
+					LongSerializer.INSTANCE,
+					stateDescriptor);
+
+			assertEquals(state.value(), timer.getNamespace());
+			getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
+		}
+	}
+
+	public static class AccumulatorCountingSink<T> extends RichSinkFunction<T> {
+		private static final long serialVersionUID = 1L;
+
+		public static final String NUM_ELEMENTS_ACCUMULATOR = AccumulatorCountingSink.class + "_NUM_ELEMENTS";
+
+		int count = 0;
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+
+			getRuntimeContext().addAccumulator(NUM_ELEMENTS_ACCUMULATOR, new IntCounter());
+		}
+
+		@Override
+		public void invoke(T value) throws Exception {
+			count++;
+			getRuntimeContext().getAccumulator(NUM_ELEMENTS_ACCUMULATOR).add(1);
+		}
+	}
+}
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint
new file mode 100644
index 0000000000000000000000000000000000000000..a8d19f2af31e54187690468b4192d409adb4a7ca
GIT binary patch
literal 25245
zcmeHPTZklA8LpY#jSx{+#S5aaNhUE!&dj;hxwsE%R7k>-2)p=T5OS`w6MJUXp5B;M
zko6@#1W|B<isFN=SMnwVf=V7#0=nXx1QntZP=v)BsNf}jr@E@Ar>E6*9YZ!<2dcZe
zy3VPpQ|El&e>?x3_pOtvs+O>oay_=|9{a<OFWmaMH^24C$NqTpmtXb8AK}>Hb}Q$V
z{p#7aV0US$S`o?W6ip(OWksatkyw{FaWYbBT@XPjocf!kM=w`Lc81k0+r#x$cgBU&
zX*CZvw%&&y8>{!b2i)rD?3qDYJykw=xf06Ka<zP-TF!J-z2ZbUX>H)P*4OUw_opyA
zUjAY4uRdHYJvgjxzM_W5&6akn2hWVEBd60~u#MY#X>$jq@zoRCTkFRke&!RGK7P~F
z%hl3d)sY9>;B3Mrj_#gx_t{OKh7UdZ$QxdK;pY!Gt+h?RsoZwi$H$@eSl7uowvtUy
zF6KZ~;-1JPBr)6>;#FqWdK+9FP_60f(#lMK`{*4*uaeATCNerpjIAwkx|Z3h>?uvC
zwvhc@y1R6B+uetsM0az^ZAvl_E^9+1r-nGr9FffESc{s<tkK)Q+O4|$R6A6U|C<qY
z?o2v6+8FGt4$p3lHa6Q~dom3-++gE_X^7@ZX~x;GddvUN((P`iywFn6+TlB^BW}1c
zI(KJv-5_nPk51oP9nBkQ5O=S=Gph8xCveHN@jP2=<{5r$tcC5(&F!tVQFr@mr?9tM
z6Eo`iW){AE;C9f=%ctgCbLco`ft>p=<Ca$^EwAovdHewFyep4_Vf~M@!}dn^1Hyl3
z?zsKP>%a2&tN;4xZssjv4%;icF^>*yCi(7jPv70Jy8XXsXYW&<+}K(lq|x@)xvzcn
z>rbwJ;GO063P2rx;7XB??cU4Mzec{R!@G8JT)#7NqeQmcW*JyzvOL|6c(w1zyW!nb
zS$+5RM!Xx<c7Weg!)EuEE8N%tci%w`INanf+sAyK4wQp?(g_H*NH`&)Lo*OrUjIsT
z(m84s;ha(;70;k>UK6SulZGk5V;LwNBQ4J`l8l@(Ya|#+fzX&&#Me?0BfM2Cc$>mP
zGZHXh4<lJ;g!91>W@sc*)PWcoAO(P>GOJU~8b<naTMe(UXQV;u8EK!;ypJOrX6qSg
zpZL`?Qjz+mWV0(BC<nwyK)?wj6*QbzI;L7G3^fIr4MWw`Bx9u{NeU=EDTASGQ8Go8
z!Rw$btx>IGs6#UhrCb6*=}25aP3ep$P*FJ%K}slNMhBTuePMN`PADi{Sac@ERE<u}
z#5)6>DJns{gwAA?tV3{2TkU(Dsnzs)o$3D|^!GYbv5H+a!5<KA7ffCZoe2myVW^IY
zI;L7K@HhpL4I_mDh+4wzX?;zk@syZoG!a%%(22Gt03yeXbm8Q?zIFNH6Q6wc(l7q<
z1Nnuu_IP2da;=87W*93*my?p2D8nV1XN*_@BdhX-$FdNxv7+j;3uIM&<vpj5ZJxXB
z*yywyy>aK*)<#G-v^`k5vAm#Cy0?d&6-{%Xg4Kk%V)4u9Knwkfz(mR%x>(!Vph_n;
z%$iz_jFAQwh)7V+q|+coUf!`2vgX2Pp^Mnizl3q8FHI&%`wkjeB5T8`K&NX76M@V!
z!qROW30&2+aZ!r^lUHwlZLG$<EYZsndu`#p{T1H0=g3fha8gqNzy<_6HtN`EuKhI_
zV^T~EN~sq_T37|WglHih*WS3CK~+;UHEVz6*+%Ilg@hnrPh=tyco97bq2|;kEt2YG
zg=tx#yo@OeLqk*Wo+_zp@DkII=e(@pR*i`SttpGDPKH|0RBx0m&dr{tdYbBKs*3=4
zuKmdeL{mV(2~BmXE`V`Q>F8^&QC27#gRmOT!V+0Qr6o2nNtjd)!4?Q-(It$h)2?QX
zvRYZGJ<b=d1P&=wcT#Y|O{#<TA#x>9z1OZfNutnJ+1i)V(3Z=LG7Yb-7ly%?=p78>
z_{g=cxs7%99vW`1UG>@(#tmlq!G~w`mfGG@i|DsKVPtc|pbm((fPfR)>gcJXulbf*
zNqsUkTv*YUfJx|CCc^_5y%wHIFSL>!O<nxV{eODTjqmv8&(`0zde7;j_qE3hg_Uc}
zT58AF-Tf@+TuOIe*jeFD^IF1JDRTtPQKN{nA;OoYRgADOa_)iaKWwTUDWQbJKK!r`
z_IM!%dn`osT>}7RGEaGQb`he%kxq!~1i9}6CcO38J8wvhxMU*hXoxWgQQ%3|j#=il
zcL1Q@x9EbZeNb2*6gFoi#Z~)^uK_`BF?`Vl78d47%fptflnxiW{pPzn*qu~lDq&P~
z!YKj~3Si0MQ#8~epti0f{EdPM3!mblzoUVrs+XzyB^?L7CgJ&ml?x$Lb#uSduMV9r
z&@f)JxbHvA`Fk(ApA>~^+EX@11}lf!>W7OACZQ)<fp8u!lZC!UJ;BnknnV+oHY_s)
zi&oW<m$cPTwU@zq8LXGVrq=P!mp;pi+H1~-zaTOgK(Juo7#^{M(n0F@*Y0OI-u*ZE
zE|@$8q~(kii?{_u*Ey1sK|lku5|QM|5tpO$wQ_>R;*}ULKkwzUUOwyPv+-i`YuA<B
zpu_|e&Nn}soD$=`_H*iZ2RM^DKn}=lKP|~3X3FIFv<rD$fAU#is8bcO+sJc~GA4Pb
zi8ol2DtWFE{)?Dumuul@&~PyO-lo<sV=2RJVV$E3V`*9tc@7|0Ahm<l0+7@DWvtJ2
zlbY;Q&0$W%;*zpfqNW7vBoR-Fs8Y?n!xGHU+tkL%UHbUP7jz)WLfO>D5)?q;e1V3&
zHnqurz>s27d+NK-eE!ttLyx@X{N>Ac-29ynUBdYt*mU2s`N0tJVBy!aBuj=M2&BHk
zLb>Eg@;YTLvNyD>!Fp}8l5JESy(10NF`tgbp<CbH+PR^<M01@N-FEliZhhH%fA_@q
zM*LG3P086*LNdx$E<YQdQI3tl4|_#8ETuU(-dPD+NW~E9EfkbhDMBEYa>O!fE|M@B
zf%VhN%BNkr(`|QhXWMiWm?!bH#%hKnDkYv=mw0jwDHIvHYC)Oc%pf?j56tTW^Drba
z&-%c;JrOVcvZrg&WLf}l7Z~xQ1KqjHo)S;F#u6ti1+@X{Ino&dFewQ#AZU)N=s86j
z4XWdN+L^O^k@2F_DJi!>--O(gF<4}y;;Cr<kYf>~9!af&b6f;0twwBlznZ5<-yVId
zKIz}S)JWH!!wLvELElbn2Wajo4Qk@Z+|@h=2`hAf-<u)J50N2otkZ{_?n9lkkxEcI
zjmFca*jY60<Y8QDEDI}_L#+%!1T_5WHYbEYW8DyvSy+TW*Dx@IUKZ}{uw}6JvT&b*
zbv9AmwTH$)zzG_6sN13PT<fb((o+*Dq1*!^nMX1J&5_z0IlVQqu-1xAQm}sZxBjM;
z*ZumgZ-40<AG_|Wx7=7@w%y9L_E|yTkT){1ghT?wgpzX9Ne+3~BY$2BPSUIt%n_)%
zkO}bc+<044(fTE;y#nhQZ!^hWBEmE)B)_v>6NgL-CdfK-i*x`B++lmPF?5)tRB}c$
z?0Ok$SIlJqBSKf2#6m<WVCjk}ZE;7~9)SrRMv>A|F=V4hKL3*R0|}~-RguE|qJm*Q
K)0h@`g#8m^^Bl?m

literal 0
HcmV?d00001

diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint-rocksdb b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.2-savepoint-rocksdb
new file mode 100644
index 0000000000000000000000000000000000000000..548993f09290b0c06a20f0e674d58be3b01c450f
GIT binary patch
literal 25256
zcmeHPTZmm(8QwEVYKfq!;ssGi+q4$xnpyX~mw8aDLR&&Br1)SFx~`LnbBSk8Vxu5Q
z1NEU41zS`UA2eR+ix&z~r4K3vt@tKB2vw;lLZ~Q+;3a<lK4;IInaR|oPQ~Qx1?QZ7
zTkGt-_FCWn{r~#^|E>oXSXEUs_)yu8&vj4!`4^Af_Jy~<^O-0Aa?4j=_vN4C*xvDn
zuA}Yh;<#XQW~Mr1JU7C7#gwQW(^5F*6c;R}y5?RdEds}-f1G*ZLbY#gP~EyZSeWxC
zd_0!t%F^P>hp@9acif-ybHkG-mh#+v^eyS1P-bSU*(23#;lt`RN9dyYCBL#Te{VRR
z<M1$@5g)ETUd=o>sBXEehDXg(yVZjyhSk1fd1+}CxAm&_4yf_DBdaS5haY+VQ_p_#
z=I3UsnY*ifr~J~%jK4UranU^|mqQ*r{KVsLdF5ljexz+}-1OV%wrLw*d&bAQPR6lA
zC2$$N$gGs4n667=P4b*YC!^)ohfoVtYx|m7+0x(Kt5kC#SSUd=Eu2Dihh1(`G{wj|
zTU49M{-*Axt{!*y-e=I=Jcq1`jjZsx!T2lz<5NUtr48IUr)-_j+rQhWI{VA<P(A!2
zBkJ^td~&$Bv^F<5xiVZ_9uM22d9di07C)K?XpTxd&IZ*x{)?9G@N0S07`3+du4<nj
zEDlfKRb97~R~ClH?ynA%#k`a@{(Eg$)%PF4U(S!_*?c?Cu(LQHSC^MpSLTP^?a$wb
zt&N(PQTy9j`0gdYhGuS<m~-u+!<Ys7k`FU#d2YSsxy>zaJOSn%bqqe&pB#*DbUPsY
zhvp8uvv2;|_g??^&vr9!26Nb5-i-xxXgkR_o_q4{2Gt!`(9Y(kJi55Du#|_ZE2qEl
ziElnL_mTI~?GZrjJ$1RrM{}PF-u2AR$oJClZag{muMPb$vlYKg1B>3obK?<jw>^0`
zyxS^s?_XU^_n_Ju@Oxs|(EXx}2m3B}-!2Zg-;N*i|8$`2(vw1#pB=rBk^H&7+qrmg
z3qaD#Kcp15JP5^d6aYxCJo8=_>;|QbkS>J=Ep+l|c8ffcIV4Ck&yvEIHnsp?OT&zf
zHrQ;Vwg%OwRvrNX+9^g|Nb;ywvW`}J7MrFp-4w|}ijo<n>X3@(S~^^Maa;|vvzJFp
zxtB-VDDyT6*~+zE9&ICEy*wh$H^Iy<ccAQ^JR%TWFP%K11N$EuA0`%3b!VXSkffS`
z&Q?<8>>_VK<&%VhsYL^=#|OgzoJMIaY$zmZI!U!>i=@(to7!i|qR2JVfhQJB(=g?Y
z48aK~3MA;JR#J83fVz+*Rn~PjUN<a4muK1~15Fb>mwarp%hCE8nr5##wfbJKIaSa}
zb{T4W&53MdFImv<5`GgjPmShu4i8JEl~|oTg3d$WI02BYJQB68ZR0XjC0OBf41#$M
zp(MD`4YB-&7gTHI(PKy7_x%g!pZ@g4XMgjzpXx8okB`$i>fiR7E%K_aGZ&?JR$>AE
zTBDc?Ua~Ck4YOgE)}!jhDYC4-_TFO$mrvh*aCpoQ-@0~iWijR($2~~hNH2&A^=Mll
z#bU9=dMTN37QGlD^}P|A@f!P13mF=+Pj{qYtGU%WFO74S3A|cZYk6di2ojPox~M=#
zl&5#xzl2ez&#X_9@jGalgpwj!Q)gy$z(kQSS2so~X1K|Udsl*2J(Cz)j(aB2Gl|Xi
z@ZJWC6}kVBv3{30(-qoab;&^*W0@%y09k4*Qq@>f4K8`>xTtfiCxp~i8?4FM80Tvy
zbQ7VeDUIbo>zGfl?P@EN>6yYNra(kt!n!ar%z{-xrR9uk#bI5DoHde5mbkNmH+Vk<
zYcsuJwpVvSFfn*~b$63Vx))McqmXLVT!3L)I(3(JuE;PerLxAU2Btv`9m@kcy)rNt
z#^|(?DhE$1&7F?5)i7&R@IvLnpwvQq32m7tMK{$t>k<>!o>a76$C}i!CX5lS_$o##
zA(Cb(Y4~B8+^|sR#Ilqo##)%#!8qPqYkTq4i?6EpbJ?~YNM4UuosM;mL>0>ASZgf|
zeZWV}vQ8N~7HohaC(ep;jrWbQbs(?PR!E)y-0{DD;HLLH^{a(<&E0$KzysssbdLV2
zy=KB%JCdRs-<PsOT4kIp*)puDS=b?K!5mkLaRCiaH8Lr3v)S`AG(S`ABbUNK?}qsD
zyM5p-{;PLG-0pz=a=IZ-i5DJbGEL^4aiXJeovHjMwCHYw0S!(dzb;|D!P9aDRxfgn
z#stvo7$N}Z1!{Tp>D|M6_pnPuRJ`O><EvoNeL7a7X?{<~lUCIYXYsI9I_gBfojn5f
zf-O)<;CHA&Fh?|8Gf`?-9cow%V0nP~+;U!W@jgi`&F=XpTIv~9AJ}m%+iRzSQH^QR
z21J~}%toQe#ck(s=>GGMKKCu0n_Mm1(9v2=j$nljjjQh^2IfUcyfFpqKmrWR1j`Tz
zVOf-=aL0MlGE~*R1LJC_+B2}8f%Ob*BAWNo-1F+QFcB<F4FelNBc6DobPS9R-SE)(
zM2EHX+g-4O$G%grY#9VA3m$-_0e%fZ(pN13q}IYO3>0neC=R_hdiAll*RBWoR8X`I
zL)IbU%m_1xqHQW?MtgL8Tb(g=`}U>^?M_SSzKhMt735L<^>3ndowkV0meMK9n`B|@
z7cm5;GZ3h8SP5vn<*8N;4~1ebd!rg<F6jMFr_;4(I*86xQMz+2YHmO*fn>+nc8F;~
z%nD-sK-QuWhG}Iqqc{f!cT(Bt=xf*L-l#Up@zNvZHI0;0!Pq*c*0HeJGb7BHG^(wi
z8#A+~e)Rko?^}NO@i&~gaN*8de)zFxQH9!>{A^7V7EL2~REQo{Mluc7g*Yk|nJdB;
zRZ9&YH8k<YHkfT#9k??O^1+Y~rh#8rU0J(v+<f~Vuekl5f86%!5B}-t9}nec&Nsiu
zIr7M$58c96?+iNDo}e$HhE&>vqm4rig1;MbTC7td*SX}Or8We72^qN>E0O``joPKE
z6DzutZFj75+;r=emqbh<!l&PRkC1hV$T6QAgXCRK3Mj_`xxCPFpM9!lp1o6f@0E9L
zc;%fXTt#WIw&R;SEaLJ4#oTAkOu~|jPRK)rh+7Z0xDw%zCmdpR(Nv4+_?=wK*^0d+
zrY5o1BdZmKNi(#9^<e-xqE5A7N=ohl`3f1X&~l$?smI<PdwcA?Hn6wj>cGMcU7xnE
zD4++qXpu~*LDGbbRDM8Z#FE1GkVz!;txb)P95b0TVbUDC6_q<yj9X|Kadf=Mh`^K(
z1Hvr1kjy37FdL&#Ry9E8o)LFhWP1z4HEm&N>B>yi=VRTLc6R5yM^~R00kHQRAuvjd
z&K)D(!2(-c5ESyyOrWen0<hcEv0*~SU^2WKHDAvK@c9?Q&4=Ffhr569mG3-y-Pdou
z34igTmJql%nbKGn8qcyrf&uNFV?iUyP~$WFCB2hUQ$m}m%z{U+XtQ|-9TuWa=o7=N
zpu7Nh35XklLQ!Jmu^0<oVrtU{wjH)fCqsv++NvZe(n)fd(^T!5YjQ?zELi%uN=bN9
s>85vtZK6vw!g8azF9Mmp5WkCj9q>2|NLHVP*4A1eK*sX)w079P0sAum1ONa4

literal 0
HcmV?d00001


From 521bc0ea6327da5a04dae94a16020c5dddc341be Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Mon, 24 Apr 2017 12:31:53 +0200
Subject: [PATCH 120/253] [FLINK-5969] Add OperatorSnapshotUtil

This has methods for storing/reading OperatorStateHandles, as returned
from stream operator test harnesses. This can be used to write binary
snapshots for use in state migration tests.
---
 .../savepoint/SavepointV1Serializer.java      |  27 ++-
 .../streaming/util/OperatorSnapshotUtil.java  | 156 ++++++++++++++++++
 2 files changed, 174 insertions(+), 9 deletions(-)
 create mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
index aaa8cdd305686..f67d54ca15149 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Serializer.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.runtime.checkpoint.savepoint;
 
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.checkpoint.SubtaskState;
 import org.apache.flink.runtime.checkpoint.TaskState;
@@ -43,12 +45,13 @@
 
 /**
  * Deserializer for checkpoints written in format {@code 1} (Flink 1.2.x format)
- * 
+ *
  * <p>In contrast to the previous versions, this serializer makes sure that no Java
- * serialization is used for serialization. Therefore, we don't rely on any involved 
+ * serialization is used for serialization. Therefore, we don't rely on any involved
  * classes to stay the same.
  */
-class SavepointV1Serializer implements SavepointSerializer<SavepointV2> {
+@Internal
+public class SavepointV1Serializer implements SavepointSerializer<SavepointV2> {
 
 	private static final byte NULL_HANDLE = 0;
 	private static final byte BYTE_STREAM_STATE_HANDLE = 1;
@@ -210,7 +213,8 @@ private static SubtaskState deserializeSubtaskState(DataInputStream dis) throws
 				keyedStateStream);
 	}
 
-	private static void serializeKeyedStateHandle(
+	@VisibleForTesting
+	public static void serializeKeyedStateHandle(
 			KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
 
 		if (stateHandle == null) {
@@ -230,7 +234,8 @@ private static void serializeKeyedStateHandle(
 		}
 	}
 
-	private static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
+	@VisibleForTesting
+	public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
 		final int type = dis.readByte();
 		if (NULL_HANDLE == type) {
 			return null;
@@ -251,7 +256,8 @@ private static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis)
 		}
 	}
 
-	private static void serializeOperatorStateHandle(
+	@VisibleForTesting
+	public static void serializeOperatorStateHandle(
 			OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {
 
 		if (stateHandle != null) {
@@ -279,7 +285,8 @@ private static void serializeOperatorStateHandle(
 		}
 	}
 
-	private static OperatorStateHandle deserializeOperatorStateHandle(
+	@VisibleForTesting
+	public static OperatorStateHandle deserializeOperatorStateHandle(
 			DataInputStream dis) throws IOException {
 
 		final int type = dis.readByte();
@@ -310,7 +317,8 @@ private static OperatorStateHandle deserializeOperatorStateHandle(
 		}
 	}
 
-	private static void serializeStreamStateHandle(
+	@VisibleForTesting
+	public static void serializeStreamStateHandle(
 			StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {
 
 		if (stateHandle == null) {
@@ -337,7 +345,8 @@ private static void serializeStreamStateHandle(
 		dos.flush();
 	}
 
-	private static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
+	@VisibleForTesting
+	public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
 		int type = dis.read();
 		if (NULL_HANDLE == type) {
 			return null;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java
new file mode 100644
index 0000000000000..92a9452ef3f52
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/OperatorSnapshotUtil.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.flink.runtime.checkpoint.savepoint.SavepointV1Serializer;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+
+/**
+ * Util for writing/reading {@link org.apache.flink.streaming.runtime.tasks.OperatorStateHandles},
+ * for use in tests.
+ */
+public class OperatorSnapshotUtil {
+
+	public static String getResourceFilename(String filename) {
+		ClassLoader cl = OperatorSnapshotUtil.class.getClassLoader();
+		URL resource = cl.getResource(filename);
+		return resource.getFile();
+	}
+
+	public static void writeStateHandle(OperatorStateHandles state, String path) throws IOException {
+		FileOutputStream out = new FileOutputStream(path);
+		DataOutputStream dos = new DataOutputStream(out);
+
+		dos.writeInt(state.getOperatorChainIndex());
+
+		SavepointV1Serializer.serializeStreamStateHandle(state.getLegacyOperatorState(), dos);
+
+		Collection<OperatorStateHandle> rawOperatorState = state.getRawOperatorState();
+		if (rawOperatorState != null) {
+			dos.writeInt(rawOperatorState.size());
+			for (OperatorStateHandle operatorStateHandle : rawOperatorState) {
+				SavepointV1Serializer.serializeOperatorStateHandle(operatorStateHandle, dos);
+			}
+		} else {
+			// this means no states, not even an empty list
+			dos.writeInt(-1);
+		}
+
+		Collection<OperatorStateHandle> managedOperatorState = state.getManagedOperatorState();
+		if (managedOperatorState != null) {
+			dos.writeInt(managedOperatorState.size());
+			for (OperatorStateHandle operatorStateHandle : managedOperatorState) {
+				SavepointV1Serializer.serializeOperatorStateHandle(operatorStateHandle, dos);
+			}
+		} else {
+			// this means no states, not even an empty list
+			dos.writeInt(-1);
+		}
+
+		Collection<KeyedStateHandle> rawKeyedState = state.getRawKeyedState();
+		if (rawKeyedState != null) {
+			dos.writeInt(rawKeyedState.size());
+			for (KeyedStateHandle keyedStateHandle : rawKeyedState) {
+				SavepointV1Serializer.serializeKeyedStateHandle(keyedStateHandle, dos);
+			}
+		} else {
+			// this means no operator states, not even an empty list
+			dos.writeInt(-1);
+		}
+
+		Collection<KeyedStateHandle> managedKeyedState = state.getManagedKeyedState();
+		if (managedKeyedState != null) {
+			dos.writeInt(managedKeyedState.size());
+			for (KeyedStateHandle keyedStateHandle : managedKeyedState) {
+				SavepointV1Serializer.serializeKeyedStateHandle(keyedStateHandle, dos);
+			}
+		} else {
+			// this means no operator states, not even an empty list
+			dos.writeInt(-1);
+		}
+
+		dos.flush();
+		out.close();
+	}
+
+	public static OperatorStateHandles readStateHandle(String path) throws IOException, ClassNotFoundException {
+		FileInputStream in = new FileInputStream(path);
+		DataInputStream dis = new DataInputStream(in);
+		int index = dis.readInt();
+
+		StreamStateHandle legacyState = SavepointV1Serializer.deserializeStreamStateHandle(dis);
+
+		List<OperatorStateHandle> rawOperatorState = null;
+		int numRawOperatorStates = dis.readInt();
+		if (numRawOperatorStates >= 0) {
+			rawOperatorState = new ArrayList<>();
+			for (int i = 0; i < numRawOperatorStates; i++) {
+				OperatorStateHandle operatorState = SavepointV1Serializer.deserializeOperatorStateHandle(
+						dis);
+				rawOperatorState.add(operatorState);
+			}
+		}
+
+		List<OperatorStateHandle> managedOperatorState = null;
+		int numManagedOperatorStates = dis.readInt();
+		if (numManagedOperatorStates >= 0) {
+			managedOperatorState = new ArrayList<>();
+			for (int i = 0; i < numManagedOperatorStates; i++) {
+				OperatorStateHandle operatorState = SavepointV1Serializer.deserializeOperatorStateHandle(
+						dis);
+				managedOperatorState.add(operatorState);
+			}
+		}
+
+		List<KeyedStateHandle> rawKeyedState = null;
+		int numRawKeyedStates = dis.readInt();
+		if (numRawKeyedStates >= 0) {
+			rawKeyedState = new ArrayList<>();
+			for (int i = 0; i < numRawKeyedStates; i++) {
+				KeyedStateHandle keyedState = SavepointV1Serializer.deserializeKeyedStateHandle(
+						dis);
+				rawKeyedState.add(keyedState);
+			}
+		}
+
+		List<KeyedStateHandle> managedKeyedState = null;
+		int numManagedKeyedStates = dis.readInt();
+		if (numManagedKeyedStates >= 0) {
+			managedKeyedState = new ArrayList<>();
+			for (int i = 0; i < numManagedKeyedStates; i++) {
+				KeyedStateHandle keyedState = SavepointV1Serializer.deserializeKeyedStateHandle(
+						dis);
+				managedKeyedState.add(keyedState);
+			}
+		}
+
+		return new OperatorStateHandles(index, legacyState, managedKeyedState, rawKeyedState, managedOperatorState, rawOperatorState);
+	}
+}

From b92b50bf099c3bdfbe07b4c58f1244f2b37e22a7 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Mon, 24 Apr 2017 11:25:32 +0200
Subject: [PATCH 121/253] [FLINK-5969] Add
 ContinuousFileProcessingFrom12MigrationTest

The binary snapshots were created on the Flink 1.2 branch.
---
 ...ousFileProcessingFrom11MigrationTest.java} |   4 +-
 ...uousFileProcessingFrom12MigrationTest.java | 366 ++++++++++++++++++
 ...ation-test-1493116191000-flink1.2-snapshot | Bin 0 -> 373 bytes
 .../reader-migration-test-flink1.2-snapshot   | Bin 0 -> 2590 bytes
 4 files changed, 368 insertions(+), 2 deletions(-)
 rename flink-fs-tests/src/test/java/org/apache/flink/hdfstests/{ContinuousFileProcessingMigrationTest.java => ContinuousFileProcessingFrom11MigrationTest.java} (98%)
 create mode 100644 flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingFrom12MigrationTest.java
 create mode 100644 flink-fs-tests/src/test/resources/monitoring-function-migration-test-1493116191000-flink1.2-snapshot
 create mode 100644 flink-fs-tests/src/test/resources/reader-migration-test-flink1.2-snapshot

diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingFrom11MigrationTest.java
similarity index 98%
rename from flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
rename to flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingFrom11MigrationTest.java
index e271a21e6afa3..ec5e1ad94ef35 100644
--- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingFrom11MigrationTest.java
@@ -56,7 +56,7 @@
 import java.io.IOException;
 import java.net.URL;
 
-public class ContinuousFileProcessingMigrationTest {
+public class ContinuousFileProcessingFrom11MigrationTest {
 
 	private static final int NO_OF_FILES = 5;
 	private static final int LINES_PER_FILE = 10;
@@ -107,7 +107,7 @@ public static void destroyHDFS() {
 	}
 
 	private static String getResourceFilename(String filename) {
-		ClassLoader cl = ContinuousFileProcessingMigrationTest.class.getClassLoader();
+		ClassLoader cl = ContinuousFileProcessingFrom11MigrationTest.class.getClassLoader();
 		URL resource = cl.getResource(filename);
 		return resource.getFile();
 	}
diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingFrom12MigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingFrom12MigrationTest.java
new file mode 100644
index 0000000000000..bf09447cd6c77
--- /dev/null
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingFrom12MigrationTest.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hdfstests;
+
+import java.io.FileOutputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.io.TextInputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.core.fs.FileInputSplit;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.functions.source.ContinuousFileMonitoringFunction;
+import org.apache.flink.streaming.api.functions.source.ContinuousFileReaderOperator;
+import org.apache.flink.streaming.api.functions.source.FileProcessingMode;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.functions.source.TimestampedFileInputSplit;
+import org.apache.flink.streaming.api.operators.StreamSource;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OperatorSnapshotUtil;
+import org.apache.flink.util.Preconditions;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+public class ContinuousFileProcessingFrom12MigrationTest {
+
+	private static final int LINES_PER_FILE = 10;
+
+	private static final long INTERVAL = 100;
+
+	@ClassRule
+	public static TemporaryFolder tempFolder = new TemporaryFolder();
+
+	/**
+	 * Manually run this to write binary snapshot data. Remove @Ignore to run.
+	 */
+	@Ignore
+	@Test
+	public void writeReaderSnapshot() throws Exception {
+
+		File testFolder = tempFolder.newFolder();
+
+		TimestampedFileInputSplit split1 =
+				new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);
+
+		TimestampedFileInputSplit split2 =
+				new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);
+
+		TimestampedFileInputSplit split3 =
+				new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);
+
+		TimestampedFileInputSplit split4 =
+				new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);
+
+		// this always blocks to ensure that the reader doesn't to any actual processing so that
+		// we keep the state for the four splits
+		final OneShotLatch blockingLatch = new OneShotLatch();
+		BlockingFileInputFormat format = new BlockingFileInputFormat(blockingLatch, new Path(testFolder.getAbsolutePath()));
+
+		TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);
+		ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(
+				format);
+		initReader.setOutputType(typeInfo, new ExecutionConfig());
+		OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> testHarness =
+				new OneInputStreamOperatorTestHarness<>(initReader);
+		testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
+		testHarness.open();
+		// create some state in the reader
+		testHarness.processElement(new StreamRecord<>(split1));
+		testHarness.processElement(new StreamRecord<>(split2));
+		testHarness.processElement(new StreamRecord<>(split3));
+		testHarness.processElement(new StreamRecord<>(split4));
+		// take a snapshot of the operator's state. This will be used
+		// to initialize another reader and compare the results of the
+		// two operators.
+
+		final OperatorStateHandles snapshot;
+		synchronized (testHarness.getCheckpointLock()) {
+			snapshot = testHarness.snapshot(0L, 0L);
+		}
+
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/reader-migration-test-flink1.2-snapshot");
+	}
+
+	@Test
+	public void testReaderRestore() throws Exception {
+		File testFolder = tempFolder.newFolder();
+
+		final OneShotLatch latch = new OneShotLatch();
+
+		BlockingFileInputFormat format = new BlockingFileInputFormat(latch, new Path(testFolder.getAbsolutePath()));
+		TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);
+
+		ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(format);
+		initReader.setOutputType(typeInfo, new ExecutionConfig());
+
+		OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> testHarness =
+			new OneInputStreamOperatorTestHarness<>(initReader);
+		testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
+
+		testHarness.setup();
+		OperatorStateHandles operatorStateHandles = OperatorSnapshotUtil.readStateHandle(
+				OperatorSnapshotUtil.getResourceFilename(
+						"reader-migration-test-flink1.2-snapshot"));
+		testHarness.initializeState(operatorStateHandles);
+		testHarness.open();
+
+		latch.trigger();
+
+		// ... and wait for the operators to close gracefully
+
+		synchronized (testHarness.getCheckpointLock()) {
+			testHarness.close();
+		}
+
+		TimestampedFileInputSplit split1 =
+				new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);
+
+		TimestampedFileInputSplit split2 =
+				new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);
+
+		TimestampedFileInputSplit split3 =
+				new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);
+
+		TimestampedFileInputSplit split4 =
+				new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);
+
+		// compare if the results contain what they should contain and also if
+		// they are the same, as they should.
+
+		Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split1)));
+		Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split2)));
+		Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split3)));
+		Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split4)));
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data. Remove @Ignore to run.
+	 */
+	@Ignore
+	@Test
+	public void writeMonitoringSourceSnapshot() throws Exception {
+
+		File testFolder = tempFolder.newFolder();
+
+		long fileModTime = Long.MIN_VALUE;
+		for (int i = 0; i < 1; i++) {
+			Tuple2<File, String> file = createFileAndFillWithData(testFolder, "file", i, "This is test line.");
+			fileModTime = file.f0.lastModified();
+		}
+
+		TextInputFormat format = new TextInputFormat(new Path(testFolder.getAbsolutePath()));
+
+		final ContinuousFileMonitoringFunction<String> monitoringFunction =
+			new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
+
+		StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> src =
+			new StreamSource<>(monitoringFunction);
+
+		final AbstractStreamOperatorTestHarness<TimestampedFileInputSplit> testHarness =
+				new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
+
+		testHarness.open();
+
+		final Throwable[] error = new Throwable[1];
+
+		final OneShotLatch latch = new OneShotLatch();
+
+		// run the source asynchronously
+		Thread runner = new Thread() {
+			@Override
+			public void run() {
+				try {
+					monitoringFunction.run(new DummySourceContext() {
+						@Override
+						public void collect(TimestampedFileInputSplit element) {
+							latch.trigger();
+						}
+
+						@Override
+						public void markAsTemporarilyIdle() {
+
+						}
+					});
+				}
+				catch (Throwable t) {
+					t.printStackTrace();
+					error[0] = t;
+				}
+			}
+		};
+		runner.start();
+
+		if (!latch.isTriggered()) {
+			latch.await();
+		}
+
+		final OperatorStateHandles snapshot;
+		synchronized (testHarness.getCheckpointLock()) {
+			snapshot = testHarness.snapshot(0L, 0L);
+		}
+
+		OperatorSnapshotUtil.writeStateHandle(
+				snapshot,
+				"src/test/resources/monitoring-function-migration-test-" + fileModTime +"-flink1.2-snapshot");
+
+		monitoringFunction.cancel();
+		runner.join();
+
+		testHarness.close();
+	}
+
+	@Test
+	public void testMonitoringSourceRestore() throws Exception {
+
+		File testFolder = tempFolder.newFolder();
+
+		Long expectedModTime = Long.parseLong("1493116191000");
+		TextInputFormat format = new TextInputFormat(new Path(testFolder.getAbsolutePath()));
+
+		final ContinuousFileMonitoringFunction<String> monitoringFunction =
+			new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
+
+		StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> src =
+			new StreamSource<>(monitoringFunction);
+
+		final AbstractStreamOperatorTestHarness<TimestampedFileInputSplit> testHarness =
+			new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
+
+		testHarness.setup();
+		OperatorStateHandles operatorStateHandles = OperatorSnapshotUtil.readStateHandle(
+				OperatorSnapshotUtil.getResourceFilename(
+						"monitoring-function-migration-test-1493116191000-flink1.2-snapshot"));
+
+		testHarness.initializeState(operatorStateHandles);
+		testHarness.open();
+
+		Assert.assertEquals((long) expectedModTime, monitoringFunction.getGlobalModificationTime());
+
+	}
+
+	private static class BlockingFileInputFormat extends FileInputFormat<FileInputSplit> {
+
+		private static final long serialVersionUID = -6727603565381560267L;
+
+		private final OneShotLatch latch;
+
+		private FileInputSplit split;
+
+		private boolean reachedEnd;
+
+		BlockingFileInputFormat(OneShotLatch latch, Path filePath) {
+			super(filePath);
+			this.latch = latch;
+			this.reachedEnd = false;
+		}
+
+		@Override
+		public void open(FileInputSplit fileSplit) throws IOException {
+			this.split = fileSplit;
+			this.reachedEnd = false;
+		}
+
+		@Override
+		public boolean reachedEnd() throws IOException {
+			if (!latch.isTriggered()) {
+				try {
+					latch.await();
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+			}
+			return reachedEnd;
+		}
+
+		@Override
+		public FileInputSplit nextRecord(FileInputSplit reuse) throws IOException {
+			this.reachedEnd = true;
+			return split;
+		}
+
+		@Override
+		public void close() {
+
+		}
+	}
+
+	private static abstract class DummySourceContext
+		implements SourceFunction.SourceContext<TimestampedFileInputSplit> {
+
+		private final Object lock = new Object();
+
+		@Override
+		public void collectWithTimestamp(TimestampedFileInputSplit element, long timestamp) {
+		}
+
+		@Override
+		public void emitWatermark(Watermark mark) {
+		}
+
+		@Override
+		public Object getCheckpointLock() {
+			return lock;
+		}
+
+		@Override
+		public void close() {
+		}
+	}
+
+	/**
+	 * Create a file with pre-determined String format of the form:
+	 * {@code fileIdx +": "+ sampleLine +" "+ lineNo}.
+	 * */
+	private Tuple2<File, String> createFileAndFillWithData(
+		File base, String fileName, int fileIdx, String sampleLine) throws IOException {
+
+		File file = new File(base, fileName + fileIdx);
+		Assert.assertFalse(file.exists());
+
+		File tmp = new File(base, "." + fileName + fileIdx);
+		FileOutputStream stream = new FileOutputStream(tmp);
+		StringBuilder str = new StringBuilder();
+		for (int i = 0; i < LINES_PER_FILE; i++) {
+			String line = fileIdx +": "+ sampleLine + " " + i +"\n";
+			str.append(line);
+			stream.write(line.getBytes());
+		}
+		stream.close();
+
+		FileUtils.moveFile(tmp, file);
+
+		Assert.assertTrue("No result file present", file.exists());
+		return new Tuple2<>(file, str.toString());
+	}
+}
diff --git a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1493116191000-flink1.2-snapshot b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1493116191000-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..25451acbdfcf37d007bb3615e6485993127c8f76
GIT binary patch
literal 373
zcmZQz0D}KO0HhdMKm>zmT4qkFZf<^FW=VcgW?s5(aY<rHDo7n8hybZ$WKc;?vP>~B
zGBwmSw@gjZHAzb`&`q*5O4BvANKP~~Oi3|IO*RB7<pHV3ZRnb}46MaP43_yt>3WF;
ziOCtMdTBYCdD%ctre1P>F3=TvC6xuKr6rj;#d=AJ#i@Ee`FZKVsYRKIIhj?dMPLU3
ztzcqcs3>A^BEg&xps@&Zf`J~*NiE6Gt6bT>>bPFHE5tw(;tWJJ3)xKtVE;2PMlV^(
KC;<v*5C8xu6n5PJ

literal 0
HcmV?d00001

diff --git a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.2-snapshot b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..b22ceee635f4a567bc6a6d133b3628f1ca0c13d1
GIT binary patch
literal 2590
zcmeHJF>ljA6n=KvqNWHTkk~3DKw=^1rb$JrZXgvUvJfH#A$2<2=j5pEv+m9)z<*$1
zVq#(={0}Au#DWkrV&FHxdvUCQlBH8BlsEXSJHPk7@4L@u17Q5dz`<Fg3v8G?k=D=x
zO;Efk2lsn9u3sz1a{~u%#G(~6BOW%wPOIH$2hF`ku)h~Hn)}gCGv0|hyPXJNbBrm@
zU)9IIVBKhFC_P|2=iyMWIFZ>2(?w=wDwwg{3fAM#cwcDA6Zt~u!~T;`@7nd(F2G1b
zdoGvfl7%WwRmSY|TohI&h8^MEtVD?%94e6$rYVr_Trp#{;Hk_ApJZ{7g;uJ}Frx|`
z3U-7b5o4N*=ujr2o8^V=lX9=Vez|`84t#UL?ZL%VMKYElr(8<$p+3>KPxLHY@t-0?
zPb#m1$b&>2AiJA$fI_8FB*Xq~HzQQ*!A2sofgScxTg$ceVJ${%7OwhO%`0{GCOUgm
z&5MoqfZHMUyS2EVRLoZhpJ2^6&YHSi*qRW!U!;N1$FMn6#%7$vfy>7wy@Y22uWvPq
zc`#AER_JPx(IT5Je)#<O$ItDT_p05Q_5pNI8EBd9#XwOQqB4CB_t)_O+eix+N)VZ_
z-lK<I1b{`(lk5zq7*36y5xe`g`@Tjw8@TU?)54`i*e%p8GV|!t995|Dm5G*MBIkcF
v(Qaj;`pQJhIEma-n&{*7ofl{6BwCqhxt}vQrHRhu=d4woL|Zxi5K{aB9aAa@

literal 0
HcmV?d00001


From 60c50e36a933331c45c9c2efcbc37a1bc8f0a5d4 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Thu, 27 Apr 2017 10:49:05 +0200
Subject: [PATCH 122/253] [FLINK-5969] Fix restore from empty state in
 KafkaConsumerBase

---
 .../streaming/connectors/kafka/FlinkKafkaConsumerBase.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
index a35e7104ecbf6..4a05efa8dccea 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
@@ -516,7 +516,8 @@ public void initializeState(FunctionInitializationContext context) throws Except
 				if (LOG.isDebugEnabled()) {
 					LOG.debug("Using the following offsets: {}", restoredState);
 				}
-			} else if (restoredState.isEmpty()) {
+			}
+			if (restoredState != null && restoredState.isEmpty()) {
 				restoredState = null;
 			}
 		} else {

From af7a8eec247fbfc04454b5573c6b80ca3982a5d6 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Mon, 24 Apr 2017 15:12:14 +0200
Subject: [PATCH 123/253] [FLINK-5969] Add KafkaConsumerBaseFrom12MigrationTest

The binary snapshots were created on the Flink 1.2 branch.
---
 ...KafkaConsumerBaseFrom11MigrationTest.java} |   4 +-
 ...kKafkaConsumerBaseFrom12MigrationTest.java | 338 ++++++++++++++++++
 ...-consumer-migration-test-flink1.2-snapshot | Bin 0 -> 1022 bytes
 ...gration-test-flink1.2-snapshot-empty-state | Bin 0 -> 240 bytes
 4 files changed, 340 insertions(+), 2 deletions(-)
 rename flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/{FlinkKafkaConsumerBaseMigrationTest.java => FlinkKafkaConsumerBaseFrom11MigrationTest.java} (98%)
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseFrom12MigrationTest.java
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot
 create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot-empty-state

diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseFrom11MigrationTest.java
similarity index 98%
rename from flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
rename to flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseFrom11MigrationTest.java
index 9fc261e833b99..7cc1f9c162aa7 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseFrom11MigrationTest.java
@@ -49,7 +49,7 @@
  * <p>For regenerating the binary snapshot file you have to run the commented out portion
  * of each test on a checkout of the Flink 1.1 branch.
  */
-public class FlinkKafkaConsumerBaseMigrationTest {
+public class FlinkKafkaConsumerBaseFrom11MigrationTest {
 
 	/** Test restoring from an legacy empty state, when no partitions could be found for topics. */
 	@Test
@@ -168,7 +168,7 @@ public void testRestoreFromFlink11() throws Exception {
 	// ------------------------------------------------------------------------
 
 	private static String getResourceFilename(String filename) {
-		ClassLoader cl = FlinkKafkaConsumerBaseMigrationTest.class.getClassLoader();
+		ClassLoader cl = FlinkKafkaConsumerBaseFrom11MigrationTest.class.getClassLoader();
 		URL resource = cl.getResource(filename);
 		if (resource == null) {
 			throw new NullPointerException("Missing snapshot resource.");
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseFrom12MigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseFrom12MigrationTest.java
new file mode 100644
index 0000000000000..6414a12bcf750
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseFrom12MigrationTest.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.operators.StreamSource;
+import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode;
+import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher;
+import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
+import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionStateSentinel;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OperatorSnapshotUtil;
+import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema;
+import org.apache.flink.util.SerializedValue;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Tests for checking whether {@link FlinkKafkaConsumerBase} can restore from snapshots that were
+ * done using the Flink 1.2 {@link FlinkKafkaConsumerBase}.
+ *
+ * <p>For regenerating the binary snapshot files run {@link #writeSnapshot()} on the Flink 1.2
+ * branch.
+ */
+public class FlinkKafkaConsumerBaseFrom12MigrationTest {
+
+	final static HashMap<KafkaTopicPartition, Long> PARTITION_STATE = new HashMap<>();
+
+	static {
+		PARTITION_STATE.put(new KafkaTopicPartition("abc", 13), 16768L);
+		PARTITION_STATE.put(new KafkaTopicPartition("def", 7), 987654321L);
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeSnapshot() throws Exception {
+		writeSnapshot("src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot", PARTITION_STATE);
+
+		final HashMap<KafkaTopicPartition, Long> emptyState = new HashMap<>();
+		writeSnapshot("src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot-empty-state", emptyState);
+	}
+
+	private void writeSnapshot(String path, HashMap<KafkaTopicPartition, Long> state) throws Exception {
+
+		final OneShotLatch latch = new OneShotLatch();
+		final AbstractFetcher<String, ?> fetcher = mock(AbstractFetcher.class);
+
+		doAnswer(new Answer<Void>() {
+			@Override
+			public Void answer(InvocationOnMock invocation) throws Throwable {
+				latch.trigger();
+				return null;
+			}
+		}).when(fetcher).runFetchLoop();
+
+		when(fetcher.snapshotCurrentState()).thenReturn(state);
+
+		final List<KafkaTopicPartition> partitions = new ArrayList<>(PARTITION_STATE.keySet());
+
+		final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(fetcher, partitions);
+
+		StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator =
+				new StreamSource<>(consumerFunction);
+
+
+		final AbstractStreamOperatorTestHarness<String> testHarness =
+				new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+		testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+		testHarness.setup();
+		testHarness.open();
+
+		final Throwable[] error = new Throwable[1];
+
+		// run the source asynchronously
+		Thread runner = new Thread() {
+			@Override
+			public void run() {
+				try {
+					consumerFunction.run(new DummySourceContext() {
+						@Override
+						public void collect(String element) {
+
+						}
+					});
+				}
+				catch (Throwable t) {
+					t.printStackTrace();
+					error[0] = t;
+				}
+			}
+		};
+		runner.start();
+
+		if (!latch.isTriggered()) {
+			latch.await();
+		}
+
+		final OperatorStateHandles snapshot;
+		synchronized (testHarness.getCheckpointLock()) {
+			snapshot = testHarness.snapshot(0L, 0L);
+		}
+
+		OperatorSnapshotUtil.writeStateHandle(snapshot, path);
+
+		consumerOperator.close();
+		runner.join();
+	}
+
+	/**
+	 * Test restoring from an legacy empty state, when no partitions could be found for topics.
+	 */
+	@Test
+	public void testRestoreFromEmptyStateNoPartitions() throws Exception {
+		final DummyFlinkKafkaConsumer<String> consumerFunction =
+				new DummyFlinkKafkaConsumer<>(Collections.<KafkaTopicPartition>emptyList());
+
+		StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
+
+		final AbstractStreamOperatorTestHarness<String> testHarness =
+				new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+		testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+		testHarness.setup();
+		// restore state from binary snapshot file
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("kafka-consumer-migration-test-flink1.2-snapshot-empty-state")));
+		testHarness.open();
+
+		// assert that no partitions were found and is empty
+		assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
+		assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
+
+		// assert that no state was restored
+		assertTrue(consumerFunction.getRestoredState() == null);
+
+		consumerOperator.close();
+		consumerOperator.cancel();
+	}
+
+	/**
+	 * Test restoring from an empty state taken using Flink 1.2, when some partitions could be
+	 * found for topics.
+	 */
+	@Test
+	public void testRestoreFromEmptyStateWithPartitions() throws Exception {
+		final List<KafkaTopicPartition> partitions = new ArrayList<>(PARTITION_STATE.keySet());
+
+		final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(partitions);
+
+		StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator =
+				new StreamSource<>(consumerFunction);
+
+		final AbstractStreamOperatorTestHarness<String> testHarness =
+				new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+		testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+		testHarness.setup();
+		// restore state from binary snapshot file
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("kafka-consumer-migration-test-flink1.2-snapshot-empty-state")));
+		testHarness.open();
+
+		// the expected state in "kafka-consumer-migration-test-flink1.2-snapshot-empty-state";
+		// since the state is empty, the consumer should reflect on the startup mode to determine start offsets.
+		final HashMap<KafkaTopicPartition, Long> expectedSubscribedPartitionsWithStartOffsets = new HashMap<>();
+		for (KafkaTopicPartition partition : PARTITION_STATE.keySet()) {
+			expectedSubscribedPartitionsWithStartOffsets.put(partition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
+		}
+
+		// assert that there are partitions and is identical to expected list
+		assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
+		assertTrue(!consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
+		Assert.assertEquals(expectedSubscribedPartitionsWithStartOffsets, consumerFunction.getSubscribedPartitionsToStartOffsets());
+
+		assertTrue(consumerFunction.getRestoredState() == null);
+
+		consumerOperator.close();
+		consumerOperator.cancel();
+	}
+
+	/**
+	 * Test restoring from a non-empty state taken using Flink 1.2, when some partitions could be
+	 * found for topics.
+	 */
+	@Test
+	public void testRestore() throws Exception {
+		final List<KafkaTopicPartition> partitions = new ArrayList<>(PARTITION_STATE.keySet());
+
+		final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(partitions);
+
+		StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator =
+				new StreamSource<>(consumerFunction);
+
+		final AbstractStreamOperatorTestHarness<String> testHarness =
+				new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+		testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+		testHarness.setup();
+		// restore state from binary snapshot file
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("kafka-consumer-migration-test-flink1.2-snapshot")));
+		testHarness.open();
+
+		// assert that there are partitions and is identical to expected list
+		assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
+		assertTrue(!consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
+
+		// on restore, subscribedPartitionsToStartOffsets should be identical to the restored state
+		Assert.assertEquals(PARTITION_STATE, consumerFunction.getSubscribedPartitionsToStartOffsets());
+
+		// assert that state is correctly restored from legacy checkpoint
+		assertTrue(consumerFunction.getRestoredState() != null);
+		Assert.assertEquals(PARTITION_STATE, consumerFunction.getRestoredState());
+
+		consumerOperator.close();
+		consumerOperator.cancel();
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static class DummyFlinkKafkaConsumer<T> extends FlinkKafkaConsumerBase<T> {
+		private static final long serialVersionUID = 1L;
+
+		private final List<KafkaTopicPartition> partitions;
+
+		private final AbstractFetcher<T, ?> fetcher;
+
+		@SuppressWarnings("unchecked")
+		DummyFlinkKafkaConsumer(AbstractFetcher<T, ?> fetcher, List<KafkaTopicPartition> partitions) {
+			super(Arrays.asList("dummy-topic"), (KeyedDeserializationSchema< T >) mock(KeyedDeserializationSchema.class));
+			this.fetcher = fetcher;
+			this.partitions = partitions;
+		}
+
+		DummyFlinkKafkaConsumer(List<KafkaTopicPartition> partitions) {
+			super(Arrays.asList("dummy-topic"), (KeyedDeserializationSchema< T >) mock(KeyedDeserializationSchema.class));
+			this.fetcher = mock(AbstractFetcher.class);
+			this.partitions = partitions;
+		}
+
+		@Override
+		protected AbstractFetcher<T, ?> createFetcher(
+				SourceContext<T> sourceContext,
+				Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
+				SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
+				SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
+				StreamingRuntimeContext runtimeContext,
+				OffsetCommitMode offsetCommitMode) throws Exception {
+			return fetcher;
+		}
+
+		@Override
+		protected List<KafkaTopicPartition> getKafkaPartitions(List<String> topics) {
+			return partitions;
+		}
+
+		@Override
+		protected boolean getIsAutoCommitEnabled() {
+			return false;
+		}
+	}
+
+
+	private static abstract class DummySourceContext
+			implements SourceFunction.SourceContext<String> {
+
+		private final Object lock = new Object();
+
+		@Override
+		public void collectWithTimestamp(String element, long timestamp) {
+		}
+
+		@Override
+		public void emitWatermark(Watermark mark) {
+		}
+
+		@Override
+		public Object getCheckpointLock() {
+			return lock;
+		}
+
+		@Override
+		public void close() {
+		}
+
+		@Override
+		public void markAsTemporarilyIdle() {
+
+		}
+	}
+}
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..f0be11a1e665d761eeb40e2ed619a3129233438b
GIT binary patch
literal 1022
zcmeHGJx?1!5S=^6LKp=_lt`(P(B0aYLqJ@mK^QEEWEV&&(A?eHUVOKA?Cn`Vfl{TR
zNW@2h=uk=;{s*F_KqN&v#6O4>W-*q9h?Ep5GsWzkcHW!!p0of|zo_W2o|y*2YXNU!
z7V0%~XPZWK+&k=X3tpGd4R{FyCkTQ$$K^BV%+9(c$DNs*#K0?;y{R%l?ua1PW5Uy0
z$fpXNShW}?=(jm*hC+6j%A^(%XQ@Wb*&=qa&XvGWY;!eVU%mM17WPQWrULf~78Akz
zIErG)^j5+%Ey9#F=xadH6AzZ5NQ|rpArrqPTc`C*XGzFe!*nupUj(;nU^hz|#%gA$
z7@}+ySG;xZ>x#LV+<+ahNadTqA|CLPEXt+&1eI#VF%q&x4yB|uu}WD7n;m3AYOW-b
z*Cq4ch!f$zN2P@pu`J|26fdt<r)|hpq2QaOg9S|6RTxhCGd0L-1A6SHt`)7TP)yGL
z4GDa%A%~t%>PGK#F_u{^rjgwrU(P?j`euW*2>C9C87Jc}21CnP<Z<=s^zh}#!S~&M
j^hEjY<Nx?$`KS2V&lAdOh~lUB`0wI>d)+=W58l)rh!{g=

literal 0
HcmV?d00001

diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot-empty-state b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.2-snapshot-empty-state
new file mode 100644
index 0000000000000000000000000000000000000000..45047eede585171ea04fcd3f896898f1be6acf44
GIT binary patch
literal 240
zcmZQz0D}KO0HhdMKm-G4d`fCsVrfoEJV*{Gpps;0kdkC>YN2bImSn1HVq#*cYiVX?
zp=+FyW?*h+Xk?LOXb4m}8>|||fHiL!Sc{7obn}bS^%4sblQUBF(sDBMvh|8e^GY&v
zQ}v2V5=&C`yb{Y2gHww#6LT`FQj6SzBhD=`;hMz6z)(@dV1l7Hu^>|~IX^cyKTofu
fvLLmzBr~U2F9gU!7y)uP5HNy_C;%})UIYOE!s$wD

literal 0
HcmV?d00001


From a802eae00f998905eb049dafb205d5445339324d Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Mon, 24 Apr 2017 17:13:49 +0200
Subject: [PATCH 124/253] [FLINK-5969] Also snapshot legacy state in operator
 test harness

---
 .../util/AbstractStreamOperatorTestHarness.java   | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
index 4a9463a1593c1..7a8488fbe5c85 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
@@ -509,9 +509,21 @@ public OperatorStateHandles snapshot(long checkpointId, long timestamp) throws E
 		OperatorStateHandle opManaged = FutureUtil.runIfNotDoneAndGet(operatorStateResult.getOperatorStateManagedFuture());
 		OperatorStateHandle opRaw = FutureUtil.runIfNotDoneAndGet(operatorStateResult.getOperatorStateRawFuture());
 
+		// also snapshot legacy state, if any
+		StreamStateHandle legacyStateHandle = null;
+
+		if (operator instanceof StreamCheckpointedOperator) {
+
+			final CheckpointStreamFactory.CheckpointStateOutputStream outStream =
+					streamFactory.createCheckpointStateOutputStream(checkpointId, timestamp);
+
+				((StreamCheckpointedOperator) operator).snapshotState(outStream, checkpointId, timestamp);
+				legacyStateHandle = outStream.closeAndGetHandle();
+		}
+
 		return new OperatorStateHandles(
 			0,
-			null,
+			legacyStateHandle,
 			keyedManaged != null ? Collections.singletonList(keyedManaged) : null,
 			keyedRaw != null ? Collections.singletonList(keyedRaw) : null,
 			opManaged != null ? Collections.singletonList(opManaged) : null,
@@ -523,7 +535,6 @@ public OperatorStateHandles snapshot(long checkpointId, long timestamp) throws E
 	 * the operator implements this interface.
 	 */
 	@Deprecated
-	@SuppressWarnings("deprecation")
 	public StreamStateHandle snapshotLegacy(long checkpointId, long timestamp) throws Exception {
 
 		CheckpointStreamFactory.CheckpointStateOutputStream outStream = stateBackend.createStreamFactory(

From a0b4f3af9af854c4e333ac391c9e56d27cf56d06 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Mon, 24 Apr 2017 17:13:27 +0200
Subject: [PATCH 125/253] [FLINK-5969] Add WindowOperatorFrom12MigrationTest

The binary snapshots for this were created on the Flink 1.2 branch.
---
 ...=> WindowOperatorFrom11MigrationTest.java} |    4 +-
 .../WindowOperatorFrom12MigrationTest.java    | 1014 +++++++++++++++++
 ...ation-test-accum-aligned-flink1.2-snapshot |  Bin 0 -> 222 bytes
 ...ration-test-aggr-aligned-flink1.2-snapshot |  Bin 0 -> 187 bytes
 ...on-test-apply-event-time-flink1.2-snapshot |  Bin 0 -> 2193 bytes
 ...st-apply-processing-time-flink1.2-snapshot |  Bin 0 -> 2081 bytes
 ...n-test-reduce-event-time-flink1.2-snapshot |  Bin 0 -> 1988 bytes
 ...t-reduce-processing-time-flink1.2-snapshot |  Bin 0 -> 1925 bytes
 ...on-with-stateful-trigger-flink1.2-snapshot |  Bin 0 -> 3703 bytes
 ...th-stateful-trigger-mint-flink1.2-snapshot |  Bin 0 -> 494 bytes
 10 files changed, 1016 insertions(+), 2 deletions(-)
 rename flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/{WindowOperatorMigrationTest.java => WindowOperatorFrom11MigrationTest.java} (99%)
 create mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorFrom12MigrationTest.java
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.2-snapshot
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.2-snapshot
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.2-snapshot
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.2-snapshot
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.2-snapshot
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.2-snapshot
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.2-snapshot
 create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.2-snapshot

diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorFrom11MigrationTest.java
similarity index 99%
rename from flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
rename to flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorFrom11MigrationTest.java
index 1168eb0343110..9ec192327f025 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorFrom11MigrationTest.java
@@ -70,10 +70,10 @@
  * <p>For regenerating the binary snapshot file you have to run the commented out portion
  * of each test on a checkout of the Flink 1.1 branch.
  */
-public class WindowOperatorMigrationTest {
+public class WindowOperatorFrom11MigrationTest {
 
 	private static String getResourceFilename(String filename) {
-		ClassLoader cl = WindowOperatorMigrationTest.class.getClassLoader();
+		ClassLoader cl = WindowOperatorFrom11MigrationTest.class.getClassLoader();
 		URL resource = cl.getResource(filename);
 		if (resource == null) {
 			throw new NullPointerException("Missing snapshot resource.");
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorFrom12MigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorFrom12MigrationTest.java
new file mode 100644
index 0000000000000..0d3a6dcc43977
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorFrom12MigrationTest.java
@@ -0,0 +1,1014 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.runtime.operators.windowing;
+
+import static org.junit.Assert.fail;
+
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.state.ReducingStateDescriptor;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.typeutils.TypeInfoParser;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.LegacyWindowOperatorType;
+import org.apache.flink.streaming.api.functions.windowing.PassThroughWindowFunction;
+import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
+import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.streaming.api.windowing.triggers.CountTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.ProcessingTimeTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.PurgingTrigger;
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.apache.flink.streaming.api.windowing.windows.Window;
+import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableWindowFunction;
+import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalSingleValueWindowFunction;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OperatorSnapshotUtil;
+import org.apache.flink.streaming.util.TestHarnessUtil;
+import org.apache.flink.util.Collector;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Tests for checking whether {@link WindowOperator} can restore from snapshots that were done
+ * using the Flink 1.2 {@link WindowOperator}.
+ *
+ * <p>This also checks whether {@link WindowOperator} can restore from a checkpoint of the Flink 1.2
+ * aligned processing-time windows operator.
+ *
+ * <p>For regenerating the binary snapshot file you have to run the {@code write*()} method on
+ * the Flink 1.2 branch.
+ */
+public class WindowOperatorFrom12MigrationTest {
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeSessionWindowsWithCountTriggerSnapshot() throws Exception {
+		final int SESSION_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
+				EventTimeSessionWindows.withGap(Time.seconds(SESSION_SIZE)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new SessionWindowFunction()),
+				PurgingTrigger.of(CountTrigger.of(4)),
+				0,
+				null /* late data output tag */);
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.open();
+
+		// add elements out-of-order
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 3500));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000));
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
+
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.2-snapshot");
+
+		testHarness.close();
+	}
+
+	@Test
+	public void testRestoreSessionWindowsWithCountTrigger() throws Exception {
+
+		final int SESSION_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
+				EventTimeSessionWindows.withGap(Time.seconds(SESSION_SIZE)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new SessionWindowFunction()),
+				PurgingTrigger.of(CountTrigger.of(4)),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-session-with-stateful-trigger-flink1.2-snapshot")));
+		testHarness.open();
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 6000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 6500));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 7000));
+
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
+
+		// add an element that merges the two "key1" sessions, they should now have count 6, and therfore fire
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 10), 4500));
+
+		expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-22", 10L, 10000L), 9999L));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
+
+		testHarness.close();
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeSessionWindowsWithCountTriggerInMintConditionSnapshot() throws Exception {
+
+		final int SESSION_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
+				EventTimeSessionWindows.withGap(Time.seconds(SESSION_SIZE)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new SessionWindowFunction()),
+				PurgingTrigger.of(CountTrigger.of(4)),
+				0,
+				null /* late data output tag */);
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.open();
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.2-snapshot");
+
+		testHarness.close();
+	}
+
+	/**
+	 * This checks that we can restore from a virgin {@code WindowOperator} that has never seen
+	 * any elements.
+	 */
+	@Test
+	public void testRestoreSessionWindowsWithCountTriggerInMintCondition() throws Exception {
+
+		final int SESSION_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
+				EventTimeSessionWindows.withGap(Time.seconds(SESSION_SIZE)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new SessionWindowFunction()),
+				PurgingTrigger.of(CountTrigger.of(4)),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-session-with-stateful-trigger-mint-flink1.2-snapshot")));
+		testHarness.open();
+
+		// add elements out-of-order
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 3500));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 6000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 6500));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 7000));
+
+		expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-10", 0L, 6500L), 6499));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
+
+		// add an element that merges the two "key1" sessions, they should now have count 6, and therfore fire
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 10), 4500));
+
+		expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-22", 10L, 10000L), 9999L));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
+
+		testHarness.close();
+	}
+
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeReducingEventTimeWindowsSnapshot() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+				new SumReducer(),
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+				EventTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.open();
+
+		// add elements out-of-order
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+
+		testHarness.processWatermark(new Watermark(999));
+		expectedOutput.add(new Watermark(999));
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		testHarness.processWatermark(new Watermark(1999));
+		expectedOutput.add(new Watermark(1999));
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-reduce-event-time-flink1.2-snapshot");
+
+		testHarness.close();
+	}
+
+	@Test
+	public void testRestoreReducingEventTimeWindows() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+				new SumReducer(),
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+				EventTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-reduce-event-time-flink1.2-snapshot")));
+		testHarness.open();
+
+		testHarness.processWatermark(new Watermark(2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
+		expectedOutput.add(new Watermark(2999));
+
+		testHarness.processWatermark(new Watermark(3999));
+		expectedOutput.add(new Watermark(3999));
+
+		testHarness.processWatermark(new Watermark(4999));
+		expectedOutput.add(new Watermark(4999));
+
+		testHarness.processWatermark(new Watermark(5999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
+		expectedOutput.add(new Watermark(5999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+		testHarness.close();
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeApplyEventTimeWindowsSnapshot() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
+				EventTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.open();
+
+		// add elements out-of-order
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+
+		testHarness.processWatermark(new Watermark(999));
+		expectedOutput.add(new Watermark(999));
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		testHarness.processWatermark(new Watermark(1999));
+		expectedOutput.add(new Watermark(1999));
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-apply-event-time-flink1.2-snapshot");
+
+		testHarness.close();
+	}
+
+	@Test
+	public void testRestoreApplyEventTimeWindows() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
+				EventTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-apply-event-time-flink1.2-snapshot")));
+		testHarness.open();
+
+		testHarness.processWatermark(new Watermark(2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
+		expectedOutput.add(new Watermark(2999));
+
+		testHarness.processWatermark(new Watermark(3999));
+		expectedOutput.add(new Watermark(3999));
+
+		testHarness.processWatermark(new Watermark(4999));
+		expectedOutput.add(new Watermark(4999));
+
+		testHarness.processWatermark(new Watermark(5999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
+		expectedOutput.add(new Watermark(5999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+		testHarness.close();
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeReducingProcessingTimeWindowsSnapshot() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+				new SumReducer(),
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingProcessingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+				ProcessingTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.open();
+
+		testHarness.setProcessingTime(10);
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1)));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1)));
+
+		testHarness.setProcessingTime(3010);
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1)));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key3", 1)));
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 1), 2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 2999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-reduce-processing-time-flink1.2-snapshot");
+
+		testHarness.close();
+
+	}
+
+	@Test
+	public void testRestoreReducingProcessingTimeWindows() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+				new SumReducer(),
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingProcessingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+				ProcessingTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-reduce-processing-time-flink1.2-snapshot")));
+		testHarness.open();
+
+		testHarness.setProcessingTime(3020);
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3)));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3)));
+
+		testHarness.setProcessingTime(6000);
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 5999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+		testHarness.close();
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeApplyProcessingTimeWindowsSnapshot() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingProcessingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
+				ProcessingTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.open();
+
+		testHarness.setProcessingTime(10);
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1)));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1)));
+
+		testHarness.setProcessingTime(3010);
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1)));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key3", 1)));
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 1), 2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 2999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-apply-processing-time-flink1.2-snapshot");
+
+		testHarness.close();
+	}
+
+	@Test
+	public void testRestoreApplyProcessingTimeWindows() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingProcessingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
+				ProcessingTimeTrigger.create(),
+				0,
+				null /* late data output tag */);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-apply-processing-time-flink1.2-snapshot")));
+		testHarness.open();
+
+		testHarness.setProcessingTime(3020);
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3)));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3)));
+
+		testHarness.setProcessingTime(6000);
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 5999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+		testHarness.close();
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeAggregatingAlignedProcessingTimeWindowsSnapshot() throws Exception {
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		AggregatingProcessingTimeWindowOperator<String, Tuple2<String, Integer>> operator =
+			new AggregatingProcessingTimeWindowOperator<>(
+				new ReduceFunction<Tuple2<String, Integer>>() {
+					private static final long serialVersionUID = -8913160567151867987L;
+
+					@Override
+					public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
+						return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
+					}
+				},
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				inputType.createSerializer(new ExecutionConfig()),
+				3000,
+				3000);
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.open();
+
+		testHarness.setProcessingTime(3);
+
+		// timestamp is ignored in processing time
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), Long.MAX_VALUE));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+
+		// do a snapshot, close and restore again
+		OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-aggr-aligned-flink1.2-snapshot");
+		testHarness.close();
+	}
+
+	@Test
+	public void testRestoreAggregatingAlignedProcessingTimeWindows() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+				new SumReducer(),
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingProcessingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+				ProcessingTimeTrigger.create(),
+				0,
+				null /* late data output tag */,
+				LegacyWindowOperatorType.FAST_AGGREGATING);
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-aggr-aligned-flink1.2-snapshot")));
+		testHarness.open();
+
+		testHarness.setProcessingTime(5000);
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), 2999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+
+		testHarness.setProcessingTime(7000);
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 5999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		testHarness.close();
+	}
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writeAlignedProcessingTimeWindowsSnapshot() throws Exception {
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		AccumulatingProcessingTimeWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>> operator =
+			new AccumulatingProcessingTimeWindowOperator<>(
+					new InternalIterableWindowFunction<>(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
+
+						private static final long serialVersionUID = 6551516443265733803L;
+
+						@Override
+						public void apply(String s, TimeWindow window, Iterable<Tuple2<String, Integer>> input, Collector<Tuple2<String, Integer>> out) throws Exception {
+							int sum = 0;
+							for (Tuple2<String, Integer> anInput : input) {
+								sum += anInput.f1;
+							}
+							out.collect(new Tuple2<>(s, sum));
+						}
+					}),
+					new TupleKeySelector(),
+					BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+					inputType.createSerializer(new ExecutionConfig()),
+					3000,
+					3000);
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		testHarness.open();
+
+		testHarness.setProcessingTime(3);
+
+		// timestamp is ignored in processing time
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), Long.MAX_VALUE));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+
+		// do a snapshot, close and restore again
+		OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/win-op-migration-test-accum-aligned-flink1.2-snapshot");
+		testHarness.close();
+	}
+
+	@Test
+	public void testRestoreAccumulatingAlignedProcessingTimeWindows() throws Exception {
+		final int WINDOW_SIZE = 3;
+
+		TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+		ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+				new SumReducer(),
+				inputType.createSerializer(new ExecutionConfig()));
+
+		WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
+				TumblingProcessingTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+				new TimeWindow.Serializer(),
+				new TupleKeySelector(),
+				BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+				stateDesc,
+				new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+				ProcessingTimeTrigger.create(),
+				0,
+				null /* late data output tag */,
+				LegacyWindowOperatorType.FAST_ACCUMULATING);
+
+		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+				new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+		ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("win-op-migration-test-accum-aligned-flink1.2-snapshot")));
+		testHarness.open();
+
+		testHarness.setProcessingTime(5000);
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), 2999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+		testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
+
+		testHarness.setProcessingTime(7000);
+
+		expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 5999));
+
+		TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+
+		testHarness.close();
+	}
+
+
+	private static class TupleKeySelector implements KeySelector<Tuple2<String, Integer>, String> {
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public String getKey(Tuple2<String, Integer> value) throws Exception {
+			return value.f0;
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	private static class Tuple2ResultSortComparator implements Comparator<Object> {
+		@Override
+		public int compare(Object o1, Object o2) {
+			if (o1 instanceof Watermark || o2 instanceof Watermark) {
+				return 0;
+			} else {
+				StreamRecord<Tuple2<String, Integer>> sr0 = (StreamRecord<Tuple2<String, Integer>>) o1;
+				StreamRecord<Tuple2<String, Integer>> sr1 = (StreamRecord<Tuple2<String, Integer>>) o2;
+				if (sr0.getTimestamp() != sr1.getTimestamp()) {
+					return (int) (sr0.getTimestamp() - sr1.getTimestamp());
+				}
+				int comparison = sr0.getValue().f0.compareTo(sr1.getValue().f0);
+				if (comparison != 0) {
+					return comparison;
+				} else {
+					return sr0.getValue().f1 - sr1.getValue().f1;
+				}
+			}
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	private static class Tuple3ResultSortComparator implements Comparator<Object> {
+		@Override
+		public int compare(Object o1, Object o2) {
+			if (o1 instanceof Watermark || o2 instanceof Watermark) {
+				return 0;
+			} else {
+				StreamRecord<Tuple3<String, Long, Long>> sr0 = (StreamRecord<Tuple3<String, Long, Long>>) o1;
+				StreamRecord<Tuple3<String, Long, Long>> sr1 = (StreamRecord<Tuple3<String, Long, Long>>) o2;
+				if (sr0.getTimestamp() != sr1.getTimestamp()) {
+					return (int) (sr0.getTimestamp() - sr1.getTimestamp());
+				}
+				int comparison = sr0.getValue().f0.compareTo(sr1.getValue().f0);
+				if (comparison != 0) {
+					return comparison;
+				} else {
+					comparison = (int) (sr0.getValue().f1 - sr1.getValue().f1);
+					if (comparison != 0) {
+						return comparison;
+					}
+					return (int) (sr0.getValue().f1 - sr1.getValue().f1);
+				}
+			}
+		}
+	}
+
+	public static class SumReducer implements ReduceFunction<Tuple2<String, Integer>> {
+		private static final long serialVersionUID = 1L;
+		@Override
+		public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1,
+				Tuple2<String, Integer> value2) throws Exception {
+			return new Tuple2<>(value2.f0, value1.f1 + value2.f1);
+		}
+	}
+
+	public static class RichSumReducer<W extends Window> extends RichWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, W> {
+		private static final long serialVersionUID = 1L;
+
+		private boolean openCalled = false;
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+			openCalled = true;
+		}
+
+		@Override
+		public void close() throws Exception {
+			super.close();
+		}
+
+		@Override
+		public void apply(String key,
+				W window,
+				Iterable<Tuple2<String, Integer>> input,
+				Collector<Tuple2<String, Integer>> out) throws Exception {
+
+			if (!openCalled) {
+				fail("Open was not called");
+			}
+			int sum = 0;
+
+			for (Tuple2<String, Integer> t: input) {
+				sum += t.f1;
+			}
+			out.collect(new Tuple2<>(key, sum));
+
+		}
+
+	}
+
+	public static class SessionWindowFunction implements WindowFunction<Tuple2<String, Integer>, Tuple3<String, Long, Long>, String, TimeWindow> {
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public void apply(String key,
+				TimeWindow window,
+				Iterable<Tuple2<String, Integer>> values,
+				Collector<Tuple3<String, Long, Long>> out) throws Exception {
+			int sum = 0;
+			for (Tuple2<String, Integer> i: values) {
+				sum += i.f1;
+			}
+			String resultString = key + "-" + sum;
+			out.collect(new Tuple3<>(resultString, window.getStart(), window.getEnd()));
+		}
+	}
+}
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..d182bee59961968c62270be1526f1ff3667c0f66
GIT binary patch
literal 222
zcmZvWJr2S!422yE6GJc2H4;hOxU5_R<;Pt(0XIV&f{mFo^aNlRvhgJQeX^b{0{|Ep
z5*<=;v08M&Nz2~m8oZstahjPj&JiVfmJ;o6&HLqX5;pf2JjFM=3tzoJ*TDP-*TM>G
rW`*}pI_#vW%-0kKERn|I(GsU(F_65;BgwBh2F}rOsuJJoik|uab}}$?

literal 0
HcmV?d00001

diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..83741c1a32fad972c56c0177e33e7cd6bb975038
GIT binary patch
literal 187
zcmXv|u?+$-3`}^aC|H8lQtZU}M51C7@<|#dU^DbX&`~o36M+2>C;9FypDi0>j5S9t
zjY4k@B71~Du|u9*gF2;@2}?!Kk~~QX_V>RY-jB0*JHO0RbhEp*8!aT=pavJx+2;q%
l4?+sN%KC%D?8`{N+!#XX7f4)3z}P9~lp~@3u+<ek^#!)HDrx`#

literal 0
HcmV?d00001

diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..3411df6dbb057969a1d8c1bee58bb9ec706ac759
GIT binary patch
literal 2193
zcmd^AJ#W)M7=GiB4iwaefh>%L&33*MD+8hkq)3q%5{aSH#XhIkI(B-#G)<9^m=OPf
zi4}nu*nl5^82Af_iIt5F33%`9BzBuhfi9Ie$?|(&&%GbdJqLj8cihm-WaR+1O!Sk+
zmeqHxMy1@)E48wtS1sGpEn0E)8nK#=ZP#hPS;vJx)V8Y=<~zYe7i>s*_~tw0LJqBf
z4-7IQ_AWJCkNNx9VTK(Hhk<X1>4=U6^FpIfLTYpcXa1l|IU^oBraX2eqX8Uo*!~No
z9s)0Lx`^sg5%|+rPhWpBCif!1>S};#tfWt)IL7;@fI`738RD@j+9)rw7g?dv!xTMK
z+1gsTnS|zWe(w-3o7<<;QVbmb*560Q`a3yUUSV>!qcDmV7lL@xNzUYsku#3nFH^Tz
z*Q-SfIn%eBdb8N5>qXbDmaRt3E;%;9+9qO1s>$!$|3DTMsRkSIEwKY%P+x>`3SL}z
zE@7dvA7d}erx?%2zP$HFD2SlO9nQ(L!$PrO_r|kFU%q}WywxDnfjsr-5TjpA3#fMx
z)QqIUR8oN*W=2wB&ZWXk-^Ugz!}+hunL@Qk4oO0$W>NOWBQGH^SVP%jp}Ku4Ia2ag
zL)u0ijKCz?r8eXUXJXohHIMoOvAYACE~B2aNL(nOvGZ@K>%e6p1G(#w5MlBi>7(fp
ze_*Pi*3khx1{6!s@SXTI=J81@==q5vSx;`f{BZr}b1A%1M6@##egR$v6*M~7Tw2HF
z4z`nAUK9{i2Cp1)xV024B89Ahp(r&|W30W_0nIH6xR$8OgZ>`1MN7_<q?dUl|IL+v
ulP<+7E`?XtHqn&OYqK|F{&-|&BWXiAtJX+EO{I+pIQ4kUSE)nN%l`o0Vw3U!

literal 0
HcmV?d00001

diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..060397f6079033d02403b33a0fda370513b91ea1
GIT binary patch
literal 2081
zcmd^AJ#W)M7=Ggt20m26Ko`csX8WAx!%+rA5lE3NF(eX0r?Y)duXXHjzBEmdkeCqv
zfQc1>7#Lw-L=5}@7R1ELFF*qB>`U!5i4^EkiIc2*@6LYwemw7U0Al=BLztyCxgka=
zo20Qxu<IdYTe@#sNUzzgs_wQ>O~)<@Y6LaxI<m!!U((Mmk7?w`W1Z6hVab~xP)-=M
zW7aqD5PQ4C2tpd|iv=~jcrb_~gHMKJ#A%oqE>4Ki<%~xCE@2dh^oTI!NJ#@YWU&1g
zNIe0Z<8%dTNO&AgUOj#N$r#_w0BfrO&SDjP8mbuY%>gBx5j+sTO{2~7B3GG}7(J1q
z2Whr8b2p373{LNz!qdw8WKzu*7vE*`#!$_%JltA#DE3d%qI@K1(QIOb-Ik{ZwpZ6{
z7;Wh;vV6T!w@~2vmc3>B0Ob$zd-Z((ANY_)tHHWz1TT&_iFl$?@Z!wVMHFl8qw=yk
zM`|{TWNRA<$DA0q8N(BYCOjwb`m;x0zkS|#t3k<u3JJ+TM4umXXgC7Y%+$b4)j%w$
znW=$!tOn+6%e6)GpS~NGYSA7(zy&3mM%fz;!-9s08p@%x=+<0vq^hlkv@N7A1Cway
zI#9-p@<|8Iha~Fr-5t0RP!jri;t~$coqtQ60~fdq<W7ha0aJ0Lk7kIYzL{>d4<Eu~
z5PArjxC(xanEKNSl6|5`ttZ!CzQ6YKxztvvF51$Bp8@etgBA|j3+uSp!A_COc>y8K
z;H3iwHy46sq>wc*Qc5W$mbKR$&;rEaYN0Ck-96&*wmh+<mlZ+&8!G`9T~hKs6<={Q
Sqx{ELLTP$8q=vi)a^nvq?}gm}

literal 0
HcmV?d00001

diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..dad4630a55beafc741e40fbedf3653de85bdc965
GIT binary patch
literal 1988
zcmd^=zi-n(6vtnj1PLmB4s>BGY_?<9X<}tSY6ViHNDPU@(CIEYr*55ddcHIb0}>Pe
z0uw6&u>*er{|I7ZV?zSo`7Xq1BNPNfC7xvY&Ug3qyL<0_4gl?kF>IE~>H}Iyzf@yu
z#~b)zGoZF#qXRo=c%dCOuvHHzZS1fn4Z}K4{F*PjI*s`to!T-UF_FFf0@X}FD-}bB
zj7k51IZ+bxBkaUZKOK!y?#S7gO=O&84ka0Lx>Ce^*kvLnN&JKf<4DN@I2Ew_2TDB@
zUg&gDHDNO4v)9kxd~~J{v|zBF;IdZHr?J`N!*fEJ6pW1UxA|`Mxu~9QWlj%|=y85p
zTMIXf(gOb8JJc&zkJ!vJ!sfN{UN<)0sX>2@!TE~Kv7Wq{nnh=Grt%qchO(wleNwXt
zCEIooF=B`HAhP|&HrpO}b)QiRP~Affm1^qw^gkp^zSA;XH&xP4xnx{s<_KP1_*}w3
zmA&5v>X4j}LQRVaS153LlW|h0M%=J6Q`Ps+)sEuBdP=(p4K1eBDs89|5zAQ{E+vc)
z<-tDOh+>uu7A2NRXzX|Jm##*Vn<EK3v8$2f{z8(wT=%UGT#;%cdkM)<=4MCv=q7{@
z-TYGv5%o`M)<PzKp9tgV8tgpdX!z{Ts}Hxoy-<)-1#4Yo3LnE$sGx(*m1(g0nrL&m
zi0C5zaH_;Ep9r|OQcUNR3Zc%(Ir%VktPWUFO~S20P#@7l)|VJ|EAo`^HH`1wwS<c<
j8Dc8PVLXbKL)FUpJco2nYky8<b006kx0DagXIMW0Ki^eE

literal 0
HcmV?d00001

diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..0edf25ec50e20028a0c13aa94f6e1f1fad55bf23
GIT binary patch
literal 1925
zcmd^=&x+GP9LIl6TS3LY2fcU^R&Pm~ZrY^uAgm}93LaYUu%~G{>CSFC=}xBGtp~xA
zui(k6C_aEbfsfz|h&P`=P=Aw|+H|){MGp%Ol=+u@^PBnozMleM`)(QzOJ%iyt)x3O
z*b3`@6vxBR4jZk&_T#8!lV-DK`$4l-r(vT`TQz*}WB%Hu8RNrrW=l4vB76G<s+oXx
zDn<^OkZ6xOal-fkPM8y=<8jIzIiJv}WJ%@>NJgEW6pW90R4|gTCsdd~N*2JefSo^3
z>Z9;dr-!Nul_{UUe*WfzGkc%~{mledwTdxK%o!h^63V2YWQ?E9Ppj`m&Gaa9`glc;
z^V`~5hFO#r@b}RfJ*#>^=XE`~_)hmXC#Fx;=Ju~Pas6ajR6b)_c!M|~zUSM6ASSjS
zh1)jqut0(+tW)aMyk^`0sNPoZo8PDZp&s(nmf?yqLX>hzxy;NJygc*eg2^g-zfIIW
zIU)tyiYZracly&wQZO7IST$qZd#CK75U`oj4st<@DYZ%mszfk3@4&f)@{!!zg{v{6
z$#ChSOhT~R#ZS7LI&MxKoG@2Y$Nfnich%bME?ki6AomlJq0F_eiqTC7AG!Hb8!7Zp
zY-u5OzfFY+bOSt|2(<O=)~ok7zP?Z#Q%<(dGH4IsDO51PaBUf^|0n2hxy<M?|8T6t
z&L0W5yH-rsluDtlNJBkUI#w5~xF+FxA*qiB`!tf6b}ROj@MX;JosEQxAsLh|K!;l^
Ql-2lMFu0WSb^+be4+u_8tpET3

literal 0
HcmV?d00001

diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..d0afbe02b659522bfaadbbbcb9750bbbd8c39200
GIT binary patch
literal 3703
zcmeHKKX21O6n}9DLsKX%5~vbmVYA(&#)*{yQ3O&Y3x-rMbh<c~)U9KO^QCP-LSjOE
z114ZYAO=QQD<K9x01E;MvGNI!fOqz}cAG|oR+U=Olaf2%`F?)??)TmSVEZi_4s&@i
zfh`wzVlZR1=9*@yMAb4eOw}NbIkjpMQze#px!Eu(v`j1b;-_fasdd+H2J0$!JCud5
ze?TE*PzzXFBVFPwQ?2E>{whAWniF(7fv@p?m-e{pg<6AzRI795`t3SpF7e!Zlu1Ey
z3cv<~`F}uKMBo&sj;J2xf!}}m=+y^p{dx>&j0U(JD=Sl1>Ud)SDCCTi4t_S$Hnxj+
zibtWgh$gxdS!*U0GYidN|I#6zQdp(^vILghbTb}IJ7TdUXUF1x;<@af$@!gOQLJNF
z7?fJptU=W|!?ILk)~u=xgH~12ELR#;$8=hh0F0kS3=vJ@_x*o}2M}i{4+SUa`CKag
z^w0;1aL+d>r8N-B!;h;S9I3;KS}HIhIm!uqPJJF8B`4+IODB~FoWwBRkW#de6Hc|O
zjFG<WhCJn#iO2UpfBAU+jRH9vCaFg|sC~N3!L$)nkBM53h#DVUJtk^>i>USOY*<5H
z-ak<dO%p3*jbxHUqUB<*>t*O2eJGkt6ITX|BO-V-q<PFjF_=I*u>b|aT;5-RlOFZk
ze0d4Zv|Q>nQ^SQEDocAaoeg7LD00msA;L`BLPp&qep`>GY8!Xq0fsDwEoSLoJtn_2
zLN?vuB<taY7w^vfd?M0bSKtjb_#w#4k<ut&byq*`F5p5omQx2pWWn(@2A6gPi-SV=
zK#M{uNtEb&#RjD{%V8?xm0OJ!>hPLaDG4bjG59Zz1e_HqasW(WiPXl5K$UzH(j;Ge
zm51CMKu*FxKIFFpw|SFvXo$7d(8F*DYbLY>CY*fbn9Cx@KIXE&+zdwGvID5UBNh^W
z2gV+!_skyg35u#}K`hzIJ(n%%G1n*6cjmp~rr4jn7hMp<9Uz?nWouWuDgdx$zm<6|
SZYK8(u{G0@yGAlqeESD6oqC-B

literal 0
HcmV?d00001

diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..7d0b6cc28385fbe643cfa35e2001b2ea257f3e1c
GIT binary patch
literal 494
zcmZQz0D}Ke0Aw;Ug9Lzp5ki3&Dkes$X~rf=X}Sg`Ny)k<$p*=~Nfw5Qx+&(SW(J9d
zrilirX+VW@!P<DsGxJjN%XLdKb5o0o*Suw5EiPiP&M!*WODsrC&Pdfu%gM~k268g>
zlJj$O^YipdDhpCeOEPnc^^y{cQ}u#NiZb)kgHww#6LT`FQj4GtVPs-ps3>A^BEh5(
zpuq@pf`O*yq?Y98RjzDbbzHC96=I+XaR#EAh3uvRs2|-h3@R=uN=?iKS_}3qBwRo|
d2&-5x1Q<l&;LuV*I1}0EA_mG`2Q?2I$pA=~pEUpg

literal 0
HcmV?d00001


From 9174952788d8c544930a6871ade9b0c9122060bb Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Mon, 24 Apr 2017 17:50:59 +0200
Subject: [PATCH 126/253] [FLINK-5969] Add BucketingSinkFrom12MigrationTest

The binary snapshots have been created on the Flink 1.2 branch.
---
 .../fs/bucketing/BucketingSink.java           |   6 +
 .../BucketingSinkFrom12MigrationTest.java     | 223 ++++++++++++++++++
 ...ting-sink-migration-test-flink1.2-snapshot | Bin 0 -> 1623 bytes
 3 files changed, 229 insertions(+)
 create mode 100644 flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkFrom12MigrationTest.java
 create mode 100644 flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.2-snapshot

diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java
index 7dbcda75c69cb..20e54b8122f24 100644
--- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java
+++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java
@@ -18,6 +18,7 @@
 package org.apache.flink.streaming.connectors.fs.bucketing;
 
 import org.apache.commons.lang3.time.StopWatch;
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.OperatorStateStore;
@@ -1033,6 +1034,11 @@ public BucketingSink<T> setAsyncTimeout(long timeout) {
 		return this;
 	}
 
+	@VisibleForTesting
+	public State<T> getState() {
+		return state;
+	}
+
 	// --------------------------------------------------------------------------------------------
 	//  Internal Classes
 	// --------------------------------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkFrom12MigrationTest.java b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkFrom12MigrationTest.java
new file mode 100644
index 0000000000000..350b7b4df1f35
--- /dev/null
+++ b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkFrom12MigrationTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.connectors.fs.bucketing;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.FileUtils;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.OperatorStateStore;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.streaming.api.operators.StreamSink;
+import org.apache.flink.streaming.connectors.fs.StringWriter;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OperatorSnapshotUtil;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Tests for checking whether {@link BucketingSink} can restore from snapshots that were done
+ * using the Flink 1.2 {@link BucketingSink}.
+ *
+ * <p>For regenerating the binary snapshot file you have to run the {@code write*()} method on
+ * the Flink 1.2 branch.
+ */
+
+public class BucketingSinkFrom12MigrationTest {
+
+	@ClassRule
+	public static TemporaryFolder tempFolder = new TemporaryFolder();
+
+	private static final String PART_PREFIX = "part";
+	private static final String PENDING_SUFFIX = ".pending";
+	private static final String IN_PROGRESS_SUFFIX = ".in-progress";
+	private static final String VALID_LENGTH_SUFFIX = ".valid";
+
+	/**
+	 * Manually run this to write binary snapshot data. Remove @Ignore to run.
+	 */
+	@Ignore
+	@Test
+	public void writeSnapshot() throws Exception {
+
+		final File outDir = tempFolder.newFolder();
+
+		BucketingSink<String> sink = new BucketingSink<String>(outDir.getAbsolutePath())
+			.setWriter(new StringWriter<String>())
+			.setBatchSize(5)
+			.setPartPrefix(PART_PREFIX)
+			.setInProgressPrefix("")
+			.setPendingPrefix("")
+			.setValidLengthPrefix("")
+			.setInProgressSuffix(IN_PROGRESS_SUFFIX)
+			.setPendingSuffix(PENDING_SUFFIX)
+			.setValidLengthSuffix(VALID_LENGTH_SUFFIX);
+
+		OneInputStreamOperatorTestHarness<String, Object> testHarness =
+			new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));
+
+		testHarness.setup();
+		testHarness.open();
+
+		testHarness.processElement(new StreamRecord<>("test1", 0L));
+		testHarness.processElement(new StreamRecord<>("test2", 0L));
+
+		checkFs(outDir, 1, 1, 0, 0);
+
+		testHarness.processElement(new StreamRecord<>("test3", 0L));
+		testHarness.processElement(new StreamRecord<>("test4", 0L));
+		testHarness.processElement(new StreamRecord<>("test5", 0L));
+
+		checkFs(outDir, 1, 4, 0, 0);
+
+		OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
+
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/bucketing-sink-migration-test-flink1.2-snapshot");
+		testHarness.close();
+	}
+
+	@Test
+	public void testRestore() throws Exception {
+		final File outDir = tempFolder.newFolder();
+
+		ValidatingBucketingSink<String> sink = (ValidatingBucketingSink<String>) new ValidatingBucketingSink<String>(outDir.getAbsolutePath())
+			.setWriter(new StringWriter<String>())
+			.setBatchSize(5)
+			.setPartPrefix(PART_PREFIX)
+			.setInProgressPrefix("")
+			.setPendingPrefix("")
+			.setValidLengthPrefix("")
+			.setInProgressSuffix(IN_PROGRESS_SUFFIX)
+			.setPendingSuffix(PENDING_SUFFIX)
+			.setValidLengthSuffix(VALID_LENGTH_SUFFIX);
+
+		OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(
+			new StreamSink<>(sink), 10, 1, 0);
+		testHarness.setup();
+		testHarness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("bucketing-sink-migration-test-flink1.2-snapshot")));
+		testHarness.open();
+
+		assertTrue(sink.initializeCalled);
+
+		testHarness.processElement(new StreamRecord<>("test1", 0L));
+		testHarness.processElement(new StreamRecord<>("test2", 0L));
+
+		checkFs(outDir, 1, 1, 0, 0);
+
+		testHarness.close();
+	}
+
+	private void checkFs(File outDir, int inprogress, int pending, int completed, int valid) throws IOException {
+		int inProg = 0;
+		int pend = 0;
+		int compl = 0;
+		int val = 0;
+
+		for (File file: FileUtils.listFiles(outDir, null, true)) {
+			if (file.getAbsolutePath().endsWith("crc")) {
+				continue;
+			}
+			String path = file.getPath();
+			if (path.endsWith(IN_PROGRESS_SUFFIX)) {
+				inProg++;
+			} else if (path.endsWith(PENDING_SUFFIX)) {
+				pend++;
+			} else if (path.endsWith(VALID_LENGTH_SUFFIX)) {
+				val++;
+			} else if (path.contains(PART_PREFIX)) {
+				compl++;
+			}
+		}
+
+		Assert.assertEquals(inprogress, inProg);
+		Assert.assertEquals(pending, pend);
+		Assert.assertEquals(completed, compl);
+		Assert.assertEquals(valid, val);
+	}
+
+	static class ValidatingBucketingSink<T> extends BucketingSink<T> {
+
+		private static final long serialVersionUID = -4263974081712009141L;
+
+		public boolean initializeCalled = false;
+
+		ValidatingBucketingSink(String basePath) {
+			super(basePath);
+		}
+
+		/**
+		 * The actual paths in this depend on the binary checkpoint so it you update this the paths
+		 * here have to be updated as well.
+		 */
+		@Override
+		public void initializeState(FunctionInitializationContext context) throws Exception {
+			OperatorStateStore stateStore = context.getOperatorStateStore();
+
+			ListState<State<T>> restoredBucketStates = stateStore.getSerializableListState("bucket-states");
+
+			if (context.isRestored()) {
+
+				for (State<T> states : restoredBucketStates.get()) {
+					for (String bucketPath : states.bucketStates.keySet()) {
+						BucketState state = states.getBucketState(new Path(bucketPath));
+						String current = state.currentFile;
+						long validLength = state.currentFileValidLength;
+
+						Assert.assertEquals("/var/folders/v_/ry2wp5fx0y7c1rvr41xy9_700000gn/T/junit9160378385359106772/junit479663758539998903/1970-01-01--01/part-0-4", current);
+						Assert.assertEquals(6, validLength);
+
+						List<String> pendingFiles = state.pendingFiles;
+						assertTrue(pendingFiles.isEmpty());
+
+						final Map<Long, List<String>> pendingFilesPerCheckpoint = state.pendingFilesPerCheckpoint;
+						Assert.assertEquals(1, pendingFilesPerCheckpoint.size());
+
+						for (Map.Entry<Long, List<String>> entry: pendingFilesPerCheckpoint.entrySet()) {
+							long checkpoint = entry.getKey();
+							List<String> files = entry.getValue();
+
+							Assert.assertEquals(0L, checkpoint);
+							Assert.assertEquals(4, files.size());
+
+							for (int i = 0; i < 4; i++) {
+								Assert.assertEquals(
+										"/var/folders/v_/ry2wp5fx0y7c1rvr41xy9_700000gn/T/junit9160378385359106772/junit479663758539998903/1970-01-01--01/part-0-" + i,
+										files.get(i));
+							}
+						}
+					}
+				}
+			}
+
+			initializeCalled = true;
+			super.initializeState(context);
+		}
+	}
+}
diff --git a/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.2-snapshot b/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..a541bada4ac04d1761cf4ee8bbdd9ae404edc688
GIT binary patch
literal 1623
zcmc&#y>HV%6n}0LKH35;NR=2m(bZpZ636PmmlV;+Af!aVRxZvtaqFDT-8pp_kXTqc
zfDjA}h=sBL1Y%%8NHDU%z{<)B?;KDnG9r;D`|fn_`S*MGd+%fbw4Z1=X(ctlscshR
zQK<__Bo)f9sR8Yk2Ad&uXuUzYdf?zttvN*Zydc!WN~KY+bQ@hK=mL~~V7N95cK992
zf`gv$zCltF45$&t5!*9(#$+_4MnP({$cVHlk4PNdr@Yy|^Wk-M>FEMM&Y?Q<ElDFI
zNQT3N8FHM`Oh&OVI=Grb6zO9|3L;gh{Z6Dg112P=WEinNLNZ1JnQ&o*!kCDn_OBH3
z;Hr&@Z?=_6Q+UmXl}Tvf5E9P#yLcK?Wo2F`=~kYi0BdT`P<zHTA_ll$KK}X5+x5F2
z7h$0ZOL0PaO`^!R;iMdJDh5g1%hQXO3aXX;6F9Hpt4T=B5#eTte#(V8+A;aKvY*z%
z+#0)q%}2ay=VNcjwUqUl*)exB7D>;pTaMds8a1cp*;d_kD-%c6_3CxUt)av7yoP5v
zrtP_wZrSR=7xVZ3S+|Lbf2mh)!FrH!PMK^*F};QL@A;JV<)8&?F%j}MkEEomlXS3I
zd{{mbEaB`#>6oy-*_IqDw1w43DeK`<;bApak5t2tgv5yRM<Z@hesw^Dy)=oKyax|J
zEBZyLeoPI@B#i&J<Yr2^)GfV=+Mk>D&SlQYSREsO_-W(CJMwZ7v~4H}yfu@w_tlk#
zT&R;Q&NdICoi5OnW`>`jVh7)zynlS*%K~UED33^-QM_}j)1VvKuuJ))!{-~z&%Pe~
jxlf7(3YKMDlnUC?9Cfnh$ZgM&yD~>^C(m>BCsFnbi%a04

literal 0
HcmV?d00001


From 8491941a97908e9968b080151002d3ec6c8768f7 Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Fri, 28 Apr 2017 12:28:50 +0200
Subject: [PATCH 127/253] [FLINK-5969] Add CEPFrom12MigrationTest

The binary snapshots have been created on the Flink 1.2 branch.
---
 .../AbstractKeyedCEPPatternOperator.java      |  11 +-
 ...3Test.java => CEPFrom12MigrationTest.java} | 394 +++++++++---------
 ...igration-after-branching-flink1.2-snapshot | Bin 0 -> 5580 bytes
 ...ingle-pattern-afterwards-flink1.2-snapshot | Bin 0 -> 2326 bytes
 ...ion-starting-new-pattern-flink1.2-snapshot | Bin 0 -> 5389 bytes
 5 files changed, 211 insertions(+), 194 deletions(-)
 rename flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/{CEPMigration12to13Test.java => CEPFrom12MigrationTest.java} (54%)
 create mode 100644 flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink1.2-snapshot
 create mode 100644 flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink1.2-snapshot
 create mode 100644 flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink1.2-snapshot

diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
index 5f2487590a7df..cd62c0d4de029 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
@@ -42,6 +42,7 @@
 import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.util.Migration;
 import org.apache.flink.util.OutputTag;
 import org.apache.flink.util.Preconditions;
 
@@ -323,9 +324,13 @@ private void updatePriorityQueue(PriorityQueue<StreamRecord<IN>> queue) throws I
 
 	@Override
 	public void restoreState(FSDataInputStream in) throws Exception {
-		// this is the flag indicating if we have udf
-		// state to restore (not needed here)
-		in.read();
+		if (in instanceof Migration) {
+			// absorb the introduced byte from the migration stream
+			int hasUdfState = in.read();
+			if (hasUdfState == 1) {
+				throw new Exception("Found UDF state but CEPOperator is not an UDF operator.");
+			}
+		}
 
 		DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(in);
 		timerService = getInternalTimerService(
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigration12to13Test.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPFrom12MigrationTest.java
similarity index 54%
rename from flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigration12to13Test.java
rename to flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPFrom12MigrationTest.java
index 82495355238d3..2f7cdeb8bd69a 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigration12to13Test.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPFrom12MigrationTest.java
@@ -26,40 +26,85 @@
 import org.apache.flink.cep.nfa.compiler.NFACompiler;
 import org.apache.flink.cep.pattern.Pattern;
 import org.apache.flink.cep.pattern.conditions.SimpleCondition;
-import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.OperatorStateHandle;
-import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
 import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
 import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OperatorSnapshotUtil;
+import org.junit.Ignore;
 import org.junit.Test;
 
-import java.io.FileInputStream;
-import java.io.ObjectInputStream;
-import java.net.URL;
-import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class CEPMigration12to13Test {
+/**
+ * Tests for checking whether CEP operator can restore from snapshots that were done
+ * using the Flink 1.2 operator.
+ *
+ * <p>For regenerating the binary snapshot file you have to run the {@code write*()} method on
+ * the Flink 1.2 branch.
+ */
 
-	private static String getResourceFilename(String filename) {
-		ClassLoader cl = CEPMigration12to13Test.class.getClassLoader();
-		URL resource = cl.getResource(filename);
-		if (resource == null) {
-			throw new NullPointerException("Missing snapshot resource.");
-		}
-		return resource.getFile();
+public class CEPFrom12MigrationTest {
+
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
+	@Test
+	public void writAfterBranchingPatternSnapshot() throws Exception {
+
+		KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
+			private static final long serialVersionUID = -4873366487571254798L;
+
+			@Override
+			public Integer getKey(Event value) throws Exception {
+				return value.getId();
+			}
+		};
+
+		final Event startEvent = new Event(42, "start", 1.0);
+		final SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
+		final SubEvent middleEvent2 = new SubEvent(42, "foo2", 2.0, 10.0);
+
+		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
+				new KeyedOneInputStreamOperatorTestHarness<>(
+						new KeyedCEPPatternOperator<>(
+								Event.createTypeSerializer(),
+								false,
+								keySelector,
+								IntSerializer.INSTANCE,
+								new NFAFactory(),
+								null,
+								true),
+						keySelector,
+						BasicTypeInfo.INT_TYPE_INFO);
+
+		harness.setup();
+		harness.open();
+
+		harness.processElement(new StreamRecord<Event>(startEvent, 1));
+		harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
+		harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
+		harness.processElement(new StreamRecord<Event>(middleEvent1, 2));
+		harness.processElement(new StreamRecord<Event>(middleEvent2, 3));
+
+		harness.processWatermark(new Watermark(5));
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = harness.snapshot(0L, 0L);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/cep-migration-after-branching-flink1.2-snapshot");
+
+		harness.close();
 	}
 
 	@Test
-	public void testMigrationAfterBranchingPattern() throws Exception {
+	public void testRestoreAfterBranchingPattern() throws Exception {
 
 		KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
 			private static final long serialVersionUID = -4873366487571254798L;
@@ -75,66 +120,23 @@ public Integer getKey(Event value) throws Exception {
 		final SubEvent middleEvent2 = new SubEvent(42, "foo2", 2.0, 10.0);
 		final Event endEvent = new Event(42, "end", 1.0);
 
-		// uncomment these lines for regenerating the snapshot on Flink 1.2
-//		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
-//			new KeyedOneInputStreamOperatorTestHarness<>(
-//				new KeyedCEPPatternOperator<>(
-//					Event.createTypeSerializer(),
-//					false,
-//					keySelector,
-//					IntSerializer.INSTANCE,
-//					new NFAFactory(),
-//					true),
-//				keySelector,
-//				BasicTypeInfo.INT_TYPE_INFO);
-//
-//		harness.setup();
-//		harness.open();
-//		harness.processElement(new StreamRecord<Event>(startEvent, 1));
-//		harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
-//		harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
-//		harness.processElement(new StreamRecord<Event>(middleEvent1, 2));
-//		harness.processElement(new StreamRecord<Event>(middleEvent2, 3));
-//		harness.processWatermark(new Watermark(5));
-//		// simulate snapshot/restore with empty element queue but NFA state
-//		OperatorStateHandles snapshot = harness.snapshot(1, 1);
-//		FileOutputStream out = new FileOutputStream(
-//				"src/test/resources/cep-branching-snapshot-1.2");
-//		ObjectOutputStream oos = new ObjectOutputStream(out);
-//		oos.writeObject(snapshot.getOperatorChainIndex());
-//		oos.writeObject(snapshot.getLegacyOperatorState());
-//		oos.writeObject(snapshot.getManagedKeyedState());
-//		oos.writeObject(snapshot.getRawKeyedState());
-//		oos.writeObject(snapshot.getManagedOperatorState());
-//		oos.writeObject(snapshot.getRawOperatorState());
-//		out.close();
-//		harness.close();
-
 		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
-			new KeyedOneInputStreamOperatorTestHarness<>(
-				new KeyedCEPPatternOperator<>(
-					Event.createTypeSerializer(),
-					false,
-					keySelector,
-					IntSerializer.INSTANCE,
-					new NFAFactory(),
-					null,
-					true),
-				keySelector,
-				BasicTypeInfo.INT_TYPE_INFO);
+				new KeyedOneInputStreamOperatorTestHarness<>(
+						new KeyedCEPPatternOperator<>(
+								Event.createTypeSerializer(),
+								false,
+								keySelector,
+								IntSerializer.INSTANCE,
+								new NFAFactory(),
+								null,
+								true),
+						keySelector,
+						BasicTypeInfo.INT_TYPE_INFO);
 
 		harness.setup();
-		final ObjectInputStream ois = new ObjectInputStream(new FileInputStream(getResourceFilename(
-			"cep-branching-snapshot-1.2")));
-		final OperatorStateHandles snapshot = new OperatorStateHandles(
-			(int) ois.readObject(),
-			(StreamStateHandle) ois.readObject(),
-			(Collection<KeyedStateHandle>) ois.readObject(),
-			(Collection<KeyedStateHandle>) ois.readObject(),
-			(Collection<OperatorStateHandle>) ois.readObject(),
-			(Collection<OperatorStateHandle>) ois.readObject()
-		);
-		harness.initializeState(snapshot);
+		harness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("cep-migration-after-branching-flink1.2-snapshot")));
 		harness.open();
 
 		harness.processElement(new StreamRecord<>(new Event(42, "start", 1.0), 4));
@@ -174,8 +176,55 @@ public Integer getKey(Event value) throws Exception {
 		harness.close();
 	}
 
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
 	@Test
-	public void testStartingNewPatternAfterMigration() throws Exception {
+	public void writeStartingNewPatternAfterMigrationSnapshot() throws Exception {
+
+		KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
+			private static final long serialVersionUID = -4873366487571254798L;
+
+			@Override
+			public Integer getKey(Event value) throws Exception {
+				return value.getId();
+			}
+		};
+
+		final Event startEvent1 = new Event(42, "start", 1.0);
+		final SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
+
+		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
+				new KeyedOneInputStreamOperatorTestHarness<>(
+						new KeyedCEPPatternOperator<>(
+								Event.createTypeSerializer(),
+								false,
+								keySelector,
+								IntSerializer.INSTANCE,
+								new NFAFactory(),
+								null,
+								true),
+						keySelector,
+						BasicTypeInfo.INT_TYPE_INFO);
+
+		harness.setup();
+		harness.open();
+		harness.processElement(new StreamRecord<Event>(startEvent1, 1));
+		harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
+		harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
+		harness.processElement(new StreamRecord<Event>(middleEvent1, 2));
+		harness.processWatermark(new Watermark(5));
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = harness.snapshot(0L, 0L);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/cep-migration-starting-new-pattern-flink1.2-snapshot");
+
+		harness.close();
+	}
+
+	@Test
+	public void testRestoreStartingNewPatternAfterMigration() throws Exception {
 
 		KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
 			private static final long serialVersionUID = -4873366487571254798L;
@@ -192,65 +241,23 @@ public Integer getKey(Event value) throws Exception {
 		final SubEvent middleEvent2 = new SubEvent(42, "foo2", 2.0, 10.0);
 		final Event endEvent = new Event(42, "end", 1.0);
 
-		// uncomment these lines for regenerating the snapshot on Flink 1.2
-//		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
-//			new KeyedOneInputStreamOperatorTestHarness<>(
-//				new KeyedCEPPatternOperator<>(
-//					Event.createTypeSerializer(),
-//					false,
-//					keySelector,
-//					IntSerializer.INSTANCE,
-//					new NFAFactory(),
-//					true),
-//				keySelector,
-//				BasicTypeInfo.INT_TYPE_INFO);
-//
-//		harness.setup();
-//		harness.open();
-//		harness.processElement(new StreamRecord<Event>(startEvent1, 1));
-//		harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
-//		harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
-//		harness.processElement(new StreamRecord<Event>(middleEvent1, 2));
-//		harness.processWatermark(new Watermark(5));
-//		// simulate snapshot/restore with empty element queue but NFA state
-//		OperatorStateHandles snapshot = harness.snapshot(1, 1);
-//		FileOutputStream out = new FileOutputStream(
-//				"src/test/resources/cep-starting-snapshot-1.2");
-//		ObjectOutputStream oos = new ObjectOutputStream(out);
-//		oos.writeObject(snapshot.getOperatorChainIndex());
-//		oos.writeObject(snapshot.getLegacyOperatorState());
-//		oos.writeObject(snapshot.getManagedKeyedState());
-//		oos.writeObject(snapshot.getRawKeyedState());
-//		oos.writeObject(snapshot.getManagedOperatorState());
-//		oos.writeObject(snapshot.getRawOperatorState());
-//		out.close();
-//		harness.close();
-
 		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
-			new KeyedOneInputStreamOperatorTestHarness<>(
-				new KeyedCEPPatternOperator<>(
-					Event.createTypeSerializer(),
-					false,
-					keySelector,
-					IntSerializer.INSTANCE,
-					new NFAFactory(),
-					null,
-					true),
-				keySelector,
-				BasicTypeInfo.INT_TYPE_INFO);
+				new KeyedOneInputStreamOperatorTestHarness<>(
+						new KeyedCEPPatternOperator<>(
+								Event.createTypeSerializer(),
+								false,
+								keySelector,
+								IntSerializer.INSTANCE,
+								new NFAFactory(),
+								null,
+								true),
+						keySelector,
+						BasicTypeInfo.INT_TYPE_INFO);
 
 		harness.setup();
-		final ObjectInputStream ois = new ObjectInputStream(new FileInputStream(getResourceFilename(
-			"cep-starting-snapshot-1.2")));
-		final OperatorStateHandles snapshot = new OperatorStateHandles(
-			(int) ois.readObject(),
-			(StreamStateHandle) ois.readObject(),
-			(Collection<KeyedStateHandle>) ois.readObject(),
-			(Collection<KeyedStateHandle>) ois.readObject(),
-			(Collection<OperatorStateHandle>) ois.readObject(),
-			(Collection<OperatorStateHandle>) ois.readObject()
-		);
-		harness.initializeState(snapshot);
+		harness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("cep-migration-starting-new-pattern-flink1.2-snapshot")));
 		harness.open();
 
 		harness.processElement(new StreamRecord<>(startEvent2, 5));
@@ -303,8 +310,12 @@ public Integer getKey(Event value) throws Exception {
 		harness.close();
 	}
 
+	/**
+	 * Manually run this to write binary snapshot data.
+	 */
+	@Ignore
 	@Test
-	public void testSinglePatternAfterMigration() throws Exception {
+	public void writeSinglePatternAfterMigrationSnapshot() throws Exception {
 
 		KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
 			private static final long serialVersionUID = -4873366487571254798L;
@@ -317,61 +328,62 @@ public Integer getKey(Event value) throws Exception {
 
 		final Event startEvent1 = new Event(42, "start", 1.0);
 
-		// uncomment these lines for regenerating the snapshot on Flink 1.2
-//		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
-//			new KeyedOneInputStreamOperatorTestHarness<>(
-//				new KeyedCEPPatternOperator<>(
-//					Event.createTypeSerializer(),
-//					false,
-//					keySelector,
-//					IntSerializer.INSTANCE,
-//					new SinglePatternNFAFactory(),
-//					true),
-//				keySelector,
-//				BasicTypeInfo.INT_TYPE_INFO);
-//
-//		harness.setup();
-//		harness.open();
-//		harness.processWatermark(new Watermark(5));
-//		// simulate snapshot/restore with empty element queue but NFA state
-//		OperatorStateHandles snapshot = harness.snapshot(1, 1);
-//		FileOutputStream out = new FileOutputStream(
-//				"src/test/resources/cep-single-pattern-snapshot-1.2");
-//		ObjectOutputStream oos = new ObjectOutputStream(out);
-//		oos.writeObject(snapshot.getOperatorChainIndex());
-//		oos.writeObject(snapshot.getLegacyOperatorState());
-//		oos.writeObject(snapshot.getManagedKeyedState());
-//		oos.writeObject(snapshot.getRawKeyedState());
-//		oos.writeObject(snapshot.getManagedOperatorState());
-//		oos.writeObject(snapshot.getRawOperatorState());
-//		out.close();
-//		harness.close();
+		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
+				new KeyedOneInputStreamOperatorTestHarness<>(
+						new KeyedCEPPatternOperator<>(
+								Event.createTypeSerializer(),
+								false,
+								keySelector,
+								IntSerializer.INSTANCE,
+								new SinglePatternNFAFactory(),
+								null,
+								true),
+						keySelector,
+						BasicTypeInfo.INT_TYPE_INFO);
+
+		harness.setup();
+		harness.open();
+		harness.processWatermark(new Watermark(5));
+
+		// do snapshot and save to file
+		OperatorStateHandles snapshot = harness.snapshot(0L, 0L);
+		OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/cep-migration-single-pattern-afterwards-flink1.2-snapshot");
+
+		harness.close();
+	}
+
+
+	@Test
+	public void testSinglePatternAfterMigration() throws Exception {
+
+		KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
+			private static final long serialVersionUID = -4873366487571254798L;
+
+			@Override
+			public Integer getKey(Event value) throws Exception {
+				return value.getId();
+			}
+		};
+
+		final Event startEvent1 = new Event(42, "start", 1.0);
 
 		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness =
-			new KeyedOneInputStreamOperatorTestHarness<>(
-				new KeyedCEPPatternOperator<>(
-					Event.createTypeSerializer(),
-					false,
-					keySelector,
-					IntSerializer.INSTANCE,
-					new SinglePatternNFAFactory(),
-					null,
-					true),
-				keySelector,
-				BasicTypeInfo.INT_TYPE_INFO);
+				new KeyedOneInputStreamOperatorTestHarness<>(
+						new KeyedCEPPatternOperator<>(
+								Event.createTypeSerializer(),
+								false,
+								keySelector,
+								IntSerializer.INSTANCE,
+								new SinglePatternNFAFactory(),
+								null,
+								true),
+						keySelector,
+						BasicTypeInfo.INT_TYPE_INFO);
 
 		harness.setup();
-		final ObjectInputStream ois = new ObjectInputStream(new FileInputStream(getResourceFilename(
-			"cep-single-pattern-snapshot-1.2")));
-		final OperatorStateHandles snapshot = new OperatorStateHandles(
-			(int) ois.readObject(),
-			(StreamStateHandle) ois.readObject(),
-			(Collection<KeyedStateHandle>) ois.readObject(),
-			(Collection<KeyedStateHandle>) ois.readObject(),
-			(Collection<OperatorStateHandle>) ois.readObject(),
-			(Collection<OperatorStateHandle>) ois.readObject()
-		);
-		harness.initializeState(snapshot);
+		harness.initializeState(
+				OperatorSnapshotUtil.readStateHandle(
+						OperatorSnapshotUtil.getResourceFilename("cep-migration-single-pattern-afterwards-flink1.2-snapshot")));
 		harness.open();
 
 		harness.processElement(new StreamRecord<>(startEvent1, 5));
@@ -414,7 +426,7 @@ private SinglePatternNFAFactory(boolean handleTimeout) {
 		public NFA<Event> createNFA() {
 
 			Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new StartFilter())
-				.within(Time.milliseconds(10L));
+					.within(Time.milliseconds(10L));
 
 			return NFACompiler.compile(pattern, Event.createTypeSerializer(), handleTimeout);
 		}
@@ -438,14 +450,14 @@ private NFAFactory(boolean handleTimeout) {
 		public NFA<Event> createNFA() {
 
 			Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new StartFilter())
-				.followedBy("middle")
-				.subtype(SubEvent.class)
-				.where(new MiddleFilter())
-				.followedBy("end")
-				.where(new EndFilter())
-				// add a window timeout to test whether timestamps of elements in the
-				// priority queue in CEP operator are correctly checkpointed/restored
-				.within(Time.milliseconds(10L));
+					.followedBy("middle")
+					.subtype(SubEvent.class)
+					.where(new MiddleFilter())
+					.followedBy("end")
+					.where(new EndFilter())
+					// add a window timeout to test whether timestamps of elements in the
+					// priority queue in CEP operator are correctly checkpointed/restored
+					.within(Time.milliseconds(10L));
 
 			return NFACompiler.compile(pattern, Event.createTypeSerializer(), handleTimeout);
 		}
diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink1.2-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..6775f2aac537b7fe5da2c620fb4faa03de894a49
GIT binary patch
literal 5580
zcmd^DTWB0r7(Tn3rfE}~q+-3dYS&wEXS2y9*|lhsrisREHnoWgt%zr5&SpnvXT~#U
zvl|~OMZ^bRd}*PmprC@{iv_PJC`F6-AS$SMK`bKP6;WRl^#9My>?J!*QmrDm53@US
z{`3FefBViqO9&wm63?p{nx?}n(Lcb{gxa5{iF7)hO7!IihBNxGni}j^L9#<i6Vm&K
z|H1nhT<{a|zNEe(tuUA$)P@p$sy>iVwX~L?X^kedOln|oK+O!Rg+UN+^S3P3(J>8}
z`n2I;eJGPoWD2y9P}Sl7L?%55x2b_le=4J@n%YOm8S8x0X#TUOz9VZ~PSQ4?SExg^
zB2x;cVciZlhN9V}l5Hts(P3p_n68qiE>k8gG0V6?P2*0+13Xbes+{aR2{m(|wXSE@
zu;xuBY-{m}d!Ia}RK`7p>gp6W>uSX05Ksh>?zRea#$lWao6ia=n0!jer+xs2S`6{B
zC5#eNT$EOBv<*E+OU#8BHNb8(_1|>gE3><NH!p2*Q)7+-u~Bjpqw#t=n*BNjSwI+@
z9d6j%5R2EBS((W!U`YR0nP_{<&%_O(nXvs5RYvo!;8YV+Y?0|><GXiLAsDxU5{~a~
z3T3Swe?RvA#%H!a`*M^-v!sWati&3-Rgj@9*iCw!o|K&qH%8Ly^kl8mlg%BqJxkUW
z0FBKT1(|DA0Y#JypeIiX`h=%-JCMiUuQDLAYg)nzab&av^rF-|8lP~c*<4p<<?{H_
zNYxu4k|i54F7@WPgX|}fDqdq?aYu_qq|V#u0&O79!1V2W(N?D5QrG5NibY9#mYk)t
z0xg>j1UqI@*JZ9CXJ=7o5{WpO1%_t&wbU(6QwNf>sb!ms@H(=ji?eydh0<X<ru2k*
zW|;sz@g>qX8KPK6>O=80Z&_A0u-}HE6QsujZ&GVMDb;MR#qUfRDs)R}a!Y+BcXPM-
z%5PB;*-JX9rZLx@;X30`Dn0c*eh}8I{qWVJn{RzPMxqmBy=hb3tE@?~P82zFi?*p(
zogE`yv@3WncjNCWkw`a$u5c0x4r29z`TpL6pY(3=PN^kuVEx5i_Z&R*{BtTC^;M^`
z0j&iVs>>L}(1t=;;N+Z^Iz#FnUw{TjNBmFKfs(u>R5WPNljIiJAA4l8_|4j5y!-9L
zxuG6(-tp|>EFs4ky!84McU~wA|FF5d^R-7)2QIumM%wq1(~8v6O*RMTtz8zAqz4XN
zE~IZ`Hk`wZUq{wg3`@5wD3c}US+=#S?3ji|1+;cETM04{PEomKHbM1oaElcBWM{ch
zU|f((!<7L3<ZOc=Y;2b^0`yi_LyuxddEvnc1V8H`HL1{;EC=jR*XR54kH7vP^g#;~
z{bfEsRdo9wCcpLn82N$bRgfP@El+-Hp&UZ_1vvd*CBKaUH8s)V3Ba8S@(VYJ<QKYf
za^!b$Xon%|G<NM>A9qIcw|%oUhF#UQKus7(Yf}^=X|Fo+psV03R}@mU<-~gN4dLdZ
z(0}^%tHCIVLE<f#YX#ZVaO%OF0Yn94(Jjehj#JAugkf7i8=D(wW6U;9rr`p}uJdH&
z5!$;99U*}DEMAcbfp!GE)ggrf?$hgM<tXQLF$?2B^}svlA3jDO0`DhDn+q>;lBuFK
zXD?gc**f}{JHC71iXUd8((QH_<^Z@LeJfzC?;owXs@aw<1F{mS70Q;}<R&K!6OM)n
z{~qGi2|mv=AtNEk1uLL{IPK>|S*@!cRLSIU*(6_hH_G7RSfh+v<x=A!<e+VCbZQ(@
z5N^WpMtWt`(wiMQ{=%I5k&-?thxj&=6$Dn|#aXh38}mi)MxOKiq0j@pv*kPvU(LGT
zJFw@=>t0I!B2SaGt|x-(H&h+yv-3kWf~L?pV>DE{HfO+V%LKPess3qW9$vhmGUi|a
zjZYgoob?{88+X2W_tE=4g~JU-2{{;@D@c1uO32#_vHSe8M3~#~(hD)Y`Y+iY4`NNf
ze?tG`r~5x|G8|);Ub9N1RZ@i#S)m*1o!6Lu1bR92;e=sPQ)=C|AD>+Y4o+T^o7poi
z2}Z5Tr&qz#oXtytC`yh>fvs{<9;;jg$|K~a$Xo<F`{x}$2FBz!&RsFwq*e~WPe?Bb
z4UnJ8pFvH*wx15PjKrI?;lvVIor)tP{>~*T@J}uY3YYQEFUVOu6cSSNE>>ZSxpFDb
z_zjP|@%p}pKTClN`$(r=Ul1g^Z?fu?Iet&Sc<Ut#Umtl)=Fc}3k>?j0^@jS1G6rN4
zZashChm^oOf=g50EPi^^fZm2HTZ3Bcj_2V8%CdmFC9A}1^%0Tv5%DsMd$1};LOoom
zBS143UPH`O4jljD@ZfRjmqb#lk2O*ZpQ(7cvMK>cxT^FE-HJ*)b<cw|;2Non$S+9D
zDwEq%AJTfwSnQM<YpEpZe$7|}Z>K8f)?H}n1(e8DI)eXNoyxl4C%or@&t65!vg}1Q
Fe*!A5wq*bS

literal 0
HcmV?d00001

diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink1.2-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..f63b7ddbe49e81332135cb9a81cb3a132a798ada
GIT binary patch
literal 2326
zcmd^AON-M`6h7^YIzG^;3ttOY1A>sGou*BPS&Xk473(-!A0Q&$B&X9jX%cU4XIdA!
z5`TfN-70Q<AP9;Jmm)3{LBzfO0tG!cNvG|!ii0i$15KLb-tT<pJLfwA07$`DzQ9Z)
zn?)s?H!Vd++fquHHx)C_47BrBAvd0*mBaLd@5v!|4#@{}q~c42RWb@j);5dEL~+7Y
z^l_#urF_X$CdLc7YPFa%3c5whBP_Xf1;)s6;$!5^YKiG4?X(D^Xl9jC$*`1M(a2g^
zTc0Qt41keyNoX1V)BP{7Cgf1^`Mk;kX4O!wx{kL%H;!ug^}6q=q7`66IBuw#EJXFF
zCo0Gt<~moA6NWreX@Dk&1FL{JO{g85isQ|@D15JV_vXEq>e6%!sQ(^dFIG$`<`e({
zk<z1Hm7NHXGvV`!U;?A303Q<pihY52!xK&&)ljZh&-#u%%jy`CMqRYKL;aiS%j5yB
zCu#2KOVh$YC2iE%nf+s(a`fi)7pfp>Yzw&Kb4RpJHn4$F6{Mv9RV9XBC6zccuo8aK
zqUwG#6r5S&7`D(pI6XJVgh1}KTX<})N6KY8etY?L^MlEUPt%YtLk3-}W4Buc6w4G>
zi=D1TPNy43i=D15J6-E_)MOdfRf&zyTLMn^Lr@cGZR*;psL#ZxlQca3yvt~i9qyBr
zbVOFC@uGGdEuV0-d~U0i==JnUr8)*km0`2grDKjOa1ByTc^;yOQ+*NT=)BApS=Zof
zhhE?<Up+>r4q6gvC=F}Nu*1eGYq(tl_Mpqc5JLew%Mw$QgIFumghu-a3v0(&K*ibG
zw=I<PM#`{-<Gd46>!2-5`cs?=3c?v%vHF}NY8|Etl>~YBFW0u?4@>C;WMc9z^X9b(
zW^!4-?U7V6Ewv{t{n_k=QTN%`G^EbMMrK(Uh9|g<oLVK**^`K{X5IVeFScEJJp}0)
z*x>rijyvlptQR$oVa<2#W-zrcR(olLuA}mQ6C$RY2EO4ikQ{0C&V76S)`z{j<FC{T
zI`sa@p&PeuKYpasXSx|IZIaR25t_dUD5V`pWs$?KzCM$qdwP)!E)z+PW<V`@ap2On
eL036jB7dBU%#xTj<GlHNXSSG;$p`JgDfj`qUEk&a

literal 0
HcmV?d00001

diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink1.2-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink1.2-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..8e0fd27703dfa2b56a21e617abdbc7d95863c0f6
GIT binary patch
literal 5389
zcmd^DU5pb|6uxbjWnmF^!Qd~TQUo`4+U|BsSE8^iySUKp0$m{iW4P_yZbzpx<DJ`W
z*9QnO#s^<~fyAhZCTfE5MHBysBp4#Z_+ZpTj6Wd|V*J+-^+gjs=gv%j=&%cl#>75M
zJ2U5=^PRu%o?Ai)36OA{CSy4@)f>qtdXtevDjAOq6k^E;O~w-}m5ZhNb8(RDkkW+o
z{NaA^J_Hy11e`Caub1^DSt2<Qi5GhNB8g-o6&YAd`yvI(Xn!G|N)=)%h_|^*F4WO6
z2AE=rfjmtX@{!ouJ{C!=rGP1xU=g*yuP+rJhz%svwS-)~#WjuQ&%E?KnPYR3GWn!J
zEt)SfrJ!rZ18}1$d9ze94Mj{_tSmI$R&vy4%BUf-jB8Za9%9_X6C|X{$;Qi2GXYxb
zda{}^sWV|3(}y2@@uX52b`&NqO<}XHMobm~MG)yOqd><k#;GuQR#3s@Q$kL=0Z6tO
z;$=f<C8pRYt?V>4HA73xh8Q)#ZZ!4Zbl)kn{jQs<TinbuOM%!ZnUTS8Jsr({U4bkh
zjHMRWOs<LP?PXSGG7A{e|5Ya1-gh%`k8dVSw?vh}oGm!bi>+*$sYAotwoxG%H@p%K
zZ)*x=tsQ?qd1~RS>s~t=B*8T4W;!de#%>iPnFhO2r_-ad)8R&oI-MS^b$Yb9qt>O#
z+ybC6`LrMtttz01q8{|<WkDZtw5|j4`1@4`M7FsltPn>=OF%D5y`%97XL*yWN>;ue
zzA#et1_-3dLX1njIUXeYNT7<>5LjH_ViBqHemX@Ph|@Fu0G~FMt#GMpb1j8}q&-cp
zRat?S^#+0+(y47TTafG0s56R098Cj5Gxct27ssdt$ywa8%tm+}X|jm3NzI1RVJfEd
zqI$ASfS&LL(z`TKtRr=yxSE&GDC^m8!_W!R?SR**F&UL=*45&7jSLmKr8K#fIF`A$
zOF#BokOX#<PMXg%+aBjC<4`Ky^)+q~=FI){?c+=Leh?zT5i(ylsp?eLD48dUoY_TF
zSF6_g4NkNxxG#6%?<$c<7x=#6#1|aI>J$Cc?t@?SEO&OP3*f-}Bby#Mc<7DS6R_1+
zt;zzl=2@uDU=Ty=3uTIv>s#s!se5<|8XO&QKUE7#@;+ZtuR$-9TV#Likj>&YYq$2%
zch6*!-RQjK*u_~wjx%`aaVhRbUl{IYb6e*-&&3biygfwPcay7%)KGOc0sF047Neva
zHeD{HZ$l>R!;Ehs^DCO6niZ5ulN$}w*i^Q3El&lsb~9TE(&tZ6nHd&A^)IlC6#8Ui
zxlmwSkk$T50Dm;wAP5WF<%|HmwWy&-v7?;u-~@u7`H-5pZ%k$ac4*PLJ-HW7JOzEw
z!bE?W&#x5S{)fqL{y#>3p!pKW52R)$zqwEjzWf55{;!hXLXVo7Xz?Q8t_1n{8$|N+
zeRFx_x5~G}kaZfncK_#{!QB1dt_WdQEt;Y_45YOw3X!x|ExFNE@W>T}R4u=_UOXY(
zTNLW~U%&1Tk`N@`fVozX#SOb2%o#vbKo(t+EG9TLY)xpU0kpBSfi{LrU1xcG0kZQP
zS-FMw%s@v7Ae_Z3G9l28fKwe(DBxaF-zx_>r_*T|2dW1?T=v9C`ZRbyO4@8V$VsLO
z)|_24v$J*dM?1cM^7bFcgVODG80G-DAhB7n*4Gc#T+N$?Dg!bbsTInGTx3T_G#$2v
z5%(VA)d@aNG9e=&$W61LfH>{rM7dN~9jKDY;j&mBco)jx;#i{$8|6~tOUOan#NgIp
zNP)iz%NgmFK|^hJ<m{Uh_Ge1!xE$i!bXE{piL26N4%a4&&W+sXdwrpMdb8ym4qwf>
zKR&SIt1WLuf04V%T-y;r^$V&N^w~0BjiAYQ&KUHSuFV;6Y#HHZDc(D#O~Sz&Dq{i$
z(D0b1!d~yd+PU$)hmSvY8a6i=CFEdqgCOlCDIsrf#_n^=5@D{yOAo~KuD@h^*o!s&
z;jsG0`Nz*S84fW+tyv|~Dyc$=%+d|@&TGs+0=*6Ta6~hxF14=QhtJl4gQJ@><2!~W
z!6d5k>7DR2WAc(Gijsp;V1=BNhbp&#@&MTtm<V8J|GfSu&zStixhsf^5|ykJ3=rP{
zxvBga)D%qfWlzh7aFaHiSfoaLivL_ez{9@!kXJXc3Y*T9OF72xdG_5Cd!9KHhw|D(
zI@S6ML4tcmt5%ug_t=rWw@!WY%JVY#uBm|Be9)*fk&l$on^Cy+zz}D0UeYv9x1rPu
zuX4J=IXLbZ1~9GUfN(9F0h!H!lLuUbHMGIkrP=BiG;`sU?|9|F*)N~yKPw%PNNU-f
zBgODp0xx$|C4i88AYDaIgAz~ORs3gAh5%nyHsBRLfdD-8g*r<%C8C<K0N!q`+~6oh
etu4ib5}8Ve_h+S5ndkk4gAy!o(czCCRP!ey9Fmj(

literal 0
HcmV?d00001


From 1b5b56c0be70fd01d4ecbc95690cb97d2881f5cf Mon Sep 17 00:00:00 2001
From: WangTaoTheTonic <wangtao111@huawei.com>
Date: Mon, 17 Apr 2017 14:55:27 +0800
Subject: [PATCH 128/253] [FLINK-6312] [build] Update curator version to 2.12.0

The updated curator version includes a bugfix for a potential block
---
 flink-contrib/flink-storm-examples/pom.xml                | 4 ++++
 .../checkpoint/ZooKeeperCompletedCheckpointStoreTest.java | 8 ++++----
 pom.xml                                                   | 2 +-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml
index b9da2141dfeb1..d042ddea25a73 100644
--- a/flink-contrib/flink-storm-examples/pom.xml
+++ b/flink-contrib/flink-storm-examples/pom.xml
@@ -84,6 +84,10 @@ under the License.
 					<groupId>org.apache.storm</groupId>
 					<artifactId>storm-core</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.apache.curator</groupId>
+					<artifactId>curator-framework</artifactId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
index 8fc0f02a86f3f..0d22dc6b4863b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java
@@ -22,7 +22,7 @@
 import org.apache.curator.framework.api.BackgroundCallback;
 import org.apache.curator.framework.api.CuratorEvent;
 import org.apache.curator.framework.api.CuratorEventType;
-import org.apache.curator.framework.api.Pathable;
+import org.apache.curator.framework.api.ErrorListenerPathable;
 import org.apache.curator.utils.EnsurePath;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.concurrent.Executors;
@@ -128,12 +128,12 @@ public void testCheckpointRecovery() throws Exception {
 				.delete()
 				.deletingChildrenIfNeeded()
 				.inBackground(any(BackgroundCallback.class), any(Executor.class))
-		).thenAnswer(new Answer<Pathable<Void>>() {
+		).thenAnswer(new Answer<ErrorListenerPathable<Void>>() {
 			@Override
-			public Pathable<Void> answer(InvocationOnMock invocation) throws Throwable {
+			public ErrorListenerPathable<Void> answer(InvocationOnMock invocation) throws Throwable {
 				final BackgroundCallback callback = (BackgroundCallback) invocation.getArguments()[0];
 
-				Pathable<Void> result = mock(Pathable.class);
+				ErrorListenerPathable<Void> result = mock(ErrorListenerPathable.class);
 
 				when(result.forPath(anyString())).thenAnswer(new Answer<Void>() {
 					@Override
diff --git a/pom.xml b/pom.xml
index f822198b903a2..71ad7792ae1cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@ under the License.
 		<chill.version>0.7.4</chill.version>
 		<asm.version>5.0.4</asm.version>
 		<zookeeper.version>3.4.6</zookeeper.version>
-		<curator.version>2.8.0</curator.version>
+		<curator.version>2.12.0</curator.version>
 		<jackson.version>2.7.4</jackson.version>
 		<metrics.version>3.1.0</metrics.version>
 		<junit.version>4.12</junit.version>

From d6104260eed5beb2fa3064be05ee2563543c4c7e Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 29 Mar 2017 22:49:54 +0200
Subject: [PATCH 129/253] [FLINK-6340] [flip-1] Add a termination future to the
 Execution

---
 .../runtime/executiongraph/Execution.java     |  26 ++-
 .../executiongraph/ExecutionGraph.java        | 182 +++++++++------
 .../executiongraph/ExecutionJobVertex.java    | 116 +++-------
 .../executiongraph/ExecutionVertex.java       |  56 +++--
 .../ExecutionGraphRestartTest.java            |  78 ++++---
 .../ExecutionGraphTestUtils.java              |   8 +-
 .../ExecutionStateProgressTest.java           |  94 --------
 .../TerminalStateDeadlockTest.java            | 216 ------------------
 8 files changed, 250 insertions(+), 526 deletions(-)
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 729e161d86eda..2680849dcec9c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -121,6 +121,9 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 	private final ConcurrentLinkedQueue<PartialInputChannelDeploymentDescriptor> partialInputChannelDeploymentDescriptors;
 
+	/** A future that completes once the Execution reaches a terminal ExecutionState */
+	private final FlinkCompletableFuture<ExecutionState> terminationFuture;
+
 	private volatile ExecutionState state = CREATED;
 
 	private volatile SimpleSlot assignedResource;     // once assigned, never changes until the execution is archived
@@ -161,6 +164,7 @@ public Execution(
 		markTimestamp(ExecutionState.CREATED, startTimestamp);
 
 		this.partialInputChannelDeploymentDescriptors = new ConcurrentLinkedQueue<>();
+		this.terminationFuture = new FlinkCompletableFuture<>();
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -234,6 +238,16 @@ public void setInitialState(TaskStateHandles checkpointStateHandles) {
 		this.taskState = checkpointStateHandles;
 	}
 
+	/**
+	 * Gets a future that completes once the task execution reaches a terminal state.
+	 * The future will be completed with specific state that the execution reached.
+	 *
+	 * @return A future for the execution's termination
+	 */
+	public Future<ExecutionState> getTerminationFuture() {
+		return terminationFuture;
+	}
+
 	// --------------------------------------------------------------------------------------------
 	//  Actions
 	// --------------------------------------------------------------------------------------------
@@ -473,7 +487,8 @@ else if (current == CREATED || current == SCHEDULED) {
 						}
 					}
 					finally {
-						vertex.executionCanceled();
+						vertex.executionCanceled(this);
+						terminationFuture.complete(CANCELED);
 					}
 					return;
 				}
@@ -741,7 +756,8 @@ void markFinished(Map<String, Accumulator<?, ?>> userAccumulators, IOMetrics met
 						vertex.getExecutionGraph().deregisterExecution(this);
 					}
 					finally {
-						vertex.executionFinished();
+						vertex.executionFinished(this);
+						terminationFuture.complete(FINISHED);
 					}
 					return;
 				}
@@ -793,7 +809,8 @@ else if (current == CANCELING || current == RUNNING || current == DEPLOYING) {
 						vertex.getExecutionGraph().deregisterExecution(this);
 					}
 					finally {
-						vertex.executionCanceled();
+						vertex.executionCanceled(this);
+						terminationFuture.complete(CANCELED);
 					}
 					return;
 				}
@@ -886,7 +903,8 @@ private boolean processFail(Throwable t, boolean isCallback, Map<String, Accumul
 					vertex.getExecutionGraph().deregisterExecution(this);
 				}
 				finally {
-					vertex.executionFailed(t);
+					vertex.executionFailed(this, t);
+					terminationFuture.complete(FAILED);
 				}
 
 				if (!isCallback && (current == RUNNING || current == DEPLOYING)) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 23ed99d26ee72..fff1ea26a4a1f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -40,6 +40,7 @@
 import org.apache.flink.runtime.checkpoint.CheckpointStatsTracker;
 import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
 import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
 import org.apache.flink.runtime.concurrent.BiFunction;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.concurrent.FutureUtils;
@@ -89,6 +90,7 @@
 import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 
 import static org.apache.flink.util.Preconditions.checkArgument;
@@ -188,6 +190,8 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	/** Registered KvState instances reported by the TaskManagers. */
 	private final KvStateLocationRegistry kvStateLocationRegistry;
 
+	private int numVerticesTotal;
+
 	// ------ Configuration of the Execution -------
 
 	/** Flag to indicate whether the scheduler may queue tasks for execution, or needs to be able
@@ -203,6 +207,8 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 
 	// ------ Execution status and progress. These values are volatile, and accessed under the lock -------
 
+	private final AtomicInteger verticesFinished;
+
 	/** Current status of the job execution */
 	private volatile JobStatus state = JobStatus.CREATED;
 
@@ -210,9 +216,6 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	 * that was not recoverable and triggered job failure */
 	private volatile Throwable failureCause;
 
-	/** The number of job vertices that have reached a terminal state */
-	private volatile int numFinishedJobVertices;
-
 	// ------ Fields that are relevant to the execution and need to be cleared before archiving  -------
 
 	/** The coordinator for checkpoints, if snapshot checkpoints are enabled */
@@ -317,6 +320,8 @@ public ExecutionGraph(
 
 		this.restartStrategy = restartStrategy;
 		this.kvStateLocationRegistry = new KvStateLocationRegistry(jobId, getAllVertices());
+
+		this.verticesFinished = new AtomicInteger();
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -454,7 +459,7 @@ private ExecutionVertex[] collectExecutionVertices(List<ExecutionJobVertex> jobV
 			return jv.getTaskVertices();
 		}
 		else {
-			ArrayList<ExecutionVertex> all = new ArrayList<ExecutionVertex>();
+			ArrayList<ExecutionVertex> all = new ArrayList<>();
 			for (ExecutionJobVertex jv : jobVertices) {
 				if (jv.getGraph() != this) {
 					throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph");
@@ -586,6 +591,10 @@ public void remove() {
 		};
 	}
 
+	public int getTotalNumberOfVertices() {
+		return numVerticesTotal;
+	}
+
 	public Map<IntermediateDataSetID, IntermediateResult> getAllIntermediateResults() {
 		return Collections.unmodifiableMap(this.intermediateResults);
 	}
@@ -620,7 +629,7 @@ public Executor getFutureExecutor() {
 	 */
 	public Map<String, Accumulator<?,?>> aggregateUserAccumulators() {
 
-		Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<String, Accumulator<?, ?>>();
+		Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<>();
 
 		for (ExecutionVertex vertex : getAllExecutionVertices()) {
 			Map<String, Accumulator<?, ?>> next = vertex.getCurrentExecutionAttempt().getUserAccumulators();
@@ -657,7 +666,7 @@ public Map<String, SerializedValue<Object>> getAccumulatorsSerialized() throws I
 
 		Map<String, Accumulator<?, ?>> accumulatorMap = aggregateUserAccumulators();
 
-		Map<String, SerializedValue<Object>> result = new HashMap<String, SerializedValue<Object>>();
+		Map<String, SerializedValue<Object>> result = new HashMap<>();
 		for (Map.Entry<String, Accumulator<?, ?>> entry : accumulatorMap.entrySet()) {
 			result.put(entry.getKey(), new SerializedValue<Object>(entry.getValue().getLocalValue()));
 		}
@@ -713,6 +722,7 @@ public void attachJobGraph(List<JobVertex> topologiallySorted) throws JobExcepti
 			}
 
 			this.verticesInCreationOrder.add(ejv);
+			this.numVerticesTotal += ejv.getParallelism();
 		}
 	}
 
@@ -878,9 +888,23 @@ public void cancel() {
 
 			if (current == JobStatus.RUNNING || current == JobStatus.CREATED) {
 				if (transitionState(current, JobStatus.CANCELLING)) {
+
+					final ArrayList<Future<?>> futures = new ArrayList<>(verticesInCreationOrder.size());
+
+					// cancel all tasks (that still need cancelling)
 					for (ExecutionJobVertex ejv : verticesInCreationOrder) {
-						ejv.cancel();
+						futures.add(ejv.cancelWithFuture());
 					}
+
+					// we build a future that is complete once all vertices have reached a terminal state
+					final ConjunctFuture allTerminal = FutureUtils.combineAll(futures);
+					allTerminal.thenAccept(new AcceptFunction<Void>() {
+						@Override
+						public void accept(Void value) {
+							allVerticesInTerminalState();
+						}
+					});
+
 					return;
 				}
 			}
@@ -968,26 +992,33 @@ public void fail(Throwable t) {
 				current == JobStatus.SUSPENDED ||
 				current.isGloballyTerminalState()) {
 				return;
-			} else if (current == JobStatus.RESTARTING) {
+			}
+			else if (current == JobStatus.RESTARTING) {
 				this.failureCause = t;
 
 				if (tryRestartOrFail()) {
 					return;
 				}
-				// concurrent job status change, let's check again
-			} else if (transitionState(current, JobStatus.FAILING, t)) {
+			}
+			else if (transitionState(current, JobStatus.FAILING, t)) {
 				this.failureCause = t;
 
-				if (!verticesInCreationOrder.isEmpty()) {
-					// cancel all. what is failed will not cancel but stay failed
-					for (ExecutionJobVertex ejv : verticesInCreationOrder) {
-						ejv.cancel();
-					}
-				} else {
-					// set the state of the job to failed
-					transitionState(JobStatus.FAILING, JobStatus.FAILED, t);
+				// we build a future that is complete once all vertices have reached a terminal state
+				final ArrayList<Future<?>> futures = new ArrayList<>(verticesInCreationOrder.size());
+
+				// cancel all tasks (that still need cancelling)
+				for (ExecutionJobVertex ejv : verticesInCreationOrder) {
+					futures.add(ejv.cancelWithFuture());
 				}
 
+				final ConjunctFuture allTerminal = FutureUtils.combineAll(futures);
+				allTerminal.thenAccept(new AcceptFunction<Void>() {
+					@Override
+					public void accept(Void value) {
+						allVerticesInTerminalState();
+					}
+				});
+
 				return;
 			}
 
@@ -1039,7 +1070,7 @@ public void restart() {
 						stateTimestamps[i] = 0;
 					}
 				}
-				numFinishedJobVertices = 0;
+
 				transitionState(JobStatus.RESTARTING, JobStatus.CREATED);
 
 				// if we have checkpointed state, reload it into the executions
@@ -1097,9 +1128,24 @@ public ArchivedExecutionConfig getArchivedExecutionConfig() {
 	 * For testing: This waits until the job execution has finished.
 	 */
 	public void waitUntilFinished() throws InterruptedException {
-		synchronized (progressLock) {
-			while (!state.isTerminalState()) {
-				progressLock.wait();
+		// we may need multiple attempts in the presence of failures / recovery
+		while (true) {
+			for (ExecutionJobVertex ejv : verticesInCreationOrder) {
+				for (ExecutionVertex ev : ejv.getTaskVertices()) {
+					try {
+						ev.getCurrentExecutionAttempt().getTerminationFuture().get();
+					}
+					catch (ExecutionException e) {
+						// this should never happen
+						throw new RuntimeException(e);
+					}
+				}
+			}
+
+			// now that all vertices have been (at some point) in a terminal state,
+			// we need to check if the job as a whole has entered a final state
+			if (state.isTerminalState()) {
+				return;
 			}
 		}
 	}
@@ -1129,59 +1175,57 @@ private boolean transitionState(JobStatus current, JobStatus newState, Throwable
 		}
 	}
 
-	void jobVertexInFinalState() {
-		synchronized (progressLock) {
-			if (numFinishedJobVertices >= verticesInCreationOrder.size()) {
-				throw new IllegalStateException("All vertices are already finished, cannot transition vertex to finished.");
-			}
-
-			numFinishedJobVertices++;
+	void vertexFinished() {
+		int numFinished = verticesFinished.incrementAndGet();
+		if (numFinished == numVerticesTotal) {
+			// done :-)
+			allVerticesInTerminalState();
+		}
+	}
 
-			if (numFinishedJobVertices == verticesInCreationOrder.size()) {
+	void vertexUnFinished() {
+		verticesFinished.getAndDecrement();
+	}
 
-				// we are done, transition to the final state
-				JobStatus current;
-				while (true) {
-					current = this.state;
+	private void allVerticesInTerminalState() {
+		// we are done, transition to the final state
+		JobStatus current;
+		while (true) {
+			current = this.state;
 
-					if (current == JobStatus.RUNNING) {
-						if (transitionState(current, JobStatus.FINISHED)) {
-							postRunCleanup();
-							break;
-						}
-					}
-					else if (current == JobStatus.CANCELLING) {
-						if (transitionState(current, JobStatus.CANCELED)) {
-							postRunCleanup();
-							break;
-						}
-					}
-					else if (current == JobStatus.FAILING) {
-						if (tryRestartOrFail()) {
-							break;
-						}
-						// concurrent job status change, let's check again
-					}
-					else if (current == JobStatus.SUSPENDED) {
-						// we've already cleaned up when entering the SUSPENDED state
-						break;
-					}
-					else if (current.isGloballyTerminalState()) {
-						LOG.warn("Job has entered globally terminal state without waiting for all " +
-							"job vertices to reach final state.");
-						break;
-					}
-					else {
-						fail(new Exception("ExecutionGraph went into final state from state " + current));
-						break;
-					}
+			if (current == JobStatus.RUNNING) {
+				if (transitionState(current, JobStatus.FINISHED)) {
+					postRunCleanup();
+					break;
 				}
-				// done transitioning the state
-
-				// also, notify waiters
-				progressLock.notifyAll();
+			}
+			else if (current == JobStatus.CANCELLING) {
+				if (transitionState(current, JobStatus.CANCELED)) {
+					postRunCleanup();
+					break;
+				}
+			}
+			else if (current == JobStatus.FAILING) {
+				if (tryRestartOrFail()) {
+					break;
+				}
+				// concurrent job status change, let's check again
+			}
+			else if (current == JobStatus.SUSPENDED) {
+				// we've already cleaned up when entering the SUSPENDED state
+				break;
+			}
+			else if (current.isGloballyTerminalState()) {
+				LOG.warn("Job has entered globally terminal state without waiting for all " +
+						"job vertices to reach final state.");
+				break;
+			}
+			else {
+				fail(new Exception("ExecutionGraph went into final state from state " + current));
+				break;
 			}
 		}
+		// done transitioning the state
 	}
 
 	/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index 2e5de64ca9298..3197e656b4500 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -32,6 +32,7 @@
 import org.apache.flink.runtime.JobException;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.FutureUtils;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.instance.SimpleSlot;
 import org.apache.flink.runtime.instance.SlotProvider;
@@ -65,9 +66,9 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 	public static final int VALUE_NOT_SET = -1;
 
 	private final Object stateMonitor = new Object();
-	
+
 	private final ExecutionGraph graph;
-	
+
 	private final JobVertex jobVertex;
 
 	/**
@@ -91,12 +92,10 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 	private final ExecutionVertex[] taskVertices;
 
 	private final IntermediateResult[] producedDataSets;
-	
+
 	private final List<IntermediateResult> inputs;
-	
-	private final int parallelism;
 
-	private final boolean[] finishedSubtasks;
+	private final int parallelism;
 
 	private final SlotSharingGroup slotSharingGroup;
 
@@ -108,8 +107,6 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 
 	private int maxParallelism;
 
-	private volatile int numSubtasksInFinalState;
-
 	/**
 	 * Serialized task information which is for all sub tasks the same. Thus, it avoids to
 	 * serialize the same information multiple times in order to create the
@@ -231,8 +228,6 @@ public ExecutionJobVertex(
 		catch (Throwable t) {
 			throw new JobException("Creating the input splits caused an error: " + t.getMessage(), t);
 		}
-		
-		finishedSubtasks = new boolean[parallelism];
 	}
 
 	/**
@@ -360,10 +355,6 @@ public SerializedValue<TaskInformation> getSerializedTaskInformation() throws IO
 		return serializedTaskInformation;
 	}
 
-	public boolean isInFinalState() {
-		return numSubtasksInFinalState == parallelism;
-	}
-
 	@Override
 	public ExecutionState getAggregateState() {
 		int[] num = new int[ExecutionState.values().length];
@@ -484,51 +475,51 @@ public ExecutionAndSlot[] allocateResourcesForAll(SlotProvider resourceProvider,
 		return slots;
 	}
 
+	/**
+	 * Cancels all currently running vertex executions.
+	 */
 	public void cancel() {
 		for (ExecutionVertex ev : getTaskVertices()) {
 			ev.cancel();
 		}
 	}
-	
-	public void fail(Throwable t) {
+
+	/**
+	 * Cancels all currently running vertex executions.
+	 * 
+	 * @return A future that is complete once all tasks have canceled.
+	 */
+	public Future<Void> cancelWithFuture() {
+		// we collect all futures from the task cancellations
+		ArrayList<Future<?>> futures = new ArrayList<>(parallelism);
+
+		// cancel each vertex
 		for (ExecutionVertex ev : getTaskVertices()) {
-			ev.fail(t);
+			futures.add(ev.cancel());
 		}
+
+		// return a conjunct future, which is complete once all individual tasks are canceled
+		return FutureUtils.combineAll(futures);
 	}
-	
-	public void waitForAllVerticesToReachFinishingState() throws InterruptedException {
-		synchronized (stateMonitor) {
-			while (numSubtasksInFinalState < parallelism) {
-				stateMonitor.wait();
-			}
+
+	public void fail(Throwable t) {
+		for (ExecutionVertex ev : getTaskVertices()) {
+			ev.fail(t);
 		}
 	}
-	
+
 	public void resetForNewExecution() {
-		if (!(numSubtasksInFinalState == 0 || numSubtasksInFinalState == parallelism)) {
-			throw new IllegalStateException("Cannot reset vertex that is not in final state");
-		}
-		
+
 		synchronized (stateMonitor) {
 			// check and reset the sharing groups with scheduler hints
 			if (slotSharingGroup != null) {
 				slotSharingGroup.clearTaskAssignment();
 			}
-			
-			// reset vertices one by one. if one reset fails, the "vertices in final state"
-			// fields will be consistent to handle triggered cancel calls
+
 			for (int i = 0; i < parallelism; i++) {
 				taskVertices[i].resetForNewExecution();
-				if (finishedSubtasks[i]) {
-					finishedSubtasks[i] = false;
-					numSubtasksInFinalState--;
-				}
-			}
-			
-			if (numSubtasksInFinalState != 0) {
-				throw new RuntimeException("Bug: resetting the execution job vertex failed.");
 			}
-			
+
 			// set up the input splits again
 			try {
 				if (this.inputSplits != null) {
@@ -548,51 +539,6 @@ public void resetForNewExecution() {
 			}
 		}
 	}
-	
-	//---------------------------------------------------------------------------------------------
-	//  Notifications
-	//---------------------------------------------------------------------------------------------
-	
-	void vertexFinished(int subtask) {
-		subtaskInFinalState(subtask);
-	}
-	
-	void vertexCancelled(int subtask) {
-		subtaskInFinalState(subtask);
-	}
-	
-	void vertexFailed(int subtask, Throwable error) {
-		subtaskInFinalState(subtask);
-	}
-	
-	private void subtaskInFinalState(int subtask) {
-		synchronized (stateMonitor) {
-			if (!finishedSubtasks[subtask]) {
-				finishedSubtasks[subtask] = true;
-				
-				if (numSubtasksInFinalState+1 == parallelism) {
-					
-					// call finalizeOnMaster hook
-					try {
-						getJobVertex().finalizeOnMaster(getGraph().getUserClassLoader());
-					}
-					catch (Throwable t) {
-						getGraph().fail(t);
-					}
-
-					numSubtasksInFinalState++;
-					
-					// we are in our final state
-					stateMonitor.notifyAll();
-					
-					// tell the graph
-					graph.jobVertexInFinalState();
-				} else {
-					numSubtasksInFinalState++;
-				}
-			}
-		}
-	}
 
 	// --------------------------------------------------------------------------------------------
 	//  Accumulators / Metrics
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index 3f6ce886e20b3..bcf7a7c4d909b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -22,6 +22,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.JobException;
+import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor;
 import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
 import org.apache.flink.runtime.deployment.PartialInputChannelDeploymentDescriptor;
@@ -60,8 +61,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.apache.flink.runtime.execution.ExecutionState.CANCELED;
-import static org.apache.flink.runtime.execution.ExecutionState.FAILED;
 import static org.apache.flink.runtime.execution.ExecutionState.FINISHED;
 
 /**
@@ -509,30 +508,41 @@ public Iterable<TaskManagerLocation> getPreferredLocationsBasedOnInputs() {
 	//   Actions
 	// --------------------------------------------------------------------------------------------
 
-	public void resetForNewExecution() {
+	public Execution resetForNewExecution() {
 
 		LOG.debug("Resetting execution vertex {} for new execution.", getTaskNameWithSubtaskIndex());
 
 		synchronized (priorExecutions) {
-			Execution execution = currentExecution;
-			ExecutionState state = execution.getState();
+			final Execution oldExecution = currentExecution;
+			final ExecutionState oldState = oldExecution.getState();
 
-			if (state == FINISHED || state == CANCELED || state == FAILED) {
-				priorExecutions.add(execution);
-				currentExecution = new Execution(
+			if (oldState.isTerminal()) {
+				priorExecutions.add(oldExecution);
+
+				final Execution newExecution = new Execution(
 					getExecutionGraph().getFutureExecutor(),
 					this,
-					execution.getAttemptNumber()+1,
+						oldExecution.getAttemptNumber()+1,
 					System.currentTimeMillis(),
 					timeout);
 
+				this.currentExecution = newExecution;
+
 				CoLocationGroup grp = jobVertex.getCoLocationGroup();
 				if (grp != null) {
 					this.locationConstraint = grp.getLocationConstraint(subTaskIndex);
 				}
+
+				// if the execution was 'FINISHED' before, tell the ExecutionGraph that
+				// we take one step back on the road to reaching global FINISHED
+				if (oldState == FINISHED) {
+					getExecutionGraph().vertexUnFinished();
+				}
+
+				return newExecution;
 			}
 			else {
-				throw new IllegalStateException("Cannot reset a vertex that is in state " + state);
+				throw new IllegalStateException("Cannot reset a vertex that is in non-terminal state " + oldState);
 			}
 		}
 	}
@@ -545,8 +555,16 @@ public void deployToSlot(SimpleSlot slot) throws JobException {
 		this.currentExecution.deployToSlot(slot);
 	}
 
-	public void cancel() {
-		this.currentExecution.cancel();
+	/**
+	 *  
+	 * @return A future that completes once the execution has reached its final state.
+	 */
+	public Future<ExecutionState> cancel() {
+		// to avoid any case of mixup in the presence of concurrent calls,
+		// we copy a reference to the stack to make sure both calls go to the same Execution 
+		final Execution exec = this.currentExecution;
+		exec.cancel();
+		return exec.getTerminationFuture();
 	}
 
 	public void stop() {
@@ -621,16 +639,18 @@ List<IntermediateResultPartition> finishAllBlockingPartitions() {
 	//   Notifications from the Execution Attempt
 	// --------------------------------------------------------------------------------------------
 
-	void executionFinished() {
-		jobVertex.vertexFinished(subTaskIndex);
+	void executionFinished(Execution execution) {
+		if (execution == currentExecution) {
+			getExecutionGraph().vertexFinished();
+		}
 	}
 
-	void executionCanceled() {
-		jobVertex.vertexCancelled(subTaskIndex);
+	void executionCanceled(Execution execution) {
+		// nothing to do
 	}
 
-	void executionFailed(Throwable t) {
-		jobVertex.vertexFailed(subTaskIndex, t);
+	void executionFailed(Execution execution, Throwable cause) {
+		// nothing to do
 	}
 
 	// --------------------------------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
index 1729582ddf400..1ebfcac80152e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
@@ -60,12 +60,12 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.SimpleActorGateway;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doCallRealMethod;
-import static org.mockito.Mockito.doNothing;
+
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
@@ -288,48 +288,58 @@ public void testFailWhileRestarting() throws Exception {
 
 	@Test
 	public void testCancelWhileFailing() throws Exception {
-		// We want to manually control the restart and delay
-		RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy();
-		Tuple2<ExecutionGraph, Instance> executionGraphInstanceTuple = createSpyExecutionGraph(restartStrategy);
-		ExecutionGraph executionGraph = executionGraphInstanceTuple.f0;
-		Instance instance = executionGraphInstanceTuple.f1;
-		doNothing().when(executionGraph).jobVertexInFinalState();
+		final RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy();
+		final ExecutionGraph graph = createExecutionGraph(restartStrategy).f0;
 
-		// Kill the instance...
-		instance.markDead();
+		assertEquals(JobStatus.RUNNING, graph.getState());
 
-		Deadline deadline = TestingUtils.TESTING_DURATION().fromNow();
+		// switch all tasks to running
+		for (ExecutionVertex vertex : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			vertex.getCurrentExecutionAttempt().switchToRunning();
+		}
 
-		// ...and wait for all vertices to be in state FAILED. The
-		// jobVertexInFinalState does nothing, that's why we don't wait on the
-		// job status.
-		boolean success = false;
-		while (deadline.hasTimeLeft() && !success) {
-			success = true;
-			for (ExecutionVertex vertex : executionGraph.getAllExecutionVertices()) {
-				ExecutionState state = vertex.getExecutionState();
-				if (state != ExecutionState.FAILED && state != ExecutionState.CANCELED) {
-					success = false;
-					Thread.sleep(100);
-					break;
-				}
-			}
+		graph.fail(new Exception("test"));
+
+		assertEquals(JobStatus.FAILING, graph.getState());
+
+		graph.cancel();
+
+		assertEquals(JobStatus.CANCELLING, graph.getState());
+
+		// let all tasks finish cancelling
+		for (ExecutionVertex vertex : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			vertex.getCurrentExecutionAttempt().cancelingComplete();
 		}
 
-		// Still in failing
-		assertEquals(JobStatus.FAILING, executionGraph.getState());
+		assertEquals(JobStatus.CANCELED, graph.getState());
+	}
 
-		// The cancel call needs to change the state to CANCELLING
-		executionGraph.cancel();
+	@Test
+	public void testFailWhileCanceling() throws Exception {
+		final RestartStrategy restartStrategy = new NoRestartStrategy();
+		final ExecutionGraph graph = createExecutionGraph(restartStrategy).f0;
 
-		assertEquals(JobStatus.CANCELLING, executionGraph.getState());
+		assertEquals(JobStatus.RUNNING, graph.getState());
 
-		// Unspy and finalize the job state
-		doCallRealMethod().when(executionGraph).jobVertexInFinalState();
+		// switch all tasks to running
+		for (ExecutionVertex vertex : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			vertex.getCurrentExecutionAttempt().switchToRunning();
+		}
 
-		executionGraph.jobVertexInFinalState();
+		graph.cancel();
 
-		assertEquals(JobStatus.CANCELED, executionGraph.getState());
+		assertEquals(JobStatus.CANCELLING, graph.getState());
+
+		graph.fail(new Exception("test"));
+
+		assertEquals(JobStatus.FAILING, graph.getState());
+
+		// let all tasks finish cancelling
+		for (ExecutionVertex vertex : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			vertex.getCurrentExecutionAttempt().cancelingComplete();
+		}
+
+		assertEquals(JobStatus.FAILED, graph.getState());
 	}
 
 	@Test
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
index b0137fa75828d..73838dc6c4524 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.runtime.executiongraph;
 
-import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 
@@ -52,9 +51,10 @@
 import org.apache.flink.runtime.messages.TaskMessages.CancelTask;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.SerializedValue;
-import org.mockito.Matchers;
+
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
+
 import scala.concurrent.ExecutionContext;
 import scala.concurrent.ExecutionContext$;
 
@@ -198,10 +198,6 @@ public Void answer(InvocationOnMock invocation) {
 			}
 		};
 
-		doAnswer(noop).when(ejv).vertexCancelled(Matchers.anyInt());
-		doAnswer(noop).when(ejv).vertexFailed(Matchers.anyInt(), Matchers.any(Throwable.class));
-		doAnswer(noop).when(ejv).vertexFinished(Matchers.anyInt());
-
 		return ejv;
 	}
 	
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java
deleted file mode 100644
index bd51c814892f0..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.executiongraph;
-
-import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.*;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.mock;
-
-import java.util.Collections;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
-import org.apache.flink.runtime.instance.SimpleSlot;
-import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
-import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.util.SerializedValue;
-import org.junit.Test;
-
-public class ExecutionStateProgressTest {
-
-	@Test
-	public void testAccumulatedStateFinished() {
-		try {
-			final JobID jid = new JobID();
-			final JobVertexID vid = new JobVertexID();
-
-			JobVertex ajv = new JobVertex("TestVertex", vid);
-			ajv.setParallelism(3);
-			ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());
-
-			ExecutionGraph graph = new ExecutionGraph(
-				TestingUtils.defaultExecutor(),
-				TestingUtils.defaultExecutor(),
-				jid, 
-				"test job", 
-				new Configuration(),
-				new SerializedValue<>(new ExecutionConfig()),
-				AkkaUtils.getDefaultTimeout(),
-				new NoRestartStrategy(),
-				new Scheduler(TestingUtils.defaultExecutionContext()));
-			graph.attachJobGraph(Collections.singletonList(ajv));
-
-			setGraphStatus(graph, JobStatus.RUNNING);
-
-			ExecutionJobVertex ejv = graph.getJobVertex(vid);
-
-			// mock resources and mock taskmanager
-			for (ExecutionVertex ee : ejv.getTaskVertices()) {
-				SimpleSlot slot = getInstance(
-					new ActorTaskManagerGateway(
-						new SimpleActorGateway(
-							TestingUtils.defaultExecutionContext()))
-				).allocateSimpleSlot(jid);
-				ee.deployToSlot(slot);
-			}
-
-			// finish all
-			for (ExecutionVertex ee : ejv.getTaskVertices()) {
-				ee.executionFinished();
-			}
-
-			assertTrue(ejv.isInFinalState());
-			assertEquals(JobStatus.FINISHED, graph.getState());
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
-	}
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
deleted file mode 100644
index d71798622d3e2..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.executiongraph;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.execution.ExecutionState;
-import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy;
-import org.apache.flink.runtime.instance.DummyActorGateway;
-import org.apache.flink.runtime.instance.HardwareDescription;
-import org.apache.flink.runtime.instance.Instance;
-import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
-import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
-import org.apache.flink.runtime.instance.InstanceID;
-import org.apache.flink.runtime.instance.SimpleSlot;
-import org.apache.flink.runtime.instance.SlotProvider;
-import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
-import org.apache.flink.runtime.operators.testutils.DummyInvokable;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.util.SerializedValue;
-
-import org.junit.Test;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.net.InetAddress;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-
-import static org.junit.Assert.*;
-
-public class TerminalStateDeadlockTest {
-
-	private final Field stateField;
-	private final Field resourceField;
-	private final Field execGraphStateField;
-	private final Field execGraphSlotProviderField;
-
-	private final SimpleSlot resource;
-
-
-	public TerminalStateDeadlockTest() {
-		try {
-			// the reflection fields to access the private fields
-			this.stateField = Execution.class.getDeclaredField("state");
-			this.stateField.setAccessible(true);
-
-			this.resourceField = Execution.class.getDeclaredField("assignedResource");
-			this.resourceField.setAccessible(true);
-
-			this.execGraphStateField = ExecutionGraph.class.getDeclaredField("state");
-			this.execGraphStateField.setAccessible(true);
-
-			this.execGraphSlotProviderField = ExecutionGraph.class.getDeclaredField("slotProvider");
-			this.execGraphSlotProviderField.setAccessible(true);
-			
-			// the dummy resource
-			ResourceID resourceId = ResourceID.generate();
-			InetAddress address = InetAddress.getByName("127.0.0.1");
-			TaskManagerLocation ci = new TaskManagerLocation(resourceId, address, 12345);
-				
-			HardwareDescription resources = new HardwareDescription(4, 4000000, 3000000, 2000000);
-			Instance instance = new Instance(
-				new ActorTaskManagerGateway(DummyActorGateway.INSTANCE), ci, new InstanceID(), resources, 4);
-
-			this.resource = instance.allocateSimpleSlot(new JobID());
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-			
-			// silence the compiler
-			throw new RuntimeException();
-		}
-	}
-
-	// ------------------------------------------------------------------------
-
-	@Test
-	public void testProvokeDeadlock() {
-		try {
-			final JobID jobId = resource.getJobID();
-			final JobVertexID vid1 = new JobVertexID();
-			final JobVertexID vid2 = new JobVertexID();
-			
-			final List<JobVertex> vertices;
-			{
-				JobVertex v1 = new JobVertex("v1", vid1);
-				JobVertex v2 = new JobVertex("v2", vid2);
-				v1.setParallelism(1);
-				v2.setParallelism(1);
-				v1.setInvokableClass(DummyInvokable.class);
-				v2.setInvokableClass(DummyInvokable.class);
-				vertices = Arrays.asList(v1, v2);
-			}
-			
-			final Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
-			
-			final Executor executor = Executors.newFixedThreadPool(4);
-			
-			// try a lot!
-			for (int i = 0; i < 20000; i++) {
-				final TestExecGraph eg = new TestExecGraph(jobId);
-				eg.attachJobGraph(vertices);
-
-				final Execution e1 = eg.getJobVertex(vid1).getTaskVertices()[0].getCurrentExecutionAttempt();
-				final Execution e2 = eg.getJobVertex(vid2).getTaskVertices()[0].getCurrentExecutionAttempt();
-
-				initializeExecution(e1);
-				initializeExecution(e2);
-
-				execGraphStateField.set(eg, JobStatus.FAILING);
-				execGraphSlotProviderField.set(eg, scheduler);
-				
-				Runnable r1 = new Runnable() {
-					@Override
-					public void run() {
-						e1.cancelingComplete();
-					}
-				};
-				Runnable r2 = new Runnable() {
-					@Override
-					public void run() {
-						e2.cancelingComplete();
-					}
-				};
-				
-				executor.execute(r1);
-				executor.execute(r2);
-				
-				eg.waitTillDone();
-			}
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
-	}
-	
-	private void initializeExecution(Execution exec) throws IllegalAccessException {
-		// set state to canceling
-		stateField.set(exec, ExecutionState.CANCELING);
-		
-		// assign a resource
-		resourceField.set(exec, resource);
-	}
-	
-	
-	static class TestExecGraph extends ExecutionGraph {
-
-		private static final Configuration EMPTY_CONFIG = new Configuration();
-
-		private static final Time TIMEOUT = Time.seconds(30L);
-
-		private volatile boolean done;
-
-		TestExecGraph(JobID jobId) throws IOException {
-			super(
-				TestingUtils.defaultExecutor(),
-				TestingUtils.defaultExecutor(),
-				jobId,
-				"test graph",
-				EMPTY_CONFIG,
-				new SerializedValue<>(new ExecutionConfig()),
-				TIMEOUT,
-				new FixedDelayRestartStrategy(1, 0),
-				new Scheduler(TestingUtils.defaultExecutionContext()));
-		}
-
-		@Override
-		public void scheduleForExecution() {
-			// notify that we are done with the "restarting"
-			synchronized (this) {
-				done = true;
-				this.notifyAll();
-			}
-		}
-
-		public void waitTillDone() {
-			try {
-				synchronized (this) {
-					while (!done) {
-						this.wait();
-					}
-				}
-			}
-			catch (InterruptedException e) {
-				throw new RuntimeException(e);
-			}
-		}
-	}
-}

From 45d9b966d6c25457d3e8011277f57f73d2a260dd Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Tue, 21 Mar 2017 19:13:34 +0100
Subject: [PATCH 130/253] [FLINK-5869] [flip-1] Add basic abstraction for
 Failover Strategies to ExecutionGraph

  - Rename 'ExecutionGraph.fail()' to 'ExecutionGraph.failGlobally()' to differentiate from fine grained failures/recovery
  - Add base class for FailoverStrategy
  - Add default implementation (restart all tasks)
  - Add logic to load the failover strategy from the configuration
---
 .../configuration/JobManagerOptions.java      |   7 +
 .../org/apache/flink/util/StringUtils.java    |  20 +
 .../runtime/execution/ExecutionState.java     |  33 +-
 .../runtime/executiongraph/Execution.java     |  56 ++-
 .../executiongraph/ExecutionGraph.java        | 315 ++++++++++----
 .../executiongraph/ExecutionGraphBuilder.java |  10 +
 .../executiongraph/ExecutionJobVertex.java    | 172 +++-----
 .../executiongraph/ExecutionVertex.java       |  87 +++-
 .../GlobalModVersionMismatch.java}            |  35 +-
 .../failover/FailoverStrategy.java            |  92 ++++
 .../failover/FailoverStrategyLoader.java      |  72 ++++
 .../failover/RestartAllStrategy.java          |  80 ++++
 .../failover/RestartIndividualStrategy.java   | 173 ++++++++
 .../metrics/NumberOfFullRestartsGauge.java    |  47 +++
 .../flink/runtime/jobmaster/JobMaster.java    |   2 +-
 .../flink/runtime/jobmanager/JobManager.scala |   6 +-
 .../checkpoint/CheckpointCoordinatorTest.java |   1 +
 .../checkpoint/CoordinatorShutdownTest.java   |   4 +-
 ...ecutionGraphCheckpointCoordinatorTest.java |   4 +-
 .../ExecutionGraphDeploymentTest.java         | 114 ++---
 .../ExecutionGraphMetricsTest.java            |  12 +-
 .../ExecutionGraphRestartTest.java            |  33 +-
 .../ExecutionGraphSchedulingTest.java         |   9 +-
 .../ExecutionGraphSignalsTest.java            | 399 ------------------
 .../ExecutionGraphStopTest.java               | 176 ++++++++
 .../ExecutionGraphSuspendTest.java            | 307 ++++++++++++++
 .../ExecutionGraphTestUtils.java              | 263 ++++++++++--
 .../ExecutionGraphVariousFailuesTest.java     | 118 ++++++
 .../ExecutionVertexCancelTest.java            |  45 +-
 .../ExecutionVertexLocalityTest.java          |   2 +-
 .../ExecutionVertexStopTest.java              | 129 ------
 .../executiongraph/FinalizeOnMasterTest.java  |  96 +++++
 .../executiongraph/GlobalModVersionTest.java  | 212 ++++++++++
 .../IndividualRestartsConcurrencyTest.java    | 332 +++++++++++++++
 .../utils/SimpleAckingTaskManagerGateway.java | 121 ++++++
 .../utils/SimpleSlotProvider.java             | 106 +++++
 .../runtime/jobmanager/JobManagerTest.java    |   2 +-
 .../LeaderChangeJobRecoveryTest.java          |   9 +-
 .../partitioner/RescalePartitionerTest.java   |   2 +
 .../ManuallyTriggeredDirectExecutor.java      |  70 +++
 40 files changed, 2780 insertions(+), 993 deletions(-)
 rename flink-runtime/src/{test/java/org/apache/flink/runtime/executiongraph/TerminalJobStatusListener.java => main/java/org/apache/flink/runtime/executiongraph/GlobalModVersionMismatch.java} (50%)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategy.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartAllStrategy.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/metrics/NumberOfFullRestartsGauge.java
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphStopTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSuspendTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphVariousFailuesTest.java
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexStopTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FinalizeOnMasterTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/GlobalModVersionTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IndividualRestartsConcurrencyTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java
 create mode 100644 flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/ManuallyTriggeredDirectExecutor.java

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index 10d9e16235708..5481d7a32f5c3 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -72,6 +72,13 @@ public class JobManagerOptions {
 			.defaultValue(16)
 			.withDeprecatedKeys("job-manager.max-attempts-history-size");
 
+	/**
+	 * The maximum number of prior execution attempts kept in history.
+	 */
+	public static final ConfigOption<String> EXECUTION_FAILOVER_STRATEGY =
+		key("jobmanager.execution.failover-strategy")
+			.defaultValue("full");
+
 	/**
 	 * This option specifies the interval in order to trigger a resource manager reconnection if the connection
 	 * to the resource manager has been lost.
diff --git a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
index abd6ba6a4fae5..6638062b37748 100644
--- a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
@@ -330,6 +330,26 @@ public static boolean isNullOrWhitespaceOnly(String str) {
 		return true;
 	}
 
+	/**
+	 * If both string arguments are non-null, this method concatenates them with ' and '.
+	 * If only one of the arguments is non-null, this method returns the non-null argument.
+	 * If both arguments are null, this method returns null.
+	 * 
+	 * @param s1 The first string argument
+	 * @param s2 The second string argument
+	 * 
+	 * @return The concatenated string, or non-null argument, or null 
+	 */
+	@Nullable
+	public static String concatenateWithAnd(@Nullable String s1, @Nullable String s2) {
+		if (s1 != null) {
+			return s2 == null ? s1 : s1 + " and " + s2;
+		}
+		else {
+			return s2 != null ? s2 : null;
+		}
+	}
+
 	// ------------------------------------------------------------------------
 
 	/** Prevent instantiation of this utility class */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java
index d6ff0cdd94af3..53ca8b3742133 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java
@@ -25,27 +25,27 @@
  * <pre>{@code
  *
  *     CREATED  -> SCHEDULED -> DEPLOYING -> RUNNING -> FINISHED
- *            |         |            |          |
- *            |         |            |   +------+
- *            |         |            V   V
- *            |         |         CANCELLING -----+----> CANCELED
- *            |         |                         |
- *            |        +-------------------------+
- *            |
- *            |                                   ... -> FAILED
- *           V
+ *        |            |            |          |
+ *        |            |            |   +------+
+ *        |            |            V   V
+ *        |            |         CANCELLING -----+----> CANCELED
+ *        |            |                         |
+ *        |            +-------------------------+
+ *        |
+ *        |                                   ... -> FAILED
+ *        V
  *    RECONCILING  -> RUNNING | FINISHED | CANCELED | FAILED
  *
  * }</pre>
  *
  * <p>It is possible to enter the {@code RECONCILING} state from {@code CREATED}
  * state if job manager fail over, and the {@code RECONCILING} state can switch into
- * any existing task state.</p>
+ * any existing task state.
  *
- * <p>It is possible to enter the {@code FAILED} state from any other state.</p>
+ * <p>It is possible to enter the {@code FAILED} state from any other state.
  *
  * <p>The states {@code FINISHED}, {@code CANCELED}, and {@code FAILED} are
- * considered terminal states.</p>
+ * considered terminal states.
  */
 public enum ExecutionState {
 
@@ -56,7 +56,14 @@ public enum ExecutionState {
 	DEPLOYING,
 	
 	RUNNING,
-	
+
+	/**
+	 * This state marks "successfully completed". It can only be reached when a
+	 * program reaches the "end of its input". The "end of input" can be reached
+	 * when consuming a bounded input (fix set of files, bounded query, etc) or
+	 * when stopping a program (not cancelling!) which make the input look like
+	 * it reached its end at a specific point.
+	 */
 	FINISHED,
 	
 	CANCELING,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 2680849dcec9c..c0f1f39fdc5f4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -74,12 +74,13 @@
 import static org.apache.flink.util.Preconditions.checkState;
 
 /**
- * A single execution of a vertex. While an {@link ExecutionVertex} can be executed multiple times (for recovery,
- * or other re-computation), this class tracks the state of a single execution of that vertex and the resources.
+ * A single execution of a vertex. While an {@link ExecutionVertex} can be executed multiple times
+ * (for recovery, re-computation, re-configuration), this class tracks the state of a single execution
+ * of that vertex and the resources.
  * 
- * <p>NOTE ABOUT THE DESIGN RATIONAL:
+ * <h2>Lock free state transitions</h2>
  * 
- * <p>In several points of the code, we need to deal with possible concurrent state changes and actions.
+ * In several points of the code, we need to deal with possible concurrent state changes and actions.
  * For example, while the call to deploy a task (send it to the TaskManager) happens, the task gets cancelled.
  * 
  * <p>We could lock the entire portion of the code (decision to deploy, deploy, set state to running) such that
@@ -113,6 +114,12 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	/** The unique ID marking the specific execution instant of the task */
 	private final ExecutionAttemptID attemptId;
 
+	/** Gets the global modification version of the execution graph when this execution was created.
+	 * This version is bumped in the ExecutionGraph whenever a global failover happens. It is used
+	 * to resolve conflicts between concurrent modification by global and local failover actions. */
+	private final long globalModVersion;
+
+	/** The timestamps when state transitions occurred, indexed by {@link ExecutionState#ordinal()} */ 
 	private final long[] stateTimestamps;
 
 	private final int attemptNumber;
@@ -146,10 +153,27 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 	// --------------------------------------------------------------------------------------------
 
+	/**
+	 * Creates a new Execution attempt.
+	 * 
+	 * @param executor
+	 *             The executor used to dispatch callbacks from futures and asynchronous RPC calls.
+	 * @param vertex
+	 *             The execution vertex to which this Execution belongs
+	 * @param attemptNumber
+	 *             The execution attempt number.
+	 * @param globalModVersion
+	 *             The global modification version of the execution graph when this execution was created
+	 * @param startTimestamp
+	 *             The timestamp that marks the creation of this Execution
+	 * @param timeout
+	 *             The timeout for RPC calls like deploy/cancel/stop.
+	 */
 	public Execution(
 			Executor executor,
 			ExecutionVertex vertex,
 			int attemptNumber,
+			long globalModVersion,
 			long startTimestamp,
 			Time timeout) {
 
@@ -158,6 +182,7 @@ public Execution(
 		this.attemptId = new ExecutionAttemptID();
 		this.timeout = checkNotNull(timeout);
 
+		this.globalModVersion = globalModVersion;
 		this.attemptNumber = attemptNumber;
 
 		this.stateTimestamps = new long[ExecutionState.values().length];
@@ -190,6 +215,16 @@ public ExecutionState getState() {
 		return state;
 	}
 
+	/**
+	 * Gets the global modification version of the execution graph when this execution was created.
+	 * 
+	 * <p>This version is bumped in the ExecutionGraph whenever a global failover happens. It is used
+	 * to resolve conflicts between concurrent modification by global and local failover actions.
+	 */
+	public long getGlobalModVersion() {
+		return globalModVersion;
+	}
+
 	public SimpleSlot getAssignedResource() {
 		return assignedResource;
 	}
@@ -252,6 +287,12 @@ public Future<ExecutionState> getTerminationFuture() {
 	//  Actions
 	// --------------------------------------------------------------------------------------------
 
+	public boolean scheduleForExecution() {
+		SlotProvider resourceProvider = getVertex().getExecutionGraph().getSlotProvider();
+		boolean allowQueued = getVertex().getExecutionGraph().isQueuedSchedulingAllowed();
+		return scheduleForExecution(resourceProvider, allowQueued);
+	}
+
 	/**
 	 * NOTE: This method only throws exceptions if it is in an illegal state to be scheduled, or if the tasks needs
 	 *       to be scheduled immediately and no resource is available. If the task is accepted by the schedule, any
@@ -381,9 +422,6 @@ public void deployToSlot(final SimpleSlot slot) throws JobException {
 				taskState,
 				attemptNumber);
 
-			// register this execution at the execution graph, to receive call backs
-			vertex.getExecutionGraph().registerExecution(this);
-			
 			final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
 
 			final Future<Acknowledge> submitResultFuture = taskManagerGateway.submitTask(deployment, timeout);
@@ -823,7 +861,7 @@ else if (current == CANCELING || current == RUNNING || current == DEPLOYING) {
 				if (current != FAILED) {
 					String message = String.format("Asynchronous race: Found state %s after successful cancel call.", state);
 					LOG.error(message);
-					vertex.getExecutionGraph().fail(new Exception(message));
+					vertex.getExecutionGraph().failGlobal(new Exception(message));
 				}
 				return;
 			}
@@ -1069,7 +1107,7 @@ private boolean transitionState(ExecutionState currentState, ExecutionState targ
 			// make sure that the state transition completes normally.
 			// potential errors (in listeners may not affect the main logic)
 			try {
-				vertex.notifyStateTransition(attemptId, targetState, error);
+				vertex.notifyStateTransition(this, targetState, error);
 			}
 			catch (Throwable t) {
 				LOG.error("Error while notifying execution graph of execution state transition.", t);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index fff1ea26a4a1f..5eaa637a9314f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -18,8 +18,6 @@
 
 package org.apache.flink.runtime.executiongraph;
 
-import org.apache.commons.lang3.StringUtils;
-
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.Archiveable;
 import org.apache.flink.api.common.ArchivedExecutionConfig;
@@ -42,11 +40,15 @@
 import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.concurrent.AcceptFunction;
 import org.apache.flink.runtime.concurrent.BiFunction;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.runtime.concurrent.FutureUtils;
 import org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.execution.SuppressRestartsException;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy;
+import org.apache.flink.runtime.executiongraph.failover.RestartAllStrategy;
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
 import org.apache.flink.runtime.instance.SimpleSlot;
 import org.apache.flink.runtime.instance.SlotProvider;
@@ -65,10 +67,12 @@
 import org.apache.flink.runtime.taskmanager.TaskExecutionState;
 import org.apache.flink.runtime.util.SerializedThrowable;
 import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FlinkException;
 import org.apache.flink.util.FlinkRuntimeException;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.SerializedValue;
 
+import org.apache.flink.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -91,6 +95,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 
 import static org.apache.flink.util.Preconditions.checkArgument;
@@ -100,9 +105,9 @@
 /**
  * The execution graph is the central data structure that coordinates the distributed
  * execution of a data flow. It keeps representations of each parallel task, each
- * intermediate result, and the communication between them.
+ * intermediate stream, and the communication between them.
  *
- * The execution graph consists of the following constructs:
+ * <p>The execution graph consists of the following constructs:
  * <ul>
  *     <li>The {@link ExecutionJobVertex} represents one vertex from the JobGraph (usually one operation like
  *         "map" or "join") during execution. It holds the aggregated state of all parallel subtasks.
@@ -118,12 +123,41 @@
  *         about deployment of tasks and updates in the task status always use the ExecutionAttemptID to
  *         address the message receiver.</li>
  * </ul>
+ * 
+ * <h2>Global and local failover</h2>
+ * 
+ * The Execution Graph has two failover modes: <i>global failover</i> and <i>local failover</i>.
+ * 
+ * <p>A <b>global failover</b> aborts the task executions for all vertices and restarts whole
+ * data flow graph from the last completed checkpoint. Global failover is considered the
+ * "fallback strategy" that is used when a local failover is unsuccessful, or when a issue is
+ * found in the state of the ExecutionGraph that could mark it as inconsistent (caused by a bug).
+ * 
+ * <p>A <b>local failover</b> is triggered when an individual vertex execution (a task) fails.
+ * The local failover is coordinated by the {@link FailoverStrategy}. A local failover typically
+ * attempts to restart as little as possible, but as much as necessary.
+ * 
+ * <p>Between local- and global failover, the global failover always takes precedence, because it
+ * is the core mechanism that the ExecutionGraph relies on to bring back consistency. The
+ * guard that, the ExecutionGraph maintains a <i>global modification version</i>, which is incremented
+ * with every global failover (and other global actions, like job cancellation, or terminal
+ * failure). Local failover is always scoped by the modification version that the execution graph
+ * had when the failover was triggered. If a new global modification version is reached during
+ * local failover (meaning there is a concurrent global failover), the failover strategy has to
+ * yield before the global failover.  
  */
 public class ExecutionGraph implements AccessExecutionGraph, Archiveable<ArchivedExecutionGraph> {
 
+	/** In place updater for the execution graph's current state. Avoids having to use an
+	 * AtomicReference and thus makes the frequent read access a bit faster */
 	private static final AtomicReferenceFieldUpdater<ExecutionGraph, JobStatus> STATE_UPDATER =
 			AtomicReferenceFieldUpdater.newUpdater(ExecutionGraph.class, JobStatus.class, "state");
 
+	/** In place updater for the execution graph's current global recovery version.
+	 * Avoids having to use an AtomicLong and thus makes the frequent read access a bit faster */
+	private static final AtomicLongFieldUpdater<ExecutionGraph> GLOBAL_VERSION_UPDATER =
+			AtomicLongFieldUpdater.newUpdater(ExecutionGraph.class, "globalModVersion");
+
 	/** The log object used for debugging. */
 	static final Logger LOG = LoggerFactory.getLogger(ExecutionGraph.class);
 
@@ -169,6 +203,9 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	/** Listeners that receive messages whenever a single task execution changes its status */
 	private final List<ExecutionStatusListener> executionListeners;
 
+	/** The implementation that decides how to recover the failures of tasks */
+	private final FailoverStrategy failoverStrategy;
+
 	/** Timestamps (in milliseconds as returned by {@code System.currentTimeMillis()} when
 	 * the execution graph transitioned into a certain state. The index into this array is the
 	 * ordinal of the enum value, i.e. the timestamp when the graph went into state "RUNNING" is
@@ -178,6 +215,10 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	/** The timeout for all messages that require a response/acknowledgement */
 	private final Time rpcCallTimeout;
 
+	/** The timeout for bulk slot allocation (eager scheduling mode). After this timeout,
+	 * slots are released and a recovery is triggered */
+	private final Time scheduleAllocationTimeout;
+
 	/** Strategy to use for restarts */
 	private final RestartStrategy restartStrategy;
 
@@ -190,6 +231,7 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	/** Registered KvState instances reported by the TaskManagers. */
 	private final KvStateLocationRegistry kvStateLocationRegistry;
 
+	/** The total number of vertices currently in the execution graph */
 	private int numVerticesTotal;
 
 	// ------ Configuration of the Execution -------
@@ -203,8 +245,6 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	 * from results than need to be materialized. */
 	private ScheduleMode scheduleMode = ScheduleMode.LAZY_FROM_SOURCES;
 
-	private final Time scheduleAllocationTimeout;
-
 	// ------ Execution status and progress. These values are volatile, and accessed under the lock -------
 
 	private final AtomicInteger verticesFinished;
@@ -212,6 +252,13 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	/** Current status of the job execution */
 	private volatile JobStatus state = JobStatus.CREATED;
 
+	/** A future that completes once the job has reached a terminal state */
+	private volatile CompletableFuture<JobStatus> terminationFuture;
+
+	/** On each global recovery, this version is incremented. The version breaks conflicts
+	 * between concurrent restart attempts by local failover strategies */
+	private volatile long globalModVersion;
+
 	/** The exception that caused the job to fail. This is set to the first root exception
 	 * that was not recoverable and triggered job failure */
 	private volatile Throwable failureCause;
@@ -233,7 +280,7 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	// --------------------------------------------------------------------------------------------
 
 	/**
-	 * This constructor is for tests only, because it does not include class loading information.
+	 * This constructor is for tests only, because it sets default values for many fields.
 	 */
 	@VisibleForTesting
 	ExecutionGraph(
@@ -255,6 +302,7 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 			serializedConfig,
 			timeout,
 			restartStrategy,
+			new RestartAllStrategy.Factory(),
 			Collections.<BlobKey>emptyList(),
 			Collections.<URL>emptyList(),
 			slotProvider,
@@ -270,6 +318,7 @@ public ExecutionGraph(
 			SerializedValue<ExecutionConfig> serializedConfig,
 			Time timeout,
 			RestartStrategy restartStrategy,
+			FailoverStrategy.Factory failoverStrategyFactory,
 			List<BlobKey> requiredJarFiles,
 			List<URL> requiredClasspaths,
 			SlotProvider slotProvider,
@@ -322,6 +371,13 @@ public ExecutionGraph(
 		this.kvStateLocationRegistry = new KvStateLocationRegistry(jobId, getAllVertices());
 
 		this.verticesFinished = new AtomicInteger();
+
+		this.globalModVersion = 1L;
+
+		// the failover strategy must be instantiated last, so that the execution graph
+		// is ready by the time the failover strategy sees it
+		this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");
+		LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -541,6 +597,18 @@ public Throwable getFailureCause() {
 		return failureCause;
 	}
 
+	/**
+	 * Gets the number of full restarts that the execution graph went through.
+	 * If a full restart recovery is currently pending, this recovery is included in the
+	 * count.
+	 * 
+	 * @return The number of full restarts so far
+	 */
+	public long getNumberOfFullRestarts() {
+		// subtract one, because the version starts at one
+		return globalModVersion - 1;
+	}
+
 	@Override
 	public String getFailureCauseAsString() {
 		return ExceptionUtils.stringifyException(failureCause);
@@ -689,11 +757,12 @@ public StringifiedAccumulatorResult[] getAccumulatorResultsStringified() {
 	// --------------------------------------------------------------------------------------------
 
 	public void attachJobGraph(List<JobVertex> topologiallySorted) throws JobException {
-		if (LOG.isDebugEnabled()) {
-			LOG.debug(String.format("Attaching %d topologically sorted vertices to existing job graph with %d "
-					+ "vertices and %d intermediate results.", topologiallySorted.size(), tasks.size(), intermediateResults.size()));
-		}
 
+		LOG.debug("Attaching {} topologically sorted vertices to existing job graph with {} " +
+				"vertices and {} intermediate results.",
+				topologiallySorted.size(), tasks.size(), intermediateResults.size());
+
+		final ArrayList<ExecutionJobVertex> newExecJobVertices = new ArrayList<>(topologiallySorted.size());
 		final long createTimestamp = System.currentTimeMillis();
 
 		for (JobVertex jobVertex : topologiallySorted) {
@@ -704,7 +773,7 @@ public void attachJobGraph(List<JobVertex> topologiallySorted) throws JobExcepti
 
 			// create the execution job vertex and attach it to the graph
 			ExecutionJobVertex ejv =
-					new ExecutionJobVertex(this, jobVertex, 1, rpcCallTimeout, createTimestamp);
+					new ExecutionJobVertex(this, jobVertex, 1, rpcCallTimeout, globalModVersion, createTimestamp);
 			ejv.connectToPredecessors(this.intermediateResults);
 
 			ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv);
@@ -723,7 +792,11 @@ public void attachJobGraph(List<JobVertex> topologiallySorted) throws JobExcepti
 
 			this.verticesInCreationOrder.add(ejv);
 			this.numVerticesTotal += ejv.getParallelism();
+			newExecJobVertices.add(ejv);
 		}
+
+		terminationFuture = new FlinkCompletableFuture<>();
+		failoverStrategy.notifyNewVertices(newExecJobVertices);
 	}
 
 	public void scheduleForExecution() throws JobException {
@@ -856,7 +929,7 @@ public Void apply(Void ignored, Throwable throwable) {
 						// ExecutionGraph notices
 						// we need to go into recovery and make sure to release all slots
 						try {
-							fail(t);
+							failGlobal(t);
 						}
 						finally {
 							ExecutionGraphUtils.releaseAllSlotsSilently(resources);
@@ -889,6 +962,9 @@ public void cancel() {
 			if (current == JobStatus.RUNNING || current == JobStatus.CREATED) {
 				if (transitionState(current, JobStatus.CANCELLING)) {
 
+					// make sure no concurrent local actions interfere with the cancellation
+					incrementGlobalModVersion();
+
 					final ArrayList<Future<?>> futures = new ArrayList<>(verticesInCreationOrder.size());
 
 					// cancel all tasks (that still need cancelling)
@@ -920,8 +996,7 @@ else if (current == JobStatus.FAILING) {
 			else if (current == JobStatus.RESTARTING) {
 				synchronized (progressLock) {
 					if (transitionState(current, JobStatus.CANCELED)) {
-						postRunCleanup();
-						progressLock.notifyAll();
+						onTerminalState(JobStatus.CANCELED);
 
 						LOG.info("Canceled during restart.");
 						return;
@@ -951,7 +1026,7 @@ public void stop() throws StoppingException {
 	 * Suspends the current ExecutionGraph.
 	 *
 	 * The JobStatus will be directly set to SUSPENDED iff the current state is not a terminal
-	 * state. All ExecutionJobVertices will be canceled and the postRunCleanup is executed.
+	 * state. All ExecutionJobVertices will be canceled and the onTerminalState() is executed.
 	 *
 	 * The SUSPENDED state is a local terminal state which stops the execution of the job but does
 	 * not remove the job from the HA job store so that it can be recovered by another JobManager.
@@ -962,21 +1037,23 @@ public void suspend(Throwable suspensionCause) {
 		while (true) {
 			JobStatus currentState = state;
 
-			if (currentState.isGloballyTerminalState()) {
+			if (currentState.isTerminalState()) {
 				// stay in a terminal state
 				return;
 			} else if (transitionState(currentState, JobStatus.SUSPENDED, suspensionCause)) {
 				this.failureCause = suspensionCause;
 
+				// make sure no concurrent local actions interfere with the cancellation
+				incrementGlobalModVersion();
+
 				for (ExecutionJobVertex ejv: verticesInCreationOrder) {
 					ejv.cancel();
 				}
 
 				synchronized (progressLock) {
-						postRunCleanup();
-						progressLock.notifyAll();
+					onTerminalState(JobStatus.SUSPENDED);
 
-						LOG.info("Job {} has been suspended.", getJobID());
+					LOG.info("Job {} has been suspended.", getJobID());
 				}
 
 				return;
@@ -984,7 +1061,18 @@ public void suspend(Throwable suspensionCause) {
 		}
 	}
 
-	public void fail(Throwable t) {
+	/**
+	 * Fails the execution graph globally. This failure will not be recovered by a specific
+	 * failover strategy, but results in a full restart of all tasks.
+	 * 
+	 * <p>This global failure is meant to be triggered in cases where the consistency of the
+	 * execution graph' state cannot be guaranteed any more (for example when catching unexpected
+	 * exceptions that indicate a bug or an unexpected call race), and where a full restart is the
+	 * safe way to get consistency back.
+	 * 
+	 * @param t The exception that caused the failure.
+	 */
+	public void failGlobal(Throwable t) {
 		while (true) {
 			JobStatus current = state;
 			// stay in these states
@@ -1003,6 +1091,9 @@ else if (current == JobStatus.RESTARTING) {
 			else if (transitionState(current, JobStatus.FAILING, t)) {
 				this.failureCause = t;
 
+				// make sure no concurrent local actions interfere with the cancellation
+				incrementGlobalModVersion();
+
 				// we build a future that is complete once all vertices have reached a terminal state
 				final ArrayList<Future<?>> futures = new ArrayList<>(verticesInCreationOrder.size());
 
@@ -1044,23 +1135,20 @@ public void restart() {
 					throw new IllegalStateException("Can only restart job from state restarting.");
 				}
 
-				if (slotProvider == null) {
-					throw new IllegalStateException("The execution graph has not been scheduled before - slotProvider is null.");
-				}
-
 				this.currentExecutions.clear();
 
-				Collection<CoLocationGroup> colGroups = new HashSet<>();
+				final Collection<CoLocationGroup> colGroups = new HashSet<>();
+				final long resetTimestamp = System.currentTimeMillis();
 
 				for (ExecutionJobVertex jv : this.verticesInCreationOrder) {
 
 					CoLocationGroup cgroup = jv.getCoLocationGroup();
-					if(cgroup != null && !colGroups.contains(cgroup)){
+					if (cgroup != null && !colGroups.contains(cgroup)){
 						cgroup.resetConstraints();
 						colGroups.add(cgroup);
 					}
 
-					jv.resetForNewExecution();
+					jv.resetForNewExecution(resetTimestamp, globalModVersion);
 				}
 
 				for (int i = 0; i < stateTimestamps.length; i++) {
@@ -1083,7 +1171,7 @@ public void restart() {
 		}
 		catch (Throwable t) {
 			LOG.warn("Failed to restart the job.", t);
-			fail(t);
+			failGlobal(t);
 		}
 	}
 
@@ -1124,32 +1212,45 @@ public ArchivedExecutionConfig getArchivedExecutionConfig() {
 		return null;
 	}
 
+	@VisibleForTesting
+	public Future<JobStatus> getTerminationFuture() {
+		return terminationFuture;
+	}
+
+	@VisibleForTesting
+	public JobStatus waitUntilTerminal() throws InterruptedException {
+		try {
+			return terminationFuture.get();
+		}
+		catch (ExecutionException e) {
+			// this should never happen
+			// it would be a bug, so we  don't expect this to be handled and throw
+			// an unchecked exception here
+			throw new RuntimeException(e);
+		}
+	}
+
 	/**
-	 * For testing: This waits until the job execution has finished.
+	 * Gets the failover strategy used by the execution graph to recover from failures of tasks.
 	 */
-	public void waitUntilFinished() throws InterruptedException {
-		// we may need multiple attempts in the presence of failures / recovery
-		while (true) {
-			for (ExecutionJobVertex ejv : verticesInCreationOrder) {
-				for (ExecutionVertex ev : ejv.getTaskVertices()) {
-					try {
-						ev.getCurrentExecutionAttempt().getTerminationFuture().get();
-					}
-					catch (ExecutionException e) {
-						// this should never happen
-						throw new RuntimeException(e);
-					}
-				}
-			}
+	public FailoverStrategy getFailoverStrategy() {
+		return this.failoverStrategy;
+	}
 
-			// now that all vertices have been (at some point) in a terminal state,
-			// we need to check if the job as a whole has entered a final state
-			if (state.isTerminalState()) {
-				return;
-			}
-		}
+	/**
+	 * Gets the current global modification version of the ExecutionGraph.
+	 * The global modification version is incremented with each global action (cancel/fail/restart)
+	 * and is used to disambiguate concurrent modifications between local and global
+	 * failover actions.
+	 */
+	long getGlobalModVersion() {
+		return globalModVersion;
 	}
 
+	// ------------------------------------------------------------------------
+	//  State Transitions
+	// ------------------------------------------------------------------------
+
 	private boolean transitionState(JobStatus current, JobStatus newState) {
 		return transitionState(current, newState, null);
 	}
@@ -1175,11 +1276,45 @@ private boolean transitionState(JobStatus current, JobStatus newState, Throwable
 		}
 	}
 
+	private long incrementGlobalModVersion() {
+		return GLOBAL_VERSION_UPDATER.incrementAndGet(this);
+	}
+
+	// ------------------------------------------------------------------------
+	//  Job Status Progress
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Called whenever a vertex reaches state FINISHED (completed successfully).
+	 * Once all vertices are in the FINISHED state, the program is successfully done.
+	 */
 	void vertexFinished() {
-		int numFinished = verticesFinished.incrementAndGet();
+		final int numFinished = verticesFinished.incrementAndGet();
 		if (numFinished == numVerticesTotal) {
 			// done :-)
-			allVerticesInTerminalState();
+
+			// check whether we are still in "RUNNING" and trigger the final cleanup
+			if (state == JobStatus.RUNNING) {
+				// we do the final cleanup in the I/O executor, because it may involve
+				// some heavier work
+
+				try {
+					for (ExecutionJobVertex ejv : verticesInCreationOrder) {
+						ejv.getJobVertex().finalizeOnMaster(getUserClassLoader());
+					}
+				}
+				catch (Throwable t) {
+					ExceptionUtils.rethrowIfFatalError(t);
+					failGlobal(new Exception("Failed to finalize execution on master", t));
+					return;
+				}
+
+				// if we do not make this state transition, then a concurrent
+				// cancellation or failure happened
+				if (transitionState(JobStatus.RUNNING, JobStatus.FINISHED)) {
+					onTerminalState(JobStatus.FINISHED);
+				}
+			}
 		}
 	}
 
@@ -1187,6 +1322,10 @@ void vertexUnFinished() {
 		verticesFinished.getAndDecrement();
 	}
 
+	/**
+	 * This method is a callback during cancellation/failover and called when all tasks
+	 * have reached a terminal state (cancelled/failed/finished).
+	 */
 	private void allVerticesInTerminalState() {
 		// we are done, transition to the final state
 		JobStatus current;
@@ -1194,14 +1333,11 @@ private void allVerticesInTerminalState() {
 			current = this.state;
 
 			if (current == JobStatus.RUNNING) {
-				if (transitionState(current, JobStatus.FINISHED)) {
-					postRunCleanup();
-					break;
-				}
+				failGlobal(new Exception("ExecutionGraph went into allVerticesInTerminalState() from RUNNING"));
 			}
 			else if (current == JobStatus.CANCELLING) {
 				if (transitionState(current, JobStatus.CANCELED)) {
-					postRunCleanup();
+					onTerminalState(JobStatus.CANCELED);
 					break;
 				}
 			}
@@ -1221,7 +1357,7 @@ else if (current.isGloballyTerminalState()) {
 				break;
 			}
 			else {
-				fail(new Exception("ExecutionGraph went into final state from state " + current));
+				failGlobal(new Exception("ExecutionGraph went into final state from state " + current));
 				break;
 			}
 		}
@@ -1255,18 +1391,16 @@ private boolean tryRestartOrFail() {
 					restartStrategy.restart(this);
 
 					return true;
-				} else if (!isRestartable && transitionState(currentState, JobStatus.FAILED, failureCause)) {
-					final List<String> reasonsForNoRestart = new ArrayList<>(2);
-					if (!isFailureCauseAllowingRestart) {
-						reasonsForNoRestart.add("a type of SuppressRestartsException was thrown");
-					}
-					if (!isRestartStrategyAllowingRestart) {
-						reasonsForNoRestart.add("the restart strategy prevented it");
-					}
+				}
+				else if (!isRestartable && transitionState(currentState, JobStatus.FAILED, failureCause)) {
+					final String cause1 = isFailureCauseAllowingRestart ? null :  
+							"a type of SuppressRestartsException was thrown";
+					final String cause2 = isRestartStrategyAllowingRestart ? null :
+						"the restart strategy prevented it";
 
 					LOG.info("Could not restart the job {} ({}) because {}.", getJobName(), getJobID(),
-						StringUtils.join(reasonsForNoRestart, " and "), failureCause);
-					postRunCleanup();
+						StringUtils.concatenateWithAnd(cause1, cause2), failureCause);
+					onTerminalState(JobStatus.FAILED);
 
 					return true;
 				} else {
@@ -1280,16 +1414,20 @@ private boolean tryRestartOrFail() {
 		}
 	}
 
-	private void postRunCleanup() {
+	private void onTerminalState(JobStatus status) {
 		try {
 			CheckpointCoordinator coord = this.checkpointCoordinator;
 			this.checkpointCoordinator = null;
 			if (coord != null) {
-				coord.shutdown(state);
+				coord.shutdown(status);
 			}
-		} catch (Exception e) {
+		}
+		catch (Exception e) {
 			LOG.error("Error while cleaning up after execution", e);
 		}
+		finally {
+			terminationFuture.complete(status);
+		}
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -1343,7 +1481,7 @@ public boolean updateState(TaskExecutionState state) {
 				ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
 
 				// failures during updates leave the ExecutionGraph inconsistent
-				fail(t);
+				failGlobal(t);
 				return false;
 			}
 		}
@@ -1405,7 +1543,7 @@ public Map<ExecutionAttemptID, Execution> getRegisteredExecutions() {
 	void registerExecution(Execution exec) {
 		Execution previous = currentExecutions.putIfAbsent(exec.getAttemptId(), exec);
 		if (previous != null) {
-			fail(new Exception("Trying to register execution " + exec + " for already used ID " + exec.getAttemptId()));
+			failGlobal(new Exception("Trying to register execution " + exec + " for already used ID " + exec.getAttemptId()));
 		}
 	}
 
@@ -1413,7 +1551,7 @@ void deregisterExecution(Execution exec) {
 		Execution contained = currentExecutions.remove(exec.getAttemptId());
 
 		if (contained != null && contained != exec) {
-			fail(new Exception("De-registering execution " + exec + " failed. Found for same ID execution " + contained));
+			failGlobal(new Exception("De-registering execution " + exec + " failed. Found for same ID execution " + contained));
 		}
 	}
 
@@ -1471,21 +1609,21 @@ private void notifyJobStatusChange(JobStatus newState, Throwable error) {
 	}
 
 	void notifyExecutionChange(
-			JobVertexID vertexId, int subtask, ExecutionAttemptID executionID,
-			ExecutionState newExecutionState, Throwable error)
-	{
-		ExecutionJobVertex vertex = getJobVertex(vertexId);
+			final Execution execution,
+			final ExecutionState newExecutionState,
+			final Throwable error) {
 
 		if (executionListeners.size() > 0) {
+			final ExecutionJobVertex vertex = execution.getVertex().getJobVertex();
 			final String message = error == null ? null : ExceptionUtils.stringifyException(error);
 			final long timestamp = System.currentTimeMillis();
 
 			for (ExecutionStatusListener listener : executionListeners) {
 				try {
 					listener.executionStatusChanged(
-							getJobID(), vertexId, vertex.getJobVertex().getName(),
-							vertex.getParallelism(), subtask, executionID, newExecutionState,
-							timestamp, message);
+							getJobID(), vertex.getJobVertexId(), vertex.getJobVertex().getName(),
+							vertex.getParallelism(), execution.getParallelSubtaskIndex(),
+							execution.getAttemptId(), newExecutionState, timestamp, message);
 				} catch (Throwable t) {
 					LOG.warn("Error while notifying ExecutionStatusListener", t);
 				}
@@ -1494,7 +1632,20 @@ void notifyExecutionChange(
 
 		// see what this means for us. currently, the first FAILED state means -> FAILED
 		if (newExecutionState == ExecutionState.FAILED) {
-			fail(error);
+			final Throwable ex = error != null ? error : new FlinkException("Unknown Error (missing cause)");
+
+			// by filtering out late failure calls, we can save some work in
+			// avoiding redundant local failover
+			if (execution.getGlobalModVersion() == globalModVersion) {
+				try {
+					failoverStrategy.onTaskFailure(execution, ex);
+				}
+				catch (Throwable t) {
+					// bug in the failover strategy - fall back to global failover
+					LOG.warn("Error in failover strategy - falling back to global restart", t);
+					failGlobal(ex);
+				}
+			}
 		}
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
index b40817ff8adc1..88863e485197e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
@@ -34,7 +34,10 @@
 import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.client.JobSubmissionException;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategyLoader;
 import org.apache.flink.runtime.executiongraph.metrics.DownTimeGauge;
+import org.apache.flink.runtime.executiongraph.metrics.NumberOfFullRestartsGauge;
 import org.apache.flink.runtime.executiongraph.metrics.RestartTimeGauge;
 import org.apache.flink.runtime.executiongraph.metrics.UpTimeGauge;
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
@@ -90,6 +93,9 @@ public static ExecutionGraph buildGraph(
 		final String jobName = jobGraph.getName();
 		final JobID jobId = jobGraph.getJobID();
 
+		final FailoverStrategy.Factory failoverStrategy = 
+				FailoverStrategyLoader.loadFailoverStrategy(jobManagerConfig, log);
+
 		// create a new execution graph, if none exists so far
 		final ExecutionGraph executionGraph = (prior != null) ? prior :
 				new ExecutionGraph(
@@ -101,6 +107,7 @@ public static ExecutionGraph buildGraph(
 						jobGraph.getSerializedExecutionConfig(),
 						timeout,
 						restartStrategy,
+						failoverStrategy,
 						jobGraph.getUserJarBlobKeys(),
 						jobGraph.getClasspaths(),
 						slotProvider,
@@ -269,6 +276,9 @@ public static ExecutionGraph buildGraph(
 		metrics.gauge(RestartTimeGauge.METRIC_NAME, new RestartTimeGauge(executionGraph));
 		metrics.gauge(DownTimeGauge.METRIC_NAME, new DownTimeGauge(executionGraph));
 		metrics.gauge(UpTimeGauge.METRIC_NAME, new UpTimeGauge(executionGraph));
+		metrics.gauge(NumberOfFullRestartsGauge.METRIC_NAME, new NumberOfFullRestartsGauge(executionGraph));
+
+		executionGraph.getFailoverStrategy().registerMetrics(metrics);
 
 		return executionGraph;
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index 3197e656b4500..3a98e0a5a1f59 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.executiongraph;
 
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.Archiveable;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.JobID;
@@ -28,7 +29,6 @@
 import org.apache.flink.core.io.InputSplit;
 import org.apache.flink.core.io.InputSplitAssigner;
 import org.apache.flink.core.io.InputSplitSource;
-import org.apache.flink.core.io.LocatableInputSplit;
 import org.apache.flink.runtime.JobException;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.concurrent.Future;
@@ -58,6 +58,13 @@
 import java.util.List;
 import java.util.Map;
 
+/**
+ * An {@code ExecutionJobVertex} is part of the {@link ExecutionGraph}, and the peer 
+ * to the {@link JobVertex}.
+ * 
+ * <p>The {@code ExecutionJobVertex} corresponds to a parallelized operation. It
+ * contains an {@link ExecutionVertex} for each parallel instance of that operation.
+ */
 public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable<ArchivedExecutionJobVertex> {
 
 	/** Use the same log for all ExecutionGraph classes */
@@ -115,21 +122,26 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 	private SerializedValue<TaskInformation> serializedTaskInformation;
 
 	private InputSplitAssigner splitAssigner;
-	
-	public ExecutionJobVertex(
+
+	/**
+	 * Convenience constructor for testing.
+	 */
+	@VisibleForTesting
+	ExecutionJobVertex(
 		ExecutionGraph graph,
 		JobVertex jobVertex,
 		int defaultParallelism,
 		Time timeout) throws JobException {
 
-		this(graph, jobVertex, defaultParallelism, timeout, System.currentTimeMillis());
+		this(graph, jobVertex, defaultParallelism, timeout, 1L, System.currentTimeMillis());
 	}
-	
+
 	public ExecutionJobVertex(
 		ExecutionGraph graph,
 		JobVertex jobVertex,
 		int defaultParallelism,
 		Time timeout,
+		long initialGlobalModVersion,
 		long createTimestamp) throws JobException {
 
 		if (graph == null || jobVertex == null) {
@@ -190,18 +202,24 @@ public ExecutionJobVertex(
 		// create all task vertices
 		for (int i = 0; i < numTaskVertices; i++) {
 			ExecutionVertex vertex = new ExecutionVertex(
-					this, i, this.producedDataSets, timeout, createTimestamp, maxPriorAttemptsHistoryLength);
+					this,
+					i,
+					producedDataSets,
+					timeout,
+					initialGlobalModVersion,
+					createTimestamp,
+					maxPriorAttemptsHistoryLength);
 
 			this.taskVertices[i] = vertex;
 		}
-		
+
 		// sanity check for the double referencing between intermediate result partitions and execution vertices
 		for (IntermediateResult ir : this.producedDataSets) {
 			if (ir.getNumberOfAssignedPartitions() != parallelism) {
 				throw new RuntimeException("The intermediate result's partitions were not correctly assigned.");
 			}
 		}
-		
+
 		// set up the input splits, if the vertex has any
 		try {
 			@SuppressWarnings("unchecked")
@@ -508,7 +526,8 @@ public void fail(Throwable t) {
 		}
 	}
 
-	public void resetForNewExecution() {
+	public void resetForNewExecution(final long timestamp, final long expectedGlobalModVersion)
+			throws GlobalModVersionMismatch {
 
 		synchronized (stateMonitor) {
 			// check and reset the sharing groups with scheduler hints
@@ -517,7 +536,7 @@ public void resetForNewExecution() {
 			}
 
 			for (int i = 0; i < parallelism; i++) {
-				taskVertices[i].resetForNewExecution();
+				taskVertices[i].resetForNewExecution(timestamp, expectedGlobalModVersion);
 			}
 
 			// set up the input splits again
@@ -558,112 +577,36 @@ public StringifiedAccumulatorResult[] getAggregatedUserAccumulatorsStringified()
 	}
 
 	// --------------------------------------------------------------------------------------------
-	//  Static / pre-assigned input splits
+	//  Archiving
 	// --------------------------------------------------------------------------------------------
 
-	private List<LocatableInputSplit>[] computeLocalInputSplitsPerTask(InputSplit[] splits) throws JobException {
-		
-		final int numSubTasks = getParallelism();
-		
-		// sanity check
-		if (numSubTasks > splits.length) {
-			throw new JobException("Strictly local assignment requires at least as many splits as subtasks.");
-		}
-		
-		// group the splits by host while preserving order per host
-		Map<String, List<LocatableInputSplit>> splitsByHost = new HashMap<String, List<LocatableInputSplit>>();
-		
-		for (InputSplit split : splits) {
-			// check that split has exactly one local host
-			if(!(split instanceof LocatableInputSplit)) {
-				throw new JobException("Invalid InputSplit type " + split.getClass().getCanonicalName() + ". " +
-						"Strictly local assignment requires LocatableInputSplit");
-			}
-			LocatableInputSplit lis = (LocatableInputSplit) split;
-
-			if (lis.getHostnames() == null) {
-				throw new JobException("LocatableInputSplit has no host information. " +
-						"Strictly local assignment requires exactly one hostname for each LocatableInputSplit.");
-			}
-			else if (lis.getHostnames().length != 1) {
-				throw new JobException("Strictly local assignment requires exactly one hostname for each LocatableInputSplit.");
-			}
-			String hostName = lis.getHostnames()[0];
-			
-			if (hostName == null) {
-				throw new JobException("For strictly local input split assignment, no null host names are allowed.");
-			}
-
-			List<LocatableInputSplit> hostSplits = splitsByHost.get(hostName);
-			if (hostSplits == null) {
-				hostSplits = new ArrayList<LocatableInputSplit>();
-				splitsByHost.put(hostName, hostSplits);
-			}
-			hostSplits.add(lis);
-		}
-		
-		
-		int numHosts = splitsByHost.size();
-		
-		if (numSubTasks < numHosts) {
-			throw new JobException("Strictly local split assignment requires at least as " +
-					"many parallel subtasks as distinct split hosts. Please increase the parallelism " +
-					"of DataSource "+this.getJobVertex().getName()+" to at least "+numHosts+".");
-		}
-
-		// get list of hosts in deterministic order
-		List<String> hosts = new ArrayList<String>(splitsByHost.keySet());
-		Collections.sort(hosts);
-		
-		@SuppressWarnings("unchecked")
-		List<LocatableInputSplit>[] subTaskSplitAssignment = (List<LocatableInputSplit>[]) new List<?>[numSubTasks];
-		
-		final int subtasksPerHost = numSubTasks / numHosts;
-		final int hostsWithOneMore = numSubTasks % numHosts;
-		
-		int subtaskNum = 0;
-		
-		// we go over all hosts and distribute the hosts' input splits
-		// over the subtasks
-		for (int hostNum = 0; hostNum < numHosts; hostNum++) {
-			String host = hosts.get(hostNum);
-			List<LocatableInputSplit> splitsOnHost = splitsByHost.get(host);
-			
-			int numSplitsOnHost = splitsOnHost.size();
-			
-			// the number of subtasks to split this over.
-			// NOTE: if the host has few splits, some subtasks will not get anything.
-			int subtasks = Math.min(numSplitsOnHost, 
-							hostNum < hostsWithOneMore ? subtasksPerHost + 1 : subtasksPerHost);
-			
-			int splitsPerSubtask = numSplitsOnHost / subtasks;
-			int subtasksWithOneMore = numSplitsOnHost % subtasks;
-			
-			int splitnum = 0;
-			
-			// go over the subtasks and grab a subrange of the input splits
-			for (int i = 0; i < subtasks; i++) {
-				int numSplitsForSubtask = (i < subtasksWithOneMore ? splitsPerSubtask + 1 : splitsPerSubtask);
-				
-				List<LocatableInputSplit> splitList;
-				
-				if (numSplitsForSubtask == numSplitsOnHost) {
-					splitList = splitsOnHost;
-				}
-				else {
-					splitList = new ArrayList<LocatableInputSplit>(numSplitsForSubtask);
-					for (int k = 0; k < numSplitsForSubtask; k++) {
-						splitList.add(splitsOnHost.get(splitnum++));
-					}
-				}
-				
-				subTaskSplitAssignment[subtaskNum++] = splitList;
-			}
-		}
-		
-		return subTaskSplitAssignment;
+	@Override
+	public ArchivedExecutionJobVertex archive() {
+		return new ArchivedExecutionJobVertex(this);
 	}
 
+	// ------------------------------------------------------------------------
+	//  Static Utilities
+	// ------------------------------------------------------------------------
+
+	/**
+	 * A utility function that computes an "aggregated" state for the vertex.
+	 * 
+	 * <p>This state is not used anywhere in the  coordination, but can be used for display
+	 * in dashboards to as a summary for how the particular parallel operation represented by
+	 * this ExecutionJobVertex is currently behaving.
+	 * 
+	 * <p>For example, if at least one parallel task is failed, the aggregate state is failed.
+	 * If not, and at least one parallel task is cancelling (or cancelled), the aggregate state
+	 * is cancelling (or cancelled). If all tasks are finished, the aggregate state is finished,
+	 * and so on.
+	 * 
+	 * @param verticesPerState The number of vertices in each state (indexed by the ordinal of
+	 *                         the ExecutionState values).
+	 * @param parallelism The parallelism of the ExecutionJobVertex
+	 * 
+	 * @return The aggregate state of this ExecutionJobVertex. 
+	 */
 	public static ExecutionState getAggregateJobVertexState(int[] verticesPerState, int parallelism) {
 		if (verticesPerState == null || verticesPerState.length != ExecutionState.values().length) {
 			throw new IllegalArgumentException("Must provide an array as large as there are execution states.");
@@ -739,9 +682,4 @@ public static Map<OperatorID, ExecutionJobVertex> includeAlternativeOperatorIDs(
 
 		return expanded;
 	}
-
-	@Override
-	public ArchivedExecutionJobVertex archive() {
-		return new ArchivedExecutionJobVertex(this);
-	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index bcf7a7c4d909b..e8c19840a0b6f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.executiongraph;
 
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.Archiveable;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
@@ -97,7 +98,11 @@ public class ExecutionVertex implements AccessExecutionVertex, Archiveable<Archi
 
 	// --------------------------------------------------------------------------------------------
 
-	public ExecutionVertex(
+	/**
+	 * Convenience constructor for tests. Sets various fields to default values.
+	 */
+	@VisibleForTesting
+	ExecutionVertex(
 			ExecutionJobVertex jobVertex,
 			int subTaskIndex,
 			IntermediateResult[] producedDataSets,
@@ -108,24 +113,28 @@ public ExecutionVertex(
 				subTaskIndex,
 				producedDataSets,
 				timeout,
+				1L,
 				System.currentTimeMillis(),
 				JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue());
 	}
 
+	/**
+	 * 
+	 * @param timeout
+	 *            The RPC timeout to use for deploy / cancel calls
+	 * @param initialGlobalModVersion
+	 *            The global modification version to initialize the first Execution with.
+	 * @param createTimestamp
+	 *            The timestamp for the vertex creation, used to initialize the first Execution with.
+	 * @param maxPriorExecutionHistoryLength
+	 *            The number of prior Executions (= execution attempts) to keep.
+	 */
 	public ExecutionVertex(
 			ExecutionJobVertex jobVertex,
 			int subTaskIndex,
 			IntermediateResult[] producedDataSets,
 			Time timeout,
-			int maxPriorExecutionHistoryLength) {
-		this(jobVertex, subTaskIndex, producedDataSets, timeout, System.currentTimeMillis(), maxPriorExecutionHistoryLength);
-	}
-
-	public ExecutionVertex(
-			ExecutionJobVertex jobVertex,
-			int subTaskIndex,
-			IntermediateResult[] producedDataSets,
-			Time timeout,
+			long initialGlobalModVersion,
 			long createTimestamp,
 			int maxPriorExecutionHistoryLength) {
 
@@ -151,6 +160,7 @@ public ExecutionVertex(
 			getExecutionGraph().getFutureExecutor(),
 			this,
 			0,
+			initialGlobalModVersion,
 			createTimestamp,
 			timeout);
 
@@ -163,6 +173,8 @@ public ExecutionVertex(
 			this.locationConstraint = null;
 		}
 
+		getExecutionGraph().registerExecution(currentExecution);
+
 		this.timeout = timeout;
 	}
 
@@ -508,11 +520,40 @@ public Iterable<TaskManagerLocation> getPreferredLocationsBasedOnInputs() {
 	//   Actions
 	// --------------------------------------------------------------------------------------------
 
-	public Execution resetForNewExecution() {
-
+	/**
+	 * Archives the current Execution and creates a new Execution for this vertex.
+	 * 
+	 * <p>This method atomically checks if the ExecutionGraph is still of an expected
+	 * global mod. version and replaces the execution if that is the case. If the ExecutionGraph
+	 * has increased its global mod. version in the meantime, this operation fails.
+	 * 
+	 * <p>This mechanism can be used to prevent conflicts between various concurrent recovery and
+	 * reconfiguration actions in a similar way as "optimistic concurrency control".
+	 * 
+	 * @param timestamp
+	 *             The creation timestamp for the new Execution
+	 * @param originatingGlobalModVersion
+	 *             The 
+	 * 
+	 * @return Returns the new created Execution. 
+	 * 
+	 * @throws GlobalModVersionMismatch Thrown, if the execution graph has a new global mod
+	 *                                  version than the one passed to this message.
+	 */
+	public Execution resetForNewExecution(final long timestamp, final long originatingGlobalModVersion)
+			throws GlobalModVersionMismatch
+	{
 		LOG.debug("Resetting execution vertex {} for new execution.", getTaskNameWithSubtaskIndex());
 
 		synchronized (priorExecutions) {
+			// check if another global modification has been triggered since the
+			// action that originally caused this reset/restart happened
+			final long actualModVersion = getExecutionGraph().getGlobalModVersion();
+			if (actualModVersion > originatingGlobalModVersion) {
+				// global change happened since, reject this action
+				throw new GlobalModVersionMismatch(originatingGlobalModVersion, actualModVersion);
+			}
+
 			final Execution oldExecution = currentExecution;
 			final ExecutionState oldState = oldExecution.getState();
 
@@ -522,8 +563,9 @@ public Execution resetForNewExecution() {
 				final Execution newExecution = new Execution(
 					getExecutionGraph().getFutureExecutor(),
 					this,
-						oldExecution.getAttemptNumber()+1,
-					System.currentTimeMillis(),
+					oldExecution.getAttemptNumber() + 1,
+					originatingGlobalModVersion,
+					timestamp,
 					timeout);
 
 				this.currentExecution = newExecution;
@@ -533,6 +575,9 @@ public Execution resetForNewExecution() {
 					this.locationConstraint = grp.getLocationConstraint(subTaskIndex);
 				}
 
+				// register this execution at the execution graph, to receive call backs
+				getExecutionGraph().registerExecution(newExecution);
+
 				// if the execution was 'FINISHED' before, tell the ExecutionGraph that
 				// we take one step back on the road to reaching global FINISHED
 				if (oldState == FINISHED) {
@@ -640,9 +685,7 @@ List<IntermediateResultPartition> finishAllBlockingPartitions() {
 	// --------------------------------------------------------------------------------------------
 
 	void executionFinished(Execution execution) {
-		if (execution == currentExecution) {
-			getExecutionGraph().vertexFinished();
-		}
+		getExecutionGraph().vertexFinished();
 	}
 
 	void executionCanceled(Execution execution) {
@@ -658,10 +701,14 @@ void executionFailed(Execution execution, Throwable cause) {
 	// --------------------------------------------------------------------------------------------
 
 	/**
-	 * Simply forward this notification. This is for logs and event archivers.
+	 * Simply forward this notification
 	 */
-	void notifyStateTransition(ExecutionAttemptID executionId, ExecutionState newState, Throwable error) {
-		getExecutionGraph().notifyExecutionChange(getJobvertexId(), subTaskIndex, executionId, newState, error);
+	void notifyStateTransition(Execution execution, ExecutionState newState, Throwable error) {
+		// only forward this notification if the execution is still the current execution
+		// otherwise we have an outdated execution
+		if (currentExecution == execution) {
+			getExecutionGraph().notifyExecutionChange(execution, newState, error);
+		}
 	}
 
 	/**
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalJobStatusListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/GlobalModVersionMismatch.java
similarity index 50%
rename from flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalJobStatusListener.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/GlobalModVersionMismatch.java
index c107d54cf6ef7..bc968050fd8ae 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalJobStatusListener.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/GlobalModVersionMismatch.java
@@ -18,28 +18,29 @@
 
 package org.apache.flink.runtime.executiongraph;
 
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.core.testutils.OneShotLatch;
-import org.apache.flink.runtime.jobgraph.JobStatus;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
 /**
- * A job status listener that waits lets one block until the job is in a terminal state.
+ * An exception that indicates a mismatch between the expected global modification version
+ * of the execution graph, and the actual modification version.
  */
-public class TerminalJobStatusListener  implements JobStatusListener {
+public class GlobalModVersionMismatch extends Exception {
+
+	private static final long serialVersionUID = 6643688395797045098L;
 
-	private final OneShotLatch terminalStateLatch = new OneShotLatch();
+	private final long expectedModVersion;
+
+	private final long actualModVersion;
+
+	public GlobalModVersionMismatch(long expectedModVersion, long actualModVersion) {
+		super("expected=" + expectedModVersion + ", actual=" + actualModVersion);
+		this.expectedModVersion = expectedModVersion;
+		this.actualModVersion = actualModVersion;
+	}
 
-	public void waitForTerminalState(long timeoutMillis) throws InterruptedException, TimeoutException {
-		terminalStateLatch.await(timeoutMillis, TimeUnit.MILLISECONDS);
+	public long expectedModVersion() {
+		return expectedModVersion;
 	}
 
-	@Override
-	public void jobStatusChanges(JobID jobId, JobStatus newJobStatus, long timestamp, Throwable error) {
-		if (newJobStatus.isGloballyTerminalState() || newJobStatus == JobStatus.SUSPENDED) {
-			terminalStateLatch.trigger();
-		}
+	public long actualModVersion() {
+		return actualModVersion;
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategy.java
new file mode 100644
index 0000000000000..2c4313fe7e71a
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategy.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.failover;
+
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+
+import java.util.List;
+
+/**
+ * A {@code FailoverStrategy} describes how the job computation recovers from task
+ * failures.
+ * 
+ * <p>Failover strategies implement recovery logic for failures of tasks. The execution
+ * graph still implements "global failure / recovery" (which restarts all tasks) as
+ * a fallback plan or safety net in cases where it deems that the state of the graph
+ * may have become inconsistent.
+ */
+public abstract class FailoverStrategy {
+
+
+	// ------------------------------------------------------------------------
+	//  failover implementation
+	// ------------------------------------------------------------------------ 
+
+	/**
+	 * Called by the execution graph when a task failure occurs.
+	 * 
+	 * @param taskExecution The execution attempt of the failed task. 
+	 * @param cause The exception that caused the task failure.
+	 */
+	public abstract void onTaskFailure(Execution taskExecution, Throwable cause);
+
+	/**
+	 * Called whenever new vertices are added to the ExecutionGraph.
+	 * 
+	 * @param newJobVerticesTopological The newly added vertices, in topological order.
+	 */
+	public abstract void notifyNewVertices(List<ExecutionJobVertex> newJobVerticesTopological);
+
+	/**
+	 * Gets the name of the failover strategy, for logging purposes.
+	 */
+	public abstract String getStrategyName();
+
+	/**
+	 * Tells the FailoverStrategy to register its metrics.
+	 * 
+	 * <p>The default implementation does nothing
+	 * 
+	 * @param metricGroup The metric group to register the metrics at
+	 */
+	public void registerMetrics(MetricGroup metricGroup) {}
+
+	// ------------------------------------------------------------------------
+	//  factory
+	// ------------------------------------------------------------------------
+
+	/**
+	 * This factory is a necessary indirection when creating the FailoverStrategy to that
+	 * we can have both the FailoverStrategy final in the ExecutionGraph, and the
+	 * ExecutionGraph final in the FailOverStrategy.
+	 */
+	public interface Factory {
+
+		/**
+		 * Instantiates the {@code FailoverStrategy}.
+		 * 
+		 * @param executionGraph The execution graph for which the strategy implements failover.
+		 * @return The instantiated failover strategy.
+		 */
+		FailoverStrategy create(ExecutionGraph executionGraph);
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java
new file mode 100644
index 0000000000000..f18a90f79c329
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.failover;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.util.StringUtils;
+
+import org.slf4j.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * A utility class to load failover strategies from the configuration. 
+ */
+public class FailoverStrategyLoader {
+
+	/** Config name for the {@link RestartAllStrategy} */
+	public static final String FULL_RESTART_STRATEGY_NAME = "full";
+
+	/** Config name for the strategy that restarts individual tasks */
+	public static final String INDIVIDUAL_RESTART_STRATEGY_NAME = "individual";
+
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Loads a FailoverStrategy Factory from the given configuration.
+	 */
+	public static FailoverStrategy.Factory loadFailoverStrategy(Configuration config, @Nullable Logger logger) {
+		final String strategyParam = config.getString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY);
+
+		if (StringUtils.isNullOrWhitespaceOnly(strategyParam)) {
+			if (logger != null) {
+				logger.warn("Null config value for {} ; using default failover strategy (full restarts).",
+						JobManagerOptions.EXECUTION_FAILOVER_STRATEGY.key());
+			}
+
+			return new RestartAllStrategy.Factory();
+		}
+		else {
+			switch (strategyParam.toLowerCase()) {
+				case FULL_RESTART_STRATEGY_NAME:
+					return new RestartAllStrategy.Factory();
+
+				case INDIVIDUAL_RESTART_STRATEGY_NAME:
+					return new RestartIndividualStrategy.Factory();
+
+				default:
+					// we could interpret the parameter as a factory class name and instantiate that
+					// for now we simply do not support this
+					throw new IllegalConfigurationException("Unknown failover strategy: " + strategyParam);
+			}
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartAllStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartAllStrategy.java
new file mode 100644
index 0000000000000..21f42b974cf19
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartAllStrategy.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.failover;
+
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Simple failover strategy that triggers a restart of all tasks in the
+ * execution graph, via {@link ExecutionGraph#failGlobal(Throwable)}.
+ */
+public class RestartAllStrategy extends FailoverStrategy {
+
+	/** The execution graph to recover */
+	private final ExecutionGraph executionGraph;
+
+	/**
+	 * Creates a new failover strategy that recovers from failures by restarting all tasks
+	 * of the execution graph.
+	 * 
+	 * @param executionGraph The execution graph to handle.
+	 */
+	public RestartAllStrategy(ExecutionGraph executionGraph) {
+		this.executionGraph = checkNotNull(executionGraph);
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public void onTaskFailure(Execution taskExecution, Throwable cause) {
+		// this strategy makes every task failure a global failure
+		executionGraph.failGlobal(cause);
+	}
+
+	@Override
+	public void notifyNewVertices(List<ExecutionJobVertex> newJobVerticesTopological) {
+		// nothing to do
+	}
+
+	@Override
+	public String getStrategyName() {
+		return "full graph restart";
+	}
+
+	// ------------------------------------------------------------------------
+	//  factory
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Factory that instantiates the RestartAllStrategy.
+	 */
+	public static class Factory implements FailoverStrategy.Factory {
+
+		@Override
+		public FailoverStrategy create(ExecutionGraph executionGraph) {
+			return new RestartAllStrategy(executionGraph);
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java
new file mode 100644
index 0000000000000..0a449b83d2e16
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.failover;
+
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.SimpleCounter;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.executiongraph.GlobalModVersionMismatch;
+import org.apache.flink.runtime.executiongraph.IntermediateResult;
+import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Simple failover strategy that restarts each task individually.
+ * This strategy is only applicable if the entire job consists unconnected
+ * tasks, meaning each task is its own component.
+ */
+public class RestartIndividualStrategy extends FailoverStrategy {
+
+	private static final Logger LOG = LoggerFactory.getLogger(RestartIndividualStrategy.class);
+
+	// ------------------------------------------------------------------------
+
+	/** The execution graph to recover */
+	private final ExecutionGraph executionGraph;
+
+	/** The executor that executes restart callbacks */
+	private final Executor callbackExecutor;
+
+	private final SimpleCounter numTaskFailures;
+
+	/**
+	 * Creates a new failover strategy that recovers from failures by restarting all tasks
+	 * of the execution graph.
+	 * 
+	 * <p>The strategy will use the ExecutionGraph's future executor for callbacks.
+	 * 
+	 * @param executionGraph The execution graph to handle.
+	 */
+	public RestartIndividualStrategy(ExecutionGraph executionGraph) {
+		this(executionGraph, executionGraph.getFutureExecutor());
+	}
+
+	/**
+	 * Creates a new failover strategy that recovers from failures by restarting all tasks
+	 * of the execution graph.
+	 *
+	 * @param executionGraph The execution graph to handle.
+	 * @param callbackExecutor The executor that executes restart callbacks
+	 */
+	public RestartIndividualStrategy(ExecutionGraph executionGraph, Executor callbackExecutor) {
+		this.executionGraph = checkNotNull(executionGraph);
+		this.callbackExecutor = checkNotNull(callbackExecutor);
+
+		this.numTaskFailures = new SimpleCounter();
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public void onTaskFailure(Execution taskExecution, Throwable cause) {
+
+		// to better handle the lack of resources (potentially by a scale-in), we
+		// make failures due to missing resources global failures 
+		if (cause instanceof NoResourceAvailableException) {
+			LOG.info("Not enough resources to schedule {} - triggering full recovery.", taskExecution);
+			executionGraph.failGlobal(cause);
+			return;
+		}
+
+		LOG.info("Recovering task failure for {} (#{}) via individual restart.", 
+				taskExecution.getVertex().getTaskNameWithSubtaskIndex(), taskExecution.getAttemptNumber());
+
+		numTaskFailures.inc();
+
+		// trigger the restart once the task has reached its terminal state
+		// Note: currently all tasks passed here are already in their terminal state,
+		//       so we could actually avoid the future. We use it anyways because it is cheap and
+		//       it helps to support better testing
+		final Future<ExecutionState> terminationFuture = taskExecution.getTerminationFuture();
+
+		final ExecutionVertex vertexToRecover = taskExecution.getVertex(); 
+		final long globalModVersion = taskExecution.getGlobalModVersion();
+
+		terminationFuture.thenAcceptAsync(new AcceptFunction<ExecutionState>() {
+			@Override
+			public void accept(ExecutionState value) {
+				try {
+					long createTimestamp = System.currentTimeMillis();
+					Execution newExecution = vertexToRecover.resetForNewExecution(createTimestamp, globalModVersion);
+					newExecution.scheduleForExecution();
+				}
+				catch (GlobalModVersionMismatch e) {
+					// this happens if a concurrent global recovery happens. simply do nothing.
+				}
+				catch (Exception e) {
+					executionGraph.failGlobal(
+							new Exception("Error during fine grained recovery - triggering full recovery", e));
+				}
+			}
+		}, callbackExecutor);
+	}
+
+	@Override
+	public void notifyNewVertices(List<ExecutionJobVertex> newJobVerticesTopological) {
+		// we validate here that the vertices are in fact not connected to
+		// any other vertices
+		for (ExecutionJobVertex ejv : newJobVerticesTopological) {
+			List<IntermediateResult> inputs = ejv.getInputs();
+			IntermediateResult[] outputs = ejv.getProducedDataSets();
+
+			if ((inputs != null && inputs.size() > 0) || (outputs != null && outputs.length > 0)) {
+				throw new FlinkRuntimeException("Incompatible failover strategy - strategy '" + 
+						getStrategyName() + "' can only handle jobs with only disconnected tasks.");
+			}
+		}
+	}
+
+	@Override
+	public String getStrategyName() {
+		return "Individual Task Restart";
+	}
+
+	@Override
+	public void registerMetrics(MetricGroup metricGroup) {
+		metricGroup.counter("task_failures", numTaskFailures);
+	}
+
+	// ------------------------------------------------------------------------
+	//  factory
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Factory that instantiates the RestartAllStrategy.
+	 */
+	public static class Factory implements FailoverStrategy.Factory {
+
+		@Override
+		public RestartIndividualStrategy create(ExecutionGraph executionGraph) {
+			return new RestartIndividualStrategy(executionGraph);
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/metrics/NumberOfFullRestartsGauge.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/metrics/NumberOfFullRestartsGauge.java
new file mode 100644
index 0000000000000..05a641468e38c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/metrics/NumberOfFullRestartsGauge.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.metrics;
+
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Gauge which returns the number of full restarts.
+ */
+public class NumberOfFullRestartsGauge implements Gauge<Long> {
+
+	public static final String METRIC_NAME = "fullRestarts";
+
+	// ------------------------------------------------------------------------
+
+	private final ExecutionGraph eg;
+
+	public NumberOfFullRestartsGauge(ExecutionGraph executionGraph) {
+		this.eg = checkNotNull(executionGraph);
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public Long getValue() {
+		return eg.getNumberOfFullRestarts();
+	}
+}
\ No newline at end of file
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 99dbc8692c28f..e60ff77983bf5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -376,7 +376,7 @@ public void run() {
 					executionGraph.scheduleForExecution();
 				}
 				catch (Throwable t) {
-					executionGraph.fail(t);
+					executionGraph.failGlobal(t);
 				}
 			}
 		});
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index b01ddc086b166..479ec51d12368 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -1345,7 +1345,7 @@ class JobManager(
           currentJobs.remove(jobId)
 
           if (executionGraph != null) {
-            executionGraph.fail(t)
+            executionGraph.failGlobal(t)
           }
 
           val rt: Throwable = if (t.isInstanceOf[JobExecutionException]) {
@@ -1426,7 +1426,7 @@ class JobManager(
           }
         } catch {
           case t: Throwable => try {
-            executionGraph.fail(t)
+            executionGraph.failGlobal(t)
           } catch {
             case tt: Throwable =>
               log.error("Error while marking ExecutionGraph as failed.", tt)
@@ -1837,7 +1837,7 @@ class JobManager(
         job =>
           future {
             // Fail the execution graph
-            job._1.fail(new IllegalStateException("Another JobManager removed the job from " +
+            job._1.failGlobal(new IllegalStateException("Another JobManager removed the job from " +
               "ZooKeeper."))
           }(context.dispatcher)
       )
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index 41b0e35d84147..9250634834f8c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -3135,6 +3135,7 @@ private static ExecutionVertex mockExecutionVertex(
 			vertex,
 			1,
 			1L,
+			1L,
 			Time.milliseconds(500L)
 		));
 		when(exec.getAttemptId()).thenReturn(attemptID);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
index 2e0bd762d3035..99a38151fd998 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
@@ -90,7 +90,7 @@ public void testCoordinatorShutsDownOnFailure() {
 
 			FailingBlockingInvokable.unblock();
 
-			graph.waitUntilFinished();
+			graph.waitUntilTerminal();
 			
 			// verify that the coordinator was shut down
 			CheckpointCoordinator coord = graph.getCheckpointCoordinator();
@@ -149,7 +149,7 @@ public void testCoordinatorShutsDownOnSuccess() {
 
 			BlockingInvokable.unblock();
 			
-			graph.waitUntilFinished();
+			graph.waitUntilTerminal();
 
 			// verify that the coordinator was shut down
 			CheckpointCoordinator coord = graph.getCheckpointCoordinator();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
index 1f038bd274743..a524e5cc3eed8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
@@ -25,6 +25,7 @@
 import org.apache.flink.runtime.blob.BlobKey;
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.failover.RestartAllStrategy;
 import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -56,7 +57,7 @@ public void testShutdownCheckpointCoordinator() throws Exception {
 		CompletedCheckpointStore store = mock(CompletedCheckpointStore.class);
 
 		ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
-		graph.fail(new Exception("Test Exception"));
+		graph.failGlobal(new Exception("Test Exception"));
 
 		verify(counter, times(1)).shutdown(JobStatus.FAILED);
 		verify(store, times(1)).shutdown(eq(JobStatus.FAILED));
@@ -91,6 +92,7 @@ private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
 			new SerializedValue<>(new ExecutionConfig()),
 			Time.days(1L),
 			new NoRestartStrategy(),
+			new RestartAllStrategy.Factory(),
 			Collections.<BlobKey>emptyList(),
 			Collections.<URL>emptyList(),
 			new Scheduler(TestingUtils.defaultExecutionContext()),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
index 866f55c5317a4..a739e14adb81b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
@@ -24,11 +24,9 @@
 import static org.junit.Assert.fail;
 import static org.junit.Assert.assertNotEquals;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -341,50 +339,6 @@ public void testRegistrationOfExecutionsCanceled() {
 		}
 	}
 
-	@Test
-	public void testRegistrationOfExecutionsFailingFinalize() {
-		try {
-
-			final JobVertexID jid1 = new JobVertexID();
-			final JobVertexID jid2 = new JobVertexID();
-
-			JobVertex v1 = new FailingFinalizeJobVertex("v1", jid1);
-			JobVertex v2 = new JobVertex("v2", jid2);
-
-			Map<ExecutionAttemptID, Execution> executions = setupExecution(v1, 6, v2, 4).f1;
-
-			List<Execution> execList = new ArrayList<Execution>();
-			execList.addAll(executions.values());
-			// sort executions by job vertex. Failing job vertex first
-			Collections.sort(execList, new Comparator<Execution>() {
-				@Override
-				public int compare(Execution o1, Execution o2) {
-					return o1.getVertex().getTaskNameWithSubtaskIndex().compareTo(o2.getVertex().getTaskNameWithSubtaskIndex());
-				}
-			});
-
-			int cnt = 0;
-			for (Execution e : execList) {
-				cnt++;
-				e.markFinished();
-				if (cnt <= 6) {
-					// the last execution of the first job vertex triggers the failing finalize hook
-					assertEquals(ExecutionState.FINISHED, e.getState());
-				}
-				else {
-					// all following executions should be canceled
-					assertEquals(ExecutionState.CANCELED, e.getState());
-				}
-			}
-
-			assertEquals(0, executions.size());
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
-	}
-
 	@Test
 	/**
 	 * Tests that a blocking batch job fails if there are not enough resources left to schedule the
@@ -473,24 +427,6 @@ public void testSettingMaxNumberOfCheckpointsToRetain() throws Exception {
 			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
 	}
 
-	@Test
-	public void testSettingIllegalMaxNumberOfCheckpointsToRetain() throws Exception {
-
-		final int negativeMaxNumberOfCheckpointsToRetain = -10;
-
-		final Configuration jobManagerConfig = new Configuration();
-		jobManagerConfig.setInteger(CoreOptions.MAX_RETAINED_CHECKPOINTS,
-			negativeMaxNumberOfCheckpointsToRetain);
-
-		final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);
-
-		assertNotEquals(negativeMaxNumberOfCheckpointsToRetain,
-			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
-
-		assertEquals(CoreOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
-			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
-	}
-
 	private Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> setupExecution(JobVertex v1, int dop1, JobVertex v2, int dop2) throws Exception {
 		final JobID jobId = new JobID();
 
@@ -503,24 +439,24 @@ private Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> setupExecutio
 		Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
 		for (int i = 0; i < dop1 + dop2; i++) {
 			scheduler.newInstanceAvailable(
-				ExecutionGraphTestUtils.getInstance(
-					new ActorTaskManagerGateway(
-						new ExecutionGraphTestUtils.SimpleActorGateway(
-							TestingUtils.directExecutionContext()))));
+					ExecutionGraphTestUtils.getInstance(
+							new ActorTaskManagerGateway(
+									new ExecutionGraphTestUtils.SimpleActorGateway(
+											TestingUtils.directExecutionContext()))));
 		}
 
 		// execution graph that executes actions synchronously
 		ExecutionGraph eg = new ExecutionGraph(
-			new DirectScheduledExecutorService(),
-			TestingUtils.defaultExecutor(),
-			jobId, 
-			"some job", 
-			new Configuration(), 
-			new SerializedValue<>(new ExecutionConfig()),
-			AkkaUtils.getDefaultTimeout(),
-			new NoRestartStrategy(),
-			scheduler);
-		
+				new DirectScheduledExecutorService(),
+				TestingUtils.defaultExecutor(),
+				jobId,
+				"some job",
+				new Configuration(),
+				new SerializedValue<>(new ExecutionConfig()),
+				AkkaUtils.getDefaultTimeout(),
+				new NoRestartStrategy(),
+				scheduler);
+
 		eg.setQueuedSchedulingAllowed(false);
 
 		List<JobVertex> ordered = Arrays.asList(v1, v2);
@@ -537,13 +473,27 @@ private Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> setupExecutio
 		return new Tuple2<>(eg, executions);
 	}
 
+	@Test
+	public void testSettingIllegalMaxNumberOfCheckpointsToRetain() throws Exception {
+
+		final int negativeMaxNumberOfCheckpointsToRetain = -10;
+
+		final Configuration jobManagerConfig = new Configuration();
+		jobManagerConfig.setInteger(CoreOptions.MAX_RETAINED_CHECKPOINTS,
+			negativeMaxNumberOfCheckpointsToRetain);
+
+		final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);
+
+		assertNotEquals(negativeMaxNumberOfCheckpointsToRetain,
+			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
+
+		assertEquals(CoreOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
+			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
+	}
+
 	@SuppressWarnings("serial")
 	public static class FailingFinalizeJobVertex extends JobVertex {
 
-		public FailingFinalizeJobVertex(String name) {
-			super(name);
-		}
-
 		public FailingFinalizeJobVertex(String name, JobVertexID id) {
 			super(name, id);
 		}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
index 97127c75ec795..6b5ceaef0479c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
@@ -22,7 +22,6 @@
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.JobException;
-import org.apache.flink.runtime.blob.BlobKey;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
@@ -52,9 +51,7 @@
 import org.mockito.Matchers;
 
 import java.io.IOException;
-import java.net.URL;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -133,10 +130,7 @@ public void testExecutionGraphRestartTimeMetric() throws JobException, IOExcepti
 				new SerializedValue<ExecutionConfig>(null),
 				timeout,
 				testingRestartStrategy,
-				Collections.<BlobKey>emptyList(),
-				Collections.<URL>emptyList(),
-				scheduler,
-				getClass().getClassLoader());
+				scheduler);
 
 			RestartTimeGauge restartingTime = new RestartTimeGauge(executionGraph);
 
@@ -242,8 +236,8 @@ public void testExecutionGraphRestartTimeMetric() throws JobException, IOExcepti
 
 			// now lets fail the job while it is in restarting and see whether the restarting time then stops to increase
 			// for this to work, we have to use a SuppressRestartException
-			executionGraph.fail(new SuppressRestartsException(new Exception()));
-
+			executionGraph.failGlobal(new SuppressRestartsException(new Exception()));
+	
 			assertEquals(JobStatus.FAILED, executionGraph.getState());
 
 			previousRestartingTime = restartingTime.getValue();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
index 1ebfcac80152e..eeb6c69b0c5cf 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
@@ -66,9 +66,8 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
+
 
 public class ExecutionGraphRestartTest extends TestLogger {
 
@@ -271,12 +270,12 @@ public void testFailWhileRestarting() throws Exception {
 		assertEquals(JobStatus.RESTARTING, executionGraph.getState());
 
 		// The restarting should not fail with an ordinary exception
-		executionGraph.fail(new Exception("Test exception"));
+		executionGraph.failGlobal(new Exception("Test exception"));
 
 		assertEquals(JobStatus.RESTARTING, executionGraph.getState());
 
 		// but it should fail when sending a SuppressRestartsException
-		executionGraph.fail(new SuppressRestartsException(new Exception("Test exception")));
+		executionGraph.failGlobal(new SuppressRestartsException(new Exception("Test exception")));
 
 		assertEquals(JobStatus.FAILED, executionGraph.getState());
 
@@ -298,7 +297,7 @@ public void testCancelWhileFailing() throws Exception {
 			vertex.getCurrentExecutionAttempt().switchToRunning();
 		}
 
-		graph.fail(new Exception("test"));
+		graph.failGlobal(new Exception("test"));
 
 		assertEquals(JobStatus.FAILING, graph.getState());
 
@@ -330,7 +329,7 @@ public void testFailWhileCanceling() throws Exception {
 
 		assertEquals(JobStatus.CANCELLING, graph.getState());
 
-		graph.fail(new Exception("test"));
+		graph.failGlobal(new Exception("test"));
 
 		assertEquals(JobStatus.FAILING, graph.getState());
 
@@ -344,8 +343,7 @@ public void testFailWhileCanceling() throws Exception {
 
 	@Test
 	public void testNoRestartOnSuppressException() throws Exception {
-		Tuple2<ExecutionGraph, Instance> executionGraphInstanceTuple = createSpyExecutionGraph(new FixedDelayRestartStrategy(1, 1000));
-		ExecutionGraph eg = executionGraphInstanceTuple.f0;
+		final ExecutionGraph eg = createExecutionGraph(new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0)).f0;
 
 		// Fail with unrecoverable Exception
 		eg.getAllExecutionVertices().iterator().next().fail(
@@ -357,21 +355,10 @@ public void testNoRestartOnSuppressException() throws Exception {
 			vertex.getCurrentExecutionAttempt().cancelingComplete();
 		}
 
-		FiniteDuration timeout = new FiniteDuration(2, TimeUnit.MINUTES);
-
-		// Wait for async restart
-		Deadline deadline = timeout.fromNow();
-		while (deadline.hasTimeLeft() && eg.getState() != JobStatus.FAILED) {
-			Thread.sleep(100);
-		}
-
+		eg.waitUntilTerminal();
 		assertEquals(JobStatus.FAILED, eg.getState());
 
-		// No restart
-		verify(eg, never()).restart();
-
 		RestartStrategy restartStrategy = eg.getRestartStrategy();
-
 		assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);
 
 		assertEquals(0, ((FixedDelayRestartStrategy) restartStrategy).getCurrentRestartAttempt());
@@ -443,7 +430,7 @@ public void testFailingExecutionAfterRestart() throws Exception {
 
 	/**
 	 * Tests that a graph is not restarted after cancellation via a call to
-	 * {@link ExecutionGraph#fail(Throwable)}. This can happen when a slot is
+	 * {@link ExecutionGraph#failGlobal(Throwable)}. This can happen when a slot is
 	 * released concurrently with cancellation.
 	 */
 	@Test
@@ -490,7 +477,7 @@ public void testFailExecutionAfterCancel() throws Exception {
 
 	/**
 	 * Tests that it is possible to fail a graph via a call to
-	 * {@link ExecutionGraph#fail(Throwable)} after cancellation.
+	 * {@link ExecutionGraph#failGlobal(Throwable)} after cancellation.
 	 */
 	@Test
 	public void testFailExecutionGraphAfterCancel() throws Exception {
@@ -523,7 +510,7 @@ public void testFailExecutionGraphAfterCancel() throws Exception {
 		eg.cancel();
 		assertEquals(JobStatus.CANCELLING, eg.getState());
 
-		eg.fail(new Exception("Test Exception"));
+		eg.failGlobal(new Exception("Test Exception"));
 		assertEquals(JobStatus.FAILING, eg.getState());
 
 		Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java
index c8a04228cfb9d..1eecd4a4f68f3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java
@@ -61,6 +61,7 @@
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.*;
 import static org.mockito.Mockito.*;
@@ -300,8 +301,6 @@ public void testOneSlotFailureAbortsDeploy() throws Exception {
 		slotProvider.addSlots(targetVertex.getID(), targetFutures);
 
 		final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
-		TerminalJobStatusListener testListener = new TerminalJobStatusListener();
-		eg.registerJobStatusListener(testListener);
 
 		//
 		//  we complete some of the futures
@@ -322,7 +321,7 @@ public void testOneSlotFailureAbortsDeploy() throws Exception {
 		sourceFutures[1].completeExceptionally(new TestRuntimeException());
 
 		// wait until the job failed as a whole
-		testListener.waitForTerminalState(2000);
+		eg.getTerminationFuture().get(2000, TimeUnit.MILLISECONDS);
 
 		// wait until all slots are back
 		verify(slotOwner, new Timeout(2000, times(6))).returnAllocatedSlot(any(Slot.class));
@@ -371,8 +370,6 @@ public void testTimeoutForSlotAllocation() throws Exception {
 		slotProvider.addSlots(vertex.getID(), slotFutures);
 
 		final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider, Time.milliseconds(20));
-		final TerminalJobStatusListener statusListener = new TerminalJobStatusListener();
-		eg.registerJobStatusListener(statusListener);
 
 		//  we complete one future
 		slotFutures[1].complete(slots[1]);
@@ -388,7 +385,7 @@ public void testTimeoutForSlotAllocation() throws Exception {
 
 		// since future[0] is still missing the while operation must time out
 		// we have no restarts allowed, so the job will go terminal
-		statusListener.waitForTerminalState(2000);
+		eg.getTerminationFuture().get(2000, TimeUnit.MILLISECONDS);
 
 		// wait until all slots are back
 		verify(slotOwner, new Timeout(2000, times(2))).returnAllocatedSlot(any(Slot.class));
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
deleted file mode 100644
index 27844c16f4734..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.executiongraph;
-
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.StoppingException;
-import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.execution.SuppressRestartsException;
-import org.apache.flink.runtime.executiongraph.restart.InfiniteDelayRestartStrategy;
-import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
-import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
-import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.jobgraph.DistributionPattern;
-import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.util.SerializedValue;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.powermock.api.mockito.PowerMockito;
-
-import scala.concurrent.duration.FiniteDuration;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.same;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(ExecutionGraph.class)
-public class ExecutionGraphSignalsTest {
-	private ExecutionJobVertex[] mockEJV = new ExecutionJobVertex[5];
-	private int[] dop = new int[] { 5, 7, 2, 11, 4 };
-	private ExecutionVertex[][] mockEV = new ExecutionVertex[mockEJV.length][];
-	private ExecutionGraph eg;
-	private Field f;
-
-	@Before
-	public void prepare() throws Exception {
-		final JobID jobId = new JobID();
-		final String jobName = "Test Job Sample Name";
-		final Configuration cfg = new Configuration();
-
-
-		assert (mockEJV.length == 5);
-		JobVertex v1 = new JobVertex("vertex1");
-		JobVertex v2 = new JobVertex("vertex2");
-		JobVertex v3 = new JobVertex("vertex3");
-		JobVertex v4 = new JobVertex("vertex4");
-		JobVertex v5 = new JobVertex("vertex5");
-
-		for(int i = 0; i < mockEJV.length; ++i) {
-			mockEJV[i] = mock(ExecutionJobVertex.class);
-
-			this.mockEV[i] = new ExecutionVertex[dop[i]];
-			for (int j = 0; j < dop[i]; ++j) {
-				this.mockEV[i][j] = mock(ExecutionVertex.class);
-			}
-
-			when(mockEJV[i].getProducedDataSets()).thenReturn(new IntermediateResult[0]);
-			when(mockEJV[i].getTaskVertices()).thenReturn(this.mockEV[i]);
-		}
-
-		PowerMockito
-			.whenNew(ExecutionJobVertex.class)
-			.withArguments(any(ExecutionGraph.class), same(v1), any(Integer.class).intValue(),
-				any(FiniteDuration.class), any(Long.class).longValue()).thenReturn(mockEJV[0]);
-		PowerMockito
-			.whenNew(ExecutionJobVertex.class)
-			.withArguments(any(ExecutionGraph.class), same(v2), any(Integer.class).intValue(),
-				any(FiniteDuration.class), any(Long.class).longValue()).thenReturn(mockEJV[1]);
-		PowerMockito
-			.whenNew(ExecutionJobVertex.class)
-			.withArguments(any(ExecutionGraph.class), same(v3), any(Integer.class).intValue(),
-				any(FiniteDuration.class), any(Long.class).longValue()).thenReturn(mockEJV[2]);
-		PowerMockito
-			.whenNew(ExecutionJobVertex.class)
-			.withArguments(any(ExecutionGraph.class), same(v4), any(Integer.class).intValue(),
-				any(FiniteDuration.class), any(Long.class).longValue()).thenReturn(mockEJV[3]);
-		PowerMockito
-			.whenNew(ExecutionJobVertex.class)
-			.withArguments(any(ExecutionGraph.class), same(v5), any(Integer.class).intValue(),
-				any(FiniteDuration.class), any(Long.class).longValue()).thenReturn(mockEJV[4]);
-
-		v1.setParallelism(dop[0]);
-		v2.setParallelism(dop[1]);
-		v3.setParallelism(dop[2]);
-		v4.setParallelism(dop[3]);
-		v5.setParallelism(dop[4]);
-
-		v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
-		mockNumberOfInputs(1,0);
-		v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
-		mockNumberOfInputs(3,1);
-		v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
-		mockNumberOfInputs(3,2);
-		v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
-		mockNumberOfInputs(4,3);
-		v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
-		mockNumberOfInputs(4,2);
-
-		List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
-
-		eg = new ExecutionGraph(
-			TestingUtils.defaultExecutor(),
-			TestingUtils.defaultExecutor(),
-			jobId,
-			jobName,
-			cfg,
-			new SerializedValue<>(new ExecutionConfig()),
-			AkkaUtils.getDefaultTimeout(),
-			new NoRestartStrategy(),
-			new Scheduler(TestingUtils.defaultExecutionContext()));
-		eg.attachJobGraph(ordered);
-
-		f = eg.getClass().getDeclaredField("state");
-		f.setAccessible(true);
-	}
-
-	private void mockNumberOfInputs(int nodeIndex, int predecessorIndex) {
-		for(int j = 0; j < dop[nodeIndex]; ++j) {
-			when(mockEV[nodeIndex][j].getNumberOfInputs()).thenReturn(dop[predecessorIndex]);
-		}
-	}
-
-	@Test
-	public void testCancel() throws Exception {
-		assertEquals(JobStatus.CREATED, eg.getState());
-		eg.cancel();
-
-		verifyCancel(1);
-
-		f.set(eg, JobStatus.RUNNING);
-		eg.cancel();
-
-		verifyCancel(2);
-		assertEquals(JobStatus.CANCELLING, eg.getState());
-
-		eg.cancel();
-
-		verifyCancel(2);
-		assertEquals(JobStatus.CANCELLING, eg.getState());
-
-		f.set(eg, JobStatus.CANCELED);
-		eg.cancel();
-
-		verifyCancel(2);
-		assertEquals(JobStatus.CANCELED, eg.getState());
-
-		f.set(eg, JobStatus.FAILED);
-		eg.cancel();
-
-		verifyCancel(2);
-		assertEquals(JobStatus.FAILED, eg.getState());
-
-		f.set(eg, JobStatus.FAILING);
-		eg.cancel();
-
-		verifyCancel(2);
-		assertEquals(JobStatus.CANCELLING, eg.getState());
-
-		f.set(eg, JobStatus.FINISHED);
-		eg.cancel();
-
-		verifyCancel(2);
-		assertEquals(JobStatus.FINISHED, eg.getState());
-
-		f.set(eg, JobStatus.RESTARTING);
-		eg.cancel();
-
-		verifyCancel(2);
-		assertEquals(JobStatus.CANCELED, eg.getState());
-	}
-
-	private void verifyCancel(int times) {
-		for (int i = 0; i < mockEJV.length; ++i) {
-			verify(mockEJV[i], times(times)).cancel();
-		}
-	}
-
-	/**
-	 * Tests that suspend cancels the ExecutionJobVertices and transitions to SUSPENDED state.
-	 * Tests also that one cannot leave the SUSPENDED state to enter a terminal state.
-	 */
-	@Test
-	public void testSuspend() throws Exception {
-		assertEquals(JobStatus.CREATED, eg.getState());
-		Exception testException = new Exception("Test exception");
-
-		eg.suspend(testException);
-
-		verifyCancel(1);
-		assertEquals(JobStatus.SUSPENDED, eg.getState());
-
-		f.set(eg, JobStatus.RUNNING);
-
-		eg.suspend(testException);
-
-		verifyCancel(2);
-		assertEquals(JobStatus.SUSPENDED, eg.getState());
-
-		f.set(eg, JobStatus.FAILING);
-
-		eg.suspend(testException);
-
-		verifyCancel(3);
-		assertEquals(JobStatus.SUSPENDED, eg.getState());
-
-		f.set(eg, JobStatus.CANCELLING);
-
-		eg.suspend(testException);
-
-		verifyCancel(4);
-		assertEquals(JobStatus.SUSPENDED, eg.getState());
-
-		f.set(eg, JobStatus.FAILED);
-
-		eg.suspend(testException);
-
-		verifyCancel(4);
-		assertEquals(JobStatus.FAILED, eg.getState());
-
-		f.set(eg, JobStatus.FINISHED);
-
-		eg.suspend(testException);
-
-		verifyCancel(4);
-		assertEquals(JobStatus.FINISHED, eg.getState());
-
-		f.set(eg, JobStatus.CANCELED);
-
-		eg.suspend(testException);
-
-		verifyCancel(4);
-		assertEquals(JobStatus.CANCELED, eg.getState());
-
-		f.set(eg, JobStatus.SUSPENDED);
-
-		eg.fail(testException);
-
-		assertEquals(JobStatus.SUSPENDED, eg.getState());
-
-		eg.cancel();
-
-		assertEquals(JobStatus.SUSPENDED, eg.getState());
-	}
-
-	// test that all source tasks receive STOP signal
-	// test that all non-source tasks do not receive STOP signal
-	@Test
-	public void testStop() throws Exception {
-		Field f = eg.getClass().getDeclaredField("isStoppable");
-		f.setAccessible(true);
-		f.set(eg, true);
-
-		eg.stop();
-
-		for (int i : new int[]{0,2}) {
-			for (int j = 0; j < mockEV[i].length; ++j) {
-				verify(mockEV[i][j], times(1)).stop();
-			}
-		}
-
-		for (int i : new int[]{1,3,4}) {
-			for (int j = 0; j < mockEV[i].length; ++j) {
-				verify(mockEV[i][j], times(0)).stop();
-			}
-		}
-	}
-
-	/**
-	 * Test that failing in state restarting will retrigger the restarting logic. This means that
-	 * it only goes into the state FAILED after the restart strategy says the job is no longer
-	 * restartable.
-	 */
-	@Test
-	public void testFailureWhileRestarting() throws IllegalAccessException, NoSuchFieldException, InterruptedException {
-		Field restartStrategyField = eg.getClass().getDeclaredField("restartStrategy");
-		restartStrategyField.setAccessible(true);
-
-		restartStrategyField.set(eg, new InfiniteDelayRestartStrategy(1));
-
-		f.set(eg, JobStatus.RESTARTING);
-
-		eg.fail(new Exception("Test"));
-
-		// we should restart since we have one restart attempt left
-		assertEquals(JobStatus.RESTARTING, eg.getState());
-
-		eg.fail(new Exception("Test"));
-
-		// after depleting all our restart attempts we should go into Failed
-		assertEquals(JobStatus.FAILED, eg.getState());
-	}
-
-	/**
-	 * Tests that a {@link SuppressRestartsException} in state RESTARTING stops the restarting
-	 * immediately and sets the execution graph's state to FAILED.
-	 */
-	@Test
-	public void testSuppressRestartFailureWhileRestarting() throws IllegalAccessException, NoSuchFieldException {
-		Field restartStrategyField = eg.getClass().getDeclaredField("restartStrategy");
-		restartStrategyField.setAccessible(true);
-
-		restartStrategyField.set(eg, new InfiniteDelayRestartStrategy());
-
-		f.set(eg, JobStatus.RESTARTING);
-
-		// suppress a possible restart
-		eg.fail(new SuppressRestartsException(new Exception("Test")));
-
-		assertEquals(JobStatus.FAILED, eg.getState());
-	}
-
-	/**
-	 * Tests that we can suspend a job when in state RESTARTING.
-	 */
-	@Test
-	public void testSuspendWhileRestarting() throws IllegalAccessException, NoSuchFieldException {
-		Field restartStrategyField = eg.getClass().getDeclaredField("restartStrategy");
-		restartStrategyField.setAccessible(true);
-
-		restartStrategyField.set(eg, new InfiniteDelayRestartStrategy());
-
-		f.set(eg, JobStatus.RESTARTING);
-
-		final Exception exception = new Exception("Suspended");
-
-		eg.suspend(exception);
-
-		assertEquals(JobStatus.SUSPENDED, eg.getState());
-
-		assertEquals(exception, eg.getFailureCause());
-	}
-
-	// STOP only supported if all sources are stoppable 
-	@Test(expected = StoppingException.class)
-	public void testStopBatching() throws StoppingException {
-		eg.stop();
-	}
-
-	/**
-	 * Tests that a failing scheduleOrUpdateConsumers call with a non-existing execution attempt
-	 * id, will not fail the execution graph.
-	 */
-	@Test
-	public void testFailingScheduleOrUpdateConsumers() throws IllegalAccessException {
-		IntermediateResultPartitionID intermediateResultPartitionId = new IntermediateResultPartitionID();
-		// The execution attempt id does not exist and thus the scheduleOrUpdateConsumers call
-		// should fail
-		ExecutionAttemptID producerId = new ExecutionAttemptID();
-		ResultPartitionID resultPartitionId = new ResultPartitionID(intermediateResultPartitionId, producerId);
-
-		f.set(eg, JobStatus.RUNNING);
-
-		assertEquals(JobStatus.RUNNING, eg.getState());
-
-		try {
-			eg.scheduleOrUpdateConsumers(resultPartitionId);
-			fail("Expected ExecutionGraphException.");
-		} catch (ExecutionGraphException e) {
-			// we've expected this exception to occur
-		}
-
-		assertEquals(JobStatus.RUNNING, eg.getState());
-	}
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphStopTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphStopTest.java
new file mode 100644
index 0000000000000..effe417764776
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphStopTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.StoppingException;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway;
+import org.apache.flink.runtime.instance.SimpleSlot;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
+import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+import org.apache.flink.runtime.testutils.StoppableInvokable;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Validates that stop() calls are handled correctly.
+ */
+public class ExecutionGraphStopTest extends TestLogger {
+
+	/**
+	 * Tests that STOP is only supported if all sources are stoppable
+	 */
+	@Test
+	public void testStopIfSourcesNotStoppable() throws Exception {
+		final ExecutionGraph graph = ExecutionGraphTestUtils.createSimpleTestGraph();
+
+		try {
+			graph.stop();
+			fail("exception expected");
+		}
+		catch (StoppingException e) {
+			// expected
+		}
+	}
+
+	/**
+	 * Validates that stop is only sent to the sources
+	 * 
+	 * This test build a simple job with two sources and two non-source vertices.
+	 */
+	@Test
+	public void testStop() throws Exception {
+		final int sourcePar1 = 11;
+		final int sourcePar2 = 7;
+
+		final JobVertex source1 = new JobVertex("source 1");
+		source1.setInvokableClass(StoppableInvokable.class);
+		source1.setParallelism(sourcePar1);
+
+		final JobVertex source2 = new JobVertex("source 2");
+		source2.setInvokableClass(StoppableInvokable.class);
+		source2.setParallelism(sourcePar2);
+
+		final JobVertex nonSource1 = new JobVertex("non-source-1");
+		nonSource1.setInvokableClass(NoOpInvokable.class);
+		nonSource1.setParallelism(10);
+
+		final JobVertex nonSource2 = new JobVertex("non-source-2");
+		nonSource2.setInvokableClass(NoOpInvokable.class);
+		nonSource2.setParallelism(10);
+
+		nonSource1.connectNewDataSetAsInput(source1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		nonSource1.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+		nonSource2.connectNewDataSetAsInput(nonSource1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+
+		final JobID jid = new JobID();
+		final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(
+				jid, source1, source2, nonSource1, nonSource2);
+
+		// we use different gateways for sources and non-sources to make sure the right ones
+		// get the RPC calls
+		final TaskManagerGateway sourceGateway = spy(new SimpleAckingTaskManagerGateway());
+		final TaskManagerGateway nonSourceGateway = spy(new SimpleAckingTaskManagerGateway());
+
+		// deploy source 1
+		for (ExecutionVertex ev : eg.getJobVertex(source1.getID()).getTaskVertices()) {
+			SimpleSlot slot = ExecutionGraphTestUtils.createMockSimpleSlot(jid, sourceGateway);
+			ev.getCurrentExecutionAttempt().deployToSlot(slot);
+		}
+
+		// deploy source 2
+		for (ExecutionVertex ev : eg.getJobVertex(source2.getID()).getTaskVertices()) {
+			SimpleSlot slot = ExecutionGraphTestUtils.createMockSimpleSlot(jid, sourceGateway);
+			ev.getCurrentExecutionAttempt().deployToSlot(slot);
+		}
+
+		// deploy non-source 1
+		for (ExecutionVertex ev : eg.getJobVertex(nonSource1.getID()).getTaskVertices()) {
+			SimpleSlot slot = ExecutionGraphTestUtils.createMockSimpleSlot(jid, nonSourceGateway);
+			ev.getCurrentExecutionAttempt().deployToSlot(slot);
+		}
+
+		// deploy non-source 2
+		for (ExecutionVertex ev : eg.getJobVertex(nonSource2.getID()).getTaskVertices()) {
+			SimpleSlot slot = ExecutionGraphTestUtils.createMockSimpleSlot(jid, nonSourceGateway);
+			ev.getCurrentExecutionAttempt().deployToSlot(slot);
+		}
+
+		eg.stop();
+
+		verify(sourceGateway, timeout(1000).times(sourcePar1 + sourcePar2)).stopTask(any(ExecutionAttemptID.class), any(Time.class));
+		verify(nonSourceGateway, times(0)).stopTask(any(ExecutionAttemptID.class), any(Time.class));
+
+		ExecutionGraphTestUtils.finishAllVertices(eg);
+	}
+
+	/**
+	 * Tests that the stopping RPC call is sent upon stopping requests.
+	 */
+	@Test
+	public void testStopRpc() throws Exception {
+		final JobID jid = new JobID();
+		final JobVertex vertex = new JobVertex("vertex");
+		vertex.setInvokableClass(NoOpInvokable.class);
+		vertex.setParallelism(5);
+
+		final ExecutionGraph graph = ExecutionGraphTestUtils.createSimpleTestGraph(jid, vertex);
+		final Execution exec = graph.getJobVertex(vertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();
+
+		final TaskManagerGateway gateway = mock(TaskManagerGateway.class);
+		when(gateway.submitTask(any(TaskDeploymentDescriptor.class), any(Time.class)))
+				.thenReturn(FlinkCompletableFuture.completed(Acknowledge.get()));
+		when(gateway.stopTask(any(ExecutionAttemptID.class), any(Time.class)))
+				.thenReturn(FlinkCompletableFuture.completed(Acknowledge.get()));
+
+		final SimpleSlot slot = ExecutionGraphTestUtils.createMockSimpleSlot(jid, gateway);
+
+		exec.deployToSlot(slot);
+		exec.switchToRunning();
+		assertEquals(ExecutionState.RUNNING, exec.getState());
+
+		exec.stop();
+		assertEquals(ExecutionState.RUNNING, exec.getState());
+
+		verify(gateway, times(1)).stopTask(any(ExecutionAttemptID.class), any(Time.class));
+
+		exec.markFinished();
+		assertEquals(ExecutionState.FINISHED, exec.getState());
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSuspendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSuspendTest.java
new file mode 100644
index 0000000000000..b3af005d3bee4
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSuspendTest.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.InfiniteDelayRestartStrategy;
+import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway;
+import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+/**
+ * Validates that suspending out of various states works correctly.
+ */
+public class ExecutionGraphSuspendTest {
+
+	/**
+	 * Going into SUSPENDED out of CREATED should immediately cancel everything and
+	 * not send out RPC calls.
+	 */
+	@Test
+	public void testSuspendedOutOfCreated() throws Exception {
+		final TaskManagerGateway gateway = spy(new SimpleAckingTaskManagerGateway());
+		final int parallelism = 10;
+		final ExecutionGraph eg = createExecutionGraph(gateway, parallelism);
+
+		assertEquals(JobStatus.CREATED, eg.getState());
+
+		// suspend
+
+		eg.suspend(new Exception("suspend"));
+
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+		validateAllVerticesInState(eg, ExecutionState.CANCELED);
+		validateCancelRpcCalls(gateway, 0);
+
+		ensureCannotLeaveSuspendedState(eg, gateway);
+	}
+
+	/**
+	 * Going into SUSPENDED out of DEPLOYING vertices should cancel all vertices once with RPC calls.
+	 */
+	@Test
+	public void testSuspendedOutOfDeploying() throws Exception {
+		final TaskManagerGateway gateway = spy(new SimpleAckingTaskManagerGateway());
+		final int parallelism = 10;
+		final ExecutionGraph eg = createExecutionGraph(gateway, parallelism);
+
+		eg.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		validateAllVerticesInState(eg, ExecutionState.DEPLOYING);
+
+		// suspend
+
+		eg.suspend(new Exception("suspend"));
+
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+		validateAllVerticesInState(eg, ExecutionState.CANCELING);
+		validateCancelRpcCalls(gateway, parallelism);
+
+		ensureCannotLeaveSuspendedState(eg, gateway);
+	}
+
+	/**
+	 * Going into SUSPENDED out of RUNNING vertices should cancel all vertices once with RPC calls.
+	 */
+	@Test
+	public void testSuspendedOutOfRunning() throws Exception {
+		final TaskManagerGateway gateway = spy(new SimpleAckingTaskManagerGateway());
+		final int parallelism = 10;
+		final ExecutionGraph eg = createExecutionGraph(gateway, parallelism);
+
+		eg.scheduleForExecution();
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		validateAllVerticesInState(eg, ExecutionState.RUNNING);
+
+		// suspend
+
+		eg.suspend(new Exception("suspend"));
+
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+		validateAllVerticesInState(eg, ExecutionState.CANCELING);
+		validateCancelRpcCalls(gateway, parallelism);
+
+		ensureCannotLeaveSuspendedState(eg, gateway);
+	}
+
+	/**
+	 * Suspending from FAILING goes to SUSPENDED and sends no additional RPC calls
+	 */
+	@Test
+	public void testSuspendedOutOfFailing() throws Exception {
+		final TaskManagerGateway gateway = spy(new SimpleAckingTaskManagerGateway());
+		final int parallelism = 10;
+		final ExecutionGraph eg = createExecutionGraph(gateway, parallelism);
+
+		eg.scheduleForExecution();
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		eg.failGlobal(new Exception("fail global"));
+
+		assertEquals(JobStatus.FAILING, eg.getState());
+		validateCancelRpcCalls(gateway, parallelism);
+
+		// suspend
+		eg.suspend(new Exception("suspend"));
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+
+		ExecutionGraphTestUtils.completeCancellingForAllVertices(eg);
+
+		ensureCannotLeaveSuspendedState(eg, gateway);
+	}
+
+	/**
+	 * Suspending from FAILED should do nothing.
+	 */
+	@Test
+	public void testSuspendedOutOfFailed() throws Exception {
+		final TaskManagerGateway gateway = spy(new SimpleAckingTaskManagerGateway());
+		final int parallelism = 10;
+		final ExecutionGraph eg = createExecutionGraph(gateway, parallelism);
+
+		eg.scheduleForExecution();
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		eg.failGlobal(new Exception("fail global"));
+
+		assertEquals(JobStatus.FAILING, eg.getState());
+		validateCancelRpcCalls(gateway, parallelism);
+
+		ExecutionGraphTestUtils.completeCancellingForAllVertices(eg);
+		assertEquals(JobStatus.FAILED, eg.getState());
+
+		// suspend
+		eg.suspend(new Exception("suspend"));
+
+		// still in failed state
+		assertEquals(JobStatus.FAILED, eg.getState());
+		validateCancelRpcCalls(gateway, parallelism);
+	}
+
+	/**
+	 * Suspending from CANCELING goes to SUSPENDED and sends no additional RPC calls. 
+	 */
+	@Test
+	public void testSuspendedOutOfCanceling() throws Exception {
+		final TaskManagerGateway gateway = spy(new SimpleAckingTaskManagerGateway());
+		final int parallelism = 10;
+		final ExecutionGraph eg = createExecutionGraph(gateway, parallelism);
+
+		eg.scheduleForExecution();
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		eg.cancel();
+
+		assertEquals(JobStatus.CANCELLING, eg.getState());
+		validateCancelRpcCalls(gateway, parallelism);
+
+		// suspend
+		eg.suspend(new Exception("suspend"));
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+
+		ExecutionGraphTestUtils.completeCancellingForAllVertices(eg);
+
+		ensureCannotLeaveSuspendedState(eg, gateway);
+	}
+
+	/**
+	 * Suspending from CANCELLED should do nothing.
+	 */
+	@Test
+	public void testSuspendedOutOfCanceled() throws Exception {
+		final TaskManagerGateway gateway = spy(new SimpleAckingTaskManagerGateway());
+		final int parallelism = 10;
+		final ExecutionGraph eg = createExecutionGraph(gateway, parallelism);
+
+		eg.scheduleForExecution();
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		eg.cancel();
+
+		assertEquals(JobStatus.CANCELLING, eg.getState());
+		validateCancelRpcCalls(gateway, parallelism);
+
+		ExecutionGraphTestUtils.completeCancellingForAllVertices(eg);
+		assertEquals(JobStatus.CANCELED, eg.getState());
+
+		// suspend
+		eg.suspend(new Exception("suspend"));
+
+		// still in failed state
+		assertEquals(JobStatus.CANCELED, eg.getState());
+		validateCancelRpcCalls(gateway, parallelism);
+	}
+
+	/**
+	 * Tests that we can suspend a job when in state RESTARTING.
+	 */
+	@Test
+	public void testSuspendWhileRestarting() throws Exception {
+		final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new InfiniteDelayRestartStrategy(10));
+		eg.scheduleForExecution();
+
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		eg.failGlobal(new Exception("test"));
+		assertEquals(JobStatus.FAILING, eg.getState());
+
+		ExecutionGraphTestUtils.completeCancellingForAllVertices(eg);
+		assertEquals(JobStatus.RESTARTING, eg.getState());
+
+		final Exception exception = new Exception("Suspended");
+
+		eg.suspend(exception);
+
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+
+		assertEquals(exception, eg.getFailureCause());
+	}
+
+	// ------------------------------------------------------------------------
+	//  utilities
+	// ------------------------------------------------------------------------
+
+	private static void ensureCannotLeaveSuspendedState(ExecutionGraph eg, TaskManagerGateway gateway) {
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+		reset(gateway);
+
+		eg.failGlobal(new Exception("fail"));
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+		verifyNoMoreInteractions(gateway);
+
+		eg.cancel();
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+		verifyNoMoreInteractions(gateway);
+
+		eg.suspend(new Exception("suspend again"));
+		assertEquals(JobStatus.SUSPENDED, eg.getState());
+		verifyNoMoreInteractions(gateway);
+
+		for (ExecutionVertex ev : eg.getAllExecutionVertices()) {
+			assertEquals(0, ev.getCurrentExecutionAttempt().getAttemptNumber());
+		}
+	}
+
+	private static void validateAllVerticesInState(ExecutionGraph eg, ExecutionState expected) {
+		for (ExecutionVertex ev : eg.getAllExecutionVertices()) {
+			assertEquals(expected, ev.getCurrentExecutionAttempt().getState());
+		}
+	}
+
+	private static void validateCancelRpcCalls(TaskManagerGateway gateway, int num) {
+		verify(gateway, times(num)).cancelTask(any(ExecutionAttemptID.class), any(Time.class));
+	}
+
+	private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
+		final JobID jobId = new JobID();
+
+		final JobVertex vertex = new JobVertex("vertex");
+		vertex.setInvokableClass(NoOpInvokable.class);
+		vertex.setParallelism(parallelism);
+
+		final SlotProvider slotProvider = new SimpleSlotProvider(jobId, parallelism, gateway);
+
+		return ExecutionGraphTestUtils.createSimpleTestGraph(
+				jobId,
+				slotProvider,
+				new FixedDelayRestartStrategy(0, 0),
+				vertex);
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
index 73838dc6c4524..0d7e389117e09 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
@@ -18,51 +18,158 @@
 
 package org.apache.flink.runtime.executiongraph;
 
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-
-import java.lang.reflect.Field;
-import java.net.InetAddress;
-import java.util.concurrent.ScheduledExecutorService;
-
 import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
+import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider;
 import org.apache.flink.runtime.instance.BaseTestingActorGateway;
 import org.apache.flink.runtime.instance.HardwareDescription;
 import org.apache.flink.runtime.instance.Instance;
-import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
-import org.apache.flink.runtime.messages.Acknowledge;
-import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.instance.SimpleSlot;
-import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
-import org.apache.flink.runtime.messages.TaskMessages.SubmitTask;
-import org.apache.flink.runtime.messages.TaskMessages.FailIntermediateResultPartitions;
+import org.apache.flink.runtime.jobmanager.slots.AllocatedSlot;
+import org.apache.flink.runtime.jobmanager.slots.SlotOwner;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.TaskMessages.CancelTask;
+import org.apache.flink.runtime.messages.TaskMessages.FailIntermediateResultPartitions;
+import org.apache.flink.runtime.messages.TaskMessages.SubmitTask;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
 import org.apache.flink.util.SerializedValue;
 
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import scala.concurrent.ExecutionContext;
 import scala.concurrent.ExecutionContext$;
 
+import java.lang.reflect.Field;
+import java.net.InetAddress;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+/**
+ * A collection of utility methods for testing the ExecutionGraph and its related classes. 
+ */
 public class ExecutionGraphTestUtils {
 
-	// --------------------------------------------------------------------------------------------
+	private static final Logger TEST_LOGGER = LoggerFactory.getLogger(ExecutionGraphTestUtils.class);
+
+	// ------------------------------------------------------------------------
+	//  reaching states
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Waits until the job has reached a certain state.
+	 * 
+	 * <p>This method is based on polling and might miss very fast state transitions!
+	 */
+	public static void waitUntilJobStatus(ExecutionGraph eg, JobStatus status, long maxWaitMillis) 
+			throws TimeoutException {
+
+		checkNotNull(eg);
+		checkNotNull(status);
+		checkArgument(maxWaitMillis >= 0);
+
+		// this is a poor implementation - we may want to improve it eventually
+		final long deadline = maxWaitMillis == 0 ? Long.MAX_VALUE : System.nanoTime() + (maxWaitMillis * 1_000_000);
+
+		while (eg.getState() != status && System.nanoTime() < deadline) {
+			try { 
+				Thread.sleep(2);
+			} catch (InterruptedException ignored) {}
+		}
+
+		if (System.nanoTime() >= deadline) {
+			throw new TimeoutException();
+		}
+	}
+
+	/**
+	 * Waits until the Execution has reached a certain state.
+	 *
+	 * <p>This method is based on polling and might miss very fast state transitions!
+	 */
+	public static void waitUntilExecutionState(Execution execution, ExecutionState state, long maxWaitMillis)
+			throws TimeoutException {
+
+		checkNotNull(execution);
+		checkNotNull(state);
+		checkArgument(maxWaitMillis >= 0);
+
+		// this is a poor implementation - we may want to improve it eventually
+		final long deadline = maxWaitMillis == 0 ? Long.MAX_VALUE : System.nanoTime() + (maxWaitMillis * 1_000_000);
+
+		while (execution.getState() != state && System.nanoTime() < deadline) {
+			try {
+				Thread.sleep(2);
+			} catch (InterruptedException ignored) {}
+		}
+
+		if (System.nanoTime() >= deadline) {
+			throw new TimeoutException();
+		}
+	}
+
+	/**
+	 * Takes all vertices in the given ExecutionGraph and switches their current
+	 * execution to RUNNING.
+	 */
+	public static void switchAllVerticesToRunning(ExecutionGraph eg) {
+		for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
+			vertex.getCurrentExecutionAttempt().switchToRunning();
+		}
+	}
+
+	/**
+	 * Takes all vertices in the given ExecutionGraph and switches their current
+	 * execution to RUNNING.
+	 */
+	public static void completeCancellingForAllVertices(ExecutionGraph eg) {
+		for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
+			vertex.getCurrentExecutionAttempt().cancelingComplete();
+		}
+	}
+
+	/**
+	 * Takes all vertices in the given ExecutionGraph and switches their current
+	 * execution to RUNNING.
+	 */
+	public static void finishAllVertices(ExecutionGraph eg) {
+		for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
+			vertex.getCurrentExecutionAttempt().markFinished();
+		}
+	}
+
+	// ------------------------------------------------------------------------
 	//  state modifications
-	// --------------------------------------------------------------------------------------------
+	// ------------------------------------------------------------------------
 	
 	public static void setVertexState(ExecutionVertex vertex, ExecutionState state) {
 		try {
@@ -89,21 +196,105 @@ public static void setVertexResource(ExecutionVertex vertex, SimpleSlot slot) {
 			throw new RuntimeException("Modifying the slot failed", e);
 		}
 	}
-	
-	public static void setGraphStatus(ExecutionGraph graph, JobStatus status) {
-		try {
-			Field f = ExecutionGraph.class.getDeclaredField("state");
-			f.setAccessible(true);
-			f.set(graph, status);
-		}
-		catch (Exception e) {
-			throw new RuntimeException("Modifying the status failed", e);
+
+	// ------------------------------------------------------------------------
+	//  Mocking Slots
+	// ------------------------------------------------------------------------
+
+	public static SimpleSlot createMockSimpleSlot(JobID jid, TaskManagerGateway gateway) {
+		final TaskManagerLocation location = new TaskManagerLocation(
+				ResourceID.generate(), InetAddress.getLoopbackAddress(), 6572);
+
+		final AllocatedSlot allocatedSlot = new AllocatedSlot(
+				new AllocationID(),
+				jid,
+				location,
+				0,
+				ResourceProfile.UNKNOWN,
+				gateway);
+
+		return new SimpleSlot(allocatedSlot, mock(SlotOwner.class), 0);
+	}
+
+	// ------------------------------------------------------------------------
+	//  Mocking ExecutionGraph
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Creates an execution graph with on job vertex of parallelism 10 that does no restarts.
+	 */
+	public static ExecutionGraph createSimpleTestGraph() throws Exception {
+		return createSimpleTestGraph(new NoRestartStrategy());
+	}
+
+	/**
+	 * Creates an execution graph with on job vertex of parallelism 10, using the given
+	 * restart strategy.
+	 */
+	public static ExecutionGraph createSimpleTestGraph(RestartStrategy restartStrategy) throws Exception {
+		JobVertex vertex = new JobVertex("vertex");
+		vertex.setInvokableClass(NoOpInvokable.class);
+		vertex.setParallelism(10);
+
+		return createSimpleTestGraph(new JobID(), restartStrategy, vertex);
+	}
+
+	/**
+	 * Creates an execution graph containing the given vertices.
+	 * 
+	 * <p>The execution graph uses {@link NoRestartStrategy} as the restart strategy.
+	 */
+	public static ExecutionGraph createSimpleTestGraph(JobID jid, JobVertex... vertices) throws Exception {
+		return createSimpleTestGraph(jid, new NoRestartStrategy(), vertices);
+	}
+
+	/**
+	 * Creates an execution graph containing the given vertices and the given restart strategy.
+	 */
+	public static ExecutionGraph createSimpleTestGraph(
+			JobID jid,
+			RestartStrategy restartStrategy,
+			JobVertex... vertices) throws Exception {
+
+		int numSlotsNeeded = 0;
+		for (JobVertex vertex : vertices) {
+			numSlotsNeeded += vertex.getParallelism();
 		}
+
+		SlotProvider slotProvider = new SimpleSlotProvider(jid, numSlotsNeeded);
+
+		return createSimpleTestGraph(jid, slotProvider, restartStrategy, vertices);
 	}
-	
-	// --------------------------------------------------------------------------------------------
+
+	public static ExecutionGraph createSimpleTestGraph(
+			JobID jid,
+			SlotProvider slotProvider,
+			RestartStrategy restartStrategy,
+			JobVertex... vertices) throws Exception {
+
+		checkNotNull(jid);
+		checkNotNull(restartStrategy);
+		checkNotNull(vertices);
+
+		return ExecutionGraphBuilder.buildGraph(
+				null,
+				new JobGraph(jid, "test job", vertices),
+				new Configuration(),
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				slotProvider,
+				ExecutionGraphTestUtils.class.getClassLoader(),
+				mock(CheckpointRecoveryFactory.class),
+				Time.seconds(10),
+				restartStrategy,
+				new UnregisteredMetricsGroup(),
+				1,
+				TEST_LOGGER);
+	}
+
+	// ------------------------------------------------------------------------
 	//  utility mocking methods
-	// --------------------------------------------------------------------------------------------
+	// ------------------------------------------------------------------------
 
 	public static Instance getInstance(final TaskManagerGateway gateway) throws Exception {
 		return getInstance(gateway, 1);
@@ -188,17 +379,7 @@ public static ExecutionJobVertex getExecutionVertex(
 			new NoRestartStrategy(),
 			new Scheduler(ExecutionContext$.MODULE$.fromExecutor(executor)));
 
-		ExecutionJobVertex ejv = spy(new ExecutionJobVertex(graph, ajv, 1,
-				AkkaUtils.getDefaultTimeout()));
-
-		Answer<Void> noop = new Answer<Void>() {
-			@Override
-			public Void answer(InvocationOnMock invocation) {
-				return null;
-			}
-		};
-
-		return ejv;
+		return spy(new ExecutionJobVertex(graph, ajv, 1, AkkaUtils.getDefaultTimeout()));
 	}
 	
 	public static ExecutionJobVertex getExecutionVertex(JobVertexID id) throws Exception {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphVariousFailuesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphVariousFailuesTest.java
new file mode 100644
index 0000000000000..0797ef91d0690
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphVariousFailuesTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.runtime.execution.SuppressRestartsException;
+import org.apache.flink.runtime.executiongraph.restart.InfiniteDelayRestartStrategy;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+
+public class ExecutionGraphVariousFailuesTest {
+
+	/**
+	 * Test that failing in state restarting will retrigger the restarting logic. This means that
+	 * it only goes into the state FAILED after the restart strategy says the job is no longer
+	 * restartable.
+	 */
+	@Test
+	public void testFailureWhileRestarting() throws Exception {
+		final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new InfiniteDelayRestartStrategy(2));
+		eg.scheduleForExecution();
+
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		eg.failGlobal(new Exception("Test 1"));
+		assertEquals(JobStatus.FAILING, eg.getState());
+		ExecutionGraphTestUtils.completeCancellingForAllVertices(eg);
+
+		// we should restart since we have two restart attempts left
+		assertEquals(JobStatus.RESTARTING, eg.getState());
+
+		eg.failGlobal(new Exception("Test 2"));
+
+		// we should restart since we have one restart attempts left
+		assertEquals(JobStatus.RESTARTING, eg.getState());
+
+		eg.failGlobal(new Exception("Test 3"));
+
+		// after depleting all our restart attempts we should go into Failed
+		assertEquals(JobStatus.FAILED, eg.getState());
+	}
+
+	/**
+	 * Tests that a {@link SuppressRestartsException} in state RESTARTING stops the restarting
+	 * immediately and sets the execution graph's state to FAILED.
+	 */
+	@Test
+	public void testSuppressRestartFailureWhileRestarting() throws Exception {
+		final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new InfiniteDelayRestartStrategy(10));
+		eg.scheduleForExecution();
+
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		eg.failGlobal(new Exception("test"));
+		assertEquals(JobStatus.FAILING, eg.getState());
+
+		ExecutionGraphTestUtils.completeCancellingForAllVertices(eg);
+		assertEquals(JobStatus.RESTARTING, eg.getState());
+
+		// suppress a possible restart
+		eg.failGlobal(new SuppressRestartsException(new Exception("Test")));
+
+		assertEquals(JobStatus.FAILED, eg.getState());
+	}
+
+	/**
+	 * Tests that a failing scheduleOrUpdateConsumers call with a non-existing execution attempt
+	 * id, will not fail the execution graph.
+	 */
+	@Test
+	public void testFailingScheduleOrUpdateConsumers() throws Exception {
+		final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new InfiniteDelayRestartStrategy(10));
+		eg.scheduleForExecution();
+
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		IntermediateResultPartitionID intermediateResultPartitionId = new IntermediateResultPartitionID();
+		ExecutionAttemptID producerId = new ExecutionAttemptID();
+		ResultPartitionID resultPartitionId = new ResultPartitionID(intermediateResultPartitionId, producerId);
+
+		// The execution attempt id does not exist and thus the scheduleOrUpdateConsumers call
+		// should fail
+
+		try {
+			eg.scheduleOrUpdateConsumers(resultPartitionId);
+			fail("Expected ExecutionGraphException.");
+		} catch (ExecutionGraphException e) {
+			// we've expected this exception to occur
+		}
+
+		assertEquals(JobStatus.RUNNING, eg.getState());
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexCancelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexCancelTest.java
index 82561b28af6a2..fe6c3cd2f6829 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexCancelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexCancelTest.java
@@ -32,6 +32,7 @@
 import org.apache.flink.runtime.instance.SimpleSlot;
 import org.apache.flink.runtime.instance.Instance;
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
 import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
@@ -403,41 +404,25 @@ public void testCancelCallFails() {
 	}
 
 	@Test
-	public void testSendCancelAndReceiveFail() {
-		try {
-			final JobVertexID jid = new JobVertexID();
-			final ExecutionJobVertex ejv = getExecutionVertex(jid);
-
-			final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
-					AkkaUtils.getDefaultTimeout());
+	public void testSendCancelAndReceiveFail() throws Exception {
+		final ExecutionGraph graph = ExecutionGraphTestUtils.createSimpleTestGraph();
 
-			final ActorGateway gateway = new CancelSequenceActorGateway(
-					TestingUtils.defaultExecutionContext(),
-					1);
+		graph.scheduleForExecution();
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(graph);
+		assertEquals(JobStatus.RUNNING, graph.getState());
 
-			Instance instance = getInstance(new ActorTaskManagerGateway(gateway));
-			SimpleSlot slot = instance.allocateSimpleSlot(new JobID());
+		final ExecutionVertex[] vertices = graph.getVerticesTopologically().iterator().next().getTaskVertices();
+		assertEquals(vertices.length, graph.getRegisteredExecutions().size());
 
-			setVertexState(vertex, ExecutionState.RUNNING);
-			setVertexResource(vertex, slot);
+		final Execution exec = vertices[3].getCurrentExecutionAttempt();
+		exec.cancel();
+		assertEquals(ExecutionState.CANCELING, exec.getState());
 
-			assertEquals(ExecutionState.RUNNING, vertex.getExecutionState());
+		exec.markFailed(new Exception("test"));
+		assertTrue(exec.getState() == ExecutionState.FAILED || exec.getState() == ExecutionState.CANCELED);
 
-			vertex.cancel();
-
-			assertTrue(vertex.getExecutionState() == ExecutionState.CANCELING || vertex.getExecutionState() == ExecutionState.FAILED);
-
-			vertex.getCurrentExecutionAttempt().markFailed(new Throwable("test"));
-
-			assertTrue(vertex.getExecutionState() == ExecutionState.CANCELED || vertex.getExecutionState() == ExecutionState.FAILED);
-
-			assertTrue(slot.isReleased());
-
-			assertEquals(0, vertex.getExecutionGraph().getRegisteredExecutions().size());
-		} catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
+		assertTrue(exec.getAssignedResource().isCanceled());
+		assertEquals(vertices.length - 1, exec.getVertex().getExecutionGraph().getRegisteredExecutions().size());
 	}
 
 	// --------------------------------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
index eb85a3387b87d..0eed90d271bcc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
@@ -156,7 +156,7 @@ public void testLocalityBasedOnState() throws Exception {
 
 		// mimic a restart: all vertices get re-initialized without actually being executed
 		for (ExecutionJobVertex ejv : graph.getVerticesTopologically()) {
-			ejv.resetForNewExecution();
+			ejv.resetForNewExecution(System.currentTimeMillis(), graph.getGlobalModVersion());
 		}
 
 		// set new location for the sources and some state for the targets
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexStopTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexStopTest.java
deleted file mode 100644
index 5edaf91a29d49..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexStopTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.executiongraph;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.execution.ExecutionState;
-import org.apache.flink.runtime.instance.ActorGateway;
-import org.apache.flink.runtime.instance.BaseTestingActorGateway;
-import org.apache.flink.runtime.instance.Instance;
-import org.apache.flink.runtime.instance.SimpleSlot;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
-import org.apache.flink.runtime.messages.Acknowledge;
-import org.apache.flink.runtime.messages.TaskMessages;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.util.TestLogger;
-import org.junit.AfterClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import akka.actor.ActorSystem;
-import akka.testkit.JavaTestKit;
-import scala.concurrent.ExecutionContext;
-import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.setVertexState;
-import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.getExecutionVertex;
-import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.getInstance;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(ExecutionVertex.class)
-public class ExecutionVertexStopTest extends TestLogger {
-
-	private static ActorSystem system;
-
-	private static boolean receivedStopSignal;
-
-	@AfterClass
-	public static void teardown(){
-		if(system != null) {
-			JavaTestKit.shutdownActorSystem(system);
-			system = null;
-		}
-	}
-
-	@Test
-	public void testStop() throws Exception {
-		final JobVertexID jid = new JobVertexID();
-		final ExecutionJobVertex ejv = getExecutionVertex(jid);
-
-		Execution executionMock = mock(Execution.class);
-		whenNew(Execution.class).withAnyArguments().thenReturn(executionMock);
-
-		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
-				AkkaUtils.getDefaultTimeout());
-
-		vertex.stop();
-
-		verify(executionMock).stop();
-	}
-
-	@Test
-	public void testStopRpc() throws Exception {
-		final JobVertexID jid = new JobVertexID();
-		final ExecutionJobVertex ejv = getExecutionVertex(jid);
-
-		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
-				AkkaUtils.getDefaultTimeout());
-		final ExecutionAttemptID execId = vertex.getCurrentExecutionAttempt().getAttemptId();
-
-		setVertexState(vertex, ExecutionState.SCHEDULED);
-		assertEquals(ExecutionState.SCHEDULED, vertex.getExecutionState());
-
-		final ActorGateway gateway = new StopSequenceInstanceGateway(
-				TestingUtils.defaultExecutionContext());
-
-		Instance instance = getInstance(new ActorTaskManagerGateway(gateway));
-		SimpleSlot slot = instance.allocateSimpleSlot(new JobID());
-
-		vertex.deployToSlot(slot);
-
-		receivedStopSignal = false;
-		vertex.stop();
-		assertTrue(receivedStopSignal);
-	}
-
-	public static class StopSequenceInstanceGateway extends BaseTestingActorGateway {
-		private static final long serialVersionUID = 7611571264006653627L;
-
-		public StopSequenceInstanceGateway(ExecutionContext executionContext) {
-			super(executionContext);
-		}
-
-		@Override
-		public Object handleMessage(Object message) throws Exception {
-			Object result = null;
-			if (message instanceof TaskMessages.SubmitTask) {
-				result = Acknowledge.get();
-			} else if (message instanceof TaskMessages.StopTask) {
-				result = Acknowledge.get();
-				receivedStopSignal = true;
-			}
-
-			return result;
-		}
-	}
-
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FinalizeOnMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FinalizeOnMasterTest.java
new file mode 100644
index 0000000000000..8e0c04d203777
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FinalizeOnMasterTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.createSimpleTestGraph;
+
+import static org.junit.Assert.assertEquals;
+
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Tests that the {@link JobVertex#finalizeOnMaster(ClassLoader)} is called properly and
+ * only when the execution graph reaches the a successful final state.
+ */
+public class FinalizeOnMasterTest extends TestLogger {
+
+	@Test
+	public void testFinalizeIsCalledUponSuccess() throws Exception {
+		final JobID jid = new JobID();
+
+		final JobVertex vertex1 = spy(new JobVertex("test vertex 1"));
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(3);
+
+		final JobVertex vertex2 = spy(new JobVertex("test vertex 2"));
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(2);
+
+		final ExecutionGraph eg = createSimpleTestGraph(jid, vertex1, vertex2);
+		eg.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		// move all vertices to finished state
+		ExecutionGraphTestUtils.finishAllVertices(eg);
+		assertEquals(JobStatus.FINISHED, eg.waitUntilTerminal());
+
+		verify(vertex1, times(1)).finalizeOnMaster(any(ClassLoader.class));
+		verify(vertex2, times(1)).finalizeOnMaster(any(ClassLoader.class));
+
+		assertEquals(0, eg.getRegisteredExecutions().size());
+	}
+
+	@Test
+	public void testFinalizeIsNotCalledUponFailure() throws Exception {
+		final JobID jid = new JobID();
+
+		final JobVertex vertex = spy(new JobVertex("test vertex 1"));
+		vertex.setInvokableClass(NoOpInvokable.class);
+		vertex.setParallelism(1);
+
+		final ExecutionGraph eg = createSimpleTestGraph(jid, vertex);
+		eg.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, eg.getState());
+
+		ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
+
+		// fail the execution
+		final Execution exec = eg.getJobVertex(vertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();
+		exec.fail(new Exception("test"));
+
+		assertEquals(JobStatus.FAILED, eg.waitUntilTerminal());
+
+		verify(vertex, times(0)).finalizeOnMaster(any(ClassLoader.class));
+
+		assertEquals(0, eg.getRegisteredExecutions().size());
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/GlobalModVersionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/GlobalModVersionTest.java
new file mode 100644
index 0000000000000..4dd55ae88eec7
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/GlobalModVersionTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory;
+import org.apache.flink.runtime.executiongraph.restart.InfiniteDelayRestartStrategy;
+import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+import org.apache.flink.util.SerializedValue;
+
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.Random;
+
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilExecutionState;
+
+import static org.junit.Assert.assertEquals;
+
+import static org.mockito.Mockito.*;
+
+public class GlobalModVersionTest {
+
+	/**
+	 * Tests that failures during a global cancellation are not handed to the local
+	 * failover strategy.
+	 */
+	@Test
+	public void testNoLocalFailoverWhileCancelling() throws Exception {
+		final FailoverStrategy mockStrategy = mock(FailoverStrategy.class);
+
+		final ExecutionGraph graph = createSampleGraph(mockStrategy);
+
+		final ExecutionVertex testVertex = getRandomVertex(graph);
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(1L, graph.getGlobalModVersion());
+
+		// wait until everything is running
+		for (ExecutionVertex v : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			final Execution exec = v.getCurrentExecutionAttempt();
+			waitUntilExecutionState(exec, ExecutionState.DEPLOYING, 1000);
+			exec.switchToRunning();
+			assertEquals(ExecutionState.RUNNING, exec.getState());
+		}
+
+		// now cancel the job
+		graph.cancel();
+		assertEquals(2L, graph.getGlobalModVersion());
+
+		// everything should be cancelling
+		for (ExecutionVertex v : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			final Execution exec = v.getCurrentExecutionAttempt();
+			assertEquals(ExecutionState.CANCELING, exec.getState());
+		}
+
+		// let a vertex fail
+		testVertex.getCurrentExecutionAttempt().fail(new Exception("test exception"));
+
+		// all cancellations are done now
+		for (ExecutionVertex v : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			final Execution exec = v.getCurrentExecutionAttempt();
+			exec.cancelingComplete();
+		}
+
+		assertEquals(JobStatus.CANCELED, graph.getState());
+
+		// no failure notification at all
+		verify(mockStrategy, times(0)).onTaskFailure(any(Execution.class), any(Throwable.class));
+	}
+
+	/**
+	 * Tests that failures during a global faiover are not handed to the local
+	 * failover strategy.
+	 */
+	@Test
+	public void testNoLocalFailoverWhileFailing() throws Exception {
+		final FailoverStrategy mockStrategy = mock(FailoverStrategy.class);
+
+		final ExecutionGraph graph = createSampleGraph(mockStrategy);
+
+		final ExecutionVertex testVertex = getRandomVertex(graph);
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		// wait until everything is running
+		for (ExecutionVertex v : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			final Execution exec = v.getCurrentExecutionAttempt();
+			waitUntilExecutionState(exec, ExecutionState.DEPLOYING, 1000);
+			exec.switchToRunning();
+			assertEquals(ExecutionState.RUNNING, exec.getState());
+		}
+
+		// now send the job into global failover
+		graph.failGlobal(new Exception("global failover"));
+		assertEquals(JobStatus.FAILING, graph.getState());
+		assertEquals(2L, graph.getGlobalModVersion());
+
+		// another attempt to fail global should not do anything
+		graph.failGlobal(new Exception("should be ignored"));
+		assertEquals(JobStatus.FAILING, graph.getState());
+		assertEquals(2L, graph.getGlobalModVersion());
+
+		// everything should be cancelling
+		for (ExecutionVertex v : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			final Execution exec = v.getCurrentExecutionAttempt();
+			assertEquals(ExecutionState.CANCELING, exec.getState());
+		}
+
+		// let a vertex fail
+		testVertex.getCurrentExecutionAttempt().fail(new Exception("test exception"));
+
+		// all cancellations are done now
+		for (ExecutionVertex v : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
+			final Execution exec = v.getCurrentExecutionAttempt();
+			exec.cancelingComplete();
+		}
+
+		assertEquals(JobStatus.RESTARTING, graph.getState());
+
+		// no failure notification at all
+		verify(mockStrategy, times(0)).onTaskFailure(any(Execution.class), any(Throwable.class));
+	}
+
+	// ------------------------------------------------------------------------
+	//  utilities
+	// ------------------------------------------------------------------------
+
+	private ExecutionGraph createSampleGraph(FailoverStrategy failoverStrategy) throws Exception {
+
+		final JobID jid = new JobID();
+		final int parallelism = new Random().nextInt(10) + 1;
+
+		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);
+
+		// build a simple execution graph with on job vertex, parallelism 2
+		final ExecutionGraph graph = new ExecutionGraph(
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				jid,
+				"test job",
+				new Configuration(),
+				new SerializedValue<>(new ExecutionConfig()),
+				Time.seconds(10),
+				new InfiniteDelayRestartStrategy(),
+				new CustomStrategy(failoverStrategy),
+				Collections.<BlobKey>emptyList(),
+				Collections.<URL>emptyList(),
+				slotProvider,
+				getClass().getClassLoader());
+
+		JobVertex jv = new JobVertex("test vertex");
+		jv.setInvokableClass(NoOpInvokable.class);
+		jv.setParallelism(parallelism);
+
+		JobGraph jg = new JobGraph(jid, "testjob", jv);
+		graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
+
+		return graph;
+	}
+
+	private static ExecutionVertex getRandomVertex(ExecutionGraph eg) {
+		final ExecutionVertex[] vertices = eg.getVerticesTopologically().iterator().next().getTaskVertices();
+		return vertices[new Random().nextInt(vertices.length)];
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static class CustomStrategy implements Factory {
+
+		private final FailoverStrategy failoverStrategy;
+
+		CustomStrategy(FailoverStrategy failoverStrategy) {
+			this.failoverStrategy = failoverStrategy;
+		}
+
+		@Override
+		public FailoverStrategy create(ExecutionGraph executionGraph) {
+			return failoverStrategy;
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IndividualRestartsConcurrencyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IndividualRestartsConcurrencyTest.java
new file mode 100644
index 0000000000000..713aece7a327f
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IndividualRestartsConcurrencyTest.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.ManuallyTriggeredDirectExecutor;
+import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory;
+import org.apache.flink.runtime.executiongraph.failover.RestartIndividualStrategy;
+import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
+import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+import org.apache.flink.util.SerializedValue;
+
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilExecutionState;
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilJobStatus;
+
+import static org.junit.Assert.*;
+
+/**
+ * These tests make sure that global failover (restart all) always takes precedence over
+ * local recovery strategies.
+ * 
+ * <p>This test must be in the package it resides in, because it uses package-private methods
+ * from the ExecutionGraph classes.
+ */
+public class IndividualRestartsConcurrencyTest {
+
+	/**
+	 * Tests that a cancellation concurrent to a local failover leads to a properly
+	 * cancelled state.
+	 */
+	@Test
+	public void testCancelWhileInLocalFailover() throws Exception {
+
+		// the logic in this test is as follows:
+		//  - start a job
+		//  - cause a task failure and delay the local recovery action via the manual executor
+		//  - cancel the job to go into cancelling
+		//  - resume in local recovery action
+		//  - validate that this does in fact not start a new task, because the graph as a
+		//    whole should now be cancelled already
+
+		final JobID jid = new JobID();
+		final int parallelism = 2;
+
+		final ManuallyTriggeredDirectExecutor executor = new ManuallyTriggeredDirectExecutor();
+
+		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);
+
+		final ExecutionGraph graph = createSampleGraph(
+				jid,
+				new IndividualFailoverWithCustomExecutor(executor),
+				slotProvider,
+				2);
+
+		final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next();
+		final ExecutionVertex vertex1 = ejv.getTaskVertices()[0];
+		final ExecutionVertex vertex2 = ejv.getTaskVertices()[1];
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		// let one of the vertices fail - that triggers a local recovery action
+		vertex1.getCurrentExecutionAttempt().fail(new Exception("test failure"));
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+
+		// graph should still be running and the failover recovery action should be queued
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(1, executor.numQueuedRunnables());
+
+		// now cancel the job
+		graph.cancel();
+
+		assertEquals(JobStatus.CANCELLING, graph.getState());
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.CANCELING, vertex2.getCurrentExecutionAttempt().getState());
+
+		// let the recovery action continue
+		executor.trigger();
+
+		// now report that cancelling is complete for the other vertex
+		vertex2.getCurrentExecutionAttempt().cancelingComplete();
+
+		assertEquals(JobStatus.CANCELED, graph.getState());
+		assertTrue(vertex1.getCurrentExecutionAttempt().getState().isTerminal());
+		assertTrue(vertex2.getCurrentExecutionAttempt().getState().isTerminal());
+
+		// make sure all slots are recycled
+		assertEquals(parallelism, slotProvider.getNumberOfAvailableSlots());
+	}
+
+	/**
+	 * Tests that a terminal global failure concurrent to a local failover
+	 * leads to a properly failed state.
+	 */
+	@Test
+	public void testGlobalFailureConcurrentToLocalFailover() throws Exception {
+
+		// the logic in this test is as follows:
+		//  - start a job
+		//  - cause a task failure and delay the local recovery action via the manual executor
+		//  - cause a global failure
+		//  - resume in local recovery action
+		//  - validate that this does in fact not start a new task, because the graph as a
+		//    whole should now be terminally failed already
+
+		final JobID jid = new JobID();
+		final int parallelism = 2;
+
+		final ManuallyTriggeredDirectExecutor executor = new ManuallyTriggeredDirectExecutor();
+
+		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);
+
+		final ExecutionGraph graph = createSampleGraph(
+				jid,
+				new IndividualFailoverWithCustomExecutor(executor),
+				slotProvider,
+				2);
+
+		final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next();
+		final ExecutionVertex vertex1 = ejv.getTaskVertices()[0];
+		final ExecutionVertex vertex2 = ejv.getTaskVertices()[1];
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		// let one of the vertices fail - that triggers a local recovery action
+		vertex1.getCurrentExecutionAttempt().fail(new Exception("test failure"));
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+
+		// graph should still be running and the failover recovery action should be queued
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(1, executor.numQueuedRunnables());
+
+		// now cancel the job
+		graph.failGlobal(new Exception("test exception"));
+
+		assertEquals(JobStatus.FAILING, graph.getState());
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.CANCELING, vertex2.getCurrentExecutionAttempt().getState());
+
+		// let the recovery action continue
+		executor.trigger();
+
+		// now report that cancelling is complete for the other vertex
+		vertex2.getCurrentExecutionAttempt().cancelingComplete();
+
+		assertEquals(JobStatus.FAILED, graph.getState());
+		assertTrue(vertex1.getCurrentExecutionAttempt().getState().isTerminal());
+		assertTrue(vertex2.getCurrentExecutionAttempt().getState().isTerminal());
+
+		// make sure all slots are recycled
+		assertEquals(parallelism, slotProvider.getNumberOfAvailableSlots());
+	}
+
+	/**
+	 * Tests that a local failover does not try to trump a global failover.
+	 */
+	@Test
+	public void testGlobalRecoveryConcurrentToLocalRecovery() throws Exception {
+
+		// the logic in this test is as follows:
+		//  - start a job
+		//  - cause a task failure and delay the local recovery action via the manual executor
+		//  - cause a global failure that is recovering immediately
+		//  - resume in local recovery action
+		//  - validate that this does in fact not cause another task restart, because the global
+		//    recovery should already have restarted the task graph
+
+		final JobID jid = new JobID();
+		final int parallelism = 2;
+
+		final ManuallyTriggeredDirectExecutor executor = new ManuallyTriggeredDirectExecutor();
+
+		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);
+
+		final ExecutionGraph graph = createSampleGraph(
+				jid,
+				new IndividualFailoverWithCustomExecutor(executor),
+				new FixedDelayRestartStrategy(1, 0), // one restart, no delay
+				slotProvider,
+				2);
+
+		final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next();
+		final ExecutionVertex vertex1 = ejv.getTaskVertices()[0];
+		final ExecutionVertex vertex2 = ejv.getTaskVertices()[1];
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		// let one of the vertices fail - that triggers a local recovery action
+		vertex2.getCurrentExecutionAttempt().fail(new Exception("test failure"));
+		assertEquals(ExecutionState.FAILED, vertex2.getCurrentExecutionAttempt().getState());
+
+		// graph should still be running and the failover recovery action should be queued
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(1, executor.numQueuedRunnables());
+
+		// now cancel the job
+		graph.failGlobal(new Exception("test exception"));
+
+		assertEquals(JobStatus.FAILING, graph.getState());
+		assertEquals(ExecutionState.FAILED, vertex2.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.CANCELING, vertex1.getCurrentExecutionAttempt().getState());
+
+		// now report that cancelling is complete for the other vertex
+		vertex1.getCurrentExecutionAttempt().cancelingComplete();
+
+		waitUntilJobStatus(graph, JobStatus.RUNNING, 1000);
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		waitUntilExecutionState(vertex1.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000);
+		waitUntilExecutionState(vertex2.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000);
+		vertex1.getCurrentExecutionAttempt().switchToRunning();
+		vertex2.getCurrentExecutionAttempt().switchToRunning();
+		assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState());
+
+		// let the recovery action continue - this should do nothing any more
+		executor.trigger();
+
+		// validate that the graph is still peachy
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState());
+		assertEquals(1, vertex1.getCurrentExecutionAttempt().getAttemptNumber());
+		assertEquals(1, vertex2.getCurrentExecutionAttempt().getAttemptNumber());
+		assertEquals(1, vertex1.getCopyOfPriorExecutionsList().size());
+		assertEquals(1, vertex2.getCopyOfPriorExecutionsList().size());
+
+		// make sure all slots are in use
+		assertEquals(0, slotProvider.getNumberOfAvailableSlots());
+	}
+
+	// ------------------------------------------------------------------------
+	//  utilities
+	// ------------------------------------------------------------------------
+
+	private ExecutionGraph createSampleGraph(
+			JobID jid,
+			Factory failoverStrategy,
+			SlotProvider slotProvider,
+			int parallelism) throws Exception {
+
+		return createSampleGraph(jid, failoverStrategy, new NoRestartStrategy(), slotProvider, parallelism);
+	}
+
+	private ExecutionGraph createSampleGraph(
+			JobID jid,
+			Factory failoverStrategy,
+			RestartStrategy restartStrategy,
+			SlotProvider slotProvider,
+			int parallelism) throws Exception {
+
+		// build a simple execution graph with on job vertex, parallelism 2
+		final ExecutionGraph graph = new ExecutionGraph(
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				jid,
+				"test job",
+				new Configuration(),
+				new SerializedValue<>(new ExecutionConfig()),
+				Time.seconds(10),
+				restartStrategy,
+				failoverStrategy,
+				Collections.<BlobKey>emptyList(),
+				Collections.<URL>emptyList(),
+				slotProvider,
+				getClass().getClassLoader());
+
+		JobVertex jv = new JobVertex("test vertex");
+		jv.setInvokableClass(NoOpInvokable.class);
+		jv.setParallelism(parallelism);
+
+		JobGraph jg = new JobGraph(jid, "testjob", jv);
+		graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
+
+		return graph;
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static class IndividualFailoverWithCustomExecutor implements Factory {
+
+		private final Executor executor;
+
+		IndividualFailoverWithCustomExecutor(Executor executor) {
+			this.executor = executor;
+		}
+
+		@Override
+		public FailoverStrategy create(ExecutionGraph executionGraph) {
+			return new RestartIndividualStrategy(executionGraph, executor);
+		}
+	}
+
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java
new file mode 100644
index 0000000000000..6e67c1a53dc81
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.utils;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.checkpoint.CheckpointOptions;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.PartitionInfo;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
+import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.messages.StackTrace;
+import org.apache.flink.runtime.messages.StackTraceSampleResponse;
+
+import java.util.UUID;
+
+/**
+ * A TaskManagerGateway that simply acks the basic operations (deploy, cancel, update) and does not
+ * support any more advanced operations.
+ */
+public class SimpleAckingTaskManagerGateway implements TaskManagerGateway {
+
+	private final String address = UUID.randomUUID().toString();
+
+	@Override
+	public String getAddress() {
+		return address;
+	}
+
+	@Override
+	public void disconnectFromJobManager(InstanceID instanceId, Exception cause) {}
+
+	@Override
+	public void stopCluster(ApplicationStatus applicationStatus, String message) {}
+
+	@Override
+	public Future<StackTrace> requestStackTrace(Time timeout) {
+		return FlinkCompletableFuture.completedExceptionally(new UnsupportedOperationException());
+	}
+
+	@Override
+	public Future<StackTraceSampleResponse> requestStackTraceSample(
+			ExecutionAttemptID executionAttemptID,
+			int sampleId,
+			int numSamples,
+			Time delayBetweenSamples,
+			int maxStackTraceDepth,
+			Time timeout) {
+		return FlinkCompletableFuture.completedExceptionally(new UnsupportedOperationException());
+	}
+
+	@Override
+	public Future<Acknowledge> submitTask(TaskDeploymentDescriptor tdd, Time timeout) {
+		return FlinkCompletableFuture.completed(Acknowledge.get());
+	}
+
+	@Override
+	public Future<Acknowledge> stopTask(ExecutionAttemptID executionAttemptID, Time timeout) {
+		return FlinkCompletableFuture.completed(Acknowledge.get());
+	}
+
+	@Override
+	public Future<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
+		return FlinkCompletableFuture.completed(Acknowledge.get());
+	}
+
+	@Override
+	public Future<Acknowledge> updatePartitions(ExecutionAttemptID executionAttemptID, Iterable<PartitionInfo> partitionInfos, Time timeout) {
+		return FlinkCompletableFuture.completed(Acknowledge.get());
+	}
+
+	@Override
+	public void failPartition(ExecutionAttemptID executionAttemptID) {}
+
+	@Override
+	public void notifyCheckpointComplete(
+			ExecutionAttemptID executionAttemptID,
+			JobID jobId,
+			long checkpointId,
+			long timestamp) {}
+
+	@Override
+	public void triggerCheckpoint(
+			ExecutionAttemptID executionAttemptID,
+			JobID jobId,
+			long checkpointId,
+			long timestamp,
+			CheckpointOptions checkpointOptions) {}
+
+	@Override
+	public Future<BlobKey> requestTaskManagerLog(Time timeout) {
+		return FlinkCompletableFuture.completedExceptionally(new UnsupportedOperationException());
+	}
+
+	@Override
+	public Future<BlobKey> requestTaskManagerStdout(Time timeout) {
+		return FlinkCompletableFuture.completedExceptionally(new UnsupportedOperationException());
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java
new file mode 100644
index 0000000000000..2cf1eec67b34e
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.utils;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.instance.SimpleSlot;
+import org.apache.flink.runtime.instance.Slot;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
+import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit;
+import org.apache.flink.runtime.jobmanager.slots.AllocatedSlot;
+import org.apache.flink.runtime.jobmanager.slots.SlotOwner;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+
+import java.net.InetAddress;
+import java.util.ArrayDeque;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A testing utility slot provider that simply has a predefined pool of slots.
+ */
+public class SimpleSlotProvider implements SlotProvider, SlotOwner {
+
+	private final ArrayDeque<AllocatedSlot> slots;
+
+	public SimpleSlotProvider(JobID jobId, int numSlots) {
+		this(jobId, numSlots, new SimpleAckingTaskManagerGateway());
+	}
+
+	public SimpleSlotProvider(JobID jobId, int numSlots, TaskManagerGateway taskManagerGateway) {
+		checkNotNull(jobId, "jobId");
+		checkArgument(numSlots >= 0, "numSlots must be >= 0");
+
+		this.slots = new ArrayDeque<>(numSlots);
+
+		for (int i = 0; i < numSlots; i++) {
+			AllocatedSlot as = new AllocatedSlot(
+					new AllocationID(),
+					jobId,
+					new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i),
+					0,
+					ResourceProfile.UNKNOWN,
+					taskManagerGateway);
+			slots.add(as);
+		}
+	}
+
+	@Override
+	public Future<SimpleSlot> allocateSlot(ScheduledUnit task, boolean allowQueued) {
+		final AllocatedSlot slot;
+
+		synchronized (slots) {
+			if (slots.isEmpty()) {
+				slot = null;
+			} else {
+				slot = slots.removeFirst();
+			}
+		}
+
+		if (slot != null) {
+			SimpleSlot result = new SimpleSlot(slot, this, 0);
+			return FlinkCompletableFuture.completed(result);
+		}
+		else {
+			return FlinkCompletableFuture.completedExceptionally(new NoResourceAvailableException());
+		}
+	}
+
+	@Override
+	public boolean returnAllocatedSlot(Slot slot) {
+		synchronized (slots) {
+			slots.add(slot.getAllocatedSlot());
+		}
+		return true;
+	}
+
+	public int getNumberOfAvailableSlots() {
+		synchronized (slots) {
+			return slots.size();
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
index 6316bfd4cb54b..195baa1fa261d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
@@ -447,7 +447,7 @@ protected void run() {
 							vertex.getCurrentExecutionAttempt().getAttemptId());
 
 						// Reset execution => new execution attempt
-						vertex.resetForNewExecution();
+						vertex.resetForNewExecution(System.currentTimeMillis(), 1L);
 
 						// Producer finished, request state
 						Object request = new JobManagerMessages.RequestPartitionProducerState(jid, rid, partitionId);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
index 49d0239cd5e6c..b1bb5483d854d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
@@ -21,7 +21,6 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
-import org.apache.flink.runtime.executiongraph.TerminalJobStatusListener;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
@@ -53,7 +52,6 @@ public class LeaderChangeJobRecoveryTest extends TestLogger {
 	private int numSlotsPerTM = 1;
 	private int parallelism = numTMs * numSlotsPerTM;
 
-	private Configuration configuration;
 	private LeaderElectionRetrievalTestingCluster cluster = null;
 	private JobGraph job = createBlockingJob(parallelism);
 
@@ -61,7 +59,7 @@ public class LeaderChangeJobRecoveryTest extends TestLogger {
 	public void before() throws TimeoutException, InterruptedException {
 		Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(true);
 
-		configuration = new Configuration();
+		Configuration configuration = new Configuration();
 
 		configuration.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 1);
 		configuration.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTMs);
@@ -110,12 +108,9 @@ public void testNotRestartedWhenLosingLeadership() throws Exception {
 
 		ExecutionGraph executionGraph = (ExecutionGraph) ((TestingJobManagerMessages.ExecutionGraphFound) responseExecutionGraph).executionGraph();
 
-		TerminalJobStatusListener testListener = new TerminalJobStatusListener();
-		executionGraph.registerJobStatusListener(testListener);
-
 		cluster.revokeLeadership();
 
-		testListener.waitForTerminalState(30000);
+		executionGraph.getTerminationFuture().get(30, TimeUnit.SECONDS);
 	}
 
 	public JobGraph createBlockingJob(int parallelism) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
index 7102f27a941e9..4019d63e143ed 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
@@ -30,6 +30,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.executiongraph.failover.RestartAllStrategy;
 import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -147,6 +148,7 @@ public void flatMap(String value,
 			new SerializedValue<>(new ExecutionConfig()),
 			AkkaUtils.getDefaultTimeout(),
 			new NoRestartStrategy(),
+			new RestartAllStrategy.Factory(),
 			new ArrayList<BlobKey>(),
 			new ArrayList<URL>(),
 			new Scheduler(TestingUtils.defaultExecutionContext()),
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/ManuallyTriggeredDirectExecutor.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/ManuallyTriggeredDirectExecutor.java
new file mode 100644
index 0000000000000..07c7bd352a640
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/ManuallyTriggeredDirectExecutor.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.testutils;
+
+import javax.annotation.Nonnull;
+import java.util.ArrayDeque;
+import java.util.concurrent.Executor;
+
+/**
+ * An executor that does not immediately execute a Runnable, but only executes it
+ * upon an explicit trigger.
+ * 
+ * <p>This executor can be used in concurrent tests to control when certain asynchronous
+ * actions should happen.
+ */
+public class ManuallyTriggeredDirectExecutor implements Executor {
+
+	private final ArrayDeque<Runnable> queuedRunnables = new ArrayDeque<>();
+
+	@Override
+	public void execute(@Nonnull Runnable command) {
+		synchronized (queuedRunnables) {
+			queuedRunnables.addLast(command);
+		}
+	}
+
+	/**
+	 * Triggers the next queued runnable and executes it synchronously.
+	 * This method throws an exception if no Runnable is currently queued.
+	 */
+	public void trigger() {
+		final Runnable next;
+
+		synchronized (queuedRunnables) {
+			next = queuedRunnables.removeFirst();
+		}
+
+		if (next != null) {
+			next.run();
+		}
+		else {
+			throw new IllegalStateException("No runnable available");
+		}
+	}
+
+	/**
+	 * Gets the number of Runnables currently queued.
+	 */
+	public int numQueuedRunnables() {
+		synchronized (queuedRunnables) {
+			return queuedRunnables.size();
+		}
+	}
+}

From 2549e74d81d1530714a3028041add6c2d68fa7df Mon Sep 17 00:00:00 2001
From: Xpray <leonxpray@gmail.com>
Date: Wed, 3 May 2017 16:52:43 +0800
Subject: [PATCH 131/253] [FLINK-6334] [table] Refactor Table API TableFunction
 join methods.

This closes #3791.
---
 .../flink/table/api/TableEnvironment.scala    |   2 +-
 .../api/scala/TableFunctionConversions.scala  |  56 ++++
 .../flink/table/api/scala/package.scala       |   4 +
 .../org/apache/flink/table/api/table.scala    | 288 +++++++++---------
 .../flink/table/functions/TableFunction.scala |  16 -
 .../utils/UserDefinedFunctionUtils.scala      |  39 ++-
 .../flink/table/plan/logical/operators.scala  |  22 +-
 .../api/scala/batch/table/JoinITCase.scala    |  25 +-
 .../table/UserDefinedTableFunctionTest.scala  |  18 +-
 .../table/UserDefinedTableFunctionTest.scala  | 126 +++++++-
 10 files changed, 404 insertions(+), 192 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableFunctionConversions.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index bd974b0eafa3a..45267d2f89954 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -79,7 +79,7 @@ abstract class TableEnvironment(val config: TableConfig) {
   private val rootSchema: SchemaPlus = internalSchema.plus()
 
   // Table API/SQL function catalog
-  private val functionCatalog: FunctionCatalog = FunctionCatalog.withBuiltIns
+  private[flink] val functionCatalog: FunctionCatalog = FunctionCatalog.withBuiltIns
 
   // the configuration to create a Calcite planner
   private lazy val frameworkConfig: FrameworkConfig = Frameworks
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableFunctionConversions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableFunctionConversions.scala
new file mode 100644
index 0000000000000..692876fe2da09
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableFunctionConversions.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.table.api.Table
+import org.apache.flink.table.expressions._
+import org.apache.flink.table.functions.TableFunction
+import org.apache.flink.table.plan.logical.LogicalTableFunctionCall
+
+/**
+  * Holds methods to convert a [[TableFunction]] call in the Scala Table API into a [[Table]].
+  *
+  * @param tf The TableFunction to convert.
+  */
+class TableFunctionConversions[T](tf: TableFunction[T]) {
+
+  /**
+    * Creates a [[Table]] from a [[TableFunction]] in Scala Table API.
+    *
+    * @param args The arguments of the table function call.
+    * @return A [[Table]] with which represents the [[LogicalTableFunctionCall]].
+    */
+  final def apply(args: Expression*)(implicit typeInfo: TypeInformation[T]): Table = {
+
+    val resultType = if (tf.getResultType == null) typeInfo else tf.getResultType
+
+    new Table(
+      tableEnv = null, // Table environment will be set later.
+      LogicalTableFunctionCall(
+        tf.getClass.getCanonicalName,
+        tf,
+        args.toList,
+        resultType,
+        Array.empty,
+        child = null // Child will be set later.
+      )
+    )
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala
index 5b431ec5b376e..e8a201777fac6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/package.scala
@@ -24,6 +24,7 @@ import org.apache.flink.api.scala.DataSet
 import org.apache.flink.streaming.api.scala.DataStream
 import org.apache.flink.table.api.scala.{StreamTableEnvironment => ScalaStreamTableEnv}
 import org.apache.flink.table.api.scala.{BatchTableEnvironment => ScalaBatchTableEnv}
+import org.apache.flink.table.functions.TableFunction
 
 import _root_.scala.language.implicitConversions
 
@@ -89,4 +90,7 @@ package object scala extends ImplicitExpressionConversions {
     tableEnv.toDataStream[Row](table)
   }
 
+  implicit def tableFunctionCall2Table[T](tf: TableFunction[T]): TableFunctionConversions[T] = {
+    new TableFunctionConversions[T](tf)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 095361152aa26..9606979643fa5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -20,9 +20,10 @@ package org.apache.flink.table.api
 import org.apache.calcite.rel.RelNode
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.operators.join.JoinType
-import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.calcite.{FlinkRelBuilder, FlinkTypeFactory}
+import org.apache.flink.table.expressions.{Alias, Asc, Expression, ExpressionParser, Ordering, UnresolvedAlias, UnresolvedFieldReference}
 import org.apache.flink.table.plan.logical.Minus
-import org.apache.flink.table.expressions.{Alias, Asc, Call, Expression, ExpressionParser, Ordering, TableFunctionCall, UnresolvedAlias}
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
 import org.apache.flink.table.plan.ProjectionTranslator._
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.sinks.TableSink
@@ -64,13 +65,34 @@ class Table(
     private[flink] val tableEnv: TableEnvironment,
     private[flink] val logicalPlan: LogicalNode) {
 
-  private val tableSchema: TableSchema = new TableSchema(
+  // Check if the plan has an unbounded TableFunctionCall as child node.
+  //   A TableFunctionCall is tolerated as root node because the Table holds the initial call.
+  if (containsUnboudedUDTFCall(logicalPlan) &&
+    !logicalPlan.isInstanceOf[LogicalTableFunctionCall]) {
+    throw new ValidationException("TableFunction can only be used in join and leftOuterJoin.")
+  }
+
+  /**
+    * Creates a [[Table]] for a TableFunction call from a String expression.
+    *
+    * @param tableEnv The TableEnvironment in which the call is created.
+    * @param udtfCall A String expression of the TableFunction call.
+    */
+  def this(tableEnv: TableEnvironment, udtfCall: String) {
+    this(tableEnv, UserDefinedFunctionUtils.createLogicalFunctionCall(tableEnv, udtfCall))
+  }
+
+  private lazy val tableSchema: TableSchema = new TableSchema(
     logicalPlan.output.map(_.name).toArray,
     logicalPlan.output.map(_.resultType).toArray)
 
-  def relBuilder = tableEnv.getRelBuilder
+  def relBuilder: FlinkRelBuilder = tableEnv.getRelBuilder
 
-  def getRelNode: RelNode = logicalPlan.toRelNode(relBuilder)
+  def getRelNode: RelNode = if (containsUnboudedUDTFCall(logicalPlan)) {
+    throw new ValidationException("Cannot translate a query with an unbounded table function call.")
+  } else {
+    logicalPlan.toRelNode(relBuilder)
+  }
 
   /**
     * Returns the schema of this table.
@@ -143,7 +165,35 @@ class Table(
     * }}}
     */
   def as(fields: Expression*): Table = {
-    new Table(tableEnv, AliasNode(fields, logicalPlan).validate(tableEnv))
+
+    logicalPlan match {
+      case functionCall: LogicalTableFunctionCall if functionCall.child == null => {
+        // If the logical plan is a TableFunctionCall, we replace its field names to avoid special
+        //   cases during the validation.
+        if (fields.length != functionCall.output.length) {
+          throw new ValidationException(
+            "List of column aliases must have same degree as TableFunction's output")
+        }
+        if (!fields.forall(_.isInstanceOf[UnresolvedFieldReference])) {
+          throw new ValidationException(
+            "Alias field must be an instance of UnresolvedFieldReference"
+          )
+        }
+        new Table(
+          tableEnv,
+          new LogicalTableFunctionCall(
+            functionCall.functionName,
+            functionCall.tableFunction,
+            functionCall.parameters,
+            functionCall.resultType,
+            fields.map(_.asInstanceOf[UnresolvedFieldReference].name).toArray,
+            functionCall.child)
+        )
+      }
+      case _ =>
+        // prepend an AliasNode
+        new Table(tableEnv, AliasNode(fields, logicalPlan).validate(tableEnv))
+    }
   }
 
   /**
@@ -309,6 +359,41 @@ class Table(
     join(right, Some(joinPredicate), JoinType.INNER)
   }
 
+  /**
+    * Joins this [[Table]] with an user-defined [[org.apache.calcite.schema.TableFunction]].
+    * This join is similar to a SQL left outer join with ON TRUE predicate, but it works with a
+    * table function. Each row of the outer table is joined with all rows produced by the table
+    * function. If the table function does not produce any row, the outer row is padded with nulls.
+    *
+    * Scala Example:
+    * {{{
+    *   class MySplitUDTF extends TableFunction[String] {
+    *     def eval(str: String): Unit = {
+    *       str.split("#").foreach(collect)
+    *     }
+    *   }
+    *
+    *   val split = new MySplitUDTF()
+    *   table.leftOuterJoin(split('c) as ('s)).select('a,'b,'c,'s)
+    * }}}
+    *
+    * Java Example:
+    * {{{
+    *   class MySplitUDTF extends TableFunction<String> {
+    *     public void eval(String str) {
+    *       str.split("#").forEach(this::collect);
+    *     }
+    *   }
+    *
+    *   TableFunction<String> split = new MySplitUDTF();
+    *   tableEnv.registerFunction("split", split);
+    *   table.leftOuterJoin(new Table(tableEnv, "split(c)").as("s"))).select("a, b, c, s");
+    * }}}
+    */
+  def leftOuterJoin(right: Table): Table = {
+    join(right, None, JoinType.LEFT_OUTER)
+  }
+
   /**
     * Joins two [[Table]]s. Similar to an SQL left outer join. The fields of the two joined
     * operations must not overlap, use [[as]] to rename fields if necessary.
@@ -417,14 +502,46 @@ class Table(
   }
 
   private def join(right: Table, joinPredicate: Option[Expression], joinType: JoinType): Table = {
-    // check that right table belongs to the same TableEnvironment
-    if (right.tableEnv != this.tableEnv) {
-      throw new ValidationException("Only tables from the same TableEnvironment can be joined.")
+
+    // check if we join with a table or a table function
+    if (!containsUnboudedUDTFCall(right.logicalPlan)) {
+      // regular table-table join
+
+      // check that the TableEnvironment of right table is not null
+      // and right table belongs to the same TableEnvironment
+      if (right.tableEnv != this.tableEnv) {
+        throw new ValidationException("Only tables from the same TableEnvironment can be joined.")
+      }
+
+      new Table(
+        tableEnv,
+        Join(this.logicalPlan, right.logicalPlan, joinType, joinPredicate, correlated = false)
+          .validate(tableEnv))
+
+    } else {
+      // join with a table function
+
+      // check join type
+      if (joinType != JoinType.INNER && joinType != JoinType.LEFT_OUTER) {
+        throw new ValidationException(
+          "TableFunctions are currently supported for join and leftOuterJoin.")
+      }
+
+      val udtf = right.logicalPlan.asInstanceOf[LogicalTableFunctionCall]
+      val udtfCall = LogicalTableFunctionCall(
+        udtf.functionName,
+        udtf.tableFunction,
+        udtf.parameters,
+        udtf.resultType,
+        udtf.fieldNames,
+        this.logicalPlan
+      ).validate(tableEnv)
+
+      new Table(
+        tableEnv,
+        Join(this.logicalPlan, udtfCall, joinType, joinPredicate, correlated = true)
+          .validate(tableEnv))
     }
-    new Table(
-      tableEnv,
-      Join(this.logicalPlan, right.logicalPlan, joinType, joinPredicate, correlated = false)
-        .validate(tableEnv))
   }
 
   /**
@@ -632,136 +749,6 @@ class Table(
     new Table(tableEnv, Limit(offset, fetch, logicalPlan).validate(tableEnv))
   }
 
-  /**
-    * Joins this [[Table]] to a user-defined [[org.apache.calcite.schema.TableFunction]]. Similar
-    * to an SQL cross join, but it works with a table function. It returns rows from the outer
-    * table (table on the left of the operator) that produces matching values from the table
-    * function (which is defined in the expression on the right side of the operator).
-    *
-    * Example:
-    *
-    * {{{
-    *   class MySplitUDTF extends TableFunction[String] {
-    *     def eval(str: String): Unit = {
-    *       str.split("#").foreach(collect)
-    *     }
-    *   }
-    *
-    *   val split = new MySplitUDTF()
-    *   table.join(split('c) as ('s)).select('a,'b,'c,'s)
-    * }}}
-    */
-  def join(udtf: Expression): Table = {
-    joinUdtfInternal(udtf, JoinType.INNER)
-  }
-
-  /**
-    * Joins this [[Table]] to a user-defined [[org.apache.calcite.schema.TableFunction]]. Similar
-    * to an SQL cross join, but it works with a table function. It returns rows from the outer
-    * table (table on the left of the operator) that produces matching values from the table
-    * function (which is defined in the expression on the right side of the operator).
-    *
-    * Example:
-    *
-    * {{{
-    *   class MySplitUDTF extends TableFunction<String> {
-    *     public void eval(String str) {
-    *       str.split("#").forEach(this::collect);
-    *     }
-    *   }
-    *
-    *   TableFunction<String> split = new MySplitUDTF();
-    *   tableEnv.registerFunction("split", split);
-    *
-    *   table.join("split(c) as (s)").select("a, b, c, s");
-    * }}}
-    */
-  def join(udtf: String): Table = {
-    joinUdtfInternal(udtf, JoinType.INNER)
-  }
-
-  /**
-    * Joins this [[Table]] to a user-defined [[org.apache.calcite.schema.TableFunction]]. Similar
-    * to an SQL left outer join with ON TRUE, but it works with a table function. It returns all
-    * the rows from the outer table (table on the left of the operator), and rows that do not match
-    * the condition from the table function (which is defined in the expression on the right
-    * side of the operator). Rows with no matching condition are filled with null values.
-    *
-    * Example:
-    *
-    * {{{
-    *   class MySplitUDTF extends TableFunction[String] {
-    *     def eval(str: String): Unit = {
-    *       str.split("#").foreach(collect)
-    *     }
-    *   }
-    *
-    *   val split = new MySplitUDTF()
-    *   table.leftOuterJoin(split('c) as ('s)).select('a,'b,'c,'s)
-    * }}}
-    */
-  def leftOuterJoin(udtf: Expression): Table = {
-    joinUdtfInternal(udtf, JoinType.LEFT_OUTER)
-  }
-
-  /**
-    * Joins this [[Table]] to a user-defined [[org.apache.calcite.schema.TableFunction]]. Similar
-    * to an SQL left outer join with ON TRUE, but it works with a table function. It returns all
-    * the rows from the outer table (table on the left of the operator), and rows that do not match
-    * the condition from the table function (which is defined in the expression on the right
-    * side of the operator). Rows with no matching condition are filled with null values.
-    *
-    * Example:
-    *
-    * {{{
-    *   class MySplitUDTF extends TableFunction<String> {
-    *     public void eval(String str) {
-    *       str.split("#").forEach(this::collect);
-    *     }
-    *   }
-    *
-    *   TableFunction<String> split = new MySplitUDTF();
-    *   tableEnv.registerFunction("split", split);
-    *
-    *   table.leftOuterJoin("split(c) as (s)").select("a, b, c, s");
-    * }}}
-    */
-  def leftOuterJoin(udtf: String): Table = {
-    joinUdtfInternal(udtf, JoinType.LEFT_OUTER)
-  }
-
-  private def joinUdtfInternal(udtfString: String, joinType: JoinType): Table = {
-    val udtf = ExpressionParser.parseExpression(udtfString)
-    joinUdtfInternal(udtf, joinType)
-  }
-
-  private def joinUdtfInternal(udtf: Expression, joinType: JoinType): Table = {
-    var alias: Option[Seq[String]] = None
-
-    // unwrap an Expression until we get a TableFunctionCall
-    def unwrap(expr: Expression): TableFunctionCall = expr match {
-      case Alias(child, name, extraNames) =>
-        alias = Some(Seq(name) ++ extraNames)
-        unwrap(child)
-      case Call(name, args) =>
-        val function = tableEnv.getFunctionCatalog.lookupFunction(name, args)
-        unwrap(function)
-      case c: TableFunctionCall => c
-      case _ =>
-        throw new TableException(
-          "Cross/Outer Apply operators only accept expressions that define table functions.")
-    }
-
-    val call = unwrap(udtf)
-      .as(alias)
-      .toLogicalTableFunctionCall(this.logicalPlan)
-      .validate(tableEnv)
-
-    new Table(
-      tableEnv,
-      Join(this.logicalPlan, call, joinType, None, correlated = true).validate(tableEnv))
-  }
-
   /**
     * Writes the [[Table]] to a [[TableSink]]. A [[TableSink]] defines an external storage location.
     *
@@ -773,7 +760,6 @@ class Table(
     * @tparam T The data type that the [[TableSink]] expects.
     */
   def writeToSink[T](sink: TableSink[T]): Unit = {
-
     // get schema information of table
     val rowType = getRelNode.getRowType
     val fieldNames: Array[String] = rowType.getFieldNames.asScala.toArray
@@ -859,6 +845,20 @@ class Table(
     }
     tableName
   }
+
+  /**
+    * Checks if the plan represented by a [[LogicalNode]] contains an unbounded UDTF call.
+    * @param n the node to check
+    * @return true if the plan contains an unbounded UDTF call, false otherwise.
+    */
+  private def containsUnboudedUDTFCall(n: LogicalNode): Boolean = {
+    n match {
+      case functionCall: LogicalTableFunctionCall if functionCall.child == null => true
+      case u: UnaryNode => containsUnboudedUDTFCall(u.child)
+      case b: BinaryNode => containsUnboudedUDTFCall(b.left) || containsUnboudedUDTFCall(b.right)
+      case _: LeafNode => false
+    }
+  }
 }
 
 /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TableFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TableFunction.scala
index d4c5021ed108a..53543494f78bd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TableFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TableFunction.scala
@@ -19,7 +19,6 @@
 package org.apache.flink.table.functions
 
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.table.expressions.{Expression, TableFunctionCall}
 import org.apache.flink.util.Collector
 
 /**
@@ -79,21 +78,6 @@ import org.apache.flink.util.Collector
   */
 abstract class TableFunction[T] extends UserDefinedFunction {
 
-  /**
-    * Creates a call to a [[TableFunction]] in Scala Table API.
-    *
-    * @param params actual parameters of function
-    * @return [[Expression]] in form of a [[TableFunctionCall]]
-    */
-  final def apply(params: Expression*)(implicit typeInfo: TypeInformation[T]): Expression = {
-    val resultType = if (getResultType == null) {
-      typeInfo
-    } else {
-      getResultType
-    }
-    TableFunctionCall(getClass.getSimpleName, this, params, resultType)
-  }
-
   override def toString: String = getClass.getCanonicalName
 
   // ----------------------------------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
index c1cfe0610ae5e..d108e312b40fc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
@@ -19,7 +19,7 @@
 
 package org.apache.flink.table.functions.utils
 
-import java.lang.{Long => JLong, Integer => JInt}
+import java.lang.{Integer => JInt, Long => JLong}
 import java.lang.reflect.{Method, Modifier}
 import java.sql.{Date, Time, Timestamp}
 
@@ -30,8 +30,10 @@ import org.apache.flink.api.common.functions.InvalidTypesException
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.table.api.{TableEnvironment, ValidationException}
+import org.apache.flink.table.api.{TableEnvironment, TableException, ValidationException}
+import org.apache.flink.table.expressions._
 import org.apache.flink.table.functions.{ScalarFunction, TableFunction, UserDefinedFunction}
+import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.plan.schema.FlinkTableFunctionImpl
 import org.apache.flink.util.InstantiationUtil
 
@@ -358,4 +360,37 @@ object UserDefinedFunctionUtils {
     InstantiationUtil
       .deserializeObject[UserDefinedFunction](byteData, Thread.currentThread.getContextClassLoader)
   }
+
+  /**
+    * Creates a [[LogicalTableFunctionCall]] by parsing a String expression.
+    *
+    * @param tableEnv The table environmenent to lookup the function.
+    * @param udtf a String expression of a TableFunctionCall, such as "split(c)"
+    * @return A LogicalTableFunctionCall.
+    */
+  def createLogicalFunctionCall(
+      tableEnv: TableEnvironment,
+      udtf: String): LogicalTableFunctionCall = {
+
+    var alias: Option[Seq[String]] = None
+
+    // unwrap an Expression until we get a TableFunctionCall
+    def unwrap(expr: Expression): TableFunctionCall = expr match {
+      case Alias(child, name, extraNames) =>
+        alias = Some(Seq(name) ++ extraNames)
+        unwrap(child)
+      case Call(name, args) =>
+        val function = tableEnv.functionCatalog.lookupFunction(name, args)
+        unwrap(function)
+      case c: TableFunctionCall => c
+      case _ =>
+        throw new TableException(
+          "Table(TableEnv, String) constructor only accept String that " +
+            "define table function followed by some Alias.")
+    }
+
+    val functionCall: LogicalTableFunctionCall = unwrap(ExpressionParser.parseExpression(udtf))
+      .as(alias).toLogicalTableFunctionCall(child = null)
+    functionCall
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index dbfd3ced562fc..c67bfd18d301e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -170,7 +170,7 @@ case class Limit(offset: Int, fetch: Int = -1, child: LogicalNode) extends Unary
     if (tableEnv.isInstanceOf[StreamTableEnvironment]) {
       failValidation(s"Limit on stream tables is currently not supported.")
     }
-    if (!child.validate(tableEnv).isInstanceOf[Sort]) {
+    if (!child.isInstanceOf[Sort]) {
       failValidation(s"Limit operator must be preceded by an OrderBy operator.")
     }
     if (offset < 0) {
@@ -662,11 +662,19 @@ case class LogicalTableFunctionCall(
     child: LogicalNode)
   extends UnaryNode {
 
-  private val (_, fieldIndexes, fieldTypes) = getFieldInfo(resultType)
+  private val (generatedNames, fieldIndexes, fieldTypes) = getFieldInfo(resultType)
   private var evalMethod: Method = _
 
-  override def output: Seq[Attribute] = fieldNames.zip(fieldTypes).map {
-    case (n, t) => ResolvedFieldReference(n, t)
+  override def output: Seq[Attribute] = {
+    if (fieldNames.isEmpty) {
+      generatedNames.zip(fieldTypes).map {
+        case (n, t) => ResolvedFieldReference(n, t)
+      }
+    } else {
+      fieldNames.zip(fieldTypes).map {
+        case (n, t) => ResolvedFieldReference(n, t)
+      }
+    }
   }
 
   override def validate(tableEnv: TableEnvironment): LogicalNode = {
@@ -691,7 +699,11 @@ case class LogicalTableFunctionCall(
 
   override protected[logical] def construct(relBuilder: RelBuilder): RelBuilder = {
     val fieldIndexes = getFieldInfo(resultType)._2
-    val function = new FlinkTableFunctionImpl(resultType, fieldIndexes, fieldNames, evalMethod)
+    val function = new FlinkTableFunctionImpl(
+      resultType,
+      fieldIndexes,
+      if (fieldNames.isEmpty) generatedNames else fieldNames, evalMethod
+    )
     val typeFactory = relBuilder.getTypeFactory.asInstanceOf[FlinkTypeFactory]
     val sqlFunction = TableSqlFunction(
       tableFunction.functionIdentifier,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/JoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/JoinITCase.scala
index 5993728a49a83..8085a3c6e4a7c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/JoinITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/JoinITCase.scala
@@ -24,8 +24,9 @@ import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableC
 import org.apache.flink.table.api.scala._
 import org.apache.flink.api.scala.util.CollectionDataSets
 import org.apache.flink.types.Row
-import org.apache.flink.table.api.{TableEnvironment, TableException, ValidationException}
+import org.apache.flink.table.api.{TableEnvironment, ValidationException}
 import org.apache.flink.table.expressions.Literal
+import org.apache.flink.table.utils.TableFunc2
 import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
 import org.apache.flink.test.util.TestBaseUtils
 import org.junit._
@@ -345,4 +346,26 @@ class JoinITCase(
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
+  @Test
+  def testUDTFJoinOnTuples(): Unit = {
+    val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val data = List("hi#world", "how#are#you")
+
+    val ds1 = env.fromCollection(data).toTable(tEnv, 'a)
+    val func2 = new TableFunc2
+
+    val joinDs = ds1.join(func2('a) as ('name, 'len))
+
+    val results = joinDs.toDataSet[Row].collect()
+    val expected = Seq(
+      "hi#world,hi,2",
+      "hi#world,world,5",
+      "how#are#you,how,3",
+      "how#are#you,are,3",
+      "how#are#you,you,3").mkString("\n")
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/UserDefinedTableFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/UserDefinedTableFunctionTest.scala
index 2dbcccf967e23..4d7f6cbd3655d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/UserDefinedTableFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/UserDefinedTableFunctionTest.scala
@@ -24,7 +24,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.types.Row
 import org.apache.flink.table.utils.TableTestUtil._
 import org.apache.flink.table.utils.{PojoTableFunc, TableFunc2, _}
-import org.apache.flink.table.api.{TableEnvironment, Types}
+import org.apache.flink.table.api.{Table, TableEnvironment, Types}
 import org.junit.Test
 import org.mockito.Mockito._
 
@@ -54,24 +54,24 @@ class UserDefinedTableFunctionTest extends TableTestBase {
     val func1 = new TableFunc1
     javaTableEnv.registerFunction("func1", func1)
     var scalaTable = in1.join(func1('c) as 's).select('c, 's)
-    var javaTable = in2.join("func1(c).as(s)").select("c, s")
+    var javaTable = in2.join(new Table(javaTableEnv, "func1(c).as(s)")).select("c, s")
     verifyTableEquals(scalaTable, javaTable)
 
     // test left outer join
     scalaTable = in1.leftOuterJoin(func1('c) as 's).select('c, 's)
-    javaTable = in2.leftOuterJoin("as(func1(c), s)").select("c, s")
+    javaTable = in2.leftOuterJoin(new Table(javaTableEnv, "as(func1(c), s)")).select("c, s")
     verifyTableEquals(scalaTable, javaTable)
 
     // test overloading
     scalaTable = in1.join(func1('c, "$") as 's).select('c, 's)
-    javaTable = in2.join("func1(c, '$') as (s)").select("c, s")
+    javaTable = in2.join(new Table(javaTableEnv, "func1(c, '$') as (s)")).select("c, s")
     verifyTableEquals(scalaTable, javaTable)
 
     // test custom result type
     val func2 = new TableFunc2
     javaTableEnv.registerFunction("func2", func2)
     scalaTable = in1.join(func2('c) as ('name, 'len)).select('c, 'name, 'len)
-    javaTable = in2.join("func2(c).as(name, len)").select("c, name, len")
+    javaTable = in2.join(new Table(javaTableEnv, "func2(c).as(name, len)")).select("c, name, len")
     verifyTableEquals(scalaTable, javaTable)
 
     // test hierarchy generic type
@@ -79,7 +79,7 @@ class UserDefinedTableFunctionTest extends TableTestBase {
     javaTableEnv.registerFunction("hierarchy", hierarchy)
     scalaTable = in1.join(hierarchy('c) as ('name, 'adult, 'len))
       .select('c, 'name, 'len, 'adult)
-    javaTable = in2.join("AS(hierarchy(c), name, adult, len)")
+    javaTable = in2.join(new Table(javaTableEnv, "AS(hierarchy(c), name, adult, len)"))
       .select("c, name, len, adult")
     verifyTableEquals(scalaTable, javaTable)
 
@@ -88,21 +88,21 @@ class UserDefinedTableFunctionTest extends TableTestBase {
     javaTableEnv.registerFunction("pojo", pojo)
     scalaTable = in1.join(pojo('c))
       .select('c, 'name, 'age)
-    javaTable = in2.join("pojo(c)")
+    javaTable = in2.join(new Table(javaTableEnv, "pojo(c)"))
       .select("c, name, age")
     verifyTableEquals(scalaTable, javaTable)
 
     // test with filter
     scalaTable = in1.join(func2('c) as ('name, 'len))
       .select('c, 'name, 'len).filter('len > 2)
-    javaTable = in2.join("func2(c) as (name, len)")
+    javaTable = in2.join(new Table(javaTableEnv, "func2(c) as (name, len)"))
       .select("c, name, len").filter("len > 2")
     verifyTableEquals(scalaTable, javaTable)
 
     // test with scalar function
     scalaTable = in1.join(func1('c.substring(2)) as 's)
       .select('a, 'c, 's)
-    javaTable = in2.join("func1(substring(c, 2)) as (s)")
+    javaTable = in2.join(new Table(javaTableEnv, "func1(substring(c, 2)) as (s)"))
       .select("a, c, s")
     verifyTableEquals(scalaTable, javaTable)
   }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UserDefinedTableFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UserDefinedTableFunctionTest.scala
index 56b9fdbe1a377..3edfd8c0cb0db 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UserDefinedTableFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UserDefinedTableFunctionTest.scala
@@ -57,24 +57,24 @@ class UserDefinedTableFunctionTest extends TableTestBase {
     val func1 = new TableFunc1
     javaTableEnv.registerFunction("func1", func1)
     var scalaTable = in1.join(func1('c) as 's).select('c, 's)
-    var javaTable = in2.join("func1(c).as(s)").select("c, s")
+    var javaTable = in2.join(new Table(javaTableEnv, "func1(c).as(s)")).select("c, s")
     verifyTableEquals(scalaTable, javaTable)
 
     // test left outer join
     scalaTable = in1.leftOuterJoin(func1('c) as 's).select('c, 's)
-    javaTable = in2.leftOuterJoin("as(func1(c), s)").select("c, s")
+    javaTable = in2.leftOuterJoin(new Table(javaTableEnv, "func1(c)").as("s")).select("c, s")
     verifyTableEquals(scalaTable, javaTable)
 
     // test overloading
     scalaTable = in1.join(func1('c, "$") as 's).select('c, 's)
-    javaTable = in2.join("func1(c, '$') as (s)").select("c, s")
+    javaTable = in2.join(new Table(javaTableEnv, "func1(c, '$') as (s)")).select("c, s")
     verifyTableEquals(scalaTable, javaTable)
 
     // test custom result type
     val func2 = new TableFunc2
     javaTableEnv.registerFunction("func2", func2)
     scalaTable = in1.join(func2('c) as ('name, 'len)).select('c, 'name, 'len)
-    javaTable = in2.join("func2(c).as(name, len)").select("c, name, len")
+    javaTable = in2.join(new Table(javaTableEnv, "func2(c).as(name, len)")).select("c, name, len")
     verifyTableEquals(scalaTable, javaTable)
 
     // test hierarchy generic type
@@ -82,7 +82,7 @@ class UserDefinedTableFunctionTest extends TableTestBase {
     javaTableEnv.registerFunction("hierarchy", hierarchy)
     scalaTable = in1.join(hierarchy('c) as ('name, 'adult, 'len))
       .select('c, 'name, 'len, 'adult)
-    javaTable = in2.join("AS(hierarchy(c), name, adult, len)")
+    javaTable = in2.join(new Table(javaTableEnv, "AS(hierarchy(c), name, adult, len)"))
       .select("c, name, len, adult")
     verifyTableEquals(scalaTable, javaTable)
 
@@ -91,21 +91,21 @@ class UserDefinedTableFunctionTest extends TableTestBase {
     javaTableEnv.registerFunction("pojo", pojo)
     scalaTable = in1.join(pojo('c))
       .select('c, 'name, 'age)
-    javaTable = in2.join("pojo(c)")
+    javaTable = in2.join(new Table(javaTableEnv, "pojo(c)"))
       .select("c, name, age")
     verifyTableEquals(scalaTable, javaTable)
 
     // test with filter
     scalaTable = in1.join(func2('c) as ('name, 'len))
       .select('c, 'name, 'len).filter('len > 2)
-    javaTable = in2.join("func2(c) as (name, len)")
+    javaTable = in2.join(new Table(javaTableEnv, "func2(c) as (name, len)"))
       .select("c, name, len").filter("len > 2")
     verifyTableEquals(scalaTable, javaTable)
 
     // test with scalar function
     scalaTable = in1.join(func1('c.substring(2)) as 's)
       .select('a, 'c, 's)
-    javaTable = in2.join("func1(substring(c, 2)) as (s)")
+    javaTable = in2.join(new Table(javaTableEnv, "func1(substring(c, 2)) as (s)"))
       .select("a, c, s")
     verifyTableEquals(scalaTable, javaTable)
 
@@ -114,8 +114,103 @@ class UserDefinedTableFunctionTest extends TableTestBase {
       tableEnv.registerFunction("func3", ObjectTableFunction), "Scala object")
     expectExceptionThrown(
       javaTableEnv.registerFunction("func3", ObjectTableFunction), "Scala object")
+    expectExceptionThrown(in1.join(ObjectTableFunction('a, 1)), "Scala object")
+
+  }
+
+  @Test
+  def testInvalidTableFunctions(): Unit = {
+    // mock
+    val ds = mock(classOf[DataStream[Row]])
+    val jDs = mock(classOf[JDataStream[Row]])
+    val typeInfo = new RowTypeInfo(Seq(Types.INT, Types.LONG, Types.STRING): _*)
+    when(ds.javaStream).thenReturn(jDs)
+    when(jDs.getType).thenReturn(typeInfo)
+
+    // Scala environment
+    val env = mock(classOf[ScalaExecutionEnv])
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+    val in1 = ds.toTable(tableEnv).as('a, 'b, 'c)
+
+    // Java environment
+    val javaEnv = mock(classOf[JavaExecutionEnv])
+    val javaTableEnv = TableEnvironment.getTableEnvironment(javaEnv)
+    val in2 = javaTableEnv.fromDataStream(jDs).as("a, b, c")
+
+    val func1 = new TableFunc1
+    javaTableEnv.registerFunction("func1", func1)
+
+    // table function call select
+    expectExceptionThrown(
+      func1('c).select("f0"),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call select
+    expectExceptionThrown(
+      func1('c).select('f0),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call writeToSink
     expectExceptionThrown(
-      in1.join(ObjectTableFunction('a, 1)), "Scala object")
+      func1('c).writeToSink(null),
+      "Cannot translate a query with an unbounded table function call."
+    )
+
+    // table function call distinct
+    expectExceptionThrown(
+      func1('c).distinct(),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call filter
+    expectExceptionThrown(
+      func1('c).filter('f0 === "?"),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call filter
+    expectExceptionThrown(
+      func1('c).filter("f0 = '?'"),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call limit
+    expectExceptionThrown(
+      func1('c).orderBy('f0).limit(3),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call limit
+    expectExceptionThrown(
+      func1('c).orderBy('f0).limit(0, 3),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call orderBy
+    expectExceptionThrown(
+      func1('c).orderBy("f0"),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call orderBy
+    expectExceptionThrown(
+      func1('c).orderBy('f0),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call where
+    expectExceptionThrown(
+      func1('c).where("f0 = '?'"),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
+
+    // table function call where
+    expectExceptionThrown(
+      func1('c).where('f0 === "?"),
+      "TableFunction can only be used in join and leftOuterJoin."
+    )
 
   }
 
@@ -137,7 +232,9 @@ class UserDefinedTableFunctionTest extends TableTestBase {
 
     //============ throw exception when table function is not registered =========
     // Java Table API call
-    expectExceptionThrown(t.join("nonexist(a)"), "Undefined function: NONEXIST")
+    expectExceptionThrown(
+      t.join(new Table(util.tEnv, "nonexist(a)")
+      ), "Undefined function: NONEXIST")
     // SQL API call
     expectExceptionThrown(
       util.tEnv.sql("SELECT * FROM MyTable, LATERAL TABLE(nonexist(a))"),
@@ -145,11 +242,12 @@ class UserDefinedTableFunctionTest extends TableTestBase {
 
 
     //========= throw exception when the called function is a scalar function ====
-    util.addFunction("func0", Func0)
+    util.tEnv.registerFunction("func0", Func0)
+
     // Java Table API call
     expectExceptionThrown(
-      t.join("func0(a)"),
-      "only accept expressions that define table functions",
+      t.join(new Table(util.tEnv, "func0(a)")),
+      "only accept String that define table function",
       classOf[TableException])
     // SQL API call
     // NOTE: it doesn't throw an exception but an AssertionError, maybe a Calcite bug
@@ -162,7 +260,7 @@ class UserDefinedTableFunctionTest extends TableTestBase {
     // Java Table API call
     util.addFunction("func2", new TableFunc2)
     expectExceptionThrown(
-      t.join("func2(c, c)"),
+      t.join(new Table(util.tEnv, "func2(c, c)")),
       "Given parameters of function 'FUNC2' do not match any signature")
     // SQL API call
     expectExceptionThrown(

From 40721466e8592afb7111b822fc8a81220f58926d Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Wed, 3 May 2017 15:16:07 +0200
Subject: [PATCH 132/253] [hotfix] Add check for aligned assigner in
 AllWindowedStream.evictor()

This check is present in WindowedStream but was missing on
AllWindowedStream.
---
 .../flink/streaming/api/datastream/AllWindowedStream.java    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
index 24003301b33be..772e635df93a7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
@@ -50,6 +50,7 @@
 import org.apache.flink.streaming.api.functions.windowing.ReduceApplyAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.ReduceApplyProcessAllWindowFunction;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.windowing.assigners.BaseAlignedWindowAssigner;
 import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
 import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;
 import org.apache.flink.streaming.api.windowing.evictors.Evictor;
@@ -178,6 +179,10 @@ public AllWindowedStream<T, W> evictor(Evictor<? super T, ? super W> evictor) {
 			throw new UnsupportedOperationException("Cannot use a merging WindowAssigner with an Evictor.");
 		}
 
+		if (windowAssigner instanceof BaseAlignedWindowAssigner) {
+			throw new UnsupportedOperationException("Cannot use a " + windowAssigner.getClass().getSimpleName() + " with an Evictor.");
+		}
+
 		this.evictor = evictor;
 		return this;
 	}

From 1f649ef84579de8d58d894825f4edb3550c6f42e Mon Sep 17 00:00:00 2001
From: Aljoscha Krettek <aljoscha.krettek@gmail.com>
Date: Wed, 3 May 2017 15:18:05 +0200
Subject: [PATCH 133/253] [FLINK-5933] Allow Evictor for merging windows

---
 .../api/datastream/AllWindowedStream.java     |  4 --
 .../api/datastream/WindowedStream.java        |  4 --
 .../windowing/AllWindowTranslationTest.java   | 39 +++++++++++++++++++
 .../windowing/WindowOperatorTest.java         | 32 ---------------
 .../windowing/WindowTranslationTest.java      | 24 ++++++++++++
 .../api/scala/AllWindowTranslationTest.scala  | 35 +++++++++++++++++
 .../api/scala/WindowTranslationTest.scala     | 35 +++++++++++++++++
 7 files changed, 133 insertions(+), 40 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
index 772e635df93a7..0d953a9fdc9a9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
@@ -175,10 +175,6 @@ public AllWindowedStream<T, W> sideOutputLateData(OutputTag<T> outputTag) {
 	 */
 	@PublicEvolving
 	public AllWindowedStream<T, W> evictor(Evictor<? super T, ? super W> evictor) {
-		if (windowAssigner instanceof MergingWindowAssigner) {
-			throw new UnsupportedOperationException("Cannot use a merging WindowAssigner with an Evictor.");
-		}
-
 		if (windowAssigner instanceof BaseAlignedWindowAssigner) {
 			throw new UnsupportedOperationException("Cannot use a " + windowAssigner.getClass().getSimpleName() + " with an Evictor.");
 		}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
index 688e9b6df4cc0..2d7dafef316a2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
@@ -191,10 +191,6 @@ public WindowedStream<T, K, W> sideOutputLateData(OutputTag<T> outputTag) {
 	 */
 	@PublicEvolving
 	public WindowedStream<T, K, W> evictor(Evictor<? super T, ? super W> evictor) {
-		if (windowAssigner instanceof MergingWindowAssigner) {
-			throw new UnsupportedOperationException("Cannot use a merging WindowAssigner with an Evictor.");
-		}
-
 		if (windowAssigner instanceof BaseAlignedWindowAssigner) {
 			throw new UnsupportedOperationException("Cannot use a " + windowAssigner.getClass().getSimpleName() + " with an Evictor.");
 		}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
index 468f14c2b0eb8..81a9275f953c8 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
@@ -235,6 +235,30 @@ public void clear(TimeWindow window, TriggerContext ctx) throws Exception {}
 		fail("The trigger call should fail.");
 	}
 
+	@Test
+	@SuppressWarnings("rawtypes")
+	public void testMergingWindowsWithEvictor() throws Exception {
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
+
+		DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
+
+		DataStream<Tuple3<String, String, Integer>> window1 = source
+				.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)))
+				.evictor(CountEvictor.of(5))
+				.process(new TestProcessAllWindowFunction());
+
+		OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
+		OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
+		Assert.assertTrue(operator instanceof WindowOperator);
+		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
+		Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
+		Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
+		Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);
+
+		processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
+	}
+
 	// ------------------------------------------------------------------------
 	//  reduce() translation tests
 	// ------------------------------------------------------------------------
@@ -1515,4 +1539,19 @@ public void apply(
 			}
 		}
 	}
+
+	private static class TestProcessAllWindowFunction
+			extends ProcessAllWindowFunction<Tuple2<String, Integer>, Tuple3<String, String, Integer>, TimeWindow> {
+
+		@Override
+		public void process(Context ctx,
+				Iterable<Tuple2<String, Integer>> values,
+				Collector<Tuple3<String, String, Integer>> out) throws Exception {
+
+			for (Tuple2<String, Integer> in : values) {
+				out.collect(new Tuple3<>(in.f0, in.f0, in.f1));
+			}
+		}
+	}
+
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
index 474548cb1b641..ce845013811f3 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
@@ -800,38 +800,6 @@ public void testSessionWindowsWithContinuousEventTimeTrigger() throws Exception
 		testHarness.close();
 	}
 
-	@Test
-	public void testMergeAndEvictor() throws Exception {
-		// verify that merging WindowAssigner and Evictor cannot be used together
-
-		StreamExecutionEnvironment env = LocalStreamEnvironment.createLocalEnvironment();
-
-		WindowedStream<String, String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao")
-				.keyBy(new KeySelector<String, String>() {
-					private static final long serialVersionUID = -887743259776124087L;
-
-					@Override
-					public String getKey(String value) throws Exception {
-						return value;
-					}
-				})
-				.window(EventTimeSessionWindows.withGap(Time.seconds(5)));
-
-		try {
-			windowedStream.evictor(CountEvictor.of(13));
-
-		} catch (UnsupportedOperationException e) {
-			// expected
-			// use a catch to ensure that the exception is thrown by the fold
-			return;
-		}
-
-		fail("The evictor call should fail.");
-
-		env.execute();
-
-	}
-
 	@Test
 	@SuppressWarnings("unchecked")
 	/**
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java
index b899948d41885..5071c37f36f4a 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java
@@ -253,6 +253,30 @@ public void clear(TimeWindow window, TriggerContext ctx) throws Exception {}
 		fail("The trigger call should fail.");
 	}
 
+	@Test
+	@SuppressWarnings("rawtypes")
+	public void testMergingWindowsWithEvictor() throws Exception {
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
+
+		DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
+
+		DataStream<Tuple3<String, String, Integer>> window1 = source
+				.keyBy(new TupleKeySelector())
+				.window(EventTimeSessionWindows.withGap(Time.seconds(5)))
+				.evictor(CountEvictor.of(5))
+				.process(new TestProcessWindowFunction());
+
+		OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
+		OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
+		Assert.assertTrue(operator instanceof WindowOperator);
+		WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
+		Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
+		Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
+		Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);
+
+		processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
+	}
 
 	// ------------------------------------------------------------------------
 	//  Reduce Translation Tests
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala
index ee9f50c6b1633..e8d5a1221bbb9 100644
--- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AllWindowTranslationTest.scala
@@ -174,6 +174,41 @@ class AllWindowTranslationTest {
     fail("The trigger call should fail.")
   }
 
+  @Test
+  def testMergingWindowsWithEvictor() {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime)
+
+    val source = env.fromElements(("hello", 1), ("hello", 2))
+
+    val window1 = source
+      .windowAll(EventTimeSessionWindows.withGap(Time.seconds(1)))
+      .evictor(CountEvictor.of(2))
+      .process(new TestProcessAllWindowFunction)
+
+    val transform = window1
+      .javaStream
+      .getTransformation
+      .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]]
+
+    val operator = transform.getOperator
+    assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]])
+
+    val winOperator = operator
+      .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]]
+
+    assertTrue(winOperator.getTrigger.isInstanceOf[EventTimeTrigger])
+    assertTrue(winOperator.getWindowAssigner.isInstanceOf[EventTimeSessionWindows])
+    assertTrue(winOperator.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]])
+
+    processElementAndEnsureOutput[String, (String, Int), (String, Int)](
+      winOperator,
+      winOperator.getKeySelector,
+      BasicTypeInfo.STRING_TYPE_INFO,
+      ("hello", 1))
+  }
+
+
   // ------------------------------------------------------------------------
   //  reduce() translation tests
   // ------------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala
index 600645b9b4f2f..cc55f0d6106b5 100644
--- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/WindowTranslationTest.scala
@@ -178,6 +178,41 @@ class WindowTranslationTest {
     fail("The trigger call should fail.")
   }
 
+  @Test
+  def testMergingWindowsWithEvictor() {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime)
+
+    val source = env.fromElements(("hello", 1), ("hello", 2))
+
+    val window1 = source
+      .keyBy(_._1)
+      .window(EventTimeSessionWindows.withGap(Time.seconds(1)))
+      .evictor(CountEvictor.of(2))
+      .process(new TestProcessWindowFunction)
+
+    val transform = window1
+      .javaStream
+      .getTransformation
+      .asInstanceOf[OneInputTransformation[(String, Int), (String, Int)]]
+
+    val operator = transform.getOperator
+    assertTrue(operator.isInstanceOf[WindowOperator[_, _, _, _, _ <: Window]])
+
+    val winOperator = operator
+      .asInstanceOf[WindowOperator[String, (String, Int), _, (String, Int), _ <: Window]]
+
+    assertTrue(winOperator.getTrigger.isInstanceOf[EventTimeTrigger])
+    assertTrue(winOperator.getWindowAssigner.isInstanceOf[EventTimeSessionWindows])
+    assertTrue(winOperator.getStateDescriptor.isInstanceOf[ListStateDescriptor[_]])
+
+    processElementAndEnsureOutput[String, (String, Int), (String, Int)](
+      winOperator,
+      winOperator.getKeySelector,
+      BasicTypeInfo.STRING_TYPE_INFO,
+      ("hello", 1))
+  }
+
   // --------------------------------------------------------------------------
   //  reduce() tests
   // --------------------------------------------------------------------------

From dfed745f3683f254ea5c4f57449971e6a4e6f508 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 6 Apr 2017 14:33:47 +0200
Subject: [PATCH 134/253] [FLINK-6274] Replaces usages of org.codehaus.jackson

This closes #3780.
---
 .../streaming/examples/twitter/TwitterExample.java    |  4 ++--
 .../util/AbstractDeserializationSchemaTest.java       |  3 +--
 .../test/optimizer/jsonplan/DumpCompiledPlanTest.java |  6 +++---
 .../test/optimizer/jsonplan/PreviewPlanDumpTest.java  | 11 ++++++-----
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterExample.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterExample.java
index 4f13f3430f6cf..6d1bce51c67fe 100644
--- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterExample.java
+++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterExample.java
@@ -17,6 +17,8 @@
 
 package org.apache.flink.streaming.examples.twitter;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.utils.ParameterTool;
@@ -25,8 +27,6 @@
 import org.apache.flink.streaming.connectors.twitter.TwitterSource;
 import org.apache.flink.streaming.examples.twitter.util.TwitterExampleData;
 import org.apache.flink.util.Collector;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
 
 import java.util.StringTokenizer;
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java
index 1ae144cfc73eb..77178d77dd307 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractDeserializationSchemaTest.java
@@ -18,14 +18,13 @@
 
 package org.apache.flink.streaming.util;
 
+import com.fasterxml.jackson.databind.util.JSONPObject;
 import org.apache.flink.api.common.functions.InvalidTypesException;
 import org.apache.flink.api.common.typeinfo.TypeHint;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.util.serialization.AbstractDeserializationSchema;
 
-import org.codehaus.jackson.map.util.JSONPObject;
-
 import org.junit.Test;
 
 import java.io.IOException;
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java
index 9f9d42be982cd..6932fd20a1fd5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java
@@ -18,6 +18,9 @@
 
 package org.apache.flink.test.optimizer.jsonplan;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.Plan;
 import org.apache.flink.client.program.OptimizerPlanEnvironment;
@@ -31,9 +34,6 @@
 import org.apache.flink.examples.java.relational.WebLogAnalysis;
 import org.apache.flink.examples.java.wordcount.WordCount;
 import org.apache.flink.optimizer.util.CompilerTestBase;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.JsonParser;
 import org.junit.Assert;
 import org.junit.Test;
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java
index 6f02ab6581c42..14f66560d5daa 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java
@@ -20,6 +20,9 @@
 
 import java.util.List;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
 import org.apache.flink.api.common.Plan;
 import org.apache.flink.client.program.OptimizerPlanEnvironment;
 import org.apache.flink.client.program.PreviewPlanEnvironment;
@@ -32,9 +35,6 @@
 import org.apache.flink.optimizer.dag.DataSinkNode;
 import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
 import org.apache.flink.optimizer.util.CompilerTestBase;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.JsonParser;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -168,8 +168,9 @@ private void dump(Plan p) {
 			List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(p);
 			PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
 			String json = dumper.getPactPlanAsJSON(sinks);
-			JsonParser parser = new JsonFactory().createJsonParser(json);
-			while (parser.nextToken() != null);
+			try (JsonParser parser = new JsonFactory().createParser(json)) {
+				while (parser.nextToken() != null) ;
+			}
 		} catch (JsonParseException e) {
 			e.printStackTrace();
 			Assert.fail("JSON Generator produced malformatted output: " + e.getMessage());

From 450338c10d4678610da2bc3e2b6394d2a129b69c Mon Sep 17 00:00:00 2001
From: WangTaoTheTonic <wangtao111@huawei.com>
Date: Tue, 11 Apr 2017 19:48:52 +0800
Subject: [PATCH 135/253] [FLINK-3709] Prevent caching of outdated suspended
 ExecutionGraphs

This closes #3709.
---
 .../flink/runtime/webmonitor/ExecutionGraphHolder.java   | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/ExecutionGraphHolder.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/ExecutionGraphHolder.java
index 3d0cfc0174b08..f9faa8573bdfb 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/ExecutionGraphHolder.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/ExecutionGraphHolder.java
@@ -22,6 +22,7 @@
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
 import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 
 import org.slf4j.Logger;
@@ -67,14 +68,18 @@ public ExecutionGraphHolder(FiniteDuration timeout) {
 	public AccessExecutionGraph getExecutionGraph(JobID jid, ActorGateway jobManager) {
 		AccessExecutionGraph cached = cache.get(jid);
 		if (cached != null) {
-			return cached;
+			if (cached.getState() == JobStatus.SUSPENDED) {
+				cache.remove(jid);
+			} else {
+				return cached;
+			}
 		}
 
 		try {
 			if (jobManager != null) {
 				Future<Object> future = jobManager.ask(new JobManagerMessages.RequestJob(jid), timeout);
 				Object result = Await.result(future, timeout);
-				
+
 				if (result instanceof JobManagerMessages.JobNotFound) {
 					return null;
 				}

From 0bf1d464de44b8352c9a075b0bf848c5bb9e0550 Mon Sep 17 00:00:00 2001
From: shijinkui <shijinkui@huawei.com>
Date: Tue, 25 Apr 2017 20:09:15 +0800
Subject: [PATCH 136/253] [FLINK-6367] support custom header settings of allow
 origin

This closes #3769.
---
 docs/setup/config.md                                       | 2 ++
 .../org/apache/flink/configuration/ConfigConstants.java    | 6 ++++++
 .../flink/runtime/webmonitor/RuntimeMonitorHandler.java    | 7 ++++++-
 .../apache/flink/runtime/webmonitor/WebMonitorConfig.java  | 4 ++++
 .../apache/flink/runtime/webmonitor/WebRuntimeMonitor.java | 7 ++++---
 5 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/docs/setup/config.md b/docs/setup/config.md
index 250d92ff31fd7..a3daac000b7c7 100644
--- a/docs/setup/config.md
+++ b/docs/setup/config.md
@@ -375,6 +375,8 @@ These parameters allow for advanced tuning. The default values are sufficient wh
 
 - `jobmanager.web.ssl.enabled`: Enable https access to the web frontend. This is applicable only when the global ssl flag security.ssl.enabled is set to true (DEFAULT: `true`).
 
+- `jobmanager.web.access-control-allow-origin`: Enable custom access control parameter for allow origin header, default is `*`.
+
 ### File Systems
 
 The parameters define the behavior of tasks that create result files.
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 61c1b27ecbeed..975a3d49cf052 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -1333,6 +1333,12 @@ public final class ConfigConstants {
 		key("jobmanager.web.address")
 			.noDefaultValue();
 
+	/** The config parameter defining the Access-Control-Allow-Origin header for all
+	 * responses from the web-frontend. */
+	public static final ConfigOption<String> JOB_MANAGER_WEB_ACCESS_CONTROL_ALLOW_ORIGIN =
+		key("jobmanager.web.access-control-allow-origin")
+			.defaultValue("*");
+
 	/** The config key for the port of the JobManager web frontend.
 	 * Setting this value to {@code -1} disables the web frontend. */
 	public static final int DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT = 8081;
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/RuntimeMonitorHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/RuntimeMonitorHandler.java
index 3d7fbed4020bd..90fe2e065f929 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/RuntimeMonitorHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/RuntimeMonitorHandler.java
@@ -66,14 +66,19 @@ public class RuntimeMonitorHandler extends RuntimeMonitorHandlerBase {
 
 	private final RequestHandler handler;
 
+	private final String allowOrigin;
+
 	public RuntimeMonitorHandler(
+			WebMonitorConfig cfg,
 			RequestHandler handler,
 			JobManagerRetriever retriever,
 			Future<String> localJobManagerAddressFuture,
 			FiniteDuration timeout,
 			boolean httpsEnabled) {
+
 		super(retriever, localJobManagerAddressFuture, timeout, httpsEnabled);
 		this.handler = checkNotNull(handler);
+		this.allowOrigin = cfg.getAllowOrigin();
 	}
 
 	@Override
@@ -122,7 +127,7 @@ protected void respondAsLeader(ChannelHandlerContext ctx, Routed routed, ActorGa
 			LOG.debug("Error while handling request", e);
 		}
 
-		response.headers().set(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+		response.headers().set(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN, allowOrigin);
 		// Content-Encoding:utf-8
 		response.headers().set(HttpHeaders.Names.CONTENT_ENCODING, ENCODING.name());
 
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
index 18fc5e825d8fe..11e94b0a32e75 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
@@ -78,4 +78,8 @@ public boolean isProgramSubmitEnabled() {
 			ConfigConstants.JOB_MANAGER_WEB_SUBMIT_ENABLED_KEY,
 			ConfigConstants.DEFAULT_JOB_MANAGER_WEB_SUBMIT_ENABLED);
 	}
+
+	public String getAllowOrigin() {
+		return config.getString(ConfigConstants.JOB_MANAGER_WEB_ACCESS_CONTROL_ALLOW_ORIGIN);
+	}
 }
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
index c1e6229d69ce3..f83fa272ab5b1 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
@@ -136,6 +136,8 @@ public class WebRuntimeMonitor implements WebMonitor {
 
 	private final BackPressureStatsTracker backPressureStatsTracker;
 
+	private final WebMonitorConfig cfg;
+
 	private AtomicBoolean cleanedUp = new AtomicBoolean();
 
 	private ExecutorService executorService;
@@ -150,8 +152,7 @@ public WebRuntimeMonitor(
 		this.leaderRetrievalService = checkNotNull(leaderRetrievalService);
 		this.timeout = AkkaUtils.getTimeout(config);
 		this.retriever = new JobManagerRetriever(this, actorSystem, AkkaUtils.getTimeout(config), timeout);
-		
-		final WebMonitorConfig cfg = new WebMonitorConfig(config);
+		this.cfg = new WebMonitorConfig(config);
 
 		final String configuredAddress = cfg.getWebFrontendAddress();
 
@@ -515,7 +516,7 @@ private void POST(Router router, RuntimeMonitorHandlerBase handler) {
 	// ------------------------------------------------------------------------
 
 	private RuntimeMonitorHandler handler(RequestHandler handler) {
-		return new RuntimeMonitorHandler(handler, retriever, jobManagerAddressPromise.future(), timeout,
+		return new RuntimeMonitorHandler(cfg, handler, retriever, jobManagerAddressPromise.future(), timeout,
 			serverSSLContext !=  null);
 	}
 

From 3d25522df5bfef49d55d181c862359421cbed0a3 Mon Sep 17 00:00:00 2001
From: Zohar Mizrahi <zohar.mizrahi@parallelmachines.com>
Date: Sun, 9 Apr 2017 12:11:57 +0300
Subject: [PATCH 137/253] [FLINK-6177] Add support for "Distributed Cache" in
 streaming applications

This closes #3741.
---
 .../StreamExecutionEnvironment.java           | 48 +++++++++++++++
 .../api/graph/StreamingJobGraphGenerator.java |  6 ++
 .../scala/StreamExecutionEnvironment.scala    | 48 +++++++++++++++
 .../DistributedCacheTest.java                 | 60 +++++++++----------
 4 files changed, 130 insertions(+), 32 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index e8279454dfc7f..aad3a4b74c33b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -33,6 +33,7 @@
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.InvalidProgramException;
 import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.cache.DistributedCache;
 import org.apache.flink.api.common.functions.InvalidTypesException;
 import org.apache.flink.api.common.functions.StoppableFunction;
 import org.apache.flink.api.common.io.FileInputFormat;
@@ -46,6 +47,7 @@
 import org.apache.flink.api.java.ClosureCleaner;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.io.TextInputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.typeutils.MissingTypeInfo;
 import org.apache.flink.api.java.typeutils.PojoTypeInfo;
@@ -136,6 +138,8 @@ public abstract class StreamExecutionEnvironment {
 	/** The time characteristic used by the data streams. */
 	private TimeCharacteristic timeCharacteristic = DEFAULT_TIME_CHARACTERISTIC;
 
+	protected final List<Tuple2<String, DistributedCache.DistributedCacheEntry>> cacheFile = new ArrayList<>();
+
 
 	// --------------------------------------------------------------------------------------------
 	// Constructor and Properties
@@ -148,6 +152,13 @@ public ExecutionConfig getConfig() {
 		return config;
 	}
 
+	/**
+	* Get the list of cached files that were registered for distribution among the task managers.
+	*/
+	public List<Tuple2<String, DistributedCache.DistributedCacheEntry>> getCachedFiles() {
+		return cacheFile;
+	}
+
 	/**
 	 * Sets the parallelism for operations executed through this environment.
 	 * Setting a parallelism of x here will cause all operators (such as map,
@@ -1774,4 +1785,41 @@ protected static void initializeContextEnvironment(StreamExecutionEnvironmentFac
 	protected static void resetContextEnvironment() {
 		contextEnvironmentFactory = null;
 	}
+
+	/**
+	 * Registers a file at the distributed cache under the given name. The file will be accessible
+	 * from any user-defined function in the (distributed) runtime under a local path. Files
+	 * may be local files (as long as all relevant workers have access to it), or files in a distributed file system.
+	 * The runtime will copy the files temporarily to a local cache, if needed.
+	 *
+	 * <p>The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
+	 * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
+	 * {@link org.apache.flink.api.common.cache.DistributedCache} via
+	 * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
+	 *
+	 * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or "hdfs://host:port/and/path")
+	 * @param name The name under which the file is registered.
+	 */
+	public void registerCachedFile(String filePath, String name) {
+		registerCachedFile(filePath, name, false);
+	}
+
+	/**
+	 * Registers a file at the distributed cache under the given name. The file will be accessible
+	 * from any user-defined function in the (distributed) runtime under a local path. Files
+	 * may be local files (as long as all relevant workers have access to it), or files in a distributed file system.
+	 * The runtime will copy the files temporarily to a local cache, if needed.
+	 *
+	 * <p>The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
+	 * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
+	 * {@link org.apache.flink.api.common.cache.DistributedCache} via
+	 * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
+	 *
+	 * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or "hdfs://host:port/and/path")
+	 * @param name The name under which the file is registered.
+	 * @param executable flag indicating whether the file should be executable
+	 */
+	public void registerCachedFile(String filePath, String name, boolean executable) {
+		this.cacheFile.add(new Tuple2<>(name, new DistributedCache.DistributedCacheEntry(filePath, executable)));
+	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index aa0f08d20b9b1..b3a6cf8260136 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -29,6 +29,7 @@
 import org.apache.commons.lang3.StringUtils;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.cache.DistributedCache;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.operators.ResourceSpec;
 import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
@@ -151,6 +152,11 @@ private JobGraph createJobGraph() {
 
 		configureCheckpointing();
 
+		// add registered cache file into job configuration
+		for (Tuple2<String, DistributedCache.DistributedCacheEntry> e : streamGraph.getEnvironment().getCachedFiles()) {
+			DistributedCache.writeFileInfoToConfig(e.f0, e.f1, jobGraph.getJobConfiguration());
+		}
+
 		// set the ExecutionConfig last when it has been finalized
 		try {
 			jobGraph.setExecutionConfig(streamGraph.getExecutionConfig());
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
index 0b2587ecee241..742baf9315452 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -49,6 +49,11 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
    */
   def getConfig = javaEnv.getConfig
 
+  /**
+    * Gets cache files.
+    */
+  def getCachedFiles = javaEnv.getCachedFiles
+
   /**
    * Sets the parallelism for operations executed through this environment.
    * Setting a parallelism of x here will cause all operators (such as join, map, reduce) to run
@@ -668,6 +673,49 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
     }
     f
   }
+
+
+  /**
+    * Registers a file at the distributed cache under the given name. The file will be accessible
+    * from any user-defined function in the (distributed) runtime under a local path. Files
+    * may be local files (as long as all relevant workers have access to it), or files in a
+    * distributed file system. The runtime will copy the files temporarily to a local cache,
+    * if needed.
+    * <p>
+    * The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs
+    * via {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and
+    * provides access {@link org.apache.flink.api.common.cache.DistributedCache} via
+    * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
+    *
+    * @param filePath The path of the file, as a URI (e.g. "file:///some/path" or
+    *                 "hdfs://host:port/and/path")
+    * @param name     The name under which the file is registered.
+    */
+  def registerCachedFile(filePath: String, name: String): Unit = {
+    javaEnv.registerCachedFile(filePath, name)
+  }
+
+
+  /**
+    * Registers a file at the distributed cache under the given name. The file will be accessible
+    * from any user-defined function in the (distributed) runtime under a local path. Files
+    * may be local files (as long as all relevant workers have access to it), or files in a
+    * distributed file system. The runtime will copy the files temporarily to a local cache,
+    * if needed.
+    * <p>
+    * The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs
+    * via {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and
+    * provides access {@link org.apache.flink.api.common.cache.DistributedCache} via
+    * {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
+    *
+    * @param filePath   The path of the file, as a URI (e.g. "file:///some/path" or
+    *                   "hdfs://host:port/and/path")
+    * @param name       The name under which the file is registered.
+    * @param executable flag indicating whether the file should be executable
+    */
+  def registerCachedFile(filePath: String, name: String, executable: Boolean): Unit = {
+    javaEnv.registerCachedFile(filePath, name, executable)
+  }
 }
 
 object StreamExecutionEnvironment {
diff --git a/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java b/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
index 4fb2d952778dc..19bcf76d22164 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
@@ -15,29 +15,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.flink.test.distributedCache;
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+package org.apache.flink.test.distributedCache;
 
 import org.apache.flink.api.common.functions.RichFlatMapFunction;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.tuple.Tuple1;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.test.util.JavaProgramTestBase;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
 import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.*;
+
+import java.util.*;
 
-/**
- * Tests the distributed cache by comparing a text file with a distributed copy.
- */
-public class DistributedCacheTest extends JavaProgramTestBase {
 
+public class DistributedCacheTest extends StreamingMultipleProgramsTestBase {
 	public static final String data
 			= "machen\n"
 			+ "zeit\n"
@@ -45,33 +42,31 @@ public class DistributedCacheTest extends JavaProgramTestBase {
 			+ "keiner\n"
 			+ "meine\n";
 
-	protected String textPath;
 
-	@Override
-	protected void preSubmit() throws Exception {
-		textPath = createTempFile("count.txt", data);
+	@Test
+	public void testStreamingDistributedCache() throws Exception {
+		String textPath = createTempFile("count.txt", data);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.registerCachedFile(textPath, "cache_test");
+		env.readTextFile(textPath).flatMap(new WordChecker());
+		env.execute();
 	}
 
-	@Override
-	protected void testProgram() throws Exception {
+	@Test
+	public void testBatchDistributedCache() throws Exception {
+		String textPath = createTempFile("count.txt", data);
 		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.registerCachedFile(textPath, "cache_test");
-
-		List<Tuple1<String>> result = env
-				.readTextFile(textPath)
-				.flatMap(new WordChecker())
-				.collect();
-
-		compareResultAsTuples(result, data);
+		env.readTextFile(textPath).flatMap(new WordChecker()).count();
 	}
 
 	public static class WordChecker extends RichFlatMapFunction<String, Tuple1<String>> {
 		private static final long serialVersionUID = 1L;
 
-		private final Set<String> wordList = new HashSet<>();
+		private final List<String> wordList = new ArrayList<>();
 
 		@Override
-		public void open(Configuration conf) throws FileNotFoundException, IOException {
+		public void open(Configuration conf) throws IOException {
 			File file = getRuntimeContext().getDistributedCache().getFile("cache_test");
 			BufferedReader reader = new BufferedReader(new FileReader(file));
 			String tempString;
@@ -83,9 +78,10 @@ public void open(Configuration conf) throws FileNotFoundException, IOException {
 
 		@Override
 		public void flatMap(String word, Collector<Tuple1<String>> out) throws Exception {
-			if (wordList.contains(word)) {
-				out.collect(new Tuple1<>(word));
-			}
+			assertTrue("Unexpected word in stream! wordFromStream: " + word + ", shouldBeOneOf: " +
+				wordList.toString(), wordList.contains(word));
+
+			out.collect(new Tuple1<>(word));
 		}
 	}
 }

From 00d4fb8c4543fdd1e1497ef5fde361beda2d56b3 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 20 Apr 2017 13:26:38 +0200
Subject: [PATCH 138/253] [hotfix] [py] Code cleanup - Functions

---
 .../flink/python/api/functions/util/IdentityGroupReduce.java    | 2 ++
 .../apache/flink/python/api/functions/util/SerializerMap.java   | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/IdentityGroupReduce.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/IdentityGroupReduce.java
index 1e7bbe6e260b9..32fd22a90d2f8 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/IdentityGroupReduce.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/IdentityGroupReduce.java
@@ -12,12 +12,14 @@
  */
 package org.apache.flink.python.api.functions.util;
 
+import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
 import org.apache.flink.util.Collector;
 import org.apache.flink.api.common.functions.GroupReduceFunction;
 
 /*
 Utility function to group and sort data.
 */
+@ForwardedFields("*->*")
 public class IdentityGroupReduce<IN> implements GroupReduceFunction<IN, IN> {
 	@Override
 	public final void reduce(Iterable<IN> values, Collector<IN> out) throws Exception {
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/SerializerMap.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/SerializerMap.java
index 9c39e5fff65ad..116efd4e94943 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/SerializerMap.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/util/SerializerMap.java
@@ -20,7 +20,7 @@
 Utility function to serialize values, usually directly from data sources.
 */
 public class SerializerMap<IN> implements MapFunction<IN, byte[]> {
-	private Serializer<IN> serializer = null;
+	private transient Serializer<IN> serializer;
 
 	@Override
 	@SuppressWarnings("unchecked")

From fd083e55e92b4a37ab7b59c5c703a108675cf5f1 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 20 Apr 2017 13:27:15 +0200
Subject: [PATCH 139/253] [hotfix] [py] Code cleanup - SerializationUtils

---
 .../api/streaming/plan/PythonPlanSender.java  |  1 -
 .../streaming/util/SerializationUtils.java    | 67 +++++++++++--------
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanSender.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanSender.java
index 8b8366a0cc69e..331c67e4561ee 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanSender.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanSender.java
@@ -28,7 +28,6 @@ public PythonPlanSender(OutputStream output) {
 		this.output = new DataOutputStream(output);
 	}
 
-	@SuppressWarnings("unchecked")
 	public void sendRecord(Object record) throws IOException {
 		byte[] data = SerializationUtils.getSerializer(record).serialize(record);
 		output.write(data);
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/SerializationUtils.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/SerializationUtils.java
index f228327e43411..721746af86973 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/SerializationUtils.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/SerializationUtils.java
@@ -20,21 +20,22 @@
 import java.util.ArrayList;
 
 public class SerializationUtils {
-	public static final byte TYPE_BOOLEAN = (byte) 34;
-	public static final byte TYPE_BYTE = (byte) 33;
-	public static final byte TYPE_INTEGER = (byte) 32;
-	public static final byte TYPE_LONG = (byte) 31;
-	public static final byte TYPE_DOUBLE = (byte) 30;
-	public static final byte TYPE_FLOAT = (byte) 29;
-	public static final byte TYPE_STRING = (byte) 28;
-	public static final byte TYPE_BYTES = (byte) 27;
-	public static final byte TYPE_NULL = (byte) 26;
+	public static final byte TYPE_BOOLEAN = 34;
+	public static final byte TYPE_BYTE = 33;
+	public static final byte TYPE_INTEGER = 32;
+	public static final byte TYPE_LONG = 31;
+	public static final byte TYPE_DOUBLE = 30;
+	public static final byte TYPE_FLOAT = 29;
+	public static final byte TYPE_STRING = 28;
+	public static final byte TYPE_BYTES = 27;
+	public static final byte TYPE_NULL = 26;
 
 	private enum SupportedTypes {
 		TUPLE, BOOLEAN, BYTE, BYTES, INTEGER, LONG, FLOAT, DOUBLE, STRING, NULL, CUSTOMTYPEWRAPPER
 	}
 
-	public static Serializer getSerializer(Object value) {
+	@SuppressWarnings("unchecked")
+	public static <IN> Serializer<IN> getSerializer(IN value) {
 		String className = value.getClass().getSimpleName().toUpperCase();
 		if (className.startsWith("TUPLE")) {
 			className = "TUPLE";
@@ -43,35 +44,48 @@ public static Serializer getSerializer(Object value) {
 			className = "BYTES";
 		}
 		SupportedTypes type = SupportedTypes.valueOf(className);
+		Serializer<?> serializer;
 		switch (type) {
 			case TUPLE:
-				return new TupleSerializer((Tuple) value);
+				serializer = new TupleSerializer((Tuple) value);
+				break;
 			case BOOLEAN:
-				return new BooleanSerializer();
+				serializer = new BooleanSerializer();
+				break;
 			case BYTE:
-				return new ByteSerializer();
+				serializer = new ByteSerializer();
+				break;
 			case BYTES:
-				return new BytesSerializer();
+				serializer = new BytesSerializer();
+				break;
 			case INTEGER:
-				return new IntSerializer();
+				serializer = new IntSerializer();
+				break;
 			case LONG:
-				return new LongSerializer();
+				serializer = new LongSerializer();
+				break;
 			case STRING:
-				return new StringSerializer();
+				serializer = new StringSerializer();
+				break;
 			case FLOAT:
-				return new FloatSerializer();
+				serializer = new FloatSerializer();
+				break;
 			case DOUBLE:
-				return new DoubleSerializer();
+				serializer = new DoubleSerializer();
+				break;
 			case NULL:
-				return new NullSerializer();
+				serializer = new NullSerializer();
+				break;
 			case CUSTOMTYPEWRAPPER:
-				return new CustomTypeWrapperSerializer((CustomTypeWrapper) value);
+				serializer = new CustomTypeWrapperSerializer((CustomTypeWrapper) value);
+				break;
 			default:
 				throw new IllegalArgumentException("Unsupported Type encountered: " + type);
 		}
+		return (Serializer<IN>) serializer;
 	}
 
-	public static abstract class Serializer<IN> {
+	public abstract static class Serializer<IN> {
 		private byte[] typeInfo = null;
 
 		public byte[] serialize(IN value) {
@@ -130,7 +144,7 @@ public void putTypeInfo(ByteBuffer buffer) {
 	public static class BooleanSerializer extends Serializer<Boolean> {
 		@Override
 		public byte[] serializeWithoutTypeInfo(Boolean value) {
-			return new byte[]{value ? (byte) 1 : (byte) 0};
+			return new byte[]{(byte) (value ? 1 : 0)};
 		}
 
 		@Override
@@ -237,7 +251,7 @@ public void putTypeInfo(ByteBuffer buffer) {
 	}
 
 	public static class TupleSerializer extends Serializer<Tuple> {
-		private final Serializer[] serializer;
+		private final Serializer<Object>[] serializer;
 
 		public TupleSerializer(Tuple value) {
 			serializer = new Serializer[value.getArity()];
@@ -246,7 +260,6 @@ public TupleSerializer(Tuple value) {
 			}
 		}
 
-		@SuppressWarnings("unchecked")
 		@Override
 		public byte[] serializeWithoutTypeInfo(Tuple value) {
 			ArrayList<byte[]> bits = new ArrayList<>();
@@ -269,7 +282,7 @@ public byte[] serializeWithoutTypeInfo(Tuple value) {
 		@Override
 		public void putTypeInfo(ByteBuffer buffer) {
 			buffer.put((byte) serializer.length);
-			for (Serializer s : serializer) {
+			for (Serializer<Object> s : serializer) {
 				s.putTypeInfo(buffer);
 			}
 		}
@@ -277,7 +290,7 @@ public void putTypeInfo(ByteBuffer buffer) {
 		@Override
 		public int getTypeInfoSize() {
 			int size = 1;
-			for (Serializer s : serializer) {
+			for (Serializer<Object> s : serializer) {
 				size += s.getTypeInfoSize();
 			}
 			return size;

From b7274e56ae45396d3c12ab8e597482aa37231621 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 20 Apr 2017 14:08:54 +0200
Subject: [PATCH 140/253] [hotfix] [py] Improve error reporting in
 Python*InputStreamer

---
 .../python/api/streaming/data/PythonDualInputStreamer.java  | 6 +++---
 .../api/streaming/data/PythonSingleInputStreamer.java       | 6 ++----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java
index b7e8a25aa5e8c..8c9fde9ae30cb 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java
@@ -21,7 +21,6 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.util.Collector;
 
-import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.util.Iterator;
 
@@ -46,9 +45,8 @@ public PythonDualInputStreamer(AbstractRichFunction function, Configuration conf
 	 * @param iterator1 first input stream
 	 * @param iterator2 second input stream
 	 * @param c         collector
-	 * @throws IOException
 	 */
-	public final void streamBufferWithGroups(Iterator<IN1> iterator1, Iterator<IN2> iterator2, Collector<OUT> c) throws IOException {
+	public final void streamBufferWithGroups(Iterator<IN1> iterator1, Iterator<IN2> iterator2, Collector<OUT> c) {
 		SingleElementPushBackIterator<IN1> i1 = new SingleElementPushBackIterator<>(iterator1);
 		SingleElementPushBackIterator<IN2> i2 = new SingleElementPushBackIterator<>(iterator2);
 		try {
@@ -93,6 +91,8 @@ public final void streamBufferWithGroups(Iterator<IN1> iterator1, Iterator<IN2>
 			}
 		} catch (SocketTimeoutException ignored) {
 			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg);
+		} catch (Exception e) {
+			throw new RuntimeException("Critical failure for task " + function.getRuntimeContext().getTaskName() + ". " + msg.get(), e);
 		}
 	}
 }
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java
index e7f018c430125..6c0a13cda3fed 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java
@@ -21,7 +21,6 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.util.Collector;
 
-import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.util.Iterator;
 
@@ -43,9 +42,8 @@ public PythonSingleInputStreamer(AbstractRichFunction function, Configuration co
 	 *
 	 * @param iterator input stream
 	 * @param c        collector
-	 * @throws IOException
 	 */
-	public final void streamBufferWithoutGroups(Iterator<IN> iterator, Collector<OUT> c) throws IOException {
+	public final void streamBufferWithoutGroups(Iterator<IN> iterator, Collector<OUT> c) {
 		SingleElementPushBackIterator<IN> i = new SingleElementPushBackIterator<>(iterator);
 		try {
 			int size;
@@ -86,7 +84,7 @@ public final void streamBufferWithoutGroups(Iterator<IN> iterator, Collector<OUT
 		} catch (SocketTimeoutException ignored) {
 			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg.get());
 		} catch (Exception e) {
-			throw new RuntimeException("Critical failure. " + msg.get(), e);
+			throw new RuntimeException("Critical failure for task " + function.getRuntimeContext().getTaskName() + ". " + msg.get(), e);
 		}
 	}
 }

From 0ffb24778cdf2bedfb8535672d0cf251498b3b1e Mon Sep 17 00:00:00 2001
From: kl0u <kkloudas@gmail.com>
Date: Thu, 4 May 2017 15:33:40 +0200
Subject: [PATCH 141/253] [FLINK-6445] [cep] Fix NPE in no-condition patterns.

---
 .../org/apache/flink/cep/pattern/Pattern.java |  7 +-
 .../cep/pattern/conditions/AndCondition.java  |  6 +-
 .../cep/pattern/conditions/NotCondition.java  |  2 +-
 .../cep/pattern/conditions/OrCondition.java   |  6 +-
 .../pattern/conditions/SubtypeCondition.java  |  4 +-
 .../org/apache/flink/cep/nfa/NFAITCase.java   | 66 +++++++++++++++++++
 6 files changed, 84 insertions(+), 7 deletions(-)

diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
index cef0f85b7a6ea..b100bc5f80135 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
@@ -127,8 +127,9 @@ public static <X> Pattern<X, X> begin(final String name) {
 	 * @return The pattern with the new condition is set.
 	 */
 	public Pattern<T, F> where(IterativeCondition<F> condition) {
-		ClosureCleaner.clean(condition, true);
+		Preconditions.checkNotNull(condition, "The condition cannot be null.");
 
+		ClosureCleaner.clean(condition, true);
 		if (this.condition == null) {
 			this.condition = condition;
 		} else {
@@ -148,6 +149,8 @@ public Pattern<T, F> where(IterativeCondition<F> condition) {
 	 * @return The pattern with the new condition is set.
 	 */
 	public Pattern<T, F> or(IterativeCondition<F> condition) {
+		Preconditions.checkNotNull(condition, "The condition cannot be null.");
+
 		ClosureCleaner.clean(condition, true);
 
 		if (this.condition == null) {
@@ -167,6 +170,8 @@ public Pattern<T, F> or(IterativeCondition<F> condition) {
 	 * @return The same pattern with the new subtype constraint
 	 */
 	public <S extends F> Pattern<T, S> subtype(final Class<S> subtypeClass) {
+		Preconditions.checkNotNull(subtypeClass, "The class cannot be null.");
+
 		if (condition == null) {
 			this.condition = new SubtypeCondition<F>(subtypeClass);
 		} else {
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/AndCondition.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/AndCondition.java
index 5df7c669f813a..ac34c41301b9e 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/AndCondition.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/AndCondition.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.cep.pattern.conditions;
 
+import org.apache.flink.util.Preconditions;
+
 /**
  * A {@link IterativeCondition condition} which combines two conditions with a logical
  * {@code AND} and returns {@code true} if both are {@code true}.
@@ -32,8 +34,8 @@ public class AndCondition<T> extends IterativeCondition<T> {
 	private final IterativeCondition<T> right;
 
 	public AndCondition(final IterativeCondition<T> left, final IterativeCondition<T> right) {
-		this.left = left;
-		this.right = right;
+		this.left = Preconditions.checkNotNull(left, "The condition cannot be null.");
+		this.right = Preconditions.checkNotNull(right, "The condition cannot be null.");
 	}
 
 	@Override
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/NotCondition.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/NotCondition.java
index 3e6ab56d57ce0..9318c2f67726f 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/NotCondition.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/NotCondition.java
@@ -35,6 +35,6 @@ public NotCondition(final IterativeCondition<T> original) {
 
 	@Override
 	public boolean filter(T value, Context<T> ctx) throws Exception {
-		return !original.filter(value, ctx);
+		return original != null && !original.filter(value, ctx);
 	}
 }
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/OrCondition.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/OrCondition.java
index 6aaa4bbb15802..d3690ab4da065 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/OrCondition.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/OrCondition.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.cep.pattern.conditions;
 
+import org.apache.flink.util.Preconditions;
+
 /**
  * A {@link IterativeCondition condition} which combines two conditions with a logical
  * {@code OR} and returns {@code true} if at least one is {@code true}.
@@ -32,8 +34,8 @@ public class OrCondition<T> extends IterativeCondition<T> {
 	private final IterativeCondition<T> right;
 
 	public OrCondition(final IterativeCondition<T> left, final IterativeCondition<T> right) {
-		this.left = left;
-		this.right = right;
+		this.left = Preconditions.checkNotNull(left, "The condition cannot be null.");
+		this.right = Preconditions.checkNotNull(right, "The condition cannot be null.");
 	}
 
 	@Override
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/SubtypeCondition.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/SubtypeCondition.java
index 91f6c21ffb55c..cff8693c588aa 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/SubtypeCondition.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/SubtypeCondition.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.cep.pattern.conditions;
 
+import org.apache.flink.util.Preconditions;
+
 /**
  * A {@link IterativeCondition condition} which filters elements of the given type.
  * An element is filtered out iff it is not assignable to the given subtype of {@code T}.
@@ -31,7 +33,7 @@ public class SubtypeCondition<T> extends SimpleCondition<T> {
 	private final Class<? extends T> subtype;
 
 	public SubtypeCondition(final Class<? extends T> subtype) {
-		this.subtype = subtype;
+		this.subtype = Preconditions.checkNotNull(subtype, "The subtype cannot be null.");
 	}
 
 	@Override
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index 4a00c1e00f7d8..fe31564af30af 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -50,6 +50,72 @@
 @SuppressWarnings("unchecked")
 public class NFAITCase extends TestLogger {
 
+	@Test
+	public void testNoConditionNFA() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a = new Event(40, "a", 1.0);
+		Event b = new Event(41, "b", 2.0);
+		Event c = new Event(42, "c", 3.0);
+		Event d = new Event(43, "d", 4.0);
+		Event e = new Event(44, "e", 5.0);
+
+		inputEvents.add(new StreamRecord<>(a, 1));
+		inputEvents.add(new StreamRecord<>(b, 2));
+		inputEvents.add(new StreamRecord<>(c, 3));
+		inputEvents.add(new StreamRecord<>(d, 4));
+		inputEvents.add(new StreamRecord<>(e, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").followedBy("end");
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(a, b),
+				Lists.newArrayList(b, c),
+				Lists.newArrayList(c, d),
+				Lists.newArrayList(d, e)
+		));
+	}
+
+	@Test
+	public void testAnyWithNoConditionNFA() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a = new Event(40, "a", 1.0);
+		Event b = new Event(41, "b", 2.0);
+		Event c = new Event(42, "c", 3.0);
+		Event d = new Event(43, "d", 4.0);
+		Event e = new Event(44, "e", 5.0);
+
+		inputEvents.add(new StreamRecord<>(a, 1));
+		inputEvents.add(new StreamRecord<>(b, 2));
+		inputEvents.add(new StreamRecord<>(c, 3));
+		inputEvents.add(new StreamRecord<>(d, 4));
+		inputEvents.add(new StreamRecord<>(e, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").followedByAny("end");
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		List<List<Event>> resultingPatterns = feedNFA(inputEvents, nfa);
+
+		compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
+				Lists.newArrayList(a, b),
+				Lists.newArrayList(a, c),
+				Lists.newArrayList(a, d),
+				Lists.newArrayList(a, e),
+				Lists.newArrayList(b, c),
+				Lists.newArrayList(b, d),
+				Lists.newArrayList(b, e),
+				Lists.newArrayList(c, d),
+				Lists.newArrayList(c, e),
+				Lists.newArrayList(d, e)
+		));
+	}
+
 	@Test
 	public void testSimplePatternNFA() {
 		List<StreamRecord<Event>> inputEvents = new ArrayList<>();

From a616b8e3b81b407f6e58af1d2d56eef69328621a Mon Sep 17 00:00:00 2001
From: shaoxuan-wang <wshaoxuan@gmail.com>
Date: Tue, 2 May 2017 23:00:51 +0800
Subject: [PATCH 142/253] [FLINK-5906] [table] Add support to register UDAGGs
 for Table API and SQL.

This closes #3809.
---
 .../flink/table/api/TableEnvironment.scala    |  25 ++-
 .../api/java/BatchTableEnvironment.scala      |  22 +-
 .../api/java/StreamTableEnvironment.scala     |  22 +-
 .../api/scala/BatchTableEnvironment.scala     |  18 +-
 .../api/scala/StreamTableEnvironment.scala    |  18 +-
 .../flink/table/api/scala/expressionDsl.scala |   3 +
 .../org/apache/flink/table/api/table.scala    |  21 +-
 .../flink/table/codegen/CodeGenerator.scala   |  97 +++++++--
 .../codegen/calls/ScalarFunctionCallGen.scala |   4 +-
 .../codegen/calls/TableFunctionCallGen.scala  |   2 +-
 .../table/expressions/UDAGGExpression.scala   |  36 ++++
 .../table/expressions/aggregations.scala      |  75 ++++++-
 .../apache/flink/table/expressions/call.scala |  13 +-
 .../table/functions/AggregateFunction.scala   |  14 +-
 .../aggfunctions/CountAggFunction.scala       |   3 +
 .../functions/utils/AggSqlFunction.scala      | 179 ++++++++++++++++
 .../functions/utils/ScalarSqlFunction.scala   |  33 +--
 .../utils/UserDefinedFunctionUtils.scala      | 198 +++++++++++++-----
 .../table/plan/ProjectionTranslator.scala     |  57 +++++
 .../flink/table/plan/logical/operators.scala  |   4 +-
 .../table/plan/nodes/CommonAggregate.scala    |   2 +-
 .../table/plan/nodes/OverAggregate.scala      |   2 +-
 .../datastream/DataStreamAggregate.scala      |  12 +-
 .../runtime/aggregate/AggregateUtil.scala     |  48 +++--
 .../table/validate/FunctionCatalog.scala      |  13 +-
 .../java/utils/UserDefinedAggFunctions.java   |  95 +++++++++
 .../scala/batch/sql/AggregationsITCase.scala  |  43 ++--
 .../scala/batch/sql/WindowAggregateTest.scala |  43 +++-
 .../batch/table/AggregationsITCase.scala      |   9 +-
 .../scala/batch/table/GroupWindowTest.scala   | 155 ++++++++++++++
 .../AggregationsStringExpressionTest.scala    |  58 +++++
 .../AggregationsValidationTest.scala          |  99 +++++++++
 .../stream/sql/WindowAggregateTest.scala      |  74 ++++---
 .../stream/table/AggregationsITCase.scala     |  43 ++--
 .../scala/stream/table/GroupWindowTest.scala  | 120 ++++++++++-
 .../scala/stream/table/OverWindowITCase.scala |  45 ++--
 .../scala/stream/table/OverWindowTest.scala   |  88 +++++---
 .../GroupWindowStringExpressionTest.scala     |  65 ++++++
 38 files changed, 1617 insertions(+), 241 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/UDAGGExpression.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala
 create mode 100644 flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/utils/UserDefinedAggFunctions.java
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 45267d2f89954..06c405eccff38 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -50,8 +50,8 @@ import org.apache.flink.table.calcite.{FlinkPlannerImpl, FlinkRelBuilder, FlinkT
 import org.apache.flink.table.catalog.{ExternalCatalog, ExternalCatalogSchema}
 import org.apache.flink.table.codegen.{CodeGenerator, ExpressionReducer}
 import org.apache.flink.table.expressions.{Alias, Expression, UnresolvedFieldReference}
-import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{checkForInstantiation, checkNotSingleton, createScalarSqlFunction, createTableSqlFunctions}
-import org.apache.flink.table.functions.{ScalarFunction, TableFunction}
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
+import org.apache.flink.table.functions.{ScalarFunction, TableFunction, AggregateFunction}
 import org.apache.flink.table.plan.cost.DataSetCostFactory
 import org.apache.flink.table.plan.logical.{CatalogNode, LogicalRelNode}
 import org.apache.flink.table.plan.rules.FlinkRuleSets
@@ -351,6 +351,27 @@ abstract class TableEnvironment(val config: TableConfig) {
     functionCatalog.registerSqlFunctions(sqlFunctions)
   }
 
+  /**
+    * Registers an [[AggregateFunction]] under a unique name. Replaces already existing
+    * user-defined functions under this name.
+    */
+  private[flink] def registerAggregateFunctionInternal[T: TypeInformation, ACC](
+      name: String, function: AggregateFunction[T, ACC]): Unit = {
+    // check if class not Scala object
+    checkNotSingleton(function.getClass)
+    // check if class could be instantiated
+    checkForInstantiation(function.getClass)
+
+    val typeInfo: TypeInformation[_] = implicitly[TypeInformation[T]]
+
+    // register in Table API
+    functionCatalog.registerFunction(name, function.getClass)
+
+    // register in SQL API
+    val sqlFunctions = createAggregateSqlFunction(name, function, typeInfo, typeFactory)
+    functionCatalog.registerSqlFunction(sqlFunctions)
+  }
+
   /**
     * Registers a [[Table]] under a unique name in the TableEnvironment's catalog.
     * Registered tables can be referenced in SQL queries.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/BatchTableEnvironment.scala
index de5f789dcd453..03fb77ee2b3ad 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/BatchTableEnvironment.scala
@@ -22,7 +22,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.api.java.{DataSet, ExecutionEnvironment}
 import org.apache.flink.table.expressions.ExpressionParser
 import org.apache.flink.table.api._
-import org.apache.flink.table.functions.TableFunction
+import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
 
 /**
   * The [[TableEnvironment]] for a Java batch [[DataSet]]
@@ -178,4 +178,24 @@ class BatchTableEnvironment(
 
     registerTableFunctionInternal[T](name, tf)
   }
+
+  /**
+    * Registers an [[AggregateFunction]] under a unique name in the TableEnvironment's catalog.
+    * Registered functions can be referenced in Table API and SQL queries.
+    *
+    * @param name The name under which the function is registered.
+    * @param f The AggregateFunction to register.
+    * @tparam T The type of the output value.
+    * @tparam ACC The type of aggregate accumulator.
+    */
+  def registerFunction[T, ACC](
+      name: String,
+      f: AggregateFunction[T, ACC])
+  : Unit = {
+    implicit val typeInfo: TypeInformation[T] = TypeExtractor
+      .createTypeInfo(f, classOf[AggregateFunction[T, ACC]], f.getClass, 0)
+      .asInstanceOf[TypeInformation[T]]
+
+    registerAggregateFunctionInternal[T, ACC](name, f)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
index 4d9f1e1e98799..a649584ae9bef 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
@@ -20,7 +20,7 @@ package org.apache.flink.table.api.java
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.table.api._
-import org.apache.flink.table.functions.TableFunction
+import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
 import org.apache.flink.table.expressions.ExpressionParser
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
@@ -180,4 +180,24 @@ class StreamTableEnvironment(
 
     registerTableFunctionInternal[T](name, tf)
   }
+
+  /**
+    * Registers an [[AggregateFunction]] under a unique name in the TableEnvironment's catalog.
+    * Registered functions can be referenced in Table API and SQL queries.
+    *
+    * @param name The name under which the function is registered.
+    * @param f The AggregateFunction to register.
+    * @tparam T The type of the output value.
+    * @tparam ACC The type of aggregate accumulator.
+    */
+  def registerFunction[T, ACC](
+      name: String,
+      f: AggregateFunction[T, ACC])
+  : Unit = {
+    implicit val typeInfo: TypeInformation[T] = TypeExtractor
+      .createTypeInfo(f, classOf[AggregateFunction[T, ACC]], f.getClass, 0)
+      .asInstanceOf[TypeInformation[T]]
+
+    registerAggregateFunctionInternal[T, ACC](name, f)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala
index 3ae8c31245052..0dd7ca0609591 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala
@@ -21,7 +21,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api._
 import org.apache.flink.table.expressions.Expression
-import org.apache.flink.table.functions.TableFunction
+import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
 
 import _root_.scala.reflect.ClassTag
 
@@ -151,4 +151,20 @@ class BatchTableEnvironment(
   def registerFunction[T: TypeInformation](name: String, tf: TableFunction[T]): Unit = {
     registerTableFunctionInternal(name, tf)
   }
+
+  /**
+    * Registers an [[AggregateFunction]] under a unique name in the TableEnvironment's catalog.
+    * Registered functions can be referenced in Table API and SQL queries.
+    *
+    * @param name The name under which the function is registered.
+    * @param f The AggregateFunction to register.
+    * @tparam T The type of the output value.
+    * @tparam ACC The type of aggregate accumulator.
+    */
+  def registerFunction[T: TypeInformation, ACC](
+      name: String,
+      f: AggregateFunction[T, ACC])
+  : Unit = {
+    registerAggregateFunctionInternal[T, ACC](name, f)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
index 01131461b0326..0552d7c64fbb7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.api.scala
 
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.{TableEnvironment, Table, TableConfig}
-import org.apache.flink.table.functions.TableFunction
+import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
 import org.apache.flink.table.expressions.Expression
 import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
 import org.apache.flink.streaming.api.scala.asScalaStream
@@ -152,4 +152,20 @@ class StreamTableEnvironment(
   def registerFunction[T: TypeInformation](name: String, tf: TableFunction[T]): Unit = {
     registerTableFunctionInternal(name, tf)
   }
+
+  /**
+    * Registers an [[AggregateFunction]] under a unique name in the TableEnvironment's catalog.
+    * Registered functions can be referenced in Table API and SQL queries.
+    *
+    * @param name The name under which the function is registered.
+    * @param f The AggregateFunction to register.
+    * @tparam T The type of the output value.
+    * @tparam ACC The type of aggregate accumulator.
+    */
+  def registerFunction[T: TypeInformation, ACC](
+      name: String,
+      f: AggregateFunction[T, ACC])
+  : Unit = {
+    registerAggregateFunctionInternal[T, ACC](name, f)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
index b3de4a4b91fab..cc58ff54fbf10 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
@@ -26,6 +26,7 @@ import org.apache.flink.table.api.{TableException, CurrentRow, CurrentRange, Unb
 import org.apache.flink.table.expressions.ExpressionUtils.{convertArray, toMilliInterval, toMonthInterval, toRowInterval}
 import org.apache.flink.table.expressions.TimeIntervalUnit.TimeIntervalUnit
 import org.apache.flink.table.expressions._
+import org.apache.flink.table.functions.AggregateFunction
 
 import scala.language.implicitConversions
 
@@ -773,6 +774,8 @@ trait ImplicitExpressionConversions {
   implicit def sqlTimestamp2Literal(sqlTimestamp: Timestamp): Expression =
     Literal(sqlTimestamp)
   implicit def array2ArrayConstructor(array: Array[_]): Expression = convertArray(array)
+  implicit def userDefinedAggFunctionConstructor[T: TypeInformation, ACC]
+      (udagg: AggregateFunction[T, ACC]): UDAGGExpression[T, ACC] = UDAGGExpression(udagg)
 }
 
 // ------------------------------------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 9606979643fa5..87dde0aa68cf0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -151,7 +151,9 @@ class Table(
     */
   def select(fields: String): Table = {
     val fieldExprs = ExpressionParser.parseExpressionList(fields)
-    select(fieldExprs: _*)
+    //get the correct expression for AggFunctionCall
+    val withResolvedAggFunctionCall = fieldExprs.map(replaceAggFunctionCall(_, tableEnv))
+    select(withResolvedAggFunctionCall: _*)
   }
 
   /**
@@ -167,7 +169,7 @@ class Table(
   def as(fields: Expression*): Table = {
 
     logicalPlan match {
-      case functionCall: LogicalTableFunctionCall if functionCall.child == null => {
+      case functionCall: LogicalTableFunctionCall if functionCall.child == null =>
         // If the logical plan is a TableFunctionCall, we replace its field names to avoid special
         //   cases during the validation.
         if (fields.length != functionCall.output.length) {
@@ -181,7 +183,7 @@ class Table(
         }
         new Table(
           tableEnv,
-          new LogicalTableFunctionCall(
+          LogicalTableFunctionCall(
             functionCall.functionName,
             functionCall.tableFunction,
             functionCall.parameters,
@@ -189,7 +191,6 @@ class Table(
             fields.map(_.asInstanceOf[UnresolvedFieldReference].name).toArray,
             functionCall.child)
         )
-      }
       case _ =>
         // prepend an AliasNode
         new Table(tableEnv, AliasNode(fields, logicalPlan).validate(tableEnv))
@@ -908,7 +909,9 @@ class GroupedTable(
     */
   def select(fields: String): Table = {
     val fieldExprs = ExpressionParser.parseExpressionList(fields)
-    select(fieldExprs: _*)
+    //get the correct expression for AggFunctionCall
+    val withResolvedAggFunctionCall = fieldExprs.map(replaceAggFunctionCall(_, table.tableEnv))
+    select(withResolvedAggFunctionCall: _*)
   }
 }
 
@@ -983,7 +986,9 @@ class OverWindowedTable(
 
   def select(fields: String): Table = {
     val fieldExprs = ExpressionParser.parseExpressionList(fields)
-    select(fieldExprs: _*)
+    //get the correct expression for AggFunctionCall
+    val withResolvedAggFunctionCall = fieldExprs.map(replaceAggFunctionCall(_, table.tableEnv))
+    select(withResolvedAggFunctionCall: _*)
   }
 }
 
@@ -1043,7 +1048,9 @@ class WindowGroupedTable(
     */
   def select(fields: String): Table = {
     val fieldExprs = ExpressionParser.parseExpressionList(fields)
-    select(fieldExprs: _*)
+    //get the correct expression for AggFunctionCall
+    val withResolvedAggFunctionCall = fieldExprs.map(replaceAggFunctionCall(_, table.tableEnv))
+    select(withResolvedAggFunctionCall: _*)
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 648efe6af7d73..5bb3b0e4bffd7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -18,6 +18,8 @@
 
 package org.apache.flink.table.codegen
 
+import java.lang.reflect.ParameterizedType
+import java.lang.{Iterable => JIterable}
 import java.math.{BigDecimal => JBigDecimal}
 
 import org.apache.calcite.avatica.util.DateTimeUtils
@@ -45,6 +47,7 @@ import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
 import org.apache.flink.table.runtime.TableFunctionCollector
 import org.apache.flink.table.typeutils.TypeCheckUtils._
 import org.apache.flink.types.Row
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{getUserDefinedMethod, signatureToString}
 
 import scala.collection.JavaConversions._
 import scala.collection.mutable
@@ -258,6 +261,9 @@ class CodeGenerator(
     * @param constantFlags An optional parameter to define where to set constant boolean flags in
     *                      the output row.
     * @param outputArity The number of fields in the output row.
+    * @param needRetract a flag to indicate if the aggregate needs the retract method
+    * @param needMerge a flag to indicate if the aggregate needs the merge method
+    * @param needReset a flag to indicate if the aggregate needs the resetAccumulator method
     *
     * @return A GeneratedAggregationsFunction
     */
@@ -274,7 +280,8 @@ class CodeGenerator(
       constantFlags: Option[Array[(Int, Boolean)]],
       outputArity: Int,
       needRetract: Boolean,
-      needMerge: Boolean)
+      needMerge: Boolean,
+      needReset: Boolean)
   : GeneratedAggregationsFunction = {
 
     // get unique function name
@@ -282,19 +289,80 @@ class CodeGenerator(
     // register UDAGGs
     val aggs = aggregates.map(a => generator.addReusableFunction(a))
     // get java types of accumulators
-    val accTypes = aggregates.map { a =>
-      a.getClass.getMethod("createAccumulator").getReturnType.getCanonicalName
+    val accTypeClasses = aggregates.map { a =>
+      a.getClass.getMethod("createAccumulator").getReturnType
     }
+    val accTypes = accTypeClasses.map(_.getCanonicalName)
 
-    // get java types of input fields
-    val javaTypes = inputType.getFieldList
-      .map(f => FlinkTypeFactory.toTypeInfo(f.getType))
-      .map(t => t.getTypeClass.getCanonicalName)
+    // get java classes of input fields
+    val javaClasses = inputType.getFieldList
+      .map(f => FlinkTypeFactory.toTypeInfo(f.getType).getTypeClass)
     // get parameter lists for aggregation functions
-    val parameters = aggFields.map {inFields =>
-      val fields = for (f <- inFields) yield s"(${javaTypes(f)}) input.getField($f)"
+    val parameters = aggFields.map { inFields =>
+      val fields = for (f <- inFields) yield
+        s"(${javaClasses(f).getCanonicalName}) input.getField($f)"
       fields.mkString(", ")
     }
+    val methodSignaturesList = aggFields.map {
+      inFields => for (f <- inFields) yield javaClasses(f)
+    }
+
+    // check and validate the needed methods
+    aggregates.zipWithIndex.map {
+      case (a, i) => {
+        getUserDefinedMethod(a, "accumulate", Array(accTypeClasses(i)) ++ methodSignaturesList(i))
+          .getOrElse(
+            throw new CodeGenException(
+              s"No matching accumulate method found for AggregateFunction " +
+                s"'${a.getClass.getCanonicalName}'" +
+                s"with parameters '${signatureToString(methodSignaturesList(i))}'.")
+          )
+
+        if (needRetract) {
+          getUserDefinedMethod(a, "retract", Array(accTypeClasses(i)) ++ methodSignaturesList(i))
+            .getOrElse(
+              throw new CodeGenException(
+                s"No matching retract method found for AggregateFunction " +
+                  s"'${a.getClass.getCanonicalName}'" +
+                  s"with parameters '${signatureToString(methodSignaturesList(i))}'.")
+            )
+        }
+
+        if (needMerge) {
+          val methods =
+            getUserDefinedMethod(a, "merge", Array(accTypeClasses(i), classOf[JIterable[Any]]))
+              .getOrElse(
+                throw new CodeGenException(
+                  s"No matching merge method found for AggregateFunction " +
+                    s"${a.getClass.getCanonicalName}'.")
+              )
+
+          var iterableTypeClass = methods.getGenericParameterTypes.apply(1)
+            .asInstanceOf[ParameterizedType].getActualTypeArguments.apply(0)
+          // further extract iterableTypeClass if the accumulator has generic type
+          iterableTypeClass match {
+            case impl: ParameterizedType => iterableTypeClass = impl.getRawType
+            case _ =>
+          }
+
+          if (iterableTypeClass != accTypeClasses(i)) {
+            throw new CodeGenException(
+              s"merge method in AggregateFunction ${a.getClass.getCanonicalName} does not have " +
+                s"the correct Iterable type. Actually: ${iterableTypeClass.toString}. " +
+                s"Expected: ${accTypeClasses(i).toString}")
+          }
+        }
+
+        if (needReset) {
+          getUserDefinedMethod(a, "resetAccumulator", Array(accTypeClasses(i)))
+            .getOrElse(
+              throw new CodeGenException(
+                s"No matching resetAccumulator method found for " +
+                  s"aggregate ${a.getClass.getCanonicalName}'.")
+            )
+        }
+      }
+    }
 
     def genSetAggregationResults: String = {
 
@@ -529,9 +597,14 @@ class CodeGenerator(
              |      ((${accTypes(i)}) accs.getField($i)));""".stripMargin
       }.mkString("\n")
 
-      j"""$sig {
-         |$reset
-         |  }""".stripMargin
+      if (needReset) {
+        j"""$sig {
+           |$reset
+           |  }""".stripMargin
+      } else {
+        j"""$sig {
+           |  }""".stripMargin
+      }
     }
 
     var funcCode =
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarFunctionCallGen.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarFunctionCallGen.scala
index b0b4e09a9159a..07a8708daa5b3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarFunctionCallGen.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarFunctionCallGen.scala
@@ -44,10 +44,10 @@ class ScalarFunctionCallGen(
       operands: Seq[GeneratedExpression])
     : GeneratedExpression = {
     // determine function method and result class
-    val matchingMethod = getEvalMethod(scalarFunction, signature)
+    val matchingMethod = getUserDefinedMethod(scalarFunction, "eval", typeInfoToClass(signature))
       .getOrElse(throw new CodeGenException("No matching signature found."))
     val matchingSignature = matchingMethod.getParameterTypes
-    val resultClass = getResultTypeClass(scalarFunction, matchingSignature)
+    val resultClass = getResultTypeClassOfScalarFunction(scalarFunction, matchingSignature)
 
     // zip for variable signatures
     var paramToOperands = matchingSignature.zip(operands)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/TableFunctionCallGen.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/TableFunctionCallGen.scala
index ba90292f93895..a3609c149faee 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/TableFunctionCallGen.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/TableFunctionCallGen.scala
@@ -45,7 +45,7 @@ class TableFunctionCallGen(
       operands: Seq[GeneratedExpression])
     : GeneratedExpression = {
     // determine function method
-    val matchingMethod = getEvalMethod(tableFunction, signature)
+    val matchingMethod = getUserDefinedMethod(tableFunction, "eval", typeInfoToClass(signature))
       .getOrElse(throw new CodeGenException("No matching signature found."))
     val matchingSignature = matchingMethod.getParameterTypes
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/UDAGGExpression.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/UDAGGExpression.scala
new file mode 100644
index 0000000000000..c0e213df4ccb1
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/UDAGGExpression.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.expressions
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.table.functions.AggregateFunction
+
+/**
+  * A class which creates a call to an aggregateFunction
+  */
+case class UDAGGExpression[T: TypeInformation, ACC](aggregateFunction: AggregateFunction[T, ACC]) {
+
+  /**
+    * Creates a call to an [[AggregateFunction]].
+    *
+    * @param params actual parameters of function
+    * @return a [[AggFunctionCall]]
+    */
+  def apply(params: Expression*): AggFunctionCall =
+    AggFunctionCall(aggregateFunction, params)
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
index 72e7e4b515a75..7b180ae7c4ebc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala
@@ -23,13 +23,18 @@ import org.apache.calcite.sql.SqlKind._
 import org.apache.calcite.sql.fun._
 import org.apache.calcite.tools.RelBuilder
 import org.apache.calcite.tools.RelBuilder.AggCall
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.table.functions.AggregateFunction
+import org.apache.flink.table.functions.utils.AggSqlFunction
+import org.apache.flink.table.typeutils.TypeCheckUtils
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
 import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.table.typeutils.TypeCheckUtils
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
+import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
-abstract sealed class Aggregation extends UnaryExpression {
+abstract sealed class Aggregation extends Expression {
 
-  override def toString = s"Aggregate($child)"
+  override def toString = s"Aggregate"
 
   override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode =
     throw new UnsupportedOperationException("Aggregate cannot be transformed to RexNode")
@@ -47,6 +52,7 @@ abstract sealed class Aggregation extends UnaryExpression {
 }
 
 case class Sum(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"sum($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -67,6 +73,7 @@ case class Sum(child: Expression) extends Aggregation {
 }
 
 case class Sum0(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"sum0($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -83,6 +90,7 @@ case class Sum0(child: Expression) extends Aggregation {
 }
 
 case class Min(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"min($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -100,6 +108,7 @@ case class Min(child: Expression) extends Aggregation {
 }
 
 case class Max(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"max($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -117,6 +126,7 @@ case class Max(child: Expression) extends Aggregation {
 }
 
 case class Count(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"count($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -131,6 +141,7 @@ case class Count(child: Expression) extends Aggregation {
 }
 
 case class Avg(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"avg($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -148,6 +159,7 @@ case class Avg(child: Expression) extends Aggregation {
 }
 
 case class StddevPop(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"stddev_pop($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -164,6 +176,7 @@ case class StddevPop(child: Expression) extends Aggregation {
 }
 
 case class StddevSamp(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"stddev_samp($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -180,6 +193,7 @@ case class StddevSamp(child: Expression) extends Aggregation {
 }
 
 case class VarPop(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"var_pop($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -196,6 +210,7 @@ case class VarPop(child: Expression) extends Aggregation {
 }
 
 case class VarSamp(child: Expression) extends Aggregation {
+  override private[flink] def children: Seq[Expression] = Seq(child)
   override def toString = s"var_samp($child)"
 
   override private[flink] def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
@@ -210,3 +225,57 @@ case class VarSamp(child: Expression) extends Aggregation {
   override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) =
     new SqlAvgAggFunction(VAR_SAMP)
 }
+
+case class AggFunctionCall(
+    aggregateFunction: AggregateFunction[_, _],
+    args: Seq[Expression])
+  extends Aggregation {
+
+  override private[flink] def children: Seq[Expression] = args
+
+  override def resultType: TypeInformation[_] = getResultTypeOfAggregateFunction(aggregateFunction)
+
+  override def validateInput(): ValidationResult = {
+    val signature = children.map(_.resultType)
+    // look for a signature that matches the input types
+    val foundSignature = getAccumulateMethodSignature(aggregateFunction, signature)
+    if (foundSignature.isEmpty) {
+      ValidationFailure(s"Given parameters do not match any signature. \n" +
+                          s"Actual: ${signatureToString(signature)} \n" +
+                          s"Expected: ${
+                            getMethodSignatures(aggregateFunction, "accumulate").drop(1)
+                              .map(signatureToString).mkString(", ")}")
+    } else {
+      ValidationSuccess
+    }
+  }
+
+  override def toString(): String = s"${aggregateFunction.getClass.getSimpleName}($args)"
+
+  override def toAggCall(name: String)(implicit relBuilder: RelBuilder): AggCall = {
+    val typeFactory = relBuilder.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+    val sqlFunction = AggSqlFunction(aggregateFunction.getClass.getSimpleName,
+                                     aggregateFunction,
+                                     resultType,
+                                     typeFactory)
+    relBuilder.aggregateCall(sqlFunction, false, null, name, args.map(_.toRexNode): _*)
+  }
+
+  override private[flink] def getSqlAggFunction()(implicit relBuilder: RelBuilder) = {
+    val typeFactory = relBuilder.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+    AggSqlFunction(aggregateFunction.getClass.getSimpleName,
+                   aggregateFunction,
+                   resultType,
+                   typeFactory)
+  }
+
+  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+    val typeFactory = relBuilder.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+    relBuilder.call(
+      AggSqlFunction(aggregateFunction.getClass.getSimpleName,
+                     aggregateFunction,
+                     resultType,
+                     typeFactory),
+      args.map(_.toRexNode): _*)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
index 68ed68827992f..5f7204a29ae71 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
@@ -106,8 +106,8 @@ case class OverCall(
       .getTypeFactory.asInstanceOf[FlinkTypeFactory]
       .createTypeFromTypeInfo(agg.resultType)
 
-    val aggChildName = agg.asInstanceOf[Aggregation].child.asInstanceOf[ResolvedFieldReference].name
-    val aggExprs = List(relBuilder.field(aggChildName).asInstanceOf[RexNode]).asJava
+    // assemble exprs by agg children
+    val aggExprs = agg.asInstanceOf[Aggregation].children.map(_.toRexNode(relBuilder)).asJava
 
     // assemble order by key
     val orderKey = orderBy match {
@@ -281,16 +281,19 @@ case class ScalarFunctionCall(
   override def toString =
     s"${scalarFunction.getClass.getCanonicalName}(${parameters.mkString(", ")})"
 
-  override private[flink] def resultType = getResultType(scalarFunction, foundSignature.get)
+  override private[flink] def resultType =
+    getResultTypeOfScalarFunction(
+      scalarFunction,
+      foundSignature.get)
 
   override private[flink] def validateInput(): ValidationResult = {
     val signature = children.map(_.resultType)
     // look for a signature that matches the input types
-    foundSignature = getSignature(scalarFunction, signature)
+    foundSignature = getEvalMethodSignature(scalarFunction, signature)
     if (foundSignature.isEmpty) {
       ValidationFailure(s"Given parameters do not match any signature. \n" +
         s"Actual: ${signatureToString(signature)} \n" +
-        s"Expected: ${signaturesToString(scalarFunction)}")
+        s"Expected: ${signaturesToString(scalarFunction, "eval")}")
     } else {
       ValidationSuccess
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala
index 7a74112dbc6a8..9c79439ed5435 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/AggregateFunction.scala
@@ -89,7 +89,7 @@ package org.apache.flink.table.functions
   *
   * {{{
   * Returns the [[org.apache.flink.api.common.typeinfo.TypeInformation]] of the accumulator. This
-  * function is optional and can be implemented if the accumulator type cannot automatically
+  * function is optional and can be implemented if the accumulator type cannot be automatically
   * inferred from the instance returned by createAccumulator method.
   *
   * @return  the type information for the accumulator.
@@ -98,6 +98,18 @@ package org.apache.flink.table.functions
   * }}}
   *
   *
+  * {{{
+  * Returns the [[org.apache.flink.api.common.typeinfo.TypeInformation]] of the return value. This
+  * function is optional and needed in case Flink's type extraction facilities are not sufficient
+  * to extract the TypeInformation. Flink's type extraction facilities can handle basic types or
+  * simple POJOs but might be wrong for more complex, custom, or composite types.
+  *
+  * @return  the type information for the return value.
+  *
+  * def getResultType: TypeInformation[_]
+  * }}}
+  *
+  *
   * @tparam T   the type of the aggregation result
   * @tparam ACC base class for aggregate Accumulator. The accumulator is used to keep the aggregated
   *             values which are needed to compute an aggregation result. AggregateFunction
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala
index 77341cd761583..2b8ec14d6d81d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunction.scala
@@ -68,4 +68,7 @@ class CountAggFunction extends AggregateFunction[Long, CountAccumulator] {
   def getAccumulatorType(): TypeInformation[_] = {
     new TupleTypeInfo((new CountAccumulator).getClass, BasicTypeInfo.LONG_TYPE_INFO)
   }
+
+  def getResultType(): TypeInformation[_] =
+    BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[_]]
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala
new file mode 100644
index 0000000000000..c3f6c4cf3e8e1
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.utils
+
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.sql._
+import org.apache.calcite.sql.`type`._
+import org.apache.calcite.sql.`type`.SqlOperandTypeChecker.Consistency
+import org.apache.calcite.sql.parser.SqlParserPos
+import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction
+import org.apache.flink.api.common.typeinfo._
+import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.functions.AggregateFunction
+import org.apache.flink.table.functions.utils.AggSqlFunction.{createOperandTypeChecker, createOperandTypeInference, createReturnTypeInference}
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
+
+/**
+  * Calcite wrapper for user-defined aggregate functions.
+  *
+  * @param name function name (used by SQL parser)
+  * @param aggregateFunction aggregate function to be called
+  * @param returnType the type information of returned value
+  * @param typeFactory type factory for converting Flink's between Calcite's types
+  */
+class AggSqlFunction(
+    name: String,
+    aggregateFunction: AggregateFunction[_, _],
+    returnType: TypeInformation[_],
+    typeFactory: FlinkTypeFactory)
+  extends SqlUserDefinedAggFunction(
+    new SqlIdentifier(name, SqlParserPos.ZERO),
+    createReturnTypeInference(returnType, typeFactory),
+    createOperandTypeInference(aggregateFunction, typeFactory),
+    createOperandTypeChecker(aggregateFunction),
+    // Do not need to provide a calcite aggregateFunction here. Flink aggregateion function
+    // will be generated when translating the calcite relnode to flink runtime execution plan
+    null
+  ) {
+
+  def getFunction: AggregateFunction[_, _] = aggregateFunction
+}
+
+object AggSqlFunction {
+
+  def apply(
+      name: String,
+      aggregateFunction: AggregateFunction[_, _],
+      returnType: TypeInformation[_],
+      typeFactory: FlinkTypeFactory): AggSqlFunction = {
+
+    new AggSqlFunction(name, aggregateFunction, returnType, typeFactory)
+  }
+
+  private[flink] def createOperandTypeInference(
+      aggregateFunction: AggregateFunction[_, _],
+      typeFactory: FlinkTypeFactory)
+  : SqlOperandTypeInference = {
+    /**
+      * Operand type inference based on [[AggregateFunction]] given information.
+      */
+    new SqlOperandTypeInference {
+      override def inferOperandTypes(
+          callBinding: SqlCallBinding,
+          returnType: RelDataType,
+          operandTypes: Array[RelDataType]): Unit = {
+
+        val operandTypeInfo = getOperandTypeInfo(callBinding)
+
+        val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandTypeInfo)
+          .getOrElse(
+            throw new ValidationException(
+              s"Operand types of ${signatureToString(operandTypeInfo)} could not be inferred."))
+
+        val inferredTypes = getParameterTypes(aggregateFunction, foundSignature.drop(1))
+          .map(typeFactory.createTypeFromTypeInfo)
+
+        for (i <- operandTypes.indices) {
+          if (i < inferredTypes.length - 1) {
+            operandTypes(i) = inferredTypes(i)
+          } else if (null != inferredTypes.last.getComponentType) {
+            // last argument is a collection, the array type
+            operandTypes(i) = inferredTypes.last.getComponentType
+          } else {
+            operandTypes(i) = inferredTypes.last
+          }
+        }
+      }
+    }
+  }
+
+  private[flink] def createReturnTypeInference(
+      resultType: TypeInformation[_],
+      typeFactory: FlinkTypeFactory)
+  : SqlReturnTypeInference = {
+
+    new SqlReturnTypeInference {
+      override def inferReturnType(opBinding: SqlOperatorBinding): RelDataType = {
+        typeFactory.createTypeFromTypeInfo(resultType)
+      }
+    }
+  }
+
+  private[flink] def createOperandTypeChecker(aggregateFunction: AggregateFunction[_, _])
+  : SqlOperandTypeChecker = {
+
+    val signatures = getMethodSignatures(aggregateFunction, "accumulate")
+
+    /**
+      * Operand type checker based on [[AggregateFunction]] given information.
+      */
+    new SqlOperandTypeChecker {
+      override def getAllowedSignatures(op: SqlOperator, opName: String): String = {
+        s"$opName[${signaturesToString(aggregateFunction, "accumulate")}]"
+      }
+
+      override def getOperandCountRange: SqlOperandCountRange = {
+        var min = 255
+        var max = -1
+        signatures.foreach(
+          sig => {
+            //do not count accumulator as input
+            val inputSig = sig.drop(1)
+            var len = inputSig.length
+            if (len > 0 && inputSig.last.isArray) {
+              max = 253 // according to JVM spec 4.3.3
+              len = sig.length - 1
+            }
+            max = Math.max(len, max)
+            min = Math.min(len, min)
+          })
+        SqlOperandCountRanges.between(min, max)
+      }
+
+      override def checkOperandTypes(
+          callBinding: SqlCallBinding,
+          throwOnFailure: Boolean)
+      : Boolean = {
+        val operandTypeInfo = getOperandTypeInfo(callBinding)
+
+        val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandTypeInfo)
+
+        if (foundSignature.isEmpty) {
+          if (throwOnFailure) {
+            throw new ValidationException(
+              s"Given parameters of function do not match any signature. \n" +
+                s"Actual: ${signatureToString(operandTypeInfo)} \n" +
+                s"Expected: ${signaturesToString(aggregateFunction, "accumulate")}")
+          } else {
+            false
+          }
+        } else {
+          true
+        }
+      }
+
+      override def isOptional(i: Int): Boolean = false
+
+      override def getConsistency: Consistency = Consistency.NONE
+
+    }
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/ScalarSqlFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/ScalarSqlFunction.scala
index e2cd272030cc4..bbfa3aa93ef0a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/ScalarSqlFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/ScalarSqlFunction.scala
@@ -23,12 +23,11 @@ import org.apache.calcite.sql._
 import org.apache.calcite.sql.`type`.SqlOperandTypeChecker.Consistency
 import org.apache.calcite.sql.`type`._
 import org.apache.calcite.sql.parser.SqlParserPos
-import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.ValidationException
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.functions.ScalarFunction
-import org.apache.flink.table.functions.utils.ScalarSqlFunction.{createOperandTypeChecker, createOperandTypeInference, createReturnTypeInference}
-import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{getResultType, getSignature, getSignatures, signatureToString, signaturesToString}
+import org.apache.flink.table.functions.utils.ScalarSqlFunction._
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
 
 import scala.collection.JavaConverters._
 
@@ -77,14 +76,14 @@ object ScalarSqlFunction {
               FlinkTypeFactory.toTypeInfo(operandType)
             }
           }
-        val foundSignature = getSignature(scalarFunction, parameters)
+        val foundSignature = getEvalMethodSignature(scalarFunction, parameters)
         if (foundSignature.isEmpty) {
           throw new ValidationException(
             s"Given parameters of function '$name' do not match any signature. \n" +
               s"Actual: ${signatureToString(parameters)} \n" +
-              s"Expected: ${signaturesToString(scalarFunction)}")
+              s"Expected: ${signaturesToString(scalarFunction, "eval")}")
         }
-        val resultType = getResultType(scalarFunction, foundSignature.get)
+        val resultType = getResultTypeOfScalarFunction(scalarFunction, foundSignature.get)
         val t = typeFactory.createTypeFromTypeInfo(resultType)
         typeFactory.createTypeWithNullability(t, nullable = true)
       }
@@ -106,7 +105,7 @@ object ScalarSqlFunction {
 
         val operandTypeInfo = getOperandTypeInfo(callBinding)
 
-        val foundSignature = getSignature(scalarFunction, operandTypeInfo)
+        val foundSignature = getEvalMethodSignature(scalarFunction, operandTypeInfo)
           .getOrElse(throw new ValidationException(s"Operand types of could not be inferred."))
 
         val inferredTypes = scalarFunction
@@ -132,14 +131,14 @@ object ScalarSqlFunction {
       scalarFunction: ScalarFunction)
     : SqlOperandTypeChecker = {
 
-    val signatures = getSignatures(scalarFunction)
+    val signatures = getMethodSignatures(scalarFunction, "eval")
 
     /**
       * Operand type checker based on [[ScalarFunction]] given information.
       */
     new SqlOperandTypeChecker {
       override def getAllowedSignatures(op: SqlOperator, opName: String): String = {
-        s"$opName[${signaturesToString(scalarFunction)}]"
+        s"$opName[${signaturesToString(scalarFunction, "eval")}]"
       }
 
       override def getOperandCountRange: SqlOperandCountRange = {
@@ -163,14 +162,14 @@ object ScalarSqlFunction {
         : Boolean = {
         val operandTypeInfo = getOperandTypeInfo(callBinding)
 
-        val foundSignature = getSignature(scalarFunction, operandTypeInfo)
+        val foundSignature = getEvalMethodSignature(scalarFunction, operandTypeInfo)
 
         if (foundSignature.isEmpty) {
           if (throwOnFailure) {
             throw new ValidationException(
               s"Given parameters of function '$name' do not match any signature. \n" +
                 s"Actual: ${signatureToString(operandTypeInfo)} \n" +
-                s"Expected: ${signaturesToString(scalarFunction)}")
+                s"Expected: ${signaturesToString(scalarFunction, "eval")}")
           } else {
             false
           }
@@ -185,16 +184,4 @@ object ScalarSqlFunction {
 
     }
   }
-
-  private[flink] def getOperandTypeInfo(callBinding: SqlCallBinding): Seq[TypeInformation[_]] = {
-    val operandTypes = for (i <- 0 until callBinding.getOperandCount)
-      yield callBinding.getOperandType(i)
-    operandTypes.map { operandType =>
-      if (operandType.getSqlTypeName == SqlTypeName.NULL) {
-        null
-      } else {
-        FlinkTypeFactory.toTypeInfo(operandType)
-      }
-    }
-  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
index d108e312b40fc..689bf0e6da528 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
@@ -25,15 +25,16 @@ import java.sql.{Date, Time, Timestamp}
 
 import org.apache.commons.codec.binary.Base64
 import com.google.common.primitives.Primitives
-import org.apache.calcite.sql.SqlFunction
+import org.apache.calcite.sql.`type`.SqlTypeName
+import org.apache.calcite.sql.{SqlCallBinding, SqlFunction}
 import org.apache.flink.api.common.functions.InvalidTypesException
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.api.{TableEnvironment, TableException, ValidationException}
 import org.apache.flink.table.expressions._
-import org.apache.flink.table.functions.{ScalarFunction, TableFunction, UserDefinedFunction}
 import org.apache.flink.table.plan.logical._
+import org.apache.flink.table.functions.{AggregateFunction, ScalarFunction, TableFunction, UserDefinedFunction}
 import org.apache.flink.table.plan.schema.FlinkTableFunctionImpl
 import org.apache.flink.util.InstantiationUtil
 
@@ -69,52 +70,89 @@ object UserDefinedFunctionUtils {
   }
 
   // ----------------------------------------------------------------------------------------------
-  // Utilities for eval methods
+  // Utilities for user-defined methods
   // ----------------------------------------------------------------------------------------------
 
   /**
-    * Returns signatures matching the given signature of [[TypeInformation]].
+    * Returns signatures of eval methods matching the given signature of [[TypeInformation]].
     * Elements of the signature can be null (act as a wildcard).
     */
-  def getSignature(
-      function: UserDefinedFunction,
-      signature: Seq[TypeInformation[_]])
+  def getEvalMethodSignature(
+    function: UserDefinedFunction,
+    signature: Seq[TypeInformation[_]])
     : Option[Array[Class[_]]] = {
-    getEvalMethod(function, signature).map(_.getParameterTypes)
+    getUserDefinedMethod(function, "eval", typeInfoToClass(signature)).map(_.getParameterTypes)
   }
 
   /**
-    * Returns eval method matching the given signature of [[TypeInformation]].
+    * Returns signatures of accumulate methods matching the given signature of [[TypeInformation]].
+    * Elements of the signature can be null (act as a wildcard).
     */
-  def getEvalMethod(
-      function: UserDefinedFunction,
+  def getAccumulateMethodSignature(
+      function: AggregateFunction[_, _],
       signature: Seq[TypeInformation[_]])
+  : Option[Array[Class[_]]] = {
+    val accType = TypeExtractor.createTypeInfo(
+      function, classOf[AggregateFunction[_, _]], function.getClass, 1)
+    val input = (Array(accType) ++ signature).toSeq
+    getUserDefinedMethod(
+      function,
+      "accumulate",
+      typeInfoToClass(input)).map(_.getParameterTypes)
+  }
+
+  def getParameterTypes(
+      function: UserDefinedFunction,
+      signature: Array[Class[_]]): Array[TypeInformation[_]] = {
+    signature.map { c =>
+      try {
+        TypeExtractor.getForClass(c)
+      } catch {
+        case ite: InvalidTypesException =>
+          throw new ValidationException(
+            s"Parameter types of function '${function.getClass.getCanonicalName}' cannot be " +
+              s"automatically determined. Please provide type information manually.")
+      }
+    }
+  }
+
+  /**
+    * Returns user defined method matching the given name and signature.
+    *
+    * @param function        function instance
+    * @param methodName      method name
+    * @param methodSignature an array of raw Java classes. We compare the raw Java classes not the
+    *                        TypeInformation. TypeInformation does not matter during runtime (e.g.
+    *                        within a MapFunction)
+    */
+  def getUserDefinedMethod(
+      function: UserDefinedFunction,
+      methodName: String,
+      methodSignature: Array[Class[_]])
     : Option[Method] = {
-    // We compare the raw Java classes not the TypeInformation.
-    // TypeInformation does not matter during runtime (e.g. within a MapFunction).
-    val actualSignature = typeInfoToClass(signature)
-    val evalMethods = checkAndExtractEvalMethods(function)
 
-    val filtered = evalMethods
-      // go over all eval methods and filter out matching methods
+    val methods = checkAndExtractMethods(function, methodName)
+
+    val filtered = methods
+      // go over all the methods and filter out matching methods
       .filter {
         case cur if !cur.isVarArgs =>
           val signatures = cur.getParameterTypes
           // match parameters of signature to actual parameters
-          actualSignature.length == signatures.length &&
+          methodSignature.length == signatures.length &&
             signatures.zipWithIndex.forall { case (clazz, i) =>
-              parameterTypeEquals(actualSignature(i), clazz)
+              parameterTypeEquals(methodSignature(i), clazz)
           }
         case cur if cur.isVarArgs =>
           val signatures = cur.getParameterTypes
-          actualSignature.zipWithIndex.forall {
+          methodSignature.zipWithIndex.forall {
             // non-varargs
             case (clazz, i) if i < signatures.length - 1  =>
               parameterTypeEquals(clazz, signatures(i))
             // varargs
             case (clazz, i) if i >= signatures.length - 1 =>
               parameterTypeEquals(clazz, signatures.last.getComponentType)
-          } || (actualSignature.isEmpty && signatures.length == 1) // empty varargs
+          } || (methodSignature.isEmpty && signatures.length == 1) // empty varargs
     }
 
     // if there is a fixed method, compiler will call this method preferentially
@@ -126,19 +164,21 @@ object UserDefinedFunctionUtils {
 
     // check if there is a Scala varargs annotation
     if (found.isEmpty &&
-      evalMethods.exists { evalMethod =>
-        val signatures = evalMethod.getParameterTypes
+      methods.exists { method =>
+        val signatures = method.getParameterTypes
         signatures.zipWithIndex.forall {
           case (clazz, i) if i < signatures.length - 1 =>
-            parameterTypeEquals(actualSignature(i), clazz)
+            parameterTypeEquals(methodSignature(i), clazz)
           case (clazz, i) if i == signatures.length - 1 =>
             clazz.getName.equals("scala.collection.Seq")
         }
       }) {
-      throw new ValidationException("Scala-style variable arguments in 'eval' methods are not " +
-        "supported. Please add a @scala.annotation.varargs annotation.")
+      throw new ValidationException(
+        s"Scala-style variable arguments in '${methodName}' methods are not supported. Please " +
+          s"add a @scala.annotation.varargs annotation.")
     } else if (found.length > 1) {
-      throw new ValidationException("Found multiple 'eval' methods which match the signature.")
+      throw new ValidationException(
+        "Found multiple '${methodName}' methods which match the signature.")
     }
     found.headOption
   }
@@ -157,16 +197,18 @@ object UserDefinedFunctionUtils {
   }
 
   /**
-    * Extracts "eval" methods and throws a [[ValidationException]] if no implementation
+    * Extracts methods and throws a [[ValidationException]] if no implementation
     * can be found, or implementation does not match the requirements.
     */
-  def checkAndExtractEvalMethods(function: UserDefinedFunction): Array[Method] = {
+  def checkAndExtractMethods(
+      function: UserDefinedFunction,
+      methodName: String): Array[Method] = {
     val methods = function
       .getClass
-      .getDeclaredMethods
+      .getMethods
       .filter { m =>
         val modifiers = m.getModifiers
-        m.getName == "eval" &&
+        m.getName == methodName &&
           Modifier.isPublic(modifiers) &&
           !Modifier.isAbstract(modifiers) &&
           !(function.isInstanceOf[TableFunction[_]] && Modifier.isStatic(modifiers))
@@ -175,15 +217,17 @@ object UserDefinedFunctionUtils {
     if (methods.isEmpty) {
       throw new ValidationException(
         s"Function class '${function.getClass.getCanonicalName}' does not implement at least " +
-          s"one method named 'eval' which is public, not abstract and " +
+          s"one method named '${methodName}' which is public, not abstract and " +
           s"(in case of table functions) not static.")
     }
 
     methods
   }
 
-  def getSignatures(function: UserDefinedFunction): Array[Array[Class[_]]] = {
-    checkAndExtractEvalMethods(function).map(_.getParameterTypes)
+  def getMethodSignatures(
+      function: UserDefinedFunction,
+      methodName: String): Array[Array[Class[_]]] = {
+    checkAndExtractMethods(function, methodName).map(_.getParameterTypes)
   }
 
   // ----------------------------------------------------------------------------------------------
@@ -222,7 +266,7 @@ object UserDefinedFunctionUtils {
       typeFactory: FlinkTypeFactory)
     : Seq[SqlFunction] = {
     val (fieldNames, fieldIndexes, _) = UserDefinedFunctionUtils.getFieldInfo(resultType)
-    val evalMethods = checkAndExtractEvalMethods(tableFunction)
+    val evalMethods = checkAndExtractMethods(tableFunction, "eval")
 
     evalMethods.map { method =>
       val function = new FlinkTableFunctionImpl(resultType, fieldIndexes, fieldNames, method)
@@ -230,29 +274,75 @@ object UserDefinedFunctionUtils {
     }
   }
 
+  /**
+    * Create [[SqlFunction]] for an [[AggregateFunction]]
+    *
+    * @param name function name
+    * @param aggFunction aggregate function
+    * @param typeFactory type factory
+    * @return the TableSqlFunction
+    */
+  def createAggregateSqlFunction(
+      name: String,
+      aggFunction: AggregateFunction[_, _],
+      typeInfo: TypeInformation[_],
+      typeFactory: FlinkTypeFactory)
+  : SqlFunction = {
+    //check if a qualified accumulate method exists before create Sql function
+    checkAndExtractMethods(aggFunction, "accumulate")
+    val resultType: TypeInformation[_] = getResultTypeOfAggregateFunction(aggFunction, typeInfo)
+    AggSqlFunction(name, aggFunction, resultType, typeFactory)
+  }
+
   // ----------------------------------------------------------------------------------------------
-  // Utilities for scalar functions
+  // Utilities for user-defined functions
   // ----------------------------------------------------------------------------------------------
 
+  /**
+    * Internal method of AggregateFunction#getResultType() that does some pre-checking and uses
+    * [[TypeExtractor]] as default return type inference.
+    */
+  def getResultTypeOfAggregateFunction(
+      aggregateFunction: AggregateFunction[_, _],
+      extractedType: TypeInformation[_] = null)
+    : TypeInformation[_] = {
+
+    val resultType = try {
+      val method: Method = aggregateFunction.getClass.getMethod("getResultType")
+      method.invoke(aggregateFunction).asInstanceOf[TypeInformation[_]]
+    } catch {
+      case _: NoSuchMethodException => null
+      case ite: Throwable => throw new TableException("Unexpected exception:", ite)
+    }
+    if (resultType != null) {
+      resultType
+    } else if(extractedType != null) {
+      extractedType
+    } else {
+      TypeExtractor
+        .createTypeInfo(aggregateFunction,
+                        classOf[AggregateFunction[_, _]],
+                        aggregateFunction.getClass,
+                        0)
+        .asInstanceOf[TypeInformation[_]]
+    }
+  }
+
   /**
     * Internal method of [[ScalarFunction#getResultType()]] that does some pre-checking and uses
     * [[TypeExtractor]] as default return type inference.
     */
-  def getResultType(
+  def getResultTypeOfScalarFunction(
       function: ScalarFunction,
       signature: Array[Class[_]])
     : TypeInformation[_] = {
-    // find method for signature
-    val evalMethod = checkAndExtractEvalMethods(function)
-      .find(m => signature.sameElements(m.getParameterTypes))
-      .getOrElse(throw new ValidationException("Given signature is invalid."))
 
     val userDefinedTypeInfo = function.getResultType(signature)
     if (userDefinedTypeInfo != null) {
       userDefinedTypeInfo
     } else {
       try {
-        TypeExtractor.getForClass(evalMethod.getReturnType)
+        TypeExtractor.getForClass(getResultTypeClassOfScalarFunction(function, signature))
       } catch {
         case ite: InvalidTypesException =>
           throw new ValidationException(
@@ -265,12 +355,12 @@ object UserDefinedFunctionUtils {
   /**
     * Returns the return type of the evaluation method matching the given signature.
     */
-  def getResultTypeClass(
+  def getResultTypeClassOfScalarFunction(
       function: ScalarFunction,
       signature: Array[Class[_]])
     : Class[_] = {
     // find method for signature
-    val evalMethod = checkAndExtractEvalMethods(function)
+    val evalMethod = checkAndExtractMethods(function, "eval")
       .find(m => signature.sameElements(m.getParameterTypes))
       .getOrElse(throw new IllegalArgumentException("Given signature is invalid."))
     evalMethod.getReturnType
@@ -317,16 +407,16 @@ object UserDefinedFunctionUtils {
   }
 
   /**
-    * Prints all eval methods signatures of a class.
+    * Prints all signatures of methods with given name in a class.
     */
-  def signaturesToString(function: UserDefinedFunction): String = {
-    getSignatures(function).map(signatureToString).mkString(", ")
+  def signaturesToString(function: UserDefinedFunction, name: String): String = {
+    getMethodSignatures(function, name).map(signatureToString).mkString(", ")
   }
 
   /**
     * Extracts type classes of [[TypeInformation]] in a null-aware way.
     */
-  private def typeInfoToClass(typeInfos: Seq[TypeInformation[_]]): Array[Class[_]] =
+  def typeInfoToClass(typeInfos: Seq[TypeInformation[_]]): Array[Class[_]] =
   typeInfos.map { typeInfo =>
     if (typeInfo == null) {
       null
@@ -393,4 +483,16 @@ object UserDefinedFunctionUtils {
       .as(alias).toLogicalTableFunctionCall(child = null)
     functionCall
   }
+
+  def getOperandTypeInfo(callBinding: SqlCallBinding): Seq[TypeInformation[_]] = {
+    val operandTypes = for (i <- 0 until callBinding.getOperandCount)
+      yield callBinding.getOperandType(i)
+    operandTypes.map { operandType =>
+      if (operandType.getSqlTypeName == SqlTypeName.NULL) {
+        null
+      } else {
+        FlinkTypeFactory.toTypeInfo(operandType)
+      }
+    }
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
index 0d45a37992123..d26cdcf208260 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
@@ -303,6 +303,11 @@ object ProjectionTranslator {
         (fieldReferences, expr) => identifyFieldReferences(expr, fieldReferences)
       }
 
+    case aggfc @ AggFunctionCall(clazz, args) =>
+      args.foldLeft(fieldReferences) {
+        (fieldReferences, expr) => identifyFieldReferences(expr, fieldReferences)
+      }
+
     // array constructor
     case c @ ArrayConstructor(args) =>
       args.foldLeft(fieldReferences) {
@@ -327,4 +332,56 @@ object ProjectionTranslator {
       }
   }
 
+  /**
+    * Find and replace UDAGG function Call to AggFunctionCall
+    *
+    * @param field    the expression to check
+    * @param tableEnv the TableEnvironment
+    * @return an expression with correct AggFunctionCall type for UDAGG functions
+    */
+  def replaceAggFunctionCall(field: Expression, tableEnv: TableEnvironment): Expression = {
+    field match {
+      case l: LeafExpression => l
+
+      case u: UnaryExpression =>
+        val c = replaceAggFunctionCall(u.child, tableEnv)
+        u.makeCopy(Array(c))
+
+      case b: BinaryExpression =>
+        val l = replaceAggFunctionCall(b.left, tableEnv)
+        val r = replaceAggFunctionCall(b.right, tableEnv)
+        b.makeCopy(Array(l, r))
+
+      // Functions calls
+      case c @ Call(name, args) =>
+        val function = tableEnv.getFunctionCatalog.lookupFunction(name, args)
+        if (function.isInstanceOf[AggFunctionCall]) {
+          function
+        } else {
+          val newArgs =
+            args.map(
+            (exp: Expression) =>
+              replaceAggFunctionCall(exp, tableEnv))
+          c.makeCopy(Array(name, newArgs))
+        }
+
+      // Scala functions
+      case sfc @ ScalarFunctionCall(clazz, args) =>
+        val newArgs: Seq[Expression] =
+          args.map(
+            (exp: Expression) =>
+              replaceAggFunctionCall(exp, tableEnv))
+        sfc.makeCopy(Array(clazz, newArgs))
+
+      // Array constructor
+      case c @ ArrayConstructor(args) =>
+        val newArgs =
+          c.elements
+            .map((exp: Expression) => replaceAggFunctionCall(exp, tableEnv))
+        c.makeCopy(Array(newArgs))
+
+      // Other expressions
+      case e: Expression => e
+    }
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index c67bfd18d301e..5f2394cd26a3a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -685,12 +685,12 @@ case class LogicalTableFunctionCall(
     checkForInstantiation(tableFunction.getClass)
     // look for a signature that matches the input types
     val signature = node.parameters.map(_.resultType)
-    val foundMethod = getEvalMethod(tableFunction, signature)
+    val foundMethod = getUserDefinedMethod(tableFunction, "eval", typeInfoToClass(signature))
     if (foundMethod.isEmpty) {
       failValidation(
         s"Given parameters of function '$functionName' do not match any signature. \n" +
           s"Actual: ${signatureToString(signature)} \n" +
-          s"Expected: ${signaturesToString(tableFunction)}")
+          s"Expected: ${signaturesToString(tableFunction, "eval")}")
     } else {
       node.evalMethod = foundMethod.get
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonAggregate.scala
index 3883b1467a468..e95747cbaf951 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonAggregate.scala
@@ -50,7 +50,7 @@ trait CommonAggregate {
     val aggs = namedAggregates.map(_.getKey)
     val aggStrings = aggs.map( a => s"${a.getAggregation}(${
       if (a.getArgList.size() > 0) {
-        inFields(a.getArgList.get(0))
+        a.getArgList.asScala.map(inFields(_)).mkString(", ")
       } else {
         "*"
       }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala
index 91c8cefabe358..6878473e0f4b0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala
@@ -88,7 +88,7 @@ trait OverAggregate {
     val aggStrings = namedAggregates.map(_.getKey).map(
       a => s"${a.getAggregation}(${
         if (a.getArgList.size() > 0) {
-          inFields(a.getArgList.get(0))
+          a.getArgList.asScala.map(inFields(_)).mkString(", ")
         } else {
           "*"
         }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
index 187773da7b55a..c232a71f02631 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
@@ -122,6 +122,12 @@ class DataStreamAggregate(
       false,
       inputDS.getType)
 
+    val needMerge = window match {
+      case ProcessingTimeSessionGroupWindow(_, _) => true
+      case EventTimeSessionGroupWindow(_, _, _) => true
+      case _ => false
+    }
+
     // grouped / keyed aggregation
     if (grouping.length > 0) {
       val windowFunction = AggregateUtil.createAggregationGroupWindowFunction(
@@ -141,7 +147,8 @@ class DataStreamAggregate(
           generator,
           namedAggregates,
           inputType,
-          rowRelDataType)
+          rowRelDataType,
+          needMerge)
 
       windowedStream
         .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, rowTypeInfo)
@@ -163,7 +170,8 @@ class DataStreamAggregate(
           generator,
           namedAggregates,
           inputType,
-          rowRelDataType)
+          rowRelDataType,
+          needMerge)
 
       windowedStream
         .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, rowTypeInfo)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 2950a78a8f933..e38207df87fb1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -39,6 +39,7 @@ import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.expressions._
 import org.apache.flink.table.functions.aggfunctions._
+import org.apache.flink.table.functions.utils.AggSqlFunction
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
 import org.apache.flink.table.functions.{AggregateFunction => TableAggregateFunction}
 import org.apache.flink.table.plan.logical._
@@ -101,7 +102,8 @@ object AggregateUtil {
       None,
       outputArity,
       needRetract,
-      needMerge = false
+      needMerge = false,
+      needReset = false
     )
 
     if (isRowTimeType) {
@@ -178,7 +180,8 @@ object AggregateUtil {
       None,
       outputArity,
       needRetract,
-      needMerge = false
+      needMerge = false,
+      needReset = true
     )
 
     if (isRowTimeType) {
@@ -303,7 +306,8 @@ object AggregateUtil {
       None,
       outputArity,
       needRetract,
-      needMerge = false
+      needMerge = false,
+      needReset = true
     )
 
     new DataSetWindowAggMapFunction(
@@ -374,12 +378,13 @@ object AggregateUtil {
           aggFieldIndexes,
           aggregates.indices.map(_ + groupings.length).toArray,
           partialResults = true,
-          groupings,
+          groupings.indices.toArray,
           Some(aggregates.indices.map(_ + groupings.length).toArray),
           None,
           keysAndAggregatesArity + 1,
           needRetract,
-          needMerge = true
+          needMerge = true,
+          needReset = true
         )
         new DataSetSlideTimeWindowAggReduceGroupFunction(
           genFunction,
@@ -481,7 +486,8 @@ object AggregateUtil {
       None,
       outputType.getFieldCount,
       needRetract,
-      needMerge = true
+      needMerge = true,
+      needReset = true
     )
 
     val genFinalAggFunction = generator.generateAggregations(
@@ -497,7 +503,8 @@ object AggregateUtil {
       None,
       outputType.getFieldCount,
       needRetract,
-      needMerge = true
+      needMerge = true,
+      needReset = true
     )
 
     val keysAndAggregatesArity = groupings.length + namedAggregates.length
@@ -636,7 +643,8 @@ object AggregateUtil {
           None,
           groupings.length + aggregates.length + 2,
           needRetract,
-          needMerge = true
+          needMerge = true,
+          needReset = true
         )
 
         new DataSetSessionWindowAggregatePreProcessor(
@@ -708,7 +716,8 @@ object AggregateUtil {
           None,
           groupings.length + aggregates.length + 2,
           needRetract,
-          needMerge = true
+          needMerge = true,
+          needReset = true
         )
 
         new DataSetSessionWindowAggregatePreProcessor(
@@ -787,7 +796,8 @@ object AggregateUtil {
         None,
         groupings.length + aggregates.length,
         needRetract,
-        needMerge = false
+        needMerge = false,
+        needReset = true
       )
 
       // compute mapping of forwarded grouping keys
@@ -813,7 +823,8 @@ object AggregateUtil {
         constantFlags,
         outputType.getFieldCount,
         needRetract,
-        needMerge = true
+        needMerge = true,
+        needReset = true
       )
 
       (
@@ -836,7 +847,8 @@ object AggregateUtil {
         constantFlags,
         outputType.getFieldCount,
         needRetract,
-        needMerge = false
+        needMerge = false,
+        needReset = true
       )
 
       (
@@ -902,7 +914,8 @@ object AggregateUtil {
       generator: CodeGenerator,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       inputType: RelDataType,
-      outputType: RelDataType)
+      outputType: RelDataType,
+      needMerge: Boolean)
     : (DataStreamAggFunction[Row, Row, Row], RowTypeInfo, RowTypeInfo) = {
 
     val needRetract = false
@@ -928,7 +941,8 @@ object AggregateUtil {
       None,
       outputArity,
       needRetract,
-      needMerge = true
+      needMerge,
+      needReset = false
     )
 
     val aggResultTypes = namedAggregates.map(a => FlinkTypeFactory.toTypeInfo(a.left.getType))
@@ -1083,9 +1097,6 @@ object AggregateUtil {
           throw new TableException("Aggregate fields should not be empty.")
         }
       } else {
-        if (argList.size() > 1) {
-          throw new TableException("Currently, do not support aggregate on multi fields.")
-        }
         aggFieldIndexes(index) = argList.asScala.map(i => i.intValue).toArray
       }
       val sqlTypeName = inputType.getFieldList.get(aggFieldIndexes(index)(0)).getType
@@ -1298,6 +1309,9 @@ object AggregateUtil {
         case _: SqlCountAggFunction =>
           aggregates(index) = new CountAggFunction
 
+        case udagg: AggSqlFunction =>
+          aggregates(index) = udagg.getFunction
+
         case unSupported: SqlAggFunction =>
           throw new TableException("unsupported Function: " + unSupported.getName)
       }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
index 1022e4d59eb6b..63dc1aedc337e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
@@ -23,8 +23,8 @@ import org.apache.calcite.sql.util.{ChainedSqlOperatorTable, ListSqlOperatorTabl
 import org.apache.calcite.sql.{SqlFunction, SqlOperator, SqlOperatorTable}
 import org.apache.flink.table.api.ValidationException
 import org.apache.flink.table.expressions._
-import org.apache.flink.table.functions.utils.{ScalarSqlFunction, TableSqlFunction}
-import org.apache.flink.table.functions.{EventTimeExtractor, RowTime, ScalarFunction, TableFunction, _}
+import org.apache.flink.table.functions.utils.{AggSqlFunction, ScalarSqlFunction, TableSqlFunction}
+import org.apache.flink.table.functions.{AggregateFunction, EventTimeExtractor, RowTime, ScalarFunction, TableFunction, _}
 
 import scala.collection.JavaConversions._
 import scala.collection.mutable
@@ -97,6 +97,15 @@ class FunctionCatalog {
         val function = tableSqlFunction.getTableFunction
         TableFunctionCall(name, function, children, typeInfo)
 
+      // user-defined aggregate function call
+      case af if classOf[AggregateFunction[_, _]].isAssignableFrom(af) =>
+        val aggregateFunction = sqlFunctions
+          .find(f => f.getName.equalsIgnoreCase(name) && f.isInstanceOf[AggSqlFunction])
+          .getOrElse(throw ValidationException(s"Undefined table function: $name"))
+          .asInstanceOf[AggSqlFunction]
+        val function = aggregateFunction.getFunction
+        AggFunctionCall(function, children)
+
       // general expression call
       case expression if classOf[Expression].isAssignableFrom(expression) =>
         // try to find a constructor accepts `Seq[Expression]`
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/utils/UserDefinedAggFunctions.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/utils/UserDefinedAggFunctions.java
new file mode 100644
index 0000000000000..cfddc57272343
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/utils/UserDefinedAggFunctions.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.api.java.utils;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.table.functions.AggregateFunction;
+
+import java.util.Iterator;
+
+public class UserDefinedAggFunctions {
+
+    // Accumulator for WeightedAvg
+    public static class WeightedAvgAccum extends Tuple2<Long, Integer> {
+        public long sum = 0;
+        public int count = 0;
+    }
+
+    // Base class for WeightedAvg
+    public static class WeightedAvg extends AggregateFunction<Long, WeightedAvgAccum> {
+        @Override
+        public WeightedAvgAccum createAccumulator() {
+            return new WeightedAvgAccum();
+        }
+
+        @Override
+        public Long getValue(WeightedAvgAccum accumulator) {
+            if (accumulator.count == 0)
+                return null;
+            else
+                return accumulator.sum/accumulator.count;
+        }
+
+        //Overloaded accumulate method
+        public void accumulate(WeightedAvgAccum accumulator, long iValue, int iWeight) {
+            accumulator.sum += iValue * iWeight;
+            accumulator.count += iWeight;
+        }
+
+        //Overloaded accumulate method
+        public void accumulate(WeightedAvgAccum accumulator, int iValue, int iWeight) {
+            accumulator.sum += iValue * iWeight;
+            accumulator.count += iWeight;
+        }
+    }
+
+    // A WeightedAvg class with merge method
+    public static class WeightedAvgWithMerge extends WeightedAvg {
+        public void merge(WeightedAvgAccum acc, Iterable<WeightedAvgAccum> it) {
+            Iterator<WeightedAvgAccum> iter = it.iterator();
+            while (iter.hasNext()) {
+                WeightedAvgAccum a = iter.next();
+                acc.count += a.count;
+                acc.sum += a.sum;
+            }
+        }
+    }
+
+    // A WeightedAvg class with merge and reset method
+    public static class WeightedAvgWithMergeAndReset extends WeightedAvgWithMerge {
+        public void resetAccumulator(WeightedAvgAccum acc) {
+            acc.count = 0;
+            acc.sum = 0L;
+        }
+    }
+
+    // A WeightedAvg class with retract method
+    public static class WeightedAvgWithRetract extends WeightedAvg {
+        //Overloaded retract method
+        public void retract(WeightedAvgAccum accumulator, long iValue, int iWeight) {
+            accumulator.sum -= iValue * iWeight;
+            accumulator.count -= iWeight;
+        }
+
+        //Overloaded retract method
+        public void retract(WeightedAvgAccum accumulator, int iValue, int iWeight) {
+            accumulator.sum -= iValue * iWeight;
+            accumulator.count -= iWeight;
+        }
+    }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
index 600c15be7ba5c..8368817c8f12d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/AggregationsITCase.scala
@@ -27,6 +27,8 @@ import org.apache.flink.api.scala.util.CollectionDataSets
 import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
 import org.apache.flink.types.Row
 import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMergeAndReset
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
 import org.apache.flink.test.util.TestBaseUtils
 import org.junit._
 import org.junit.runner.RunWith
@@ -300,9 +302,11 @@ class AggregationsITCase(
 
     val env = ExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    tEnv.registerFunction("countFun", new CountAggFunction)
+    tEnv.registerFunction("wAvgWithMergeAndReset", new WeightedAvgWithMergeAndReset)
 
     val sqlQuery =
-      "SELECT b, SUM(a), COUNT(*)" +
+      "SELECT b, SUM(a), countFun(c), wAvgWithMergeAndReset(b, a), wAvgWithMergeAndReset(a, a)" +
         "FROM T " +
         "GROUP BY b, TUMBLE(ts, INTERVAL '3' SECOND)"
 
@@ -313,12 +317,12 @@ class AggregationsITCase(
 
     val result = tEnv.sql(sqlQuery).toDataSet[Row].collect()
     val expected = Seq(
-      "1,1,1",
-      "2,2,1", "2,3,1",
-      "3,9,2", "3,6,1",
-      "4,15,2", "4,19,2",
-      "5,11,1", "5,39,3", "5,15,1",
-      "6,33,2", "6,57,3", "6,21,1"
+      "1,1,1,1,1",
+      "2,2,1,2,2", "2,3,1,2,3",
+      "3,9,2,3,4", "3,6,1,3,6",
+      "4,15,2,4,7", "4,19,2,4,9",
+      "5,11,1,5,11", "5,39,3,5,13", "5,15,1,5,15",
+      "6,33,2,6,16", "6,57,3,6,19", "6,21,1,6,21"
     ).mkString("\n")
 
     TestBaseUtils.compareResultAsText(result.asJava, expected)
@@ -329,11 +333,13 @@ class AggregationsITCase(
 
     val env = ExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    tEnv.registerFunction("countFun", new CountAggFunction)
+    tEnv.registerFunction("wAvgWithMergeAndReset", new WeightedAvgWithMergeAndReset)
 
     env.setParallelism(1)
 
     val sqlQuery =
-      "SELECT b, SUM(a), COUNT(*)" +
+      "SELECT b, SUM(a), countFun(c), wAvgWithMergeAndReset(b, a), wAvgWithMergeAndReset(a, a)" +
         "FROM T " +
         "GROUP BY b, HOP(ts, INTERVAL '2' SECOND, INTERVAL '4' SECOND)"
 
@@ -344,12 +350,12 @@ class AggregationsITCase(
 
     val result = tEnv.sql(sqlQuery).toDataSet[Row].collect()
     val expected = Seq(
-      "1,1,1","1,1,1",
-      "2,5,2","2,5,2",
-      "3,9,2", "3,15,3", "3,6,1",
-      "4,7,1", "4,24,3", "4,27,3", "4,10,1",
-      "5,11,1", "5,36,3", "5,54,4", "5,29,2",
-      "6,33,2", "6,70,4", "6,78,4", "6,41,2"
+      "1,1,1,1,1","1,1,1,1,1",
+      "2,5,2,2,2","2,5,2,2,2",
+      "3,9,2,3,4", "3,15,3,3,5", "3,6,1,3,6",
+      "4,7,1,4,7", "4,24,3,4,8", "4,27,3,4,9", "4,10,1,4,10",
+      "5,11,1,5,11", "5,36,3,5,12", "5,54,4,5,13", "5,29,2,5,14",
+      "6,33,2,6,16", "6,70,4,6,17", "6,78,4,6,19", "6,41,2,6,20"
     ).mkString("\n")
 
     TestBaseUtils.compareResultAsText(result.asJava, expected)
@@ -360,11 +366,14 @@ class AggregationsITCase(
 
     val env = ExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    tEnv.registerFunction("countFun", new CountAggFunction)
+    tEnv.registerFunction("wAvgWithMergeAndReset", new WeightedAvgWithMergeAndReset)
 
     env.setParallelism(1)
 
     val sqlQuery =
-      "SELECT MIN(a), MAX(a), SUM(a), COUNT(*)" +
+      "SELECT MIN(a), MAX(a), SUM(a), countFun(c), wAvgWithMergeAndReset(b, a), " +
+        "wAvgWithMergeAndReset(a, a)" +
         "FROM T " +
         "GROUP BY SESSION(ts, INTERVAL '4' SECOND)"
 
@@ -376,8 +385,8 @@ class AggregationsITCase(
 
     val result = tEnv.sql(sqlQuery).toDataSet[Row].collect()
     val expected = Seq(
-      "2,10,39,6",
-      "16,21,111,6"
+      "2,10,39,6,3,7",
+      "16,21,111,6,6,18"
     ).mkString("\n")
 
     TestBaseUtils.compareResultAsText(result.asJava, expected)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
index 4f46a73f90a3d..0ccb55723a397 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
@@ -21,7 +21,8 @@ package org.apache.flink.table.api.scala.batch.sql
 import java.sql.Timestamp
 
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.TableException
+import org.apache.flink.table.api.{TableException, ValidationException}
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.utils.TableTestBase
@@ -85,6 +86,33 @@ class WindowAggregateTest extends TableTestBase {
     util.verifySql(sqlQuery, expected)
   }
 
+  @Test
+  def testTumbleWindowWithUdAgg() = {
+    val util = batchTestUtil()
+    util.addTable[(Int, Long, String, Timestamp)]("T", 'a, 'b, 'c, 'ts)
+
+    val weightedAvg = new WeightedAvgWithMerge
+    util.tEnv.registerFunction("weightedAvg", weightedAvg)
+
+    val sql = "SELECT weightedAvg(b, a) AS wAvg " +
+      "FROM T " +
+      "GROUP BY TUMBLE(ts, INTERVAL '4' MINUTE)"
+
+    val expected =
+      unaryNode(
+        "DataSetWindowAggregate",
+        unaryNode(
+          "DataSetCalc",
+          batchTableNode(0),
+          term("select", "ts, b, a")
+        ),
+        term("window", EventTimeTumblingGroupWindow('w$, 'ts, 240000.millis)),
+        term("select", "weightedAvg(b, a) AS wAvg")
+      )
+
+    util.verifySql(sql, expected)
+  }
+
   @Test
   def testNonPartitionedHopWindow(): Unit = {
     val util = batchTestUtil()
@@ -280,4 +308,17 @@ class WindowAggregateTest extends TableTestBase {
     util.verifySql(sql, "n/a")
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testTumbleWindowWithInvalidUdAggArgs() = {
+    val util = batchTestUtil()
+    util.addTable[(Int, Long, String, Timestamp)]("T", 'a, 'b, 'c, 'ts)
+
+    val weightedAvg = new WeightedAvgWithMerge
+    util.tEnv.registerFunction("weightedAvg", weightedAvg)
+
+    val sql = "SELECT weightedAvg(c, a) AS wAvg " +
+      "FROM T " +
+      "GROUP BY TUMBLE(ts, INTERVAL '4' MINUTE)"
+    util.verifySql(sql, "n/a")
+  }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
index ba6809dad1535..45abf09f86bdc 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/AggregationsITCase.scala
@@ -26,6 +26,8 @@ import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.batch.utils.TableProgramsCollectionTestBase
 import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMergeAndReset
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
 import org.apache.flink.test.util.TestBaseUtils
 import org.apache.flink.types.Row
 import org.junit._
@@ -205,12 +207,15 @@ class AggregationsITCase(
 
     val env = ExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val countFun = new CountAggFunction
+    val wAvgFun = new WeightedAvgWithMergeAndReset
 
     val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
       .groupBy('b)
-      .select('b, 'a.sum)
+      .select('b, 'a.sum, countFun('c), wAvgFun('b, 'a), wAvgFun('a, 'a))
 
-    val expected = "1,1\n" + "2,5\n" + "3,15\n" + "4,34\n" + "5,65\n" + "6,111\n"
+    val expected = "1,1,1,1,1\n" + "2,5,2,2,2\n" + "3,15,3,3,5\n" + "4,34,4,4,8\n" +
+      "5,65,5,5,13\n" + "6,111,6,6,18\n"
     val results = t.toDataSet[Row].collect()
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
index 8f4eea54fd017..8a20f6d0d11e7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.api.scala.batch.table
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
 import org.apache.flink.table.expressions.WindowReference
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.utils.TableTestBase
@@ -83,6 +84,20 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testTumblingGroupWindowWithInvalidUdAggArgs(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    table
+      .window(Tumble over 2.minutes on 'rowtime as 'w)
+      .groupBy('w, 'long)
+      // invalid function arguments
+      .select(myWeightedAvg('int, 'string))
+  }
+
   @Test
   def testEventTimeTumblingGroupWindowOverCount(): Unit = {
     val util = batchTestUtil()
@@ -104,6 +119,29 @@ class GroupWindowTest extends TableTestBase {
     util.verifyTable(windowedTable, expected)
   }
 
+  @Test
+  def testEventTimeTumblingGroupWindowOverTimeWithUdAgg(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    val windowedTable = table
+      .window(Tumble over 5.milli on 'long as 'w)
+      .groupBy('w, 'string)
+      .select('string, myWeightedAvg('long, 'int))
+
+    val expected = unaryNode(
+      "DataSetWindowAggregate",
+      batchTableNode(0),
+      term("groupBy", "string"),
+      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
+      term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
   @Test
   def testEventTimeTumblingGroupWindowOverTime(): Unit = {
     val util = batchTestUtil()
@@ -147,6 +185,20 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testAllTumblingGroupWindowWithInvalidUdAggArgs(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    table
+      .window(Tumble over 2.minutes on 'rowtime as 'w)
+      .groupBy('w)
+      // invalid function arguments
+      .select(myWeightedAvg('int, 'string))
+  }
+
   @Test
   def testAllEventTimeTumblingGroupWindowOverTime(): Unit = {
     val util = batchTestUtil()
@@ -221,6 +273,20 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testSlidingGroupWindowWithInvalidUdAggArgs(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    table
+      .window(Slide over 2.minutes every 1.minute on 'rowtime as 'w)
+      .groupBy('w, 'long)
+      // invalid function arguments
+      .select(myWeightedAvg('int, 'string))
+  }
+
   @Test
   def testEventTimeSlidingGroupWindowOverTime(): Unit = {
     val util = batchTestUtil()
@@ -265,6 +331,30 @@ class GroupWindowTest extends TableTestBase {
     util.verifyTable(windowedTable, expected)
   }
 
+  @Test
+  def testEventTimeSlidingGroupWindowOverTimeWithUdAgg(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    val windowedTable = table
+      .window(Slide over 8.milli every 10.milli on 'long as 'w)
+      .groupBy('w, 'string)
+      .select('string, myWeightedAvg('long, 'int))
+
+    val expected = unaryNode(
+      "DataSetWindowAggregate",
+      batchTableNode(0),
+      term("groupBy", "string"),
+      term("window",
+           EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
+      term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
   @Test(expected = classOf[ValidationException])
   def testAllProcessingTimeSlidingGroupWindowOverCount(): Unit = {
     val util = batchTestUtil()
@@ -276,6 +366,20 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testAllSlidingGroupWindowWithInvalidUdAggArgs(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    table
+      .window(Slide over 2.minutes every 1.minute on 'rowtime as 'w)
+      .groupBy('w)
+      // invalid function arguments
+      .select(myWeightedAvg('int, 'string))
+  }
+
   @Test
   def testAllEventTimeSlidingGroupWindowOverTime(): Unit = {
     val util = batchTestUtil()
@@ -351,6 +455,29 @@ class GroupWindowTest extends TableTestBase {
     util.verifyTable(windowedTable, expected)
   }
 
+  @Test
+  def testEventTimeSessionGroupWindowOverTimeWithUdAgg(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    val windowedTable = table
+      .window(Session withGap 7.milli on 'long as 'w)
+      .groupBy('w, 'string)
+      .select('string, myWeightedAvg('long, 'int))
+
+    val expected = unaryNode(
+      "DataSetWindowAggregate",
+      batchTableNode(0),
+      term("groupBy", "string"),
+      term("window", EventTimeSessionGroupWindow(WindowReference("w"), 'long, 7.milli)),
+      term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
   @Test(expected = classOf[ValidationException])
   def testProcessingTimeSessionGroupWindowOverTime(): Unit = {
     val util = batchTestUtil()
@@ -362,4 +489,32 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testSessionGroupWindowWithInvalidUdAggArgs(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    table
+      .window(Session withGap 2.minutes on 'rowtime as 'w)
+      .groupBy('w, 'long)
+      // invalid function arguments
+      .select(myWeightedAvg('int, 'string))
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testAllSessionGroupWindowWithInvalidUdAggArgs(): Unit = {
+    val util = batchTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val myWeightedAvg = new WeightedAvgWithMerge
+
+    table
+      .window(Session withGap 2.minutes on 'rowtime as 'w)
+      .groupBy('w)
+      // invalid function arguments
+      .select(myWeightedAvg('int, 'string))
+  }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala
index 528ffd8abc44e..eda958c3c295b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/stringexpr/AggregationsStringExpressionTest.scala
@@ -23,6 +23,9 @@ import org.apache.flink.api.scala.util.CollectionDataSets
 import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.utils.TableTestBase
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMergeAndReset
+import org.apache.flink.table.api.scala.batch.utils.LogicalPlanFormatUtils
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
 import org.junit._
 
 class AggregationsStringExpressionTest extends TableTestBase {
@@ -290,4 +293,59 @@ class AggregationsStringExpressionTest extends TableTestBase {
     verifyTableEquals(resScala, resJava)
   }
 
+  @Test
+  def testAggregateWithUDAGG(): Unit = {
+
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+
+    val myCnt = new CountAggFunction
+    tEnv.registerFunction("myCnt", myCnt)
+    val myWeightedAvg = new WeightedAvgWithMergeAndReset
+    tEnv.registerFunction("myWeightedAvg", myWeightedAvg)
+
+    val t1 = t.select(myCnt('a) as 'aCnt, myWeightedAvg('b, 'a) as 'wAvg)
+    val t2 = t.select("myCnt(a) as aCnt, myWeightedAvg(b, a) as wAvg")
+
+    val lPlan1 = t1.logicalPlan
+    val lPlan2 = t2.logicalPlan
+
+    val x = LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString)
+    val y = LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString)
+
+    Assert.assertEquals(
+      "Logical Plans do not match",
+      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
+      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+  }
+
+  @Test
+  def testGroupedAggregateWithUDAGG(): Unit = {
+
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+
+    val myCnt = new CountAggFunction
+    tEnv.registerFunction("myCnt", myCnt)
+    val myWeightedAvg = new WeightedAvgWithMergeAndReset
+    tEnv.registerFunction("myWeightedAvg", myWeightedAvg)
+
+    val t1 = t.groupBy('b)
+      .select('b, myCnt('a) + 9 as 'aCnt, myWeightedAvg('b, 'a) * 2 as 'wAvg, myWeightedAvg('a, 'a))
+    val t2 = t.groupBy("b")
+      .select("b, myCnt(a) + 9 as aCnt, myWeightedAvg(b, a) * 2 as wAvg, myWeightedAvg(a, a)")
+
+    val lPlan1 = t1.logicalPlan
+    val lPlan2 = t2.logicalPlan
+
+    Assert.assertEquals(
+      "Logical Plans do not match",
+      LogicalPlanFormatUtils.formatTempTableId(lPlan1.toString),
+      LogicalPlanFormatUtils.formatTempTableId(lPlan2.toString))
+  }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/validation/AggregationsValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/validation/AggregationsValidationTest.scala
index 5cf3f8433cd70..278711c2c231f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/validation/AggregationsValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/validation/AggregationsValidationTest.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.api.scala.batch.table.validation
 
 import org.apache.flink.api.scala._
 import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMergeAndReset
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.{TableEnvironment, ValidationException}
 import org.junit._
@@ -83,6 +84,50 @@ class AggregationsValidationTest {
       .select('foo.avg)
   }
 
+  @Test(expected = classOf[ValidationException])
+  @throws[Exception]
+  def testInvalidUdAggArgs() {
+    val env= ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+
+    val myWeightedAvg = new WeightedAvgWithMergeAndReset
+
+    val input = CollectionDataSets.get3TupleDataSet(env).toTable(tableEnv, 'a, 'b, 'c)
+    input
+      // must fail. UDAGG does not accept String type
+      .select(myWeightedAvg('c, 'a))
+  }
+
+  @Test(expected = classOf[ValidationException])
+  @throws[Exception]
+  def testGroupingInvalidUdAggArgs() {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+
+    val myWeightedAvg = new WeightedAvgWithMergeAndReset
+
+    val input = CollectionDataSets.get3TupleDataSet(env).toTable(tableEnv, 'a, 'b, 'c)
+    input
+      .groupBy('b)
+      // must fail. UDAGG does not accept String type
+      .select(myWeightedAvg('c, 'a))
+  }
+
+  @Test(expected = classOf[ValidationException])
+  @throws[Exception]
+  def testGroupingNestedUdAgg() {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+
+    val myWeightedAvg = new WeightedAvgWithMergeAndReset
+
+    val input = CollectionDataSets.get3TupleDataSet(env).toTable(tableEnv, 'a, 'b, 'c)
+    input
+      .groupBy('c)
+      // must fail. UDAGG does not accept String type
+      .select(myWeightedAvg(myWeightedAvg('b, 'a), 'a))
+  }
+
   @Test(expected = classOf[ValidationException])
   @throws[Exception]
   def testAggregationOnNonExistingFieldJava() {
@@ -136,4 +181,58 @@ class AggregationsValidationTest {
       .select("c")
   }
 
+  @Test(expected = classOf[ValidationException])
+  @throws[Exception]
+  def testUnknownUdAggJava() {
+    val env= ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+    val input = CollectionDataSets.get3TupleDataSet(env).toTable(tableEnv, 'a, 'b, 'c)
+    input
+      // must fail. unknown is not known
+      .select("unknown(c)")
+  }
+
+  @Test(expected = classOf[ValidationException])
+  @throws[Exception]
+  def testGroupingUnknownUdAggJava() {
+    val env= ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+    val input = CollectionDataSets.get3TupleDataSet(env).toTable(tableEnv, 'a, 'b, 'c)
+    input
+      .groupBy("a, b")
+      // must fail. unknown is not known
+      .select("unknown(c)")
+  }
+
+  @Test(expected = classOf[ValidationException])
+  @throws[Exception]
+  def testInvalidUdAggArgsJava() {
+    val env= ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+
+    val myWeightedAvg = new WeightedAvgWithMergeAndReset
+    tableEnv.registerFunction("myWeightedAvg", myWeightedAvg)
+
+    val input = CollectionDataSets.get3TupleDataSet(env).toTable(tableEnv, 'a, 'b, 'c)
+    input
+      // must fail. UDAGG does not accept String type
+      .select("myWeightedAvg(c, a)")
+  }
+
+  @Test(expected = classOf[ValidationException])
+  @throws[Exception]
+  def testGroupingInvalidUdAggArgsJava() {
+    val env= ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env)
+
+    val myWeightedAvg = new WeightedAvgWithMergeAndReset
+    tableEnv.registerFunction("myWeightedAvg", myWeightedAvg)
+
+    val input = CollectionDataSets.get3TupleDataSet(env).toTable(tableEnv, 'a, 'b, 'c)
+    input
+      .groupBy("b")
+      // must fail. UDAGG does not accept String type
+      .select("myWeightedAvg(c, a)")
+  }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
index 578a6a84d123a..cef2665692da7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
@@ -18,7 +18,8 @@
 package org.apache.flink.table.api.scala.stream.sql
 
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.TableException
+import org.apache.flink.table.api.{TableException, ValidationException}
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.plan.logical.{EventTimeTumblingGroupWindow, ProcessingTimeSessionGroupWindow, ProcessingTimeSlidingGroupWindow}
 import org.apache.flink.table.utils.TableTestUtil._
@@ -33,9 +34,9 @@ class WindowAggregateTest extends TableTestBase {
   def testNonPartitionedProcessingTimeBoundedWindow() = {
 
     val sqlQuery = "SELECT a, Count(c) OVER (ORDER BY procTime()" +
-      "RANGE BETWEEN INTERVAL '10' SECOND PRECEDING AND CURRENT ROW) AS countA " +
-      "FROM MyTable"
-      val expected =
+      "RANGE BETWEEN INTERVAL '10' SECOND PRECEDING AND CURRENT ROW) AS countA FROM MyTable"
+
+    val expected =
       unaryNode(
         "DataStreamCalc",
         unaryNode(
@@ -84,10 +85,11 @@ class WindowAggregateTest extends TableTestBase {
 
   @Test
   def testTumbleFunction() = {
+    streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
 
     val sql =
       "SELECT " +
-        "  COUNT(*), " +
+        "  COUNT(*), weightedAvg(c, a) AS wAvg, " +
         "  TUMBLE_START(rowtime(), INTERVAL '15' MINUTE), " +
         "  TUMBLE_END(rowtime(), INTERVAL '15' MINUTE)" +
         "FROM MyTable " +
@@ -97,23 +99,25 @@ class WindowAggregateTest extends TableTestBase {
         "DataStreamCalc",
         unaryNode(
           "DataStreamAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "1970-01-01 00:00:00 AS $f0")
-          ),
+          streamTableNode(0),
           term("window", EventTimeTumblingGroupWindow('w$, 'rowtime, 900000.millis)),
-          term("select", "COUNT(*) AS EXPR$0, start('w$) AS w$start, end('w$) AS w$end")
+          term("select",
+            "COUNT(*) AS EXPR$0, " +
+              "weightedAvg(c, a) AS wAvg, " +
+              "start('w$) AS w$start, " +
+              "end('w$) AS w$end")
         ),
-        term("select", "EXPR$0, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
+        term("select", "EXPR$0, wAvg, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
       )
     streamUtil.verifySql(sql, expected)
   }
 
   @Test
   def testHoppingFunction() = {
+    streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
+
     val sql =
-      "SELECT COUNT(*), " +
+      "SELECT COUNT(*), weightedAvg(c, a) AS wAvg, " +
         "  HOP_START(proctime(), INTERVAL '15' MINUTE, INTERVAL '1' HOUR), " +
         "  HOP_END(proctime(), INTERVAL '15' MINUTE, INTERVAL '1' HOUR) " +
         "FROM MyTable " +
@@ -123,25 +127,27 @@ class WindowAggregateTest extends TableTestBase {
         "DataStreamCalc",
         unaryNode(
           "DataStreamAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "1970-01-01 00:00:00 AS $f0")
-          ),
+          streamTableNode(0),
           term("window", ProcessingTimeSlidingGroupWindow('w$,
             3600000.millis, 900000.millis)),
-          term("select", "COUNT(*) AS EXPR$0, start('w$) AS w$start, end('w$) AS w$end")
+          term("select",
+            "COUNT(*) AS EXPR$0, " +
+              "weightedAvg(c, a) AS wAvg, " +
+              "start('w$) AS w$start, " +
+              "end('w$) AS w$end")
         ),
-        term("select", "EXPR$0, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
+        term("select", "EXPR$0, wAvg, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
       )
     streamUtil.verifySql(sql, expected)
   }
 
   @Test
   def testSessionFunction() = {
+    streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
+
     val sql =
       "SELECT " +
-        "  COUNT(*), " +
+        "  COUNT(*), weightedAvg(c, a) AS wAvg, " +
         "  SESSION_START(proctime(), INTERVAL '15' MINUTE), " +
         "  SESSION_END(proctime(), INTERVAL '15' MINUTE) " +
         "FROM MyTable " +
@@ -151,15 +157,15 @@ class WindowAggregateTest extends TableTestBase {
         "DataStreamCalc",
         unaryNode(
           "DataStreamAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "1970-01-01 00:00:00 AS $f0")
-          ),
+          streamTableNode(0),
           term("window", ProcessingTimeSessionGroupWindow('w$, 900000.millis)),
-          term("select", "COUNT(*) AS EXPR$0, start('w$) AS w$start, end('w$) AS w$end")
+          term("select",
+            "COUNT(*) AS EXPR$0, " +
+              "weightedAvg(c, a) AS wAvg, " +
+              "start('w$) AS w$start, " +
+              "end('w$) AS w$end")
         ),
-        term("select", "EXPR$0, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
+        term("select", "EXPR$0, wAvg, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
       )
     streamUtil.verifySql(sql, expected)
   }
@@ -215,6 +221,18 @@ class WindowAggregateTest extends TableTestBase {
     streamUtil.verifySql(sql, expected)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testWindowUdAggInvalidArgs(): Unit = {
+    streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
+
+    val sqlQuery =
+      "SELECT SUM(a) AS sumA, weightedAvg(a, b) AS wAvg " +
+        "FROM MyTable " +
+        "GROUP BY TUMBLE(proctime(), INTERVAL '2' HOUR, TIME '10:00:00')"
+
+    streamUtil.verifySql(sqlQuery, "n/a")
+  }
+
   @Test
   def testUnboundPartitionedProcessingWindowWithRange() = {
     val sql = "SELECT " +
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
index 9f366a8434f5f..36517498d7817 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
@@ -25,9 +25,11 @@ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.apache.flink.streaming.api.watermark.Watermark
 import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
 import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge}
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.stream.table.AggregationsITCase.TimestampAndWatermarkWithOffset
 import org.apache.flink.table.api.scala.stream.utils.StreamITCase
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
 import org.apache.flink.types.Row
 import org.junit.Assert._
 import org.junit.Test
@@ -50,23 +52,28 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
   @Test
   def testProcessingTimeSlidingGroupWindowOverCount(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setParallelism(1)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.testResults = mutable.MutableList()
 
     val stream = env.fromCollection(data)
     val table = stream.toTable(tEnv, 'long, 'int, 'string)
 
+    val countFun = new CountAggFunction
+    val weightAvgFun = new WeightedAvg
+
     val windowedTable = table
       .window(Slide over 2.rows every 1.rows as 'w)
       .groupBy('w, 'string)
-      .select('string, 'int.count, 'int.avg)
+      .select('string, countFun('int), 'int.avg,
+              weightAvgFun('long, 'int), weightAvgFun('int, 'int))
 
     val results = windowedTable.toDataStream[Row]
     results.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = Seq("Hello world,1,3", "Hello world,2,3", "Hello,1,2", "Hello,2,2", "Hi,1,1")
+    val expected = Seq("Hello world,1,3,8,3", "Hello world,2,3,12,3", "Hello,1,2,2,2",
+                       "Hello,2,2,3,2", "Hi,1,1,1,1")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -89,6 +96,9 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.testResults = mutable.MutableList()
 
+    val countFun = new CountAggFunction
+    val weightAvgFun = new WeightedAvgWithMerge
+
     val stream = env
       .fromCollection(sessionWindowTestdata)
       .assignTimestampsAndWatermarks(new TimestampAndWatermarkWithOffset(10L))
@@ -97,36 +107,40 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
     val windowedTable = table
       .window(Session withGap 5.milli on 'rowtime as 'w)
       .groupBy('w, 'string)
-      .select('string, 'int.count, 'int.sum)
+      .select('string, countFun('int), 'int.avg,
+              weightAvgFun('long, 'int), weightAvgFun('int, 'int))
 
     val results = windowedTable.toDataStream[Row]
     results.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = Seq("Hello World,1,9", "Hello,1,16", "Hello,4,15")
+    val expected = Seq("Hello World,1,9,9,9", "Hello,1,16,16,16", "Hello,4,3,5,5")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
   @Test
   def testAllProcessingTimeTumblingGroupWindowOverCount(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setParallelism(1)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.testResults = mutable.MutableList()
 
     val stream = env.fromCollection(data)
     val table = stream.toTable(tEnv, 'long, 'int, 'string)
+    val countFun = new CountAggFunction
+    val weightAvgFun = new WeightedAvg
 
     val windowedTable = table
       .window(Tumble over 2.rows as 'w)
       .groupBy('w)
-      .select('int.count)
+      .select(countFun('string), 'int.avg,
+              weightAvgFun('long, 'int), weightAvgFun('int, 'int))
 
     val results = windowedTable.toDataStream[Row]
     results.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = Seq("2", "2")
+    val expected = Seq("2,1,1,1", "2,2,6,2")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -141,21 +155,24 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampAndWatermarkWithOffset(0L))
     val table = stream.toTable(tEnv, 'long, 'int, 'string)
+    val countFun = new CountAggFunction
+    val weightAvgFun = new WeightedAvg
 
     val windowedTable = table
       .window(Tumble over 5.milli on 'rowtime as 'w)
       .groupBy('w, 'string)
-      .select('string, 'int.count, 'int.avg, 'int.min, 'int.max, 'int.sum, 'w.start, 'w.end)
+      .select('string, countFun('string), 'int.avg, weightAvgFun('long, 'int),
+              weightAvgFun('int, 'int), 'int.min, 'int.max, 'int.sum, 'w.start, 'w.end)
 
     val results = windowedTable.toDataStream[Row]
     results.addSink(new StreamITCase.StringSink)
     env.execute()
 
     val expected = Seq(
-      "Hello world,1,3,3,3,3,1970-01-01 00:00:00.005,1970-01-01 00:00:00.01",
-      "Hello world,1,3,3,3,3,1970-01-01 00:00:00.015,1970-01-01 00:00:00.02",
-      "Hello,2,2,2,2,4,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
-      "Hi,1,1,1,1,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
+      "Hello world,1,3,8,3,3,3,3,1970-01-01 00:00:00.005,1970-01-01 00:00:00.01",
+      "Hello world,1,3,16,3,3,3,3,1970-01-01 00:00:00.015,1970-01-01 00:00:00.02",
+      "Hello,2,2,3,2,2,2,4,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
+      "Hi,1,1,1,1,1,1,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index 402ebbeb781ab..084ee1457e559 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -19,8 +19,8 @@
 package org.apache.flink.table.api.scala.stream.table
 
 import org.apache.flink.api.scala._
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table._
 import org.apache.flink.table.api.{TableException, ValidationException}
 import org.apache.flink.table.expressions.{RowtimeAttribute, WindowReference}
 import org.apache.flink.table.plan.logical._
@@ -112,6 +112,18 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testTumbleUdAggWithInvalidArgs(): Unit = {
+    val util = streamTestUtil()
+    val weightedAvg = new WeightedAvgWithMerge
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    table
+      .window(Tumble over 2.hours on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, weightedAvg('string, 'int)) // invalid UDAGG args
+  }
+
   @Test(expected = classOf[ValidationException])
   def testInvalidSlidingSize(): Unit = {
     val util = streamTestUtil()
@@ -134,6 +146,18 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testSlideUdAggWithInvalidArgs(): Unit = {
+    val util = streamTestUtil()
+    val weightedAvg = new WeightedAvgWithMerge
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    table
+      .window(Slide over 2.hours every 30.minutes on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, weightedAvg('string, 'int)) // invalid UDAGG args
+  }
+
   @Test(expected = classOf[ValidationException])
   def testInvalidSessionGap(): Unit = {
     val util = streamTestUtil()
@@ -167,6 +191,18 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testSessionUdAggWithInvalidArgs(): Unit = {
+    val util = streamTestUtil()
+    val weightedAvg = new WeightedAvgWithMerge
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    table
+      .window(Session withGap 2.hours on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, weightedAvg('string, 'int)) // invalid UDAGG args
+  }
+
   @Test
   def testMultiWindow(): Unit = {
     val util = streamTestUtil()
@@ -324,6 +360,34 @@ class GroupWindowTest extends TableTestBase {
     util.verifyTable(windowedTable, expected)
   }
 
+  @Test
+  def testEventTimeTumblingGroupWindowWithUdAgg(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val weightedAvg = new WeightedAvgWithMerge
+
+    val windowedTable = table
+      .window(Tumble over 5.milli on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, weightedAvg('long, 'int))
+
+    val expected = unaryNode(
+      "DataStreamAggregate",
+      streamTableNode(0),
+      term("groupBy", "string"),
+      term(
+        "window",
+        EventTimeTumblingGroupWindow(
+          WindowReference("w"),
+          RowtimeAttribute(),
+          5.milli)),
+      term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
   @Test
   def testProcessingTimeSlidingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
@@ -437,6 +501,33 @@ class GroupWindowTest extends TableTestBase {
     util.verifyTable(windowedTable, expected)
   }
 
+  @Test
+  def testEventTimeSlidingGroupWindowWithUdAgg(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val weightedAvg = new WeightedAvgWithMerge
+
+    val windowedTable = table
+      .window(Slide over 8.milli every 10.milli on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, weightedAvg('long, 'int))
+
+    val expected = unaryNode(
+      "DataStreamAggregate",
+      streamTableNode(0),
+      term("groupBy", "string"),
+      term(
+        "window",
+        EventTimeSlidingGroupWindow(
+          WindowReference("w"),
+          RowtimeAttribute(), 8.milli, 10.milli)),
+      term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
   @Test
   def testEventTimeSessionGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
@@ -466,6 +557,33 @@ class GroupWindowTest extends TableTestBase {
     util.verifyTable(windowedTable, expected)
   }
 
+  @Test
+  def testEventTimeSessionGroupWindowWithUdAgg(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+    val weightedAvg = new WeightedAvgWithMerge
+
+    val windowedTable = table
+      .window(Session withGap 7.milli on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, weightedAvg('long, 'int))
+
+    val expected = unaryNode(
+      "DataStreamAggregate",
+      streamTableNode(0),
+      term("groupBy", "string"),
+      term(
+        "window",
+        EventTimeSessionGroupWindow(
+          WindowReference("w"),
+          RowtimeAttribute(), 7.milli)),
+      term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
   @Test
   def testAllProcessingTimeTumblingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
index 8464ba166119a..4c0fea7c3ddb6 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
@@ -25,9 +25,11 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceCont
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.apache.flink.streaming.api.watermark.Watermark
 import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvg
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.stream.table.OverWindowITCase.RowTimeSourceFunction
 import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamingWithStateTestBase}
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
 import org.apache.flink.types.Row
 import org.junit.Assert._
 import org.junit.Test
@@ -57,20 +59,22 @@ class OverWindowITCase extends StreamingWithStateTestBase {
     StreamITCase.clear
     val stream = env.fromCollection(data)
     val table = stream.toTable(tEnv, 'a, 'b, 'c)
+    val countFun = new CountAggFunction
+    val weightAvgFun = new WeightedAvg
 
     val windowedTable = table
       .window(
         Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_ROW as 'w)
-      .select('c, 'b.count over 'w as 'mycount)
-      .select('c, 'mycount)
+      .select('c, countFun('b) over 'w as 'mycount, weightAvgFun('a, 'b) over 'w as 'wAvg)
+      .select('c, 'mycount, 'wAvg)
 
     val results = windowedTable.toDataStream[Row]
     results.addSink(new StreamITCase.StringSink)
     env.execute()
 
     val expected = Seq(
-      "Hello World,1", "Hello World,2", "Hello World,3",
-      "Hello,1", "Hello,2", "Hello,3", "Hello,4", "Hello,5", "Hello,6")
+      "Hello World,1,7", "Hello World,2,7", "Hello World,3,14",
+      "Hello,1,1", "Hello,2,1", "Hello,3,2", "Hello,4,3", "Hello,5,3", "Hello,6,4")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -104,6 +108,8 @@ class OverWindowITCase extends StreamingWithStateTestBase {
     val table = env
       .addSource(new RowTimeSourceFunction[(Int, Long, String)](data))
       .toTable(tEnv).as('a, 'b, 'c)
+    val countFun = new CountAggFunction
+    val weightAvgFun = new WeightedAvg
 
     val windowedTable = table
       .window(Over partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE following
@@ -111,29 +117,30 @@ class OverWindowITCase extends StreamingWithStateTestBase {
       .select(
         'a, 'b, 'c,
         'b.sum over 'w,
-        'b.count over 'w,
+        countFun('b) over 'w,
         'b.avg over 'w,
         'b.max over 'w,
-        'b.min over 'w)
+        'b.min over 'w,
+        weightAvgFun('b, 'a) over 'w)
 
     val result = windowedTable.toDataStream[Row]
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
     val expected = mutable.MutableList(
-      "1,1,Hello,6,3,2,3,1",
-      "1,2,Hello,6,3,2,3,1",
-      "1,3,Hello world,6,3,2,3,1",
-      "1,1,Hi,7,4,1,3,1",
-      "2,1,Hello,1,1,1,1,1",
-      "2,2,Hello world,6,3,2,3,1",
-      "2,3,Hello world,6,3,2,3,1",
-      "1,4,Hello world,11,5,2,4,1",
-      "1,5,Hello world,29,8,3,7,1",
-      "1,6,Hello world,29,8,3,7,1",
-      "1,7,Hello world,29,8,3,7,1",
-      "2,4,Hello world,15,5,3,5,1",
-      "2,5,Hello world,15,5,3,5,1"
+      "1,1,Hello,6,3,2,3,1,2",
+      "1,2,Hello,6,3,2,3,1,2",
+      "1,3,Hello world,6,3,2,3,1,2",
+      "1,1,Hi,7,4,1,3,1,1",
+      "2,1,Hello,1,1,1,1,1,1",
+      "2,2,Hello world,6,3,2,3,1,2",
+      "2,3,Hello world,6,3,2,3,1,2",
+      "1,4,Hello world,11,5,2,4,1,2",
+      "1,5,Hello world,29,8,3,7,1,3",
+      "1,6,Hello world,29,8,3,7,1,3",
+      "1,7,Hello world,29,8,3,7,1,3",
+      "2,4,Hello world,15,5,3,5,1,3",
+      "2,5,Hello world,15,5,3,5,1,3"
     )
 
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
index 26f4eadafe4dc..7dea521462725 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
@@ -18,6 +18,7 @@
 package org.apache.flink.table.api.scala.stream.table
 
 import org.apache.flink.api.scala._
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.{WeightedAvgWithRetract}
 import org.apache.flink.table.api.{Table, ValidationException}
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.utils.TableTestUtil._
@@ -94,11 +95,23 @@ class OverWindowTest extends TableTestBase {
     streamUtil.tEnv.optimize(result.getRelNode)
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testUdAggWithInvalidArgs(): Unit = {
+    val weightedAvg = new WeightedAvgWithRetract
+
+    val result = table
+      .window(Over orderBy 'rowtime preceding 1.minutes as 'w)
+      .select('c, weightedAvg('b, 'a) over 'w)
+    streamUtil.tEnv.optimize(result.getRelNode)
+  }
+
   @Test
   def testProcTimeBoundedPartitionedRowsOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
-      .window(Over partitionBy 'c orderBy 'proctime preceding 2.rows following CURRENT_ROW as 'w)
-      .select('c, 'b.count over 'w)
+      .window(Over partitionBy 'b orderBy 'proctime preceding 2.rows following CURRENT_ROW as 'w)
+      .select('c, weightedAvg('c, 'a) over 'w)
 
     val expected =
       unaryNode(
@@ -108,12 +121,12 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "b", "c", "PROCTIME() AS $2")
+            term("select", "a", "b", "c", "PROCTIME() AS $3")
           ),
-          term("partitionBy", "c"),
+          term("partitionBy", "b"),
           term("orderBy", "PROCTIME"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "b", "c", "PROCTIME", "COUNT(b) AS w0$o0")
+          term("select", "a", "b", "c", "PROCTIME", "WeightedAvgWithRetract(c, a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS _c1")
       )
@@ -122,10 +135,12 @@ class OverWindowTest extends TableTestBase {
 
   @Test
   def testProcTimeBoundedPartitionedRangeOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
       .window(
         Over partitionBy 'a orderBy 'proctime preceding 2.hours following CURRENT_RANGE as 'w)
-      .select('a, 'c.avg over 'w as 'myAvg)
+      .select('a, weightedAvg('c, 'a) over 'w as 'myAvg)
 
     val expected =
       unaryNode(
@@ -145,7 +160,7 @@ class OverWindowTest extends TableTestBase {
             "a",
             "c",
             "PROCTIME",
-            "AVG(c) AS w0$o0"
+            "WeightedAvgWithRetract(c, a) AS w0$o0"
           )
         ),
         term("select", "a", "w0$o0 AS myAvg")
@@ -208,10 +223,12 @@ class OverWindowTest extends TableTestBase {
 
   @Test
   def testProcTimeUnboundedPartitionedRangeOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
       .window(Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_RANGE following
          CURRENT_RANGE as 'w)
-      .select('a, 'c, 'a.count over 'w, 'a.sum over 'w)
+      .select('a, 'c, 'a.count over 'w, weightedAvg('c, 'a) over 'w)
 
     val expected =
       unaryNode(
@@ -232,7 +249,7 @@ class OverWindowTest extends TableTestBase {
             "c",
             "PROCTIME",
             "COUNT(a) AS w0$o0",
-            "SUM(a) AS w0$o1"
+            "WeightedAvgWithRetract(c, a) AS w0$o1"
           )
         ),
         term(
@@ -248,10 +265,12 @@ class OverWindowTest extends TableTestBase {
 
   @Test
   def testProcTimeUnboundedPartitionedRowsOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
       .window(
         Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_ROW following CURRENT_ROW as 'w)
-      .select('c, 'a.count over 'w)
+      .select('c, 'a.count over 'w, weightedAvg('c, 'a) over 'w)
 
     val expected =
       unaryNode(
@@ -266,9 +285,11 @@ class OverWindowTest extends TableTestBase {
           term("partitionBy", "c"),
           term("orderBy", "PROCTIME"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "PROCTIME",
+               "COUNT(a) AS w0$o0",
+               "WeightedAvgWithRetract(c, a) AS w0$o1")
         ),
-        term("select", "c", "w0$o0 AS _c1")
+        term("select", "c", "w0$o0 AS _c1", "w0$o1 AS _c2")
       )
 
     streamUtil.verifyTable(result, expected)
@@ -342,10 +363,12 @@ class OverWindowTest extends TableTestBase {
 
   @Test
   def testRowTimeBoundedPartitionedRowsOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
       .window(
-        Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
-      .select('c, 'b.count over 'w)
+        Over partitionBy 'b orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
+      .select('c, 'b.count over 'w, weightedAvg('c, 'a) over 'w as 'wAvg)
 
     val expected =
       unaryNode(
@@ -355,14 +378,16 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "b", "c", "ROWTIME() AS $2")
+            term("select", "a", "b", "c", "ROWTIME() AS $3")
           ),
-          term("partitionBy", "c"),
+          term("partitionBy", "b"),
           term("orderBy", "ROWTIME"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "b", "c", "ROWTIME", "COUNT(b) AS w0$o0")
+          term("select", "a", "b", "c", "ROWTIME",
+               "COUNT(b) AS w0$o0",
+               "WeightedAvgWithRetract(c, a) AS w0$o1")
         ),
-        term("select", "c", "w0$o0 AS _c1")
+        term("select", "c", "w0$o0 AS _c1", "w0$o1 AS wAvg")
       )
 
     streamUtil.verifyTable(result, expected)
@@ -370,10 +395,12 @@ class OverWindowTest extends TableTestBase {
 
   @Test
   def testRowTimeBoundedPartitionedRangeOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
       .window(
         Over partitionBy 'a orderBy 'rowtime preceding 2.hours following CURRENT_RANGE as 'w)
-      .select('a, 'c.avg over 'w)
+      .select('a, 'c.avg over 'w, weightedAvg('c, 'a) over 'w as 'wAvg)
 
     val expected =
       unaryNode(
@@ -393,10 +420,11 @@ class OverWindowTest extends TableTestBase {
             "a",
             "c",
             "ROWTIME",
-            "AVG(c) AS w0$o0"
+            "AVG(c) AS w0$o0",
+            "WeightedAvgWithRetract(c, a) AS w0$o1"
           )
         ),
-        term("select", "a", "w0$o0 AS _c1")
+        term("select", "a", "w0$o0 AS _c1", "w0$o1 AS wAvg")
       )
 
     streamUtil.verifyTable(result, expected)
@@ -456,10 +484,12 @@ class OverWindowTest extends TableTestBase {
 
   @Test
   def testRowTimeUnboundedPartitionedRangeOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
       .window(Over partitionBy 'c orderBy 'rowtime preceding UNBOUNDED_RANGE following
          CURRENT_RANGE as 'w)
-      .select('a, 'c, 'a.count over 'w, 'a.sum over 'w)
+      .select('a, 'c, 'a.count over 'w, weightedAvg('c, 'a) over 'w as 'wAvg)
 
     val expected =
       unaryNode(
@@ -480,7 +510,7 @@ class OverWindowTest extends TableTestBase {
             "c",
             "ROWTIME",
             "COUNT(a) AS w0$o0",
-            "SUM(a) AS w0$o1"
+            "WeightedAvgWithRetract(c, a) AS w0$o1"
           )
         ),
         term(
@@ -488,7 +518,7 @@ class OverWindowTest extends TableTestBase {
           "a",
           "c",
           "w0$o0 AS _c2",
-          "w0$o1 AS _c3"
+          "w0$o1 AS wAvg"
         )
       )
 
@@ -497,10 +527,12 @@ class OverWindowTest extends TableTestBase {
 
   @Test
   def testRowTimeUnboundedPartitionedRowsOver() = {
+    val weightedAvg = new WeightedAvgWithRetract
+
     val result = table
       .window(Over partitionBy 'c orderBy 'rowtime preceding UNBOUNDED_ROW following
          CURRENT_ROW as 'w)
-      .select('c, 'a.count over 'w)
+      .select('c, 'a.count over 'w, weightedAvg('c, 'a) over 'w as 'wAvg)
 
     val expected =
       unaryNode(
@@ -515,9 +547,11 @@ class OverWindowTest extends TableTestBase {
           term("partitionBy", "c"),
           term("orderBy", "ROWTIME"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "ROWTIME",
+               "COUNT(a) AS w0$o0",
+               "WeightedAvgWithRetract(c, a) AS w0$o1")
         ),
-        term("select", "c", "w0$o0 AS _c1")
+        term("select", "c", "w0$o0 AS _c1", "w0$o1 AS wAvg")
       )
 
     streamUtil.verifyTable(result, expected)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala
new file mode 100644
index 0000000000000..d314c9ae6b022
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.api.scala.stream.table.stringexpr
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvg
+import org.apache.flink.table.api.java.{Slide => JSlide}
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
+import org.apache.flink.table.utils.TableTestBase
+import org.junit.{Assert, Test}
+
+
+class GroupWindowStringExpressionTest extends TableTestBase {
+
+  @Test
+  def testJavaScalaTableAPIEquality(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Int, Long, String)]('int, 'long, 'string)
+
+    val myCountFun = new CountAggFunction
+    util.tEnv.registerFunction("myCountFun", myCountFun)
+    val weightAvgFun = new WeightedAvg
+    util.tEnv.registerFunction("weightAvgFun", weightAvgFun)
+
+    // Expression / Scala API
+    val resScala = t
+      .window(Slide over 4.rows every 2.rows as 'w)
+      .groupBy('w, 'string)
+      .select(
+        'string,
+        myCountFun('string),
+        'int.sum,
+        weightAvgFun('long, 'int),
+        weightAvgFun('int, 'int) * 2)
+
+    // String / Java API
+    val resJava = t
+      .window(JSlide.over("4.rows").every("2.rows").as("w"))
+      .groupBy("w, string")
+      .select(
+        "string, " +
+          "myCountFun(string), " +
+          "int.sum, " +
+          "weightAvgFun(long, int), " +
+          "weightAvgFun(int, int) * 2")
+
+    Assert.assertEquals("Logical Plans do not match", resJava.logicalPlan, resJava.logicalPlan)
+  }
+}

From 774c9ffc4617007f4f9abafc93ceb901293452aa Mon Sep 17 00:00:00 2001
From: Haohui Mai <wheat9@apache.org>
Date: Wed, 29 Mar 2017 13:31:38 -0700
Subject: [PATCH 143/253] [FLINK-5998] Un-fat Hadoop from Flink fat jar.

This closes #3604
---
 flink-dist/pom.xml                     | 21 ++++++++++++++
 flink-dist/src/main/assemblies/lib.xml | 39 ++++++++++++++++++++++++++
 flink-dist/src/main/assemblies/opt.xml |  7 +++++
 3 files changed, 67 insertions(+)
 create mode 100644 flink-dist/src/main/assemblies/lib.xml

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 507e78d31716e..ff87d537dda67 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -246,6 +246,13 @@ under the License.
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
 		<!-- end optional Flink libraries -->
 	</dependencies>
 
@@ -404,6 +411,20 @@ under the License.
 							<appendAssemblyId>false</appendAssemblyId>
 						</configuration>
 					</execution>
+					<execution>
+						<id>lib</id>
+						<phase>package</phase>
+						<goals>
+							<goal>single</goal>
+						</goals>
+						<configuration>
+							<descriptors>
+								<descriptor>src/main/assemblies/lib.xml</descriptor>
+							</descriptors>
+							<finalName>flink-${project.version}-bin</finalName>
+							<appendAssemblyId>false</appendAssemblyId>
+						</configuration>
+					</execution>
 				</executions>
 			</plugin>
 
diff --git a/flink-dist/src/main/assemblies/lib.xml b/flink-dist/src/main/assemblies/lib.xml
new file mode 100644
index 0000000000000..a1839103930b7
--- /dev/null
+++ b/flink-dist/src/main/assemblies/lib.xml
@@ -0,0 +1,39 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly
+		xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+		xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+	<id>lib</id>
+	<formats>
+		<format>dir</format>
+	</formats>
+
+	<includeBaseDirectory>true</includeBaseDirectory>
+	<baseDirectory>flink-${project.version}</baseDirectory>
+
+	<files>
+		<file>
+			<source>../flink-shaded-hadoop/flink-shaded-hadoop2/target/flink-shaded-hadoop2-${project.version}.jar</source>
+			<outputDirectory>lib/</outputDirectory>
+			<destName>flink-shaded-hadoop2-${project.version}.jar</destName>
+			<fileMode>0644</fileMode>
+		</file>
+	</files>
+</assembly>
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 0282315fc9854..95218d73997ce 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -103,5 +103,12 @@
 			<destName>flink-metrics-statsd-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
+
+		<file>
+			<source>../flink-shaded-hadoop/flink-shaded-hadoop2/target/flink-shaded-hadoop2-${project.version}.jar</source>
+			<outputDirectory>opt/</outputDirectory>
+			<destName>flink-shaded-hadoop2-${project.version}.jar</destName>
+			<fileMode>0644</fileMode>
+		</file>
 	</files>
 </assembly>

From eccb51915f6397ca8500cb1218d9b90ec10b7464 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Tue, 21 Mar 2017 15:10:15 +0100
Subject: [PATCH 144/253] [FLINK-6136] Separate EmbeddedHaServices and
 StandaloneHaServices

This PR introduces a standalone high availability services implementation which can be used
in a distributed setting with no HA guarantees. Additionally, it introduces a common base
class which is also used by the EmbeddedHaServices. This base class instantiates the
standalone variants of the checkpoint recovery factory, submitted job graphs store, running
jobs registry and blob store.

The StandaloneHaServices are instantiated with a fixed address for the Job- and
ResourceManager. This address and the HighAvailability.DEFAULT_LEADER_ID is returned by
the corresponding LeaderRetrievalServices when being started.

This closes #3622.
---
 .../flink/client/program/ClusterClient.java   |   8 +-
 .../RemoteExecutorHostnameResolutionTest.java |   3 +-
 .../flink/client/program/ClientTest.java      |  41 +++--
 ...etrievalServiceHostnameResolutionTest.java |  11 +-
 .../apache/flink/storm/api/FlinkClient.java   |  16 +-
 .../flink/util}/ConfigurationException.java   |   6 +-
 .../MesosApplicationMasterRunner.java         |   7 +-
 .../handlers/HandlerRedirectUtils.java        |   5 +-
 .../apache/flink/runtime/blob/BlobUtils.java  |   4 +-
 .../runtime/client/JobListeningContext.java   |   2 +-
 .../EmbeddedNonHaServices.java                |  70 --------
 .../HighAvailabilityServicesUtils.java        |  90 ++++++++--
 .../nonha/AbstractNonHaServices.java          | 117 ++++--------
 .../nonha/embedded/EmbeddedHaServices.java    | 126 +++++++++++++
 .../{ => embedded}/EmbeddedLeaderService.java |   5 +-
 .../SingleLeaderElectionService.java          |   2 +-
 .../standalone/StandaloneHaServices.java}     |  52 ++++--
 .../StandaloneRunningJobsRegistry.java}       |   4 +-
 .../ZooKeeperHaServices.java}                 |  10 +-
 .../ZooKeeperRunningJobsRegistry.java}        |   7 +-
 .../flink/runtime/jobmaster/JobMaster.java    |   9 +-
 .../runtime/minicluster/MiniCluster.java      |   4 +-
 .../runtime/query/QueryableStateClient.java   |   2 +-
 .../resourcemanager/ResourceManager.java      |   2 +
 .../ResourceManagerConfiguration.java         |   2 +-
 ...ceManagerRuntimeServicesConfiguration.java |   2 +-
 .../slotmanager/SlotManagerConfiguration.java |   2 +-
 .../AkkaRpcServiceUtils.java}                 |  59 +++++--
 .../runtime/taskexecutor/TaskExecutor.java    |   7 +-
 .../taskexecutor/TaskManagerRunner.java       |   4 +-
 .../runtime/util/LeaderRetrievalUtils.java    |  17 +-
 .../flink/runtime/util/StandaloneUtils.java   |  74 +++-----
 .../apache/flink/runtime/akka/AkkaUtils.scala |  19 +-
 .../flink/runtime/jobmanager/JobManager.scala | 148 ++--------------
 .../minicluster/LocalFlinkMiniCluster.scala   |  15 +-
 .../runtime/taskmanager/TaskManager.scala     |  45 +----
 .../TestingHighAvailabilityServices.java      |   4 +-
 .../embedded/EmbeddedHaServicesTest.java      | 167 ++++++++++++++++++
 .../SingleLeaderElectionServiceTest.java      |   2 +-
 .../standalone/StandaloneHaServicesTest.java  | 104 +++++++++++
 .../ZooKeeperRegistryTest.java                |   7 +-
 .../JobManagerProcessReapingTest.java         |  16 +-
 .../runtime/jobmanager/JobSubmitTest.java     |   2 +-
 .../ZooKeeperLeaderElectionTest.java          |   4 +-
 .../ZooKeeperLeaderRetrievalTest.java         |  28 +--
 .../taskexecutor/TaskExecutorTest.java        |   7 +-
 .../TaskManagerProcessReapingTestBase.java    |   3 +-
 .../TaskManagerRegistrationTest.java          |   5 +-
 .../runtime/testutils/JobManagerProcess.java  |  18 +-
 .../flink/runtime/akka/AkkaUtilsTest.scala    |  17 +-
 .../jobmanager/JobManagerConnectionTest.scala |  10 +-
 .../runtime/testingUtils/TestingUtils.scala   |   9 +-
 .../ZooKeeperLeaderElectionITCase.java        |   3 +-
 .../yarn/YARNHighAvailabilityITCase.java      |   4 +-
 .../yarn/YarnApplicationMasterRunner.java     |   3 +-
 .../apache/flink/yarn/YarnClusterClient.java  |   2 +-
 .../YarnFlinkApplicationMasterRunner.java     |   7 +-
 .../flink/yarn/YarnTaskExecutorRunner.java    |   9 +-
 .../YarnHighAvailabilityServices.java         |   6 +-
 .../YarnIntraNonHaMasterServices.java         |   2 +-
 .../YarnPreConfiguredMasterNonHaServices.java |  16 +-
 ...YarnPreConfiguredMasterHaServicesTest.java |  26 ++-
 62 files changed, 897 insertions(+), 581 deletions(-)
 rename {flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions => flink-core/src/main/java/org/apache/flink/util}/ConfigurationException.java (85%)
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServices.java
 rename flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/{ => embedded}/EmbeddedLeaderService.java (98%)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/{ => nonha}/leaderelection/SingleLeaderElectionService.java (99%)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/{NonHaServices.java => nonha/standalone/StandaloneHaServices.java} (61%)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/{NonHaRegistry.java => standalone/StandaloneRunningJobsRegistry.java} (93%)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/{ZookeeperHaServices.java => zookeeper/ZooKeeperHaServices.java} (94%)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/{ZookeeperRegistry.java => zookeeper/ZooKeeperRunningJobsRegistry.java} (92%)
 rename flink-runtime/src/main/java/org/apache/flink/runtime/rpc/{RpcServiceUtils.java => akka/AkkaRpcServiceUtils.java} (76%)
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java
 rename flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/{ => nonha}/leaderelection/SingleLeaderElectionServiceTest.java (99%)
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneHaServicesTest.java
 rename flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/{ => zookeeper}/ZooKeeperRegistryTest.java (88%)

diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
index ab4daa95155c6..0f88f7cf9b4ce 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
@@ -241,7 +241,7 @@ public InetSocketAddress getJobManagerAddress() {
 		try {
 			LeaderConnectionInfo leaderConnectionInfo =
 				LeaderRetrievalUtils.retrieveLeaderConnectionInfo(
-					LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig), timeout);
+					LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true), timeout);
 
 			return AkkaUtils.getInetSockeAddressFromAkkaURL(leaderConnectionInfo.getAddress());
 		} catch (Exception e) {
@@ -464,7 +464,7 @@ public JobSubmissionResult runDetached(JobGraph jobGraph, ClassLoader classLoade
 	public JobExecutionResult retrieveJob(JobID jobID) throws JobExecutionException {
 		final LeaderRetrievalService leaderRetrievalService;
 		try {
-			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig);
+			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true);
 		} catch (Exception e) {
 			throw new JobRetrievalException(jobID, "Could not create the leader retrieval service", e);
 		}
@@ -498,7 +498,7 @@ public JobExecutionResult retrieveJob(JobID jobID) throws JobExecutionException
 	public JobListeningContext connectToJob(JobID jobID) throws JobExecutionException {
 		final LeaderRetrievalService leaderRetrievalService;
 		try {
-			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig);
+			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true);
 		} catch (Exception e) {
 			throw new JobRetrievalException(jobID, "Could not create the leader retrieval service", e);
 		}
@@ -721,7 +721,7 @@ private JobGraph getJobGraph(FlinkPlan optPlan, List<URL> jarFiles, List<URL> cl
 	public ActorGateway getJobManagerGateway() throws Exception {
 		LOG.debug("Looking up JobManager");
 		return LeaderRetrievalUtils.retrieveLeaderGateway(
-			LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig),
+			LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true),
 			actorSystemLoader.get(),
 			lookupTimeout);
 	}
diff --git a/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java b/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java
index 07edb3a9d290b..d8fb3de09aa5c 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java
@@ -23,6 +23,7 @@
 import org.apache.flink.api.java.io.DiscardingOutputFormat;
 import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.TestLogger;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -36,7 +37,7 @@
 import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 
-public class RemoteExecutorHostnameResolutionTest {
+public class RemoteExecutorHostnameResolutionTest extends TestLogger {
 
 	private static final String nonExistingHostname = "foo.bar.com.invalid";
 	private static final int port = 14451;
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
index 75cb0e71a3a79..da297d6041cdf 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
@@ -43,7 +43,7 @@
 import org.apache.flink.runtime.akka.FlinkUntypedActor;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.jobmanager.JobManager;
+import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.util.SerializedThrowable;
 import org.apache.flink.util.NetUtils;
@@ -67,7 +67,6 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-
 /**
  * Simple and maybe stupid test to check the {@link ClusterClient} class.
  */
@@ -129,7 +128,9 @@ public void shutDownActorSystem() {
 	 */
 	@Test
 	public void testDetachedMode() throws Exception{
-		jobManagerSystem.actorOf(Props.create(SuccessReturningActor.class), JobManager.JOB_MANAGER_NAME());
+		jobManagerSystem.actorOf(
+			Props.create(SuccessReturningActor.class),
+			JobMaster.JOB_MANAGER_NAME);
 		ClusterClient out = new StandaloneClusterClient(config);
 		out.setDetached(true);
 
@@ -198,22 +199,18 @@ public void testDetachedMode() throws Exception{
 	 * This test verifies correct job submission messaging logic and plan translation calls.
 	 */
 	@Test
-	public void shouldSubmitToJobClient() {
-		try {
-			jobManagerSystem.actorOf(Props.create(SuccessReturningActor.class), JobManager.JOB_MANAGER_NAME());
+	public void shouldSubmitToJobClient() throws IOException, ProgramInvocationException {
+		jobManagerSystem.actorOf(
+			Props.create(SuccessReturningActor.class),
+			JobMaster.JOB_MANAGER_NAME);
 
-			ClusterClient out = new StandaloneClusterClient(config);
-			out.setDetached(true);
-			JobSubmissionResult result = out.run(program.getPlanWithJars(), 1);
+		ClusterClient out = new StandaloneClusterClient(config);
+		out.setDetached(true);
+		JobSubmissionResult result = out.run(program.getPlanWithJars(), 1);
 
-			assertNotNull(result);
+		assertNotNull(result);
 
-			program.deleteExtractedLibraries();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
+		program.deleteExtractedLibraries();
 	}
 
 	/**
@@ -221,7 +218,9 @@ public void shouldSubmitToJobClient() {
 	 */
 	@Test
 	public void shouldSubmitToJobClientFails() throws IOException {
-		jobManagerSystem.actorOf(Props.create(FailureReturningActor.class), JobManager.JOB_MANAGER_NAME());
+			jobManagerSystem.actorOf(
+				Props.create(FailureReturningActor.class),
+				JobMaster.JOB_MANAGER_NAME);
 
 		ClusterClient out = new StandaloneClusterClient(config);
 		out.setDetached(true);
@@ -245,7 +244,9 @@ public void shouldSubmitToJobClientFails() throws IOException {
 	@Test
 	public void tryLocalExecution() {
 		try {
-			jobManagerSystem.actorOf(Props.create(SuccessReturningActor.class), JobManager.JOB_MANAGER_NAME());
+			jobManagerSystem.actorOf(
+				Props.create(SuccessReturningActor.class),
+				JobMaster.JOB_MANAGER_NAME);
 			
 			PackagedProgram packagedProgramMock = mock(PackagedProgram.class);
 			when(packagedProgramMock.isUsingInteractiveMode()).thenReturn(true);
@@ -276,7 +277,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
 	@Test
 	public void testGetExecutionPlan() {
 		try {
-			jobManagerSystem.actorOf(Props.create(FailureReturningActor.class), JobManager.JOB_MANAGER_NAME());
+			jobManagerSystem.actorOf(
+				Props.create(FailureReturningActor.class),
+				JobMaster.JOB_MANAGER_NAME);
 			
 			PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
 			assertNotNull(prg.getPreviewPlan());
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java
index dd7d8bc7698c2..fc10f65e59ce5 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java
@@ -56,7 +56,7 @@ public void testUnresolvableHostname1() {
 			config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, nonExistingHostname);
 			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 17234);
 
-			LeaderRetrievalUtils.createLeaderRetrievalService(config);
+			LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
 		}
 		catch (Exception e) {
 			System.err.println("Shouldn't throw an exception!");
@@ -69,7 +69,7 @@ public void testUnresolvableHostname1() {
 	 * Tests that the StandaloneLeaderRetrievalService does not resolve host names by default.
 	 */
 	@Test
-	public void testUnresolvableHostname2() {
+	public void testUnresolvableHostname2() throws Exception {
 
 		try {
 			Configuration config = new Configuration();
@@ -78,16 +78,11 @@ public void testUnresolvableHostname2() {
 			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 17234);
 
 			LeaderRetrievalUtils.createLeaderRetrievalService(config, true);
-			fail("This should fail with an IllegalConfigurationException");
+			fail("This should fail with an UnknownHostException");
 		}
 		catch (UnknownHostException e) {
 			// that is what we want!
 		}
-		catch (Exception e) {
-			System.err.println("Wrong exception!");
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
 	}
 
 	private static void checkPreconditions() {
diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java
index c58a8eecfa404..2b7f357ea7daa 100644
--- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java
+++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java
@@ -22,6 +22,9 @@
 import akka.actor.ActorSystem;
 import akka.pattern.Patterns;
 import akka.util.Timeout;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution;
+import org.apache.flink.runtime.jobmaster.JobMaster;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.storm.Config;
 import org.apache.storm.generated.AlreadyAliveException;
 import org.apache.storm.generated.InvalidTopologyException;
@@ -43,12 +46,10 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.client.JobStatusMessage;
 import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.messages.JobManagerMessages.RunningJobsStatus;
 import org.apache.flink.storm.util.StormConfig;
 import org.apache.flink.streaming.api.graph.StreamGraph;
-import org.apache.flink.util.NetUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import scala.Some;
@@ -326,9 +327,14 @@ private ActorRef getJobManager() throws IOException {
 			throw new RuntimeException("Could not start actor system to communicate with JobManager", e);
 		}
 
-		return JobManager.getJobManagerActorRef(AkkaUtils.getAkkaProtocol(configuration),
-				NetUtils.unresolvedHostAndPortToNormalizedString(this.jobManagerHost, this.jobManagerPort),
-				actorSystem, AkkaUtils.getLookupTimeout(configuration));
+		final String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
+			jobManagerHost,
+			jobManagerPort,
+			JobMaster.JOB_MANAGER_NAME,
+			AddressResolution.TRY_ADDRESS_RESOLUTION,
+			configuration);
+
+		return AkkaUtils.getActorRef(jobManagerAkkaUrl, actorSystem, AkkaUtils.getLookupTimeout(configuration));
 	}
 
 	@SuppressWarnings({ "unchecked", "rawtypes" })
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java b/flink-core/src/main/java/org/apache/flink/util/ConfigurationException.java
similarity index 85%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
rename to flink-core/src/main/java/org/apache/flink/util/ConfigurationException.java
index 0007318489d58..4623fa351a16a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
+++ b/flink-core/src/main/java/org/apache/flink/util/ConfigurationException.java
@@ -16,12 +16,12 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.resourcemanager.exceptions;
+package org.apache.flink.util;
 
 /**
- * Exception which occures when creating a configuration object fails.
+ * Exception which occurs when creating a configuration object fails.
  */
-public class ConfigurationException extends Exception {
+public class ConfigurationException extends FlinkException {
 	private static final long serialVersionUID = 3971647332059381556L;
 
 	public ConfigurationException(String message) {
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
index a23c9f6767f2b..0c127457afb18 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
@@ -50,6 +50,7 @@
 import org.apache.flink.runtime.clusterframework.overlays.SSLStoreOverlay;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
+import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.process.ProcessReaper;
 import org.apache.flink.runtime.security.SecurityUtils;
@@ -65,6 +66,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import scala.Option;
+import scala.Some;
 import scala.concurrent.duration.Duration;
 import scala.concurrent.duration.FiniteDuration;
 
@@ -301,8 +304,8 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 				actorSystem,
 				futureExecutor,
 				ioExecutor,
-				new scala.Some<>(JobManager.JOB_MANAGER_NAME()),
-				scala.Option.<String>empty(),
+				new Some<>(JobMaster.JOB_MANAGER_NAME),
+				Option.<String>empty(),
 				getJobManagerClass(),
 				getArchivistClass())._1();
 
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtils.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtils.java
index 6616a2a0e30d2..144e5190b2620 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtils.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtils.java
@@ -25,12 +25,11 @@
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpVersion;
 import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
-import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.webmonitor.files.MimeTypes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import scala.Option;
 import scala.Tuple2;
 
 import java.util.regex.Matcher;
@@ -62,7 +61,7 @@ public static String getRedirectAddress(
 		final String jobManagerName = localJobManagerAddress.substring(localJobManagerAddress.lastIndexOf("/") + 1);
 
 		if (!localJobManagerAddress.equals(leaderAddress) &&
-			!leaderAddress.equals(JobManager.getLocalJobManagerAkkaURL(Option.apply(jobManagerName)))) {
+			!leaderAddress.equals(AkkaUtils.getLocalAkkaURL(jobManagerName))) {
 			// We are not the leader and need to redirect
 			Matcher matcher = LeaderAddressHostPattern.matcher(leaderAddress);
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
index c540f7401579d..b0cab78241adc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
@@ -24,7 +24,7 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
-import org.apache.flink.runtime.highavailability.ZookeeperHaServices;
+import org.apache.flink.runtime.highavailability.zookeeper.ZooKeeperHaServices;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
 import org.apache.flink.util.StringUtils;
 import org.slf4j.Logger;
@@ -84,7 +84,7 @@ static BlobStore createBlobStoreFromConfig(Configuration config) throws IOExcept
 		if (highAvailabilityMode == HighAvailabilityMode.NONE) {
 			return new VoidBlobStore();
 		} else if (highAvailabilityMode == HighAvailabilityMode.ZOOKEEPER) {
-			return ZookeeperHaServices.createBlobStore(config);
+			return ZooKeeperHaServices.createBlobStore(config);
 		} else {
 			throw new IllegalConfigurationException("Unexpected high availability mode '" + highAvailabilityMode + "'.");
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java
index b5d7cb77cebf0..b944ba8f7561c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java
@@ -135,7 +135,7 @@ public ClassLoader getClassLoader() throws JobRetrievalException {
 	private ActorGateway getJobManager() throws JobRetrievalException {
 		try {
 			return LeaderRetrievalUtils.retrieveLeaderGateway(
-				LeaderRetrievalUtils.createLeaderRetrievalService(configuration),
+				LeaderRetrievalUtils.createLeaderRetrievalService(configuration, true),
 				actorSystem,
 				AkkaUtils.getLookupTimeout(configuration));
 		} catch (Exception e) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
deleted file mode 100644
index 8bf81ebbcab3a..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.highavailability;
-
-import org.apache.flink.runtime.highavailability.nonha.AbstractNonHaServices;
-import org.apache.flink.runtime.highavailability.nonha.EmbeddedLeaderService;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-
-/**
- * An implementation of the {@link HighAvailabilityServices} for the non-high-availability case
- * where all participants (ResourceManager, JobManagers, TaskManagers) run in the same process.
- *
- * <p>This implementation has no dependencies on any external services. It returns a fix
- * pre-configured ResourceManager, and stores checkpoints and metadata simply on the heap or
- * on a local file system and therefore in a storage without guarantees.
- */
-public class EmbeddedNonHaServices extends AbstractNonHaServices implements HighAvailabilityServices {
-
-	private final EmbeddedLeaderService resourceManagerLeaderService;
-
-	public EmbeddedNonHaServices() {
-		super();
-		this.resourceManagerLeaderService = new EmbeddedLeaderService(getExecutorService());
-	}
-
-	// ------------------------------------------------------------------------
-
-	@Override
-	public LeaderRetrievalService getResourceManagerLeaderRetriever() {
-		return resourceManagerLeaderService.createLeaderRetrievalService();
-	}
-
-	@Override
-	public LeaderElectionService getResourceManagerLeaderElectionService() {
-		return resourceManagerLeaderService.createLeaderElectionService();
-	}
-
-	// ------------------------------------------------------------------------
-
-	@Override
-	public void close() throws Exception {
-		try {
-			super.close();
-		} finally {
-			resourceManagerLeaderService.shutdown();
-		}
-	}
-
-	@Override
-	public void closeAndCleanupAllData() throws Exception {
-		close();
-	}
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
index fe180ded56203..106be5a4d1b14 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
@@ -18,43 +18,111 @@
 
 package org.apache.flink.runtime.highavailability;
 
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
+import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneHaServices;
+import org.apache.flink.runtime.highavailability.zookeeper.ZooKeeperHaServices;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
+import org.apache.flink.runtime.jobmaster.JobMaster;
+import org.apache.flink.runtime.resourcemanager.ResourceManager;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.runtime.util.ZooKeeperUtils;
+import org.apache.flink.util.ConfigurationException;
 
+import java.util.concurrent.Executor;
+
+/**
+ * Utils class to instantiate {@link HighAvailabilityServices} implementations.
+ */
 public class HighAvailabilityServicesUtils {
 
-	public static HighAvailabilityServices createAvailableOrEmbeddedServices(Configuration config) throws Exception {
+	public static HighAvailabilityServices createAvailableOrEmbeddedServices(
+		Configuration config,
+		Executor executor) throws Exception {
 		HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);
 
 		switch (highAvailabilityMode) {
 			case NONE:
-				return new EmbeddedNonHaServices();
+				return new EmbeddedHaServices(executor);
 
 			case ZOOKEEPER:
-				return new ZookeeperHaServices(ZooKeeperUtils.startCuratorFramework(config), 
-						Executors.directExecutor(), config);
+				return new ZooKeeperHaServices(
+					ZooKeeperUtils.startCuratorFramework(config),
+					executor,
+					config);
 
 			default:
 				throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
 		}
 	}
 	
-	
-	public static HighAvailabilityServices createHighAvailabilityServices(Configuration configuration) throws Exception {
+	public static HighAvailabilityServices createHighAvailabilityServices(
+		Configuration configuration,
+		Executor executor,
+		AddressResolution addressResolution) throws Exception {
+
 		HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);
 
 		switch(highAvailabilityMode) {
 			case NONE:
-				final String resourceManagerAddress = null;
-				return new NonHaServices(resourceManagerAddress);
+				final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);
+
+				final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
+					hostnamePort.f0,
+					hostnamePort.f1,
+					JobMaster.JOB_MANAGER_NAME,
+					addressResolution,
+					configuration);
+				final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
+					hostnamePort.f0,
+					hostnamePort.f1,
+					ResourceManager.RESOURCE_MANAGER_NAME,
+					addressResolution,
+					configuration);
+
+				return new StandaloneHaServices(resourceManagerRpcUrl, jobManagerRpcUrl);
 			case ZOOKEEPER:
-				return new ZookeeperHaServices(ZooKeeperUtils.startCuratorFramework(configuration), 
-						Executors.directExecutor(), configuration);
+				return new ZooKeeperHaServices(
+					ZooKeeperUtils.startCuratorFramework(configuration),
+					executor,
+					configuration);
 			default:
 				throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
 		}
 	}
+
+	/**
+	 * Returns the JobManager's hostname and port extracted from the given
+	 * {@link Configuration}.
+	 *
+	 * @param configuration Configuration to extract the JobManager's address from
+	 * @return The JobManager's hostname and port
+	 * @throws ConfigurationException if the JobManager's address cannot be extracted from the configuration
+	 */
+	public static Tuple2<String, Integer> getJobManagerAddress(Configuration configuration) throws ConfigurationException {
+
+		final String hostname = configuration.getString(JobManagerOptions.ADDRESS);
+		final int port = configuration.getInteger(JobManagerOptions.PORT);
+
+		if (hostname == null) {
+			throw new ConfigurationException("Config parameter '" + JobManagerOptions.ADDRESS +
+				"' is missing (hostname/address of JobManager to connect to).");
+		}
+
+		if (port <= 0 || port >= 65536) {
+			throw new ConfigurationException("Invalid value for '" + JobManagerOptions.PORT +
+				"' (port of the JobManager actor system) : " + port +
+				".  it must be great than 0 and less than 65536.");
+		}
+
+		return Tuple2.of(hostname, port);
+	}
+
+	public enum AddressResolution {
+		TRY_ADDRESS_RESOLUTION,
+		NO_ADDRESS_RESOLUTION
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
index b10e414983f72..ac90e3fb4a542 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
@@ -18,131 +18,85 @@
 
 package org.apache.flink.runtime.highavailability.nonha;
 
-import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.blob.BlobStore;
 import org.apache.flink.runtime.blob.VoidBlobStore;
 import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
 import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
-import org.apache.flink.runtime.highavailability.ServicesThreadFactory;
+import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneRunningJobsRegistry;
 import org.apache.flink.runtime.jobmanager.StandaloneSubmittedJobGraphStore;
 import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 
 import javax.annotation.concurrent.GuardedBy;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
-import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkState;
 
 /**
- * Base for all {@link HighAvailabilityServices} that are not highly available, but are backed
- * by storage that has no availability guarantees and leader election services that cannot
- * elect among multiple distributed leader contenders.
+ * Abstract base class for non high-availability services.
+ *
+ * This class returns the standalone variants for the checkpoint recovery factory, the submitted
+ * job graph store, the running jobs registry and the blob store.
  */
 public abstract class AbstractNonHaServices implements HighAvailabilityServices {
+	protected final Object lock = new Object();
 
-	private final Object lock = new Object();
-
-	private final ExecutorService executor;
-
-	private final HashMap<JobID, EmbeddedLeaderService> jobManagerLeaderServices;
-
-	private final NonHaRegistry runningJobsRegistry;
+	private final RunningJobsRegistry runningJobsRegistry;
 
 	private boolean shutdown;
 
-	// ------------------------------------------------------------------------
-
 	public AbstractNonHaServices() {
-		this.executor = Executors.newCachedThreadPool(new ServicesThreadFactory());
-		this.jobManagerLeaderServices = new HashMap<>();
-		this.runningJobsRegistry = new NonHaRegistry();
+		this.runningJobsRegistry = new StandaloneRunningJobsRegistry();
+
+		shutdown = false;
 	}
 
-	// ------------------------------------------------------------------------
-	//  services
-	// ------------------------------------------------------------------------
+	// ----------------------------------------------------------------------
+	// HighAvailabilityServices method implementations
+	// ----------------------------------------------------------------------
 
 	@Override
-	public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
-		checkNotNull(jobID);
-
+	public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
 		synchronized (lock) {
 			checkNotShutdown();
-			EmbeddedLeaderService service = getOrCreateJobManagerService(jobID);
-			return service.createLeaderRetrievalService();
+
+			return new StandaloneCheckpointRecoveryFactory();
 		}
 	}
 
 	@Override
-	public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
-		checkNotNull(jobID);
-
+	public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
 		synchronized (lock) {
 			checkNotShutdown();
-			EmbeddedLeaderService service = getOrCreateJobManagerService(jobID);
-			return service.createLeaderElectionService();
-		}
-	}
 
-	@GuardedBy("lock")
-	private EmbeddedLeaderService getOrCreateJobManagerService(JobID jobID) {
-		EmbeddedLeaderService service = jobManagerLeaderServices.get(jobID);
-		if (service == null) {
-			service = new EmbeddedLeaderService(executor);
-			jobManagerLeaderServices.put(jobID, service);
+			return new StandaloneSubmittedJobGraphStore();
 		}
-		return service;
-	}
-
-	@Override
-	public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
-		checkNotShutdown();
-		return new StandaloneCheckpointRecoveryFactory();
 	}
 
 	@Override
-	public SubmittedJobGraphStore getSubmittedJobGraphStore() {
-		checkNotShutdown();
-		return new StandaloneSubmittedJobGraphStore();
-	}
+	public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+		synchronized (lock) {
+			checkNotShutdown();
 
-	@Override
-	public RunningJobsRegistry getRunningJobsRegistry() {
-		checkNotShutdown();
-		return runningJobsRegistry;
+			return runningJobsRegistry;
+		}
 	}
 
 	@Override
 	public BlobStore createBlobStore() throws IOException {
-		checkNotShutdown();
-		return new VoidBlobStore();
-	}
+		synchronized (lock) {
+			checkNotShutdown();
 
-	// ------------------------------------------------------------------------
-	//  shutdown
-	// ------------------------------------------------------------------------
+			return new VoidBlobStore();
+		}
+	}
 
 	@Override
 	public void close() throws Exception {
 		synchronized (lock) {
 			if (!shutdown) {
 				shutdown = true;
-
-				// no further calls should be dispatched
-				executor.shutdownNow();
-
-				// stop all job manager leader services
-				for (EmbeddedLeaderService service : jobManagerLeaderServices.values()) {
-					service.shutdown();
-				}
-				jobManagerLeaderServices.clear();
 			}
 		}
 	}
@@ -153,15 +107,16 @@ public void closeAndCleanupAllData() throws Exception {
 		close();
 	}
 
-	private void checkNotShutdown() {
+	// ----------------------------------------------------------------------
+	// Helper methods
+	// ----------------------------------------------------------------------
+
+	@GuardedBy("lock")
+	protected void checkNotShutdown() {
 		checkState(!shutdown, "high availability services are shut down");
 	}
 
-	// ------------------------------------------------------------------------
-	//  utilities
-	// ------------------------------------------------------------------------
-
-	protected ExecutorService getExecutorService() {
-		return executor;
+	protected boolean isShutDown() {
+		return shutdown;
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServices.java
new file mode 100644
index 0000000000000..a338edc46ec80
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServices.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability.nonha.embedded;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.AbstractNonHaServices;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.concurrent.GuardedBy;
+import java.util.HashMap;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * An implementation of the {@link HighAvailabilityServices} for the non-high-availability case
+ * where all participants (ResourceManager, JobManagers, TaskManagers) run in the same process.
+ *
+ * <p>This implementation has no dependencies on any external services. It returns a fix
+ * pre-configured ResourceManager, and stores checkpoints and metadata simply on the heap or
+ * on a local file system and therefore in a storage without guarantees.
+ */
+public class EmbeddedHaServices extends AbstractNonHaServices {
+
+	private final Executor executor;
+
+	private final EmbeddedLeaderService resourceManagerLeaderService;
+
+	private final HashMap<JobID, EmbeddedLeaderService> jobManagerLeaderServices;
+
+	public EmbeddedHaServices(Executor executor) {
+		this.executor = Preconditions.checkNotNull(executor);
+		this.resourceManagerLeaderService = new EmbeddedLeaderService(executor);
+		this.jobManagerLeaderServices = new HashMap<>();
+	}
+
+	// ------------------------------------------------------------------------
+	//  services
+	// ------------------------------------------------------------------------
+
+	@Override
+	public LeaderRetrievalService getResourceManagerLeaderRetriever() {
+		return resourceManagerLeaderService.createLeaderRetrievalService();
+	}
+
+	@Override
+	public LeaderElectionService getResourceManagerLeaderElectionService() {
+		return resourceManagerLeaderService.createLeaderElectionService();
+	}
+
+	@Override
+	public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
+		checkNotNull(jobID);
+
+		synchronized (lock) {
+			checkNotShutdown();
+			EmbeddedLeaderService service = getOrCreateJobManagerService(jobID);
+			return service.createLeaderRetrievalService();
+		}
+	}
+
+	@Override
+	public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
+		checkNotNull(jobID);
+
+		synchronized (lock) {
+			checkNotShutdown();
+			EmbeddedLeaderService service = getOrCreateJobManagerService(jobID);
+			return service.createLeaderElectionService();
+		}
+	}
+
+	// ------------------------------------------------------------------------
+	// internal
+	// ------------------------------------------------------------------------
+
+	@GuardedBy("lock")
+	private EmbeddedLeaderService getOrCreateJobManagerService(JobID jobID) {
+		EmbeddedLeaderService service = jobManagerLeaderServices.get(jobID);
+		if (service == null) {
+			service = new EmbeddedLeaderService(executor);
+			jobManagerLeaderServices.put(jobID, service);
+		}
+		return service;
+	}
+
+	// ------------------------------------------------------------------------
+	//  shutdown
+	// ------------------------------------------------------------------------
+
+	@Override
+	public void close() throws Exception {
+		synchronized (lock) {
+			if (!isShutDown()) {
+				// stop all job manager leader services
+				for (EmbeddedLeaderService service : jobManagerLeaderServices.values()) {
+					service.shutdown();
+				}
+				jobManagerLeaderServices.clear();
+
+				resourceManagerLeaderService.shutdown();
+			}
+
+			super.close();
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/EmbeddedLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java
similarity index 98%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/EmbeddedLeaderService.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java
index d4eba26bb7cf0..5eb43750656fd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/EmbeddedLeaderService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability.nonha;
+package org.apache.flink.runtime.highavailability.nonha.embedded;
 
 import org.apache.flink.runtime.leaderelection.LeaderContender;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
@@ -32,7 +32,6 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkState;
@@ -72,7 +71,7 @@ public class EmbeddedLeaderService {
 
 	// ------------------------------------------------------------------------
 
-	public EmbeddedLeaderService(ExecutorService notificationsDispatcher) {
+	public EmbeddedLeaderService(Executor notificationsDispatcher) {
 		this.notificationExecutor = checkNotNull(notificationsDispatcher);
 		this.allLeaderContenders = new HashSet<>();
 		this.listeners = new HashSet<>();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/leaderelection/SingleLeaderElectionService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionService.java
similarity index 99%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/leaderelection/SingleLeaderElectionService.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionService.java
index 96e13908202f4..a56b077fd865b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/leaderelection/SingleLeaderElectionService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionService.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability.leaderelection;
+package org.apache.flink.runtime.highavailability.nonha.leaderelection;
 
 import org.apache.flink.runtime.leaderelection.LeaderContender;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneHaServices.java
similarity index 61%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneHaServices.java
index beb596326271e..c7b54c32260c2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneHaServices.java
@@ -16,11 +16,13 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability;
+package org.apache.flink.runtime.highavailability.nonha.standalone;
 
-import org.apache.flink.runtime.highavailability.leaderelection.SingleLeaderElectionService;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.highavailability.nonha.AbstractNonHaServices;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderelection.StandaloneLeaderElectionService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 
@@ -32,10 +34,10 @@
  * tolerate failures of the master processes (JobManager, ResourceManager).
  * 
  * <p>This implementation has no dependencies on any external services. It returns a fix
- * pre-configured ResourceManager, and stores checkpoints and metadata simply on the heap or
- * on a local file system and therefore in a storage without guarantees.
+ * pre-configured ResourceManager and JobManager, and stores checkpoints and metadata simply on the
+ * heap or on a local file system and therefore in a storage without guarantees.
  */
-public class NonHaServices extends AbstractNonHaServices implements HighAvailabilityServices {
+public class StandaloneHaServices extends AbstractNonHaServices {
 
 	/** The constant name of the ResourceManager RPC endpoint */
 	private static final String RESOURCE_MANAGER_RPC_ENDPOINT_NAME = "resource_manager";
@@ -43,14 +45,17 @@ public class NonHaServices extends AbstractNonHaServices implements HighAvailabi
 	/** The fix address of the ResourceManager */
 	private final String resourceManagerAddress;
 
+	/** The fix address of the JobManager */
+	private final String jobManagerAddress;
+
 	/**
 	 * Creates a new services class for the fix pre-defined leaders.
 	 * 
 	 * @param resourceManagerAddress    The fix address of the ResourceManager
 	 */
-	public NonHaServices(String resourceManagerAddress) {
-		super();
-		this.resourceManagerAddress = checkNotNull(resourceManagerAddress);
+	public StandaloneHaServices(String resourceManagerAddress, String jobManagerAddress) {
+		this.resourceManagerAddress = checkNotNull(resourceManagerAddress, "resourceManagerAddress");
+		this.jobManagerAddress = checkNotNull(jobManagerAddress, "jobManagerAddress");
 	}
 
 	// ------------------------------------------------------------------------
@@ -59,11 +64,38 @@ public NonHaServices(String resourceManagerAddress) {
 
 	@Override
 	public LeaderRetrievalService getResourceManagerLeaderRetriever() {
-		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
+		synchronized (lock) {
+			checkNotShutdown();
+
+			return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
+		}
+
 	}
 
 	@Override
 	public LeaderElectionService getResourceManagerLeaderElectionService() {
-		return new SingleLeaderElectionService(getExecutorService(), DEFAULT_LEADER_ID);
+		synchronized (lock) {
+			checkNotShutdown();
+
+			return new StandaloneLeaderElectionService();
+		}
+	}
+
+	@Override
+	public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
+		synchronized (lock) {
+			checkNotShutdown();
+
+			return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
+		}
+	}
+
+	@Override
+	public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
+		synchronized (lock) {
+			checkNotShutdown();
+
+			return new StandaloneLeaderElectionService();
+		}
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/NonHaRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneRunningJobsRegistry.java
similarity index 93%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/NonHaRegistry.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneRunningJobsRegistry.java
index ab1ce47eec13c..585ef34b7f79d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/NonHaRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneRunningJobsRegistry.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability.nonha;
+package org.apache.flink.runtime.highavailability.nonha.standalone;
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
@@ -28,7 +28,7 @@
 /**
  * A registry for running jobs, not-highly available.
  */
-public class NonHaRegistry implements RunningJobsRegistry {
+public class StandaloneRunningJobsRegistry implements RunningJobsRegistry {
 
 	/** The currently running jobs */
 	private final HashMap<JobID, JobSchedulingStatus> jobStatus = new HashMap<>();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java
similarity index 94%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java
index 4d0db0a33ad6f..5d895c1e14261 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability;
+package org.apache.flink.runtime.highavailability.zookeeper;
 
 import org.apache.curator.framework.CuratorFramework;
 
@@ -30,6 +30,8 @@
 import org.apache.flink.runtime.blob.FileSystemBlobStore;
 import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
 import org.apache.flink.runtime.checkpoint.ZooKeeperCheckpointRecoveryFactory;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
 import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
@@ -79,7 +81,7 @@
  * {@link HighAvailabilityOptions#HA_CLUSTER_ID}. All nodes with the same cluster id will join the same
  * cluster and participate in the execution of the same set of jobs.
  */
-public class ZookeeperHaServices implements HighAvailabilityServices {
+public class ZooKeeperHaServices implements HighAvailabilityServices {
 
 	private static final String RESOURCE_MANAGER_LEADER_PATH = "/resource_manager_lock";
 
@@ -100,11 +102,11 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
 	/** The zookeeper based running jobs registry */
 	private final RunningJobsRegistry runningJobsRegistry;
 
-	public ZookeeperHaServices(CuratorFramework client, Executor executor, Configuration configuration) {
+	public ZooKeeperHaServices(CuratorFramework client, Executor executor, Configuration configuration) {
 		this.client = checkNotNull(client);
 		this.executor = checkNotNull(executor);
 		this.configuration = checkNotNull(configuration);
-		this.runningJobsRegistry = new ZookeeperRegistry(client, configuration);
+		this.runningJobsRegistry = new ZooKeeperRunningJobsRegistry(client, configuration);
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperRunningJobsRegistry.java
similarity index 92%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperRegistry.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperRunningJobsRegistry.java
index a8be35a05679a..8a083d1fc5eef 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperRunningJobsRegistry.java
@@ -16,12 +16,13 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability;
+package org.apache.flink.runtime.highavailability.zookeeper;
 
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
 import org.apache.zookeeper.data.Stat;
 
 import java.io.IOException;
@@ -33,7 +34,7 @@
 /**
  * A zookeeper based registry for running jobs, highly available.
  */
-public class ZookeeperRegistry implements RunningJobsRegistry {
+public class ZooKeeperRunningJobsRegistry implements RunningJobsRegistry {
 
 	private static final Charset ENCODING = Charset.forName("utf-8");
 
@@ -42,7 +43,7 @@ public class ZookeeperRegistry implements RunningJobsRegistry {
 
 	private final String runningJobPath;
 
-	public ZookeeperRegistry(final CuratorFramework client, final Configuration configuration) {
+	public ZooKeeperRunningJobsRegistry(final CuratorFramework client, final Configuration configuration) {
 		this.client = checkNotNull(client, "client");
 		this.runningJobPath = configuration.getString(HighAvailabilityOptions.ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH);
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index e60ff77983bf5..3a55f2eb2e260 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -64,7 +64,6 @@
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.OnCompletionActions;
 import org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException;
 import org.apache.flink.runtime.jobmanager.slots.AllocatedSlot;
@@ -88,8 +87,8 @@
 import org.apache.flink.runtime.rpc.RpcEndpoint;
 import org.apache.flink.runtime.rpc.RpcMethod;
 import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.RpcServiceUtils;
 import org.apache.flink.runtime.rpc.StartStoppable;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
 import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
@@ -127,6 +126,10 @@
  */
 public class JobMaster extends RpcEndpoint<JobMasterGateway> {
 
+	/** Default names for Flink's distributed components */
+	public static final String JOB_MANAGER_NAME = "jobmanager";
+	public static final String ARCHIVE_NAME = "archive";
+
 	private static final AtomicReferenceFieldUpdater<JobMaster, UUID> LEADER_ID_UPDATER =
 			AtomicReferenceFieldUpdater.newUpdater(JobMaster.class, UUID.class, "leaderSessionID");
 
@@ -208,7 +211,7 @@ public JobMaster(
 			FatalErrorHandler errorHandler,
 			ClassLoader userCodeLoader) throws Exception {
 
-		super(rpcService, RpcServiceUtils.createRandomName(JobManager.JOB_MANAGER_NAME()));
+		super(rpcService, AkkaRpcServiceUtils.createRandomName(JobMaster.JOB_MANAGER_NAME));
 
 		this.resourceId = checkNotNull(resourceId);
 		this.jobGraph = checkNotNull(jobGraph);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 03fbef5d44357..3e7f2f3cfcc44 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -235,7 +235,9 @@ public void start() throws Exception {
 
 				// create the high-availability services
 				LOG.info("Starting high-availability services");
-				haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(configuration);
+				haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+					configuration,
+					commonRpcService.getExecutor());
 
 				heartbeatServices = HeartbeatServices.fromConfiguration(configuration);
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java
index 5d9db19943959..f9cf01de75214 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java
@@ -109,7 +109,7 @@ public QueryableStateClient(Configuration config) throws Exception {
 
 		// Create a leader retrieval service
 		LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils
-				.createLeaderRetrievalService(config);
+				.createLeaderRetrievalService(config, true);
 
 		// Get the ask timeout
 		String askTimeoutString = config.getString(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 2ef827648f482..bef0aa3256794 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -87,6 +87,8 @@ public abstract class ResourceManager<WorkerType extends Serializable>
 		extends RpcEndpoint<ResourceManagerGateway>
 		implements LeaderContender {
 
+	public static final String RESOURCE_MANAGER_NAME = "resourcemanager";
+
 	/** Unique id of the resource manager */
 	private final ResourceID resourceId;
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java
index fa75bbba4b360..2c64f08d382de 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java
@@ -21,7 +21,7 @@
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.AkkaOptions;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.resourcemanager.exceptions.ConfigurationException;
+import org.apache.flink.util.ConfigurationException;
 import org.apache.flink.util.Preconditions;
 import scala.concurrent.duration.Duration;
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java
index 79d1c02a60271..d4bc2d5dea0aa 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServicesConfiguration.java
@@ -21,7 +21,7 @@
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.ResourceManagerOptions;
-import org.apache.flink.runtime.resourcemanager.exceptions.ConfigurationException;
+import org.apache.flink.util.ConfigurationException;
 import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerConfiguration;
 import org.apache.flink.util.Preconditions;
 import scala.concurrent.duration.Duration;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java
index d21c25167abb3..a6511688369bd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerConfiguration.java
@@ -23,7 +23,7 @@
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.ConfigOptions;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.resourcemanager.exceptions.ConfigurationException;
+import org.apache.flink.util.ConfigurationException;
 import org.apache.flink.util.Preconditions;
 import scala.concurrent.duration.Duration;
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceUtils.java
similarity index 76%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceUtils.java
index e555e7f509ab7..eab4de8a602b3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceUtils.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.rpc;
+package org.apache.flink.runtime.rpc.akka;
 
 import akka.actor.ActorSystem;
 import com.typesafe.config.Config;
@@ -24,8 +24,10 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution;
 import org.apache.flink.runtime.net.SSLUtils;
-import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
+import org.apache.flink.runtime.rpc.RpcService;
 import org.apache.flink.util.NetUtils;
 
 import org.apache.flink.util.Preconditions;
@@ -35,6 +37,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -45,9 +48,12 @@
  * These RPC utilities contain helper methods around RPC use, such as starting an RPC service,
  * or constructing RPC addresses.
  */
-public class RpcServiceUtils {
+public class AkkaRpcServiceUtils {
 
-	private static final Logger LOG = LoggerFactory.getLogger(RpcServiceUtils.class);
+	private static final Logger LOG = LoggerFactory.getLogger(AkkaRpcServiceUtils.class);
+
+	private static final String AKKA_TCP = "akka.tcp";
+	private static final String AkKA_SSL_TCP = "akka.ssl.tcp";
 
 	private static final AtomicLong nextNameOffset = new AtomicLong(0L);
 
@@ -113,8 +119,12 @@ public static RpcService createRpcService(String hostname, int port, Configurati
 	 *
 	 * @return The RPC URL of the specified RPC endpoint.
 	 */
-	public static String getRpcUrl(String hostname, int port, String endpointName, Configuration config)
-			throws UnknownHostException {
+	public static String getRpcUrl(
+		String hostname,
+		int port,
+		String endpointName,
+		HighAvailabilityServicesUtils.AddressResolution addressResolution,
+		Configuration config) throws UnknownHostException {
 
 		checkNotNull(config, "config is null");
 
@@ -123,7 +133,12 @@ public static String getRpcUrl(String hostname, int port, String endpointName, C
 					ConfigConstants.DEFAULT_AKKA_SSL_ENABLED) &&
 				SSLUtils.getSSLEnabled(config);
 
-		return getRpcUrl(hostname, port, endpointName, sslEnabled);
+		return getRpcUrl(
+			hostname,
+			port,
+			endpointName,
+			addressResolution,
+			sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
 	}
 
 	/**
@@ -131,21 +146,37 @@ public static String getRpcUrl(String hostname, int port, String endpointName, C
 	 * @param hostname     The hostname or address where the target RPC service is listening.
 	 * @param port         The port where the target RPC service is listening.
 	 * @param endpointName The name of the RPC endpoint.
-	 * @param secure       True, if security/encryption is enabled, false otherwise.
+	 * @param akkaProtocol       True, if security/encryption is enabled, false otherwise.
 	 * 
 	 * @return The RPC URL of the specified RPC endpoint.
 	 */
-	public static String getRpcUrl(String hostname, int port, String endpointName, boolean secure)
-			throws UnknownHostException {
+	public static String getRpcUrl(
+			String hostname,
+			int port,
+			String endpointName,
+			HighAvailabilityServicesUtils.AddressResolution addressResolution,
+			AkkaProtocol akkaProtocol) throws UnknownHostException {
 
 		checkNotNull(hostname, "hostname is null");
 		checkNotNull(endpointName, "endpointName is null");
 		checkArgument(port > 0 && port <= 65535, "port must be in [1, 65535]");
 
-		final String protocol = secure ? "akka.ssl.tcp" : "akka.tcp";
-		final String hostPort = NetUtils.hostAndPortToUrlString(hostname, port);
+		final String protocolPrefix = akkaProtocol == AkkaProtocol.SSL_TCP ? AkKA_SSL_TCP : AKKA_TCP;
+
+		if (addressResolution == AddressResolution.TRY_ADDRESS_RESOLUTION) {
+			// Fail fast if the hostname cannot be resolved
+			//noinspection ResultOfMethodCallIgnored
+			InetAddress.getByName(hostname);
+		}
+
+		final String hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(hostname, port);
+
+		return String.format("%s://flink@%s/user/%s", protocolPrefix, hostPort, endpointName);
+	}
 
-		return String.format("%s://flink@%s/user/%s", protocol, hostPort, endpointName);
+	public enum AkkaProtocol {
+		TCP,
+		SSL_TCP
 	}
 
 	/**
@@ -170,5 +201,5 @@ public static String createRandomName(String prefix) {
 	// ------------------------------------------------------------------------
 
 	/** This class is not meant to be instantiated */
-	private RpcServiceUtils() {}
+	private AkkaRpcServiceUtils() {}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 5b8c8ee2e2e25..d05d900a9992d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -65,7 +65,7 @@
 import org.apache.flink.runtime.rpc.RpcEndpoint;
 import org.apache.flink.runtime.rpc.RpcMethod;
 import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.RpcServiceUtils;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.runtime.taskexecutor.exceptions.CheckpointException;
 import org.apache.flink.runtime.taskexecutor.exceptions.PartitionException;
 import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
@@ -85,7 +85,6 @@
 import org.apache.flink.runtime.taskmanager.CheckpointResponder;
 import org.apache.flink.runtime.taskmanager.Task;
 import org.apache.flink.runtime.taskmanager.TaskExecutionState;
-import org.apache.flink.runtime.taskmanager.TaskManager;
 import org.apache.flink.runtime.taskmanager.TaskManagerActions;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.util.ExceptionUtils;
@@ -111,6 +110,8 @@
  */
 public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
 
+	public static final String TASK_MANAGER_NAME = "taskmanager";
+
 	/** The connection information of this task manager */
 	private final TaskManagerLocation taskManagerLocation;
 
@@ -186,7 +187,7 @@ public TaskExecutor(
 			JobLeaderService jobLeaderService,
 			FatalErrorHandler fatalErrorHandler) {
 
-		super(rpcService, RpcServiceUtils.createRandomName(TaskManager.TASK_MANAGER_NAME()));
+		super(rpcService, AkkaRpcServiceUtils.createRandomName(TaskExecutor.TASK_MANAGER_NAME));
 
 		checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index 2be8ff1663763..2ed157896be13 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -30,7 +30,7 @@
 import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.RpcServiceUtils;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.runtime.taskexecutor.utils.TaskExecutorMetricsInitializer;
 import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 
@@ -219,6 +219,6 @@ public static RpcService createRpcService(
 				"use 0 to let the system choose port automatically.",
 			ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, rpcPort);
 
-		return RpcServiceUtils.createRpcService(taskManagerHostname, rpcPort, configuration);
+		return AkkaRpcServiceUtils.createRpcService(taskManagerHostname, rpcPort, configuration);
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
index b22f1a684efb0..05749c43cf631 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
@@ -52,18 +52,6 @@ public class LeaderRetrievalUtils {
 
 	private static final Logger LOG = LoggerFactory.getLogger(LeaderRetrievalUtils.class);
 
-	/**
-	 * Creates a {@link LeaderRetrievalService} based on the provided {@link Configuration} object.
-	 *
-	 * @param configuration Configuration containing the settings for the {@link LeaderRetrievalService}
-	 * @return The {@link LeaderRetrievalService} specified in the configuration object
-	 * @throws Exception
-	 */
-	public static LeaderRetrievalService createLeaderRetrievalService(Configuration configuration)
-		throws Exception {
-		return createLeaderRetrievalService(configuration, false);
-	}
-
 	/**
 	 * Creates a {@link LeaderRetrievalService} based on the provided {@link Configuration} object.
 	 *
@@ -73,14 +61,15 @@ public static LeaderRetrievalService createLeaderRetrievalService(Configuration
 	 * @throws Exception
 	 */
 	public static LeaderRetrievalService createLeaderRetrievalService(
-			Configuration configuration, boolean resolveInitialHostName)
+			Configuration configuration,
+			boolean resolveInitialHostName)
 		throws Exception {
 
 		HighAvailabilityMode highAvailabilityMode = getRecoveryMode(configuration);
 
 		switch (highAvailabilityMode) {
 			case NONE:
-				return StandaloneUtils.createLeaderRetrievalService(configuration, resolveInitialHostName);
+				return StandaloneUtils.createLeaderRetrievalService(configuration, resolveInitialHostName, null);
 			case ZOOKEEPER:
 				return ZooKeeperUtils.createLeaderRetrievalService(configuration);
 			default:
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/StandaloneUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/StandaloneUtils.java
index 8436ced5533a5..1719b38c3bd2f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/StandaloneUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/StandaloneUtils.java
@@ -18,15 +18,15 @@
 
 package org.apache.flink.runtime.util;
 
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.jobmanager.JobManager;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution;
+import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
-import org.apache.flink.runtime.taskmanager.TaskManager;
-import org.apache.flink.util.NetUtils;
-import scala.Option;
-import scala.Tuple3;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
+import org.apache.flink.util.ConfigurationException;
 
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 /**
@@ -40,27 +40,15 @@ public final class StandaloneUtils {
 	 *
 	 * @param configuration Configuration instance containing the host and port information
 	 * @return StandaloneLeaderRetrievalService
+	 * @throws ConfigurationException
 	 * @throws UnknownHostException
 	 */
-	public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
-		Configuration configuration)
-		throws UnknownHostException {
-		return createLeaderRetrievalService(configuration, false);
-	}
-
-	/**
-	 * Creates a {@link StandaloneLeaderRetrievalService} from the given configuration. The
-	 * host and port for the remote Akka URL are retrieved from the provided configuration.
-	 *
-	 * @param configuration Configuration instance containing the host and port information
-	 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
-	 * @return StandaloneLeaderRetrievalService
-	 * @throws UnknownHostException
-	 */
-	public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
-			Configuration configuration, boolean resolveInitialHostName)
-		throws UnknownHostException {
-		return createLeaderRetrievalService(configuration, resolveInitialHostName, null);
+	public static StandaloneLeaderRetrievalService createLeaderRetrievalService(Configuration configuration)
+		throws ConfigurationException, UnknownHostException {
+		return createLeaderRetrievalService(
+			configuration,
+			false,
+			null);
 	}
 
 	/**
@@ -73,38 +61,22 @@ public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
 	 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 	 * @param jobManagerName Name of the JobManager actor
 	 * @return StandaloneLeaderRetrievalService
-	 * @throws UnknownHostException if the host name cannot be resolved into an {@link InetAddress}
+	 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
+	 * @throws UnknownHostException if the job manager address cannot be resolved
 	 */
 	public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
 			Configuration configuration,
 			boolean resolveInitialHostName,
 			String jobManagerName)
-		throws UnknownHostException {
-
-		Tuple3<String, String, Object> stringIntPair = TaskManager.getAndCheckJobManagerAddress(configuration);
-
-		String protocol = stringIntPair._1();
-		String jobManagerHostname = stringIntPair._2();
-		int jobManagerPort = (Integer) stringIntPair._3();
-
-		// Do not try to resolve a hostname to prevent resolving to the wrong IP address
-		String hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(jobManagerHostname, jobManagerPort);
-
-		if (resolveInitialHostName) {
-			try {
-				//noinspection ResultOfMethodCallIgnored
-				InetAddress.getByName(jobManagerHostname);
-			}
-			catch (UnknownHostException e) {
-				throw new UnknownHostException("Cannot resolve the JobManager hostname '" + jobManagerHostname
-					+ "' specified in the configuration");
-			}
-		}
+		throws ConfigurationException, UnknownHostException {
+		Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);
 
-		String jobManagerAkkaUrl = JobManager.getRemoteJobManagerAkkaURL(
-				protocol,
-				hostPort,
-				Option.apply(jobManagerName));
+		String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
+			hostnamePort.f0,
+			hostnamePort.f1,
+			jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
+			resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
+			configuration);
 
 		return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
 	}
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
index d94eb7a73f873..62fa73d1ddb31 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
@@ -28,7 +28,7 @@ import com.typesafe.config.{Config, ConfigFactory}
 import org.apache.flink.api.common.time.Time
 import org.apache.flink.configuration.{AkkaOptions, ConfigConstants, Configuration}
 import org.apache.flink.runtime.net.SSLUtils
-import org.apache.flink.util.NetUtils
+import org.apache.flink.util.{ConfigurationException, NetUtils, Preconditions}
 import org.jboss.netty.logging.{InternalLoggerFactory, Slf4JLoggerFactory}
 import org.slf4j.LoggerFactory
 
@@ -707,18 +707,15 @@ object AkkaUtils {
       "(d|day)|(h|hour)|(min|minute)|s|sec|second)|(ms|milli|millisecond)|" +
       "(µs|micro|microsecond)|(ns|nano|nanosecond)"
   }
-  
-  /** Returns the protocol field for the URL of the remote actor system given the user configuration
+
+  /**
+    * Returns the local akka url for the given actor name.
     *
-    * @param config instance containing the user provided configuration values
-    * @return the remote url's protocol field
+    * @param actorName Actor name identifying the actor
+    * @return Local Akka URL for the given actor
     */
-  def getAkkaProtocol(config: Configuration): String = {
-    val sslEnabled = config.getBoolean(ConfigConstants.AKKA_SSL_ENABLED,
-        ConfigConstants.DEFAULT_AKKA_SSL_ENABLED) &&
-      SSLUtils.getSSLEnabled(config)
-    if (sslEnabled) "akka.ssl.tcp" else "akka.tcp"
+  def getLocalAkkaURL(actorName: String): String = {
+    "akka://flink/user/" + actorName
   }
-
 }
 
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 479ec51d12368..68f43da8ba0cf 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -49,11 +49,15 @@ import org.apache.flink.runtime.execution.SuppressRestartsException
 import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory
 import org.apache.flink.runtime.executiongraph._
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution
 import org.apache.flink.runtime.instance.{AkkaActorGateway, InstanceID, InstanceManager}
 import org.apache.flink.runtime.jobgraph.{JobGraph, JobStatus}
 import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore.SubmittedJobGraphListener
 import org.apache.flink.runtime.jobmanager.scheduler.{Scheduler => FlinkScheduler}
 import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway
+import org.apache.flink.runtime.jobmaster.JobMaster
+import org.apache.flink.runtime.jobmaster.JobMaster.{ARCHIVE_NAME, JOB_MANAGER_NAME}
 import org.apache.flink.runtime.leaderelection.{LeaderContender, LeaderElectionService, StandaloneLeaderElectionService}
 import org.apache.flink.runtime.messages.ArchiveMessages.ArchiveExecutionGraph
 import org.apache.flink.runtime.messages.ExecutionGraphMessages.JobStatusChanged
@@ -72,8 +76,11 @@ import org.apache.flink.runtime.metrics.util.MetricUtils
 import org.apache.flink.runtime.process.ProcessReaper
 import org.apache.flink.runtime.query.KvStateMessage.{LookupKvStateLocation, NotifyKvStateRegistered, NotifyKvStateUnregistered}
 import org.apache.flink.runtime.query.{KvStateMessage, UnknownKvStateLocation}
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils
 import org.apache.flink.runtime.security.SecurityUtils
 import org.apache.flink.runtime.security.SecurityUtils.SecurityConfiguration
+import org.apache.flink.runtime.taskexecutor.TaskExecutor
+import org.apache.flink.runtime.taskexecutor.TaskExecutor.TASK_MANAGER_NAME
 import org.apache.flink.runtime.taskmanager.TaskManager
 import org.apache.flink.runtime.util._
 import org.apache.flink.runtime.webmonitor.{WebMonitor, WebMonitorUtils}
@@ -1923,12 +1930,6 @@ object JobManager {
   val STARTUP_FAILURE_RETURN_CODE = 1
   val RUNTIME_FAILURE_RETURN_CODE = 2
 
-  /** Name of the JobManager actor */
-  val JOB_MANAGER_NAME = "jobmanager"
-
-  /** Name of the archive actor */
-  val ARCHIVE_NAME = "archive"
-
 
   /**
    * Entry point (main method) to run the JobManager in a standalone fashion.
@@ -2237,7 +2238,7 @@ object JobManager {
       if (configuration.getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0) >= 0) {
         LOG.info("Starting JobManager web frontend")
         val leaderRetrievalService = LeaderRetrievalUtils
-          .createLeaderRetrievalService(configuration)
+          .createLeaderRetrievalService(configuration, false)
 
         // start the web frontend. we need to load this dynamically
         // because it is not in the same project/dependencies
@@ -2287,7 +2288,7 @@ object JobManager {
           ResourceID.generate(),
           jobManagerSystem,
           externalHostname,
-          Some(TaskManager.TASK_MANAGER_NAME),
+          Some(TaskExecutor.TASK_MANAGER_NAME),
           None,
           localTaskManagerCommunication = true,
           classOf[TaskManager])
@@ -2305,7 +2306,13 @@ object JobManager {
       // start web monitor
       webMonitor.foreach {
         monitor =>
-          val jobManagerAkkaUrl = JobManager.getRemoteJobManagerAkkaURL(configuration)
+          val hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration)
+          val jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
+            hostnamePort.f0,
+            hostnamePort.f1,
+            JobMaster.JOB_MANAGER_NAME,
+            AddressResolution.NO_ADDRESS_RESOLUTION,
+            configuration)
           monitor.start(jobManagerAkkaUrl)
       }
 
@@ -2317,7 +2324,7 @@ object JobManager {
               FlinkResourceManager.startResourceManagerActors(
                 configuration,
                 jobManagerSystem,
-                LeaderRetrievalUtils.createLeaderRetrievalService(configuration),
+                LeaderRetrievalUtils.createLeaderRetrievalService(configuration, false),
                 rmClass))
           case None =>
             LOG.info("Resource Manager class not provided. No resource manager will be started.")
@@ -2631,8 +2638,8 @@ object JobManager {
       actorSystem,
       futureExecutor,
       ioExecutor,
-      Some(JOB_MANAGER_NAME),
-      Some(ARCHIVE_NAME),
+      Some(JobMaster.JOB_MANAGER_NAME),
+      Some(JobMaster.ARCHIVE_NAME),
       jobManagerClass,
       archiveClass)
   }
@@ -2761,121 +2768,4 @@ object JobManager {
       jobRecoveryTimeout,
       metricsRegistry)
   }
-
-  // --------------------------------------------------------------------------
-  //  Resolving the JobManager endpoint
-  // --------------------------------------------------------------------------
-
-  /**
-   * Builds the akka actor path for the JobManager actor, given the socket address
-   * where the JobManager's actor system runs.
-   *
-   * @param protocol The protocol to be used to connect to the remote JobManager's actor system.
-   * @param hostPort The external address of the JobManager's actor system in format host:port
-   * @return The akka URL of the JobManager actor.
-   */
-  def getRemoteJobManagerAkkaURL(
-      protocol: String,
-      hostPort: String,
-      name: Option[String] = None)
-    : String = {
-
-    require(protocol == "akka.tcp" || protocol == "akka.ssl.tcp",
-        "protocol field should be either akka.tcp or akka.ssl.tcp")
-
-    getJobManagerAkkaURLHelper(s"$protocol://flink@$hostPort", name)
-  }
-
-  /**
-   * Returns the JobManager actor's remote Akka URL, given the configured hostname and port.
-   *
-   * @param config The configuration to parse
-   * @return JobManager actor remote Akka URL
-   */
-  def getRemoteJobManagerAkkaURL(config: Configuration) : String = {
-    val (protocol, hostname, port) = TaskManager.getAndCheckJobManagerAddress(config)
-
-    val hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(hostname, port)
-
-    JobManager.getRemoteJobManagerAkkaURL(protocol, hostPort, Option.empty)
-  }
-
-  /**
-   * Builds the akka actor path for the JobManager actor to address the actor within
-   * its own actor system.
-   *
-   * @return The local akka URL of the JobManager actor.
-   */
-  def getLocalJobManagerAkkaURL(name: Option[String] = None): String = {
-    getJobManagerAkkaURLHelper("akka://flink", name)
-  }
-
-  def getJobManagerAkkaURL(system: ActorSystem, name: Option[String] = None): String = {
-    getJobManagerAkkaURLHelper(AkkaUtils.getAddress(system).toString, name)
-  }
-
-  private def getJobManagerAkkaURLHelper(address: String, name: Option[String]): String = {
-    address + "/user/" + name.getOrElse(JOB_MANAGER_NAME)
-  }
-
-  /**
-   * Resolves the JobManager actor reference in a blocking fashion.
-   *
-   * @param jobManagerUrl The akka URL of the JobManager.
-   * @param system The local actor system that should perform the lookup.
-   * @param timeout The maximum time to wait until the lookup fails.
-   * @throws java.io.IOException Thrown, if the lookup fails.
-   * @return The ActorRef to the JobManager
-   */
-  @throws(classOf[IOException])
-  def getJobManagerActorRef(
-      jobManagerUrl: String,
-      system: ActorSystem,
-      timeout: FiniteDuration)
-    : ActorRef = {
-    AkkaUtils.getActorRef(jobManagerUrl, system, timeout)
-  }
-
-  /**
-   * Resolves the JobManager actor reference in a blocking fashion.
-   *
-   * @param protocol The protocol to be used to connect to the remote JobManager's actor system.
-   * @param hostPort The external address of the JobManager's actor system in format host:port.
-   * @param system The local actor system that should perform the lookup.
-   * @param timeout The maximum time to wait until the lookup fails.
-   * @throws java.io.IOException Thrown, if the lookup fails.
-   * @return The ActorRef to the JobManager
-   */
-  @throws(classOf[IOException])
-  def getJobManagerActorRef(
-      protocol: String,
-      hostPort: String,
-      system: ActorSystem,
-      timeout: FiniteDuration)
-    : ActorRef = {
-
-    val jmAddress = getRemoteJobManagerAkkaURL(protocol, hostPort)
-    getJobManagerActorRef(jmAddress, system, timeout)
-  }
-
-  /**
-   * Resolves the JobManager actor reference in a blocking fashion.
-   *
-   * @param hostPort The address of the JobManager's actor system in format host:port.
-   * @param system The local actor system that should perform the lookup.
-   * @param config The config describing the maximum time to wait until the lookup fails.
-   * @throws java.io.IOException Thrown, if the lookup fails.
-   * @return The ActorRef to the JobManager
-   */
-  @throws(classOf[IOException])
-  def getJobManagerActorRef(
-      hostPort: String,
-      system: ActorSystem,
-      config: Configuration)
-    : ActorRef = {
-
-    val timeout = AkkaUtils.getLookupTimeout(config)
-    val protocol = AkkaUtils.getAkkaProtocol(config)
-    getJobManagerActorRef(protocol, hostPort, system, timeout)
-  }
 }
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index 3d43da5610e72..2f835486cd105 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -38,13 +38,14 @@ import org.apache.flink.runtime.io.network.NetworkEnvironment
 import org.apache.flink.runtime.io.network.netty.NettyConfig
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler
 import org.apache.flink.runtime.jobmanager.{JobManager, MemoryArchivist, SubmittedJobGraphStore}
+import org.apache.flink.runtime.jobmaster.JobMaster
 import org.apache.flink.runtime.leaderelection.LeaderElectionService
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
 import org.apache.flink.runtime.memory.MemoryManager
 import org.apache.flink.runtime.messages.JobManagerMessages
 import org.apache.flink.runtime.messages.JobManagerMessages.{RunningJobsStatus, StoppingFailure, StoppingResponse}
 import org.apache.flink.runtime.metrics.MetricRegistry
-import org.apache.flink.runtime.taskexecutor.{TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
+import org.apache.flink.runtime.taskexecutor.{TaskExecutor, TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
 import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
 import org.apache.flink.runtime.util.EnvironmentInformation
 
@@ -207,9 +208,9 @@ class LocalFlinkMiniCluster(
     val localExecution = numTaskManagers == 1
 
     val taskManagerActorName = if (singleActorSystem) {
-      TaskManager.TASK_MANAGER_NAME + "_" + (index + 1)
+      TaskExecutor.TASK_MANAGER_NAME + "_" + (index + 1)
     } else {
-      TaskManager.TASK_MANAGER_NAME
+      TaskExecutor.TASK_MANAGER_NAME
     }
 
     val resourceID = ResourceID.generate() // generate random resource id
@@ -397,9 +398,9 @@ class LocalFlinkMiniCluster(
 
   protected def getJobManagerName(index: Int): String = {
     if(singleActorSystem) {
-      JobManager.JOB_MANAGER_NAME + "_" + (index + 1)
+      JobMaster.JOB_MANAGER_NAME + "_" + (index + 1)
     } else {
-      JobManager.JOB_MANAGER_NAME
+      JobMaster.JOB_MANAGER_NAME
     }
   }
 
@@ -413,9 +414,9 @@ class LocalFlinkMiniCluster(
 
   protected def getArchiveName(index: Int): String = {
     if(singleActorSystem) {
-      JobManager.ARCHIVE_NAME + "_" + (index + 1)
+      JobMaster.ARCHIVE_NAME + "_" + (index + 1)
     } else {
-      JobManager.ARCHIVE_NAME
+      JobMaster.ARCHIVE_NAME
     }
   }
 
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 97e55f0452e23..4065660099658 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -64,7 +64,7 @@ import org.apache.flink.runtime.metrics.{MetricRegistry => FlinkMetricRegistry}
 import org.apache.flink.runtime.process.ProcessReaper
 import org.apache.flink.runtime.security.SecurityUtils
 import org.apache.flink.runtime.security.SecurityUtils.SecurityConfiguration
-import org.apache.flink.runtime.taskexecutor.{TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
+import org.apache.flink.runtime.taskexecutor.{TaskExecutor, TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
 import org.apache.flink.runtime.util._
 import org.apache.flink.runtime.{FlinkActor, LeaderSessionMessageFilter, LogMessages}
 
@@ -1496,9 +1496,6 @@ object TaskManager {
   /** Return code for critical errors during the runtime */
   val RUNTIME_FAILURE_RETURN_CODE = 2
 
-  /** The name of the TaskManager actor */
-  val TASK_MANAGER_NAME = "taskmanager"
-
   /** Maximum time (milli seconds) that the TaskManager will spend searching for a
     * suitable network interface to use for communication */
   val MAX_STARTUP_CONNECT_TIME = 120000L
@@ -1664,7 +1661,9 @@ object TaskManager {
       LOG.info("Using configured hostname/address for TaskManager: " + taskManagerHostname)
     }
     else {
-      val leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(configuration)
+      val leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(
+        configuration,
+        true)
       val lookupTimeout = AkkaUtils.getLookupTimeout(configuration)
 
       val taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(
@@ -1780,7 +1779,7 @@ object TaskManager {
         resourceID,
         taskManagerSystem,
         taskManagerHostname,
-        Some(TASK_MANAGER_NAME),
+        Some(TaskExecutor.TASK_MANAGER_NAME),
         None,
         localTaskManagerCommunication = false,
         taskManagerClass)
@@ -1969,38 +1968,4 @@ object TaskManager {
         throw new IOException("Could not connect to TaskManager at " + taskManagerUrl, e)
     }
   }
-
-  // --------------------------------------------------------------------------
-  //  Parsing and checking the TaskManager Configuration
-  // --------------------------------------------------------------------------
-
-  /**
-   * Gets the protocol, hostname and port of the JobManager from the configuration. Also checks that
-   * the hostname is not null and the port non-negative.
-   *
-   * @param configuration The configuration to read the config values from.
-   * @return A 3-tuple (protocol, hostname, port).
-   */
-  def getAndCheckJobManagerAddress(configuration: Configuration) : (String, String, Int) = {
-
-    val protocol = AkkaUtils.getAkkaProtocol(configuration)
-
-    val hostname = configuration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null)
-
-    val port = configuration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
-      ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT)
-
-    if (hostname == null) {
-      throw new Exception("Config parameter '" + ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY +
-        "' is missing (hostname/address of JobManager to connect to).")
-    }
-
-    if (port <= 0 || port >= 65536) {
-      throw new Exception("Invalid value for '" + ConfigConstants.JOB_MANAGER_IPC_PORT_KEY +
-        "' (port of the JobManager actor system) : " + port +
-        ".  it must be great than 0 and less than 65536.")
-    }
-
-    (protocol, hostname, port)
-  }
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index 37a86c714d241..830dbf9222e71 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -22,7 +22,7 @@
 import org.apache.flink.runtime.blob.BlobStore;
 import org.apache.flink.runtime.blob.VoidBlobStore;
 import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
-import org.apache.flink.runtime.highavailability.nonha.NonHaRegistry;
+import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneRunningJobsRegistry;
 import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
@@ -147,7 +147,7 @@ public SubmittedJobGraphStore getSubmittedJobGraphStore() {
 
 	@Override
 	public RunningJobsRegistry getRunningJobsRegistry() {
-		return new NonHaRegistry();
+		return new StandaloneRunningJobsRegistry();
 	}
 
 	@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java
new file mode 100644
index 0000000000000..7bf9364c8d3b3
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability.nonha.embedded;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.leaderelection.LeaderContender;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.util.TestLogger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.UUID;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for the {@link EmbeddedHaServices}.
+ */
+public class EmbeddedHaServicesTest extends TestLogger {
+
+	private EmbeddedHaServices embeddedHaServices;
+
+	@Before
+	public void setupTest() {
+		embeddedHaServices = new EmbeddedHaServices(Executors.directExecutor());
+	}
+
+	@After
+	public void teardownTest() throws Exception {
+		if (embeddedHaServices != null) {
+			embeddedHaServices.closeAndCleanupAllData();
+			embeddedHaServices = null;
+		}
+	}
+
+	/**
+	 * Tests that exactly one JobManager is elected as the leader for a given job id.
+	 */
+	@Test
+	public void testJobManagerLeaderElection() throws Exception {
+		JobID jobId1 = new JobID();
+		JobID jobId2 = new JobID();
+
+		LeaderContender leaderContender1 = mock(LeaderContender.class);
+		LeaderContender leaderContender2 = mock(LeaderContender.class);
+		LeaderContender leaderContenderDifferentJobId = mock(LeaderContender.class);
+
+		LeaderElectionService leaderElectionService1 = embeddedHaServices.getJobManagerLeaderElectionService(jobId1);
+		LeaderElectionService leaderElectionService2 = embeddedHaServices.getJobManagerLeaderElectionService(jobId1);
+		LeaderElectionService leaderElectionServiceDifferentJobId = embeddedHaServices.getJobManagerLeaderElectionService(jobId2);
+
+		leaderElectionService1.start(leaderContender1);
+		leaderElectionService2.start(leaderContender2);
+		leaderElectionServiceDifferentJobId.start(leaderContenderDifferentJobId);
+
+		ArgumentCaptor<UUID> leaderIdArgumentCaptor1 = ArgumentCaptor.forClass(UUID.class);
+		ArgumentCaptor<UUID> leaderIdArgumentCaptor2 = ArgumentCaptor.forClass(UUID.class);
+		verify(leaderContender1, atLeast(0)).grantLeadership(leaderIdArgumentCaptor1.capture());
+		verify(leaderContender2, atLeast(0)).grantLeadership(leaderIdArgumentCaptor2.capture());
+
+		assertTrue(leaderIdArgumentCaptor1.getAllValues().isEmpty() ^ leaderIdArgumentCaptor2.getAllValues().isEmpty());
+
+		verify(leaderContenderDifferentJobId).grantLeadership(any(UUID.class));
+	}
+
+	/**
+	 * Tests that exactly one ResourceManager is elected as the leader.
+	 */
+	@Test
+	public void testResourceManagerLeaderElection() throws Exception {
+		LeaderContender leaderContender1 = mock(LeaderContender.class);
+		LeaderContender leaderContender2 = mock(LeaderContender.class);
+
+		LeaderElectionService leaderElectionService1 = embeddedHaServices.getResourceManagerLeaderElectionService();
+		LeaderElectionService leaderElectionService2 = embeddedHaServices.getResourceManagerLeaderElectionService();
+
+		leaderElectionService1.start(leaderContender1);
+		leaderElectionService2.start(leaderContender2);
+
+		ArgumentCaptor<UUID> leaderIdArgumentCaptor1 = ArgumentCaptor.forClass(UUID.class);
+		ArgumentCaptor<UUID> leaderIdArgumentCaptor2 = ArgumentCaptor.forClass(UUID.class);
+		verify(leaderContender1, atLeast(0)).grantLeadership(leaderIdArgumentCaptor1.capture());
+		verify(leaderContender2, atLeast(0)).grantLeadership(leaderIdArgumentCaptor2.capture());
+
+		assertTrue(leaderIdArgumentCaptor1.getAllValues().isEmpty() ^ leaderIdArgumentCaptor2.getAllValues().isEmpty());
+	}
+
+	/**
+	 * Tests the JobManager leader retrieval for a given job.
+	 */
+	@Test
+	public void testJobManagerLeaderRetrieval() throws Exception {
+		final String address = "foobar";
+		JobID jobId = new JobID();
+		LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
+		LeaderContender leaderContender = mock(LeaderContender.class);
+		when(leaderContender.getAddress()).thenReturn(address);
+
+		LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
+		LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);
+
+		leaderRetrievalService.start(leaderRetrievalListener);
+		leaderElectionService.start(leaderContender);
+
+		ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
+		verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());
+
+		final UUID leaderId = leaderIdArgumentCaptor.getValue();
+
+		leaderElectionService.confirmLeaderSessionID(leaderId);
+
+		verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
+	}
+
+	/**
+	 * Tests the ResourceManager leader retrieval for a given job.
+	 */
+	@Test
+	public void testResourceManagerLeaderRetrieval() throws Exception {
+		final String address = "foobar";
+		LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
+		LeaderContender leaderContender = mock(LeaderContender.class);
+		when(leaderContender.getAddress()).thenReturn(address);
+
+		LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService();
+		LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever();
+
+		leaderRetrievalService.start(leaderRetrievalListener);
+		leaderElectionService.start(leaderContender);
+
+		ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
+		verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());
+
+		final UUID leaderId = leaderIdArgumentCaptor.getValue();
+
+		leaderElectionService.confirmLeaderSessionID(leaderId);
+
+		verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/leaderelection/SingleLeaderElectionServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionServiceTest.java
similarity index 99%
rename from flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/leaderelection/SingleLeaderElectionServiceTest.java
rename to flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionServiceTest.java
index a9805a1cc8a7e..3875c4f171c50 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/leaderelection/SingleLeaderElectionServiceTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionServiceTest.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability.leaderelection;
+package org.apache.flink.runtime.highavailability.nonha.leaderelection;
 
 import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.runtime.leaderelection.LeaderContender;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneHaServicesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneHaServicesTest.java
new file mode 100644
index 0000000000000..537ed204eef4f
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/standalone/StandaloneHaServicesTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability.nonha.standalone;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.leaderelection.LeaderContender;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.util.TestLogger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Tests for the {@link StandaloneHaServices}.
+ */
+public class StandaloneHaServicesTest extends TestLogger {
+
+	private final String jobManagerAddress = "jobManager";
+	private final String resourceManagerAddress = "resourceManager";
+
+	private StandaloneHaServices standaloneHaServices;
+
+	@Before
+	public void setupTest() {
+
+		standaloneHaServices = new StandaloneHaServices(resourceManagerAddress, jobManagerAddress);
+	}
+
+	@After
+	public void teardownTest() throws Exception {
+		if (standaloneHaServices != null) {
+			standaloneHaServices.closeAndCleanupAllData();
+			standaloneHaServices = null;
+		}
+	}
+
+	/**
+	 * Tests that the standalone leader election services return a fixed address and leader session
+	 * id.
+	 */
+	@Test
+	public void testLeaderElection() throws Exception {
+		JobID jobId = new JobID();
+		LeaderContender jmLeaderContender = mock(LeaderContender.class);
+		LeaderContender rmLeaderContender = mock(LeaderContender.class);
+
+		LeaderElectionService jmLeaderElectionService = standaloneHaServices.getJobManagerLeaderElectionService(jobId);
+		LeaderElectionService rmLeaderElectionService = standaloneHaServices.getResourceManagerLeaderElectionService();
+
+		jmLeaderElectionService.start(jmLeaderContender);
+		rmLeaderElectionService.start(rmLeaderContender);
+
+		verify(jmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
+		verify(rmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
+	}
+
+	/**
+	 * Tests that the standalone leader retrieval services return the specified address and the
+	 * fixed leader session id.
+	 */
+	@Test
+	public void testJobManagerLeaderRetrieval() throws Exception {
+		JobID jobId1 = new JobID();
+		JobID jobId2 = new JobID();
+		LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
+		LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);
+		LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class);
+
+		LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1);
+		LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2);
+		LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever();
+
+		jmLeaderRetrievalService1.start(jmListener1);
+		jmLeaderRetrievalService2.start(jmListener2);
+		rmLeaderRetrievalService.start(rmListener);
+
+		verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
+		verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
+		verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ZooKeeperRegistryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperRegistryTest.java
similarity index 88%
rename from flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ZooKeeperRegistryTest.java
rename to flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperRegistryTest.java
index b1881e574a455..06ffe3cded04c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ZooKeeperRegistryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperRegistryTest.java
@@ -16,14 +16,17 @@
  * limitations under the License.
  */
 
-package org.apache.flink.runtime.highavailability;
+package org.apache.flink.runtime.highavailability.zookeeper;
 
 import org.apache.curator.test.TestingServer;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
 import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
 import org.apache.flink.runtime.highavailability.RunningJobsRegistry.JobSchedulingStatus;
+import org.apache.flink.runtime.highavailability.zookeeper.ZooKeeperHaServices;
 import org.apache.flink.runtime.util.ZooKeeperUtils;
 import org.apache.flink.util.TestLogger;
 
@@ -58,7 +61,7 @@ public void testZooKeeperRegistry() throws Exception {
 		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
 		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
 
-		final HighAvailabilityServices zkHaService = new ZookeeperHaServices(
+		final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
 				ZooKeeperUtils.startCuratorFramework(configuration), Executors.directExecutor(), configuration);
 
 		final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
index 49c28e65d0f83..2ac3ea774b666 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
@@ -28,8 +28,10 @@
 import akka.actor.PoisonPill;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.runtime.jobmaster.JobMaster;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
-import org.apache.flink.util.NetUtils;
 import org.junit.Test;
 
 import org.apache.flink.configuration.Configuration;
@@ -117,10 +119,14 @@ public void testReapProcessOnFailure() {
 
 			if (jobManagerPort != -1) {
 				try {
-					jobManagerRef = JobManager.getJobManagerActorRef(
-						"akka.tcp",
-						NetUtils.unresolvedHostAndPortToNormalizedString("localhost", jobManagerPort),
-						localSystem, new FiniteDuration(25, TimeUnit.SECONDS));
+					final String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
+						"localhost",
+						jobManagerPort,
+						JobMaster.JOB_MANAGER_NAME,
+						HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION,
+						AkkaRpcServiceUtils.AkkaProtocol.TCP);
+
+					jobManagerRef = AkkaUtils.getActorRef(jobManagerAkkaUrl, localSystem, new FiniteDuration(25L, TimeUnit.SECONDS));
 				} catch (Throwable t) {
 					// job manager probably not ready yet
 					lastError = t;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
index bdff4019ad970..fcca173215d1b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
@@ -91,7 +91,7 @@ public static void setupJobManager() {
 			MemoryArchivist.class)._1();
 
 		try {
-			LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(jmConfig);
+			LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(jmConfig, false);
 
 			jmGateway = LeaderRetrievalUtils.retrieveLeaderGateway(
 					lrs,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java
index 1f1eb62d94a6b..bc8c0b60d0810 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java
@@ -140,7 +140,7 @@ public void testZooKeeperReelection() throws Exception {
 
 		Deadline deadline = new FiniteDuration(5, TimeUnit.MINUTES).fromNow();
 
-		int num = 20;
+		int num = 10;
 
 		ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
 		TestingContender[] contenders = new TestingContender[num];
@@ -194,7 +194,7 @@ public void testZooKeeperReelection() throws Exception {
 				}
 			}
 
-			assertFalse(deadline.isOverdue());
+			assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
 			assertEquals(num, numberSeenLeaders);
 
 		} finally {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java
index 6a8ff173ffd78..b79093ffd1fec 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java
@@ -22,10 +22,11 @@
 import org.apache.curator.test.TestingServer;
 
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.configuration.HighAvailabilityOptions;
-import org.apache.flink.runtime.jobmanager.JobManager;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.runtime.util.ZooKeeperUtils;
 import org.apache.flink.util.NetUtils;
@@ -35,7 +36,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import scala.Option;
 import scala.concurrent.duration.FiniteDuration;
 
 import java.io.IOException;
@@ -104,10 +104,12 @@ public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exc
 			client[0] = ZooKeeperUtils.startCuratorFramework(config);
 			client[1] = ZooKeeperUtils.startCuratorFramework(config);
 
-			String wrongHostPort = NetUtils.unresolvedHostAndPortToNormalizedString("1.1.1.1", 1234);
-
-			String wrongAddress = JobManager.getRemoteJobManagerAkkaURL(AkkaUtils.getAkkaProtocol(config),
-					wrongHostPort, Option.<String>empty());
+			String wrongAddress = AkkaRpcServiceUtils.getRpcUrl(
+				"1.1.1.1",
+				1234,
+				"foobar",
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION,
+				config);
 
 			try {
 				localHost = InetAddress.getLocalHost();
@@ -126,8 +128,12 @@ public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exc
 			InetSocketAddress correctInetSocketAddress = new InetSocketAddress(localHost, serverSocket.getLocalPort());
 			String hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(localHost.getHostName(), correctInetSocketAddress.getPort());
 
-			String correctAddress = JobManager.getRemoteJobManagerAkkaURL(AkkaUtils.getAkkaProtocol(config),
-				hostPort, Option.<String>empty());
+			String correctAddress = AkkaRpcServiceUtils.getRpcUrl(
+				localHost.getHostName(),
+				correctInetSocketAddress.getPort(),
+				JobMaster.JOB_MANAGER_NAME,
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION,
+				config);
 
 			faultyLeaderElectionService = ZooKeeperUtils.createLeaderElectionService(client[0], config);
 			TestingContender wrongLeaderAddressContender = new TestingContender(wrongAddress, faultyLeaderElectionService);
@@ -192,7 +198,7 @@ public void testTimeoutOfFindConnectingAddress() throws Exception {
 
 		FiniteDuration timeout = new FiniteDuration(10, TimeUnit.SECONDS);
 
-		LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(config);
+		LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
 		InetAddress result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout);
 
 		assertEquals(InetAddress.getLocalHost(), result);
@@ -214,7 +220,7 @@ public FindConnectingAddress(Configuration config, FiniteDuration timeout) {
 		@Override
 		public void run() {
 			try {
-				LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(config);
+				LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
 				result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout);
 			} catch (Exception e) {
 				exception = e;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index d3d4d43485e29..cc79c5d9510d0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -44,7 +44,7 @@
 import org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.highavailability.NonHaServices;
+import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneHaServices;
 import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
@@ -317,6 +317,7 @@ public void testImmediatelyRegistersIfLeaderIsKnown() throws Exception {
 		final ResourceID resourceID = ResourceID.generate();
 		final String resourceManagerAddress = "/resource/manager/address/one";
 		final ResourceID resourceManagerResourceId = new ResourceID(resourceManagerAddress);
+		final String jobManagerAddress = "localhost";
 
 		final TestingSerialRpcService rpc = new TestingSerialRpcService();
 		try {
@@ -335,7 +336,9 @@ public void testImmediatelyRegistersIfLeaderIsKnown() throws Exception {
 			TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
 			when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
 
-			NonHaServices haServices = new NonHaServices(resourceManagerAddress);
+			StandaloneHaServices haServices = new StandaloneHaServices(
+				resourceManagerAddress,
+				jobManagerAddress);
 
 			final TaskSlotTable taskSlotTable = mock(TaskSlotTable.class);
 			final SlotReport slotReport = new SlotReport();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java
index 2fafe5b379c24..d90400434047a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java
@@ -30,6 +30,7 @@
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
+import org.apache.flink.runtime.taskexecutor.TaskExecutor;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.util.NetUtils;
@@ -138,7 +139,7 @@ public void testReapProcessOnFailure() {
 			// is started and the TaskManager is up
 			String taskManagerActorName = String.format("akka.tcp://flink@%s/user/%s",
 					"localhost:" + taskManagerPort,
-					TaskManager.TASK_MANAGER_NAME());
+					TaskExecutor.TASK_MANAGER_NAME);
 
 			ActorRef taskManagerRef = null;
 			Throwable lastError = null;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java
index f3b1d4a1db8eb..5f352298e7a17 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java
@@ -33,6 +33,7 @@
 import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
+import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.messages.JobManagerMessages.LeaderSessionMessage;
@@ -182,7 +183,7 @@ public void testDelayedRegistration() {
 				// available. we give it the regular JobManager akka URL
 				taskManager = createTaskManager(
 						actorSystem,
-						JobManager.getLocalJobManagerAkkaURL(Option.<String>empty()),
+						AkkaUtils.getLocalAkkaURL(JobMaster.JOB_MANAGER_NAME),
 						new Configuration(),
 						true,
 						false);
@@ -248,7 +249,7 @@ public void testShutdownAfterRegistrationDurationExpired() {
 				// start the taskManager actor
 				taskManager = createTaskManager(
 						actorSystem,
-						JobManager.getLocalJobManagerAkkaURL(Option.<String>empty()),
+						AkkaUtils.getLocalAkkaURL(JobMaster.JOB_MANAGER_NAME),
 						tmConfig,
 						true,
 						false);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/JobManagerProcess.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/JobManagerProcess.java
index 48c65c048264a..6e1fb740c22b8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/JobManagerProcess.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/JobManagerProcess.java
@@ -23,15 +23,17 @@
 import org.apache.flink.api.java.utils.ParameterTool;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.JobManagerMode;
-import org.apache.flink.util.NetUtils;
+import org.apache.flink.runtime.jobmaster.JobMaster;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import scala.Option;
 import scala.concurrent.duration.Deadline;
 import scala.concurrent.duration.FiniteDuration;
 
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
@@ -143,13 +145,15 @@ public int getJobManagerPort(FiniteDuration timeout) throws InterruptedException
 	/**
 	 * Returns the Akka URL of this JobManager.
 	 */
-	public String getJobManagerAkkaURL(FiniteDuration timeout) throws InterruptedException {
+	public String getJobManagerAkkaURL(FiniteDuration timeout) throws InterruptedException, UnknownHostException {
 		int port = getJobManagerPort(timeout);
 
-		return JobManager.getRemoteJobManagerAkkaURL(
-				AkkaUtils.getAkkaProtocol(config),
-				NetUtils.unresolvedHostAndPortToNormalizedString("localhost", port),
-				Option.<String>empty());
+		return AkkaRpcServiceUtils.getRpcUrl(
+			"localhost",
+			port,
+			JobMaster.JOB_MANAGER_NAME,
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION,
+			config);
 	}
 
 	@Override
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala
index 0daac2eb1119d..2404fb9265df5 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala
@@ -20,7 +20,10 @@ package org.apache.flink.runtime.akka
 
 import java.net.InetSocketAddress
 
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution
 import org.apache.flink.runtime.jobmanager.JobManager
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils.AkkaProtocol
 import org.apache.flink.util.NetUtils
 import org.junit.runner.RunWith
 import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers}
@@ -38,19 +41,21 @@ class AkkaUtilsTest
 
     val address = new InetSocketAddress(host, port)
 
-    val remoteAkkaURL = JobManager.getRemoteJobManagerAkkaURL(
-      "akka.tcp",
-      NetUtils.unresolvedHostAndPortToNormalizedString(host, port),
-      Some("actor"))
+    val remoteAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
+      host,
+      port,
+      "actor",
+      AddressResolution.NO_ADDRESS_RESOLUTION,
+      AkkaProtocol.TCP)
 
-    val result = AkkaUtils.getInetSockeAddressFromAkkaURL(remoteAkkaURL)
+    val result = AkkaUtils.getInetSockeAddressFromAkkaURL(remoteAkkaUrl)
 
     result should equal(address)
   }
 
   test("getHostFromAkkaURL should throw an exception if the InetSocketAddress cannot be " +
     "retrieved") {
-    val localAkkaURL = JobManager.getLocalJobManagerAkkaURL(Some("actor"))
+    val localAkkaURL = AkkaUtils.getLocalAkkaURL("actor")
 
     intercept[Exception] {
       AkkaUtils.getInetSockeAddressFromAkkaURL(localAkkaURL)
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerConnectionTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerConnectionTest.scala
index 1489fb2ba71aa..97a001d07d554 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerConnectionTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerConnectionTest.scala
@@ -59,7 +59,10 @@ class JobManagerConnectionTest {
       mustReturnWithinTimeout(Duration(5*timeout, TimeUnit.MILLISECONDS)) {
         () => {
           try {
-            JobManager.getJobManagerActorRef(endpoint, actorSystem, config)
+            AkkaUtils.getActorRef(
+              endpoint,
+              actorSystem,
+              AkkaUtils.getLookupTimeout(config))
             fail("Should fail since the JobManager is not reachable")
           }
           catch {
@@ -95,7 +98,10 @@ class JobManagerConnectionTest {
       mustReturnWithinTimeout(Duration(5*timeout, TimeUnit.MILLISECONDS)) {
         () => {
           try {
-            JobManager.getJobManagerActorRef(endpoint, actorSystem, config)
+            AkkaUtils.getActorRef(
+              endpoint,
+              actorSystem,
+              AkkaUtils.getLookupTimeout(config))
             fail("Should fail since the JobManager is not reachable")
           }
           catch {
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index 876e26b1bb6db..d139a3f032e66 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -35,10 +35,12 @@ import org.apache.flink.runtime.client.JobClient
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager
 import org.apache.flink.runtime.clusterframework.types.ResourceID
 import org.apache.flink.runtime.concurrent.{ScheduledExecutor, ScheduledExecutorServiceAdapter}
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices
+import org.apache.flink.runtime.highavailability.{HighAvailabilityServices, HighAvailabilityServicesUtils}
 import org.apache.flink.runtime.instance.{ActorGateway, AkkaActorGateway}
 import org.apache.flink.runtime.jobgraph.JobGraph
 import org.apache.flink.runtime.jobmanager.{JobManager, MemoryArchivist}
+import org.apache.flink.runtime.jobmaster.JobMaster
+import org.apache.flink.runtime.jobmaster.JobMaster.{ARCHIVE_NAME, JOB_MANAGER_NAME}
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService
 import org.apache.flink.runtime.messages.TaskManagerMessages.NotifyWhenRegisteredAtJobManager
 import org.apache.flink.runtime.taskmanager.TaskManager
@@ -516,8 +518,8 @@ object TestingUtils {
         actorSystem,
         futureExecutor,
         ioExecutor,
-        Some(prefix + JobManager.JOB_MANAGER_NAME),
-        Some(prefix + JobManager.ARCHIVE_NAME),
+        Some(prefix + JobMaster.JOB_MANAGER_NAME),
+        Some(prefix + JobMaster.ARCHIVE_NAME),
         jobManagerClass,
         classOf[MemoryArchivist])
 
@@ -605,7 +607,6 @@ object TestingUtils {
     new AkkaActorGateway(actor, HighAvailabilityServices.DEFAULT_LEADER_ID)
   }
 
-
   class ForwardingActor(val target: ActorRef, val leaderSessionID: Option[UUID])
     extends FlinkActor with LeaderSessionMessageFilter with LogMessages {
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
index df4f370883ec0..5eadba6203486 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
@@ -275,7 +275,8 @@ public void run() {
 			try {
 				LeaderRetrievalService lrService =
 						LeaderRetrievalUtils.createLeaderRetrievalService(
-								cluster.configuration());
+							cluster.configuration(),
+							false);
 
 				JobExecutionResult result = JobClient.submitJobAndWait(
 						clientActorSystem,
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index 546e3d7ccdb76..bc1af65ebf8dd 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -138,7 +138,7 @@ public void testMultipleAMKill() throws Exception {
 						@Override
 						protected void run() {
 							try {
-								LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config);
+								LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
 								ActorGateway gateway = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, actorSystem, timeout);
 								ActorGateway selfGateway = new AkkaActorGateway(getRef(), gateway.leaderSessionID());
 
@@ -158,7 +158,7 @@ protected void run() {
 					@Override
 					protected void run() {
 						try {
-							LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config);
+							LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
 							ActorGateway gateway2 = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, actorSystem, timeout);
 							ActorGateway selfGateway = new AkkaActorGateway(getRef(), gateway2.leaderSessionID());
 							gateway2.tell(new TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered(1), selfGateway);
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
index 492fc0bf08663..ee87cfe147a67 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
@@ -32,6 +32,7 @@
 import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
+import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.net.SSLUtils;
 import org.apache.flink.runtime.process.ProcessReaper;
@@ -341,7 +342,7 @@ protected int runApplicationMaster(Configuration config) {
 				actorSystem,
 				futureExecutor,
 				ioExecutor,
-				new Some<>(JobManager.JOB_MANAGER_NAME()),
+				new Some<>(JobMaster.JOB_MANAGER_NAME),
 				Option.<String>empty(),
 				getJobManagerClass(),
 				getArchivistClass())._1();
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java
index cd447d756c331..f044cdd5198f5 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java
@@ -564,7 +564,7 @@ public ActorRef get() {
 				/* The leader retrieval service for connecting to the cluster and finding the active leader. */
 				LeaderRetrievalService leaderRetrievalService;
 				try {
-					leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig);
+					leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true);
 				} catch (Exception e) {
 					throw new RuntimeException("Could not create the leader retrieval service.", e);
 				}
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
index 4e178e6024568..05b2be8cd2c64 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
@@ -136,12 +136,15 @@ protected int runApplicationMaster(Configuration config) {
 
 			synchronized (lock) {
 				LOG.info("Starting High Availability Services");
-				haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config);
+				commonRpcService = createRpcService(config, appMasterHostname, amPortRange);
+
+				haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+					config,
+					commonRpcService.getExecutor());
 
 				heartbeatServices = HeartbeatServices.fromConfiguration(config);
 				
 				metricRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
-				commonRpcService = createRpcService(config, appMasterHostname, amPortRange);
 
 				// ---- (2) init resource manager -------
 				resourceManager = createResourceManager(config);
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskExecutorRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskExecutorRunner.java
index 23697656c6f59..1da52d496242f 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskExecutorRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskExecutorRunner.java
@@ -208,13 +208,18 @@ protected int runTaskExecutor(Configuration config) {
 			ResourceID resourceID = new ResourceID(containerId);
 			LOG.info("YARN assigned resource id {} for the task executor.", resourceID.toString());
 
-			haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config);
+			taskExecutorRpcService = TaskManagerRunner.createRpcService(config, haServices);
+
+			haServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				config,
+				taskExecutorRpcService.getExecutor(),
+				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);
+
 			HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(config);
 
 			metricRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
 
 			// ---- (2) init task manager runner -------
-			taskExecutorRpcService = TaskManagerRunner.createRpcService(config, haServices);
 			taskManagerRunner = new TaskManagerRunner(
 				config,
 				resourceID,
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnHighAvailabilityServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnHighAvailabilityServices.java
index 4c787267c516b..e9c3904b4220c 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnHighAvailabilityServices.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnHighAvailabilityServices.java
@@ -26,6 +26,7 @@
 import org.apache.flink.runtime.blob.FileSystemBlobStore;
 import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.InstantiationUtil;
@@ -331,7 +332,10 @@ public static YarnHighAvailabilityServices forYarnTaskManager(
 		final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
 		switch (mode) {
 			case NONE:
-				return new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
+				return new YarnPreConfiguredMasterNonHaServices(
+					flinkConfig,
+					hadoopConfig,
+					HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);
 
 			case ZOOKEEPER:
 				throw  new UnsupportedOperationException("to be implemented");
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServices.java
index fd1a45e8af7db..abfdb5ccd28e4 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServices.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServices.java
@@ -23,7 +23,7 @@
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.highavailability.ServicesThreadFactory;
-import org.apache.flink.runtime.highavailability.leaderelection.SingleLeaderElectionService;
+import org.apache.flink.runtime.highavailability.nonha.leaderelection.SingleLeaderElectionService;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java
index 72d780f4595b7..9d05bbec06bc7 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java
@@ -21,12 +21,13 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
-import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
-import org.apache.flink.runtime.rpc.RpcServiceUtils;
+import org.apache.flink.runtime.resourcemanager.ResourceManager;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.yarn.configuration.YarnConfigOptions;
 
 import java.io.IOException;
@@ -82,7 +83,8 @@ public class YarnPreConfiguredMasterNonHaServices extends AbstractYarnNonHaServi
 	 */
 	public YarnPreConfiguredMasterNonHaServices(
 			Configuration config,
-			org.apache.hadoop.conf.Configuration hadoopConf) throws IOException {
+			org.apache.hadoop.conf.Configuration hadoopConf,
+			HighAvailabilityServicesUtils.AddressResolution addressResolution) throws IOException {
 
 		super(config, hadoopConf);
 
@@ -107,8 +109,12 @@ public YarnPreConfiguredMasterNonHaServices(
 						YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' - port must be in [1, 65535]");
 			}
 
-			this.resourceManagerRpcUrl = RpcServiceUtils.getRpcUrl(
-					rmHost, rmPort, FlinkResourceManager.RESOURCE_MANAGER_NAME, config);
+			this.resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
+				rmHost,
+				rmPort,
+				ResourceManager.RESOURCE_MANAGER_NAME,
+				addressResolution,
+				config);
 
 			// all well!
 			successful = true;
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
index c7301029b768f..83ec37da98028 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
@@ -23,6 +23,7 @@
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.util.TestLogger;
 import org.apache.flink.yarn.configuration.YarnConfigOptions;
 
@@ -93,7 +94,10 @@ public void testMissingRmConfiguration() throws Exception {
 
 		// missing resource manager address
 		try {
-			new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
+			new YarnPreConfiguredMasterNonHaServices(
+				flinkConfig,
+				hadoopConfig,
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 			fail();
 		} catch (IllegalConfigurationException e) {
 			// expected
@@ -103,7 +107,10 @@ public void testMissingRmConfiguration() throws Exception {
 
 		// missing resource manager port
 		try {
-			new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
+			new YarnPreConfiguredMasterNonHaServices(
+				flinkConfig,
+				hadoopConfig,
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 			fail();
 		} catch (IllegalConfigurationException e) {
 			// expected
@@ -112,7 +119,10 @@ public void testMissingRmConfiguration() throws Exception {
 		flinkConfig.setInteger(YarnConfigOptions.APP_MASTER_RPC_PORT, 1427);
 
 		// now everything is good ;-)
-		new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig).closeAndCleanupAllData();
+		new YarnPreConfiguredMasterNonHaServices(
+			flinkConfig,
+			hadoopConfig,
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION).closeAndCleanupAllData();
 	}
 
 	@Test
@@ -122,7 +132,10 @@ public void testCloseAndCleanup() throws Exception {
 		flinkConfig.setInteger(YarnConfigOptions.APP_MASTER_RPC_PORT, 1427);
 
 		// create the services
-		YarnHighAvailabilityServices services = new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
+		YarnHighAvailabilityServices services = new YarnPreConfiguredMasterNonHaServices(
+			flinkConfig,
+			hadoopConfig,
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 		services.closeAndCleanupAllData();
 
 		final FileSystem fileSystem = HDFS_ROOT_PATH.getFileSystem();
@@ -153,7 +166,10 @@ public void testCallsOnClosedServices() throws Exception {
 		flinkConfig.setString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS, "localhost");
 		flinkConfig.setInteger(YarnConfigOptions.APP_MASTER_RPC_PORT, 1427);
 
-		YarnHighAvailabilityServices services = new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
+		YarnHighAvailabilityServices services = new YarnPreConfiguredMasterNonHaServices(
+			flinkConfig,
+			hadoopConfig,
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
 		// this method is not supported
 		try {

From 1c5ab425813bcd671f65bf8517df4954d62aebf8 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Thu, 16 Mar 2017 17:03:03 +0100
Subject: [PATCH 145/253] [FLINK-6078] Remove CuratorFramework#close calls from
 ZooKeeper based HA services

Remove client less factory methods from ZooKeeperUtils

Introduce default job id

This closes #3781.
---
 .../org/apache/flink/client/CliFrontend.java  |   6 +-
 .../flink/client/cli/CustomCommandLine.java   |   4 +-
 .../flink/client/program/ClusterClient.java   |  89 ++++---
 .../program/StandaloneClusterClient.java      |   8 +-
 .../RemoteExecutorHostnameResolutionTest.java |  22 +-
 .../flink/client/program/ClientTest.java      |   5 +-
 ...etrievalServiceHostnameResolutionTest.java |  32 +--
 .../avro/AvroExternalJarProgramITCase.java    |  22 +-
 .../src/test/resources/log4j-test.properties  |   2 +-
 .../connectors/kafka/Kafka010ITCase.java      |  14 +-
 .../connectors/kafka/Kafka08ITCase.java       |   4 +-
 .../kafka/Kafka09SecuredRunITCase.java        |   2 +-
 .../kafka/KafkaConsumerTestBase.java          |  96 ++++----
 .../kafka/KafkaProducerTestBase.java          |   2 +-
 .../kafka/KafkaShortRetentionTestBase.java    |  18 +-
 .../connectors/kafka/KafkaTestBase.java       |  23 +-
 .../src/test/resources/log4j-test.properties  |   2 +-
 .../apache/flink/storm/api/FlinkClient.java   |   4 +-
 .../contrib/streaming/CollectITCase.java      |   9 +-
 .../DegreesWithExceptionITCase.java           |  42 ++--
 .../ReduceOnEdgesWithExceptionITCase.java     |  37 ++-
 .../ReduceOnNeighborsWithExceptionITCase.java |  43 ++--
 .../apache/flink/ml/util/FlinkTestBase.scala  |   2 +-
 .../src/test/resources/log4j-test.properties  |  38 +++
 .../src/test/resources/logback-test.xml       |  42 ++++
 .../MesosApplicationMasterRunner.java         |  39 +++-
 .../MesosFlinkResourceManagerTest.java        |  37 ++-
 .../BackPressureStatsTrackerITCase.java       |  19 +-
 .../StackTraceSampleCoordinatorITCase.java    |  19 +-
 .../webmonitor/WebRuntimeMonitorITCase.java   |  31 ++-
 .../ZooKeeperCheckpointRecoveryFactory.java   |   2 +-
 .../flink/runtime/client/JobClient.java       |  47 ++--
 .../flink/runtime/client/JobClientActor.java  |   5 +-
 .../runtime/client/JobListeningContext.java   |  22 +-
 .../clusterframework/BootstrapTools.java      |  17 +-
 .../HighAvailabilityServices.java             |   7 +
 .../HighAvailabilityServicesUtils.java        |   2 +-
 .../flink/runtime/jobgraph/JobGraph.java      |   9 +
 .../ZooKeeperSubmittedJobGraphStore.java      |  12 +-
 .../ZooKeeperLeaderElectionService.java       | 184 +++++++++------
 .../ZooKeeperLeaderRetrievalService.java      | 122 ++++++----
 .../minicluster/StandaloneMiniCluster.java    | 154 +++++++++++++
 .../runtime/query/QueryableStateClient.java   |  10 +-
 .../runtime/rpc/akka/AkkaRpcServiceUtils.java |  16 +-
 .../runtime/util/LeaderRetrievalUtils.java    |  67 ++----
 .../runtime/util/SerializedThrowable.java     |   4 +
 .../flink/runtime/util/ZooKeeperUtils.java    |  27 ---
 .../flink/runtime/jobmanager/JobManager.scala | 158 ++++++-------
 .../messages/TaskManagerMessages.scala        |  13 +-
 .../minicluster/FlinkMiniCluster.scala        | 142 +++++++-----
 .../minicluster/LocalFlinkMiniCluster.scala   |  41 ++--
 .../runtime/taskmanager/TaskManager.scala     |  71 +++---
 .../checkpoint/CoordinatorShutdownTest.java   |   3 +-
 .../runtime/client/JobClientActorTest.java    |   8 +-
 .../ClusterShutdownITCase.java                | 201 ++++++++++------
 .../ResourceManagerITCase.java                | 178 +++++++++-----
 .../clusterframework/ResourceManagerTest.java | 128 ++++++++++-
 .../highavailability/ManualLeaderService.java | 116 ++++++++++
 ...TestingManualHighAvailabilityServices.java | 150 ++++++++++++
 .../jobmanager/JobManagerHARecoveryTest.java  |  35 +--
 .../JobManagerProcessReapingTest.java         |   3 +-
 .../jobmanager/JobManagerStartupTest.java     |   5 +-
 .../runtime/jobmanager/JobManagerTest.java    | 188 +++++++++------
 .../runtime/jobmanager/JobSubmitTest.java     |  15 +-
 .../jobmaster/JobManagerRunnerMockTest.java   |   3 +-
 .../LeaderChangeJobRecoveryTest.java          |  32 ++-
 .../LeaderChangeStateCleanupTest.java         |  50 ++--
 ...LeaderElectionRetrievalTestingCluster.java | 121 ----------
 .../TestingLeaderElectionService.java         |   4 +
 .../TestingLeaderRetrievalService.java        |   4 +
 .../ZooKeeperLeaderElectionTest.java          | 109 +++++----
 .../ZooKeeperLeaderRetrievalTest.java         |  91 ++++----
 .../metrics/TaskManagerMetricsTest.java       |  26 ++-
 ...kManagerComponentsStartupShutdownTest.java |  47 ++--
 .../TaskManagerConfigurationTest.java         |  76 ++++--
 .../TaskManagerProcessReapingTestBase.java    |  73 ++++--
 .../TaskManagerRegistrationTest.java          | 217 ++++++++++--------
 .../taskmanager/TaskManagerStartupTest.java   |  62 ++++-
 .../runtime/taskmanager/TaskManagerTest.java  | 136 +++++++++--
 .../JobManagerRegistrationTest.scala          |  88 ++++---
 .../runtime/testingUtils/TestingCluster.scala |  37 ++-
 .../runtime/testingUtils/TestingUtils.scala   | 176 +++++---------
 .../apache/flink/api/scala/FlinkShell.scala   |  14 +-
 .../flink/api/scala/ScalaShellITCase.scala    | 104 ++++-----
 .../environment/RemoteStreamEnvironment.java  |   6 +-
 .../streaming/util/TestStreamEnvironment.java |  79 ++++++-
 .../flink/test/util/JavaProgramTestBase.java  |   4 +-
 .../test/util/MultipleProgramsTestBase.java   |   2 +-
 .../flink/test/util/TestEnvironment.java      | 133 +++++++++--
 .../accumulators/AccumulatorErrorITCase.java  |  49 ++--
 ...actEventTimeWindowCheckpointingITCase.java |  21 +-
 ...EventTimeAllWindowCheckpointingITCase.java |  21 +-
 .../StreamFaultToleranceTestBase.java         |   4 +-
 .../WindowCheckpointingITCase.java            |  18 +-
 .../test/classloading/ClassLoaderITCase.java  | 136 ++++++-----
 .../jar/CheckpointedStreamingProgram.java     |   9 +-
 .../CheckpointingCustomKvStateProgram.java    |   9 +-
 .../jar/CustomInputSplitProgram.java          |  12 +-
 .../jar/CustomKvStateProgram.java             |  15 +-
 .../test/classloading/jar/KMeansForTest.java  |  17 +-
 .../LegacyCheckpointedStreamingProgram.java   |   6 +-
 .../jar/StreamingCustomInputSplitProgram.java |  11 +-
 .../classloading/jar/StreamingProgram.java    |   7 +-
 .../test/classloading/jar/UserCodeType.java   |   6 +-
 .../clients/examples/JobRetrievalITCase.java  |   7 +-
 .../CustomDistributionITCase.java             |   2 +-
 .../RemoteEnvironmentITCase.java              |  46 ++--
 .../test/misc/AutoParallelismITCase.java      |   8 +-
 .../test/misc/CustomSerializationITCase.java  |  57 ++---
 .../test/misc/MiscellaneousIssuesITCase.java  |  52 ++---
 ...ccessAfterNetworkBuffersFailureITCase.java |  28 +--
 .../query/AbstractQueryableStateITCase.java   |  33 ++-
 ...TaskManagerProcessFailureRecoveryTest.java |  18 +-
 .../test/recovery/ChaosMonkeyITCase.java      |  13 +-
 .../test/recovery/FastFailuresITCase.java     |  10 +-
 .../JobManagerHACheckpointRecoveryITCase.java |  42 +++-
 .../JobManagerHAJobGraphRecoveryITCase.java   |  24 +-
 ...erHAProcessFailureBatchRecoveryITCase.java |  28 ++-
 .../ProcessFailureCancelingITCase.java        |  30 ++-
 .../TaskManagerFailureRecoveryITCase.java     |   4 +-
 .../ZooKeeperLeaderElectionITCase.java        |  17 +-
 .../AbstractOperatorRestoreTestBase.java      |  39 +++-
 .../streaming/runtime/TimestampITCase.java    |  63 ++---
 .../jobmanager/JobManagerFailsITCase.scala    |  17 +-
 .../taskmanager/TaskManagerFailsITCase.scala  |   2 +-
 .../flink/yarn/FlinkYarnSessionCliTest.java   |   7 +-
 .../yarn/YARNHighAvailabilityITCase.java      |  40 +++-
 .../flink/yarn/YARNSessionFIFOITCase.java     |   2 +-
 .../yarn/AbstractYarnClusterDescriptor.java   |   2 +-
 .../yarn/YarnApplicationMasterRunner.java     |  38 ++-
 .../apache/flink/yarn/YarnClusterClient.java  |  36 +--
 .../flink/yarn/YarnClusterClientV2.java       |   5 +-
 .../YarnFlinkApplicationMasterRunner.java     |   5 +-
 .../apache/flink/yarn/cli/FlinkYarnCLI.java   |  13 +-
 .../flink/yarn/cli/FlinkYarnSessionCli.java   |   7 +-
 135 files changed, 3474 insertions(+), 2131 deletions(-)
 create mode 100644 flink-libraries/flink-python/src/test/resources/log4j-test.properties
 create mode 100644 flink-libraries/flink-python/src/test/resources/logback-test.xml
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingManualHighAvailabilityServices.java
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionRetrievalTestingCluster.java

diff --git a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java
index 0d61cbda879ca..74d5f5de0b28d 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java
@@ -263,7 +263,11 @@ protected int run(String[] args) {
 		}
 		finally {
 			if (client != null) {
-				client.shutdown();
+				try {
+					client.shutdown();
+				} catch (Exception e) {
+					LOG.warn("Could not properly shut down the cluster client.", e);
+				}
 			}
 			if (program != null) {
 				program.deleteExtractedLibraries();
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
index c58c74c8e824a..a4cb47993a6db 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CustomCommandLine.java
@@ -75,11 +75,11 @@ ClusterType retrieveCluster(
 	 * @param config The Flink config to use
 	 * @param userJarFiles User jar files to include in the classpath of the cluster.
 	 * @return The client to communicate with the cluster which the CustomCommandLine brought up.
-	 * @throws UnsupportedOperationException if the operation is not supported
+	 * @throws Exception if the cluster could not be created
 	 */
 	ClusterType createCluster(
 			String applicationName,
 			CommandLine commandLine,
 			Configuration config,
-			List<URL> userJarFiles) throws UnsupportedOperationException;
+			List<URL> userJarFiles) throws Exception;
 }
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
index 0f88f7cf9b4ce..6770eee089cf4 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
@@ -42,6 +42,9 @@
 import org.apache.flink.runtime.client.JobListeningContext;
 import org.apache.flink.runtime.client.JobRetrievalException;
 import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
@@ -72,7 +75,6 @@
 import java.util.List;
 import java.util.Map;
 
-
 /**
  * Encapsulates the functionality necessary to submit a program to a remote cluster.
  */
@@ -95,6 +97,9 @@ public abstract class ClusterClient {
 	/** Lookup timeout for the job manager retrieval service */
 	private final FiniteDuration lookupTimeout;
 
+	/** Service factory for high available */
+	protected final HighAvailabilityServices highAvailabilityServices;
+
 	/** Flag indicating whether to sysout print execution updates */
 	private boolean printStatusDuringExecution = true;
 
@@ -119,10 +124,25 @@ public abstract class ClusterClient {
 	 *
 	 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 	 *
-	 * @throws java.io.IOException Thrown, if the client's actor system could not be started.
+	 * @throws Exception we cannot create the high availability services
 	 */
-	public ClusterClient(Configuration flinkConfig) throws IOException {
+	public ClusterClient(Configuration flinkConfig) throws Exception {
+		this(flinkConfig,
+			HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				flinkConfig,
+				Executors.directExecutor(),
+				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION));
+	}
 
+	/**
+	 * Creates a instance that submits the programs to the JobManager defined in the
+	 * configuration. This method will try to resolve the JobManager hostname and throw an exception
+	 * if that is not possible.
+	 *
+	 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
+	 * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval
+	 */
+	public ClusterClient(Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices) {
 		this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
 		this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);
 
@@ -130,6 +150,8 @@ public ClusterClient(Configuration flinkConfig) throws IOException {
 		this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig);
 
 		this.actorSystemLoader = new LazyActorSystemLoader(flinkConfig, LOG);
+
+		this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
 	}
 
 	// ------------------------------------------------------------------------
@@ -202,12 +224,16 @@ public ActorSystem get() {
 	/**
 	 * Shuts down the client. This stops the internal actor system and actors.
 	 */
-	public void shutdown() {
+	public void shutdown() throws Exception {
 		synchronized (this) {
 			try {
 				finalizeCluster();
 			} finally {
-				this.actorSystemLoader.shutdown();
+				actorSystemLoader.shutdown();
+			}
+
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
 			}
 		}
 	}
@@ -241,7 +267,8 @@ public InetSocketAddress getJobManagerAddress() {
 		try {
 			LeaderConnectionInfo leaderConnectionInfo =
 				LeaderRetrievalUtils.retrieveLeaderConnectionInfo(
-					LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true), timeout);
+					highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+					timeout);
 
 			return AkkaUtils.getInetSockeAddressFromAkkaURL(leaderConnectionInfo.getAddress());
 		} catch (Exception e) {
@@ -411,17 +438,17 @@ public JobExecutionResult run(JobGraph jobGraph, ClassLoader classLoader) throws
 
 		waitForClusterToBeReady();
 
-		final LeaderRetrievalService leaderRetrievalService;
-		try {
-			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true);
-		} catch (Exception e) {
-			throw new ProgramInvocationException("Could not create the leader retrieval service", e);
-		}
-
 		try {
 			logAndSysout("Submitting job with JobID: " + jobGraph.getJobID() + ". Waiting for job completion.");
-			this.lastJobExecutionResult = JobClient.submitJobAndWait(actorSystemLoader.get(), flinkConfig,
-				leaderRetrievalService, jobGraph, timeout, printStatusDuringExecution, classLoader);
+			this.lastJobExecutionResult = JobClient.submitJobAndWait(
+				actorSystemLoader.get(),
+				flinkConfig,
+				highAvailabilityServices,
+				jobGraph,
+				timeout,
+				printStatusDuringExecution,
+				classLoader);
+
 			return this.lastJobExecutionResult;
 		} catch (JobExecutionException e) {
 			throw new ProgramInvocationException("The program execution failed: " + e.getMessage(), e);
@@ -462,13 +489,6 @@ public JobSubmissionResult runDetached(JobGraph jobGraph, ClassLoader classLoade
 	 * @throws JobExecutionException if an error occurs during monitoring the job execution
 	 */
 	public JobExecutionResult retrieveJob(JobID jobID) throws JobExecutionException {
-		final LeaderRetrievalService leaderRetrievalService;
-		try {
-			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true);
-		} catch (Exception e) {
-			throw new JobRetrievalException(jobID, "Could not create the leader retrieval service", e);
-		}
-
 		ActorGateway jobManagerGateway;
 		try {
 			jobManagerGateway = getJobManagerGateway();
@@ -477,13 +497,13 @@ public JobExecutionResult retrieveJob(JobID jobID) throws JobExecutionException
 		}
 
 		final JobListeningContext listeningContext = JobClient.attachToRunningJob(
-				jobID,
-				jobManagerGateway,
-				flinkConfig,
-				actorSystemLoader.get(),
-				leaderRetrievalService,
-				timeout,
-				printStatusDuringExecution);
+			jobID,
+			jobManagerGateway,
+			flinkConfig,
+			actorSystemLoader.get(),
+			highAvailabilityServices,
+			timeout,
+			printStatusDuringExecution);
 
 		return JobClient.awaitJobResult(listeningContext);
 	}
@@ -496,13 +516,6 @@ public JobExecutionResult retrieveJob(JobID jobID) throws JobExecutionException
 	 * @throws JobExecutionException if an error occurs during monitoring the job execution
 	 */
 	public JobListeningContext connectToJob(JobID jobID) throws JobExecutionException {
-		final LeaderRetrievalService leaderRetrievalService;
-		try {
-			leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true);
-		} catch (Exception e) {
-			throw new JobRetrievalException(jobID, "Could not create the leader retrieval service", e);
-		}
-
 		ActorGateway jobManagerGateway;
 		try {
 			jobManagerGateway = getJobManagerGateway();
@@ -515,7 +528,7 @@ public JobListeningContext connectToJob(JobID jobID) throws JobExecutionExceptio
 				jobManagerGateway,
 				flinkConfig,
 				actorSystemLoader.get(),
-				leaderRetrievalService,
+				highAvailabilityServices,
 				timeout,
 				printStatusDuringExecution);
 	}
@@ -721,7 +734,7 @@ private JobGraph getJobGraph(FlinkPlan optPlan, List<URL> jarFiles, List<URL> cl
 	public ActorGateway getJobManagerGateway() throws Exception {
 		LOG.debug("Looking up JobManager");
 		return LeaderRetrievalUtils.retrieveLeaderGateway(
-			LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true),
+			highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 			actorSystemLoader.get(),
 			lookupTimeout);
 	}
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java
index 296ddc96367dd..fd179c0e0a966 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java
@@ -22,12 +22,12 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.clusterframework.messages.GetClusterStatus;
 import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import scala.concurrent.Await;
 import scala.concurrent.Future;
 
-import java.io.IOException;
 import java.net.URL;
 import java.util.Collections;
 import java.util.List;
@@ -38,10 +38,14 @@
  */
 public class StandaloneClusterClient extends ClusterClient {
 
-	public StandaloneClusterClient(Configuration config) throws IOException {
+	public StandaloneClusterClient(Configuration config) throws Exception {
 		super(config);
 	}
 
+	public StandaloneClusterClient(Configuration config, HighAvailabilityServices highAvailabilityServices) {
+		super(config, highAvailabilityServices);
+	}
+
 	@Override
 	public void waitForClusterToBeReady() {}
 
diff --git a/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java b/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java
index d8fb3de09aa5c..be939492cbee1 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/RemoteExecutorHostnameResolutionTest.java
@@ -21,7 +21,6 @@
 import org.apache.flink.api.common.Plan;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.util.TestLogger;
 import org.junit.BeforeClass;
@@ -33,7 +32,6 @@
 import java.net.UnknownHostException;
 import java.util.Collections;
 
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 
@@ -48,26 +46,20 @@ public static void check() {
 	}
 
 	@Test
-	public void testUnresolvableHostname1() {
+	public void testUnresolvableHostname1() throws Exception {
 
 		RemoteExecutor exec = new RemoteExecutor(nonExistingHostname, port);
 		try {
 			exec.executePlan(getProgram());
 			fail("This should fail with an ProgramInvocationException");
 		}
-		catch (ProgramInvocationException e) {
+		catch (UnknownHostException ignored) {
 			// that is what we want!
-			assertTrue(e.getCause() instanceof UnknownHostException);
-		}
-		catch (Exception e) {
-			System.err.println("Wrong exception!");
-			e.printStackTrace();
-			fail(e.getMessage());
 		}
 	}
 
 	@Test
-	public void testUnresolvableHostname2() {
+	public void testUnresolvableHostname2() throws Exception {
 
 		InetSocketAddress add = new InetSocketAddress(nonExistingHostname, port);
 		RemoteExecutor exec = new RemoteExecutor(add, new Configuration(),
@@ -76,14 +68,8 @@ public void testUnresolvableHostname2() {
 			exec.executePlan(getProgram());
 			fail("This should fail with an ProgramInvocationException");
 		}
-		catch (ProgramInvocationException e) {
+		catch (UnknownHostException ignored) {
 			// that is what we want!
-			assertTrue(e.getCause() instanceof UnknownHostException);
-		}
-		catch (Exception e) {
-			System.err.println("Wrong exception!");
-			e.printStackTrace();
-			fail(e.getMessage());
 		}
 	}
 	
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
index da297d6041cdf..b7ade2a5e09fe 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
@@ -56,7 +56,6 @@
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
-import java.io.IOException;
 import java.net.URL;
 import java.util.Collections;
 import java.util.UUID;
@@ -199,7 +198,7 @@ public void testDetachedMode() throws Exception{
 	 * This test verifies correct job submission messaging logic and plan translation calls.
 	 */
 	@Test
-	public void shouldSubmitToJobClient() throws IOException, ProgramInvocationException {
+	public void shouldSubmitToJobClient() throws Exception {
 		jobManagerSystem.actorOf(
 			Props.create(SuccessReturningActor.class),
 			JobMaster.JOB_MANAGER_NAME);
@@ -217,7 +216,7 @@ public void shouldSubmitToJobClient() throws IOException, ProgramInvocationExcep
 	 * This test verifies correct that the correct exception is thrown when the job submission fails.
 	 */
 	@Test
-	public void shouldSubmitToJobClientFails() throws IOException {
+	public void shouldSubmitToJobClientFails() throws Exception {
 			jobManagerSystem.actorOf(
 				Props.create(FailureReturningActor.class),
 				JobMaster.JOB_MANAGER_NAME);
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java
index fc10f65e59ce5..0ecdc2cf5cb65 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/LeaderRetrievalServiceHostnameResolutionTest.java
@@ -20,7 +20,9 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
+import org.apache.flink.runtime.jobmaster.JobMaster;
+import org.apache.flink.runtime.util.StandaloneUtils;
+import org.apache.flink.util.ConfigurationException;
 import org.apache.flink.util.TestLogger;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -32,7 +34,7 @@
 import static org.junit.Assume.assumeTrue;
 
 /**
- * Tests that verify that the LeaderRetrievalSevice correctly handles non-resolvable host names
+ * Tests that verify that the LeaderRetrievalService correctly handles non-resolvable host names
  * and does not fail with another exception
  */
 public class LeaderRetrievalServiceHostnameResolutionTest extends TestLogger {
@@ -48,21 +50,16 @@ public static void check() {
 	 * Tests that the StandaloneLeaderRetrievalService resolves host names if specified.
 	 */
 	@Test
-	public void testUnresolvableHostname1() {
+	public void testUnresolvableHostname1() throws UnknownHostException, ConfigurationException {
+		Configuration config = new Configuration();
 
-		try {
-			Configuration config = new Configuration();
+		config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, nonExistingHostname);
+		config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 17234);
 
-			config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, nonExistingHostname);
-			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 17234);
-
-			LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
-		}
-		catch (Exception e) {
-			System.err.println("Shouldn't throw an exception!");
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
+		StandaloneUtils.createLeaderRetrievalService(
+			config,
+			false,
+			JobMaster.JOB_MANAGER_NAME);
 	}
 
 	/*
@@ -77,7 +74,10 @@ public void testUnresolvableHostname2() throws Exception {
 			config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, nonExistingHostname);
 			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 17234);
 
-			LeaderRetrievalUtils.createLeaderRetrievalService(config, true);
+			StandaloneUtils.createLeaderRetrievalService(
+				config,
+				true,
+				JobMaster.JOB_MANAGER_NAME);
 			fail("This should fail with an UnknownHostException");
 		}
 		catch (UnknownHostException e) {
diff --git a/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
index 5f5209a69dac5..063a363214140 100644
--- a/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
+++ b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
@@ -19,14 +19,16 @@
 package org.apache.flink.api.avro;
 
 import java.io.File;
+import java.net.URL;
+import java.util.Collections;
 
-import org.apache.flink.client.program.ClusterClient;
 import org.apache.flink.client.program.PackagedProgram;
-import org.apache.flink.client.program.StandaloneClusterClient;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.TestLogger;
 import org.junit.Assert;
 import org.junit.Test;
@@ -43,8 +45,9 @@ public void testExternalProgram() {
 		LocalFlinkMiniCluster testMiniCluster = null;
 
 		try {
+			int parallelism = 4;
 			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
+			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, parallelism);
 			testMiniCluster = new LocalFlinkMiniCluster(config, false);
 			testMiniCluster.start();
 
@@ -53,15 +56,16 @@ public void testExternalProgram() {
 
 			PackagedProgram program = new PackagedProgram(new File(jarFile), new String[] { testData });
 
+			TestEnvironment.setAsContext(
+				testMiniCluster,
+				parallelism,
+				Collections.singleton(new Path(jarFile)),
+				Collections.<URL>emptyList());
 
 			config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
 			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, testMiniCluster.getLeaderRPCPort());
 
-			ClusterClient client = new StandaloneClusterClient(config);
-
-			client.setPrintStatusDuringExecution(false);
-			client.run(program, 4);
-
+			program.invokeInteractiveModeForExecution();
 		}
 		catch (Throwable t) {
 			System.err.println(t.getMessage());
@@ -69,6 +73,8 @@ public void testExternalProgram() {
 			Assert.fail("Error during the packaged program execution: " + t.getMessage());
 		}
 		finally {
+			TestEnvironment.unsetAsContext();
+
 			if (testMiniCluster != null) {
 				try {
 					testMiniCluster.stop();
diff --git a/flink-connectors/flink-avro/src/test/resources/log4j-test.properties b/flink-connectors/flink-avro/src/test/resources/log4j-test.properties
index 0b686e543bb23..881dc0609b8fd 100644
--- a/flink-connectors/flink-avro/src/test/resources/log4j-test.properties
+++ b/flink-connectors/flink-avro/src/test/resources/log4j-test.properties
@@ -24,4 +24,4 @@ log4j.appender.A1=org.apache.log4j.ConsoleAppender
 
 # A1 uses PatternLayout.
 log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java
index 208516934fae9..39b2b8f3d447c 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java
@@ -180,13 +180,14 @@ public void testTimestamps() throws Exception {
 
 		// ---------- Produce an event time stream into Kafka -------------------
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(1);
 		env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env.getConfig().disableSysoutLogging();
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 
 		DataStream<Long> streamWithTimestamps = env.addSource(new SourceFunction<Long>() {
+			private static final long serialVersionUID = -2255105836471289626L;
 			boolean running = true;
 
 			@Override
@@ -208,6 +209,8 @@ public void cancel() {
 
 		final TypeInformationSerializationSchema<Long> longSer = new TypeInformationSerializationSchema<>(TypeInfoParser.<Long>parse("Long"), env.getConfig());
 		FlinkKafkaProducer010.FlinkKafkaProducer010Configuration prod = FlinkKafkaProducer010.writeToKafkaWithTimestamps(streamWithTimestamps, topic, new KeyedSerializationSchemaWrapper<>(longSer), standardProps, new KafkaPartitioner<Long>() {
+			private static final long serialVersionUID = -6730989584364230617L;
+
 			@Override
 			public int partition(Long next, byte[] serializedKey, byte[] serializedValue, int numPartitions) {
 				return (int)(next % 3);
@@ -219,7 +222,7 @@ public int partition(Long next, byte[] serializedKey, byte[] serializedValue, in
 
 		// ---------- Consume stream from Kafka -------------------
 
-		env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(1);
 		env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env.getConfig().disableSysoutLogging();
@@ -227,6 +230,8 @@ public int partition(Long next, byte[] serializedKey, byte[] serializedValue, in
 
 		FlinkKafkaConsumer010<Long> kafkaSource = new FlinkKafkaConsumer010<>(topic, new LimitedLongDeserializer(), standardProps);
 		kafkaSource.assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Long>() {
+			private static final long serialVersionUID = -4834111073247835189L;
+
 			@Nullable
 			@Override
 			public Watermark checkAndGetNextWatermark(Long lastElement, long extractedTimestamp) {
@@ -253,8 +258,12 @@ public long extractTimestamp(Long element, long previousElementTimestamp) {
 
 	private static class TimestampValidatingOperator extends StreamSink<Long> {
 
+		private static final long serialVersionUID = 1353168781235526806L;
+
 		public TimestampValidatingOperator() {
 			super(new SinkFunction<Long>() {
+				private static final long serialVersionUID = -6676565693361786524L;
+
 				@Override
 				public void invoke(Long value) throws Exception {
 					throw new RuntimeException("Unexpected");
@@ -304,6 +313,7 @@ public void close() throws Exception {
 
 	private static class LimitedLongDeserializer implements KeyedDeserializationSchema<Long> {
 
+		private static final long serialVersionUID = 6966177118923713521L;
 		private final TypeInformation<Long> ti;
 		private final TypeSerializer<Long> ser;
 		long cnt = 0;
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
index f5cb8c026a704..8cc735d0a420f 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
@@ -87,7 +87,7 @@ public void testInvalidOffset() throws Exception {
 		final int valuesCount = 20;
 		final int startFrom = 0;
 
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 
 		readSequence(env, StartupMode.GROUP_OFFSETS, null, standardProps, parallelism, topic, valuesCount, startFrom);
@@ -190,7 +190,7 @@ public void testOffsetAutocommitTest() throws Exception {
 		// write a sequence from 0 to 99 to each of the 3 partitions.
 		final String topicName = writeSequence("testOffsetAutocommit", 100, parallelism, 1);
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		// NOTE: We are not enabling the checkpointing!
 		env.getConfig().disableSysoutLogging();
 		env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09SecuredRunITCase.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09SecuredRunITCase.java
index e7485377ef69e..16a13c061b695 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09SecuredRunITCase.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09SecuredRunITCase.java
@@ -34,7 +34,7 @@ public class Kafka09SecuredRunITCase extends KafkaConsumerTestBase {
 	protected static final Logger LOG = LoggerFactory.getLogger(Kafka09SecuredRunITCase.class);
 
 	@BeforeClass
-	public static void prepare() throws IOException, ClassNotFoundException {
+	public static void prepare() throws ClassNotFoundException {
 		LOG.info("-------------------------------------------------------------------------");
 		LOG.info("    Starting Kafka09SecuredRunITCase ");
 		LOG.info("-------------------------------------------------------------------------");
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
index ddac61c655ede..ba83460886543 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
@@ -154,7 +154,7 @@ public void runFailOnNoBrokerTest() throws Exception {
 		try {
 			Properties properties = new Properties();
 
-			StreamExecutionEnvironment see = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+			StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
 			see.getConfig().disableSysoutLogging();
 			see.setRestartStrategy(RestartStrategies.noRestart());
 			see.setParallelism(1);
@@ -173,22 +173,14 @@ public void runFailOnNoBrokerTest() throws Exception {
 			DataStream<String> stream = see.addSource(source);
 			stream.print();
 			see.execute("No broker test");
-		} catch(ProgramInvocationException pie) {
+		} catch(JobExecutionException jee) {
 			if(kafkaServer.getVersion().equals("0.9") || kafkaServer.getVersion().equals("0.10")) {
-				assertTrue(pie.getCause() instanceof JobExecutionException);
-
-				JobExecutionException jee = (JobExecutionException) pie.getCause();
-
 				assertTrue(jee.getCause() instanceof TimeoutException);
 
 				TimeoutException te = (TimeoutException) jee.getCause();
 
 				assertEquals("Timeout expired while fetching topic metadata", te.getMessage());
 			} else {
-				assertTrue(pie.getCause() instanceof JobExecutionException);
-
-				JobExecutionException jee = (JobExecutionException) pie.getCause();
-
 				assertTrue(jee.getCause() instanceof RuntimeException);
 
 				RuntimeException re = (RuntimeException) jee.getCause();
@@ -208,7 +200,7 @@ public void runCommitOffsetsToKafka() throws Exception {
 
 		final String topicName = writeSequence("testCommitOffsetsToKafkaTopic", recordsInEachPartition, parallelism, 1);
 
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env.setParallelism(parallelism);
@@ -280,6 +272,8 @@ public void run() {
 	public void runStartFromKafkaCommitOffsets() throws Exception {
 		final int parallelism = 3;
 		final int recordsInEachPartition = 300;
+		final int recordsToConsume = 150;
+		final int consumePause = 50;
 
 		final String topicName = writeSequence("testStartFromKafkaCommitOffsetsTopic", recordsInEachPartition, parallelism, 1);
 
@@ -294,7 +288,7 @@ public void runStartFromKafkaCommitOffsets() throws Exception {
 			attempt++;
 			LOG.info("Attempt " + attempt + " to read records and commit some offsets to Kafka");
 
-			final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+			final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 			env.getConfig().disableSysoutLogging();
 			env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 			env.setParallelism(parallelism);
@@ -302,13 +296,13 @@ public void runStartFromKafkaCommitOffsets() throws Exception {
 
 			env
 				.addSource(kafkaServer.getConsumer(topicName, new SimpleStringSchema(), standardProps))
-				.map(new ThrottledMapper<String>(50))
+				.map(new ThrottledMapper<String>(consumePause))
 				.map(new MapFunction<String, Object>() {
 					int count = 0;
 					@Override
 					public Object map(String value) throws Exception {
 						count++;
-						if (count == 150) {
+						if (count == recordsToConsume) {
 							throw new SuccessException();
 						}
 						return null;
@@ -329,7 +323,7 @@ public Object map(String value) throws Exception {
 
 		LOG.info("Got final committed offsets from Kafka o1={}, o2={}, o3={}", o1, o2, o3);
 
-		final StreamExecutionEnvironment env2 = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env2 = StreamExecutionEnvironment.getExecutionEnvironment();
 		env2.getConfig().disableSysoutLogging();
 		env2.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env2.setParallelism(parallelism);
@@ -375,7 +369,7 @@ public void runAutoOffsetRetrievalAndCommitToKafka() throws Exception {
 
 		final String topicName = writeSequence("testAutoOffsetRetrievalAndCommitToKafkaTopic", recordsInEachPartition, parallelism, 1);
 
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env.setParallelism(parallelism);
@@ -452,7 +446,7 @@ public void runStartFromEarliestOffsets() throws Exception {
 
 		final String topicName = writeSequence("testStartFromEarliestOffsetsTopic", recordsInEachPartition, parallelism, 1);
 
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		env.setParallelism(parallelism);
 
@@ -510,7 +504,7 @@ public void runStartFromLatestOffsets() throws Exception {
 				new TypeInformationSerializationSchema<>(resultType, new ExecutionConfig()));
 
 		// setup and run the latest-consuming job
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		env.setParallelism(parallelism);
 
@@ -541,7 +535,7 @@ public void run() {
 				try {
 					env.execute(consumeExtraRecordsJobName);
 				} catch (Throwable t) {
-					if (!(t.getCause() instanceof JobCancellationException)) {
+					if (!(t instanceof JobCancellationException)) {
 						error.set(t);
 					}
 				}
@@ -555,7 +549,9 @@ public void run() {
 			consumeExtraRecordsJobName);
 
 		// setup the extra records writing job
-		final StreamExecutionEnvironment env2 = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env2 = StreamExecutionEnvironment.getExecutionEnvironment();
+
+		env2.setParallelism(parallelism);
 
 		DataStream<Tuple2<Integer, Integer>> extraRecordsStream = env2
 			.addSource(new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {
@@ -577,7 +573,7 @@ public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
 				public void cancel() {
 					running = false;
 				}
-			}).setParallelism(parallelism);
+			});
 
 		kafkaServer.produceIntoKafka(extraRecordsStream, topicName, serSchema, readProps, null);
 
@@ -626,7 +622,7 @@ public void runStartFromGroupOffsets() throws Exception {
 
 		final String topicName = writeSequence("testStartFromGroupOffsetsTopic", recordsInEachPartition, parallelism, 1);
 
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		env.setParallelism(parallelism);
 
@@ -685,7 +681,7 @@ public void runStartFromSpecificOffsets() throws Exception {
 
 		final String topicName = writeSequence("testStartFromSpecificOffsetsTopic", recordsInEachPartition, parallelism, 1);
 
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		env.setParallelism(parallelism);
 
@@ -751,7 +747,7 @@ public void runSimpleConcurrentProducerConsumerTopology() throws Exception {
 		createTestTopic(additionalEmptyTopic, parallelism, 1); // create an empty topic which will remain empty all the time
 
 		final StreamExecutionEnvironment env =
-				StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+				StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(parallelism);
 		env.enableCheckpointing(500);
 		env.setRestartStrategy(RestartStrategies.noRestart()); // fail immediately
@@ -878,7 +874,7 @@ public void runOneToOneExactlyOnceTest() throws Exception {
 		createTestTopic(topic, parallelism, 1);
 
 		DataGenerators.generateRandomizedIntegerSequence(
-				StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort),
+				StreamExecutionEnvironment.getExecutionEnvironment(),
 				kafkaServer,
 				topic, parallelism, numElementsPerPartition, true);
 
@@ -887,7 +883,7 @@ public void runOneToOneExactlyOnceTest() throws Exception {
 		DeserializationSchema<Integer> schema =
 				new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.enableCheckpointing(500);
 		env.setParallelism(parallelism);
 		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
@@ -927,7 +923,7 @@ public void runOneSourceMultiplePartitionsExactlyOnceTest() throws Exception {
 		createTestTopic(topic, numPartitions, 1);
 
 		DataGenerators.generateRandomizedIntegerSequence(
-				StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort),
+				StreamExecutionEnvironment.getExecutionEnvironment(),
 				kafkaServer,
 				topic, numPartitions, numElementsPerPartition, false);
 
@@ -936,7 +932,7 @@ public void runOneSourceMultiplePartitionsExactlyOnceTest() throws Exception {
 		DeserializationSchema<Integer> schema =
 				new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.enableCheckpointing(500);
 		env.setParallelism(parallelism);
 		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
@@ -975,7 +971,7 @@ public void runMultipleSourcesOnePartitionExactlyOnceTest() throws Exception {
 		createTestTopic(topic, numPartitions, 1);
 
 		DataGenerators.generateRandomizedIntegerSequence(
-				StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort),
+				StreamExecutionEnvironment.getExecutionEnvironment(),
 				kafkaServer,
 				topic, numPartitions, numElementsPerPartition, true);
 
@@ -984,7 +980,7 @@ public void runMultipleSourcesOnePartitionExactlyOnceTest() throws Exception {
 		DeserializationSchema<Integer> schema =
 				new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.enableCheckpointing(500);
 		env.setParallelism(parallelism);
 		// set the number of restarts to one. The failing mapper will fail once, then it's only success exceptions.
@@ -1033,7 +1029,7 @@ public void runCancelingOnFullInputTest() throws Exception {
 			@Override
 			public void run() {
 				try {
-					final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+					final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 					env.setParallelism(parallelism);
 					env.enableCheckpointing(100);
 					env.getConfig().disableSysoutLogging();
@@ -1107,7 +1103,7 @@ public void runCancelingOnEmptyInputTest() throws Exception {
 			@Override
 			public void run() {
 				try {
-					final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+					final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 					env.setParallelism(parallelism);
 					env.enableCheckpointing(100);
 					env.getConfig().disableSysoutLogging();
@@ -1163,7 +1159,7 @@ public void runFailOnDeployTest() throws Exception {
 		DeserializationSchema<Integer> schema =
 				new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(12); // needs to be more that the mini cluster has slots
 		env.getConfig().disableSysoutLogging();
 
@@ -1180,7 +1176,7 @@ public void runFailOnDeployTest() throws Exception {
 			env.execute("test fail on deploy");
 			fail("this test should fail with an exception");
 		}
-		catch (ProgramInvocationException e) {
+		catch (JobExecutionException e) {
 
 			// validate that we failed due to a NoResourceAvailableException
 			Throwable cause = e.getCause();
@@ -1209,7 +1205,7 @@ public void runProduceConsumeMultipleTopics() throws java.lang.Exception {
 		final int NUM_TOPICS = 5;
 		final int NUM_ELEMENTS = 20;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		
 		// create topics with content
@@ -1220,6 +1216,10 @@ public void runProduceConsumeMultipleTopics() throws java.lang.Exception {
 			// create topic
 			createTestTopic(topic, i + 1 /*partitions*/, 1);
 		}
+
+		// before FLINK-6078 the RemoteExecutionEnvironment set the parallelism to 1 as well
+		env.setParallelism(1);
+
 		// run first job, producing into all topics
 		DataStream<Tuple3<Integer, Integer, String>> stream = env.addSource(new RichParallelSourceFunction<Tuple3<Integer, Integer, String>>() {
 
@@ -1249,7 +1249,7 @@ public void cancel() {
 		env.execute("Write to topics");
 
 		// run second job consuming from multiple topics
-		env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 
 		stream = env.addSource(kafkaServer.getConsumer(topics, schema, props));
@@ -1357,7 +1357,7 @@ public void runBigRecordTestTopology() throws Exception {
 		final TypeInformationSerializationSchema<Tuple2<Long, byte[]>> serSchema =
 				new TypeInformationSerializationSchema<>(longBytesInfo, new ExecutionConfig());
 
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setRestartStrategy(RestartStrategies.noRestart());
 		env.getConfig().disableSysoutLogging();
 		env.enableCheckpointing(100);
@@ -1457,7 +1457,7 @@ public void runBrokerFailureTest() throws Exception {
 		createTestTopic(topic, parallelism, 2);
 
 		DataGenerators.generateRandomizedIntegerSequence(
-				StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort),
+				StreamExecutionEnvironment.getExecutionEnvironment(),
 				kafkaServer,
 				topic, parallelism, numElementsPerPartition, true);
 
@@ -1472,7 +1472,7 @@ public void runBrokerFailureTest() throws Exception {
 		DeserializationSchema<Integer> schema =
 				new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(parallelism);
 		env.enableCheckpointing(500);
 		env.setRestartStrategy(RestartStrategies.noRestart());
@@ -1503,7 +1503,7 @@ public void runKeyValueTest() throws Exception {
 
 		// ----------- Write some data into Kafka -------------------
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(1);
 		env.setRestartStrategy(RestartStrategies.noRestart());
 		env.getConfig().disableSysoutLogging();
@@ -1535,7 +1535,7 @@ public void cancel() {
 
 		// ----------- Read the data again -------------------
 
-		env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(1);
 		env.setRestartStrategy(RestartStrategies.noRestart());
 		env.getConfig().disableSysoutLogging();
@@ -1590,7 +1590,7 @@ public void runAllDeletesTest() throws Exception {
 
 		// ----------- Write some data into Kafka -------------------
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(1);
 		env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env.getConfig().disableSysoutLogging();
@@ -1621,7 +1621,7 @@ public void cancel() {
 
 		// ----------- Read the data again -------------------
 
-		env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(1);
 		env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env.getConfig().disableSysoutLogging();
@@ -1661,7 +1661,7 @@ public void runEndOfStreamTest() throws Exception {
 		final String topic = writeSequence("testEndOfStream", ELEMENT_COUNT, 1, 1);
 
 		// read using custom schema
-		final StreamExecutionEnvironment env1 = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+		final StreamExecutionEnvironment env1 = StreamExecutionEnvironment.getExecutionEnvironment();
 		env1.setParallelism(1);
 		env1.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 		env1.getConfig().disableSysoutLogging();
@@ -1700,7 +1700,7 @@ public void runMetricsTest() throws Throwable {
 			public void run() {
 				try {
 					// start job writing & reading data.
-					final StreamExecutionEnvironment env1 = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+					final StreamExecutionEnvironment env1 = StreamExecutionEnvironment.getExecutionEnvironment();
 					env1.setParallelism(1);
 					env1.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 					env1.getConfig().disableSysoutLogging();
@@ -1741,7 +1741,7 @@ public void cancel() {
 					env1.execute("Metrics test job");
 				} catch(Throwable t) {
 					LOG.warn("Got exception during execution", t);
-					if(!(t.getCause() instanceof JobCancellationException)) { // we'll cancel the job
+					if(!(t instanceof JobCancellationException)) { // we'll cancel the job
 						error.f0 = t;
 					}
 				}
@@ -1994,7 +1994,7 @@ protected String writeSequence(
 
 			createTestTopic(topicName, parallelism, replicationFactor);
 
-			StreamExecutionEnvironment writeEnv = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+			StreamExecutionEnvironment writeEnv = StreamExecutionEnvironment.getExecutionEnvironment();
 			writeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 			writeEnv.getConfig().disableSysoutLogging();
 			
@@ -2046,7 +2046,7 @@ public void cancel() {
 
 			JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout));
 			
-			final StreamExecutionEnvironment readEnv = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+			final StreamExecutionEnvironment readEnv = StreamExecutionEnvironment.getExecutionEnvironment();
 			readEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart());
 			readEnv.getConfig().disableSysoutLogging();
 			readEnv.setParallelism(parallelism);
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java
index c925c8f188c85..6f6139291e101 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java
@@ -71,7 +71,7 @@ public void runCustomPartitioningTest() {
 
 			TypeInformation<Tuple2<Long, String>> longStringInfo = TypeInfoParser.parse("Tuple2<Long, String>");
 
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
+			StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 			env.setRestartStrategy(RestartStrategies.noRestart());
 			env.getConfig().disableSysoutLogging();
 
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
index 954dc7d953f6c..f688660b2aea0 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
@@ -29,6 +29,7 @@
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
 import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema;
 import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper;
 import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
@@ -59,6 +60,12 @@
 public class KafkaShortRetentionTestBase implements Serializable {
 	
 	protected static final Logger LOG = LoggerFactory.getLogger(KafkaShortRetentionTestBase.class);
+
+	protected static final int NUM_TMS = 1;
+
+	protected static final int TM_SLOTS = 8;
+
+	protected static final int PARALLELISM = NUM_TMS * TM_SLOTS;
 	
 	private static KafkaTestEnvironment kafkaServer;
 	private static Properties standardProps;
@@ -97,17 +104,21 @@ public static void prepare() throws IOException, ClassNotFoundException {
 		standardProps = kafkaServer.getStandardProperties();
 
 		// start also a re-usable Flink mini cluster
-		flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
-		flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
+		flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
+		flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, TM_SLOTS);
 		flinkConfig.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L);
 		flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
 
 		flink = new LocalFlinkMiniCluster(flinkConfig, false);
 		flink.start();
+
+		TestStreamEnvironment.setAsContext(flink, PARALLELISM);
 	}
 
 	@AfterClass
 	public static void shutDownServices() {
+		TestStreamEnvironment.unsetAsContext();
+
 		if (flink != null) {
 			flink.shutdown();
 		}
@@ -135,8 +146,7 @@ public void runAutoOffsetResetTest() throws Exception {
 		tprops.setProperty("retention.ms", "250");
 		kafkaServer.createTestTopic(topic, parallelism, 1, tprops);
 
-		final StreamExecutionEnvironment env =
-				StreamExecutionEnvironment.createRemoteEnvironment("localhost", flink.getLeaderRPCPort());
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(parallelism);
 		env.setRestartStrategy(RestartStrategies.noRestart()); // fail immediately
 		env.getConfig().disableSysoutLogging();
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
index a21a2399cf83c..1837af695b809 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
@@ -26,6 +26,7 @@
 import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.test.util.SuccessException;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.TestLogger;
@@ -40,7 +41,6 @@
 
 import scala.concurrent.duration.FiniteDuration;
 
-import java.io.IOException;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
@@ -65,14 +65,18 @@ public abstract class KafkaTestBase extends TestLogger {
 	
 	protected static final int NUMBER_OF_KAFKA_SERVERS = 3;
 
+	protected static final int NUM_TMS = 1;
+
+	protected static final int TM_SLOTS = 8;
+
+	protected static final int PARALLELISM = NUM_TMS * TM_SLOTS;
+
 	protected static String brokerConnectionStrings;
 
 	protected static Properties standardProps;
 	
 	protected static LocalFlinkMiniCluster flink;
 
-	protected static int flinkPort;
-
 	protected static FiniteDuration timeout = new FiniteDuration(10, TimeUnit.SECONDS);
 
 	protected static KafkaTestEnvironment kafkaServer;
@@ -87,7 +91,7 @@ public abstract class KafkaTestBase extends TestLogger {
 	// ------------------------------------------------------------------------
 	
 	@BeforeClass
-	public static void prepare() throws IOException, ClassNotFoundException {
+	public static void prepare() throws ClassNotFoundException {
 
 		LOG.info("-------------------------------------------------------------------------");
 		LOG.info("    Starting KafkaTestBase ");
@@ -95,6 +99,7 @@ public static void prepare() throws IOException, ClassNotFoundException {
 
 		startClusters(false);
 
+		TestStreamEnvironment.setAsContext(flink, PARALLELISM);
 	}
 
 	@AfterClass
@@ -104,6 +109,8 @@ public static void shutDownServices() {
 		LOG.info("    Shut down KafkaTestBase ");
 		LOG.info("-------------------------------------------------------------------------");
 
+		TestStreamEnvironment.unsetAsContext();
+
 		shutdownClusters();
 
 		LOG.info("-------------------------------------------------------------------------");
@@ -113,8 +120,8 @@ public static void shutDownServices() {
 
 	protected static Configuration getFlinkConfiguration() {
 		Configuration flinkConfig = new Configuration();
-		flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
-		flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
+		flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
+		flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, TM_SLOTS);
 		flinkConfig.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L);
 		flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
 		flinkConfig.setString(MetricOptions.REPORTERS_LIST, "my_reporter");
@@ -147,14 +154,10 @@ protected static void startClusters(boolean secureMode) throws ClassNotFoundExce
 		// start also a re-usable Flink mini cluster
 		flink = new LocalFlinkMiniCluster(getFlinkConfiguration(), false);
 		flink.start();
-
-		flinkPort = flink.getLeaderRPCPort();
-
 	}
 
 	protected static void shutdownClusters() {
 
-		flinkPort = -1;
 		if (flink != null) {
 			flink.shutdown();
 		}
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/log4j-test.properties b/flink-connectors/flink-connector-kafka-base/src/test/resources/log4j-test.properties
index 6bdfb48cd550f..16c226fc1bba1 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/resources/log4j-test.properties
+++ b/flink-connectors/flink-connector-kafka-base/src/test/resources/log4j-test.properties
@@ -16,7 +16,7 @@
 # limitations under the License.
 ################################################################################
 
-log4j.rootLogger=INFO, testlogger
+log4j.rootLogger=OFF, testlogger
 
 log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
 log4j.appender.testlogger.target = System.err
diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java
index 2b7f357ea7daa..21794f9fefc36 100644
--- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java
+++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java
@@ -205,7 +205,7 @@ public void submitTopologyWithOpts(final String name, final String uploadedJarLo
 		final ClusterClient client;
 		try {
 			client = new StandaloneClusterClient(configuration);
-		} catch (final IOException e) {
+		} catch (final Exception e) {
 			throw new RuntimeException("Could not establish a connection to the job manager", e);
 		}
 
@@ -245,7 +245,7 @@ public void killTopologyWithOpts(final String name, final KillOptions options) t
 		final ClusterClient client;
 		try {
 			client = new StandaloneClusterClient(configuration);
-		} catch (final IOException e) {
+		} catch (final Exception e) {
 			throw new RuntimeException("Could not establish a connection to the job manager", e);
 		}
 
diff --git a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java
index 7a2563642450e..f9b6a2130fa17 100644
--- a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java
+++ b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java
@@ -22,8 +22,9 @@
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.util.TestLogger;
+
 import org.junit.Test;
 
 import java.util.Iterator;
@@ -41,8 +42,9 @@ public void testCollect() throws Exception {
 		try {
 			cluster.start();
 
-			final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
+			TestStreamEnvironment.setAsContext(cluster, 1);
+
+			final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 	
 			final long N = 10;
 			DataStream<Long> stream = env.generateSequence(1, N);
@@ -57,6 +59,7 @@ public void testCollect() throws Exception {
 			assertEquals("received wrong number of elements", N + 1, i);
 		}
 		finally {
+			TestStreamEnvironment.unsetAsContext();
 			cluster.stop();
 		}
 	}
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/DegreesWithExceptionITCase.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/DegreesWithExceptionITCase.java
index ad434d4bf0ca3..111d4211fa911 100644
--- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/DegreesWithExceptionITCase.java
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/DegreesWithExceptionITCase.java
@@ -28,6 +28,7 @@
 import org.apache.flink.graph.test.TestGraphUtils;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
@@ -45,27 +46,19 @@ public class DegreesWithExceptionITCase extends TestLogger {
 
 	@BeforeClass
 	public static void setupCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
-			cluster = new LocalFlinkMiniCluster(config, false);
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Error starting test cluster: " + e.getMessage());
-		}
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
+		cluster = new LocalFlinkMiniCluster(config, false);
+		cluster.start();
+
+		TestEnvironment.setAsContext(cluster, PARALLELISM);
 	}
 
 	@AfterClass
 	public static void tearDownCluster() {
-		try {
-			cluster.stop();
-		}
-		catch (Throwable t) {
-			t.printStackTrace();
-			fail("ClusterClient shutdown caused an exception: " + t.getMessage());
-		}
+		cluster.stop();
+
+		TestEnvironment.unsetAsContext();
 	}
 
 	/**
@@ -74,8 +67,7 @@ public static void tearDownCluster() {
 	@Test
 	public void testOutDegreesInvalidEdgeSrcId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 		
@@ -98,8 +90,7 @@ public void testOutDegreesInvalidEdgeSrcId() throws Exception {
 	@Test
 	public void testInDegreesInvalidEdgeTrgId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -122,8 +113,7 @@ public void testInDegreesInvalidEdgeTrgId() throws Exception {
 	@Test
 	public void testGetDegreesInvalidEdgeTrgId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -146,8 +136,7 @@ public void testGetDegreesInvalidEdgeTrgId() throws Exception {
 	@Test
 	public void testGetDegreesInvalidEdgeSrcId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -170,8 +159,7 @@ public void testGetDegreesInvalidEdgeSrcId() throws Exception {
 	@Test
 	public void testGetDegreesInvalidEdgeSrcTrgId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnEdgesWithExceptionITCase.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnEdgesWithExceptionITCase.java
index d090d3c68c0a5..7a0a30c2ab047 100644
--- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnEdgesWithExceptionITCase.java
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnEdgesWithExceptionITCase.java
@@ -31,6 +31,7 @@
 import org.apache.flink.graph.Vertex;
 import org.apache.flink.graph.test.TestGraphUtils;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
@@ -48,27 +49,19 @@ public class ReduceOnEdgesWithExceptionITCase extends TestLogger {
 
 	@BeforeClass
 	public static void setupCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
-			cluster = new LocalFlinkMiniCluster(config, false);
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Error starting test cluster: " + e.getMessage());
-		}
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
+		cluster = new LocalFlinkMiniCluster(config, false);
+		cluster.start();
+
+		TestEnvironment.setAsContext(cluster, PARALLELISM);
 	}
 
 	@AfterClass
 	public static void tearDownCluster() {
-		try {
-			cluster.stop();
-		}
-		catch (Throwable t) {
-			t.printStackTrace();
-			fail("ClusterClient shutdown caused an exception: " + t.getMessage());
-		}
+		cluster.stop();
+
+		TestEnvironment.unsetAsContext();
 	}
 
 	/**
@@ -77,8 +70,7 @@ public static void tearDownCluster() {
 	@Test
 	public void testGroupReduceOnEdgesInvalidEdgeSrcId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -91,6 +83,8 @@ public void testGroupReduceOnEdgesInvalidEdgeSrcId() throws Exception {
 
 			verticesWithAllNeighbors.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
 			env.execute();
+
+			fail("Expected an exception.");
 		} catch (Exception e) {
 			// We expect the job to fail with an exception
 		}
@@ -102,8 +96,7 @@ public void testGroupReduceOnEdgesInvalidEdgeSrcId() throws Exception {
 	@Test
 	public void testGroupReduceOnEdgesInvalidEdgeTrgId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -116,6 +109,8 @@ public void testGroupReduceOnEdgesInvalidEdgeTrgId() throws Exception {
 
 			verticesWithAllNeighbors.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
 			env.execute();
+
+			fail("Expected an exception.");
 		} catch (Exception e) {
 			// We expect the job to fail with an exception
 		}
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnNeighborsWithExceptionITCase.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnNeighborsWithExceptionITCase.java
index afe2e18f51dfd..b337bcab4ceb9 100644
--- a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnNeighborsWithExceptionITCase.java
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/test/operations/ReduceOnNeighborsWithExceptionITCase.java
@@ -32,6 +32,7 @@
 import org.apache.flink.graph.Vertex;
 import org.apache.flink.graph.test.TestGraphUtils;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
@@ -49,27 +50,19 @@ public class ReduceOnNeighborsWithExceptionITCase extends TestLogger {
 
 	@BeforeClass
 	public static void setupCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
-			cluster = new LocalFlinkMiniCluster(config, false);
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Error starting test cluster: " + e.getMessage());
-		}
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
+		cluster = new LocalFlinkMiniCluster(config, false);
+		cluster.start();
+
+		TestEnvironment.setAsContext(cluster, PARALLELISM);
 	}
 
 	@AfterClass
 	public static void tearDownCluster() {
-		try {
-			cluster.stop();
-		}
-		catch (Throwable t) {
-			t.printStackTrace();
-			fail("ClusterClient shutdown caused an exception: " + t.getMessage());
-		}
+		cluster.stop();
+
+		TestEnvironment.unsetAsContext();
 	}
 
 	/**
@@ -79,8 +72,7 @@ public static void tearDownCluster() {
 	@Test
 	public void testGroupReduceOnNeighborsWithVVInvalidEdgeSrcId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -93,6 +85,8 @@ public void testGroupReduceOnNeighborsWithVVInvalidEdgeSrcId() throws Exception
 
 			verticesWithSumOfOutNeighborValues.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
 			env.execute();
+
+			fail("Expected an exception.");
 		} catch (Exception e) {
 			// We expect the job to fail with an exception
 		}
@@ -105,8 +99,7 @@ public void testGroupReduceOnNeighborsWithVVInvalidEdgeSrcId() throws Exception
 	@Test
 	public void testGroupReduceOnNeighborsWithVVInvalidEdgeTrgId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -119,6 +112,8 @@ public void testGroupReduceOnNeighborsWithVVInvalidEdgeTrgId() throws Exception
 
 			verticesWithSumOfOutNeighborValues.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
 			env.execute();
+
+			fail("Expected an exception.");
 		} catch (Exception e) {
 			// We expect the job to fail with an exception
 		}
@@ -131,8 +126,7 @@ public void testGroupReduceOnNeighborsWithVVInvalidEdgeTrgId() throws Exception
 	@Test
 	public void testGroupReduceOnNeighborsInvalidEdgeSrcId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
@@ -157,8 +151,7 @@ public void testGroupReduceOnNeighborsInvalidEdgeSrcId() throws Exception {
 	@Test
 	public void testGroupReduceOnNeighborsInvalidEdgeTrgId() throws Exception {
 
-		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
index 6353d6a61e44c..3ee7a99ab6662 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
@@ -62,7 +62,7 @@ trait FlinkTestBase extends BeforeAndAfter {
       false,
       true)
 
-    val clusterEnvironment = new TestEnvironment(cl, parallelism)
+    val clusterEnvironment = new TestEnvironment(cl, parallelism, false)
     clusterEnvironment.setAsContext()
 
     cluster = Some(cl)
diff --git a/flink-libraries/flink-python/src/test/resources/log4j-test.properties b/flink-libraries/flink-python/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000000000..18b51cc276da3
--- /dev/null
+++ b/flink-libraries/flink-python/src/test/resources/log4j-test.properties
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger=OFF, console
+
+# -----------------------------------------------------------------------------
+# Console (use 'console')
+# -----------------------------------------------------------------------------
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+# -----------------------------------------------------------------------------
+# File (use 'file')
+# -----------------------------------------------------------------------------
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.file=${log.dir}/${mvn.forkNumber}.log
+log4j.appender.file.append=false
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+# suppress the irrelevant (wrong) warnings from the netty channel handler
+log4j.logger.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, console
diff --git a/flink-libraries/flink-python/src/test/resources/logback-test.xml b/flink-libraries/flink-python/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000000..1d64d46284302
--- /dev/null
+++ b/flink-libraries/flink-python/src/test/resources/logback-test.xml
@@ -0,0 +1,42 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<configuration>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] [%X{sourceThread} - %X{akkaSource}] %-5level %logger{60} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="WARN">
+        <appender-ref ref="STDOUT"/>
+    </root>
+
+    <!-- The following loggers are disabled during tests, because many tests deliberately
+         throw error to test failing scenarios. Logging those would overflow the log. -->
+         <!---->
+    <logger name="org.apache.flink.runtime.operators.DataSinkTask" level="OFF"/>
+    <logger name="org.apache.flink.runtime.operators.BatchTask" level="OFF"/>
+    <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
+    <logger name="org.apache.flink.runtime.taskmanager.Task" level="OFF"/>
+    <logger name="org.apache.flink.runtime.jobmanager.JobManager" level="OFF"/>
+    <logger name="org.apache.flink.runtime.testingUtils" level="OFF"/>
+    <logger name="org.apache.flink.runtime.executiongraph.ExecutionGraph" level="OFF"/>
+    <logger name="org.apache.flink.runtime.jobmanager.EventCollector" level="OFF"/>
+    <logger name="org.apache.flink.runtime.instance.InstanceManager" level="OFF"/>
+</configuration>
\ No newline at end of file
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
index 0c127457afb18..5513df43cc88f 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
@@ -48,17 +48,17 @@
 import org.apache.flink.runtime.clusterframework.overlays.KeytabOverlay;
 import org.apache.flink.runtime.clusterframework.overlays.Krb5ConfOverlay;
 import org.apache.flink.runtime.clusterframework.overlays.SSLStoreOverlay;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.jobmaster.JobMaster;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.process.ProcessReaper;
 import org.apache.flink.runtime.security.SecurityUtils;
 import org.apache.flink.runtime.util.EnvironmentInformation;
 import org.apache.flink.runtime.util.ExecutorThreadFactory;
 import org.apache.flink.runtime.util.Hardware;
 import org.apache.flink.runtime.util.JvmShutdownSafeguard;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.runtime.util.SignalHandler;
 import org.apache.flink.runtime.webmonitor.WebMonitor;
 import org.apache.mesos.Protos;
@@ -67,7 +67,6 @@
 import org.slf4j.LoggerFactory;
 
 import scala.Option;
-import scala.Some;
 import scala.concurrent.duration.Duration;
 import scala.concurrent.duration.FiniteDuration;
 
@@ -206,6 +205,7 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 		ScheduledExecutorService futureExecutor = null;
 		ExecutorService ioExecutor = null;
 		MesosServices mesosServices = null;
+		HighAvailabilityServices highAvailabilityServices = null;
 
 		try {
 			// ------- (1) load and parse / validate all configurations -------
@@ -295,6 +295,12 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 			// 3) Resource Master for Mesos
 			// 4) Process reapers for the JobManager and Resource Master
 
+			// 0: Start the JobManager services
+			highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				config,
+				ioExecutor,
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
+
 			// 1: the JobManager
 			LOG.debug("Starting JobManager actor");
 
@@ -304,8 +310,9 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 				actorSystem,
 				futureExecutor,
 				ioExecutor,
-				new Some<>(JobMaster.JOB_MANAGER_NAME),
-				Option.<String>empty(),
+				highAvailabilityServices,
+				Option.apply(JobMaster.JOB_MANAGER_NAME),
+				Option.apply(JobMaster.ARCHIVE_NAME),
 				getJobManagerClass(),
 				getArchivistClass())._1();
 
@@ -313,7 +320,12 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 			// 2: the web monitor
 			LOG.debug("Starting Web Frontend");
 
-			webMonitor = BootstrapTools.startWebMonitorIfConfigured(config, actorSystem, jobManager, LOG);
+			webMonitor = BootstrapTools.startWebMonitorIfConfigured(
+				config,
+				highAvailabilityServices,
+				actorSystem,
+				jobManager,
+				LOG);
 			if(webMonitor != null) {
 				final URL webMonitorURL = new URL("http", appMasterHostname, webMonitor.getServerPort(), "/");
 				mesosConfig.frameworkInfo().setWebuiUrl(webMonitorURL.toExternalForm());
@@ -327,17 +339,12 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 				config,
 				ioExecutor);
 
-			// we need the leader retrieval service here to be informed of new
-			// leader session IDs, even though there can be only one leader ever
-			LeaderRetrievalService leaderRetriever =
-				LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager);
-
 			Props resourceMasterProps = MesosFlinkResourceManager.createActorProps(
 				getResourceManagerClass(),
 				config,
 				mesosConfig,
 				workerStore,
-				leaderRetriever,
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 				taskManagerParameters,
 				taskManagerContainerSpec,
 				artifactServer,
@@ -435,6 +442,14 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 			LOG.error("Failed to stop the artifact server", t);
 		}
 
+		if (highAvailabilityServices != null) {
+			try {
+				highAvailabilityServices.close();
+			} catch (Throwable t) {
+				LOG.error("Could not properly stop the high availability services.");
+			}
+		}
+
 		org.apache.flink.runtime.concurrent.Executors.gracefulShutdown(
 			AkkaUtils.getTimeout(config).toMillis(),
 			TimeUnit.MILLISECONDS,
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
index 7915e8ae03f8b..7fe5db5531563 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
@@ -43,11 +43,12 @@
 import org.apache.flink.runtime.clusterframework.messages.*;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
+import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.util.TestLogger;
 import org.apache.mesos.SchedulerDriver;
 import org.apache.mesos.Protos;
@@ -142,11 +143,10 @@ protected void fatalError(String message, Throwable error) {
 	/**
 	 * The context fixture.
 	 */
-	static class Context extends JavaTestKit {
+	static class Context extends JavaTestKit implements AutoCloseable {
 
 		// mocks
 		public ActorGateway jobManager;
-		public LeaderRetrievalService retrievalService;
 		public MesosConfiguration mesosConfig;
 		public MesosWorkerStore workerStore;
 		public MesosArtifactResolver artifactResolver;
@@ -163,6 +163,8 @@ static class Context extends JavaTestKit {
 		public Protos.TaskID task2 = Protos.TaskID.newBuilder().setValue("taskmanager-00002").build();
 		public Protos.TaskID task3 = Protos.TaskID.newBuilder().setValue("taskmanager-00003").build();
 
+		private final TestingHighAvailabilityServices highAvailabilityServices;
+
 		/**
 		 * Create mock RM dependencies.
 		 */
@@ -170,8 +172,19 @@ public Context() {
 			super(system);
 
 			try {
-				jobManager = TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-				retrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager.actor());
+				jobManager = TestingUtils.createForwardingActor(
+					system,
+					getTestActor(),
+					HighAvailabilityServices.DEFAULT_LEADER_ID,
+					Option.<String>empty());
+
+				highAvailabilityServices = new TestingHighAvailabilityServices();
+
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new TestingLeaderRetrievalService(
+						jobManager.path(),
+						HighAvailabilityServices.DEFAULT_LEADER_ID));
 
 				// scheduler driver
 				schedulerDriver = mock(SchedulerDriver.class);
@@ -206,7 +219,14 @@ public void initialize() {
 			TestActorRef<TestingMesosFlinkResourceManager> resourceManagerRef =
 				TestActorRef.create(system, MesosFlinkResourceManager.createActorProps(
 					TestingMesosFlinkResourceManager.class,
-					config, mesosConfig, workerStore, retrievalService, tmParams, containerSpecification, artifactResolver, LOG));
+					config,
+					mesosConfig,
+					workerStore,
+					highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+					tmParams,
+					containerSpecification,
+					artifactResolver,
+					LOG));
 			resourceManagerInstance = resourceManagerRef.underlyingActor();
 			resourceManager = new AkkaActorGateway(resourceManagerRef, HighAvailabilityServices.DEFAULT_LEADER_ID);
 
@@ -235,6 +255,11 @@ public Protos.Offer.Operation launch(Protos.TaskInfo... taskInfo) {
 				.setLaunch(Protos.Offer.Operation.Launch.newBuilder().addAllTaskInfos(Arrays.asList(taskInfo))
 				).build();
 		}
+
+		@Override
+		public void close() throws Exception {
+			highAvailabilityServices.closeAndCleanupAllData();
+		}
 	}
 
 	/**
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java
index e80c509cc57da..f31c932585058 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java
@@ -29,6 +29,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
@@ -103,6 +104,12 @@ public void testBackPressuredProducer() throws Exception {
 
 			jobGraph.addVertex(task);
 
+			final Configuration config = new Configuration();
+
+			final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+				config,
+				TestingUtils.defaultExecutor());
+
 			ActorGateway jobManger = null;
 			ActorGateway taskManager = null;
 
@@ -125,13 +132,17 @@ public void testBackPressuredProducer() throws Exception {
 					testActorSystem,
 					TestingUtils.defaultExecutor(),
 					TestingUtils.defaultExecutor(),
-					new Configuration());
+					config,
+					highAvailabilityServices);
 
-				final Configuration config = new Configuration();
 				config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, parallelism);
 
 				taskManager = TestingUtils.createTaskManager(
-						testActorSystem, jobManger, config, true, true);
+					testActorSystem,
+					highAvailabilityServices,
+					config,
+					true,
+					true);
 
 				final ActorGateway jm = jobManger;
 
@@ -264,6 +275,8 @@ protected void run() {
 				TestingUtils.stopActor(jobManger);
 				TestingUtils.stopActor(taskManager);
 
+				highAvailabilityServices.closeAndCleanupAllData();
+
 				for (Buffer buf : buffers) {
 					buf.recycle();
 				}
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java
index 54633842b2b18..b67e7358591d0 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java
@@ -29,6 +29,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -89,6 +90,12 @@ public void testTaskClearedWhileSampling() throws Exception {
 
 			jobGraph.addVertex(task);
 
+			final Configuration config = new Configuration();
+
+			final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+				config,
+				TestingUtils.defaultExecutor());
+
 			ActorGateway jobManger = null;
 			ActorGateway taskManager = null;
 
@@ -97,13 +104,17 @@ public void testTaskClearedWhileSampling() throws Exception {
 					testActorSystem,
 					TestingUtils.defaultExecutor(),
 					TestingUtils.defaultExecutor(),
-					new Configuration());
+					config,
+					highAvailabilityServices);
 
-				final Configuration config = new Configuration();
 				config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, parallelism);
 
 				taskManager = TestingUtils.createTaskManager(
-						testActorSystem, jobManger, config, true, true);
+					testActorSystem,
+					highAvailabilityServices,
+					config,
+					true,
+					true);
 
 				final ActorGateway jm = jobManger;
 
@@ -182,6 +193,8 @@ protected void run() {
 			} finally {
 				TestingUtils.stopActor(jobManger);
 				TestingUtils.stopActor(taskManager);
+
+				highAvailabilityServices.closeAndCleanupAllData();
 			}
 		}};
 	}
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
index 8af1f46bc428c..5ccfe90921804 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
@@ -26,6 +26,8 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.leaderelection.TestingListener;
@@ -33,7 +35,6 @@
 import org.apache.flink.runtime.testingUtils.TestingCluster;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
-import org.apache.flink.runtime.util.ZooKeeperUtils;
 import org.apache.flink.runtime.webmonitor.files.MimeTypes;
 import org.apache.flink.runtime.webmonitor.testutils.HttpTestClient;
 import org.apache.flink.util.TestLogger;
@@ -49,8 +50,6 @@
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Scanner;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
@@ -127,7 +126,7 @@ public void testRedirectToLeader() throws Exception {
 
 		ActorSystem[] jobManagerSystem = new ActorSystem[2];
 		WebRuntimeMonitor[] webMonitor = new WebRuntimeMonitor[2];
-		List<LeaderRetrievalService> leaderRetrievalServices = new ArrayList<>();
+		HighAvailabilityServices highAvailabilityServices = null;
 
 		try (TestingServer zooKeeper = new TestingServer()) {
 			final Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig(
@@ -141,15 +140,20 @@ public void testRedirectToLeader() throws Exception {
 			config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0);
 			config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
 
+			highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+				config,
+				TestingUtils.defaultExecutor());
+
 			for (int i = 0; i < jobManagerSystem.length; i++) {
 				jobManagerSystem[i] = AkkaUtils.createActorSystem(new Configuration(),
 						new Some<>(new Tuple2<String, Object>("localhost", 0)));
 			}
 
 			for (int i = 0; i < webMonitor.length; i++) {
-				LeaderRetrievalService lrs = ZooKeeperUtils.createLeaderRetrievalService(config);
-				leaderRetrievalServices.add(lrs);
-				webMonitor[i] = new WebRuntimeMonitor(config, lrs, jobManagerSystem[i]);
+				webMonitor[i] = new WebRuntimeMonitor(
+					config,
+					highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+					jobManagerSystem[i]);
 			}
 
 			ActorRef[] jobManager = new ActorRef[2];
@@ -164,6 +168,7 @@ public void testRedirectToLeader() throws Exception {
 					jobManagerSystem[i],
 					TestingUtils.defaultExecutor(),
 					TestingUtils.defaultExecutor(),
+					highAvailabilityServices,
 					JobManager.class,
 					MemoryArchivist.class)._1();
 
@@ -171,8 +176,7 @@ public void testRedirectToLeader() throws Exception {
 				webMonitor[i].start(jobManagerAddress[i]);
 			}
 
-			LeaderRetrievalService lrs = ZooKeeperUtils.createLeaderRetrievalService(config);
-			leaderRetrievalServices.add(lrs);
+			LeaderRetrievalService lrs = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 			TestingListener leaderListener = new TestingListener();
 			lrs.start(leaderListener);
 
@@ -247,6 +251,8 @@ public void testRedirectToLeader() throws Exception {
 				assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("json"));
 				assertTrue(response.getContent().contains("\"taskmanagers\":1") ||
 						response.getContent().contains("\"taskmanagers\":0"));
+			} finally {
+				lrs.stop();
 			}
 		}
 		finally {
@@ -260,8 +266,8 @@ public void testRedirectToLeader() throws Exception {
 				monitor.stop();
 			}
 
-			for (LeaderRetrievalService lrs : leaderRetrievalServices) {
-				lrs.stop();
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
 			}
 		}
 	}
@@ -462,7 +468,8 @@ private WebRuntimeMonitor startWebRuntimeMonitor(
 
 		WebRuntimeMonitor webMonitor = new WebRuntimeMonitor(
 			config,
-			flink.createLeaderRetrievalService(),
+			flink.highAvailabilityServices().getJobManagerLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID),
 			jmActorSystem);
 
 		webMonitor.start(jobManagerAddress);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCheckpointRecoveryFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCheckpointRecoveryFactory.java
index 481559b41c019..d60b26f750029 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCheckpointRecoveryFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCheckpointRecoveryFactory.java
@@ -55,7 +55,7 @@ public void start() {
 
 	@Override
 	public void stop() {
-		client.close();
+		// Nothing to do
 	}
 
 	@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClient.java
index 76d6d86e5ded8..b570383cfd654 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClient.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClient.java
@@ -34,9 +34,9 @@
 import org.apache.flink.runtime.blob.BlobCache;
 import org.apache.flink.runtime.blob.BlobKey;
 import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoader;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobClientMessages;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.util.SerializedThrowable;
@@ -96,14 +96,14 @@ public static ActorSystem startJobClientActorSystem(Configuration config)
 	public static JobListeningContext submitJob(
 			ActorSystem actorSystem,
 			Configuration config,
-			LeaderRetrievalService leaderRetrievalService,
+			HighAvailabilityServices highAvailabilityServices,
 			JobGraph jobGraph,
 			FiniteDuration timeout,
 			boolean sysoutLogUpdates,
 			ClassLoader classLoader) {
 
 		checkNotNull(actorSystem, "The actorSystem must not be null.");
-		checkNotNull(leaderRetrievalService, "The jobManagerGateway must not be null.");
+		checkNotNull(highAvailabilityServices, "The high availability services must not be null.");
 		checkNotNull(jobGraph, "The jobGraph must not be null.");
 		checkNotNull(timeout, "The timeout must not be null.");
 
@@ -112,7 +112,7 @@ public static JobListeningContext submitJob(
 		// update messages, watches for disconnect between client and JobManager, ...
 
 		Props jobClientActorProps = JobSubmissionClientActor.createActorProps(
-			leaderRetrievalService,
+			highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 			timeout,
 			sysoutLogUpdates,
 			config);
@@ -125,11 +125,12 @@ public static JobListeningContext submitJob(
 				new Timeout(AkkaUtils.INF_TIMEOUT()));
 
 		return new JobListeningContext(
-				jobGraph.getJobID(),
-				submissionFuture,
-				jobClientActor,
-				timeout,
-				classLoader);
+			jobGraph.getJobID(),
+			submissionFuture,
+			jobClientActor,
+			timeout,
+			classLoader,
+			highAvailabilityServices);
 	}
 
 
@@ -142,7 +143,7 @@ public static JobListeningContext attachToRunningJob(
 			ActorGateway jobManagerGateWay,
 			Configuration configuration,
 			ActorSystem actorSystem,
-			LeaderRetrievalService leaderRetrievalService,
+			HighAvailabilityServices highAvailabilityServices,
 			FiniteDuration timeout,
 			boolean sysoutLogUpdates) {
 
@@ -150,14 +151,14 @@ public static JobListeningContext attachToRunningJob(
 		checkNotNull(jobManagerGateWay, "The jobManagerGateWay must not be null.");
 		checkNotNull(configuration, "The configuration must not be null.");
 		checkNotNull(actorSystem, "The actorSystem must not be null.");
-		checkNotNull(leaderRetrievalService, "The jobManagerGateway must not be null.");
+		checkNotNull(highAvailabilityServices, "The high availability services must not be null.");
 		checkNotNull(timeout, "The timeout must not be null.");
 
 		// we create a proxy JobClientActor that deals with all communication with
 		// the JobManager. It forwards the job attachments, checks the success/failure responses, logs
 		// update messages, watches for disconnect between client and JobManager, ...
 		Props jobClientActorProps = JobAttachmentClientActor.createActorProps(
-			leaderRetrievalService,
+			highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 			timeout,
 			sysoutLogUpdates);
 
@@ -169,12 +170,13 @@ public static JobListeningContext attachToRunningJob(
 				new Timeout(AkkaUtils.INF_TIMEOUT()));
 
 		return new JobListeningContext(
-				jobID,
-				attachmentFuture,
-				jobClientActor,
-				timeout,
-				actorSystem,
-				configuration);
+			jobID,
+			attachmentFuture,
+			jobClientActor,
+			timeout,
+			actorSystem,
+			configuration,
+			highAvailabilityServices);
 	}
 
 	/**
@@ -357,20 +359,19 @@ else if (answer instanceof JobManagerMessages.JobNotFound) {
 	 *
 	 * @param actorSystem The actor system that performs the communication.
 	 * @param config The cluster wide configuration.
-	 * @param leaderRetrievalService Leader retrieval service which used to find the current leading
-	 *                               JobManager
+	 * @param highAvailabilityServices Service factory for high availability services
 	 * @param jobGraph    JobGraph describing the Flink job
 	 * @param timeout     Timeout for futures
 	 * @param sysoutLogUpdates prints log updates to system out if true
 	 * @param classLoader The class loader for deserializing the results
 	 * @return The job execution result
-	 * @throws org.apache.flink.runtime.client.JobExecutionException Thrown if the job
+	 * @throws JobExecutionException Thrown if the job
 	 *                                                               execution fails.
 	 */
 	public static JobExecutionResult submitJobAndWait(
 			ActorSystem actorSystem,
 			Configuration config,
-			LeaderRetrievalService leaderRetrievalService,
+			HighAvailabilityServices highAvailabilityServices,
 			JobGraph jobGraph,
 			FiniteDuration timeout,
 			boolean sysoutLogUpdates,
@@ -379,7 +380,7 @@ public static JobExecutionResult submitJobAndWait(
 		JobListeningContext jobListeningContext = submitJob(
 				actorSystem,
 				config,
-				leaderRetrievalService,
+				highAvailabilityServices,
 				jobGraph,
 				timeout,
 				sysoutLogUpdates,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
index a0bf97dbfccbd..793041fe8f6fe 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
@@ -167,7 +167,8 @@ public void onSuccess(ActorRef result) throws Throwable {
 			JobManagerActorRef msg = (JobManagerActorRef) message;
 			connectToJobManager(msg.jobManager());
 
-			logAndPrintMessage("Connected to JobManager at " + msg.jobManager());
+			logAndPrintMessage("Connected to JobManager at " + msg.jobManager() +
+				" with leader session id " + leaderSessionID + '.');
 
 			connectedToJobManager();
 		}
@@ -326,8 +327,6 @@ private void connectToJobManager(ActorRef jobManager) {
 			getContext().unwatch(jobManager);
 		}
 
-		LOG.info("Connected to new JobManager {}.", jobManager.path());
-
 		this.jobManager = jobManager;
 		getContext().watch(jobManager);
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java
index b944ba8f7561c..fe8c34cd09b61 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobListeningContext.java
@@ -22,6 +22,7 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.slf4j.Logger;
@@ -47,6 +48,9 @@ public final class JobListeningContext {
 	/** Timeout used Asks */
 	private final FiniteDuration timeout;
 
+	/** Service factory for high availability services */
+	private final HighAvailabilityServices highAvailabilityServices;
+
 	/** ActorSystem for leader retrieval */
 	private ActorSystem actorSystem;
 	/** Flink configuration for initializing the BlobService */
@@ -59,16 +63,18 @@ public final class JobListeningContext {
 	 * Constructor to use when the class loader is available.
 	 */
 	public JobListeningContext(
-		JobID jobID,
-		Future<Object> jobResultFuture,
-		ActorRef jobClientActor,
-		FiniteDuration timeout,
-		ClassLoader classLoader) {
+			JobID jobID,
+			Future<Object> jobResultFuture,
+			ActorRef jobClientActor,
+			FiniteDuration timeout,
+			ClassLoader classLoader,
+			HighAvailabilityServices highAvailabilityServices) {
 		this.jobID = checkNotNull(jobID);
 		this.jobResultFuture = checkNotNull(jobResultFuture);
 		this.jobClientActor = checkNotNull(jobClientActor);
 		this.timeout = checkNotNull(timeout);
 		this.classLoader = checkNotNull(classLoader);
+		this.highAvailabilityServices = checkNotNull(highAvailabilityServices, "highAvailabilityServices");
 	}
 
 	/**
@@ -80,13 +86,15 @@ public JobListeningContext(
 		ActorRef jobClientActor,
 		FiniteDuration timeout,
 		ActorSystem actorSystem,
-		Configuration configuration) {
+		Configuration configuration,
+		HighAvailabilityServices highAvailabilityServices) {
 		this.jobID = checkNotNull(jobID);
 		this.jobResultFuture = checkNotNull(jobResultFuture);
 		this.jobClientActor = checkNotNull(jobClientActor);
 		this.timeout = checkNotNull(timeout);
 		this.actorSystem = checkNotNull(actorSystem);
 		this.configuration = checkNotNull(configuration);
+		this.highAvailabilityServices = checkNotNull(highAvailabilityServices, "highAvailabilityServices");
 	}
 
 	/**
@@ -135,7 +143,7 @@ public ClassLoader getClassLoader() throws JobRetrievalException {
 	private ActorGateway getJobManager() throws JobRetrievalException {
 		try {
 			return LeaderRetrievalUtils.retrieveLeaderGateway(
-				LeaderRetrievalUtils.createLeaderRetrievalService(configuration, true),
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 				actorSystem,
 				AkkaUtils.getLookupTimeout(configuration));
 		} catch (Exception e) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
index e356d2b1cb560..9bcaa18cf74dd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
@@ -29,8 +29,8 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.runtime.webmonitor.WebMonitor;
 import org.apache.flink.runtime.webmonitor.WebMonitorUtils;
 import org.apache.flink.util.NetUtils;
@@ -165,17 +165,20 @@ public static ActorSystem startActorSystem(
 
 	/**
 	 * Starts the web frontend.
+	 *
 	 * @param config The Flink config.
+	 * @param highAvailabilityServices Service factory for high availability services
 	 * @param actorSystem The ActorSystem to start the web frontend in.
 	 * @param logger Logger for log output
 	 * @return WebMonitor instance.
 	 * @throws Exception
 	 */
 	public static WebMonitor startWebMonitorIfConfigured(
-				Configuration config,
-				ActorSystem actorSystem,
-				ActorRef jobManager,
-				Logger logger) throws Exception {
+			Configuration config,
+			HighAvailabilityServices highAvailabilityServices,
+			ActorSystem actorSystem,
+			ActorRef jobManager,
+			Logger logger) throws Exception {
 
 
 		// this ensures correct values are present in the web frontend
@@ -186,8 +189,8 @@ public static WebMonitor startWebMonitorIfConfigured(
 		if (config.getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0) >= 0) {
 			logger.info("Starting JobManager Web Frontend");
 
-			LeaderRetrievalService leaderRetrievalService = 
-				LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager);
+			LeaderRetrievalService leaderRetrievalService =
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 
 			// start the web frontend. we need to load this dynamically
 			// because it is not in the same project/dependencies
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index 3de75d5cd01c5..69a65ddd5ba28 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -56,6 +56,13 @@ public interface HighAvailabilityServices extends AutoCloseable {
 	 */
 	UUID DEFAULT_LEADER_ID = new UUID(0, 0);
 
+	/**
+	 * This JobID should be used to identify the old JobManager when using the
+	 * {@link HighAvailabilityServices}. With Flip-6 every JobManager will have a distinct
+	 * JobID assigned.
+	 */
+	JobID DEFAULT_JOB_ID = new JobID(0L, 0L);
+
 	// ------------------------------------------------------------------------
 	//  Services
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
index 106be5a4d1b14..c9e295702f3f5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
@@ -115,7 +115,7 @@ public static Tuple2<String, Integer> getJobManagerAddress(Configuration configu
 		if (port <= 0 || port >= 65536) {
 			throw new ConfigurationException("Invalid value for '" + JobManagerOptions.PORT +
 				"' (port of the JobManager actor system) : " + port +
-				".  it must be great than 0 and less than 65536.");
+				".  it must be greater than 0 and less than 65536.");
 		}
 
 		return Tuple2.of(hostname, port);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
index 2a8af3735851f..2f5cd25760b22 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
@@ -483,6 +483,15 @@ public void addJar(Path jar) {
 		}
 	}
 
+	/**
+	 * Gets the list of assigned user jar paths.
+	 *
+	 * @return The list of assigned user jar paths
+	 */
+	public List<Path> getUserJars() {
+		return userJars;
+	}
+
 	/**
 	 * Adds the BLOB referenced by the key to the JobGraph's dependencies.
 	 *
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java
index aaafa762e81b3..2552088f5cf78 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java
@@ -143,11 +143,13 @@ public void stop() throws Exception {
 			if (isRunning) {
 				jobGraphListener = null;
 
-				pathCache.close();
-
-				client.close();
-
-				isRunning = false;
+				try {
+					pathCache.close();
+				} catch (Exception e) {
+					throw new Exception("Could not properly stop the ZooKeeperSubmittedJobGraphStore.", e);
+				} finally {
+					isRunning = false;
+				}
 			}
 		}
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java
index 0fa6a9eeb5db5..21e2ca1f90796 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.leaderelection;
 
+import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.Preconditions;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.recipes.cache.ChildData;
@@ -48,6 +49,8 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le
 
 	private static final Logger LOG = LoggerFactory.getLogger(ZooKeeperLeaderElectionService.class);
 
+	private final Object lock = new Object();
+
 	/** Client to the ZooKeeper quorum */
 	private final CuratorFramework client;
 
@@ -62,12 +65,12 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le
 
 	private UUID issuedLeaderSessionID;
 
-	private UUID confirmedLeaderSessionID;
+	private volatile UUID confirmedLeaderSessionID;
 
 	/** The leader contender which applies for leadership */
 	private volatile LeaderContender leaderContender;
 
-	private final Object lock = new Object();
+	private volatile boolean running;
 
 	private final ConnectionStateListener listener = new ConnectionStateListener() {
 		@Override
@@ -84,11 +87,17 @@ public void stateChanged(CuratorFramework client, ConnectionState newState) {
 	 * @param leaderPath ZooKeeper node path for the node which stores the current leader information
 	 */
 	public ZooKeeperLeaderElectionService(CuratorFramework client, String latchPath, String leaderPath) {
-		this.client = client;
-		this.leaderPath = leaderPath;
+		this.client = Preconditions.checkNotNull(client, "CuratorFramework client");
+		this.leaderPath = Preconditions.checkNotNull(leaderPath, "leaderPath");
 
 		leaderLatch = new LeaderLatch(client, latchPath);
 		cache = new NodeCache(client, leaderPath);
+
+		issuedLeaderSessionID = null;
+		confirmedLeaderSessionID = null;
+		leaderContender = null;
+
+		running = false;
 	}
 
 	/**
@@ -105,32 +114,56 @@ public void start(LeaderContender contender) throws Exception {
 		Preconditions.checkNotNull(contender, "Contender must not be null.");
 		Preconditions.checkState(leaderContender == null, "Contender was already set.");
 
-		LOG.info("Starting ZooKeeperLeaderElectionService.");
+		LOG.info("Starting ZooKeeperLeaderElectionService {}.", this);
+
+		synchronized (lock) {
 
-		leaderContender = contender;
+			leaderContender = contender;
 
-		leaderLatch.addListener(this);
-		leaderLatch.start();
+			leaderLatch.addListener(this);
+			leaderLatch.start();
 
-		cache.getListenable().addListener(this);
-		cache.start();
+			cache.getListenable().addListener(this);
+			cache.start();
 
-		client.getConnectionStateListenable().addListener(listener);
+			client.getConnectionStateListenable().addListener(listener);
+
+			running = true;
+		}
 	}
 
 	@Override
 	public void stop() throws Exception{
-		LOG.info("Stopping ZooKeeperLeaderElectionService.");
+		synchronized (lock) {
+			if (!running) {
+				return;
+			}
+
+			running = false;
+			confirmedLeaderSessionID = null;
+			issuedLeaderSessionID = null;
+		}
+
+		LOG.info("Stopping ZooKeeperLeaderElectionService {}.", this);
 
 		client.getConnectionStateListenable().removeListener(listener);
 
-		cache.close();
-		leaderLatch.close();
-		client.close();
+		Exception exception = null;
 
-		synchronized (lock) {
-			confirmedLeaderSessionID = null;
-			issuedLeaderSessionID = null;
+		try {
+			cache.close();
+		} catch (Exception e) {
+			exception = ExceptionUtils.firstOrSuppressed(e, exception);
+		}
+
+		try {
+			leaderLatch.close();
+		} catch(Exception e) {
+			exception = ExceptionUtils.firstOrSuppressed(e, exception);
+		}
+
+		if (exception != null) {
+			throw new Exception("Could not properly stop the ZooKeeperLeaderElectionService.", exception);
 		}
 	}
 
@@ -148,9 +181,14 @@ public void confirmLeaderSessionID(UUID leaderSessionID) {
 		if(leaderLatch.hasLeadership()) {
 			// check if this is an old confirmation call
 			synchronized (lock) {
-				if (leaderSessionID.equals(this.issuedLeaderSessionID)) {
-					confirmedLeaderSessionID = leaderSessionID;
-					writeLeaderInformation(confirmedLeaderSessionID);
+				if (running) {
+					if (leaderSessionID.equals(this.issuedLeaderSessionID)) {
+						confirmedLeaderSessionID = leaderSessionID;
+						writeLeaderInformation(confirmedLeaderSessionID);
+					}
+				} else {
+					LOG.debug("Ignoring the leader session Id {} confirmation, since the " +
+						"ZooKeeperLeaderElectionService has already been stopped.", leaderSessionID);
 				}
 			}
 		} else {
@@ -167,31 +205,41 @@ public boolean hasLeadership() {
 	@Override
 	public void isLeader() {
 		synchronized (lock) {
-			issuedLeaderSessionID = UUID.randomUUID();
-			confirmedLeaderSessionID = null;
+			if (running) {
+				issuedLeaderSessionID = UUID.randomUUID();
+				confirmedLeaderSessionID = null;
+
+				if (LOG.isDebugEnabled()) {
+					LOG.debug(
+						"Grant leadership to contender {} with session ID {}.",
+						leaderContender.getAddress(),
+						issuedLeaderSessionID);
+				}
 
-			if (LOG.isDebugEnabled()) {
-				LOG.debug(
-					"Grant leadership to contender {} with session ID {}.",
-					leaderContender.getAddress(),
-					issuedLeaderSessionID);
+				leaderContender.grantLeadership(issuedLeaderSessionID);
+			} else {
+				LOG.debug("Ignoring the grant leadership notification since the service has " +
+					"already been stopped.");
 			}
-
-			leaderContender.grantLeadership(issuedLeaderSessionID);
 		}
 	}
 
 	@Override
 	public void notLeader() {
 		synchronized (lock) {
-			issuedLeaderSessionID = null;
-			confirmedLeaderSessionID = null;
+			if (running) {
+				issuedLeaderSessionID = null;
+				confirmedLeaderSessionID = null;
 
-			if (LOG.isDebugEnabled()) {
-				LOG.debug("Revoke leadership of {}.", leaderContender.getAddress());
-			}
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Revoke leadership of {}.", leaderContender.getAddress());
+				}
 
-			leaderContender.revokeLeadership();
+				leaderContender.revokeLeadership();
+			} else {
+				LOG.debug("Ignoring the revoke leadership notification since the service " +
+					"has already been stopped.");
+			}
 		}
 	}
 
@@ -201,53 +249,57 @@ public void nodeChanged() throws Exception {
 			// leaderSessionID is null if the leader contender has not yet confirmed the session ID
 			if (leaderLatch.hasLeadership()) {
 				synchronized (lock) {
-					if (LOG.isDebugEnabled()) {
-						LOG.debug(
-							"Leader node changed while {} is the leader with session ID {}.",
-							leaderContender.getAddress(),
-							confirmedLeaderSessionID);
-					}
-
-					if (confirmedLeaderSessionID != null) {
-						ChildData childData = cache.getCurrentData();
+					if (running) {
+						if (LOG.isDebugEnabled()) {
+							LOG.debug(
+								"Leader node changed while {} is the leader with session ID {}.",
+								leaderContender.getAddress(),
+								confirmedLeaderSessionID);
+						}
 
-						if (childData == null) {
-							if (LOG.isDebugEnabled()) {
-								LOG.debug(
-									"Writing leader information into empty node by {}.",
-									leaderContender.getAddress());
-							}
-							writeLeaderInformation(confirmedLeaderSessionID);
-						} else {
-							byte[] data = childData.getData();
+						if (confirmedLeaderSessionID != null) {
+							ChildData childData = cache.getCurrentData();
 
-							if (data == null || data.length == 0) {
-								// the data field seems to be empty, rewrite information
+							if (childData == null) {
 								if (LOG.isDebugEnabled()) {
 									LOG.debug(
-										"Writing leader information into node with empty data field by {}.",
+										"Writing leader information into empty node by {}.",
 										leaderContender.getAddress());
 								}
 								writeLeaderInformation(confirmedLeaderSessionID);
 							} else {
-								ByteArrayInputStream bais = new ByteArrayInputStream(data);
-								ObjectInputStream ois = new ObjectInputStream(bais);
-
-								String leaderAddress = ois.readUTF();
-								UUID leaderSessionID = (UUID) ois.readObject();
+								byte[] data = childData.getData();
 
-								if (!leaderAddress.equals(this.leaderContender.getAddress()) ||
-										(leaderSessionID == null || !leaderSessionID.equals(confirmedLeaderSessionID))) {
-									// the data field does not correspond to the expected leader information
+								if (data == null || data.length == 0) {
+									// the data field seems to be empty, rewrite information
 									if (LOG.isDebugEnabled()) {
 										LOG.debug(
-											"Correcting leader information by {}.",
+											"Writing leader information into node with empty data field by {}.",
 											leaderContender.getAddress());
 									}
 									writeLeaderInformation(confirmedLeaderSessionID);
+								} else {
+									ByteArrayInputStream bais = new ByteArrayInputStream(data);
+									ObjectInputStream ois = new ObjectInputStream(bais);
+
+									String leaderAddress = ois.readUTF();
+									UUID leaderSessionID = (UUID) ois.readObject();
+
+									if (!leaderAddress.equals(this.leaderContender.getAddress()) ||
+										(leaderSessionID == null || !leaderSessionID.equals(confirmedLeaderSessionID))) {
+										// the data field does not correspond to the expected leader information
+										if (LOG.isDebugEnabled()) {
+											LOG.debug(
+												"Correcting leader information by {}.",
+												leaderContender.getAddress());
+										}
+										writeLeaderInformation(confirmedLeaderSessionID);
+									}
 								}
 							}
 						}
+					} else {
+						LOG.debug("Ignoring node change notification since the service has already been stopped.");
 					}
 				}
 			}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java
index f74fb1a793afa..7f18fd9394a18 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java
@@ -29,6 +29,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.util.Objects;
 import java.util.UUID;
@@ -43,6 +44,8 @@ public class ZooKeeperLeaderRetrievalService implements LeaderRetrievalService,
 	private static final Logger LOG = LoggerFactory.getLogger(
 		ZooKeeperLeaderRetrievalService.class);
 
+	private final Object lock = new Object();
+
 	/** Connection to the used ZooKeeper quorum */
 	private final CuratorFramework client;
 
@@ -53,8 +56,11 @@ public class ZooKeeperLeaderRetrievalService implements LeaderRetrievalService,
 	private volatile LeaderRetrievalListener leaderListener;
 
 	private String lastLeaderAddress;
+
 	private UUID lastLeaderSessionID;
 
+	private volatile boolean running;
+
 	private final ConnectionStateListener connectionStateListener = new ConnectionStateListener() {
 		@Override
 		public void stateChanged(CuratorFramework client, ConnectionState newState) {
@@ -69,8 +75,14 @@ public void stateChanged(CuratorFramework client, ConnectionState newState) {
 	 * @param retrievalPath Path of the ZooKeeper node which contains the leader information
 	 */
 	public ZooKeeperLeaderRetrievalService(CuratorFramework client, String retrievalPath) {
-		this.client = client;
+		this.client = Preconditions.checkNotNull(client, "CuratorFramework client");
 		this.cache = new NodeCache(client, retrievalPath);
+
+		this.leaderListener = null;
+		this.lastLeaderAddress = null;
+		this.lastLeaderSessionID = null;
+
+		running = false;
 	}
 
 	@Override
@@ -81,65 +93,87 @@ public void start(LeaderRetrievalListener listener) throws Exception {
 
 		LOG.info("Starting ZooKeeperLeaderRetrievalService.");
 
-		leaderListener = listener;
+		synchronized (lock) {
+			leaderListener = listener;
+
+			cache.getListenable().addListener(this);
+			cache.start();
 
-		cache.getListenable().addListener(this);
-		cache.start();
+			client.getConnectionStateListenable().addListener(connectionStateListener);
 
-		client.getConnectionStateListenable().addListener(connectionStateListener);
+			running = true;
+		}
 	}
 
 	@Override
 	public void stop() throws Exception {
 		LOG.info("Stopping ZooKeeperLeaderRetrievalService.");
 
+		synchronized (lock) {
+			if (!running) {
+				return;
+			}
+
+			running = false;
+		}
+
 		client.getConnectionStateListenable().removeListener(connectionStateListener);
 
-		cache.close();
+		try {
+			cache.close();
+		} catch (IOException e) {
+			throw new Exception("Could not properly stop the ZooKeeperLeaderRetrievalService.", e);
+		}
 	}
 
 	@Override
 	public void nodeChanged() throws Exception {
-		try {
-			LOG.debug("Leader node has changed.");
-
-			ChildData childData = cache.getCurrentData();
-
-			String leaderAddress;
-			UUID leaderSessionID;
-
-			if (childData == null) {
-				leaderAddress = null;
-				leaderSessionID = null;
-			} else {
-				byte[] data = childData.getData();
-
-				if (data == null || data.length == 0) {
-					leaderAddress = null;
-					leaderSessionID = null;
-				} else {
-					ByteArrayInputStream bais = new ByteArrayInputStream(data);
-					ObjectInputStream ois = new ObjectInputStream(bais);
-
-					leaderAddress = ois.readUTF();
-					leaderSessionID = (UUID) ois.readObject();
+		synchronized (lock) {
+			if (running) {
+				try {
+					LOG.debug("Leader node has changed.");
+
+					ChildData childData = cache.getCurrentData();
+
+					String leaderAddress;
+					UUID leaderSessionID;
+
+					if (childData == null) {
+						leaderAddress = null;
+						leaderSessionID = null;
+					} else {
+						byte[] data = childData.getData();
+
+						if (data == null || data.length == 0) {
+							leaderAddress = null;
+							leaderSessionID = null;
+						} else {
+							ByteArrayInputStream bais = new ByteArrayInputStream(data);
+							ObjectInputStream ois = new ObjectInputStream(bais);
+
+							leaderAddress = ois.readUTF();
+							leaderSessionID = (UUID) ois.readObject();
+						}
+					}
+
+					if (!(Objects.equals(leaderAddress, lastLeaderAddress) &&
+						Objects.equals(leaderSessionID, lastLeaderSessionID))) {
+						LOG.debug(
+							"New leader information: Leader={}, session ID={}.",
+							leaderAddress,
+							leaderSessionID);
+
+						lastLeaderAddress = leaderAddress;
+						lastLeaderSessionID = leaderSessionID;
+						leaderListener.notifyLeaderAddress(leaderAddress, leaderSessionID);
+					}
+				} catch (Exception e) {
+					leaderListener.handleError(new Exception("Could not handle node changed event.", e));
+					throw e;
 				}
+			} else {
+				LOG.debug("Ignoring node change notification since the service has already been stopped.");
 			}
-
-			if(!(Objects.equals(leaderAddress, lastLeaderAddress) &&
-					Objects.equals(leaderSessionID, lastLeaderSessionID))) {
-				LOG.debug(
-					"New leader information: Leader={}, session ID={}.",
-					leaderAddress,
-					leaderSessionID);
-
-				lastLeaderAddress = leaderAddress;
-				lastLeaderSessionID = leaderSessionID;
-				leaderListener.notifyLeaderAddress(leaderAddress, leaderSessionID);
-			}
-		} catch (Exception e) {
-			leaderListener.handleError(new Exception("Could not handle node changed event.", e));
-			throw e;
 		}
 	}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java
new file mode 100644
index 0000000000000..07178fcf2d889
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.minicluster;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.pattern.Patterns;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.runtime.jobmanager.JobManager;
+import org.apache.flink.runtime.jobmanager.MemoryArchivist;
+import org.apache.flink.runtime.messages.TaskManagerMessages;
+import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import scala.Option;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Mini cluster to run the old JobManager code without embedded high availability services. This
+ * class has been implemented because the normal {@link FlinkMiniCluster} has been changed to use
+ * the {@link HighAvailabilityServices}. With this change we can no longer use the
+ * {@link org.apache.flink.api.java.RemoteEnvironment} to connect against the
+ * {@link FlinkMiniCluster}, because the remote environment cannot retrieve the current leader
+ * session id.
+ */
+public class StandaloneMiniCluster {
+
+	private static final String LOCAL_HOSTNAME = "localhost";
+
+	private final Configuration configuration;
+
+	private final ActorSystem actorSystem;
+
+	private final ScheduledExecutorService scheduledExecutorService;
+
+	private final HighAvailabilityServices highAvailabilityServices;
+
+	private final FiniteDuration timeout;
+
+	private final int port;
+
+	public StandaloneMiniCluster(Configuration configuration) throws Exception {
+		this.configuration = Preconditions.checkNotNull(configuration);
+
+		timeout = AkkaUtils.getTimeout(configuration);
+
+		actorSystem = JobManager.startActorSystem(
+			configuration,
+			LOCAL_HOSTNAME,
+			0);
+
+		port = configuration.getInteger(JobManagerOptions.PORT);
+
+		scheduledExecutorService = new ScheduledThreadPoolExecutor(1);
+
+		highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			configuration,
+			Executors.directExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);
+
+		JobManager.startJobManagerActors(
+			configuration,
+			actorSystem,
+			scheduledExecutorService,
+			scheduledExecutorService,
+			highAvailabilityServices,
+			JobManager.class,
+			MemoryArchivist.class);
+
+		ActorRef taskManager = TaskManager.startTaskManagerComponentsAndActor(
+			configuration,
+			ResourceID.generate(),
+			actorSystem,
+			highAvailabilityServices,
+			LOCAL_HOSTNAME,
+			Option.<String>empty(),
+			true,
+			TaskManager.class);
+
+		Future<Object> registrationFuture = Patterns.ask(
+			taskManager,
+			TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(),
+			timeout.toMillis());
+
+		Await.ready(registrationFuture, timeout);
+	}
+
+	public String getHostname() {
+		return LOCAL_HOSTNAME;
+	}
+
+	public int getPort() {
+		return port;
+	}
+
+	public Configuration getConfiguration() {
+		return configuration;
+	}
+
+	public void close() throws Exception {
+		Exception exception = null;
+
+		actorSystem.shutdown();
+		actorSystem.awaitTermination();
+
+		try {
+			highAvailabilityServices.closeAndCleanupAllData();
+		} catch (Exception e) {
+			exception = e;
+		}
+
+		scheduledExecutorService.shutdownNow();
+
+		try {
+			scheduledExecutorService.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS);
+		} catch (InterruptedException e) {
+			Thread.currentThread().interrupt();
+
+			exception = ExceptionUtils.firstOrSuppressed(e, exception);
+		}
+
+		if (exception != null) {
+			throw exception;
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java
index f9cf01de75214..9b05273bcb5bd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/query/QueryableStateClient.java
@@ -29,6 +29,7 @@
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.configuration.QueryableStateOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.query.AkkaKvStateLocationLookupService.FixedDelayLookupRetryStrategyFactory;
 import org.apache.flink.runtime.query.AkkaKvStateLocationLookupService.LookupRetryStrategyFactory;
@@ -38,7 +39,6 @@
 import org.apache.flink.runtime.query.netty.UnknownKeyOrNamespace;
 import org.apache.flink.runtime.query.netty.UnknownKvStateID;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -102,14 +102,16 @@ public class QueryableStateClient {
 	 * system and another for the network client.
 	 *
 	 * @param config Configuration to use.
+	 * @param highAvailabilityServices Service factory for high availability services
 	 * @throws Exception Failures are forwarded
 	 */
-	public QueryableStateClient(Configuration config) throws Exception {
+	public QueryableStateClient(
+			Configuration config,
+			HighAvailabilityServices highAvailabilityServices) throws Exception {
 		Preconditions.checkNotNull(config, "Configuration");
 
 		// Create a leader retrieval service
-		LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils
-				.createLeaderRetrievalService(config, true);
+		LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 
 		// Get the ask timeout
 		String askTimeoutString = config.getString(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceUtils.java
index eab4de8a602b3..8789eedf0427b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceUtils.java
@@ -112,10 +112,12 @@ public static RpcService createRpcService(String hostname, int port, Configurati
 
 	/**
 	 *
-	 * @param hostname     The hostname or address where the target RPC service is listening.
-	 * @param port         The port where the target RPC service is listening.
+	 * @param hostname The hostname or address where the target RPC service is listening.
+	 * @param port The port where the target RPC service is listening.
 	 * @param endpointName The name of the RPC endpoint.
-	 * @param config       The configuration from which to deduce further settings.
+	 * @param addressResolution Whether to try address resolution of the given hostname or not.
+	 *                          This allows to fail fast in case that the hostname cannot be resolved.
+	 * @param config The configuration from which to deduce further settings.
 	 *
 	 * @return The RPC URL of the specified RPC endpoint.
 	 */
@@ -143,10 +145,12 @@ public static String getRpcUrl(
 
 	/**
 	 * 
-	 * @param hostname     The hostname or address where the target RPC service is listening.
-	 * @param port         The port where the target RPC service is listening.
+	 * @param hostname The hostname or address where the target RPC service is listening.
+	 * @param port The port where the target RPC service is listening.
 	 * @param endpointName The name of the RPC endpoint.
-	 * @param akkaProtocol       True, if security/encryption is enabled, false otherwise.
+	 * @param addressResolution Whether to try address resolution of the given hostname or not.
+	 *                          This allows to fail fast in case that the hostname cannot be resolved.
+	 * @param akkaProtocol True, if security/encryption is enabled, false otherwise.
 	 * 
 	 * @return The RPC URL of the specified RPC endpoint.
 	 */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
index 05749c43cf631..073c52b17e0c3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
@@ -33,7 +33,6 @@
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.net.ConnectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,57 +50,6 @@
 public class LeaderRetrievalUtils {
 
 	private static final Logger LOG = LoggerFactory.getLogger(LeaderRetrievalUtils.class);
-
-	/**
-	 * Creates a {@link LeaderRetrievalService} based on the provided {@link Configuration} object.
-	 *
-	 * @param configuration Configuration containing the settings for the {@link LeaderRetrievalService}
-	 * @param resolveInitialHostName If true, resolves the initial hostname
-	 * @return The {@link LeaderRetrievalService} specified in the configuration object
-	 * @throws Exception
-	 */
-	public static LeaderRetrievalService createLeaderRetrievalService(
-			Configuration configuration,
-			boolean resolveInitialHostName)
-		throws Exception {
-
-		HighAvailabilityMode highAvailabilityMode = getRecoveryMode(configuration);
-
-		switch (highAvailabilityMode) {
-			case NONE:
-				return StandaloneUtils.createLeaderRetrievalService(configuration, resolveInitialHostName, null);
-			case ZOOKEEPER:
-				return ZooKeeperUtils.createLeaderRetrievalService(configuration);
-			default:
-				throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
-		}
-	}
-
-	/**
-	 * Creates a {@link LeaderRetrievalService} that either uses the distributed leader election
-	 * configured in the configuration, or, in standalone mode, the given actor reference.
-	 *
-	 * @param configuration Configuration containing the settings for the {@link LeaderRetrievalService}
-	 * @param standaloneRef Actor reference to be used in standalone mode. 
-	 *                      
-	 * @return The {@link LeaderRetrievalService} specified in the configuration object
-	 * @throws Exception
-	 */
-	public static LeaderRetrievalService createLeaderRetrievalService(
-				Configuration configuration, ActorRef standaloneRef) throws Exception {
-
-		HighAvailabilityMode highAvailabilityMode = getRecoveryMode(configuration);
-
-		switch (highAvailabilityMode) {
-			case NONE:
-				String akkaUrl = standaloneRef.path().toSerializationFormat();
-				return new StandaloneLeaderRetrievalService(akkaUrl);
-			case ZOOKEEPER:
-				return ZooKeeperUtils.createLeaderRetrievalService(configuration);
-			default:
-				throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
-		}
-	}
 	
 	/**
 	 * Retrieves the current leader gateway using the given {@link LeaderRetrievalService}. If the
@@ -173,6 +121,21 @@ public static LeaderConnectionInfo retrieveLeaderConnectionInfo(
 		}
 	}
 
+	/**
+	 * Retrieves the current leader session id of the component identified by the given leader
+	 * retrieval service.
+	 *
+	 * @param leaderRetrievalService Leader retrieval service to be used for the leader retrieval
+	 * @param timeout Timeout for the leader retrieval
+	 * @return The leader session id of the retrieved leader
+	 * @throws LeaderRetrievalException if the leader retrieval operation fails (including a timeout)
+	 */
+	public static UUID retrieveLeaderSessionId(
+			LeaderRetrievalService leaderRetrievalService,
+			FiniteDuration timeout) throws LeaderRetrievalException {
+		return retrieveLeaderConnectionInfo(leaderRetrievalService, timeout).getLeaderSessionID();
+	}
+
 	public static InetAddress findConnectingAddress(
 		LeaderRetrievalService leaderRetrievalService,
 		Time timeout) throws LeaderRetrievalException {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
index ed485d0213b44..f6f08e429a5d0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
@@ -135,6 +135,10 @@ public Throwable deserializeError(ClassLoader classloader) {
 		return cached;
 	}
 
+	public String getOriginalErrorClassName() {
+		return originalErrorClassName;
+	}
+
 	// ------------------------------------------------------------------------
 	//  Override the behavior of Throwable
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
index 1b73dc76e13e9..9ade5ece5324c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
@@ -156,18 +156,6 @@ public static String getZooKeeperEnsemble(Configuration flinkConf)
 		return zkQuorum;
 	}
 
-	/**
-	 * Creates a {@link ZooKeeperLeaderRetrievalService} instance.
-	 *
-	 * @param configuration {@link Configuration} object containing the configuration values
-	 * @return {@link ZooKeeperLeaderRetrievalService} instance.
-	 */
-	public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
-			Configuration configuration) throws Exception {
-		final CuratorFramework client = startCuratorFramework(configuration);
-		return createLeaderRetrievalService(client, configuration);
-	}
-
 	/**
 	 * Creates a {@link ZooKeeperLeaderRetrievalService} instance.
 	 *
@@ -206,21 +194,6 @@ public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
 		return new ZooKeeperLeaderRetrievalService(client, leaderPath);
 	}
 
-	/**
-	 * Creates a {@link ZooKeeperLeaderElectionService} instance and a new {@link
-	 * CuratorFramework} client.
-	 *
-	 * @param configuration {@link Configuration} object containing the configuration values
-	 * @return {@link ZooKeeperLeaderElectionService} instance.
-	 */
-	public static ZooKeeperLeaderElectionService createLeaderElectionService(
-			Configuration configuration) throws Exception {
-
-		CuratorFramework client = startCuratorFramework(configuration);
-
-		return createLeaderElectionService(client, configuration);
-	}
-
 	/**
 	 * Creates a {@link ZooKeeperLeaderElectionService} instance.
 	 *
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 68f43da8ba0cf..5092643c490e5 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -49,7 +49,7 @@ import org.apache.flink.runtime.execution.SuppressRestartsException
 import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory
 import org.apache.flink.runtime.executiongraph._
-import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils
+import org.apache.flink.runtime.highavailability.{HighAvailabilityServices, HighAvailabilityServicesUtils}
 import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution
 import org.apache.flink.runtime.instance.{AkkaActorGateway, InstanceID, InstanceManager}
 import org.apache.flink.runtime.jobgraph.{JobGraph, JobStatus}
@@ -57,8 +57,8 @@ import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore.SubmittedJobGr
 import org.apache.flink.runtime.jobmanager.scheduler.{Scheduler => FlinkScheduler}
 import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway
 import org.apache.flink.runtime.jobmaster.JobMaster
+import org.apache.flink.runtime.leaderelection.{LeaderContender, LeaderElectionService}
 import org.apache.flink.runtime.jobmaster.JobMaster.{ARCHIVE_NAME, JOB_MANAGER_NAME}
-import org.apache.flink.runtime.leaderelection.{LeaderContender, LeaderElectionService, StandaloneLeaderElectionService}
 import org.apache.flink.runtime.messages.ArchiveMessages.ArchiveExecutionGraph
 import org.apache.flink.runtime.messages.ExecutionGraphMessages.JobStatusChanged
 import org.apache.flink.runtime.messages.JobManagerMessages._
@@ -2037,14 +2037,27 @@ object JobManager {
 
     val timeout = AkkaUtils.getTimeout(configuration)
 
-    val (jobManagerSystem, _, _, webMonitorOption, _) = try {
-      startActorSystemAndJobManagerActors(
+    // we have to first start the JobManager ActorSystem because this determines the port if 0
+    // was chosen before. The method startActorSystem will update the configuration correspondingly.
+    val jobManagerSystem = startActorSystem(
+      configuration,
+      listeningAddress,
+      listeningPort)
+
+    val highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+      configuration,
+      ioExecutor,
+      AddressResolution.NO_ADDRESS_RESOLUTION);
+
+    val (_, _, webMonitorOption, _) = try {
+      startJobManagerActors(
+        jobManagerSystem,
         configuration,
         executionMode,
         listeningAddress,
-        listeningPort,
         futureExecutor,
         ioExecutor,
+        highAvailabilityServices,
         classOf[JobManager],
         classOf[MemoryArchivist],
         Option(classOf[StandaloneResourceManager])
@@ -2070,6 +2083,13 @@ object JobManager {
         }
     }
 
+    try {
+      highAvailabilityServices.close()
+    } catch {
+      case t: Throwable =>
+        LOG.warn("Could not properly stop the high availability services.", t)
+    }
+
     FlinkExecutors.gracefulShutdown(
       timeout.toMillis,
       TimeUnit.MILLISECONDS,
@@ -2175,32 +2195,18 @@ object JobManager {
     }
   }
 
-  /** Starts an ActorSystem, the JobManager and all its components including the WebMonitor.
+  /**
+    * Starts the JobManager actor system.
     *
-    * @param configuration The configuration object for the JobManager
-    * @param executionMode The execution mode in which to run. Execution mode LOCAL with spawn an
-    *                      additional TaskManager in the same process.
-    * @param externalHostname The hostname where the JobManager is reachable for rpc communication
-    * @param port The port where the JobManager is reachable for rpc communication
-    * @param futureExecutor to run the JobManager's futures
-    * @param ioExecutor to run blocking io operations
-    * @param jobManagerClass The class of the JobManager to be started
-    * @param archiveClass The class of the Archivist to be started
-    * @param resourceManagerClass Optional class of resource manager if one should be started
-    * @return A tuple containing the started ActorSystem, ActorRefs to the JobManager and the
-    *         Archivist and an Option containing a possibly started WebMonitor
+    * @param configuration Configuration to use for the job manager actor system
+    * @param externalHostname External hostname to bind to
+    * @param port Port to bind to
+    * @return Actor system for the JobManager and its components
     */
-  def startActorSystemAndJobManagerActors(
+  def startActorSystem(
       configuration: Configuration,
-      executionMode: JobManagerMode,
       externalHostname: String,
-      port: Int,
-      futureExecutor: ScheduledExecutorService,
-      ioExecutor: Executor,
-      jobManagerClass: Class[_ <: JobManager],
-      archiveClass: Class[_ <: MemoryArchivist],
-      resourceManagerClass: Option[Class[_ <: FlinkResourceManager[_]]])
-    : (ActorSystem, ActorRef, ActorRef, Option[WebMonitor], Option[ActorRef]) = {
+      port: Int): ActorSystem = {
 
     val hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(externalHostname, port)
 
@@ -2223,7 +2229,7 @@ object JobManager {
           val cause = t.getCause()
           if (cause != null && t.getCause().isInstanceOf[java.net.BindException]) {
             throw new Exception("Unable to create JobManager at address " + hostPort +
-              " - " + cause.getMessage(), t)
+                                  " - " + cause.getMessage(), t)
           }
         }
         throw new Exception("Could not create JobManager actor system", t)
@@ -2234,11 +2240,42 @@ object JobManager {
     configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, address.host.get)
     configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, address.port.get)
 
+    jobManagerSystem
+  }
+
+  /** Starts the JobManager and all its components including the WebMonitor.
+    *
+    * @param configuration The configuration object for the JobManager
+    * @param executionMode The execution mode in which to run. Execution mode LOCAL with spawn an
+    *                      additional TaskManager in the same process.
+    * @param externalHostname The hostname where the JobManager is reachable for rpc communication
+    * @param futureExecutor to run the JobManager's futures
+    * @param ioExecutor to run blocking io operations
+    * @param highAvailabilityServices to instantiate high availability services
+    * @param jobManagerClass The class of the JobManager to be started
+    * @param archiveClass The class of the Archivist to be started
+    * @param resourceManagerClass Optional class of resource manager if one should be started
+    * @return A tuple containing the started ActorSystem, ActorRefs to the JobManager and the
+    *         Archivist and an Option containing a possibly started WebMonitor
+    */
+  def startJobManagerActors(
+      jobManagerSystem: ActorSystem,
+      configuration: Configuration,
+      executionMode: JobManagerMode,
+      externalHostname: String,
+      futureExecutor: ScheduledExecutorService,
+      ioExecutor: Executor,
+      highAvailabilityServices: HighAvailabilityServices,
+      jobManagerClass: Class[_ <: JobManager],
+      archiveClass: Class[_ <: MemoryArchivist],
+      resourceManagerClass: Option[Class[_ <: FlinkResourceManager[_]]])
+    : (ActorRef, ActorRef, Option[WebMonitor], Option[ActorRef]) = {
+
     val webMonitor: Option[WebMonitor] =
       if (configuration.getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0) >= 0) {
         LOG.info("Starting JobManager web frontend")
-        val leaderRetrievalService = LeaderRetrievalUtils
-          .createLeaderRetrievalService(configuration, false)
+        val leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
+          HighAvailabilityServices.DEFAULT_JOB_ID)
 
         // start the web frontend. we need to load this dynamically
         // because it is not in the same project/dependencies
@@ -2265,6 +2302,7 @@ object JobManager {
         jobManagerSystem,
         futureExecutor,
         ioExecutor,
+        highAvailabilityServices,
         jobManagerClass,
         archiveClass)
 
@@ -2287,9 +2325,9 @@ object JobManager {
           configuration,
           ResourceID.generate(),
           jobManagerSystem,
+          highAvailabilityServices,
           externalHostname,
           Some(TaskExecutor.TASK_MANAGER_NAME),
-          None,
           localTaskManagerCommunication = true,
           classOf[TaskManager])
 
@@ -2324,14 +2362,15 @@ object JobManager {
               FlinkResourceManager.startResourceManagerActors(
                 configuration,
                 jobManagerSystem,
-                LeaderRetrievalUtils.createLeaderRetrievalService(configuration, false),
+                highAvailabilityServices.getJobManagerLeaderRetriever(
+                  HighAvailabilityServices.DEFAULT_JOB_ID),
                 rmClass))
           case None =>
             LOG.info("Resource Manager class not provided. No resource manager will be started.")
             None
         }
 
-      (jobManagerSystem, jobManager, archive, webMonitor, resourceManager)
+      (jobManager, archive, webMonitor, resourceManager)
     }
     catch {
       case t: Throwable =>
@@ -2468,15 +2507,12 @@ object JobManager {
    * @param configuration The configuration from which to parse the config values.
    * @param futureExecutor to run JobManager's futures
    * @param ioExecutor to run blocking io operations
-   * @param leaderElectionServiceOption LeaderElectionService which shall be returned if the option
-   *                                    is defined
    * @return The members for a default JobManager.
    */
   def createJobManagerComponents(
       configuration: Configuration,
       futureExecutor: ScheduledExecutorService,
-      ioExecutor: Executor,
-      leaderElectionServiceOption: Option[LeaderElectionService]) :
+      ioExecutor: Executor) :
     (InstanceManager,
     FlinkScheduler,
     BlobLibraryCacheManager,
@@ -2484,9 +2520,6 @@ object JobManager {
     FiniteDuration, // timeout
     Int, // number of archived jobs
     Option[Path], // archive path
-    LeaderElectionService,
-    SubmittedJobGraphStore,
-    CheckpointRecoveryFactory,
     FiniteDuration, // timeout for job recovery
     Option[FlinkMetricRegistry]
    ) = {
@@ -2550,32 +2583,6 @@ object JobManager {
         throw t
     }
 
-    // Create recovery related components
-    val (leaderElectionService, submittedJobGraphs, checkpointRecoveryFactory) =
-      HighAvailabilityMode.fromConfig(configuration) match {
-        case HighAvailabilityMode.NONE =>
-          val leaderElectionService = leaderElectionServiceOption match {
-            case Some(les) => les
-            case None => new StandaloneLeaderElectionService()
-          }
-
-          (leaderElectionService,
-            new StandaloneSubmittedJobGraphStore(),
-            new StandaloneCheckpointRecoveryFactory())
-
-        case HighAvailabilityMode.ZOOKEEPER =>
-          val client = ZooKeeperUtils.startCuratorFramework(configuration)
-
-          val leaderElectionService = leaderElectionServiceOption match {
-            case Some(les) => les
-            case None => ZooKeeperUtils.createLeaderElectionService(client, configuration)
-          }
-
-          (leaderElectionService,
-            ZooKeeperUtils.createSubmittedJobGraphs(client, configuration, ioExecutor),
-            new ZooKeeperCheckpointRecoveryFactory(client, configuration, ioExecutor))
-      }
-
     val jobRecoveryTimeoutStr = configuration.getValue(HighAvailabilityOptions.HA_JOB_DELAY)
 
     val jobRecoveryTimeout = if (jobRecoveryTimeoutStr == null || jobRecoveryTimeoutStr.isEmpty) {
@@ -2605,9 +2612,6 @@ object JobManager {
       timeout,
       archiveCount,
       archivePath,
-      leaderElectionService,
-      submittedJobGraphs,
-      checkpointRecoveryFactory,
       jobRecoveryTimeout,
       metricRegistry)
   }
@@ -2629,6 +2633,7 @@ object JobManager {
       actorSystem: ActorSystem,
       futureExecutor: ScheduledExecutorService,
       ioExecutor: Executor,
+      highAvailabilityServices: HighAvailabilityServices,
       jobManagerClass: Class[_ <: JobManager],
       archiveClass: Class[_ <: MemoryArchivist])
     : (ActorRef, ActorRef) = {
@@ -2638,6 +2643,7 @@ object JobManager {
       actorSystem,
       futureExecutor,
       ioExecutor,
+      highAvailabilityServices,
       Some(JobMaster.JOB_MANAGER_NAME),
       Some(JobMaster.ARCHIVE_NAME),
       jobManagerClass,
@@ -2665,6 +2671,7 @@ object JobManager {
       actorSystem: ActorSystem,
       futureExecutor: ScheduledExecutorService,
       ioExecutor: Executor,
+      highAvailabilityServices: HighAvailabilityServices,
       jobManagerActorName: Option[String],
       archiveActorName: Option[String],
       jobManagerClass: Class[_ <: JobManager],
@@ -2678,15 +2685,11 @@ object JobManager {
     timeout,
     archiveCount,
     archivePath,
-    leaderElectionService,
-    submittedJobGraphs,
-    checkpointRecoveryFactory,
     jobRecoveryTimeout,
     metricsRegistry) = createJobManagerComponents(
       configuration,
       futureExecutor,
-      ioExecutor,
-      None)
+      ioExecutor)
 
     val archiveProps = getArchiveProps(archiveClass, archiveCount, archivePath)
 
@@ -2707,9 +2710,10 @@ object JobManager {
       archive,
       restartStrategy,
       timeout,
-      leaderElectionService,
-      submittedJobGraphs,
-      checkpointRecoveryFactory,
+      highAvailabilityServices.getJobManagerLeaderElectionService(
+        HighAvailabilityServices.DEFAULT_JOB_ID),
+      highAvailabilityServices.getSubmittedJobGraphStore(),
+      highAvailabilityServices.getCheckpointRecoveryFactory(),
       jobRecoveryTimeout,
       metricsRegistry)
 
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
index a493b3d5e5a77..79141f1bf9297 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
@@ -101,9 +101,10 @@ object TaskManagerMessages {
 
   /**
    * Acknowledges that the task manager has been successfully registered at any job manager. This
-   * message is a response to [[NotifyWhenRegisteredAtJobManager]].
+   * message is a response to [[NotifyWhenRegisteredAtJobManager]] and contains the current leader
+   * session id.
    */
-  case object RegisteredAtJobManager
+  case class RegisteredAtJobManager(leaderId: UUID)
 
   /** Tells the address of the new leading [[org.apache.flink.runtime.jobmanager.JobManager]]
     * and the new leader session ID.
@@ -150,14 +151,6 @@ object TaskManagerMessages {
   def getNotifyWhenRegisteredAtJobManagerMessage:
   NotifyWhenRegisteredAtJobManager.type = NotifyWhenRegisteredAtJobManager
 
-  /**
-   * Accessor for the case object instance, to simplify Java interoperability.
-   *
-   * @return The RegisteredAtJobManager case object instance.
-   */
-  def getRegisteredAtJobManagerMessage:
-            RegisteredAtJobManager.type = RegisteredAtJobManager
-
   /**
     * Accessor for the case object instance, to simplify Java interoperability.
     * @return The RequestTaskManagerLog case object instance.
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
index 07fb9969c3e8e..46c440464efda 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.minicluster
 
+import java.net.URL
 import java.util.UUID
 import java.util.concurrent.{Executors, TimeUnit}
 
@@ -25,20 +26,21 @@ import akka.pattern.Patterns.gracefulStop
 import akka.pattern.ask
 import akka.actor.{ActorRef, ActorSystem}
 import com.typesafe.config.Config
-
 import org.apache.flink.api.common.{JobExecutionResult, JobID, JobSubmissionResult}
 import org.apache.flink.configuration.{ConfigConstants, Configuration}
+import org.apache.flink.core.fs.Path
 import org.apache.flink.runtime.akka.AkkaUtils
 import org.apache.flink.runtime.client.{JobClient, JobExecutionException}
 import org.apache.flink.runtime.concurrent.{Executors => FlinkExecutors}
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoader
+import org.apache.flink.runtime.highavailability.{HighAvailabilityServices, HighAvailabilityServicesUtils}
 import org.apache.flink.runtime.instance.{ActorGateway, AkkaActorGateway}
 import org.apache.flink.runtime.jobgraph.JobGraph
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode
 import org.apache.flink.runtime.leaderretrieval.{LeaderRetrievalListener, LeaderRetrievalService, StandaloneLeaderRetrievalService}
 import org.apache.flink.runtime.messages.TaskManagerMessages.NotifyWhenRegisteredAtJobManager
-import org.apache.flink.runtime.util.{ExecutorThreadFactory, Hardware, ZooKeeperUtils}
+import org.apache.flink.runtime.util.{ExecutorThreadFactory, Hardware}
 import org.apache.flink.runtime.webmonitor.{WebMonitor, WebMonitorUtils}
-
 import org.slf4j.LoggerFactory
 
 import scala.concurrent.duration.{Duration, FiniteDuration}
@@ -56,6 +58,7 @@ import scala.concurrent._
  */
 abstract class FlinkMiniCluster(
     val userConfiguration: Configuration,
+    val highAvailabilityServices: HighAvailabilityServices,
     val useSingleActorSystem: Boolean)
   extends LeaderRetrievalListener {
 
@@ -115,6 +118,15 @@ abstract class FlinkMiniCluster(
     Hardware.getNumberCPUCores(),
     new ExecutorThreadFactory("mini-cluster-io"))
 
+  def this(configuration: Configuration, useSingleActorSystem: Boolean) {
+    this(
+      configuration,
+      HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+        configuration,
+        ExecutionContext.global),
+      useSingleActorSystem)
+  }
+
   def configuration: Configuration = {
     if (originalConfiguration.getInteger(
       ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
@@ -148,25 +160,17 @@ abstract class FlinkMiniCluster(
   // --------------------------------------------------------------------------
 
   def getNumberOfJobManagers: Int = {
-    if(haMode == HighAvailabilityMode.NONE) {
-      1
-    } else {
-      originalConfiguration.getInteger(
-        ConfigConstants.LOCAL_NUMBER_JOB_MANAGER,
-        ConfigConstants.DEFAULT_LOCAL_NUMBER_JOB_MANAGER
-      )
-    }
+    originalConfiguration.getInteger(
+      ConfigConstants.LOCAL_NUMBER_JOB_MANAGER,
+      ConfigConstants.DEFAULT_LOCAL_NUMBER_JOB_MANAGER
+    )
   }
 
   def getNumberOfResourceManagers: Int = {
-    if(haMode == HighAvailabilityMode.NONE) {
-      1
-    } else {
-      originalConfiguration.getInteger(
-        ConfigConstants.LOCAL_NUMBER_RESOURCE_MANAGER,
-        ConfigConstants.DEFAULT_LOCAL_NUMBER_RESOURCE_MANAGER
-      )
-    }
+    originalConfiguration.getInteger(
+      ConfigConstants.LOCAL_NUMBER_RESOURCE_MANAGER,
+      ConfigConstants.DEFAULT_LOCAL_NUMBER_RESOURCE_MANAGER
+    )
   }
 
   def getJobManagersAsJava = {
@@ -327,10 +331,11 @@ abstract class FlinkMiniCluster(
     jobManagerActorSystems = Some(jmActorSystems)
     jobManagerActors = Some(jmActors)
 
-    // start leader retrieval service
-    val lrs = createLeaderRetrievalService()
-    jobManagerLeaderRetrievalService = Some(lrs)
-    lrs.start(this)
+    // find out which job manager the leader is
+    jobManagerLeaderRetrievalService = Option(highAvailabilityServices.getJobManagerLeaderRetriever(
+      HighAvailabilityServices.DEFAULT_JOB_ID))
+
+    jobManagerLeaderRetrievalService.foreach(_.start(this))
 
     // start as many resource managers as job managers
     val (rmActorSystems, rmActors) =
@@ -383,7 +388,8 @@ abstract class FlinkMiniCluster(
         config.getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0) >= 0) {
 
       // TODO: Add support for HA: Make web server work independently from the JM
-      val leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManagerAkkaURL)
+      val leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
+        HighAvailabilityServices.DEFAULT_JOB_ID)
 
       LOG.info("Starting JobManger web frontend")
       // start the new web frontend. we need to load this dynamically
@@ -409,6 +415,9 @@ abstract class FlinkMiniCluster(
     awaitTermination()
 
     jobManagerLeaderRetrievalService.foreach(_.stop())
+
+    highAvailabilityServices.closeAndCleanupAllData()
+
     isRunning = false
 
     FlinkExecutors.gracefulShutdown(
@@ -509,33 +518,37 @@ abstract class FlinkMiniCluster(
     submitJobAndWait(jobGraph, printUpdates, timeout)
   }
 
-  def submitJobAndWait(
-    jobGraph: JobGraph,
-    printUpdates: Boolean,
-    timeout: FiniteDuration)
-  : JobExecutionResult = {
-    submitJobAndWait(jobGraph, printUpdates, timeout, createLeaderRetrievalService())
-  }
-
   @throws(classOf[JobExecutionException])
   def submitJobAndWait(
       jobGraph: JobGraph,
       printUpdates: Boolean,
-      timeout: FiniteDuration,
-      leaderRetrievalService: LeaderRetrievalService)
+      timeout: FiniteDuration)
     : JobExecutionResult = {
 
     val clientActorSystem = startJobClientActorSystem(jobGraph.getJobID)
 
-     try {
-     JobClient.submitJobAndWait(
+    val userCodeClassLoader =
+      try {
+        createUserCodeClassLoader(
+          jobGraph.getUserJars,
+          jobGraph.getClasspaths,
+          Thread.currentThread().getContextClassLoader)
+      } catch {
+        case e: Exception => throw new JobExecutionException(
+          jobGraph.getJobID,
+          "Could not create the user code class loader.",
+          e)
+      }
+
+    try {
+      JobClient.submitJobAndWait(
        clientActorSystem,
        configuration,
-       leaderRetrievalService,
+       highAvailabilityServices,
        jobGraph,
        timeout,
        printUpdates,
-       this.getClass.getClassLoader())
+       userCodeClassLoader)
     } finally {
        if(!useSingleActorSystem) {
          // we have to shutdown the just created actor system
@@ -558,11 +571,24 @@ abstract class FlinkMiniCluster(
         )
     }
 
+    val userCodeClassLoader =
+      try {
+        createUserCodeClassLoader(
+          jobGraph.getUserJars,
+          jobGraph.getClasspaths,
+          Thread.currentThread().getContextClassLoader)
+      } catch {
+        case e: Exception => throw new JobExecutionException(
+          jobGraph.getJobID,
+          "Could not create the user code class loader.",
+          e)
+      }
+
     JobClient.submitJobDetached(jobManagerGateway,
       configuration,
       jobGraph,
       timeout,
-      this.getClass.getClassLoader())
+      userCodeClassLoader)
 
     new JobSubmissionResult(jobGraph.getJobID)
   }
@@ -573,20 +599,6 @@ abstract class FlinkMiniCluster(
     }
   }
 
-  protected def createLeaderRetrievalService(): LeaderRetrievalService = {
-    (jobManagerActorSystems, jobManagerActors) match {
-      case (Some(jmActorSystems), Some(jmActors)) =>
-        if (haMode == HighAvailabilityMode.NONE) {
-          new StandaloneLeaderRetrievalService(
-            AkkaUtils.getAkkaURL(jmActorSystems(0), jmActors(0)))
-        } else {
-          ZooKeeperUtils.createLeaderRetrievalService(originalConfiguration)
-        }
-
-      case _ => throw new Exception("The FlinkMiniCluster has not been started properly.")
-    }
-  }
-
   protected def clearLeader(): Unit = {
     futureLock.synchronized{
       leaderGateway = Promise()
@@ -643,4 +655,28 @@ abstract class FlinkMiniCluster(
       }
     }
   }
+
+  private def createUserCodeClassLoader(
+      jars: java.util.List[Path],
+      classPaths: java.util.List[URL],
+      parentClassLoader: ClassLoader): FlinkUserCodeClassLoader = {
+
+    val urls = new Array[URL](jars.size() + classPaths.size())
+
+    import scala.collection.JavaConverters._
+
+    var counter = 0
+
+    for (path <- jars.asScala) {
+      urls(counter) = path.makeQualified(path.getFileSystem).toUri.toURL
+      counter += 1
+    }
+
+    for (classPath <- classPaths.asScala) {
+      urls(counter) = classPath
+      counter += 1
+    }
+
+    new FlinkUserCodeClassLoader(urls, parentClassLoader)
+  }
 }
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index 2f835486cd105..3db91b7d5dd5c 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -32,6 +32,7 @@ import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceMa
 import org.apache.flink.runtime.clusterframework.types.{ResourceID, ResourceIDRetrievable}
 import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory
+import org.apache.flink.runtime.highavailability.{HighAvailabilityServices, HighAvailabilityServicesUtils}
 import org.apache.flink.runtime.instance.InstanceManager
 import org.apache.flink.runtime.io.disk.iomanager.IOManager
 import org.apache.flink.runtime.io.network.NetworkEnvironment
@@ -49,7 +50,7 @@ import org.apache.flink.runtime.taskexecutor.{TaskExecutor, TaskManagerConfigura
 import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
 import org.apache.flink.runtime.util.EnvironmentInformation
 
-import scala.concurrent.Await
+import scala.concurrent.{Await, ExecutionContext}
 import scala.concurrent.duration.FiniteDuration
 
 /**
@@ -63,7 +64,20 @@ import scala.concurrent.duration.FiniteDuration
  */
 class LocalFlinkMiniCluster(
     userConfiguration: Configuration,
-    singleActorSystem: Boolean) extends FlinkMiniCluster(userConfiguration, singleActorSystem) {
+    highAvailabilityServices: HighAvailabilityServices,
+    singleActorSystem: Boolean) extends FlinkMiniCluster(
+  userConfiguration,
+  highAvailabilityServices,
+  singleActorSystem) {
+
+  def this(userConfiguration: Configuration, useSingleActorSystem: Boolean) = {
+    this(
+      userConfiguration,
+       HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+         userConfiguration,
+         ExecutionContext.global),
+      useSingleActorSystem)
+  }
 
   def this(userConfiguration: Configuration) = this(userConfiguration, true)
 
@@ -125,15 +139,11 @@ class LocalFlinkMiniCluster(
     timeout,
     archiveCount,
     archivePath,
-    leaderElectionService,
-    submittedJobGraphStore,
-    checkpointRecoveryFactory,
     jobRecoveryTimeout,
     metricsRegistry) = JobManager.createJobManagerComponents(
       config,
       futureExecutor,
-      ioExecutor,
-      createLeaderElectionService())
+      ioExecutor)
 
     if (config.getBoolean(ConfigConstants.LOCAL_START_WEBSERVER, false)) {
       metricsRegistry.get.startQueryService(system, null)
@@ -158,9 +168,10 @@ class LocalFlinkMiniCluster(
         archive,
         restartStrategyFactory,
         timeout,
-        leaderElectionService,
-        submittedJobGraphStore,
-        checkpointRecoveryFactory,
+        highAvailabilityServices.getJobManagerLeaderElectionService(
+          HighAvailabilityServices.DEFAULT_JOB_ID),
+        highAvailabilityServices.getSubmittedJobGraphStore(),
+        highAvailabilityServices.getCheckpointRecoveryFactory(),
         jobRecoveryTimeout,
         metricsRegistry),
       jobManagerName)
@@ -182,7 +193,8 @@ class LocalFlinkMiniCluster(
     val resourceManagerProps = getResourceManagerProps(
       resourceManagerClass,
       config,
-      createLeaderRetrievalService())
+      highAvailabilityServices.getJobManagerLeaderRetriever(
+        HighAvailabilityServices.DEFAULT_JOB_ID))
 
     system.actorOf(resourceManagerProps, resourceManagerName)
   }
@@ -237,7 +249,8 @@ class LocalFlinkMiniCluster(
       taskManagerServices.getMemoryManager(),
       taskManagerServices.getIOManager(),
       taskManagerServices.getNetworkEnvironment,
-      createLeaderRetrievalService(),
+      highAvailabilityServices.getJobManagerLeaderRetriever(
+        HighAvailabilityServices.DEFAULT_JOB_ID),
       metricRegistry)
 
     if (config.getBoolean(ConfigConstants.LOCAL_START_WEBSERVER, false)) {
@@ -332,10 +345,6 @@ class LocalFlinkMiniCluster(
   // Helper methods
   //------------------------------------------------------------------------------------------------
 
-  def createLeaderElectionService(): Option[LeaderElectionService] = {
-    None
-  }
-
   def initializeIOFormatClasses(configuration: Configuration): Unit = {
     try {
       val om = classOf[FileOutputFormat[_]].getDeclaredMethod("initDefaultsFromConfiguration",
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 4065660099658..a3110a4909c90 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -39,11 +39,14 @@ import org.apache.flink.runtime.blob.{BlobCache, BlobClient, BlobService}
 import org.apache.flink.runtime.broadcast.BroadcastVariableManager
 import org.apache.flink.runtime.clusterframework.messages.StopCluster
 import org.apache.flink.runtime.clusterframework.types.ResourceID
+import org.apache.flink.runtime.concurrent.Executors
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor
 import org.apache.flink.runtime.execution.ExecutionState
 import org.apache.flink.runtime.execution.librarycache.{BlobLibraryCacheManager, FallbackLibraryCacheManager, LibraryCacheManager}
 import org.apache.flink.runtime.executiongraph.{ExecutionAttemptID, PartitionInfo}
 import org.apache.flink.runtime.filecache.FileCache
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution
+import org.apache.flink.runtime.highavailability.{HighAvailabilityServices, HighAvailabilityServicesUtils}
 import org.apache.flink.runtime.instance.{AkkaActorGateway, HardwareDescription, InstanceID}
 import org.apache.flink.runtime.io.disk.iomanager.IOManager
 import org.apache.flink.runtime.io.network.NetworkEnvironment
@@ -293,7 +296,7 @@ class TaskManager(
     // its registration at the JobManager
     case NotifyWhenRegisteredAtJobManager =>
       if (isConnected) {
-        sender ! decorateMessage(RegisteredAtJobManager)
+        sender ! decorateMessage(RegisteredAtJobManager(leaderSessionID.orNull))
       } else {
         waitForRegistration += sender
       }
@@ -993,7 +996,7 @@ class TaskManager(
 
     // notify all the actors that listen for a successful registration
     for (listener <- waitForRegistration) {
-      listener ! RegisteredAtJobManager
+      listener ! RegisteredAtJobManager(leaderSessionID.orNull)
     }
     waitForRegistration.clear()
   }
@@ -1638,20 +1641,37 @@ object TaskManager {
       taskManagerClass: Class[_ <: TaskManager])
     : Unit = {
 
-    val (taskManagerHostname, actorSystemPort) = selectNetworkInterfaceAndPort(configuration)
+    val highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+      configuration,
+      Executors.directExecutor(),
+      AddressResolution.TRY_ADDRESS_RESOLUTION)
 
-    runTaskManager(
-      taskManagerHostname,
-      resourceID,
-      actorSystemPort,
+    val (taskManagerHostname, actorSystemPort) = selectNetworkInterfaceAndPort(
       configuration,
-      taskManagerClass)
+      highAvailabilityServices)
+
+    try {
+      runTaskManager(
+        taskManagerHostname,
+        resourceID,
+        actorSystemPort,
+        configuration,
+        highAvailabilityServices,
+        taskManagerClass)
+    } finally {
+      try {
+        highAvailabilityServices.close()
+      } catch {
+        case t: Throwable => LOG.warn("Could not properly stop the high availability services.", t)
+      }
+    }
   }
 
   @throws(classOf[IOException])
   @throws(classOf[IllegalConfigurationException])
   def selectNetworkInterfaceAndPort(
-      configuration: Configuration)
+      configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices)
     : (String, Int) = {
 
     var taskManagerHostname = configuration.getString(
@@ -1661,13 +1681,11 @@ object TaskManager {
       LOG.info("Using configured hostname/address for TaskManager: " + taskManagerHostname)
     }
     else {
-      val leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(
-        configuration,
-        true)
       val lookupTimeout = AkkaUtils.getLookupTimeout(configuration)
 
       val taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(
-        leaderRetrievalService,
+        highAvailabilityServices.getJobManagerLeaderRetriever(
+          HighAvailabilityServices.DEFAULT_JOB_ID),
         lookupTimeout)
 
       taskManagerHostname = taskManagerAddress.getHostName()
@@ -1700,13 +1718,15 @@ object TaskManager {
    * @param resourceID The id of the resource which the task manager will run on.
    * @param actorSystemPort The port at which the actor system will communicate.
    * @param configuration The configuration for the TaskManager.
+   * @param highAvailabilityServices Service factory for high availability services
    */
   @throws(classOf[Exception])
   def runTaskManager(
       taskManagerHostname: String,
       resourceID: ResourceID,
       actorSystemPort: Int,
-      configuration: Configuration)
+      configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices)
     : Unit = {
 
     runTaskManager(
@@ -1714,6 +1734,7 @@ object TaskManager {
       resourceID,
       actorSystemPort,
       configuration,
+      highAvailabilityServices,
       classOf[TaskManager])
   }
 
@@ -1730,6 +1751,7 @@ object TaskManager {
    * @param resourceID The id of the resource which the task manager will run on.
    * @param actorSystemPort The port at which the actor system will communicate.
    * @param configuration The configuration for the TaskManager.
+   * @param highAvailabilityServices Service factory for high availability services
    * @param taskManagerClass The actor class to instantiate. Allows the use of TaskManager
    *                         subclasses for example for YARN.
    */
@@ -1739,6 +1761,7 @@ object TaskManager {
       resourceID: ResourceID,
       actorSystemPort: Int,
       configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices,
       taskManagerClass: Class[_ <: TaskManager])
     : Unit = {
 
@@ -1778,9 +1801,9 @@ object TaskManager {
         configuration,
         resourceID,
         taskManagerSystem,
+        highAvailabilityServices,
         taskManagerHostname,
         Some(TaskExecutor.TASK_MANAGER_NAME),
-        None,
         localTaskManagerCommunication = false,
         taskManagerClass)
 
@@ -1821,8 +1844,7 @@ object TaskManager {
 
       // block until everything is done
       taskManagerSystem.awaitTermination()
-    }
-    catch {
+    } catch {
       case t: Throwable =>
         LOG.error("Error while starting up taskManager", t)
         try {
@@ -1840,17 +1862,15 @@ object TaskManager {
    * @param configuration The configuration for the TaskManager.
    * @param resourceID The id of the resource which the task manager will run on.
    * @param actorSystem The actor system that should run the TaskManager actor.
+   * @param highAvailabilityServices Factory to create high availability services
    * @param taskManagerHostname The hostname/address that describes the TaskManager's data location.
    * @param taskManagerActorName Optionally the name of the TaskManager actor. If none is given,
    *                             the actor will use a random name.
-   * @param leaderRetrievalServiceOption Optionally, a leader retrieval service can be provided. If
-   *                                     none is given, then a LeaderRetrievalService is
-   *                                     constructed from the configuration.
    * @param localTaskManagerCommunication If true, the TaskManager will not initiate the
    *                                      TCP network stack.
    * @param taskManagerClass The class of the TaskManager actor. May be used to give
    *                         subclasses that understand additional actor messages.
-    * @throws org.apache.flink.configuration.IllegalConfigurationException
+   * @throws org.apache.flink.configuration.IllegalConfigurationException
    *                              Thrown, if the given config contains illegal values.
    * @throws java.io.IOException Thrown, if any of the I/O components (such as buffer pools,
    *                             I/O manager, ...) cannot be properly started.
@@ -1865,9 +1885,9 @@ object TaskManager {
       configuration: Configuration,
       resourceID: ResourceID,
       actorSystem: ActorSystem,
+      highAvailabilityServices: HighAvailabilityServices,
       taskManagerHostname: String,
       taskManagerActorName: Option[String],
-      leaderRetrievalServiceOption: Option[LeaderRetrievalService],
       localTaskManagerCommunication: Boolean,
       taskManagerClass: Class[_ <: TaskManager])
     : ActorRef = {
@@ -1885,11 +1905,8 @@ object TaskManager {
 
     val metricRegistry = taskManagerServices.getMetricRegistry()
 
-    val leaderRetrievalService = leaderRetrievalServiceOption match {
-      case Some(lrs) => lrs
-      // validate the address if possible (e.g. we're in Standalone mode)
-      case None => LeaderRetrievalUtils.createLeaderRetrievalService(configuration, true)
-    }
+    val leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
+      HighAvailabilityServices.DEFAULT_JOB_ID)
 
     // create the actor properties (which define the actor constructor parameters)
     val tmProps = getTaskManagerProps(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
index 99a38151fd998..ec1bbd8e2157a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
@@ -33,6 +33,7 @@
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 
+import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
 import scala.concurrent.Await;
@@ -47,7 +48,7 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-public class CoordinatorShutdownTest {
+public class CoordinatorShutdownTest extends TestLogger {
 	
 	@Test
 	public void testCoordinatorShutsDownOnFailure() {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java
index 0ec00df870917..8530b0f05d392 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java
@@ -29,6 +29,7 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.akka.FlinkUntypedActor;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
 import org.apache.flink.runtime.messages.Acknowledge;
@@ -306,11 +307,14 @@ public void testGuaranteedAnswerIfJobClientDies() throws Exception {
 			leaderSessionID
 		);
 
+		TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
+		highAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, testingLeaderRetrievalService);
+
 		JobListeningContext jobListeningContext =
 			JobClient.submitJob(
 				system,
 				clientConfig,
-				testingLeaderRetrievalService,
+				highAvailabilityServices,
 				testJobGraph,
 				timeout,
 				false,
@@ -328,6 +332,8 @@ public void testGuaranteedAnswerIfJobClientDies() throws Exception {
 			Assert.fail();
 		} catch (JobExecutionException e) {
 			// this is what we want
+		} finally {
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ClusterShutdownITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ClusterShutdownITCase.java
index 41018ddb62106..388572c70ef5d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ClusterShutdownITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ClusterShutdownITCase.java
@@ -25,6 +25,8 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.messages.StopCluster;
 import org.apache.flink.runtime.clusterframework.messages.StopClusterSuccessful;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.testingUtils.TestingMessages;
@@ -32,12 +34,16 @@
 import org.apache.flink.runtime.testutils.TestingResourceManager;
 import org.apache.flink.util.TestLogger;
 
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import scala.Option;
 
+import java.util.Arrays;
+
 /**
  * Runs tests to ensure that a cluster is shutdown properly.
  */
@@ -47,6 +53,19 @@ public class ClusterShutdownITCase extends TestLogger {
 
 	private static Configuration config = new Configuration();
 
+	private HighAvailabilityServices highAvailabilityServices;
+
+	@Before
+	public void setupTest() {
+		highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
+	}
+
+	@After
+	public void tearDownTest() throws Exception {
+		highAvailabilityServices.closeAndCleanupAllData();
+		highAvailabilityServices = null;
+	}
+
 	@BeforeClass
 	public static void setup() {
 		system = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
@@ -68,37 +87,51 @@ public void testClusterShutdownWithoutResourceManager() {
 		@Override
 		protected void run() {
 
-			ActorGateway me =
-				TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-
-			// start job manager which doesn't shutdown the actor system
-			ActorGateway jobManager =
-				TestingUtils.createJobManager(
-					system,
-					TestingUtils.defaultExecutor(),
-					TestingUtils.defaultExecutor(),
-					config,
-					"jobmanager1");
-
-			// Tell the JobManager to inform us of shutdown actions
-			jobManager.tell(TestingMessages.getNotifyOfComponentShutdown(), me);
-
-			// Register a TaskManager
-			ActorGateway taskManager =
-				TestingUtils.createTaskManager(system, jobManager, config, true, true);
-
-			// Tell the TaskManager to inform us of TaskManager shutdowns
-			taskManager.tell(TestingMessages.getNotifyOfComponentShutdown(), me);
-
-
-			// No resource manager connected
-			jobManager.tell(new StopCluster(ApplicationStatus.SUCCEEDED, "Shutting down."), me);
-
-			expectMsgAllOf(
-				new TestingMessages.ComponentShutdown(taskManager.actor()),
-				new TestingMessages.ComponentShutdown(jobManager.actor()),
-				StopClusterSuccessful.getInstance()
-			);
+			ActorGateway jobManager = null;
+			ActorGateway taskManager = null;
+			ActorGateway forwardingActor = null;
+
+			try {
+				// start job manager which doesn't shutdown the actor system
+				jobManager =
+					TestingUtils.createJobManager(
+						system,
+						TestingUtils.defaultExecutor(),
+						TestingUtils.defaultExecutor(),
+						config,
+						highAvailabilityServices,
+						"jobmanager1");
+
+				forwardingActor =
+					TestingUtils.createForwardingActor(
+						system,
+						getTestActor(),
+						jobManager.leaderSessionID(),
+						Option.<String>empty());
+
+				// Tell the JobManager to inform us of shutdown actions
+				jobManager.tell(TestingMessages.getNotifyOfComponentShutdown(), forwardingActor);
+
+				// Register a TaskManager
+				taskManager =
+					TestingUtils.createTaskManager(system, highAvailabilityServices, config, true, true);
+
+				// Tell the TaskManager to inform us of TaskManager shutdowns
+				taskManager.tell(TestingMessages.getNotifyOfComponentShutdown(), forwardingActor);
+
+
+				// No resource manager connected
+				jobManager.tell(new StopCluster(ApplicationStatus.SUCCEEDED, "Shutting down."), forwardingActor);
+
+				expectMsgAllOf(
+					new TestingMessages.ComponentShutdown(taskManager.actor()),
+					new TestingMessages.ComponentShutdown(jobManager.actor()),
+					StopClusterSuccessful.getInstance()
+				);
+			} finally {
+				TestingUtils.stopActorGatewaysGracefully(Arrays.asList(
+					jobManager, taskManager, forwardingActor));
+			}
 
 		}};
 		}};
@@ -115,50 +148,68 @@ public void testClusterShutdownWithResourceManager() {
 		@Override
 		protected void run() {
 
-			ActorGateway me =
-				TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-
-			// start job manager which doesn't shutdown the actor system
-			ActorGateway jobManager =
-				TestingUtils.createJobManager(
-					system,
-					TestingUtils.defaultExecutor(),
-					TestingUtils.defaultExecutor(),
-					config,
-					"jobmanager2");
-
-			// Tell the JobManager to inform us of shutdown actions
-			jobManager.tell(TestingMessages.getNotifyOfComponentShutdown(), me);
-
-			// Register a TaskManager
-			ActorGateway taskManager =
-				TestingUtils.createTaskManager(system, jobManager, config, true, true);
-
-			// Tell the TaskManager to inform us of TaskManager shutdowns
-			taskManager.tell(TestingMessages.getNotifyOfComponentShutdown(), me);
-
-			// Start resource manager and let it register
-			ActorGateway resourceManager =
-				TestingUtils.createResourceManager(system, jobManager.actor(), config);
-
-			// Tell the ResourceManager to inform us of ResourceManager shutdowns
-			resourceManager.tell(TestingMessages.getNotifyOfComponentShutdown(), me);
-
-			// notify about a resource manager registration at the job manager
-			resourceManager.tell(new TestingResourceManager.NotifyWhenResourceManagerConnected(), me);
-
-			// Wait for resource manager
-			expectMsgEquals(Acknowledge.get());
-
-			// Shutdown cluster with resource manager connected
-			jobManager.tell(new StopCluster(ApplicationStatus.SUCCEEDED, "Shutting down."), me);
-
-			expectMsgAllOf(
-				new TestingMessages.ComponentShutdown(taskManager.actor()),
-				new TestingMessages.ComponentShutdown(jobManager.actor()),
-				new TestingMessages.ComponentShutdown(resourceManager.actor()),
-				StopClusterSuccessful.getInstance()
-			);
+			ActorGateway jobManager = null;
+			ActorGateway taskManager = null;
+			ActorGateway resourceManager = null;
+			ActorGateway forwardingActor = null;
+
+			try {
+				// start job manager which doesn't shutdown the actor system
+				jobManager =
+					TestingUtils.createJobManager(
+						system,
+						TestingUtils.defaultExecutor(),
+						TestingUtils.defaultExecutor(),
+						config,
+						highAvailabilityServices,
+						"jobmanager2");
+
+				forwardingActor =
+					TestingUtils.createForwardingActor(
+						system,
+						getTestActor(),
+						jobManager.leaderSessionID(),
+						Option.<String>empty());
+
+				// Tell the JobManager to inform us of shutdown actions
+				jobManager.tell(TestingMessages.getNotifyOfComponentShutdown(), forwardingActor);
+
+				// Register a TaskManager
+				taskManager =
+					TestingUtils.createTaskManager(system, highAvailabilityServices, config, true, true);
+
+				// Tell the TaskManager to inform us of TaskManager shutdowns
+				taskManager.tell(TestingMessages.getNotifyOfComponentShutdown(), forwardingActor);
+
+				// Start resource manager and let it register
+				resourceManager =
+					TestingUtils.createResourceManager(
+						system,
+						config,
+						highAvailabilityServices);
+
+				// Tell the ResourceManager to inform us of ResourceManager shutdowns
+				resourceManager.tell(TestingMessages.getNotifyOfComponentShutdown(), forwardingActor);
+
+				// notify about a resource manager registration at the job manager
+				resourceManager.tell(new TestingResourceManager.NotifyWhenResourceManagerConnected(), forwardingActor);
+
+				// Wait for resource manager
+				expectMsgEquals(Acknowledge.get());
+
+				// Shutdown cluster with resource manager connected
+				jobManager.tell(new StopCluster(ApplicationStatus.SUCCEEDED, "Shutting down."), forwardingActor);
+
+				expectMsgAllOf(
+					new TestingMessages.ComponentShutdown(taskManager.actor()),
+					new TestingMessages.ComponentShutdown(jobManager.actor()),
+					new TestingMessages.ComponentShutdown(resourceManager.actor()),
+					StopClusterSuccessful.getInstance()
+				);
+			} finally {
+				TestingUtils.stopActorGatewaysGracefully(Arrays.asList(
+					jobManager, taskManager, resourceManager, forwardingActor));
+			}
 
 		}};
 		}};
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerITCase.java
index 6191195ec2a59..d52afe4b4eae9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerITCase.java
@@ -23,6 +23,8 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.HardwareDescription;
 import org.apache.flink.runtime.messages.Acknowledge;
@@ -32,12 +34,17 @@
 import org.apache.flink.runtime.testutils.TestingResourceManager;
 import org.apache.flink.util.TestLogger;
 
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import scala.Option;
 
+
+import java.util.Arrays;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
@@ -53,6 +60,8 @@ public class ResourceManagerITCase extends TestLogger {
 
 	private static Configuration config = new Configuration();
 
+	private HighAvailabilityServices highAvailabilityServices;
+
 	@BeforeClass
 	public static void setup() {
 		system = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
@@ -63,6 +72,17 @@ public static void teardown() {
 		JavaTestKit.shutdownActorSystem(system);
 	}
 
+	@Before
+	public void setupTest() {
+		highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
+	}
+
+	@After
+	public void tearDownTest() throws Exception {
+		highAvailabilityServices.closeAndCleanupAllData();
+		highAvailabilityServices = null;
+	}
+
 	/**
 	 * Tests whether the resource manager connects and reconciles existing task managers.
 	 */
@@ -74,49 +94,67 @@ public void testResourceManagerReconciliation() {
 		@Override
 		protected void run() {
 
-			ActorGateway jobManager =
-				TestingUtils.createJobManager(
-					system,
-					TestingUtils.defaultExecutor(),
-					TestingUtils.defaultExecutor(),
-					config,
-					"ReconciliationTest");
-			ActorGateway me =
-				TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
+			ActorGateway jobManager = null;
+			ActorGateway resourceManager = null;
+			ActorGateway forwardingActor = null;
+
+			try {
+				jobManager =
+					TestingUtils.createJobManager(
+						system,
+						TestingUtils.defaultExecutor(),
+						TestingUtils.defaultExecutor(),
+						config,
+						highAvailabilityServices,
+						"ReconciliationTest");
 
-			// !! no resource manager started !!
+				forwardingActor =
+					TestingUtils.createForwardingActor(
+						system,
+						getTestActor(),
+						jobManager.leaderSessionID(),
+						Option.<String>empty());
 
-			ResourceID resourceID = ResourceID.generate();
+				// !! no resource manager started !!
 
-			TaskManagerLocation location = mock(TaskManagerLocation.class);
-			when(location.getResourceID()).thenReturn(resourceID);
+				ResourceID resourceID = ResourceID.generate();
 
-			HardwareDescription resourceProfile = HardwareDescription.extractFromSystem(1_000_000);
+				TaskManagerLocation location = mock(TaskManagerLocation.class);
+				when(location.getResourceID()).thenReturn(resourceID);
 
-			jobManager.tell(
-				new RegistrationMessages.RegisterTaskManager(resourceID, location, resourceProfile, 1),
-				me);
+				HardwareDescription resourceProfile = HardwareDescription.extractFromSystem(1_000_000);
 
-			expectMsgClass(RegistrationMessages.AcknowledgeRegistration.class);
+				jobManager.tell(
+					new RegistrationMessages.RegisterTaskManager(resourceID, location, resourceProfile, 1),
+					forwardingActor);
 
-			// now start the resource manager
-			ActorGateway resourceManager =
-				TestingUtils.createResourceManager(system, jobManager.actor(), config);
+				expectMsgClass(RegistrationMessages.AcknowledgeRegistration.class);
 
-			// register at testing job manager to receive a message once a resource manager registers
-			resourceManager.tell(new TestingResourceManager.NotifyWhenResourceManagerConnected(), me);
+				// now start the resource manager
+				resourceManager =
+					TestingUtils.createResourceManager(
+						system,
+						config,
+						highAvailabilityServices);
 
-			// Wait for resource manager
-			expectMsgEquals(Acknowledge.get());
+				// register at testing job manager to receive a message once a resource manager registers
+				resourceManager.tell(new TestingResourceManager.NotifyWhenResourceManagerConnected(), forwardingActor);
 
-			// check if we registered the task manager resource
-			resourceManager.tell(new TestingResourceManager.GetRegisteredResources(), me);
+				// Wait for resource manager
+				expectMsgEquals(Acknowledge.get());
 
-			TestingResourceManager.GetRegisteredResourcesReply reply =
-				expectMsgClass(TestingResourceManager.GetRegisteredResourcesReply.class);
+				// check if we registered the task manager resource
+				resourceManager.tell(new TestingResourceManager.GetRegisteredResources(), forwardingActor);
 
-			assertEquals(1, reply.resources.size());
-			assertTrue(reply.resources.contains(resourceID));
+				TestingResourceManager.GetRegisteredResourcesReply reply =
+					expectMsgClass(TestingResourceManager.GetRegisteredResourcesReply.class);
+
+				assertEquals(1, reply.resources.size());
+				assertTrue(reply.resources.contains(resourceID));
+			} finally {
+				TestingUtils.stopActorGatewaysGracefully(Arrays.asList(
+					jobManager, resourceManager, forwardingActor));
+			}
 
 		}};
 		}};
@@ -133,37 +171,53 @@ public void testResourceManagerTaskManagerRegistration() {
 		@Override
 		protected void run() {
 
-			ActorGateway jobManager =
-				TestingUtils.createJobManager(
-					system,
-					TestingUtils.defaultExecutor(),
-					TestingUtils.defaultExecutor(),
-					config,
-					"RegTest");
-			ActorGateway me =
-				TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-
-			// start the resource manager
-			ActorGateway resourceManager =
-				TestingUtils.createResourceManager(system, jobManager.actor(), config);
-
-			// notify about a resource manager registration at the job manager
-			resourceManager.tell(new TestingResourceManager.NotifyWhenResourceManagerConnected(), me);
-
-			// Wait for resource manager
-			expectMsgEquals(Acknowledge.get());
-
-			// start task manager and wait for registration
-			ActorGateway taskManager =
-				TestingUtils.createTaskManager(system, jobManager.actor(), config, true, true);
-
-			// check if we registered the task manager resource
-			resourceManager.tell(new TestingResourceManager.GetRegisteredResources(), me);
-
-			TestingResourceManager.GetRegisteredResourcesReply reply =
-				expectMsgClass(TestingResourceManager.GetRegisteredResourcesReply.class);
-
-			assertEquals(1, reply.resources.size());
+			ActorGateway jobManager = null;
+			ActorGateway taskManager = null;
+			ActorGateway resourceManager = null;
+			ActorGateway forwardingActor = null;
+
+			try {
+				jobManager =
+					TestingUtils.createJobManager(
+						system,
+						TestingUtils.defaultExecutor(),
+						TestingUtils.defaultExecutor(),
+						config,
+						highAvailabilityServices,
+						"RegTest");
+
+				forwardingActor =
+					TestingUtils.createForwardingActor(
+						system,
+						getTestActor(),
+						jobManager.leaderSessionID(),
+						Option.<String>empty());
+
+				// start the resource manager
+				resourceManager =
+					TestingUtils.createResourceManager(system, config, highAvailabilityServices);
+
+				// notify about a resource manager registration at the job manager
+				resourceManager.tell(new TestingResourceManager.NotifyWhenResourceManagerConnected(), forwardingActor);
+
+				// Wait for resource manager
+				expectMsgEquals(Acknowledge.get());
+
+				// start task manager and wait for registration
+				taskManager =
+					TestingUtils.createTaskManager(system, highAvailabilityServices, config, true, true);
+
+				// check if we registered the task manager resource
+				resourceManager.tell(new TestingResourceManager.GetRegisteredResources(), forwardingActor);
+
+				TestingResourceManager.GetRegisteredResourcesReply reply =
+					expectMsgClass(TestingResourceManager.GetRegisteredResourcesReply.class);
+
+				assertEquals(1, reply.resources.size());
+			} finally {
+				TestingUtils.stopActorGatewaysGracefully(Arrays.asList(
+					jobManager, resourceManager, taskManager, forwardingActor));
+			}
 
 		}};
 		}};
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
index c7405187d3bb8..5aa31ff9aa793 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java
@@ -36,6 +36,7 @@
 import org.apache.flink.runtime.concurrent.ScheduledExecutor;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.heartbeat.TestingHeartbeatServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobmaster.JobMasterGateway;
@@ -58,7 +59,9 @@
 import org.apache.flink.runtime.testutils.TestingResourceManager;
 import org.apache.flink.runtime.util.TestingFatalErrorHandler;
 import org.apache.flink.util.TestLogger;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
@@ -90,6 +93,9 @@ public class ResourceManagerTest extends TestLogger {
 
 	private static Configuration config = new Configuration();
 
+	private TestingHighAvailabilityServices highAvailabilityServices;
+	private TestingLeaderRetrievalService jobManagerLeaderRetrievalService;
+
 	@BeforeClass
 	public static void setup() {
 		system = AkkaUtils.createLocalActorSystem(config);
@@ -100,6 +106,32 @@ public static void teardown() {
 		JavaTestKit.shutdownActorSystem(system);
 	}
 
+	@Before
+	public void setupTest() {
+		jobManagerLeaderRetrievalService = new TestingLeaderRetrievalService();
+
+		highAvailabilityServices = new TestingHighAvailabilityServices();
+
+		highAvailabilityServices.setJobMasterLeaderRetriever(
+			HighAvailabilityServices.DEFAULT_JOB_ID,
+			jobManagerLeaderRetrievalService);
+	}
+
+	@After
+	public void teardownTest() throws Exception {
+		if (jobManagerLeaderRetrievalService != null) {
+			jobManagerLeaderRetrievalService.stop();
+
+			jobManagerLeaderRetrievalService = null;
+		}
+
+		if (highAvailabilityServices != null) {
+			highAvailabilityServices.closeAndCleanupAllData();
+
+			highAvailabilityServices = null;
+		}
+	}
+
 	/**
 	 * Tests the registration and reconciliation of the ResourceManager with the JobManager
 	 */
@@ -109,8 +141,20 @@ public void testJobManagerRegistrationAndReconciliation() {
 		new Within(duration("10 seconds")) {
 		@Override
 		protected void run() {
-			fakeJobManager = TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-			resourceManager = TestingUtils.createResourceManager(system, fakeJobManager.actor(), config);
+			fakeJobManager = TestingUtils.createForwardingActor(
+				system,
+				getTestActor(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				Option.<String>empty());
+
+			jobManagerLeaderRetrievalService.notifyListener(
+				fakeJobManager.path(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+			resourceManager = TestingUtils.createResourceManager(
+				system,
+				config,
+				highAvailabilityServices);
 
 			expectMsgClass(RegisterResourceManager.class);
 
@@ -150,8 +194,20 @@ protected void run() {
 			Configuration shortTimeoutConfig = config.clone();
 			shortTimeoutConfig.setString(ConfigConstants.AKKA_LOOKUP_TIMEOUT, "1 s");
 
-			fakeJobManager = TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-			resourceManager = TestingUtils.createResourceManager(system, fakeJobManager.actor(), shortTimeoutConfig);
+			fakeJobManager = TestingUtils.createForwardingActor(
+				system,
+				getTestActor(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				Option.<String>empty());
+
+			jobManagerLeaderRetrievalService.notifyListener(
+				fakeJobManager.path(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+			resourceManager = TestingUtils.createResourceManager(
+				system,
+				shortTimeoutConfig,
+				highAvailabilityServices);
 
 			// wait for registration message
 			RegisterResourceManager msg = expectMsgClass(RegisterResourceManager.class);
@@ -180,8 +236,20 @@ protected void run() {
 			Configuration shortTimeoutConfig = config.clone();
 			shortTimeoutConfig.setString(ConfigConstants.AKKA_LOOKUP_TIMEOUT, "99999 s");
 
-			fakeJobManager = TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-			resourceManager = TestingUtils.createResourceManager(system, fakeJobManager.actor(), shortTimeoutConfig);
+			fakeJobManager = TestingUtils.createForwardingActor(
+				system,
+				getTestActor(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				Option.<String>empty());
+
+			jobManagerLeaderRetrievalService.notifyListener(
+				fakeJobManager.path(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+			resourceManager = TestingUtils.createResourceManager(
+				system,
+				shortTimeoutConfig,
+				highAvailabilityServices);
 
 			// wait for registration message
 			RegisterResourceManager msg = expectMsgClass(RegisterResourceManager.class);
@@ -212,8 +280,20 @@ public void testTaskManagerRegistration() {
 		@Override
 		protected void run() {
 
-			fakeJobManager = TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-			resourceManager = TestingUtils.createResourceManager(system, fakeJobManager.actor(), config);
+			fakeJobManager = TestingUtils.createForwardingActor(
+				system,
+				getTestActor(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				Option.<String>empty());
+
+			jobManagerLeaderRetrievalService.notifyListener(
+				fakeJobManager.path(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+			resourceManager = TestingUtils.createResourceManager(
+				system,
+				config,
+				highAvailabilityServices);
 
 			// register with JM
 			expectMsgClass(RegisterResourceManager.class);
@@ -270,8 +350,20 @@ public void testResourceRemoval() {
 		@Override
 		protected void run() {
 
-			fakeJobManager = TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-			resourceManager = TestingUtils.createResourceManager(system, fakeJobManager.actor(), config);
+			fakeJobManager = TestingUtils.createForwardingActor(
+				system,
+				getTestActor(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				Option.<String>empty());
+
+			jobManagerLeaderRetrievalService.notifyListener(
+				fakeJobManager.path(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+			resourceManager = TestingUtils.createResourceManager(
+				system,
+				config,
+				highAvailabilityServices);
 
 			// register with JM
 			expectMsgClass(RegisterResourceManager.class);
@@ -321,8 +413,20 @@ public void testResourceFailureNotification() {
 		@Override
 		protected void run() {
 
-			fakeJobManager = TestingUtils.createForwardingActor(system, getTestActor(), Option.<String>empty());
-			resourceManager = TestingUtils.createResourceManager(system, fakeJobManager.actor(), config);
+			fakeJobManager = TestingUtils.createForwardingActor(
+				system,
+				getTestActor(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				Option.<String>empty());
+
+			jobManagerLeaderRetrievalService.notifyListener(
+				fakeJobManager.path(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+			resourceManager = TestingUtils.createResourceManager(
+				system,
+				config,
+				highAvailabilityServices);
 
 			// register with JM
 			expectMsgClass(RegisterResourceManager.class);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java
new file mode 100644
index 0000000000000..423c0ceded065
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability;
+
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
+import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * Leader service for {@link TestingManualHighAvailabilityServices} implementation. The leader
+ * service allows to create multiple {@link TestingLeaderElectionService} and
+ * {@link TestingLeaderRetrievalService} and allows to manually trigger the services identified
+ * by a continuous index.
+ */
+public class ManualLeaderService {
+
+	private final List<TestingLeaderElectionService> leaderElectionServices;
+	private final List<TestingLeaderRetrievalService> leaderRetrievalServices;
+
+	private int currentLeaderIndex;
+
+	@Nullable
+	private UUID currentLeaderId;
+
+	public ManualLeaderService() {
+		leaderElectionServices = new ArrayList<>(4);
+		leaderRetrievalServices = new ArrayList<>(4);
+
+		currentLeaderIndex = -1;
+		currentLeaderId = null;
+	}
+
+	public LeaderRetrievalService createLeaderRetrievalService() {
+		final TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService(
+			getLeaderAddress(currentLeaderIndex),
+			currentLeaderId);
+
+		leaderRetrievalServices.add(testingLeaderRetrievalService);
+
+		return testingLeaderRetrievalService;
+	}
+
+	public LeaderElectionService createLeaderElectionService() {
+		TestingLeaderElectionService testingLeaderElectionService = new TestingLeaderElectionService();
+
+		leaderElectionServices.add(testingLeaderElectionService);
+
+		return testingLeaderElectionService;
+	}
+
+	public void grantLeadership(int index, UUID leaderId) {
+		if (currentLeaderId != null) {
+			revokeLeadership();
+		}
+
+		Preconditions.checkNotNull(leaderId);
+		Preconditions.checkArgument(0 <= index && index < leaderElectionServices.size());
+
+		TestingLeaderElectionService testingLeaderElectionService = leaderElectionServices.get(index);
+
+		testingLeaderElectionService.isLeader(leaderId);
+
+		currentLeaderIndex = index;
+		currentLeaderId = leaderId;
+	}
+
+	public void revokeLeadership() {
+		assert(currentLeaderId != null);
+		assert(0 <= currentLeaderIndex &&  currentLeaderIndex < leaderElectionServices.size());
+
+		TestingLeaderElectionService testingLeaderElectionService = leaderElectionServices.get(currentLeaderIndex);
+
+		testingLeaderElectionService.notLeader();
+
+		currentLeaderIndex = -1;
+		currentLeaderId = null;
+	}
+
+	public void notifyRetrievers(int index, UUID leaderId) {
+		for (TestingLeaderRetrievalService retrievalService: leaderRetrievalServices) {
+			retrievalService.notifyListener(getLeaderAddress(index), leaderId);
+		}
+	}
+
+	private String getLeaderAddress(int index) {
+		if (0 <= index && index < leaderElectionServices.size()) {
+			TestingLeaderElectionService testingLeaderElectionService = leaderElectionServices.get(index);
+			return testingLeaderElectionService.getAddress();
+		} else {
+			return null;
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingManualHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingManualHighAvailabilityServices.java
new file mode 100644
index 0000000000000..676b0d8cb0d03
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingManualHighAvailabilityServices.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.blob.BlobStore;
+import org.apache.flink.runtime.blob.VoidBlobStore;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
+import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneRunningJobsRegistry;
+import org.apache.flink.runtime.jobmanager.StandaloneSubmittedJobGraphStore;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Testing high availability service which can be manually controlled. The leader election and
+ * notification of new leaders is triggered manually via {@link #grantLeadership(JobID, int, UUID)}
+ * and {@link #notifyRetrievers(JobID, int, UUID)}.
+ */
+public class TestingManualHighAvailabilityServices implements HighAvailabilityServices {
+
+	private final Map<JobID, ManualLeaderService> jobManagerLeaderServices;
+
+	private final ManualLeaderService resourceManagerLeaderService;
+
+	public TestingManualHighAvailabilityServices() {
+		jobManagerLeaderServices = new HashMap<>(4);
+		resourceManagerLeaderService = new ManualLeaderService();
+	}
+
+	@Override
+	public LeaderRetrievalService getResourceManagerLeaderRetriever() {
+		return resourceManagerLeaderService.createLeaderRetrievalService();
+	}
+
+	@Override
+	public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
+		ManualLeaderService leaderService = getOrCreateJobManagerLeaderService(jobID);
+
+		return leaderService.createLeaderRetrievalService();
+	}
+
+	@Override
+	public LeaderElectionService getResourceManagerLeaderElectionService() {
+		return resourceManagerLeaderService.createLeaderElectionService();
+	}
+
+	@Override
+	public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
+		ManualLeaderService leaderService = getOrCreateJobManagerLeaderService(jobID);
+
+		return leaderService.createLeaderElectionService();
+	}
+
+	@Override
+	public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
+		return new StandaloneCheckpointRecoveryFactory();
+	}
+
+	@Override
+	public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+		return new StandaloneSubmittedJobGraphStore();
+	}
+
+	@Override
+	public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+		return new StandaloneRunningJobsRegistry();
+	}
+
+	@Override
+	public BlobStore createBlobStore() throws IOException {
+		return new VoidBlobStore();
+	}
+
+	@Override
+	public void close() throws Exception {
+		// nothing to do
+	}
+
+	@Override
+	public void closeAndCleanupAllData() throws Exception {
+		// nothing to do
+	}
+
+	public void grantLeadership(JobID jobId, int index, UUID leaderId) {
+		ManualLeaderService manualLeaderService = jobManagerLeaderServices.get(jobId);
+
+		if (manualLeaderService != null) {
+			manualLeaderService.grantLeadership(index, leaderId);
+		} else {
+			throw new IllegalStateException("No manual leader service for job id " + jobId +
+				" has been initialized.");
+		}
+	}
+
+	public void revokeLeadership(JobID jobId) {
+		ManualLeaderService manualLeaderService = jobManagerLeaderServices.get(jobId);
+
+		if (manualLeaderService != null) {
+			manualLeaderService.revokeLeadership();
+		} else {
+			throw new IllegalStateException("No manual leader service for job id " + jobId +
+				" has been initialized.");
+		}
+	}
+
+	public void notifyRetrievers(JobID jobId, int index, UUID leaderId) {
+		ManualLeaderService manualLeaderService = jobManagerLeaderServices.get(jobId);
+
+		if (manualLeaderService != null) {
+			manualLeaderService.notifyRetrievers(index, leaderId);
+		} else {
+			throw new IllegalStateException("No manual leader service for job id " + jobId +
+				" has been initialized.");
+		}
+	}
+
+	private ManualLeaderService getOrCreateJobManagerLeaderService(JobID jobId) {
+		ManualLeaderService manualLeaderService = jobManagerLeaderServices.get(jobId);
+
+		if (manualLeaderService == null) {
+			manualLeaderService = new ManualLeaderService();
+			jobManagerLeaderServices.put(jobId, manualLeaderService);
+		}
+
+		return manualLeaderService;
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
index 77eb566349a44..b8b5984f0b20b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
@@ -52,6 +52,7 @@
 import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy;
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.instance.InstanceManager;
@@ -67,7 +68,6 @@
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
 import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.state.ChainedStateHandle;
@@ -86,6 +86,7 @@
 import org.apache.flink.runtime.util.TestByteStreamStateHandleDeepCompare;
 import org.apache.flink.util.InstantiationUtil;
 
+import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -123,7 +124,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class JobManagerHARecoveryTest {
+public class JobManagerHARecoveryTest extends TestLogger {
 
 	private static ActorSystem system;
 
@@ -171,8 +172,11 @@ public void testJobRecoveryWhenLosingLeadership() throws Exception {
 			CheckpointRecoveryFactory checkpointStateFactory = new MyCheckpointRecoveryFactory(checkpointStore, checkpointCounter);
 			TestingLeaderElectionService myLeaderElectionService = new TestingLeaderElectionService();
 			TestingLeaderRetrievalService myLeaderRetrievalService = new TestingLeaderRetrievalService(
-				"localhost",
-				HighAvailabilityServices.DEFAULT_LEADER_ID);
+				null,
+				null);
+			TestingHighAvailabilityServices testingHighAvailabilityServices = new TestingHighAvailabilityServices();
+
+			testingHighAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, myLeaderRetrievalService);
 
 			InstanceManager instanceManager = new InstanceManager();
 			instanceManager.addInstanceListener(scheduler);
@@ -200,14 +204,14 @@ public void testJobRecoveryWhenLosingLeadership() throws Exception {
 			ActorGateway gateway = new AkkaActorGateway(jobManager, leaderSessionID);
 
 			taskManager = TaskManager.startTaskManagerComponentsAndActor(
-					flinkConfiguration,
-					ResourceID.generate(),
-					system,
-					"localhost",
-					Option.apply("taskmanager"),
-					Option.apply((LeaderRetrievalService) myLeaderRetrievalService),
-					true,
-					TestingTaskManager.class);
+				flinkConfiguration,
+				ResourceID.generate(),
+				system,
+				testingHighAvailabilityServices,
+				"localhost",
+				Option.apply("taskmanager"),
+				true,
+				TestingTaskManager.class);
 
 			ActorGateway tmGateway = new AkkaActorGateway(taskManager, leaderSessionID);
 
@@ -226,9 +230,9 @@ public void testJobRecoveryWhenLosingLeadership() throws Exception {
 					vertexId,
 					vertexId,
 					vertexId,
-					100,
-					10 * 60 * 1000,
-					0,
+					100L,
+					10L * 60L * 1000L,
+					0L,
 					1,
 					ExternalizedCheckpointSettings.none(),
 					null,
@@ -604,5 +608,4 @@ public static long[] getRecoveredStates() {
 			return recoveredStates;
 		}
 	}
-
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
index 2ac3ea774b666..35ac1e5a20587 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
@@ -32,6 +32,7 @@
 import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
+import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
 import org.apache.flink.configuration.Configuration;
@@ -50,7 +51,7 @@
 /**
  * Tests that the JobManager process properly exits when the JobManager actor dies.
  */
-public class JobManagerProcessReapingTest {
+public class JobManagerProcessReapingTest extends TestLogger {
 
 	@Test
 	public void testReapProcessOnFailure() {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java
index 0ab1b671521a2..9ac6873767026 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java
@@ -36,6 +36,7 @@
 import org.apache.flink.util.NetUtils;
 
 import org.apache.flink.util.OperatingSystem;
+import org.apache.flink.util.TestLogger;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -44,7 +45,7 @@
  * Tests that verify the startup behavior of the JobManager in failure
  * situations, when the JobManager cannot be started.
  */
-public class JobManagerStartupTest {
+public class JobManagerStartupTest extends TestLogger {
 
 	private final static String DOES_NOT_EXISTS_NO_SIR = "does-not-exist-no-sir";
 
@@ -101,7 +102,7 @@ public void testStartupWithPortInUse() throws BindException {
 					throw (BindException) cause;
 				}	
 			}
-			fail("this should throw a BindException");
+			throw e;
 		}
 		finally {
 			try {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
index 195baa1fa261d..c8459e7fbab2c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
@@ -18,7 +18,10 @@
 
 package org.apache.flink.runtime.jobmanager;
 
-import akka.actor.*;
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.PoisonPill;
+import akka.actor.Status;
 import akka.testkit.JavaTestKit;
 import akka.testkit.TestProbe;
 import com.typesafe.config.Config;
@@ -42,6 +45,7 @@
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.executiongraph.IntermediateResultPartition;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.instance.HardwareDescription;
@@ -55,8 +59,6 @@
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
 import org.apache.flink.runtime.jobmanager.JobManagerHARecoveryTest.BlockingStatefulInvokable;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.messages.JobManagerMessages.CancelJob;
 import org.apache.flink.runtime.messages.JobManagerMessages.CancellationFailure;
@@ -95,16 +97,15 @@
 import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
 import org.apache.flink.runtime.testtasks.NoOpInvokable;
 import org.apache.flink.runtime.testutils.StoppableInvokable;
+import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.util.TestLogger;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.mockito.ArgumentCaptor;
 import scala.Option;
 import scala.Some;
 import scala.Tuple2;
@@ -117,6 +118,7 @@
 import java.io.File;
 import java.net.InetAddress;
 import java.util.Collections;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
@@ -127,15 +129,14 @@
 import static org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.JobStatusIs;
 import static org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered;
 import static org.apache.flink.runtime.testingUtils.TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT;
+import static org.apache.flink.runtime.testingUtils.TestingUtils.TESTING_TIMEOUT;
 import static org.apache.flink.runtime.testingUtils.TestingUtils.startTestingCluster;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
 
 public class JobManagerTest extends TestLogger {
 
@@ -144,6 +145,8 @@ public class JobManagerTest extends TestLogger {
 
 	private static ActorSystem system;
 
+	private HighAvailabilityServices highAvailabilityServices;
+
 	@BeforeClass
 	public static void setup() {
 		system = AkkaUtils.createLocalActorSystem(new Configuration());
@@ -154,6 +157,17 @@ public static void teardown() {
 		JavaTestKit.shutdownActorSystem(system);
 	}
 
+	@Before
+	public void setupTest() {
+		highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
+	}
+
+	@After
+	public void tearDownTest() throws Exception {
+		highAvailabilityServices.closeAndCleanupAllData();
+		highAvailabilityServices = null;
+	}
+
 	@Test
 	public void testNullHostnameGoesToLocalhost() {
 		try {
@@ -589,32 +603,36 @@ public void testKvStateMessages() throws Exception {
 		Configuration config = new Configuration();
 		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100ms");
 
+		ActorRef jobManagerActor = JobManager.startJobManagerActors(
+			config,
+			system,
+			TestingUtils.defaultExecutor(),
+			TestingUtils.defaultExecutor(),
+			highAvailabilityServices,
+			TestingJobManager.class,
+			MemoryArchivist.class)._1();
+
+		UUID leaderId = LeaderRetrievalUtils.retrieveLeaderSessionId(
+			highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+			TestingUtils.TESTING_TIMEOUT());
+
 		ActorGateway jobManager = new AkkaActorGateway(
-				JobManager.startJobManagerActors(
-					config,
-					system,
-					TestingUtils.defaultExecutor(),
-					TestingUtils.defaultExecutor(),
-					TestingJobManager.class,
-					MemoryArchivist.class)._1(),
-				HighAvailabilityServices.DEFAULT_LEADER_ID);
-
-		LeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(
-				AkkaUtils.getAkkaURL(system, jobManager.actor()));
+				jobManagerActor,
+				leaderId);
 
 		Configuration tmConfig = new Configuration();
 		tmConfig.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L);
 		tmConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
 
 		ActorRef taskManager = TaskManager.startTaskManagerComponentsAndActor(
-				tmConfig,
-				ResourceID.generate(),
-				system,
-				"localhost",
-				scala.Option.<String>empty(),
-				scala.Option.apply(leaderRetrievalService),
-				true,
-				TestingTaskManager.class);
+			tmConfig,
+			ResourceID.generate(),
+			system,
+			highAvailabilityServices,
+			"localhost",
+			scala.Option.<String>empty(),
+			true,
+			TestingTaskManager.class);
 
 		Future<Object> registrationFuture = jobManager
 				.ask(new NotifyWhenAtLeastNumTaskManagerAreRegistered(1), deadline.timeLeft());
@@ -783,6 +801,7 @@ public void testKvStateMessages() throws Exception {
 		// Wait for failure
 		JobStatusIs jobStatus = Await.result(failedFuture, deadline.timeLeft());
 		assertEquals(JobStatus.FAILED, jobStatus.state());
+
 	}
 
 	@Test
@@ -805,25 +824,30 @@ public void testCancelWithSavepoint() throws Exception {
 				actorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				Option.apply("jm"),
 				Option.apply("arch"),
 				TestingJobManager.class,
 				TestingMemoryArchivist.class);
 
-			jobManager = new AkkaActorGateway(master._1(), HighAvailabilityServices.DEFAULT_LEADER_ID);
-			archiver = new AkkaActorGateway(master._2(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+			UUID leaderId = LeaderRetrievalUtils.retrieveLeaderSessionId(
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+				TestingUtils.TESTING_TIMEOUT());
+
+			jobManager = new AkkaActorGateway(master._1(), leaderId);
+			archiver = new AkkaActorGateway(master._2(), leaderId);
 
 			ActorRef taskManagerRef = TaskManager.startTaskManagerComponentsAndActor(
-					config,
-					ResourceID.generate(),
-					actorSystem,
-					"localhost",
-					Option.apply("tm"),
-					Option.<LeaderRetrievalService>apply(new StandaloneLeaderRetrievalService(jobManager.path())),
-					true,
-					TestingTaskManager.class);
+				config,
+				ResourceID.generate(),
+				actorSystem,
+				highAvailabilityServices,
+				"localhost",
+				Option.apply("tm"),
+				true,
+				TestingTaskManager.class);
 
-			taskManager = new AkkaActorGateway(taskManagerRef, HighAvailabilityServices.DEFAULT_LEADER_ID);
+			taskManager = new AkkaActorGateway(taskManagerRef, leaderId);
 
 			// Wait until connected
 			Object msg = new TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager(jobManager.actor());
@@ -907,6 +931,10 @@ public void testCancelWithSavepoint() throws Exception {
 			if (taskManager != null) {
 				taskManager.actor().tell(PoisonPill.getInstance(), ActorRef.noSender());
 			}
+
+			if (actorSystem != null) {
+				actorSystem.awaitTermination(TESTING_TIMEOUT());
+			}
 		}
 	}
 
@@ -931,25 +959,30 @@ public void testCancelWithSavepointNoDirectoriesConfigured() throws Exception {
 				actorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				Option.apply("jm"),
 				Option.apply("arch"),
 				TestingJobManager.class,
 				TestingMemoryArchivist.class);
 
-			jobManager = new AkkaActorGateway(master._1(), HighAvailabilityServices.DEFAULT_LEADER_ID);
-			archiver = new AkkaActorGateway(master._2(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+			UUID leaderId = LeaderRetrievalUtils.retrieveLeaderSessionId(
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+				TestingUtils.TESTING_TIMEOUT());
+
+			jobManager = new AkkaActorGateway(master._1(), leaderId);
+			archiver = new AkkaActorGateway(master._2(), leaderId);
 
 			ActorRef taskManagerRef = TaskManager.startTaskManagerComponentsAndActor(
 				config,
 				ResourceID.generate(),
 				actorSystem,
+				highAvailabilityServices,
 				"localhost",
 				Option.apply("tm"),
-				Option.<LeaderRetrievalService>apply(new StandaloneLeaderRetrievalService(jobManager.path())),
 				true,
 				TestingTaskManager.class);
 
-			taskManager = new AkkaActorGateway(taskManagerRef, HighAvailabilityServices.DEFAULT_LEADER_ID);
+			taskManager = new AkkaActorGateway(taskManagerRef, leaderId);
 
 			// Wait until connected
 			Object msg = new TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager(jobManager.actor());
@@ -1037,25 +1070,30 @@ public void testSavepointWithDeactivatedPeriodicCheckpointing() throws Exception
 				actorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				Option.apply("jm"),
 				Option.apply("arch"),
 				TestingJobManager.class,
 				TestingMemoryArchivist.class);
 
-			jobManager = new AkkaActorGateway(master._1(), HighAvailabilityServices.DEFAULT_LEADER_ID);
-			archiver = new AkkaActorGateway(master._2(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+			UUID leaderId = LeaderRetrievalUtils.retrieveLeaderSessionId(
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+				TestingUtils.TESTING_TIMEOUT());
+
+			jobManager = new AkkaActorGateway(master._1(), leaderId);
+			archiver = new AkkaActorGateway(master._2(), leaderId);
 
 			ActorRef taskManagerRef = TaskManager.startTaskManagerComponentsAndActor(
-					config,
-					ResourceID.generate(),
-					actorSystem,
-					"localhost",
-					Option.apply("tm"),
-					Option.<LeaderRetrievalService>apply(new StandaloneLeaderRetrievalService(jobManager.path())),
-					true,
-					TestingTaskManager.class);
+				config,
+				ResourceID.generate(),
+				actorSystem,
+				highAvailabilityServices,
+				"localhost",
+				Option.apply("tm"),
+				true,
+				TestingTaskManager.class);
 
-			taskManager = new AkkaActorGateway(taskManagerRef, HighAvailabilityServices.DEFAULT_LEADER_ID);
+			taskManager = new AkkaActorGateway(taskManagerRef, leaderId);
 
 			// Wait until connected
 			Object msg = new TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager(jobManager.actor());
@@ -1115,6 +1153,10 @@ public void testSavepointWithDeactivatedPeriodicCheckpointing() throws Exception
 			if (taskManager != null) {
 				taskManager.actor().tell(PoisonPill.getInstance(), ActorRef.noSender());
 			}
+
+			if (actorSystem != null) {
+				actorSystem.awaitTermination(TestingUtils.TESTING_TIMEOUT());
+			}
 		}
 	}
 
@@ -1137,28 +1179,33 @@ public void testSavepointRestoreSettings() throws Exception {
 				actorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				Option.apply("jm"),
 				Option.apply("arch"),
 				TestingJobManager.class,
 				TestingMemoryArchivist.class);
 
-			jobManager = new AkkaActorGateway(master._1(), HighAvailabilityServices.DEFAULT_LEADER_ID);
-			archiver = new AkkaActorGateway(master._2(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+			UUID leaderId = LeaderRetrievalUtils.retrieveLeaderSessionId(
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+				TestingUtils.TESTING_TIMEOUT());
+
+			jobManager = new AkkaActorGateway(master._1(), leaderId);
+			archiver = new AkkaActorGateway(master._2(), leaderId);
 
 			Configuration tmConfig = new Configuration();
 			tmConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
 
 			ActorRef taskManagerRef = TaskManager.startTaskManagerComponentsAndActor(
-					tmConfig,
-					ResourceID.generate(),
-					actorSystem,
-					"localhost",
-					Option.apply("tm"),
-					Option.<LeaderRetrievalService>apply(new StandaloneLeaderRetrievalService(jobManager.path())),
-					true,
-					TestingTaskManager.class);
+				tmConfig,
+				ResourceID.generate(),
+				actorSystem,
+				highAvailabilityServices,
+				"localhost",
+				Option.apply("tm"),
+				true,
+				TestingTaskManager.class);
 
-			taskManager = new AkkaActorGateway(taskManagerRef, HighAvailabilityServices.DEFAULT_LEADER_ID);
+			taskManager = new AkkaActorGateway(taskManagerRef, leaderId);
 
 			// Wait until connected
 			Object msg = new TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager(jobManager.actor());
@@ -1274,6 +1321,10 @@ public void testSavepointRestoreSettings() throws Exception {
 			if (taskManager != null) {
 				taskManager.actor().tell(PoisonPill.getInstance(), ActorRef.noSender());
 			}
+
+			if (actorSystem != null) {
+				actorSystem.awaitTermination(TestingUtils.TESTING_TIMEOUT());
+			}
 		}
 	}
 
@@ -1298,7 +1349,8 @@ public void testResourceManagerConnection() throws TimeoutException, Interrupted
 				actorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
-				configuration);
+				configuration,
+				highAvailabilityServices);
 
 			final TestProbe probe = TestProbe.apply(actorSystem);
 			final AkkaActorGateway rmGateway = new AkkaActorGateway(probe.ref(), HighAvailabilityServices.DEFAULT_LEADER_ID);
@@ -1311,7 +1363,7 @@ public void testResourceManagerConnection() throws TimeoutException, Interrupted
 
 			JobManagerMessages.LeaderSessionMessage leaderSessionMessage = probe.expectMsgClass(JobManagerMessages.LeaderSessionMessage.class);
 
-			assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, leaderSessionMessage.leaderSessionID());
+			assertEquals(jmGateway.leaderSessionID(), leaderSessionMessage.leaderSessionID());
 			assertTrue(leaderSessionMessage.message() instanceof RegisterResourceManagerSuccessful);
 
 			jmGateway.tell(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
index fcca173215d1b..37f503f248e93 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
@@ -26,6 +26,8 @@
 import org.apache.flink.runtime.blob.BlobClient;
 import org.apache.flink.runtime.blob.BlobKey;
 import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -68,6 +70,7 @@ public class JobSubmitTest {
 	private static ActorSystem jobManagerSystem;
 	private static ActorGateway jmGateway;
 	private static Configuration jmConfig;
+	private static HighAvailabilityServices highAvailabilityServices;
 
 	@BeforeClass
 	public static void setupJobManager() {
@@ -81,17 +84,20 @@ public static void setupJobManager() {
 		scala.Option<Tuple2<String, Object>> listeningAddress = scala.Option.apply(new Tuple2<String, Object>("localhost", port));
 		jobManagerSystem = AkkaUtils.createActorSystem(jmConfig, listeningAddress);
 
+		highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
+
 		// only start JobManager (no ResourceManager)
 		JobManager.startJobManagerActors(
 			jmConfig,
 			jobManagerSystem,
 			TestingUtils.defaultExecutor(),
 			TestingUtils.defaultExecutor(),
+			highAvailabilityServices,
 			JobManager.class,
 			MemoryArchivist.class)._1();
 
 		try {
-			LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(jmConfig, false);
+			LeaderRetrievalService lrs = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 
 			jmGateway = LeaderRetrievalUtils.retrieveLeaderGateway(
 					lrs,
@@ -104,10 +110,15 @@ public static void setupJobManager() {
 	}
 
 	@AfterClass
-	public static void teardownJobmanager() {
+	public static void teardownJobmanager() throws Exception {
 		if (jobManagerSystem != null) {
 			jobManagerSystem.shutdown();
 		}
+
+		if (highAvailabilityServices != null) {
+			highAvailabilityServices.closeAndCleanupAllData();
+			highAvailabilityServices = null;
+		}
 	}
 
 	@Test
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
index d2221c50137e7..753c797b26948 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
@@ -35,6 +35,7 @@
 import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.util.TestLogger;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -56,7 +57,7 @@
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(JobManagerRunner.class)
-public class JobManagerRunnerMockTest {
+public class JobManagerRunnerMockTest extends TestLogger {
 
 	private JobManagerRunner runner;
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
index b1bb5483d854d..942fcf3d13c54 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
@@ -18,9 +18,13 @@
 
 package org.apache.flink.runtime.leaderelection;
 
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingManualHighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
@@ -28,6 +32,7 @@
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobmanager.Tasks;
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.runtime.testingUtils.TestingCluster;
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
 import org.apache.flink.util.TestLogger;
 
@@ -52,11 +57,15 @@ public class LeaderChangeJobRecoveryTest extends TestLogger {
 	private int numSlotsPerTM = 1;
 	private int parallelism = numTMs * numSlotsPerTM;
 
-	private LeaderElectionRetrievalTestingCluster cluster = null;
+	private JobID jobId;
+	private TestingCluster cluster = null;
 	private JobGraph job = createBlockingJob(parallelism);
+	private TestingManualHighAvailabilityServices highAvailabilityServices;
 
 	@Before
 	public void before() throws TimeoutException, InterruptedException {
+		jobId = HighAvailabilityServices.DEFAULT_JOB_ID;
+
 		Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(true);
 
 		Configuration configuration = new Configuration();
@@ -69,7 +78,13 @@ public void before() throws TimeoutException, InterruptedException {
 		configuration.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 9999);
 		configuration.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "100 milli");
 
-		cluster = new LeaderElectionRetrievalTestingCluster(configuration, true, false);
+		highAvailabilityServices = new TestingManualHighAvailabilityServices();
+
+		cluster = new TestingCluster(
+			configuration,
+			highAvailabilityServices,
+			true,
+			false);
 		cluster.start(false);
 
 		// wait for actors to be alive so that they have started their leader retrieval service
@@ -86,8 +101,15 @@ public void before() throws TimeoutException, InterruptedException {
 	public void testNotRestartedWhenLosingLeadership() throws Exception {
 		UUID leaderSessionID = UUID.randomUUID();
 
-		cluster.grantLeadership(0, leaderSessionID);
-		cluster.notifyRetrievalListeners(0, leaderSessionID);
+		highAvailabilityServices.grantLeadership(
+			jobId,
+			0,
+			leaderSessionID);
+
+		highAvailabilityServices.notifyRetrievers(
+			jobId,
+			0,
+			leaderSessionID);
 
 		cluster.waitForTaskManagersToBeRegistered(timeout);
 
@@ -108,7 +130,7 @@ public void testNotRestartedWhenLosingLeadership() throws Exception {
 
 		ExecutionGraph executionGraph = (ExecutionGraph) ((TestingJobManagerMessages.ExecutionGraphFound) responseExecutionGraph).executionGraph();
 
-		cluster.revokeLeadership();
+		highAvailabilityServices.revokeLeadership(jobId);
 
 		executionGraph.getTerminationFuture().get(30, TimeUnit.SECONDS);
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java
index 7ae99746831dc..bbcbbf06e0e48 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java
@@ -18,8 +18,11 @@
 
 package org.apache.flink.runtime.leaderelection;
 
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingManualHighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
@@ -28,6 +31,7 @@
 import org.apache.flink.runtime.jobmanager.Tasks;
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
 import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.testingUtils.TestingCluster;
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobRemoved;
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.WaitForAllVerticesToBeRunningOrFinished;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
@@ -58,12 +62,16 @@ public class LeaderChangeStateCleanupTest extends TestLogger {
 	private int numSlotsPerTM = 2;
 	private int parallelism = numTMs * numSlotsPerTM;
 
+	private JobID jobId;
 	private Configuration configuration;
-	private LeaderElectionRetrievalTestingCluster cluster = null;
+	private TestingManualHighAvailabilityServices highAvailabilityServices;
+	private TestingCluster cluster = null;
 	private JobGraph job = createBlockingJob(parallelism);
 
 	@Before
 	public void before() throws Exception {
+		jobId = HighAvailabilityServices.DEFAULT_JOB_ID;
+
 		Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(true);
 
 		configuration = new Configuration();
@@ -72,7 +80,13 @@ public void before() throws Exception {
 		configuration.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTMs);
 		configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numSlotsPerTM);
 
-		cluster = new LeaderElectionRetrievalTestingCluster(configuration, true, false);
+		highAvailabilityServices = new TestingManualHighAvailabilityServices();
+
+		cluster = new TestingCluster(
+			configuration,
+			highAvailabilityServices,
+			true,
+			false);
 		cluster.start(false); // TaskManagers don't have to register at the JobManager
 
 		cluster.waitForActorsToBeAlive(); // we only wait until all actors are alive
@@ -96,9 +110,9 @@ public void testStateCleanupAfterNewLeaderElectionAndListenerNotification() thro
 		UUID leaderSessionID2 = UUID.randomUUID();
 
 		// first make JM(0) the leader
-		cluster.grantLeadership(0, leaderSessionID1);
+		highAvailabilityServices.grantLeadership(jobId, 0, leaderSessionID1);
 		// notify all listeners
-		cluster.notifyRetrievalListeners(0, leaderSessionID1);
+		highAvailabilityServices.notifyRetrievers(jobId, 0, leaderSessionID1);
 
 		cluster.waitForTaskManagersToBeRegistered(timeout);
 
@@ -114,9 +128,9 @@ public void testStateCleanupAfterNewLeaderElectionAndListenerNotification() thro
 		Future<Object> jobRemoval = jm.ask(new NotifyWhenJobRemoved(job.getJobID()), timeout);
 
 		// make the JM(1) the new leader
-		cluster.grantLeadership(1, leaderSessionID2);
+		highAvailabilityServices.grantLeadership(jobId, 1, leaderSessionID2);
 		// notify all listeners about the event
-		cluster.notifyRetrievalListeners(1, leaderSessionID2);
+		highAvailabilityServices.notifyRetrievers(jobId, 1, leaderSessionID2);
 
 		Await.ready(jobRemoval, timeout);
 
@@ -133,7 +147,7 @@ public void testStateCleanupAfterNewLeaderElectionAndListenerNotification() thro
 
 		// try to resubmit now the non-blocking job, it should complete successfully
 		Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(false);
-		cluster.submitJobAndWait(job, false, timeout, new TestingLeaderRetrievalService(jm2.path(), jm2.leaderSessionID()));
+		cluster.submitJobAndWait(job, false, timeout);
 	}
 
 	/**
@@ -146,8 +160,8 @@ public void testStateCleanupAfterNewLeaderElection() throws Exception {
 		UUID leaderSessionID = UUID.randomUUID();
 		UUID newLeaderSessionID = UUID.randomUUID();
 
-		cluster.grantLeadership(0, leaderSessionID);
-		cluster.notifyRetrievalListeners(0, leaderSessionID);
+		highAvailabilityServices.grantLeadership(jobId, 0, leaderSessionID);
+		highAvailabilityServices.notifyRetrievers(jobId, 0, leaderSessionID);
 
 		cluster.waitForTaskManagersToBeRegistered(timeout);
 
@@ -163,7 +177,7 @@ public void testStateCleanupAfterNewLeaderElection() throws Exception {
 		Future<Object> jobRemoval = jm.ask(new NotifyWhenJobRemoved(job.getJobID()), timeout);
 
 		// only notify the JMs about the new leader JM(1)
-		cluster.grantLeadership(1, newLeaderSessionID);
+		highAvailabilityServices.grantLeadership(jobId, 1, newLeaderSessionID);
 
 		// job should be removed anyway
 		Await.ready(jobRemoval, timeout);
@@ -179,8 +193,8 @@ public void testStateCleanupAfterListenerNotification() throws Exception {
 		UUID leaderSessionID = UUID.randomUUID();
 		UUID newLeaderSessionID = UUID.randomUUID();
 
-		cluster.grantLeadership(0, leaderSessionID);
-		cluster.notifyRetrievalListeners(0, leaderSessionID);
+		highAvailabilityServices.grantLeadership(jobId, 0, leaderSessionID);
+		highAvailabilityServices.notifyRetrievers(jobId, 0, leaderSessionID);
 
 		cluster.waitForTaskManagersToBeRegistered(timeout);
 
@@ -196,7 +210,7 @@ public void testStateCleanupAfterListenerNotification() throws Exception {
 		Future<Object> jobRemoval = jm.ask(new NotifyWhenJobRemoved(job.getJobID()), timeout);
 
 		// notify listeners (TMs) about the leader change
-		cluster.notifyRetrievalListeners(1, newLeaderSessionID);
+		highAvailabilityServices.notifyRetrievers(jobId, 1, newLeaderSessionID);
 
 		Await.ready(jobRemoval, timeout);
 	}
@@ -213,8 +227,8 @@ public void testReelectionOfSameJobManager() throws Exception {
 
 		FiniteDuration shortTimeout = new FiniteDuration(10, TimeUnit.SECONDS);
 
-		cluster.grantLeadership(0, leaderSessionID);
-		cluster.notifyRetrievalListeners(0, leaderSessionID);
+		highAvailabilityServices.grantLeadership(jobId, 0, leaderSessionID);
+		highAvailabilityServices.notifyRetrievers(jobId, 0, leaderSessionID);
 
 		cluster.waitForTaskManagersToBeRegistered(timeout);
 
@@ -232,7 +246,7 @@ public void testReelectionOfSameJobManager() throws Exception {
 		LOG.info("Make JM(0) again the leader. This should first revoke the leadership.");
 
 		// make JM(0) again the leader --> this implies first a leadership revocation
-		cluster.grantLeadership(0, newLeaderSessionID);
+		highAvailabilityServices.grantLeadership(jobId, 0, newLeaderSessionID);
 
 		Await.ready(jobRemoval, timeout);
 
@@ -250,7 +264,7 @@ public void testReelectionOfSameJobManager() throws Exception {
 		LOG.info("Notify TMs about the new (old) leader.");
 
 		// notify the TMs about the new (old) leader
-		cluster.notifyRetrievalListeners(0, newLeaderSessionID);
+		highAvailabilityServices.notifyRetrievers(jobId,0, newLeaderSessionID);
 
 		cluster.waitForTaskManagersToBeRegistered(timeout);
 
@@ -258,7 +272,7 @@ public void testReelectionOfSameJobManager() throws Exception {
 
 		// try to resubmit now the non-blocking job, it should complete successfully
 		Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(false);
-		cluster.submitJobAndWait(job, false, timeout, new TestingLeaderRetrievalService(leaderGateway.path(), leaderGateway.leaderSessionID()));
+		cluster.submitJobAndWait(job, false, timeout);
 	}
 
 	public JobGraph createBlockingJob(int parallelism) {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionRetrievalTestingCluster.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionRetrievalTestingCluster.java
deleted file mode 100644
index 1cab0eabdaffa..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionRetrievalTestingCluster.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.leaderelection;
-
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.testingUtils.TestingCluster;
-import scala.Option;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * A testing cluster which allows to manually trigger grantLeadership and notifyRetrievalListener
- * events. The grantLeadership event assigns the specified JobManager the leadership. The
- * notifyRetrievalListener notifies all listeners that the specified JobManager (index) has been
- * granted the leadership.
- */
-public class LeaderElectionRetrievalTestingCluster extends TestingCluster {
-
-	private final Configuration userConfiguration;
-	private final boolean useSingleActorSystem;
-
-	public List<TestingLeaderElectionService> leaderElectionServices;
-	public List<TestingLeaderRetrievalService> leaderRetrievalServices;
-
-	private int leaderIndex = -1;
-
-	public LeaderElectionRetrievalTestingCluster(
-			Configuration userConfiguration,
-			boolean singleActorSystem,
-			boolean synchronousDispatcher) {
-		super(userConfiguration, singleActorSystem, synchronousDispatcher);
-
-		this.userConfiguration = userConfiguration;
-		this.useSingleActorSystem = singleActorSystem;
-
-		leaderElectionServices = new ArrayList<>();
-		leaderRetrievalServices = new ArrayList<>();
-	}
-
-	@Override
-	public Configuration userConfiguration() {
-		return this.userConfiguration;
-	}
-
-	@Override
-	public boolean useSingleActorSystem() {
-		return useSingleActorSystem;
-	}
-
-	@Override
-	public Option<LeaderElectionService> createLeaderElectionService() {
-		leaderElectionServices.add(new TestingLeaderElectionService());
-
-		LeaderElectionService result = leaderElectionServices.get(leaderElectionServices.size() - 1);
-
-		return Option.apply(result);
-	}
-
-	@Override
-	public LeaderRetrievalService createLeaderRetrievalService() {
-		leaderRetrievalServices.add(new TestingLeaderRetrievalService(
-			null,
-			null));
-
-		return leaderRetrievalServices.get(leaderRetrievalServices.size() - 1);
-	}
-
-	@Override
-	public int getNumberOfJobManagers() {
-		return this.originalConfiguration().getInteger(
-				ConfigConstants.LOCAL_NUMBER_JOB_MANAGER,
-				ConfigConstants.DEFAULT_LOCAL_NUMBER_JOB_MANAGER);
-	}
-
-	public void grantLeadership(int index, UUID leaderSessionID) {
-		if(leaderIndex >= 0) {
-			// first revoke leadership
-			leaderElectionServices.get(leaderIndex).notLeader();
-		}
-
-		// make the JM with index the new leader
-		leaderElectionServices.get(index).isLeader(leaderSessionID);
-
-		leaderIndex = index;
-	}
-
-	public void notifyRetrievalListeners(int index, UUID leaderSessionID) {
-		String address = jobManagerActors().get().apply(index).path().toString();
-
-		for(TestingLeaderRetrievalService service: leaderRetrievalServices) {
-			service.notifyListener(address, leaderSessionID);
-		}
-	}
-
-	public void revokeLeadership() {
-		if (leaderIndex >= 0) {
-			leaderElectionServices.get(leaderIndex).notLeader();
-			leaderIndex = -1;
-		}
-	}
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java
index 16779face895c..d4560833e8ab5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java
@@ -63,4 +63,8 @@ public void reset() {
 		contender = null;
 		hasLeadership  = false;
 	}
+
+	public String getAddress() {
+		return contender.getAddress();
+	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java
index 887772af70541..15d3bde75042d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java
@@ -35,6 +35,10 @@ public class TestingLeaderRetrievalService implements LeaderRetrievalService {
 
 	private volatile LeaderRetrievalListener listener;
 
+	public TestingLeaderRetrievalService() {
+		this(null, null);
+	}
+
 	public TestingLeaderRetrievalService(String leaderAddress, UUID leaderSessionID) {
 		this.leaderAddress = leaderAddress;
 		this.leaderSessionID = leaderSessionID;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java
index bc8c0b60d0810..6efd2700ee87a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java
@@ -20,7 +20,6 @@
 
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.api.CreateBuilder;
-import org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable;
 import org.apache.curator.framework.recipes.cache.ChildData;
 import org.apache.curator.framework.recipes.cache.NodeCache;
 import org.apache.curator.framework.recipes.cache.NodeCacheListener;
@@ -37,6 +36,7 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Matchers;
+import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 import org.slf4j.Logger;
@@ -48,6 +48,7 @@
 import scala.concurrent.duration.FiniteDuration;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.ObjectOutputStream;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -59,6 +60,11 @@
 
 public class ZooKeeperLeaderElectionTest extends TestLogger {
 	private TestingServer testingServer;
+
+	private Configuration configuration;
+
+	private CuratorFramework client;
+
 	private static final String TEST_URL = "akka//user/jobmanager";
 	private static final FiniteDuration timeout = new FiniteDuration(200, TimeUnit.SECONDS);
 
@@ -71,17 +77,26 @@ public void before() {
 		} catch (Exception e) {
 			throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
 		}
+
+		configuration = new Configuration();
+
+		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
+		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
+
+		client = ZooKeeperUtils.startCuratorFramework(configuration);
 	}
 
 	@After
-	public void after() {
-		try {
-			testingServer.stop();
-		} catch (Exception e) {
-			throw new RuntimeException("Could not stop ZooKeeper testing cluster.", e);
+	public void after() throws IOException {
+		if (client != null) {
+			client.close();
+			client = null;
 		}
 
-		testingServer = null;
+		if (testingServer != null) {
+			testingServer.stop();
+			testingServer = null;
+		}
 	}
 
 	/**
@@ -89,16 +104,12 @@ public void after() {
 	 */
 	@Test
 	public void testZooKeeperLeaderElectionRetrieval() throws Exception {
-		Configuration configuration = new Configuration();
-		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
-
 		ZooKeeperLeaderElectionService leaderElectionService = null;
 		ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
 
 		try {
-			leaderElectionService = ZooKeeperUtils.createLeaderElectionService(configuration);
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
+			leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
+			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);
 
 			TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
 			TestingListener listener = new TestingListener();
@@ -134,10 +145,6 @@ public void testZooKeeperLeaderElectionRetrieval() throws Exception {
 	 */
 	@Test
 	public void testZooKeeperReelection() throws Exception {
-		Configuration configuration = new Configuration();
-		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
-
 		Deadline deadline = new FiniteDuration(5, TimeUnit.MINUTES).fromNow();
 
 		int num = 10;
@@ -149,14 +156,14 @@ public void testZooKeeperReelection() throws Exception {
 		TestingListener listener = new TestingListener();
 
 		try {
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
+			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);
 
 			LOG.debug("Start leader retrieval service for the TestingListener.");
 
 			leaderRetrievalService.start(listener);
 
 			for (int i = 0; i < num; i++) {
-				leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(configuration);
+				leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
 				contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);
 
 				LOG.debug("Start leader election service for contender #{}.", i);
@@ -217,10 +224,6 @@ public void testZooKeeperReelection() throws Exception {
 	 */
 	@Test
 	public void testZooKeeperReelectionWithReplacement() throws Exception {
-		Configuration configuration = new Configuration();
-		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
-
 		int num = 3;
 		int numTries = 30;
 
@@ -231,12 +234,12 @@ public void testZooKeeperReelectionWithReplacement() throws Exception {
 		TestingListener listener = new TestingListener();
 
 		try {
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
+			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);
 
 			leaderRetrievalService.start(listener);
 
 			for (int i = 0; i < num; i++) {
-				leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(configuration);
+				leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
 				contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);
 
 				leaderElectionService[i].start(contenders[i]);
@@ -261,7 +264,7 @@ public void testZooKeeperReelectionWithReplacement() throws Exception {
 					// stop leader election service = revoke leadership
 					leaderElectionService[index].stop();
 					// create new leader election service which takes part in the leader election
-					leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(configuration);
+					leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
 					contenders[index] = new TestingContender(
 							TEST_URL + "_" + index + "_" + (lastTry + 1),
 							leaderElectionService[index]);
@@ -295,9 +298,6 @@ public void testMultipleLeaders() throws Exception {
 		final String FAULTY_CONTENDER_URL = "faultyContender";
 		final String leaderPath = "/leader";
 
-		Configuration configuration = new Configuration();
-		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
 		configuration.setString(ConfigConstants.HA_ZOOKEEPER_LEADER_PATH, leaderPath);
 
 		ZooKeeperLeaderElectionService leaderElectionService = null;
@@ -308,9 +308,9 @@ public void testMultipleLeaders() throws Exception {
 		TestingContender contender;
 
 		try {
-			leaderElectionService = ZooKeeperUtils.createLeaderElectionService(configuration);
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
-			leaderRetrievalService2 = ZooKeeperUtils.createLeaderRetrievalService(configuration);
+			leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
+			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);
+			leaderRetrievalService2 = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);
 
 			contender = new TestingContender(TEST_URL, leaderElectionService);
 
@@ -379,18 +379,13 @@ public void testMultipleLeaders() throws Exception {
 	 */
 	@Test
 	public void testExceptionForwarding() throws Exception {
-		Configuration configuration = new Configuration();
-		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
-
 		ZooKeeperLeaderElectionService leaderElectionService = null;
 		ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
 		TestingListener listener = new TestingListener();
 		TestingContender testingContender;
 
 		CuratorFramework client;
-		final CreateBuilder mockCreateBuilder = mock(CreateBuilder.class);
-		final ProtectACLCreateModePathAndBytesable<String> mockCreateParentsIfNeeded = mock (ProtectACLCreateModePathAndBytesable.class);
+		final CreateBuilder mockCreateBuilder = mock(CreateBuilder.class, Mockito.RETURNS_DEEP_STUBS);
 		final Exception testException = new Exception("Test exception");
 
 		try {
@@ -414,12 +409,14 @@ public CreateBuilder answer(InvocationOnMock invocation) throws Throwable {
 
 			doAnswer(answer).when(client).create();
 
-			when(mockCreateBuilder.creatingParentsIfNeeded()).thenReturn(mockCreateParentsIfNeeded);
-			when(mockCreateParentsIfNeeded.withMode(Matchers.any(CreateMode.class))).thenReturn(mockCreateParentsIfNeeded);
-			when(mockCreateParentsIfNeeded.forPath(Matchers.any(String.class),  Matchers.any(byte[].class))).thenThrow(testException);
+			when(
+				mockCreateBuilder
+				.creatingParentsIfNeeded()
+				.withMode(Matchers.any(CreateMode.class))
+				.forPath(anyString(), any(byte[].class))).thenThrow(testException);
 
 			leaderElectionService = new ZooKeeperLeaderElectionService(client, "/latch", "/leader");
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
+			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);
 
 			testingContender = new TestingContender(TEST_URL, leaderElectionService);
 
@@ -442,34 +439,33 @@ public CreateBuilder answer(InvocationOnMock invocation) throws Throwable {
 	}
 
 	/**
-	 * Tests that there is no information left in the ZooKeeper cluster after all JobManagers
-	 * have terminated. In other words, checks that the ZooKeeperLeaderElection service uses
+	 * Tests that there is no information left in the ZooKeeper cluster after the ZooKeeper client
+	 * has terminated. In other words, checks that the ZooKeeperLeaderElection service uses
 	 * ephemeral nodes.
 	 */
 	@Test
 	public void testEphemeralZooKeeperNodes() throws Exception {
-		Configuration configuration = new Configuration();
-		configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-		configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
-
 		ZooKeeperLeaderElectionService leaderElectionService;
 		ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
 		TestingContender testingContender;
 		TestingListener listener;
 
 		CuratorFramework client = null;
+		CuratorFramework client2 = null;
 		NodeCache cache = null;
 
 		try {
-			leaderElectionService = ZooKeeperUtils.createLeaderElectionService(configuration);
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
+			client = ZooKeeperUtils.startCuratorFramework(configuration);
+			client2 = ZooKeeperUtils.startCuratorFramework(configuration);
+
+			leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
+			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client2, configuration);
 			testingContender = new TestingContender(TEST_URL, leaderElectionService);
 			listener = new TestingListener();
 
-			client = ZooKeeperUtils.startCuratorFramework(configuration);
 			final String leaderPath = configuration.getString(ConfigConstants.HA_ZOOKEEPER_LEADER_PATH,
 					ConfigConstants.DEFAULT_ZOOKEEPER_LEADER_PATH);
-			cache = new NodeCache(client, leaderPath);
+			cache = new NodeCache(client2, leaderPath);
 
 			ExistsCacheListener existsListener = new ExistsCacheListener(cache);
 			DeletedCacheListener deletedCacheListener = new DeletedCacheListener(cache);
@@ -489,6 +485,9 @@ public void testEphemeralZooKeeperNodes() throws Exception {
 
 			leaderElectionService.stop();
 
+			// now stop the underlying client
+			client.close();
+
 			Future<Boolean> deletedFuture = deletedCacheListener.nodeDeleted();
 
 			// make sure that the leader node has been deleted
@@ -497,7 +496,7 @@ public void testEphemeralZooKeeperNodes() throws Exception {
 			leaderRetrievalService.start(listener);
 
 			try {
-				listener.waitForNewLeader(1000);
+				listener.waitForNewLeader(1000L);
 
 				fail("TimeoutException was expected because there is no leader registered and " +
 						"thus there shouldn't be any leader information in ZooKeeper.");
@@ -513,8 +512,8 @@ public void testEphemeralZooKeeperNodes() throws Exception {
 				cache.close();
 			}
 
-			if (client != null) {
-				client.close();
+			if (client2 != null) {
+				client2.close();
 			}
 		}
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java
index b79093ffd1fec..0ea47f2d865ca 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java
@@ -18,18 +18,17 @@
 
 package org.apache.flink.runtime.leaderelection;
 
-import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.test.TestingServer;
 
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.util.LeaderRetrievalUtils;
-import org.apache.flink.runtime.util.ZooKeeperUtils;
-import org.apache.flink.util.NetUtils;
 import org.apache.flink.util.TestLogger;
 
 import org.junit.After;
@@ -53,29 +52,41 @@ public class ZooKeeperLeaderRetrievalTest extends TestLogger{
 
 	private TestingServer testingServer;
 
+	private Configuration config;
+
+	private HighAvailabilityServices highAvailabilityServices;
+
 	@Before
-	public void before() {
-		try {
-			testingServer = new TestingServer();
-		} catch (Exception e) {
-			throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
-		}
+	public void before() throws Exception {
+		testingServer = new TestingServer();
+
+		config = new Configuration();
+		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
+		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
+
+		highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			TestingUtils.defaultExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 	}
 
 	@After
-	public void after() {
+	public void after() throws Exception {
 		if(testingServer != null) {
-			try {
-				testingServer.stop();
-			} catch (IOException e) {
-				throw new RuntimeException("Could not stop ZooKeeper testing cluster.", e);
-			}
+			testingServer.stop();
+
 			testingServer = null;
 		}
+
+		if (highAvailabilityServices != null) {
+			highAvailabilityServices.closeAndCleanupAllData();
+
+			highAvailabilityServices = null;
+		}
 	}
 
 	/**
-	 * Tests that LeaderRetrievalUtils.findConnectingAdress finds the correct connecting address
+	 * Tests that LeaderRetrievalUtils.findConnectingAddress finds the correct connecting address
 	 * in case of an old leader address in ZooKeeper and a subsequent election of a new leader.
 	 * The findConnectingAddress should block until the new leader has been elected and his
 	 * address has been written to ZooKeeper.
@@ -83,13 +94,9 @@ public void after() {
 	@Test
 	public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exception {
 		FiniteDuration timeout = new FiniteDuration(1, TimeUnit.MINUTES);
-		Configuration config = new Configuration();
 
 		long sleepingTime = 1000;
 
-		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
-		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-
 		LeaderElectionService leaderElectionService = null;
 		LeaderElectionService faultyLeaderElectionService;
 
@@ -98,12 +105,7 @@ public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exc
 
 		Thread thread;
 
-		CuratorFramework[] client = new CuratorFramework[2];
-
 		try {
-			client[0] = ZooKeeperUtils.startCuratorFramework(config);
-			client[1] = ZooKeeperUtils.startCuratorFramework(config);
-
 			String wrongAddress = AkkaRpcServiceUtils.getRpcUrl(
 				"1.1.1.1",
 				1234,
@@ -126,7 +128,6 @@ public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exc
 			}
 
 			InetSocketAddress correctInetSocketAddress = new InetSocketAddress(localHost, serverSocket.getLocalPort());
-			String hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(localHost.getHostName(), correctInetSocketAddress.getPort());
 
 			String correctAddress = AkkaRpcServiceUtils.getRpcUrl(
 				localHost.getHostName(),
@@ -135,18 +136,21 @@ public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exc
 				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION,
 				config);
 
-			faultyLeaderElectionService = ZooKeeperUtils.createLeaderElectionService(client[0], config);
+			faultyLeaderElectionService = highAvailabilityServices.getJobManagerLeaderElectionService(
+				HighAvailabilityServices.DEFAULT_JOB_ID);
 			TestingContender wrongLeaderAddressContender = new TestingContender(wrongAddress, faultyLeaderElectionService);
 
 			faultyLeaderElectionService.start(wrongLeaderAddressContender);
 
-			FindConnectingAddress findConnectingAddress = new FindConnectingAddress(config, timeout);
+			FindConnectingAddress findConnectingAddress = new FindConnectingAddress(
+				timeout,
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID));
 
 			thread = new Thread(findConnectingAddress);
 
 			thread.start();
 
-			leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client[1], config);
+			leaderElectionService = highAvailabilityServices.getJobManagerLeaderElectionService(HighAvailabilityServices.DEFAULT_JOB_ID);
 			TestingContender correctLeaderAddressContender = new TestingContender(correctAddress, leaderElectionService);
 
 			Thread.sleep(sleepingTime);
@@ -174,14 +178,6 @@ public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exc
 			if (leaderElectionService != null) {
 				leaderElectionService.stop();
 			}
-
-			if (client[0] != null) {
-				client[0].close();
-			}
-
-			if (client[1] != null) {
-				client[1].close();
-			}
 		}
 	}
 
@@ -192,13 +188,9 @@ public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exc
 	 */
 	@Test
 	public void testTimeoutOfFindConnectingAddress() throws Exception {
-		Configuration config = new Configuration();
-		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
-		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
-
-		FiniteDuration timeout = new FiniteDuration(10, TimeUnit.SECONDS);
+		FiniteDuration timeout = new FiniteDuration(10L, TimeUnit.SECONDS);
 
-		LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
+		LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 		InetAddress result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout);
 
 		assertEquals(InetAddress.getLocalHost(), result);
@@ -206,22 +198,25 @@ public void testTimeoutOfFindConnectingAddress() throws Exception {
 
 	static class FindConnectingAddress implements Runnable {
 
-		private final Configuration config;
 		private final FiniteDuration timeout;
+		private final LeaderRetrievalService leaderRetrievalService;
 
 		private InetAddress result;
 		private Exception exception;
 
-		public FindConnectingAddress(Configuration config, FiniteDuration timeout) {
-			this.config = config;
+		public FindConnectingAddress(
+				FiniteDuration timeout,
+				LeaderRetrievalService leaderRetrievalService) {
 			this.timeout = timeout;
+			this.leaderRetrievalService = leaderRetrievalService;
 		}
 
 		@Override
 		public void run() {
 			try {
-				LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
-				result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout);
+				result = LeaderRetrievalUtils.findConnectingAddress(
+					leaderRetrievalService,
+					timeout);
 			} catch (Exception e) {
 				exception = e;
 			}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
index 100c83d38e8ee..58f223113ca5d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
@@ -26,10 +26,9 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.messages.TaskManagerMessages;
 import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration;
 import org.apache.flink.runtime.taskexecutor.TaskManagerServices;
@@ -37,15 +36,17 @@
 import org.apache.flink.runtime.taskmanager.TaskManager;
 
 import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.TestLogger;
 import org.junit.Assert;
 import org.junit.Test;
 
 import scala.concurrent.duration.FiniteDuration;
 
 import java.net.InetAddress;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
-public class TaskManagerMetricsTest {
+public class TaskManagerMetricsTest extends TestLogger {
 
 	/**
 	 * Tests the metric registry life cycle on JobManager re-connects.
@@ -53,6 +54,9 @@ public class TaskManagerMetricsTest {
 	@Test
 	public void testMetricRegistryLifeCycle() throws Exception {
 		ActorSystem actorSystem = null;
+
+		HighAvailabilityServices highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
+
 		try {
 			actorSystem = AkkaUtils.createLocalActorSystem(new Configuration());
 
@@ -64,11 +68,10 @@ public void testMetricRegistryLifeCycle() throws Exception {
 				actorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				JobManager.class,
 				MemoryArchivist.class)._1();
 
-			LeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManager.path().toString());
-
 			// ================================================================
 			// Start TaskManager
 			// ================================================================
@@ -94,7 +97,7 @@ public void testMetricRegistryLifeCycle() throws Exception {
 				taskManagerServices.getMemoryManager(),
 				taskManagerServices.getIOManager(),
 				taskManagerServices.getNetworkEnvironment(),
-				leaderRetrievalService,
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 				tmRegistry);
 
 			final ActorRef taskManager = actorSystem.actorOf(tmProps);
@@ -107,20 +110,21 @@ protected void run() {
 							getTestActor());
 
 						// wait for the TM to be registered
-						expectMsgEquals(TaskManagerMessages.getRegisteredAtJobManagerMessage());
+						TaskManagerMessages.RegisteredAtJobManager registeredAtJobManager = expectMsgClass(TaskManagerMessages.RegisteredAtJobManager.class);
+						UUID leaderId = registeredAtJobManager.leaderId();
 
 						// trigger re-registration of TM; this should include a disconnect from the current JM
 						taskManager.tell(
 							new TaskManagerMessages.JobManagerLeaderAddress(
 								jobManager.path().toString(),
-								HighAvailabilityServices.DEFAULT_LEADER_ID),
+								leaderId),
 							jobManager);
 
 						// wait for re-registration to be completed
 						taskManager.tell(TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(),
 							getTestActor());
 
-						expectMsgEquals(TaskManagerMessages.getRegisteredAtJobManagerMessage());
+						expectMsgClass(TaskManagerMessages.RegisteredAtJobManager.class);
 					}
 				};
 			}};
@@ -135,6 +139,10 @@ protected void run() {
 			if (actorSystem != null) {
 				actorSystem.shutdown();
 			}
+
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
+			}
 		}
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
index cecfe6a66b62f..973fddfecce4e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
@@ -34,6 +34,8 @@
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
 import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
 import org.apache.flink.runtime.io.network.LocalConnectionManager;
@@ -44,7 +46,6 @@
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.memory.MemoryManager;
 import org.apache.flink.runtime.messages.TaskManagerMessages;
 import org.apache.flink.runtime.metrics.MetricRegistry;
@@ -52,23 +53,24 @@
 import org.apache.flink.runtime.query.KvStateRegistry;
 import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 
+import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
 import scala.concurrent.duration.FiniteDuration;
 
 import java.net.InetAddress;
+import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
 
-public class TaskManagerComponentsStartupShutdownTest {
+public class TaskManagerComponentsStartupShutdownTest extends TestLogger {
 
 	/**
 	 * Makes sure that all components are shut down when the TaskManager
 	 * actor is shut down.
 	 */
 	@Test
-	public void testComponentsStartupShutdown() {
+	public void testComponentsStartupShutdown() throws Exception {
 
 		final String[] TMP_DIR = new String[] { ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH };
 		final Time timeout = Time.seconds(100);
@@ -80,21 +82,28 @@ public void testComponentsStartupShutdown() {
 		config.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 1);
 
 		ActorSystem actorSystem = null;
+
+		HighAvailabilityServices highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
+
+		ActorRef jobManager = null;
+		ActorRef taskManager = null;
+
 		try {
 			actorSystem = AkkaUtils.createLocalActorSystem(config);
 
-			final ActorRef jobManager = JobManager.startJobManagerActors(
+			jobManager = JobManager.startJobManagerActors(
 				config,
 				actorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				JobManager.class,
 				MemoryArchivist.class)._1();
 
 			FlinkResourceManager.startResourceManagerActors(
 				config,
 				actorSystem,
-				LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager),
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 				StandaloneResourceManager.class);
 
 			final int numberOfSlots = 1;
@@ -137,7 +146,8 @@ public void testComponentsStartupShutdown() {
 
 			network.start();
 
-			LeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManager.path().toString());
+			LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID);
 
 			MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(config);
 
@@ -154,18 +164,20 @@ public void testComponentsStartupShutdown() {
 				leaderRetrievalService,
 				new MetricRegistry(metricRegistryConfiguration));
 
-			final ActorRef taskManager = actorSystem.actorOf(tmProps);
+			taskManager = actorSystem.actorOf(tmProps);
+
+			final ActorRef finalTaskManager = taskManager;
 
 			new JavaTestKit(actorSystem) {{
 
 				// wait for the TaskManager to be registered
-				new Within(new FiniteDuration(5000, TimeUnit.SECONDS)) {
+				new Within(new FiniteDuration(5000L, TimeUnit.SECONDS)) {
 					@Override
 					protected void run() {
-						taskManager.tell(TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(),
+						finalTaskManager.tell(TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(),
 								getTestActor());
 
-						expectMsgEquals(TaskManagerMessages.getRegisteredAtJobManagerMessage());
+						expectMsgClass(TaskManagerMessages.RegisteredAtJobManager.class);
 					}
 				};
 			}};
@@ -184,15 +196,16 @@ protected void run() {
 			assertTrue(network.isShutdown());
 			assertTrue(ioManager.isProperlyShutDown());
 			assertTrue(memManager.isShutdown());
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
-		finally {
+		} finally {
+			TestingUtils.stopActorsGracefully(Arrays.asList(jobManager, taskManager));
+
 			if (actorSystem != null) {
 				actorSystem.shutdown();
+
+				actorSystem.awaitTermination(TestingUtils.TESTING_TIMEOUT());
 			}
+
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
index c0d04551728b1..a760760f92ce7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
@@ -24,6 +24,9 @@
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.junit.Test;
 
 import scala.Tuple2;
@@ -44,16 +47,23 @@
 public class TaskManagerConfigurationTest {
 
 	@Test
-	public void testUsePreconfiguredNetworkInterface() {
+	public void testUsePreconfiguredNetworkInterface() throws Exception {
+		final String TEST_HOST_NAME = "testhostname";
+
+		Configuration config = new Configuration();
+		config.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, TEST_HOST_NAME);
+		config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
+		config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 7891);
+
+		HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			Executors.directExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
+
 		try {
-			final String TEST_HOST_NAME = "testhostname";
 
-			Configuration config = new Configuration();
-			config.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, TEST_HOST_NAME);
-			config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
-			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 7891);
 
-			Tuple2<String, Object> address = TaskManager.selectNetworkInterfaceAndPort(config);
+			Tuple2<String, Object> address = TaskManager.selectNetworkInterfaceAndPort(config, highAvailabilityServices);
 
 			// validate the configured test host name
 			assertEquals(TEST_HOST_NAME, address._1());
@@ -61,30 +71,37 @@ public void testUsePreconfiguredNetworkInterface() {
 		catch (Exception e) {
 			e.printStackTrace();
 			fail(e.getMessage());
+		} finally {
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
 	@Test
-	public void testActorSystemPortConfig() {
-		try {
-			// config with pre-configured hostname to speed up tests (no interface selection)
-			Configuration config = new Configuration();
-			config.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, "localhost");
-			config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
-			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 7891);
+	public void testActorSystemPortConfig() throws Exception {
+		// config with pre-configured hostname to speed up tests (no interface selection)
+		Configuration config = new Configuration();
+		config.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, "localhost");
+		config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
+		config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 7891);
+
+		HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			Executors.directExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
+		try {
 			// auto port
-			assertEquals(0, TaskManager.selectNetworkInterfaceAndPort(config)._2());
+			assertEquals(0, TaskManager.selectNetworkInterfaceAndPort(config, highAvailabilityServices)._2());
 
 			// pre-defined port
 			final int testPort = 22551;
 			config.setInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, testPort);
-			assertEquals(testPort, TaskManager.selectNetworkInterfaceAndPort(config)._2());
+			assertEquals(testPort, TaskManager.selectNetworkInterfaceAndPort(config, highAvailabilityServices)._2());
 
 			// invalid port
 			try {
 				config.setInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, -1);
-				TaskManager.selectNetworkInterfaceAndPort(config);
+				TaskManager.selectNetworkInterfaceAndPort(config, highAvailabilityServices);
 				fail("should fail with an exception");
 			}
 			catch (IllegalConfigurationException e) {
@@ -94,7 +111,7 @@ public void testActorSystemPortConfig() {
 			// invalid port
 			try {
 				config.setInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, 100000);
-				TaskManager.selectNetworkInterfaceAndPort(config);
+				TaskManager.selectNetworkInterfaceAndPort(config, highAvailabilityServices);
 				fail("should fail with an exception");
 			}
 			catch (IllegalConfigurationException e) {
@@ -104,6 +121,8 @@ public void testActorSystemPortConfig() {
 		catch (Exception e) {
 			e.printStackTrace();
 			fail(e.getMessage());
+		} finally {
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
@@ -136,7 +155,7 @@ public void testDefaultFsParameterLoading() {
 	}
 
 	@Test
-	public void testNetworkInterfaceSelection() {
+	public void testNetworkInterfaceSelection() throws Exception {
 		ServerSocket server;
 		String hostname = "localhost";
 
@@ -149,20 +168,27 @@ public void testNetworkInterfaceSelection() {
 			return;
 		}
 
-		try {
-			// open a server port to allow the system to connect
-			Configuration config = new Configuration();
+		// open a server port to allow the system to connect
+		Configuration config = new Configuration();
+
+		config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, hostname);
+		config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, server.getLocalPort());
 
-			config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, hostname);
-			config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, server.getLocalPort());
+		HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			Executors.directExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
-			assertNotNull(TaskManager.selectNetworkInterfaceAndPort(config)._1());
+		try {
+			assertNotNull(TaskManager.selectNetworkInterfaceAndPort(config, highAvailabilityServices)._1());
 		}
 		catch (Exception e) {
 			e.printStackTrace();
 			fail(e.getMessage());
 		}
 		finally {
+			highAvailabilityServices.closeAndCleanupAllData();
+
 			try {
 				server.close();
 			} catch (IOException e) {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java
index d90400434047a..130610ce87d58 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTestBase.java
@@ -23,17 +23,20 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceManager;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
-import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.taskexecutor.TaskExecutor;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.util.NetUtils;
+import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 import scala.Some;
 import scala.Tuple2;
@@ -56,7 +59,7 @@
 /**
  * Tests that the TaskManager process properly exits when the TaskManager actor dies.
  */
-public abstract class TaskManagerProcessReapingTestBase {
+public abstract class TaskManagerProcessReapingTestBase extends TestLogger {
 
 	/**
 	 * Called after the task manager has been started up. After calling this
@@ -72,12 +75,25 @@ void onTaskManagerProcessTerminated(String processOutput) {
 	}
 
 	@Test
-	public void testReapProcessOnFailure() {
+	public void testReapProcessOnFailure() throws Exception {
 		Process taskManagerProcess = null;
 		ActorSystem jmActorSystem = null;
 
 		final StringWriter processOutput = new StringWriter();
 
+		final Configuration config = new Configuration();
+
+		final String jobManagerHostname = "localhost";
+		final int jobManagerPort = NetUtils.getAvailablePort();
+
+		config.setString(JobManagerOptions.ADDRESS, jobManagerHostname);
+		config.setInteger(JobManagerOptions.PORT, jobManagerPort);
+
+		final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			TestingUtils.defaultExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
+
 		try {
 			String javaCommand = getJavaCommandPath();
 
@@ -93,29 +109,23 @@ public void testReapProcessOnFailure() {
 			tempLogFile.deleteOnExit();
 			CommonTestUtils.printLog4jDebugConfig(tempLogFile);
 
-			final int jobManagerPort = NetUtils.getAvailablePort();
-
 			// start a JobManager
-			Tuple2<String, Object> localAddress = new Tuple2<String, Object>("localhost", jobManagerPort);
-			jmActorSystem = AkkaUtils.createActorSystem(
-					new Configuration(), new Some<Tuple2<String, Object>>(localAddress));
+			Tuple2<String, Object> localAddress = new Tuple2<String, Object>(jobManagerHostname, jobManagerPort);
+			jmActorSystem = AkkaUtils.createActorSystem(config, new Some<>(localAddress));
 
 			ActorRef jmActor = JobManager.startJobManagerActors(
-				new Configuration(),
+				config,
 				jmActorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				JobManager.class,
 				MemoryArchivist.class)._1;
 
-			// start a ResourceManager
-			StandaloneLeaderRetrievalService standaloneLeaderRetrievalService =
-				new StandaloneLeaderRetrievalService(AkkaUtils.getAkkaURL(jmActorSystem, jmActor));
-
 			FlinkResourceManager.startResourceManagerActors(
 				new Configuration(),
 				jmActorSystem,
-				standaloneLeaderRetrievalService,
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 				StandaloneResourceManager.class);
 
 			final int taskManagerPort = NetUtils.getAvailablePort();
@@ -205,6 +215,9 @@ public void testReapProcessOnFailure() {
 			if (jmActorSystem != null) {
 				jmActorSystem.shutdown();
 			}
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
+			}
 		}
 	}
 
@@ -222,18 +235,28 @@ private static void printProcessLog(String log) {
 
 	public static class TaskManagerTestEntryPoint {
 
-		public static void main(String[] args) {
-			try {
-				int jobManagerPort = Integer.parseInt(args[0]);
-				int taskManagerPort = Integer.parseInt(args[1]);
+		public static void main(String[] args) throws Exception {
+			int jobManagerPort = Integer.parseInt(args[0]);
+			int taskManagerPort = Integer.parseInt(args[1]);
 
-				Configuration cfg = new Configuration();
-				cfg.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
-				cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort);
-				cfg.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L);
-				cfg.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 256);
+			Configuration cfg = new Configuration();
+			cfg.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
+			cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort);
+			cfg.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L);
+			cfg.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 256);
 
-				TaskManager.runTaskManager("localhost", ResourceID.generate(), taskManagerPort, cfg);
+			final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				cfg,
+				TestingUtils.defaultExecutor(),
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
+
+			try {
+				TaskManager.runTaskManager(
+					"localhost",
+					ResourceID.generate(),
+					taskManagerPort,
+					cfg,
+					highAvailabilityServices);
 
 				// wait forever
 				Object lock = new Object();
@@ -243,6 +266,8 @@ public static void main(String[] args) {
 			}
 			catch (Throwable t) {
 				System.exit(1);
+			} finally {
+				highAvailabilityServices.closeAndCleanupAllData();
 			}
 		}
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java
index 5f352298e7a17..92de31ac531d5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java
@@ -28,12 +28,14 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
 import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceManager;
+import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
 import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.instance.InstanceID;
-import org.apache.flink.runtime.jobmanager.JobManager;
-import org.apache.flink.runtime.jobmanager.MemoryArchivist;
-import org.apache.flink.runtime.jobmaster.JobMaster;
+import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.messages.JobManagerMessages.LeaderSessionMessage;
@@ -43,22 +45,29 @@
 import org.apache.flink.runtime.messages.TaskManagerMessages;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.TestLogger;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.mockito.Matchers;
 import scala.Option;
 import scala.concurrent.Await;
 import scala.concurrent.Future;
 import scala.concurrent.duration.Deadline;
 import scala.concurrent.duration.FiniteDuration;
 
+import java.util.Arrays;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.flink.runtime.testingUtils.TestingUtils.stopActor;
 import static org.apache.flink.runtime.testingUtils.TestingUtils.createTaskManager;
-import static org.apache.flink.runtime.testingUtils.TestingUtils.createJobManager;
+import static org.apache.flink.runtime.testingUtils.TestingUtils.stopActorGatewaysGracefully;
+import static org.apache.flink.runtime.testingUtils.TestingUtils.stopActorGracefully;
 import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * The tests in this class verify the behavior of the TaskManager
@@ -67,8 +76,6 @@
  */
 public class TaskManagerRegistrationTest extends TestLogger {
 
-	private static final Option<String> NONE_STRING = Option.empty();
-
 	// use one actor system throughout all tests
 	private static ActorSystem actorSystem;
 
@@ -76,6 +83,8 @@ public class TaskManagerRegistrationTest extends TestLogger {
 
 	private static FiniteDuration timeout = new FiniteDuration(20, TimeUnit.SECONDS);
 
+	private TestingHighAvailabilityServices highAvailabilityServices;
+
 	@BeforeClass
 	public static void startActorSystem() {
 		config = new Configuration();
@@ -94,38 +103,58 @@ public static void shutdownActorSystem() {
 		}
 	}
 
+	@Before
+	public void setupTest() {
+		highAvailabilityServices = new TestingHighAvailabilityServices();
+	}
+
+	@After
+	public void tearDownTest() throws Exception {
+		highAvailabilityServices.closeAndCleanupAllData();
+		highAvailabilityServices = null;
+	}
+
 	/**
 	 * A test that verifies that two TaskManagers correctly register at the
 	 * JobManager.
 	 */
 	@Test
-	public void testSimpleRegistration() {
+	public void testSimpleRegistration() throws Exception {
 		new JavaTestKit(actorSystem) {{
 
 			ActorGateway jobManager = null;
 			ActorGateway taskManager1 = null;
 			ActorGateway taskManager2 = null;
+			ActorGateway resourceManager = null;
+
+			EmbeddedHaServices embeddedHaServices = null;
 
 			try {
+				embeddedHaServices = new EmbeddedHaServices(Executors.directExecutor());
+
 				// a simple JobManager
-				jobManager = createJobManager(
+				jobManager = TestingUtils.createJobManager(
 					actorSystem,
 					TestingUtils.defaultExecutor(),
 					TestingUtils.defaultExecutor(),
-					config);
-				startResourceManager(config, jobManager.actor());
+					config,
+					embeddedHaServices);
+
+				resourceManager = new AkkaActorGateway(
+					startResourceManager(config, embeddedHaServices),
+					jobManager.leaderSessionID());
 
 				// start two TaskManagers. it will automatically try to register
 				taskManager1 = createTaskManager(
 						actorSystem,
-						jobManager,
+						embeddedHaServices,
 						config,
 						true,
 						false);
 
 				taskManager2 = createTaskManager(
 						actorSystem,
-						jobManager,
+						embeddedHaServices,
 						config,
 						true,
 						false);
@@ -142,10 +171,8 @@ public void testSimpleRegistration() {
 				Object response1 = Await.result(responseFuture1, timeout);
 				Object response2 = Await.result(responseFuture2, timeout);
 
-				// this is a hack to work around the way Java can interact with scala case objects
-				Class<?> confirmClass = TaskManagerMessages.getRegisteredAtJobManagerMessage().getClass();
-				assertTrue(response1 != null && confirmClass.isAssignableFrom(response1.getClass()));
-				assertTrue(response2 != null && confirmClass.isAssignableFrom(response2.getClass()));
+				assertTrue(response1 instanceof TaskManagerMessages.RegisteredAtJobManager);
+				assertTrue(response2 instanceof TaskManagerMessages.RegisteredAtJobManager);
 
 				// check that the JobManager has 2 TaskManagers registered
 				Future<Object> numTaskManagersFuture = jobManager.ask(
@@ -159,9 +186,9 @@ public void testSimpleRegistration() {
 				e.printStackTrace();
 				fail(e.getMessage());
 			} finally {
-				stopActor(taskManager1);
-				stopActor(taskManager2);
-				stopActor(jobManager);
+				stopActorGatewaysGracefully(Arrays.asList(taskManager1, taskManager2, jobManager, resourceManager));
+
+				embeddedHaServices.closeAndCleanupAllData();
 			}
 		}};
 	}
@@ -171,36 +198,35 @@ public void testSimpleRegistration() {
 	 * JobManager.
 	 */
 	@Test
-	public void testDelayedRegistration() {
+	public void testDelayedRegistration() throws Exception {
 		new JavaTestKit(actorSystem) {{
 			ActorGateway jobManager = null;
 			ActorGateway taskManager = null;
 
-			FiniteDuration delayedTimeout = timeout.$times(3);
+			FiniteDuration delayedTimeout = timeout.$times(3L);
+
+			final EmbeddedHaServices embeddedHaServices = new EmbeddedHaServices(Executors.directExecutor());
 
 			try {
 				// start a TaskManager that tries to register at the JobManager before the JobManager is
 				// available. we give it the regular JobManager akka URL
 				taskManager = createTaskManager(
 						actorSystem,
-						AkkaUtils.getLocalAkkaURL(JobMaster.JOB_MANAGER_NAME),
+						embeddedHaServices,
 						new Configuration(),
 						true,
 						false);
 
 				// let it try for a bit
-				Thread.sleep(6000);
+				Thread.sleep(6000L);
 
 				// now start the JobManager, with the regular akka URL
-				jobManager = createJobManager(
+				jobManager = TestingUtils.createJobManager(
 					actorSystem,
 					TestingUtils.defaultExecutor(),
 					TestingUtils.defaultExecutor(),
-					new Configuration());
-
-				startResourceManager(config, jobManager.actor());
-
-				startResourceManager(config, jobManager.actor());
+					new Configuration(),
+					embeddedHaServices);
 
 				// check that the TaskManagers are registered
 				Future<Object> responseFuture = taskManager.ask(
@@ -209,17 +235,11 @@ public void testDelayedRegistration() {
 
 				Object response = Await.result(responseFuture, delayedTimeout);
 
-				// this is a hack to work around the way Java can interact with scala case objects
-				Class<?> confirmClass = TaskManagerMessages.getRegisteredAtJobManagerMessage().getClass();
-				assertTrue(response != null && confirmClass.isAssignableFrom(response.getClass()));
-
-			}
-			catch (Exception e) {
-				e.printStackTrace();
-				fail(e.getMessage());
+				assertTrue(response instanceof TaskManagerMessages.RegisteredAtJobManager);
 			} finally {
-				stopActor(taskManager);
-				stopActor(jobManager);
+				stopActorGatewaysGracefully(Arrays.asList(taskManager, jobManager));
+
+				embeddedHaServices.closeAndCleanupAllData();
 			}
 		}};
 	}
@@ -246,10 +266,17 @@ public void testShutdownAfterRegistrationDurationExpired() {
 				Configuration tmConfig = new Configuration();
 				tmConfig.setString(ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, "500 ms");
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					// Give a non-existent job manager address to the task manager
+					new TestingLeaderRetrievalService(
+						"foobar",
+						HighAvailabilityServices.DEFAULT_LEADER_ID));
+
 				// start the taskManager actor
 				taskManager = createTaskManager(
 						actorSystem,
-						AkkaUtils.getLocalAkkaURL(JobMaster.JOB_MANAGER_NAME),
+						highAvailabilityServices,
 						tmConfig,
 						true,
 						false);
@@ -271,7 +298,7 @@ protected void run() {
 				e.printStackTrace();
 				fail(e.getMessage());
 			} finally {
-				stopActor(taskManager);
+				stopActorGracefully(taskManager);
 			}
 		}};
 	}
@@ -286,18 +313,28 @@ public void testTaskManagerResumesConnectAfterRefusedRegistration() {
 			ActorGateway jm = null;
 			ActorGateway taskManager =null;
 			try {
-				jm = TestingUtils.createForwardingActor(actorSystem, getTestActor(), Option.<String>empty());
+				jm = TestingUtils.createForwardingActor(
+					actorSystem,
+					getTestActor(),
+					HighAvailabilityServices.DEFAULT_LEADER_ID,
+					Option.<String>empty());
 				final ActorGateway jmGateway = jm;
 
 				FiniteDuration refusedRegistrationPause = new FiniteDuration(500, TimeUnit.MILLISECONDS);
 				Configuration tmConfig = new Configuration(config);
 				tmConfig.setString(ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE, refusedRegistrationPause.toString());
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new TestingLeaderRetrievalService(
+						jm.path(),
+						HighAvailabilityServices.DEFAULT_LEADER_ID));
+
 				// we make the test actor (the test kit) the JobManager to intercept
 				// the messages
 				taskManager = createTaskManager(
 					actorSystem,
-					jmGateway,
+					highAvailabilityServices,
 					tmConfig,
 					true,
 					false);
@@ -330,13 +367,8 @@ protected void run() {
 						expectMsgClass(RegisterTaskManager.class);
 					}
 				};
-			}
-			catch (Throwable e) {
-				e.printStackTrace();
-				fail(e.getMessage());
 			} finally {
-				stopActor(taskManager);
-				stopActor(jm);
+				stopActorGatewaysGracefully(Arrays.asList(taskManager, jm));
 			}
 		}};
 	}
@@ -353,7 +385,18 @@ public void testTaskManagerNoExcessiveRegistrationMessages() throws Exception {
 			try {
 				FiniteDuration timeout = new FiniteDuration(5, TimeUnit.SECONDS);
 
-				jm = TestingUtils.createForwardingActor(actorSystem, getTestActor(), Option.<String>empty());
+				jm = TestingUtils.createForwardingActor(
+					actorSystem,
+					getTestActor(),
+					HighAvailabilityServices.DEFAULT_LEADER_ID,
+					Option.<String>empty());
+
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new TestingLeaderRetrievalService(
+						jm.path(),
+						HighAvailabilityServices.DEFAULT_LEADER_ID));
+
 				final ActorGateway jmGateway = jm;
 
 				long refusedRegistrationPause = 500;
@@ -368,7 +411,7 @@ public void testTaskManagerNoExcessiveRegistrationMessages() throws Exception {
 				// the messages
 				taskManager = createTaskManager(
 					actorSystem,
-					jmGateway,
+					highAvailabilityServices,
 					tmConfig,
 					true,
 					false);
@@ -423,8 +466,7 @@ protected RegisterTaskManager match(Object msg) throws Exception {
 					+ maxExpectedNumberOfRegisterTaskManagerMessages,
 					registerTaskManagerMessages.length <= maxExpectedNumberOfRegisterTaskManagerMessages);
 			} finally {
-				stopActor(taskManager);
-				stopActor(jm);
+				stopActorGatewaysGracefully(Arrays.asList(taskManager, jm));
 			}
 		}};
 	}
@@ -444,16 +486,25 @@ public void testTaskManagerResumesConnectAfterJobManagerFailure() {
 
 			try {
 				fakeJobManager1Gateway = TestingUtils.createForwardingActor(
-						actorSystem,
-						getTestActor(),
-						Option.apply(JOB_MANAGER_NAME));
+					actorSystem,
+					getTestActor(),
+					HighAvailabilityServices.DEFAULT_LEADER_ID,
+					Option.apply(JOB_MANAGER_NAME));
 				final ActorGateway fakeJM1Gateway = fakeJobManager1Gateway;
 
+				TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService(
+					fakeJM1Gateway.path(),
+					HighAvailabilityServices.DEFAULT_LEADER_ID);
+
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					testingLeaderRetrievalService);
+
 				// we make the test actor (the test kit) the JobManager to intercept
 				// the messages
 				taskManagerGateway = createTaskManager(
 						actorSystem,
-						fakeJobManager1Gateway,
+						highAvailabilityServices,
 						config,
 						true,
 						false);
@@ -512,9 +563,10 @@ protected void run() {
 				do {
 					try {
 						fakeJobManager2Gateway = TestingUtils.createForwardingActor(
-								actorSystem,
-								getTestActor(),
-								Option.apply(JOB_MANAGER_NAME));
+							actorSystem,
+							getTestActor(),
+							HighAvailabilityServices.DEFAULT_LEADER_ID,
+							Option.apply(JOB_MANAGER_NAME));
 					} catch (InvalidActorNameException e) {
 						// wait and retry
 						Thread.sleep(100);
@@ -543,9 +595,7 @@ protected void run() {
 				e.printStackTrace();
 				fail(e.getMessage());
 			} finally {
-				stopActor(taskManagerGateway);
-				stopActor(fakeJobManager1Gateway);
-				stopActor(fakeJobManager2Gateway);
+				stopActorGatewaysGracefully(Arrays.asList(taskManagerGateway, fakeJobManager2Gateway));
 			}
 		}};
 	}
@@ -556,21 +606,25 @@ public void testCheckForValidRegistrationSessionIDs() {
 
 			ActorGateway taskManagerGateway = null;
 
+			final UUID falseLeaderSessionID = UUID.randomUUID();
+			final UUID trueLeaderSessionID = UUID.randomUUID();
+
+			HighAvailabilityServices mockedHighAvailabilityServices = mock(HighAvailabilityServices.class);
+			when(mockedHighAvailabilityServices.getJobManagerLeaderRetriever(Matchers.eq(HighAvailabilityServices.DEFAULT_JOB_ID)))
+				.thenReturn(new StandaloneLeaderRetrievalService(getTestActor().path().toString(), trueLeaderSessionID));
+
 			try {
 				// we make the test actor (the test kit) the JobManager to intercept
 				// the messages
 				taskManagerGateway = createTaskManager(
 						actorSystem,
-						getTestActor(),
+						mockedHighAvailabilityServices,
 						config,
 						true,
 						false);
 
 				final ActorRef taskManager = taskManagerGateway.actor();
 
-				final UUID falseLeaderSessionID = UUID.randomUUID();
-				final UUID trueLeaderSessionID = HighAvailabilityServices.DEFAULT_LEADER_ID;
-
 				new Within(timeout) {
 
 					@Override
@@ -581,7 +635,7 @@ protected void run() {
 
 						LeaderSessionMessage lsm = expectMsgClass(LeaderSessionMessage.class);
 
-						assertTrue(lsm.leaderSessionID() == trueLeaderSessionID);
+						assertTrue(lsm.leaderSessionID().equals(trueLeaderSessionID));
 						assertTrue(lsm.message() instanceof RegisterTaskManager);
 
 						final ActorRef tm = getLastSender();
@@ -606,9 +660,8 @@ protected void run() {
 								getTestActor());
 
 						Object message = null;
-						Object confirmMessageClass = TaskManagerMessages.getRegisteredAtJobManagerMessage().getClass();
 
-						while(message == null || !(message.getClass().equals(confirmMessageClass))) {
+						while(!(message instanceof TaskManagerMessages.RegisteredAtJobManager)) {
 							message = receiveOne(TestingUtils.TESTING_DURATION());
 						}
 
@@ -617,12 +670,8 @@ protected void run() {
 						expectMsgEquals(new JobManagerMessages.ResponseLeaderSessionID(trueLeaderSessionID));
 					}
 				};
-			}
-			catch (Throwable e) {
-				e.printStackTrace();
-				fail(e.getMessage());
 			} finally {
-				stopActor(taskManagerGateway);
+				stopActorGracefully(taskManagerGateway);
 			}
 		}};
 	}
@@ -631,25 +680,11 @@ protected void run() {
 	//  Utility Functions
 	// --------------------------------------------------------------------------------------------
 
-	private static ActorRef startJobManager(Configuration configuration) throws Exception {
-		// start the actors. don't give names, so they get generated names and we
-		// avoid conflicts with the actor names
-		return JobManager.startJobManagerActors(
-			configuration,
-			actorSystem,
-			TestingUtils.defaultExecutor(),
-			TestingUtils.defaultExecutor(),
-			NONE_STRING,
-			NONE_STRING,
-			JobManager.class,
-			MemoryArchivist.class)._1();
-	}
-
-	private static ActorRef startResourceManager(Configuration config, ActorRef jobManager) {
+	private static ActorRef startResourceManager(Configuration config, HighAvailabilityServices highAvailabilityServices) {
 		return FlinkResourceManager.startResourceManagerActors(
 			config,
 			actorSystem,
-			new StandaloneLeaderRetrievalService(jobManager.path().toString()),
+			highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 			StandaloneResourceManager.class);
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerStartupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerStartupTest.java
index 4df8db3204a31..0e77ddd0971ea 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerStartupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerStartupTest.java
@@ -26,9 +26,13 @@
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.util.StartupUtils;
 import org.apache.flink.util.NetUtils;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 import scala.Option;
 
@@ -46,6 +50,21 @@
  * problems.
  */
 public class TaskManagerStartupTest {
+
+	private HighAvailabilityServices highAvailabilityServices;
+
+	@Before
+	public void setupTest() {
+		highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
+	}
+
+	@After
+	public void tearDownTest() throws Exception {
+		if (highAvailabilityServices != null) {
+			highAvailabilityServices.closeAndCleanupAllData();
+			highAvailabilityServices = null;
+		}
+	}
 	
 
 	/**
@@ -55,8 +74,9 @@ public class TaskManagerStartupTest {
 	 * @throws Throwable
 	 */
 	@Test(expected = BindException.class)
-	public void testStartupWhenTaskmanagerActorPortIsUsed() throws BindException {
+	public void testStartupWhenTaskmanagerActorPortIsUsed() throws Exception {
 		ServerSocket blocker = null;
+
 		try {
 			final String localHostName = "localhost";
 			final InetAddress localBindAddress = InetAddress.getByName(NetUtils.getWildcardIPAddress());
@@ -65,8 +85,13 @@ public void testStartupWhenTaskmanagerActorPortIsUsed() throws BindException {
 			blocker = new ServerSocket(0, 50, localBindAddress);
 			final int port = blocker.getLocalPort();
 
-			TaskManager.runTaskManager(localHostName, ResourceID.generate(), port, new Configuration(),
-					TaskManager.class);
+			TaskManager.runTaskManager(
+				localHostName,
+				ResourceID.generate(),
+				port,
+				new Configuration(),
+				highAvailabilityServices,
+				TaskManager.class);
 			fail("This should fail with an IOException");
 
 		}
@@ -92,6 +117,8 @@ public void testStartupWhenTaskmanagerActorPortIsUsed() throws BindException {
 					// no need to log here
 				}
 			}
+
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
@@ -102,7 +129,7 @@ public void testStartupWhenTaskmanagerActorPortIsUsed() throws BindException {
 	 * directories are not writable.
 	 */
 	@Test
-	public void testIODirectoryNotWritable() {
+	public void testIODirectoryNotWritable() throws Exception {
 		File tempDir = new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH);
 		File nonWritable = new File(tempDir, UUID.randomUUID().toString());
 
@@ -119,7 +146,12 @@ public void testIODirectoryNotWritable() {
 			cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 21656);
 
 			try {
-				TaskManager.runTaskManager("localhost", ResourceID.generate(), 0, cfg);
+				TaskManager.runTaskManager(
+					"localhost",
+					ResourceID.generate(),
+					0,
+					cfg,
+					highAvailabilityServices);
 				fail("Should fail synchronously with an exception");
 			}
 			catch (IOException e) {
@@ -139,6 +171,8 @@ public void testIODirectoryNotWritable() {
 			catch (IOException e) {
 				// best effort
 			}
+
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
@@ -157,7 +191,12 @@ public void testMemoryConfigWrong() {
 			// something invalid
 			cfg.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, -42L);
 			try {
-				TaskManager.runTaskManager("localhost", ResourceID.generate(), 0, cfg);
+				TaskManager.runTaskManager(
+					"localhost",
+					ResourceID.generate(),
+					0,
+					cfg,
+					highAvailabilityServices);
 				fail("Should fail synchronously with an exception");
 			}
 			catch (IllegalConfigurationException e) {
@@ -169,7 +208,12 @@ public void testMemoryConfigWrong() {
 					TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()) >> 20;
 			cfg.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, memSize);
 			try {
-				TaskManager.runTaskManager("localhost", ResourceID.generate(), 0, cfg);
+				TaskManager.runTaskManager(
+					"localhost",
+					ResourceID.generate(),
+					0,
+					cfg,
+					highAvailabilityServices);
 				fail("Should fail synchronously with an exception");
 			}
 			catch (Exception e) {
@@ -204,9 +248,9 @@ public void testStartupWhenNetworkStackFailsToInitialize() throws Exception {
 				cfg,
 				ResourceID.generate(),
 				null,
+				highAvailabilityServices,
 				"localhost",
 				Option.<String>empty(),
-				Option.<LeaderRetrievalService>empty(),
 				false,
 				TaskManager.class);
 		}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index 4530adebee0f2..0f5afc095d7c7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -47,6 +47,7 @@
 import org.apache.flink.runtime.executiongraph.JobInformation;
 import org.apache.flink.runtime.executiongraph.TaskInformation;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.instance.InstanceID;
@@ -60,6 +61,8 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobmanager.Tasks;
+import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
+import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.RegistrationMessages;
 import org.apache.flink.runtime.messages.StackTraceSampleMessages.TriggerStackTraceSample;
@@ -80,8 +83,10 @@
 import org.apache.flink.util.NetUtils;
 import org.apache.flink.util.SerializedValue;
 import org.apache.flink.util.TestLogger;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -126,7 +131,9 @@ public class TaskManagerTest extends TestLogger {
 
 	private static ActorSystem system;
 
-	final static UUID leaderSessionID = HighAvailabilityServices.DEFAULT_LEADER_ID;
+	final static UUID leaderSessionID = UUID.randomUUID();
+
+	private TestingHighAvailabilityServices highAvailabilityServices;
 
 	@BeforeClass
 	public static void setup() {
@@ -138,20 +145,39 @@ public static void teardown() {
 		JavaTestKit.shutdownActorSystem(system);
 	}
 
+	@Before
+	public void setupTest() {
+		highAvailabilityServices = new TestingHighAvailabilityServices();
+	}
+
+	@After
+	public void tearDownTest() throws Exception {
+		if (highAvailabilityServices != null) {
+			highAvailabilityServices.closeAndCleanupAllData();
+
+			highAvailabilityServices = null;
+		}
+	}
+
 	@Test
 	public void testSubmitAndExecuteTask() throws IOException {
 		new JavaTestKit(system){{
 
 			ActorGateway taskManager = null;
 			final ActorGateway jobManager = TestingUtils.createForwardingActor(
-					system,
-					getTestActor(),
-					Option.<String>empty());
+				system,
+				getTestActor(),
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				Option.<String>empty());
+
+			highAvailabilityServices.setJobMasterLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID,
+				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
 
 			try {
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						new Configuration(),
 						true,
 						false);
@@ -261,9 +287,13 @@ public void testJobSubmissionAndCanceling() {
 				ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
 				jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						new Configuration(),
 						true,
 						true);
@@ -398,9 +428,13 @@ public void testJobSubmissionAndStop() throws Exception {
 				ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
 				jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						new Configuration(),
 						true,
 						true);
@@ -524,9 +558,13 @@ public void testGateChannelEdgeMismatch() {
 				ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
 				jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						new Configuration(),
 						true,
 						true);
@@ -624,9 +662,13 @@ public void testRunJobWithForwardChannel() {
 				ActorRef jm = system.actorOf(Props.create(new SimpleLookupJobManagerCreator(leaderSessionID)));
 				jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						new Configuration(),
 						true,
 						true);
@@ -769,9 +811,13 @@ public void testCancellingDependentAndStateUpdateFails() {
 
 				jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						new Configuration(),
 						true,
 						true);
@@ -906,6 +952,10 @@ public void testRemotePartitionNotFound() throws Exception {
 
 				jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				final int dataPort = NetUtils.getAvailablePort();
 				Configuration config = new Configuration();
 				config.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, dataPort);
@@ -914,7 +964,7 @@ public void testRemotePartitionNotFound() throws Exception {
 
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						config,
 						false,
 						true);
@@ -1024,13 +1074,17 @@ public void testLocalPartitionNotFound() throws Exception {
 
 				jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				final Configuration config = new Configuration();
 				config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
 				config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
 
 				taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						config,
 						true,
 						true);
@@ -1122,9 +1176,13 @@ public void testLogNotFoundHandling() throws Exception {
 				config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
 				config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");
 
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				taskManager = TestingUtils.createTaskManager(
 					system,
-					jobManager,
+					highAvailabilityServices,
 					config,
 					false,
 					true);
@@ -1166,18 +1224,26 @@ public void testTriggerStackTraceSampleMessage() throws Exception {
 
 			// We need this to be a JM that answers to update messages for
 			// robustness on Travis (if jobs need to be resubmitted in (4)).
-			ActorRef jm = system.actorOf(Props.create(new SimpleLookupJobManagerCreator(leaderSessionID)));
-			ActorGateway jobManagerActorGateway = new AkkaActorGateway(jm, leaderSessionID);
+			ActorRef jm = system.actorOf(Props.create(new SimpleLookupJobManagerCreator(
+				HighAvailabilityServices.DEFAULT_LEADER_ID)));
+			ActorGateway jobManagerActorGateway = new AkkaActorGateway(
+				jm,
+				HighAvailabilityServices.DEFAULT_LEADER_ID);
 
 			final ActorGateway testActorGateway = new AkkaActorGateway(
 					getTestActor(),
-					leaderSessionID);
+					HighAvailabilityServices.DEFAULT_LEADER_ID);
 
 			try {
 				final ActorGateway jobManager = jobManagerActorGateway;
+
+				highAvailabilityServices.setJobMasterLeaderRetriever(
+					HighAvailabilityServices.DEFAULT_JOB_ID,
+					new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 				final ActorGateway taskManager = TestingUtils.createTaskManager(
 						system,
-						jobManager,
+						highAvailabilityServices,
 						new Configuration(),
 						true,
 						false);
@@ -1459,11 +1525,15 @@ protected void run() {
 
 	@Test
 	public void testTerminationOnFatalError() {
+		highAvailabilityServices.setJobMasterLeaderRetriever(
+			HighAvailabilityServices.DEFAULT_JOB_ID,
+			new TestingLeaderRetrievalService());
+
 		new JavaTestKit(system){{
 
 			final ActorGateway taskManager = TestingUtils.createTaskManager(
 					system,
-					system.deadLetters(), // no jobmanager
+					highAvailabilityServices, // no jobmanager
 					new Configuration(),
 					true,
 					false);
@@ -1524,9 +1594,13 @@ public void testFailingScheduleOrUpdateConsumersMessage() throws Exception {
 			ActorRef jmActorRef = system.actorOf(Props.create(FailingScheduleOrUpdateConsumersJobManager.class, leaderSessionID), "jobmanager");
 			ActorGateway jobManager = new AkkaActorGateway(jmActorRef, leaderSessionID);
 
+			highAvailabilityServices.setJobMasterLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID,
+				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 			final ActorGateway taskManager = TestingUtils.createTaskManager(
 				system,
-				jobManager,
+				highAvailabilityServices,
 				configuration,
 				true,
 				true);
@@ -1562,9 +1636,13 @@ public void testSubmitTaskFailure() throws Exception {
 			ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
 			jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+			highAvailabilityServices.setJobMasterLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID,
+				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 			taskManager = TestingUtils.createTaskManager(
 				system,
-				jobManager,
+				highAvailabilityServices,
 				new Configuration(),
 				true,
 				true);
@@ -1619,9 +1697,13 @@ public void testStopTaskFailure() throws Exception {
 			ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
 			jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+			highAvailabilityServices.setJobMasterLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID,
+				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 			taskManager = TestingUtils.createTaskManager(
 				system,
-				jobManager,
+				highAvailabilityServices,
 				new Configuration(),
 				true,
 				true);
@@ -1679,9 +1761,13 @@ public void testStackTraceSampleFailure() throws Exception {
 			ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
 			jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+			highAvailabilityServices.setJobMasterLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID,
+				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 			taskManager = TestingUtils.createTaskManager(
 				system,
-				jobManager,
+				highAvailabilityServices,
 				new Configuration(),
 				true,
 				true);
@@ -1724,9 +1810,13 @@ public void testUpdateTaskInputPartitionsFailure() throws Exception {
 			ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
 			jobManager = new AkkaActorGateway(jm, leaderSessionID);
 
+			highAvailabilityServices.setJobMasterLeaderRetriever(
+				HighAvailabilityServices.DEFAULT_JOB_ID,
+				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));
+
 			taskManager = TestingUtils.createTaskManager(
 				system,
-				jobManager,
+				highAvailabilityServices,
 				new Configuration(),
 				true,
 				true);
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
index b2e8005f8b010..4be3299bf0346 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
@@ -19,7 +19,7 @@
 package org.apache.flink.runtime.jobmanager
 
 import java.net.InetAddress
-import java.util.concurrent.{Executors, ScheduledExecutorService}
+import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit}
 
 import akka.actor._
 import akka.testkit.{ImplicitSender, TestKit, TestProbe}
@@ -28,6 +28,7 @@ import org.apache.flink.runtime.akka.AkkaUtils
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager
 import org.apache.flink.runtime.clusterframework.types.ResourceID
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices
+import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices
 import org.apache.flink.runtime.instance._
 import org.apache.flink.runtime.jobmanager.JobManagerRegistrationTest.PlainForwardingActor
 import org.apache.flink.runtime.messages.JobManagerMessages.LeaderSessionMessage
@@ -40,7 +41,7 @@ import org.apache.flink.runtime.util.LeaderRetrievalUtils
 import org.junit.Assert.{assertNotEquals, assertNotNull}
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner
-import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Matchers, WordSpecLike}
 
 import scala.concurrent.Await
 import scala.concurrent.duration._
@@ -51,31 +52,45 @@ import scala.language.postfixOps
  */
 @RunWith(classOf[JUnitRunner])
 class JobManagerRegistrationTest(_system: ActorSystem) extends TestKit(_system) with
-ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
+ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll with BeforeAndAfterEach {
 
   def this() = this(AkkaUtils.createLocalActorSystem(new Configuration()))
 
   val executor: ScheduledExecutorService = Executors.newScheduledThreadPool(2)
+
+  var highAvailabilityServices: HighAvailabilityServices = _
+
+  val timeout = FiniteDuration(30, TimeUnit.SECONDS)
   
   override def afterAll(): Unit = {
     executor.shutdownNow()
     TestKit.shutdownActorSystem(system)
   }
 
+  override def beforeEach(): Unit = {
+    highAvailabilityServices = new EmbeddedHaServices(executor)
+  }
+
+  override def afterEach(): Unit = {
+    if (highAvailabilityServices != null) {
+      highAvailabilityServices.closeAndCleanupAllData()
+    }
+  }
+
   "The JobManager" should {
 
     "assign a TaskManager a unique instance ID" in {
 
       var jmOption: Option[ActorGateway] = None
-      var rmOption: Option[ActorGateway] = None
+      var rmOption: Option[ActorRef] = None
       var tm1Option: Option[ActorRef] = None
       var tm2Option: Option[ActorRef] = None
 
       try {
-        val jm = startTestingJobManager(_system)
+        val jm = startTestingJobManager(_system, highAvailabilityServices)
         jmOption = Some(jm)
 
-        val rm = startTestingResourceManager(_system, jm.actor())
+        val rm = startTestingResourceManager(_system, highAvailabilityServices)
         rmOption = Some(rm)
 
         val probe = TestProbe()
@@ -149,19 +164,19 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
     "handle repeated registration calls" in {
 
       var jmOption: Option[ActorGateway] = None
-      var rmOption: Option[ActorGateway] = None
+      var rmOption: Option[ActorRef] = None
 
       try {
         val probe = TestProbe()
 
-        val jm = startTestingJobManager(_system)
+        val jm = startTestingJobManager(_system, highAvailabilityServices)
         jmOption = Some(jm)
-        val rm = startTestingResourceManager(_system, jm.actor())
+        val rm = startTestingResourceManager(_system, highAvailabilityServices)
         rmOption = Some(rm)
 
-        val selfGateway = new AkkaActorGateway(
-          probe.ref,
-          HighAvailabilityServices.DEFAULT_LEADER_ID)
+        val leaderId = jm.leaderSessionID()
+
+        val selfGateway = new AkkaActorGateway(probe.ref, leaderId)
 
         val resourceID = ResourceID.generate()
         val connectionInfo = new TaskManagerLocation(resourceID, InetAddress.getLocalHost, 1)
@@ -199,22 +214,22 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
 
           probe.expectMsgType[LeaderSessionMessage] match {
             case LeaderSessionMessage(
-            HighAvailabilityServices.DEFAULT_LEADER_ID,
-            AcknowledgeRegistration(_, _)) =>
+              `leaderId`,
+              AcknowledgeRegistration(_, _)) =>
             case m => fail("Wrong message type: " + m)
           }
 
           probe.expectMsgType[LeaderSessionMessage] match {
             case LeaderSessionMessage(
-            HighAvailabilityServices.DEFAULT_LEADER_ID,
-            AlreadyRegistered(_, _)) =>
+              `leaderId`,
+              AlreadyRegistered(_, _)) =>
             case m => fail("Wrong message type: " + m)
           }
 
           probe.expectMsgType[LeaderSessionMessage] match {
             case LeaderSessionMessage(
-            HighAvailabilityServices.DEFAULT_LEADER_ID,
-            AlreadyRegistered(_, _)) =>
+              `leaderId`,
+              AlreadyRegistered(_, _)) =>
             case m => fail("Wrong message type: " + m)
           }
         }
@@ -225,14 +240,16 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
     }
   }
 
-  private def startTestingJobManager(system: ActorSystem): ActorGateway = {
-    val config = new Configuration()
+  private def startTestingJobManager(
+      system: ActorSystem,
+      highAvailabilityServices: HighAvailabilityServices): ActorGateway = {
 
+    val config = new Configuration()
+    
     val components = JobManager.createJobManagerComponents(
       config,
       executor,
-      executor,
-      None)
+      executor)
 
     // Start the JobManager without a MetricRegistry so that we don't start the MetricQueryService.
     // The problem of the MetricQueryService is that it starts an actor with a fixed name. Thus,
@@ -249,25 +266,34 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
       ActorRef.noSender,
       components._4,
       components._5,
+      highAvailabilityServices.getJobManagerLeaderElectionService(
+        HighAvailabilityServices.DEFAULT_JOB_ID),
+      highAvailabilityServices.getSubmittedJobGraphStore(),
+      highAvailabilityServices.getCheckpointRecoveryFactory(),
       components._8,
-      components._9,
-      components._10,
-      components._11,
       None)
 
-    val jm = _system.actorOf(props)
+    _system.actorOf(props)
 
-    new AkkaActorGateway(jm, HighAvailabilityServices.DEFAULT_LEADER_ID)
+    LeaderRetrievalUtils.retrieveLeaderGateway(
+      highAvailabilityServices.getJobManagerLeaderRetriever(
+        HighAvailabilityServices.DEFAULT_JOB_ID),
+      system,
+      timeout)
   }
 
-  private def startTestingResourceManager(system: ActorSystem, jm: ActorRef): ActorGateway = {
+  private def startTestingResourceManager(
+      system: ActorSystem,
+      highAvailabilityServices: HighAvailabilityServices)
+    : ActorRef = {
     val config = new Configuration()
-    val rm: ActorRef = FlinkResourceManager.startResourceManagerActors(
+
+    FlinkResourceManager.startResourceManagerActors(
       config,
       _system,
-      LeaderRetrievalUtils.createLeaderRetrievalService(config, jm),
+      highAvailabilityServices.getJobManagerLeaderRetriever(
+        HighAvailabilityServices.DEFAULT_JOB_ID),
       classOf[TestingResourceManager])
-    new AkkaActorGateway(rm, HighAvailabilityServices.DEFAULT_LEADER_ID)
   }
 }
 
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
index bd4a8fca09f06..7980cdfeaffe7 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
@@ -34,6 +34,7 @@ import org.apache.flink.runtime.clusterframework.FlinkResourceManager
 import org.apache.flink.runtime.clusterframework.types.ResourceIDRetrievable
 import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory
+import org.apache.flink.runtime.highavailability.{HighAvailabilityServices, HighAvailabilityServicesUtils}
 import org.apache.flink.runtime.instance.{ActorGateway, InstanceManager}
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler
 import org.apache.flink.runtime.jobmanager.{JobManager, MemoryArchivist, SubmittedJobGraphStore}
@@ -48,7 +49,7 @@ import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.NotifyWh
 import org.apache.flink.runtime.testutils.TestingResourceManager
 
 import scala.concurrent.duration.FiniteDuration
-import scala.concurrent.{Await, Future}
+import scala.concurrent.{Await, ExecutionContext, Future}
 
 /**
  * Testing cluster which starts the [[JobManager]] and [[TaskManager]] actors with testing support
@@ -60,16 +61,35 @@ import scala.concurrent.{Await, Future}
  */
 class TestingCluster(
     userConfiguration: Configuration,
+    highAvailabilityServices: HighAvailabilityServices,
     singleActorSystem: Boolean,
     synchronousDispatcher: Boolean)
   extends LocalFlinkMiniCluster(
     userConfiguration,
+    highAvailabilityServices,
     singleActorSystem) {
 
-  def this(userConfiguration: Configuration, singleActorSystem: Boolean) =
-    this(userConfiguration, singleActorSystem, false)
+  def this(
+      userConfiguration: Configuration,
+      singleActorSystem: Boolean,
+      synchronousDispatcher: Boolean) = {
+    this(
+      userConfiguration,
+      HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+        userConfiguration,
+        ExecutionContext.global),
+      singleActorSystem,
+      synchronousDispatcher)
+  }
+
+  def this(userConfiguration: Configuration, singleActorSystem: Boolean) = {
+    this(
+      userConfiguration,
+      singleActorSystem,
+      false)
+  }
 
-  def this(userConfiguration: Configuration) = this(userConfiguration, true, false)
+  def this(userConfiguration: Configuration) = this(userConfiguration, true)
 
   // --------------------------------------------------------------------------
 
@@ -224,10 +244,13 @@ class TestingCluster(
             Seq(newJobManagerActorSystem),
             1))
 
-          val lrs = createLeaderRetrievalService()
+          jobManagerLeaderRetrievalService.foreach(_.stop())
+
+          jobManagerLeaderRetrievalService = Option(
+            highAvailabilityServices.getJobManagerLeaderRetriever(
+              HighAvailabilityServices.DEFAULT_JOB_ID))
 
-          jobManagerLeaderRetrievalService = Some(lrs)
-          lrs.start(this)
+          jobManagerLeaderRetrievalService.foreach(_.start(this))
 
         case _ => throw new Exception("The JobManager of the TestingCluster have not " +
                                         "been started properly.")
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index d139a3f032e66..c8977f095ded7 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -27,22 +27,18 @@ import akka.pattern.{Patterns, ask}
 import com.google.common.util.concurrent.MoreExecutors
 import com.typesafe.config.ConfigFactory
 import grizzled.slf4j.Logger
-import org.apache.flink.api.common.JobExecutionResult
 import org.apache.flink.api.common.time.Time
 import org.apache.flink.configuration.{ConfigConstants, Configuration, HighAvailabilityOptions, TaskManagerOptions}
 import org.apache.flink.runtime.akka.AkkaUtils
-import org.apache.flink.runtime.client.JobClient
 import org.apache.flink.runtime.clusterframework.FlinkResourceManager
 import org.apache.flink.runtime.clusterframework.types.ResourceID
 import org.apache.flink.runtime.concurrent.{ScheduledExecutor, ScheduledExecutorServiceAdapter}
-import org.apache.flink.runtime.highavailability.{HighAvailabilityServices, HighAvailabilityServicesUtils}
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices
 import org.apache.flink.runtime.instance.{ActorGateway, AkkaActorGateway}
-import org.apache.flink.runtime.jobgraph.JobGraph
 import org.apache.flink.runtime.jobmanager.{JobManager, MemoryArchivist}
 import org.apache.flink.runtime.jobmaster.JobMaster
-import org.apache.flink.runtime.jobmaster.JobMaster.{ARCHIVE_NAME, JOB_MANAGER_NAME}
 import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService
-import org.apache.flink.runtime.messages.TaskManagerMessages.NotifyWhenRegisteredAtJobManager
+import org.apache.flink.runtime.messages.TaskManagerMessages.{NotifyWhenRegisteredAtJobManager, RegisteredAtJobManager}
 import org.apache.flink.runtime.taskmanager.TaskManager
 import org.apache.flink.runtime.testutils.TestingResourceManager
 import org.apache.flink.runtime.util.LeaderRetrievalUtils
@@ -222,69 +218,13 @@ object TestingUtils {
     }
   }
 
-  def createTaskManager(
-    actorSystem: ActorSystem,
-    jobManager: ActorRef,
-    configuration: Configuration,
-    useLocalCommunication: Boolean,
-    waitForRegistration: Boolean)
-  : ActorGateway = {
-    val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager)
-
-    createTaskManager(
-      actorSystem,
-      jobManagerURL,
-      configuration,
-      useLocalCommunication,
-      waitForRegistration
-    )
-  }
-
-  def createTaskManager(
-    actorSystem: ActorSystem,
-    jobManager: ActorGateway,
-    configuration: Configuration,
-    useLocalCommunication: Boolean,
-    waitForRegistration: Boolean,
-    taskManagerClass: Class[_ <: TaskManager])
-  : ActorGateway = {
-    val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager.actor)
-
-    createTaskManager(
-      actorSystem,
-      jobManagerURL,
-      configuration,
-      useLocalCommunication,
-      waitForRegistration,
-      taskManagerClass
-    )
-  }
-
-  def createTaskManager(
-      actorSystem: ActorSystem,
-      jobManager: ActorGateway,
-      configuration: Configuration,
-      useLocalCommunication: Boolean,
-      waitForRegistration: Boolean)
-    : ActorGateway = {
-    val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager.actor)
-
-    createTaskManager(
-      actorSystem,
-      jobManagerURL,
-      configuration,
-      useLocalCommunication,
-      waitForRegistration
-    )
-  }
-
   /** Creates a local TaskManager in the given ActorSystem. It is given a
     * [[StandaloneLeaderRetrievalService]] which returns the given jobManagerURL. After creating
     * the TaskManager, waitForRegistration specifies whether one waits until the TaskManager has
     * registered at the JobManager. An ActorGateway to the TaskManager is returned.
     *
     * @param actorSystem ActorSystem in which the TaskManager shall be started
-    * @param jobManagerURL URL of the JobManager to connect to
+    * @param highAvailabilityServices Service factory for high availability
     * @param configuration Configuration
     * @param useLocalCommunication true if the network stack shall use exclusively local
     *                              communication
@@ -293,15 +233,15 @@ object TestingUtils {
     * @return ActorGateway of the created TaskManager
     */
   def createTaskManager(
-    actorSystem: ActorSystem,
-    jobManagerURL: String,
-    configuration: Configuration,
-    useLocalCommunication: Boolean,
-    waitForRegistration: Boolean)
-  : ActorGateway = {
+      actorSystem: ActorSystem,
+      highAvailabilityServices: HighAvailabilityServices,
+      configuration: Configuration,
+      useLocalCommunication: Boolean,
+      waitForRegistration: Boolean)
+    : ActorGateway = {
     createTaskManager(
       actorSystem,
-      jobManagerURL,
+      highAvailabilityServices,
       configuration,
       useLocalCommunication,
       waitForRegistration,
@@ -311,7 +251,7 @@ object TestingUtils {
 
   def createTaskManager(
       actorSystem: ActorSystem,
-      jobManagerURL: String,
+      highAvailabilityServices: HighAvailabilityServices,
       configuration: Configuration,
       useLocalCommunication: Boolean,
       waitForRegistration: Boolean,
@@ -324,26 +264,27 @@ object TestingUtils {
 
     resultingConfiguration.addAll(configuration)
 
-    val leaderRetrievalService = Option(new StandaloneLeaderRetrievalService(jobManagerURL))
-
     val taskManager = TaskManager.startTaskManagerComponentsAndActor(
       resultingConfiguration,
       ResourceID.generate(),
       actorSystem,
+      highAvailabilityServices,
       "localhost",
       None,
-      leaderRetrievalService,
       useLocalCommunication,
       taskManagerClass
     )
 
-    if (waitForRegistration) {
+    val leaderId = if (waitForRegistration) {
       val notificationResult = (taskManager ? NotifyWhenRegisteredAtJobManager)(TESTING_DURATION)
+        .mapTo[RegisteredAtJobManager]
 
-      Await.ready(notificationResult, TESTING_DURATION)
+      Await.result(notificationResult, TESTING_DURATION).leaderId
+    } else {
+      HighAvailabilityServices.DEFAULT_LEADER_ID
     }
 
-    new AkkaActorGateway(taskManager, HighAvailabilityServices.DEFAULT_LEADER_ID)
+    new AkkaActorGateway(taskManager, leaderId)
   }
 
   /** Stops the given actor by sending it a Kill message
@@ -428,13 +369,15 @@ object TestingUtils {
       actorSystem: ActorSystem,
       futureExecutor: ScheduledExecutorService,
       ioExecutor: Executor,
-      configuration: Configuration)
+      configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices)
     : ActorGateway = {
     createJobManager(
       actorSystem,
       futureExecutor,
       ioExecutor,
       configuration,
+      highAvailabilityServices,
       classOf[TestingJobManager],
       ""
     )
@@ -455,6 +398,7 @@ object TestingUtils {
       futureExecutor: ScheduledExecutorService,
       ioExecutor: Executor,
       configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices,
       prefix: String)
     : ActorGateway = {
     createJobManager(
@@ -462,6 +406,7 @@ object TestingUtils {
       futureExecutor,
       ioExecutor,
       configuration,
+      highAvailabilityServices,
       classOf[TestingJobManager],
       prefix
     )
@@ -474,6 +419,7 @@ object TestingUtils {
     * @param futureExecutor to run the JobManager's futures
     * @param ioExecutor to run blocking io operations
     * @param configuration Configuration to use
+    * @param highAvailabilityServices Service factory for high availability
     * @param jobManagerClass JobManager class to instantiate
     * @return
     */
@@ -482,10 +428,18 @@ object TestingUtils {
       futureExecutor: ScheduledExecutorService,
       ioExecutor: Executor,
       configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices,
       jobManagerClass: Class[_ <: JobManager])
     : ActorGateway = {
 
-    createJobManager(actorSystem, futureExecutor, ioExecutor, configuration, jobManagerClass, "")
+    createJobManager(
+      actorSystem,
+      futureExecutor,
+      ioExecutor,
+      configuration,
+      highAvailabilityServices,
+      jobManagerClass,
+      "")
   }
 
   /**
@@ -496,6 +450,7 @@ object TestingUtils {
     * @param futureExecutor to run the JobManager's futures
     * @param ioExecutor to run blocking io operations
     * @param configuration Configuration to use
+    * @param highAvailabilityServices Service factory for high availability
     * @param jobManagerClass JobManager class to instantiate
     * @param prefix The prefix to use for the Actor names
    * @return
@@ -505,6 +460,7 @@ object TestingUtils {
       futureExecutor: ScheduledExecutorService,
       ioExecutor: Executor,
       configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices,
       jobManagerClass: Class[_ <: JobManager],
       prefix: String)
     : ActorGateway = {
@@ -518,24 +474,33 @@ object TestingUtils {
         actorSystem,
         futureExecutor,
         ioExecutor,
+        highAvailabilityServices,
         Some(prefix + JobMaster.JOB_MANAGER_NAME),
         Some(prefix + JobMaster.ARCHIVE_NAME),
         jobManagerClass,
         classOf[MemoryArchivist])
 
-    new AkkaActorGateway(actor, HighAvailabilityServices.DEFAULT_LEADER_ID)
+
+    val leaderId = LeaderRetrievalUtils.retrieveLeaderSessionId(
+        highAvailabilityServices.getJobManagerLeaderRetriever(
+          HighAvailabilityServices.DEFAULT_JOB_ID),
+        TestingUtils.TESTING_TIMEOUT)
+
+    new AkkaActorGateway(actor, leaderId)
   }
 
   /** Creates a forwarding JobManager which sends all received message to the forwarding target.
     *
     * @param actorSystem The actor system to start the actor in.
     * @param forwardingTarget Target to forward to.
+    * @param leaderId leader id for the actor gateway
     * @param actorName Name for forwarding Actor
     * @return
     */
   def createForwardingActor(
       actorSystem: ActorSystem,
       forwardingTarget: ActorRef,
+      leaderId: UUID,
       actorName: Option[String] = None)
     : ActorGateway = {
 
@@ -545,7 +510,7 @@ object TestingUtils {
           Props(
             classOf[ForwardingActor],
             forwardingTarget,
-            Option(HighAvailabilityServices.DEFAULT_LEADER_ID)),
+            Option(leaderId)),
           name
         )
       case None =>
@@ -553,58 +518,39 @@ object TestingUtils {
           Props(
             classOf[ForwardingActor],
             forwardingTarget,
-            Option(HighAvailabilityServices.DEFAULT_LEADER_ID))
+            Option(leaderId))
         )
     }
 
-    new AkkaActorGateway(actor, HighAvailabilityServices.DEFAULT_LEADER_ID)
-  }
-
-  def submitJobAndWait(
-      actorSystem: ActorSystem,
-      jobManager: ActorGateway,
-      jobGraph: JobGraph,
-      config: Configuration)
-    : JobExecutionResult = {
-
-    val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager.actor)
-    val leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManagerURL)
-
-    JobClient.submitJobAndWait(
-      actorSystem,
-      config,
-      leaderRetrievalService,
-      jobGraph,
-      TESTING_DURATION,
-      false,
-      Thread.currentThread().getContextClassLoader
-    )
+    new AkkaActorGateway(actor, leaderId)
   }
 
-  /** Creates a testing JobManager using the default recovery mode (standalone)
+  /** Creates a testing JobManager using the given configuration and high availability services.
     *
     * @param actorSystem The actor system to start the actor
-    * @param jobManager The jobManager for the standalone leader service.
     * @param configuration The configuration
+    * @param highAvailabilityServices The high availability services to use
     * @return
     */
   def createResourceManager(
       actorSystem: ActorSystem,
-      jobManager: ActorRef,
-      configuration: Configuration)
+      configuration: Configuration,
+      highAvailabilityServices: HighAvailabilityServices)
   : ActorGateway = {
 
-    configuration.setString(
-      HighAvailabilityOptions.HA_MODE,
-      ConfigConstants.DEFAULT_HA_MODE)
-
-    val actor = FlinkResourceManager.startResourceManagerActors(
+    val resourceManager = FlinkResourceManager.startResourceManagerActors(
       configuration,
       actorSystem,
-      LeaderRetrievalUtils.createLeaderRetrievalService(configuration, jobManager),
+      highAvailabilityServices.getJobManagerLeaderRetriever(
+        HighAvailabilityServices.DEFAULT_JOB_ID),
       classOf[TestingResourceManager])
 
-    new AkkaActorGateway(actor, HighAvailabilityServices.DEFAULT_LEADER_ID)
+    val leaderId = LeaderRetrievalUtils.retrieveLeaderSessionId(
+      highAvailabilityServices.getJobManagerLeaderRetriever(
+        HighAvailabilityServices.DEFAULT_JOB_ID),
+      TestingUtils.TESTING_TIMEOUT)
+
+    new AkkaActorGateway(resourceManager, leaderId)
   }
 
   class ForwardingActor(val target: ActorRef, val leaderSessionID: Option[UUID])
diff --git a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala
index 4d3adb2c9f284..bb2645473bb6a 100644
--- a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala
+++ b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkShell.scala
@@ -25,6 +25,7 @@ import org.apache.commons.cli.CommandLine
 import org.apache.flink.client.cli.CliFrontendParser
 import org.apache.flink.client.program.ClusterClient
 import org.apache.flink.client.CliFrontend
+import org.apache.flink.runtime.minicluster.StandaloneMiniCluster
 import org.apache.flink.configuration.{ConfigConstants, GlobalConfiguration}
 import org.apache.flink.runtime.minicluster.{FlinkMiniCluster, LocalFlinkMiniCluster}
 
@@ -140,18 +141,17 @@ object FlinkShell {
 
   def fetchConnectionInfo(
     config: Config
-  ): (String, Int, Option[Either[FlinkMiniCluster, ClusterClient]]) = {
+  ): (String, Int, Option[Either[StandaloneMiniCluster, ClusterClient]]) = {
     config.executionMode match {
       case ExecutionMode.LOCAL => // Local mode
         val config = GlobalConfiguration.loadConfiguration()
         config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 0)
 
-        val miniCluster = new LocalFlinkMiniCluster(config, false)
-        miniCluster.start()
+        val miniCluster = new StandaloneMiniCluster(config)
 
         println("\nStarting local Flink cluster (host: localhost, " +
-          s"port: ${miniCluster.getLeaderRPCPort}).\n")
-        ("localhost", miniCluster.getLeaderRPCPort, Some(Left(miniCluster)))
+          s"port: ${miniCluster.getPort}).\n")
+        ("localhost", miniCluster.getPort, Some(Left(miniCluster)))
 
       case ExecutionMode.REMOTE => // Remote mode
         if (config.host.isEmpty || config.port.isEmpty) {
@@ -188,7 +188,7 @@ object FlinkShell {
     val (repl, cluster) = try {
       val (host, port, cluster) = fetchConnectionInfo(config)
       val conf = cluster match {
-        case Some(Left(miniCluster)) => miniCluster.userConfiguration
+        case Some(Left(miniCluster)) => miniCluster.getConfiguration
         case Some(Right(yarnCluster)) => yarnCluster.getFlinkConfiguration
         case None => GlobalConfiguration.loadConfiguration()
       }
@@ -218,7 +218,7 @@ object FlinkShell {
     } finally {
       repl.closeInterpreter()
       cluster match {
-        case Some(Left(miniCluster)) => miniCluster.stop()
+        case Some(Left(miniCluster)) => miniCluster.close()
         case Some(Right(yarnCluster)) => yarnCluster.shutdown()
         case _ =>
       }
diff --git a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala
index 00410cc703199..0e89da3b9a295 100644
--- a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala
+++ b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala
@@ -19,14 +19,16 @@
 package org.apache.flink.api.scala
 
 import java.io._
-import java.util.concurrent.TimeUnit
 
-import org.apache.flink.configuration.GlobalConfiguration
-import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster
+import akka.actor.ActorRef
+import akka.pattern.Patterns
+import org.apache.flink.runtime.minicluster.StandaloneMiniCluster
+import org.apache.flink.configuration.{ConfigConstants, Configuration, GlobalConfiguration}
 import org.apache.flink.test.util.TestBaseUtils
 import org.apache.flink.util.TestLogger
 import org.junit.{AfterClass, Assert, BeforeClass, Test}
 
+import scala.concurrent.{Await, Future}
 import scala.concurrent.duration.FiniteDuration
 import scala.tools.nsc.Settings
 
@@ -299,18 +301,19 @@ class ScalaShellITCase extends TestLogger {
       .getFile
     val confDir = new File(confFile).getAbsoluteFile.getParent
 
-    val (c, args) = cluster match{
+    val args = cluster match {
       case Some(cl) =>
-        val arg = Array("remote",
-          cl.hostname,
-          Integer.toString(cl.getLeaderRPCPort),
+        Array(
+          "remote",
+          cl.getHostname,
+          Integer.toString(cl.getPort),
           "--configDir",
           confDir)
-        (cl, arg)
-      case None =>
-        throw new AssertionError("Cluster creation failed.")
+      case None => throw new IllegalStateException("Cluster has not been started.")
     }
 
+
+
     //start scala shell with initialized
     // buffered reader for testing
     FlinkShell.bufferedReader = Some(in)
@@ -335,26 +338,24 @@ class ScalaShellITCase extends TestLogger {
 }
 
 object ScalaShellITCase {
-  var cluster: Option[LocalFlinkMiniCluster] = None
+  var cluster: Option[StandaloneMiniCluster] = None
+
   val parallelism = 4
+  val configuration = new Configuration()
 
   @BeforeClass
   def beforeAll(): Unit = {
-    val cl = TestBaseUtils.startCluster(
-      1,
-      parallelism,
-      false,
-      false,
-      false)
-
-    cluster = Some(cl)
+    configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, parallelism)
+
+    cluster = Option(new StandaloneMiniCluster(configuration))
   }
 
   @AfterClass
   def afterAll(): Unit = {
     // The Scala interpreter somehow changes the class loader. Therfore, we have to reset it
     Thread.currentThread().setContextClassLoader(classOf[ScalaShellITCase].getClassLoader)
-    cluster.foreach(c => TestBaseUtils.stopCluster(c, new FiniteDuration(1000, TimeUnit.SECONDS)))
+
+    cluster.foreach(_.close)
   }
 
   /**
@@ -371,47 +372,46 @@ object ScalaShellITCase {
     val oldOut = System.out
     System.setOut(new PrintStream(baos))
 
-    // new local cluster
-    val host = "localhost"
-    val port = cluster match {
-      case Some(c) => c.getLeaderRPCPort
-      case _ => throw new RuntimeException("Test cluster not initialized.")
-    }
-
-    val repl = externalJars match {
-      case Some(ej) => new FlinkILoop(
-        host, port,
-        GlobalConfiguration.loadConfiguration(),
-        Option(Array(ej)),
-        in, new PrintWriter(out))
-
-      case None => new FlinkILoop(
-        host, port,
-        GlobalConfiguration.loadConfiguration(),
-        in, new PrintWriter(out))
-    }
+    cluster match {
+      case Some(cl) =>
+        val repl = externalJars match {
+          case Some(ej) => new FlinkILoop(
+            cl.getHostname,
+            cl.getPort,
+            GlobalConfiguration.loadConfiguration(),
+            Option(Array(ej)),
+            in, new PrintWriter(out))
+
+          case None => new FlinkILoop(
+            cl.getHostname,
+            cl.getPort,
+            GlobalConfiguration.loadConfiguration(),
+            in, new PrintWriter(out))
+        }
 
-    repl.settings = new Settings()
+        repl.settings = new Settings()
 
-    // enable this line to use scala in intellij
-    repl.settings.usejavacp.value = true
+        // enable this line to use scala in intellij
+        repl.settings.usejavacp.value = true
 
-    externalJars match {
-      case Some(ej) => repl.settings.classpath.value = ej
-      case None =>
-    }
+        externalJars match {
+          case Some(ej) => repl.settings.classpath.value = ej
+          case None =>
+        }
 
-    repl.process(repl.settings)
+        repl.process(repl.settings)
 
-    repl.closeInterpreter()
+        repl.closeInterpreter()
 
-    System.setOut(oldOut)
+        System.setOut(oldOut)
 
-    baos.flush()
+        baos.flush()
 
-    val stdout = baos.toString
+        val stdout = baos.toString
 
-    out.toString + stdout
+        out.toString + stdout
+      case _ => throw new IllegalStateException("The cluster has not been started.")
+    }
   }
 
   def findLibraryFolder(paths: String*): File = {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
index ff5af87b634cb..2ff45baaf3ac6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
@@ -219,7 +219,11 @@ protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL>
 			throw new ProgramInvocationException("The program execution failed" + term, e);
 		}
 		finally {
-			client.shutdown();
+			try {
+				client.shutdown();
+			} catch (Exception e) {
+				LOG.warn("Could not properly shut down the cluster client.", e);
+			}
 		}
 	}
 
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
index 64c68dc168a2e..5719456c53406 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
@@ -19,6 +19,7 @@
 package org.apache.flink.streaming.util;
 
 import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -26,49 +27,105 @@
 import org.apache.flink.streaming.api.graph.StreamGraph;
 import org.apache.flink.util.Preconditions;
 
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
 /**
- * A StreamExecutionEnvironment that executes its jobs on a test cluster.
+ * A {@link StreamExecutionEnvironment} that executes its jobs on {@link LocalFlinkMiniCluster}.
  */
 public class TestStreamEnvironment extends StreamExecutionEnvironment {
 	
 	/** The mini cluster in which this environment executes its jobs */
-	private LocalFlinkMiniCluster executor;
-	
+	private final LocalFlinkMiniCluster miniCluster;
+
+	private final Collection<Path> jarFiles;
+
+	private final Collection<URL> classPaths;
+
+	public TestStreamEnvironment(
+			LocalFlinkMiniCluster miniCluster,
+			int parallelism,
+			Collection<Path> jarFiles,
+			Collection<URL> classPaths) {
+
+		this.miniCluster = Preconditions.checkNotNull(miniCluster);
+		this.jarFiles = Preconditions.checkNotNull(jarFiles);
+		this.classPaths = Preconditions.checkNotNull(classPaths);
 
-	public TestStreamEnvironment(LocalFlinkMiniCluster executor, int parallelism) {
-		this.executor = Preconditions.checkNotNull(executor);
 		setParallelism(parallelism);
 	}
+
+	public TestStreamEnvironment(
+			LocalFlinkMiniCluster miniCluster,
+			int parallelism) {
+		this(miniCluster, parallelism, Collections.<Path>emptyList(), Collections.<URL>emptyList());
+	}
+
 	
 	@Override
 	public JobExecutionResult execute(String jobName) throws Exception {
 		final StreamGraph streamGraph = getStreamGraph();
 		streamGraph.setJobName(jobName);
 		final JobGraph jobGraph = streamGraph.getJobGraph();
-		return executor.submitJobAndWait(jobGraph, false);
+
+		for (Path jarFile: jarFiles) {
+			jobGraph.addJar(jarFile);
+		}
+
+		jobGraph.setClasspaths(new ArrayList<>(classPaths));
+
+		return miniCluster.submitJobAndWait(jobGraph, false);
 	}
 
 	// ------------------------------------------------------------------------
 
 	/**
 	 * Sets the streaming context environment to a TestStreamEnvironment that runs its programs on
-	 * the given cluster with the given default parallelism.
-	 * 
+	 * the given cluster with the given default parallelism and the specified jar files and class
+	 * paths.
+	 *
 	 * @param cluster The test cluster to run the test program on.
 	 * @param parallelism The default parallelism for the test programs.
+	 * @param jarFiles Additional jar files to execute the job with
+	 * @param classpaths Additional class paths to execute the job with
 	 */
-	public static void setAsContext(final LocalFlinkMiniCluster cluster, final int parallelism) {
-		
+	public static void setAsContext(
+			final LocalFlinkMiniCluster cluster,
+			final int parallelism,
+			final Collection<Path> jarFiles,
+			final Collection<URL> classpaths) {
+
 		StreamExecutionEnvironmentFactory factory = new StreamExecutionEnvironmentFactory() {
 			@Override
 			public StreamExecutionEnvironment createExecutionEnvironment() {
-				return new TestStreamEnvironment(cluster, parallelism);
+				return new TestStreamEnvironment(
+					cluster,
+					parallelism,
+					jarFiles,
+					classpaths);
 			}
 		};
 
 		initializeContextEnvironment(factory);
 	}
 
+	/**
+	 * Sets the streaming context environment to a TestStreamEnvironment that runs its programs on
+	 * the given cluster with the given default parallelism.
+	 * 
+	 * @param cluster The test cluster to run the test program on.
+	 * @param parallelism The default parallelism for the test programs.
+	 */
+	public static void setAsContext(final LocalFlinkMiniCluster cluster, final int parallelism) {
+		setAsContext(
+			cluster,
+			parallelism,
+			Collections.<Path>emptyList(),
+			Collections.<URL>emptyList());
+	}
+
 	/**
 	 * Resets the streaming context environment to null.
 	 */
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
index 2bcf66dd0ce1d..f06cdb7ded786 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
@@ -111,7 +111,7 @@ public void testJobWithObjectReuse() throws Exception {
 			}
 			
 			// prepare the test environment
-			TestEnvironment env = new TestEnvironment(this.executor, this.parallelism);
+			TestEnvironment env = new TestEnvironment(this.executor, this.parallelism, false);
 			env.getConfig().enableObjectReuse();
 			env.setAsContext();
 
@@ -163,7 +163,7 @@ public void testJobWithoutObjectReuse() throws Exception {
 			}
 
 			// prepare the test environment
-			TestEnvironment env = new TestEnvironment(this.executor, this.parallelism);
+			TestEnvironment env = new TestEnvironment(this.executor, this.parallelism, false);
 			env.getConfig().disableObjectReuse();
 			env.setAsContext();
 
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
index 2043cd0fc6242..ed141d12ebc98 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
@@ -86,7 +86,7 @@ public MultipleProgramsTestBase(TestExecutionMode mode) {
 		
 		switch(mode){
 			case CLUSTER:
-				new TestEnvironment(cluster, 4).setAsContext();
+				new TestEnvironment(cluster, 4, false).setAsContext();
 				break;
 			case CLUSTER_OBJECT_REUSE:
 				new TestEnvironment(cluster, 4, true).setAsContext();
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
index aea81522150fb..a30db40df78e9 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
@@ -23,6 +23,7 @@
 import org.apache.flink.api.common.CodeAnalysisMode;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.ExecutionEnvironmentFactory;
+import org.apache.flink.core.fs.Path;
 import org.apache.flink.optimizer.DataStatistics;
 import org.apache.flink.optimizer.Optimizer;
 import org.apache.flink.optimizer.plan.OptimizedPlan;
@@ -30,39 +31,71 @@
 import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.util.Preconditions;
 
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * A {@link ExecutionEnvironment} implementation which executes its jobs on a
+ * {@link LocalFlinkMiniCluster}.
+ */
 public class TestEnvironment extends ExecutionEnvironment {
 
-	private final LocalFlinkMiniCluster executor;
+	private final LocalFlinkMiniCluster miniCluster;
 
-	private TestEnvironment lastEnv = null;
+	private final Collection<Path> jarFiles;
 
-	@Override
-	public JobExecutionResult getLastJobExecutionResult() {
-		if (lastEnv == null) {
-			return this.lastJobExecutionResult;
-		}
-		else {
-			return lastEnv.getLastJobExecutionResult();
-		}
-	}
+	private final Collection<URL> classPaths;
+
+	private TestEnvironment lastEnv;
+
+	public TestEnvironment(
+			LocalFlinkMiniCluster miniCluster,
+			int parallelism,
+			boolean isObjectReuseEnabled,
+			Collection<Path> jarFiles,
+			Collection<URL> classPaths) {
+		this.miniCluster = Preconditions.checkNotNull(miniCluster);
+		this.jarFiles = Preconditions.checkNotNull(jarFiles);
+		this.classPaths = Preconditions.checkNotNull(classPaths);
 
-	public TestEnvironment(LocalFlinkMiniCluster executor, int parallelism) {
-		this.executor = executor;
 		setParallelism(parallelism);
 
 		// disabled to improve build time
 		getConfig().setCodeAnalysisMode(CodeAnalysisMode.DISABLE);
-	}
-
-	public TestEnvironment(LocalFlinkMiniCluster executor, int parallelism, boolean isObjectReuseEnabled) {
-		this(executor, parallelism);
 
 		if (isObjectReuseEnabled) {
 			getConfig().enableObjectReuse();
 		} else {
 			getConfig().disableObjectReuse();
 		}
+
+		lastEnv = null;
+	}
+
+	public TestEnvironment(
+			LocalFlinkMiniCluster executor,
+			int parallelism,
+			boolean isObjectReuseEnabled) {
+		this(
+			executor,
+			parallelism,
+			isObjectReuseEnabled,
+			Collections.<Path>emptyList(),
+			Collections.<URL>emptyList());
+	}
+
+	@Override
+	public JobExecutionResult getLastJobExecutionResult() {
+		if (lastEnv == null) {
+			return lastJobExecutionResult;
+		}
+		else {
+			return lastEnv.getLastJobExecutionResult();
+		}
 	}
 
 	@Override
@@ -76,7 +109,13 @@ public JobExecutionResult execute(String jobName) throws Exception {
 		JobGraphGenerator jgg = new JobGraphGenerator();
 		JobGraph jobGraph = jgg.compileJobGraph(op);
 
-		this.lastJobExecutionResult = executor.submitJobAndWait(jobGraph, false);
+		for (Path jarFile: jarFiles) {
+			jobGraph.addJar(jarFile);
+		}
+
+		jobGraph.setClasspaths(new ArrayList<>(classPaths));
+
+		this.lastJobExecutionResult = miniCluster.submitJobAndWait(jobGraph, false);
 		return this.lastJobExecutionResult;
 	}
 
@@ -93,7 +132,7 @@ public String getExecutionPlan() throws Exception {
 	private OptimizedPlan compileProgram(String jobName) {
 		Plan p = createProgramPlan(jobName);
 
-		Optimizer pc = new Optimizer(new DataStatistics(), this.executor.configuration());
+		Optimizer pc = new Optimizer(new DataStatistics(), this.miniCluster.configuration());
 		return pc.compile(p);
 	}
 
@@ -101,11 +140,65 @@ public void setAsContext() {
 		ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {
 			@Override
 			public ExecutionEnvironment createExecutionEnvironment() {
-				lastEnv = new TestEnvironment(executor, getParallelism(), getConfig().isObjectReuseEnabled());
+				lastEnv = new TestEnvironment(miniCluster, getParallelism(), getConfig().isObjectReuseEnabled());
 				return lastEnv;
 			}
 		};
 
 		initializeContextEnvironment(factory);
 	}
+
+	// ---------------------------------------------------------------------------------------------
+
+	/**
+	 * Sets the current {@link ExecutionEnvironment} to be a {@link TestEnvironment}. The test
+	 * environment executes the given jobs on a Flink mini cluster with the given default
+	 * parallelism and the additional jar files and class paths.
+	 *
+	 * @param miniCluster The mini cluster on which to execute the jobs
+	 * @param parallelism The default parallelism
+	 * @param jarFiles Additional jar files to execute the job with
+	 * @param classPaths Additional class paths to execute the job with
+	 */
+	public static void setAsContext(
+		final LocalFlinkMiniCluster miniCluster,
+		final int parallelism,
+		final Collection<Path> jarFiles,
+		final Collection<URL> classPaths) {
+
+		ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {
+			@Override
+			public ExecutionEnvironment createExecutionEnvironment() {
+				return new TestEnvironment(
+					miniCluster,
+					parallelism,
+					false, jarFiles,
+					classPaths
+				);
+			}
+		};
+
+		initializeContextEnvironment(factory);
+	}
+
+
+	/**
+	 * Sets the current {@link ExecutionEnvironment} to be a {@link TestEnvironment}. The test
+	 * environment executes the given jobs on a Flink mini cluster with the given default
+	 * parallelism and the additional jar files and class paths.
+	 *
+	 * @param miniCluster The mini cluster on which to execute the jobs
+	 * @param parallelism The default parallelism
+	 */
+	public static void setAsContext(final LocalFlinkMiniCluster miniCluster, final int parallelism) {
+		setAsContext(
+			miniCluster,
+			parallelism,
+			Collections.<Path>emptyList(),
+			Collections.<URL>emptyList());
+	}
+
+	public static void unsetAsContext() {
+		resetContextEnvironment();
+	}
 }
diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java
index 56d1525662b8c..25d9228b17e11 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java
@@ -30,6 +30,7 @@
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -48,39 +49,30 @@ public class AccumulatorErrorITCase extends TestLogger {
 
 	private static LocalFlinkMiniCluster cluster;
 
+	private static ExecutionEnvironment env;
+
 	@BeforeClass
 	public static void startCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 3);
-			config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L);
-			cluster = new LocalFlinkMiniCluster(config, false);
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 3);
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L);
+		cluster = new LocalFlinkMiniCluster(config, false);
 
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to start test cluster: " + e.getMessage());
-		}
+		cluster.start();
+
+		env = new TestEnvironment(cluster, 6, false);
 	}
 
 	@AfterClass
 	public static void shutdownCluster() {
-		try {
-			cluster.shutdown();
-			cluster = null;
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to stop test cluster: " + e.getMessage());
-		}
+		cluster.shutdown();
+		cluster = null;
 	}
 
 	@Test
 	public void testFaultyAccumulator() throws Exception {
 
-		ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
 		env.getConfig().disableSysoutLogging();
 
 		// Test Exception forwarding with faulty Accumulator implementation
@@ -93,11 +85,9 @@ public void testFaultyAccumulator() throws Exception {
 		try {
 			env.execute();
 			fail("Should have failed.");
-		} catch (ProgramInvocationException e) {
-			Assert.assertTrue("Exception should be passed:",
-					e.getCause() instanceof JobExecutionException);
+		} catch (JobExecutionException e) {
 			Assert.assertTrue("Root cause should be:",
-					e.getCause().getCause() instanceof CustomException);
+					e.getCause() instanceof CustomException);
 		}
 	}
 
@@ -105,7 +95,6 @@ public void testFaultyAccumulator() throws Exception {
 	@Test
 	public void testInvalidTypeAccumulator() throws Exception {
 
-		ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
 		env.getConfig().disableSysoutLogging();
 
 		// Test Exception forwarding with faulty Accumulator implementation
@@ -119,13 +108,11 @@ public void testInvalidTypeAccumulator() throws Exception {
 		try {
 			env.execute();
 			fail("Should have failed.");
-		} catch (ProgramInvocationException e) {
-			Assert.assertTrue("Exception should be passed:",
-					e.getCause() instanceof JobExecutionException);
+		} catch (JobExecutionException e) {
 			Assert.assertTrue("Root cause should be:",
-					e.getCause().getCause() instanceof Exception);
+					e.getCause() instanceof Exception);
 			Assert.assertTrue("Root cause should be:",
-					e.getCause().getCause().getCause() instanceof UnsupportedOperationException);
+					e.getCause().getCause() instanceof UnsupportedOperationException);
 		}
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
index e8ceebabf2202..d91c57f0f619c 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
@@ -36,13 +36,13 @@
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
 import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
 import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.test.util.SuccessException;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
@@ -81,6 +81,8 @@ public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLog
 
 	private static LocalFlinkMiniCluster cluster;
 
+	private static TestStreamEnvironment env;
+
 	@Rule
 	public TemporaryFolder tempFolder = new TemporaryFolder();
 
@@ -104,6 +106,8 @@ public static void startTestCluster() {
 
 		cluster = new LocalFlinkMiniCluster(config, false);
 		cluster.start();
+
+		env = new TestStreamEnvironment(cluster, PARALLELISM);
 	}
 
 	@AfterClass
@@ -153,9 +157,6 @@ public void testTumblingTimeWindow() {
 		FailingSource.reset();
 		
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-			
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
@@ -226,9 +227,6 @@ public void doTestTumblingTimeWindowWithKVState(int maxParallelism) {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setMaxParallelism(maxParallelism);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
@@ -296,9 +294,6 @@ public void testSlidingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setMaxParallelism(2 * PARALLELISM);
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
@@ -361,9 +356,6 @@ public void testPreAggregatedTumblingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
@@ -434,9 +426,6 @@ public void testPreAggregatedSlidingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java
index 3345b9ce3e7fa..a573be63deca2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java
@@ -30,13 +30,13 @@
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.runtime.state.CheckpointListener;
 import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
 import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
 import org.apache.flink.streaming.api.functions.windowing.RichAllWindowFunction;
 import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.test.util.SuccessException;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
@@ -65,6 +65,8 @@ public class EventTimeAllWindowCheckpointingITCase extends TestLogger {
 
 	private static LocalFlinkMiniCluster cluster;
 
+	private static TestStreamEnvironment env;
+
 
 	@BeforeClass
 	public static void startTestCluster() {
@@ -76,6 +78,8 @@ public static void startTestCluster() {
 		config.setString(ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT, "60 s");
 		cluster = new LocalFlinkMiniCluster(config, false);
 		cluster.start();
+
+		env = new TestStreamEnvironment(cluster, PARALLELISM);
 	}
 
 	@AfterClass
@@ -95,9 +99,6 @@ public void testTumblingTimeWindow() {
 		FailingSource.reset();
 		
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-			
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
@@ -159,9 +160,6 @@ public void testSlidingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
@@ -220,9 +218,6 @@ public void testPreAggregatedTumblingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
@@ -292,9 +287,6 @@ public void testPreAggregatedFoldingTumblingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
@@ -364,9 +356,6 @@ public void testPreAggregatedSlidingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 			env.enableCheckpointing(100);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamFaultToleranceTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamFaultToleranceTestBase.java
index 2839bc124b443..5f56def6be6ab 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamFaultToleranceTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamFaultToleranceTestBase.java
@@ -24,6 +24,7 @@
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.test.util.TestUtils;
 import org.apache.flink.util.TestLogger;
 
@@ -95,8 +96,7 @@ public static void stopCluster() {
 	@Test
 	public void runCheckpointedProgram() throws Exception {
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
+			TestStreamEnvironment env = new TestStreamEnvironment(cluster, PARALLELISM);
 			env.setParallelism(PARALLELISM);
 			env.enableCheckpointing(500);
 			env.getConfig().disableSysoutLogging();
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java
index 56d8c663fb369..7004f75041339 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java
@@ -30,12 +30,12 @@
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.runtime.state.CheckpointListener;
 import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
 import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
 import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.test.util.SuccessException;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
@@ -76,6 +76,8 @@ public WindowCheckpointingITCase(TimeCharacteristic timeCharacteristic) {
 
 	private static LocalFlinkMiniCluster cluster;
 
+	private static TestStreamEnvironment env;
+
 
 	@BeforeClass
 	public static void startTestCluster() {
@@ -86,6 +88,8 @@ public static void startTestCluster() {
 
 		cluster = new LocalFlinkMiniCluster(config, false);
 		cluster.start();
+
+		env = new TestStreamEnvironment(cluster, PARALLELISM);
 	}
 
 	@AfterClass
@@ -103,9 +107,6 @@ public void testTumblingProcessingTimeWindow() {
 		FailingSource.reset();
 		
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-			
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(timeCharacteristic);
 			env.getConfig().setAutoWatermarkInterval(10);
@@ -161,9 +162,6 @@ public void testSlidingProcessingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-			
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(timeCharacteristic);
 			env.getConfig().setAutoWatermarkInterval(10);
@@ -219,9 +217,6 @@ public void testAggregatingTumblingProcessingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(timeCharacteristic);
 			env.getConfig().setAutoWatermarkInterval(10);
@@ -267,9 +262,6 @@ public void testAggregatingSlidingProcessingTimeWindow() {
 		FailingSource.reset();
 
 		try {
-			StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARALLELISM);
 			env.setStreamTimeCharacteristic(timeCharacteristic);
 			env.getConfig().setAutoWatermarkInterval(10);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java
index f25a302e9f1ea..75eb1129e8937 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java
@@ -38,8 +38,10 @@
 import org.apache.flink.runtime.state.filesystem.FsStateBackendFactory;
 import org.apache.flink.runtime.testingUtils.TestingCluster;
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.WaitForAllVerticesToBeRunning;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.test.testdata.KMeansData;
 import org.apache.flink.test.util.SuccessException;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -54,13 +56,14 @@
 import scala.concurrent.duration.FiniteDuration;
 
 import java.io.File;
+import java.io.IOException;
+import java.net.URL;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
 
 public class ClassLoaderITCase extends TestLogger {
 
@@ -117,61 +120,69 @@ public static void tearDown() throws Exception {
 		}
 
 		FOLDER.delete();
+
+		TestStreamEnvironment.unsetAsContext();
+		TestEnvironment.unsetAsContext();
 	}
 
 	@Test
-	public void testJobsWithCustomClassLoader() {
+	public void testJobsWithCustomClassLoader() throws IOException, ProgramInvocationException {
 		try {
 			int port = testCluster.getLeaderRPCPort();
 
-			PackagedProgram inputSplitTestProg = new PackagedProgram(
-					new File(INPUT_SPLITS_PROG_JAR_FILE),
-					new String[] { INPUT_SPLITS_PROG_JAR_FILE,
-							"", // classpath
-							"localhost",
-							String.valueOf(port),
-							"4" // parallelism
-					});
+			PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE));
+
+			TestEnvironment.setAsContext(
+				testCluster,
+				parallelism,
+				Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)),
+				Collections.<URL>emptyList());
+
 			inputSplitTestProg.invokeInteractiveModeForExecution();
 
-			PackagedProgram streamingInputSplitTestProg = new PackagedProgram(
-					new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE),
-					new String[] { STREAMING_INPUT_SPLITS_PROG_JAR_FILE,
-							"localhost",
-							String.valueOf(port),
-							"4" // parallelism
-					});
+			PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE));
+
+			TestStreamEnvironment.setAsContext(
+				testCluster,
+				parallelism,
+				Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)),
+				Collections.<URL>emptyList());
+
 			streamingInputSplitTestProg.invokeInteractiveModeForExecution();
 
-			String classpath = new File(INPUT_SPLITS_PROG_JAR_FILE).toURI().toURL().toString();
-			PackagedProgram inputSplitTestProg2 = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE),
-					new String[] { "",
-							classpath, // classpath
-							"localhost",
-							String.valueOf(port),
-							"4" // parallelism
-					});
+			URL classpath = new File(INPUT_SPLITS_PROG_JAR_FILE).toURI().toURL();
+			PackagedProgram inputSplitTestProg2 = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE));
+
+			TestEnvironment.setAsContext(
+				testCluster,
+				parallelism,
+				Collections.<Path>emptyList(),
+				Collections.singleton(classpath));
+
 			inputSplitTestProg2.invokeInteractiveModeForExecution();
 
 			// regular streaming job
-			PackagedProgram streamingProg = new PackagedProgram(
-					new File(STREAMING_PROG_JAR_FILE),
-					new String[] {
-							STREAMING_PROG_JAR_FILE,
-							"localhost",
-							String.valueOf(port)
-					});
+			PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE));
+
+			TestStreamEnvironment.setAsContext(
+				testCluster,
+				parallelism,
+				Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)),
+				Collections.<URL>emptyList());
+
 			streamingProg.invokeInteractiveModeForExecution();
 
 			// checkpointed streaming job with custom classes for the checkpoint (FLINK-2543)
 			// the test also ensures that user specific exceptions are serializable between JobManager <--> JobClient.
 			try {
-				PackagedProgram streamingCheckpointedProg = new PackagedProgram(
-						new File(STREAMING_CHECKPOINTED_PROG_JAR_FILE),
-						new String[] {
-								STREAMING_CHECKPOINTED_PROG_JAR_FILE,
-								"localhost",
-								String.valueOf(port) });
+				PackagedProgram streamingCheckpointedProg = new PackagedProgram(new File(STREAMING_CHECKPOINTED_PROG_JAR_FILE));
+
+				TestStreamEnvironment.setAsContext(
+					testCluster,
+					parallelism,
+					Collections.singleton(new Path(STREAMING_CHECKPOINTED_PROG_JAR_FILE)),
+					Collections.<URL>emptyList());
+
 				streamingCheckpointedProg.invokeInteractiveModeForExecution();
 			} catch (Exception e) {
 				// we can not access the SuccessException here when executing the tests with maven, because its not available in the jar.
@@ -182,14 +193,18 @@ public void testJobsWithCustomClassLoader() {
 
 			PackagedProgram kMeansProg = new PackagedProgram(
 					new File(KMEANS_JAR_PATH),
-					new String[] { KMEANS_JAR_PATH,
-							"localhost",
-							String.valueOf(port),
-							"4", // parallelism
-							KMeansData.DATAPOINTS,
-							KMeansData.INITIAL_CENTERS,
-							"25"
+					new String[] {
+						KMeansData.DATAPOINTS,
+						KMeansData.INITIAL_CENTERS,
+						"25"
 					});
+
+			TestEnvironment.setAsContext(
+				testCluster,
+				parallelism,
+				Collections.singleton(new Path(KMEANS_JAR_PATH)),
+				Collections.<URL>emptyList());
+
 			kMeansProg.invokeInteractiveModeForExecution();
 
 			// test FLINK-3633
@@ -200,6 +215,12 @@ public void testJobsWithCustomClassLoader() {
 							String.valueOf(port),
 					});
 
+			TestEnvironment.setAsContext(
+				testCluster,
+				parallelism,
+				Collections.singleton(new Path(USERCODETYPE_JAR_PATH)),
+				Collections.<URL>emptyList());
+
 			userCodeTypeProg.invokeInteractiveModeForExecution();
 
 			File checkpointDir = FOLDER.newFolder();
@@ -208,18 +229,21 @@ public void testJobsWithCustomClassLoader() {
 			final PackagedProgram program = new PackagedProgram(
 					new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH),
 					new String[] {
-							CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH,
-							"localhost",
-							String.valueOf(port),
 							checkpointDir.toURI().toString(),
 							outputDir.toURI().toString()
 					});
 
+			TestStreamEnvironment.setAsContext(
+				testCluster,
+				parallelism,
+				Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
+				Collections.<URL>emptyList());
+
 			program.invokeInteractiveModeForExecution();
 
 		} catch (Exception e) {
 			if (!(e.getCause().getCause() instanceof SuccessException)) {
-				fail(e.getMessage());
+				throw e;
 			}
 		}
 	}
@@ -231,23 +255,25 @@ public void testJobsWithCustomClassLoader() {
 	public void testDisposeSavepointWithCustomKvState() throws Exception {
 		Deadline deadline = new FiniteDuration(100, TimeUnit.SECONDS).fromNow();
 
-		int port = testCluster.getLeaderRPCPort();
-
 		File checkpointDir = FOLDER.newFolder();
 		File outputDir = FOLDER.newFolder();
 
 		final PackagedProgram program = new PackagedProgram(
 				new File(CUSTOM_KV_STATE_JAR_PATH),
 				new String[] {
-						CUSTOM_KV_STATE_JAR_PATH,
-						"localhost",
-						String.valueOf(port),
 						String.valueOf(parallelism),
 						checkpointDir.toURI().toString(),
 						"5000",
 						outputDir.toURI().toString()
 				});
 
+		TestStreamEnvironment.setAsContext(
+			testCluster,
+			parallelism,
+			Collections.singleton(new Path(CUSTOM_KV_STATE_JAR_PATH)),
+			Collections.<URL>emptyList()
+		);
+
 		// Execute detached
 		Thread invokeThread = new Thread(new Runnable() {
 			@Override
@@ -283,7 +309,7 @@ public void run() {
 
 			// Retry if job is not available yet
 			if (jobId == null) {
-				Thread.sleep(100);
+				Thread.sleep(100L);
 			}
 		}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java
index 52a3ba8ca7b08..795ae41dccd0f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java
@@ -41,14 +41,11 @@ public class CheckpointedStreamingProgram {
 	private static final int CHECKPOINT_INTERVALL = 100;
 	
 	public static void main(String[] args) throws Exception {
-		final String jarFile = args[0];
-		final String host = args[1];
-		final int port = Integer.parseInt(args[2]);
-		
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(host, port, jarFile);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
 		env.getConfig().disableSysoutLogging();
 		env.enableCheckpointing(CHECKPOINT_INTERVALL);
-		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 10000));
+		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 100L));
 		env.disableOperatorChaining();
 		
 		DataStream<String> text = env.addSource(new SimpleStringGenerator());
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java
index d3baa7dae67e4..a24a3a88571a7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java
@@ -49,14 +49,11 @@
 public class CheckpointingCustomKvStateProgram {
 
 	public static void main(String[] args) throws Exception {
-		final String jarFile = args[0];
-		final String host = args[1];
-		final int port = Integer.parseInt(args[2]);
-		final String checkpointPath = args[3];
-		final String outputPath = args[4];
+		final String checkpointPath = args[0];
+		final String outputPath = args[1];
 		final int parallelism = 1;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(host, port, jarFile);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setParallelism(parallelism);
 		env.getConfig().disableSysoutLogging();
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomInputSplitProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomInputSplitProgram.java
index b18e8e84d8cb9..2caa7cfb45771 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomInputSplitProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomInputSplitProgram.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.test.classloading.jar;
 
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -30,7 +29,6 @@
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.RemoteEnvironment;
 import org.apache.flink.api.java.io.DiscardingOutputFormat;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
@@ -42,14 +40,8 @@
 public class CustomInputSplitProgram {
 	
 	public static void main(String[] args) throws Exception {
-		final String[] jarFile = (args[0].equals(""))? null : new String[] { args[0] };
-		final URL[] classpath = (args[1].equals(""))? null : new URL[] { new URL(args[1]) };
-		final String host = args[2];
-		final int port = Integer.parseInt(args[3]);
-		final int parallelism = Integer.parseInt(args[4]);
-
-		RemoteEnvironment env = new RemoteEnvironment(host, port, null, jarFile, classpath);
-		env.setParallelism(parallelism);
+
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 
 		DataSet<Integer> data = env.createInput(new CustomInputFormat());
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomKvStateProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomKvStateProgram.java
index 8de479769a7f3..cbd553ce6816b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomKvStateProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CustomKvStateProgram.java
@@ -43,15 +43,12 @@
 public class CustomKvStateProgram {
 
 	public static void main(String[] args) throws Exception {
-		final String jarFile = args[0];
-		final String host = args[1];
-		final int port = Integer.parseInt(args[2]);
-		final int parallelism = Integer.parseInt(args[3]);
-		final String checkpointPath = args[4];
-		final int checkpointingInterval = Integer.parseInt(args[5]);
-		final String outputPath = args[6];
-
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(host, port, jarFile);
+		final int parallelism = Integer.parseInt(args[0]);
+		final String checkpointPath = args[1];
+		final int checkpointingInterval = Integer.parseInt(args[2]);
+		final String outputPath = args[3];
+
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(parallelism);
 		env.getConfig().disableSysoutLogging();
 		env.enableCheckpointing(checkpointingInterval);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/KMeansForTest.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/KMeansForTest.java
index 785464a2035d5..b8e6c85e453e2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/KMeansForTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/KMeansForTest.java
@@ -52,22 +52,15 @@ public class KMeansForTest {
 	// *************************************************************************
 
 	public static void main(String[] args) throws Exception {
-		if (args.length < 7) {
+		if (args.length < 3) {
 			throw new IllegalArgumentException("Missing parameters");
 		}
 
-		final String jarFile = args[0];
-		final String host = args[1];
-		final int port = Integer.parseInt(args[2]);
+		final String pointsData = args[0];
+		final String centersData = args[1];
+		final int numIterations = Integer.parseInt(args[2]);
 
-		final int parallelism = Integer.parseInt(args[3]);
-
-		final String pointsData = args[4];
-		final String centersData = args[5];
-		final int numIterations = Integer.parseInt(args[6]);
-
-		ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(host, port, jarFile);
-		env.setParallelism(parallelism);
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 
 		// get input data
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/LegacyCheckpointedStreamingProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/LegacyCheckpointedStreamingProgram.java
index 60253fa3e5589..210973ffb6d03 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/LegacyCheckpointedStreamingProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/LegacyCheckpointedStreamingProgram.java
@@ -38,11 +38,7 @@ public class LegacyCheckpointedStreamingProgram {
 	private static final int CHECKPOINT_INTERVALL = 100;
 
 	public static void main(String[] args) throws Exception {
-		final String jarFile = args[0];
-		final String host = args[1];
-		final int port = Integer.parseInt(args[2]);
-
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(host, port, jarFile);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		env.enableCheckpointing(CHECKPOINT_INTERVALL);
 		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 10000));
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingCustomInputSplitProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingCustomInputSplitProgram.java
index 0f0ee0cbfb4d9..e7bd5225b6aef 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingCustomInputSplitProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingCustomInputSplitProgram.java
@@ -42,18 +42,13 @@
 public class StreamingCustomInputSplitProgram {
 	
 	public static void main(String[] args) throws Exception {
-		final String jarFile = args[0];
-		final String host = args[1];
-		final int port = Integer.parseInt(args[2]);
-		final int parallelism = Integer.parseInt(args[3]);
-
-		Configuration config = new Configuration();
+				Configuration config = new Configuration();
 
 		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "5 s");
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(host, port, config, jarFile);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
 		env.getConfig().disableSysoutLogging();
-		env.setParallelism(parallelism);
 
 		DataStream<Integer> data = env.createInput(new CustomInputFormat());
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java
index a19d8f276ce05..0fdc7447cf51c 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java
@@ -31,12 +31,7 @@
 public class StreamingProgram {
 	
 	public static void main(String[] args) throws Exception {
-		
-		final String jarFile = args[0];
-		final String host = args[1];
-		final int port = Integer.parseInt(args[2]);
-		
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(host, port, jarFile);
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 		
 		DataStream<String> text = env.fromElements(WordCountData.TEXT).rebalance();
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/UserCodeType.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/UserCodeType.java
index a073cba154a8f..f12fd5fc89a38 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/UserCodeType.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/UserCodeType.java
@@ -43,11 +43,7 @@ public String toString() {
 	}
 
 	public static void main(String[] args) throws Exception {
-		String jarFile = args[0];
-		String host = args[1];
-		int port = Integer.parseInt(args[2]);
-
-		ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(host, port, jarFile);
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.getConfig().disableSysoutLogging();
 
 		DataSet<Integer> input = env.fromElements(1,2,3,4,5);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/clients/examples/JobRetrievalITCase.java b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/JobRetrievalITCase.java
index a74ed34e02286..61595f29c6dd6 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/clients/examples/JobRetrievalITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/JobRetrievalITCase.java
@@ -53,7 +53,8 @@ public class JobRetrievalITCase extends TestLogger {
 
 	@BeforeClass
 	public static void before() {
-		cluster = new TestingCluster(new Configuration(), false);
+		Configuration configuration = new Configuration();
+		cluster = new TestingCluster(configuration, false);
 		cluster.start();
 	}
 
@@ -72,7 +73,7 @@ public void testJobRetrieval() throws Exception {
 
 		final JobGraph jobGraph = new JobGraph(jobID, "testjob", imalock);
 
-		final ClusterClient client = new StandaloneClusterClient(cluster.configuration());
+		final ClusterClient client = new StandaloneClusterClient(cluster.configuration(), cluster.highAvailabilityServices());
 
 		// acquire the lock to make sure that the job cannot complete until the job client
 		// has been attached in resumingThread
@@ -122,7 +123,7 @@ public void testNonExistingJobRetrieval() throws Exception {
 		try {
 			client.retrieveJob(jobID);
 			fail();
-		} catch (JobRetrievalException e) {
+		} catch (JobRetrievalException ignored) {
 			// this is what we want
 		}
 	}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
index 133ebd0920b6d..da92c05ab02a3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
@@ -66,7 +66,7 @@ public static void teardown() throws Exception {
 
 	@Before
 	public void prepare() {
-		TestEnvironment clusterEnv = new TestEnvironment(cluster, 1);
+		TestEnvironment clusterEnv = new TestEnvironment(cluster, 1, false);
 		clusterEnv.setAsContext();
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/RemoteEnvironmentITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/RemoteEnvironmentITCase.java
index d9d1b424f05bc..00915710aeba7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/RemoteEnvironmentITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/RemoteEnvironmentITCase.java
@@ -26,7 +26,7 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.GenericInputSplit;
-import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.runtime.minicluster.StandaloneMiniCluster;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
@@ -39,47 +39,35 @@
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 
 @SuppressWarnings("serial")
 public class RemoteEnvironmentITCase extends TestLogger {
 
 	private static final int TM_SLOTS = 4;
 
-	private static final int NUM_TM = 1;
-
 	private static final int USER_DOP = 2;
 
 	private static final String INVALID_STARTUP_TIMEOUT = "0.001 ms";
 
 	private static final String VALID_STARTUP_TIMEOUT = "100 s";
 
-	private static LocalFlinkMiniCluster cluster;
+	private static Configuration configuration;
+
+	private static StandaloneMiniCluster cluster;
+
 
 	@BeforeClass
-	public static void setupCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TM);
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, TM_SLOTS);
-			cluster = new LocalFlinkMiniCluster(config, false);
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Error starting test cluster: " + e.getMessage());
-		}
+	public static void setupCluster() throws Exception {
+		configuration = new Configuration();
+
+		configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, TM_SLOTS);
+
+		cluster = new StandaloneMiniCluster(configuration);
 	}
 
 	@AfterClass
-	public static void tearDownCluster() {
-		try {
-			cluster.stop();
-		}
-		catch (Throwable t) {
-			t.printStackTrace();
-			fail("ClusterClient shutdown caused an exception: " + t.getMessage());
-		}
+	public static void tearDownCluster() throws Exception {
+		cluster.close();
 	}
 
 	/**
@@ -91,8 +79,8 @@ public void testInvalidAkkaConfiguration() throws Throwable {
 		config.setString(ConfigConstants.AKKA_STARTUP_TIMEOUT, INVALID_STARTUP_TIMEOUT);
 
 		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				cluster.hostname(),
-				cluster.getLeaderRPCPort(),
+				cluster.getHostname(),
+				cluster.getPort(),
 				config
 		);
 		env.getConfig().disableSysoutLogging();
@@ -116,8 +104,8 @@ public void testUserSpecificParallelism() throws Exception {
 		config.setString(ConfigConstants.AKKA_STARTUP_TIMEOUT, VALID_STARTUP_TIMEOUT);
 
 		final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-				cluster.hostname(),
-				cluster.getLeaderRPCPort(),
+				cluster.getHostname(),
+				cluster.getPort(),
 				config
 		);
 		env.setParallelism(USER_DOP);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java
index 2910f06b5218e..eea2509d5f451 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java
@@ -30,6 +30,7 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.GenericInputSplit;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
@@ -52,6 +53,8 @@ public class AutoParallelismITCase extends TestLogger {
 
 	private static LocalFlinkMiniCluster cluster;
 
+	private static TestEnvironment env;
+
 	@BeforeClass
 	public static void setupCluster() {
 		Configuration config = new Configuration();
@@ -60,6 +63,8 @@ public static void setupCluster() {
 		cluster = new LocalFlinkMiniCluster(config, false);
 
 		cluster.start();
+
+		env = new TestEnvironment(cluster, NUM_TM * SLOTS_PER_TM, false);
 	}
 
 	@AfterClass
@@ -78,9 +83,6 @@ public static void teardownCluster() {
 	@Test
 	public void testProgramWithAutoParallelism() {
 		try {
-			ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(ExecutionConfig.PARALLELISM_AUTO_MAX);
 			env.getConfig().disableSysoutLogging();
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java
index b4a7f9945e4e2..76480baf63763 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java
@@ -19,7 +19,6 @@
 package org.apache.flink.test.misc;
 
 import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.io.DiscardingOutputFormat;
 import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.configuration.ConfigConstants;
@@ -27,7 +26,9 @@
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.types.Value;
 
 import org.apache.flink.util.TestLogger;
@@ -47,39 +48,28 @@ public class CustomSerializationITCase extends TestLogger {
 	
 	private static LocalFlinkMiniCluster cluster;
 
+	private static TestEnvironment env;
+
 	@BeforeClass
 	public static void startCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARLLELISM);
-			config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 30L);
-			cluster = new LocalFlinkMiniCluster(config, false);
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to start test cluster: " + e.getMessage());
-		}
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARLLELISM);
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 30L);
+		cluster = new LocalFlinkMiniCluster(config, false);
+		cluster.start();
+
+		env = new TestEnvironment(cluster, PARLLELISM, false);
 	}
 
 	@AfterClass
 	public static void shutdownCluster() {
-		try {
-			cluster.shutdown();
-			cluster = null;
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to stop test cluster: " + e.getMessage());
-		}
+		cluster.shutdown();
+		cluster = null;
 	}
 	
 	@Test
 	public void testIncorrectSerializer1() {
 		try {
-			ExecutionEnvironment env =
-					ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-			
 			env.setParallelism(PARLLELISM);
 			env.getConfig().disableSysoutLogging();
 			
@@ -96,8 +86,8 @@ public ConsumesTooMuch map(Long value) throws Exception {
 			
 			env.execute();
 		}
-		catch (ProgramInvocationException e) {
-			Throwable rootCause = e.getCause().getCause();
+		catch (JobExecutionException e) {
+			Throwable rootCause = e.getCause();
 			assertTrue(rootCause instanceof IOException);
 			assertTrue(rootCause.getMessage().contains("broken serialization"));
 		}
@@ -110,9 +100,6 @@ public ConsumesTooMuch map(Long value) throws Exception {
 	@Test
 	public void testIncorrectSerializer2() {
 		try {
-			ExecutionEnvironment env =
-					ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARLLELISM);
 			env.getConfig().disableSysoutLogging();
 
@@ -129,8 +116,8 @@ public ConsumesTooMuchSpanning map(Long value) throws Exception {
 
 			env.execute();
 		}
-		catch (ProgramInvocationException e) {
-			Throwable rootCause = e.getCause().getCause();
+		catch (JobExecutionException e) {
+			Throwable rootCause = e.getCause();
 			assertTrue(rootCause instanceof IOException);
 			assertTrue(rootCause.getMessage().contains("broken serialization"));
 		}
@@ -143,9 +130,6 @@ public ConsumesTooMuchSpanning map(Long value) throws Exception {
 	@Test
 	public void testIncorrectSerializer3() {
 		try {
-			ExecutionEnvironment env =
-					ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARLLELISM);
 			env.getConfig().disableSysoutLogging();
 
@@ -162,8 +146,8 @@ public ConsumesTooLittle map(Long value) throws Exception {
 
 			env.execute();
 		}
-		catch (ProgramInvocationException e) {
-			Throwable rootCause = e.getCause().getCause();
+		catch (JobExecutionException e) {
+			Throwable rootCause = e.getCause();
 			assertTrue(rootCause instanceof IOException);
 			assertTrue(rootCause.getMessage().contains("broken serialization"));
 		}
@@ -176,9 +160,6 @@ public ConsumesTooLittle map(Long value) throws Exception {
 	@Test
 	public void testIncorrectSerializer4() {
 		try {
-			ExecutionEnvironment env =
-					ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(PARLLELISM);
 			env.getConfig().disableSysoutLogging();
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/MiscellaneousIssuesITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/MiscellaneousIssuesITCase.java
index f885321e972bf..7dab0f1ef1e17 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/misc/MiscellaneousIssuesITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/misc/MiscellaneousIssuesITCase.java
@@ -23,15 +23,15 @@
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.common.functions.RichFlatMapFunction;
 import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.Collector;
 
 import org.apache.flink.util.TestLogger;
@@ -54,43 +54,34 @@
 @SuppressWarnings("serial")
 public class MiscellaneousIssuesITCase extends TestLogger {
 
+	private static final int PARALLELISM = 6;
+
 	private static LocalFlinkMiniCluster cluster;
+
+	private static TestEnvironment env;
 	
 	@BeforeClass
 	public static void startCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 3);
-			config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L);
-			cluster = new LocalFlinkMiniCluster(config, false);
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 3);
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L);
+		cluster = new LocalFlinkMiniCluster(config, false);
 
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to start test cluster: " + e.getMessage());
-		}
+		cluster.start();
+
+		env = new TestEnvironment(cluster, PARALLELISM, false);
 	}
 	
 	@AfterClass
 	public static void shutdownCluster() {
-		try {
-			cluster.shutdown();
-			cluster = null;
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to stop test cluster: " + e.getMessage());
-		}
+		cluster.shutdown();
+		cluster = null;
 	}
 	
 	@Test
 	public void testNullValues() {
 		try {
-			ExecutionEnvironment env =
-					ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(1);
 			env.getConfig().disableSysoutLogging();
 
@@ -107,10 +98,9 @@ public String map(String value) throws Exception {
 				env.execute();
 				fail("this should fail due to null values.");
 			}
-			catch (ProgramInvocationException e) {
+			catch (JobExecutionException e) {
 				assertNotNull(e.getCause());
-				assertNotNull(e.getCause().getCause());
-				assertTrue(e.getCause().getCause() instanceof NullPointerException);
+				assertTrue(e.getCause() instanceof NullPointerException);
 			}
 		}
 		catch (Exception e) {
@@ -122,9 +112,6 @@ public String map(String value) throws Exception {
 	@Test
 	public void testDisjointDataflows() {
 		try {
-			ExecutionEnvironment env =
-					ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(5);
 			env.getConfig().disableSysoutLogging();
 
@@ -145,9 +132,6 @@ public void testAccumulatorsAfterNoOp() {
 		final String ACC_NAME = "test_accumulator";
 		
 		try {
-			ExecutionEnvironment env =
-					ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
 			env.setParallelism(6);
 			env.getConfig().disableSysoutLogging();
 			
diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
index 4f24452305dc6..a5103cc22b802 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
@@ -25,7 +25,6 @@
 import org.apache.flink.api.java.operators.DeltaIteration;
 import org.apache.flink.api.java.operators.IterativeDataSet;
 import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.TaskManagerOptions;
@@ -34,14 +33,17 @@
 import org.apache.flink.examples.java.graph.ConnectedComponents;
 import org.apache.flink.examples.java.graph.util.ConnectedComponentsData;
 
+import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
 
 public class SuccessAfterNetworkBuffersFailureITCase extends TestLogger {
-	
+
+	private static final int PARALLELISM = 16;
 	@Test
 	public void testSuccessfulProgramAfterFailure() {
 		LocalFlinkMiniCluster cluster = null;
@@ -56,9 +58,11 @@ public void testSuccessfulProgramAfterFailure() {
 			cluster = new LocalFlinkMiniCluster(config, false);
 
 			cluster.start();
+
+			TestEnvironment env = new TestEnvironment(cluster, PARALLELISM, false);
 			
 			try {
-				runConnectedComponents(cluster.getLeaderRPCPort());
+				runConnectedComponents(env);
 			}
 			catch (Exception e) {
 				e.printStackTrace();
@@ -66,15 +70,15 @@ public void testSuccessfulProgramAfterFailure() {
 			}
 	
 			try {
-				runKMeans(cluster.getLeaderRPCPort());
+				runKMeans(env);
 				fail("This program execution should have failed.");
 			}
-			catch (ProgramInvocationException e) {
-				assertTrue(e.getCause().getCause().getMessage().contains("Insufficient number of network buffers"));
+			catch (JobExecutionException e) {
+				assertTrue(e.getCause().getMessage().contains("Insufficient number of network buffers"));
 			}
 	
 			try {
-				runConnectedComponents(cluster.getLeaderRPCPort());
+				runConnectedComponents(env);
 			}
 			catch (Exception e) {
 				e.printStackTrace();
@@ -92,10 +96,9 @@ public void testSuccessfulProgramAfterFailure() {
 		}
 	}
 	
-	private static void runConnectedComponents(int jmPort) throws Exception {
+	private static void runConnectedComponents(ExecutionEnvironment env) throws Exception {
 		
-		ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", jmPort);
-		env.setParallelism(16);
+		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
 		// read vertex and edge data
@@ -134,10 +137,9 @@ private static void runConnectedComponents(int jmPort) throws Exception {
 		env.execute();
 	}
 
-	private static void runKMeans(int jmPort) throws Exception {
+	private static void runKMeans(ExecutionEnvironment env) throws Exception {
 
-		ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", jmPort);
-		env.setParallelism(16);
+		env.setParallelism(PARALLELISM);
 		env.getConfig().disableSysoutLogging();
 
 		// get input data
diff --git a/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java b/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java
index 3c8eb482a4656..6c8e758508fc9 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java
@@ -170,7 +170,9 @@ public void testQueryableState() throws Exception {
 		final Deadline deadline = TEST_TIMEOUT.fromNow();
 		final int numKeys = 256;
 
-		final QueryableStateClient client = new QueryableStateClient(cluster.configuration());
+		final QueryableStateClient client = new QueryableStateClient(
+			cluster.configuration(),
+			cluster.highAvailabilityServices());
 
 		JobID jobId = null;
 
@@ -396,7 +398,9 @@ public void testValueState() throws Exception {
 
 		final int numElements = 1024;
 
-		final QueryableStateClient client = new QueryableStateClient(cluster.configuration());
+		final QueryableStateClient client = new QueryableStateClient(
+			cluster.configuration(),
+			cluster.highAvailabilityServices());
 
 		JobID jobId = null;
 		try {
@@ -461,7 +465,9 @@ public void testQueryNonStartedJobState() throws Exception {
 
 		final int numElements = 1024;
 
-		final QueryableStateClient client = new QueryableStateClient(cluster.configuration());
+		final QueryableStateClient client = new QueryableStateClient(
+			cluster.configuration(),
+			cluster.highAvailabilityServices());
 
 		JobID jobId = null;
 		try {
@@ -586,7 +592,9 @@ public void testValueStateDefault() throws
 
 		final int numElements = 1024;
 
-		final QueryableStateClient client = new QueryableStateClient(cluster.configuration());
+		final QueryableStateClient client = new QueryableStateClient(
+			cluster.configuration(),
+			cluster.highAvailabilityServices());
 
 		JobID jobId = null;
 		try {
@@ -679,7 +687,9 @@ public void testValueStateShortcut() throws Exception {
 
 		final int numElements = 1024;
 
-		final QueryableStateClient client = new QueryableStateClient(cluster.configuration());
+		final QueryableStateClient client = new QueryableStateClient(
+			cluster.configuration(),
+			cluster.highAvailabilityServices());
 
 		JobID jobId = null;
 		try {
@@ -743,7 +753,9 @@ public void testFoldingState() throws Exception {
 
 		final int numElements = 1024;
 
-		final QueryableStateClient client = new QueryableStateClient(cluster.configuration());
+		final QueryableStateClient client = new QueryableStateClient(
+			cluster.configuration(),
+			cluster.highAvailabilityServices());
 
 		JobID jobId = null;
 		try {
@@ -844,7 +856,9 @@ public void testReducingState() throws Exception {
 
 		final int numElements = 1024;
 
-		final QueryableStateClient client = new QueryableStateClient(cluster.configuration());
+		final QueryableStateClient client = new QueryableStateClient(
+			cluster.configuration(),
+			cluster.highAvailabilityServices());
 
 		JobID jobId = null;
 		try {
@@ -1008,6 +1022,7 @@ public void cancel() {
 	 */
 	private static class TestKeyRangeSource extends RichParallelSourceFunction<Tuple2<Integer, Long>>
 			implements CheckpointListener {
+		private static final long serialVersionUID = -5744725196953582710L;
 
 		private final static AtomicLong LATEST_CHECKPOINT_ID = new AtomicLong();
 		private final int numKeys;
@@ -1055,6 +1070,8 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception {
 	}
 
 	private static class SumFold implements FoldFunction<Tuple2<Integer, Long>, String> {
+		private static final long serialVersionUID = -6249227626701264599L;
+
 		@Override
 		public String fold(String accumulator, Tuple2<Integer, Long> value) throws Exception {
 			long acc = Long.valueOf(accumulator);
@@ -1064,6 +1081,8 @@ public String fold(String accumulator, Tuple2<Integer, Long> value) throws Excep
 	}
 
 	private static class SumReduce implements ReduceFunction<Tuple2<Integer, Long>> {
+		private static final long serialVersionUID = -8651235077342052336L;
+
 		@Override
 		public Tuple2<Integer, Long> reduce(Tuple2<Integer, Long> value1, Tuple2<Integer, Long> value2) throws Exception {
 			value1.f1 += value2.f1;
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
index 27d1aa1fc41bb..c7c07ce1472ef 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
@@ -26,9 +26,12 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.messages.JobManagerMessages;
@@ -86,13 +89,14 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test
 	protected static final int PARALLELISM = 4;
 
 	@Test
-	public void testTaskManagerProcessFailure() {
+	public void testTaskManagerProcessFailure() throws Exception {
 
 		final StringWriter processOutput1 = new StringWriter();
 		final StringWriter processOutput2 = new StringWriter();
 		final StringWriter processOutput3 = new StringWriter();
 
 		ActorSystem jmActorSystem = null;
+		HighAvailabilityServices highAvailabilityServices = null;
 		Process taskManagerProcess1 = null;
 		Process taskManagerProcess2 = null;
 		Process taskManagerProcess3 = null;
@@ -128,6 +132,13 @@ public void testTaskManagerProcessFailure() {
 			jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 9);
 			jmConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "10 s");
 			jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
+			jmConfig.setString(JobManagerOptions.ADDRESS, localAddress._1());
+			jmConfig.setInteger(JobManagerOptions.PORT, jobManagerPort);
+
+			highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				jmConfig,
+				TestingUtils.defaultExecutor(),
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
 			jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<>(localAddress));
 			ActorRef jmActor = JobManager.startJobManagerActors(
@@ -135,6 +146,7 @@ public void testTaskManagerProcessFailure() {
 				jmActorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				JobManager.class,
 				MemoryArchivist.class)._1();
 
@@ -263,6 +275,10 @@ public void run() {
 					// we can ignore this
 				}
 			}
+
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
+			}
 		}
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
index bba218faa9a2d..6d53b9f6d575a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
@@ -27,6 +27,8 @@
 import org.apache.flink.configuration.HighAvailabilityOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.akka.ListeningBehaviour;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
@@ -34,13 +36,13 @@
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.state.filesystem.FsStateBackendFactory;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerActorTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerProcess;
 import org.apache.flink.runtime.testutils.TaskManagerProcess;
 import org.apache.flink.runtime.testutils.TestJvmProcess;
 import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
-import org.apache.flink.runtime.util.ZooKeeperUtils;
 import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
 import org.apache.flink.runtime.state.CheckpointListener;
 import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
@@ -169,6 +171,11 @@ public void testChaosMonkey() throws Exception {
 		// Task manager
 		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numberOfSlotsPerTaskManager);
 
+		final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			TestingUtils.defaultExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
+
 		ActorSystem testActorSystem = null;
 		LeaderRetrievalService leaderRetrievalService = null;
 		List<JobManagerProcess> jobManagerProcesses = new ArrayList<>();
@@ -187,7 +194,7 @@ public void testChaosMonkey() throws Exception {
 			testActorSystem = AkkaUtils.createDefaultActorSystem();
 
 			// Leader listener
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
+			leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 			TestingListener leaderListener = new TestingListener();
 			leaderRetrievalService.start(leaderListener);
 
@@ -358,6 +365,8 @@ else if (jobStatus == JobStatus.FINISHED) {
 			if (testActorSystem != null) {
 				testActorSystem.shutdown();
 			}
+
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java
index d80c8266bb870..0b498142d9e24 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java
@@ -30,6 +30,7 @@
 import org.apache.flink.streaming.api.functions.sink.SinkFunction;
 import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
 
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
@@ -45,18 +46,19 @@ public class FastFailuresITCase extends TestLogger {
 	
 	@Test
 	public void testThis() {
+		final int parallelism = 4;
+
 		Configuration config = new Configuration();
 		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
 		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
 		
 		LocalFlinkMiniCluster cluster = new LocalFlinkMiniCluster(config, false);
 		cluster.start();
-		
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
 
+		TestStreamEnvironment env = new TestStreamEnvironment(cluster, parallelism);
+		
 		env.getConfig().disableSysoutLogging();
-		env.setParallelism(4);
+		env.setParallelism(parallelism);
 		env.enableCheckpointing(1000);
 		env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(210, 0));
 		
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
index f910e498d5e18..6c70b87e715ef 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
@@ -26,6 +26,8 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.akka.ListeningBehaviour;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -34,12 +36,12 @@
 import org.apache.flink.runtime.leaderelection.TestingListener;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerActorTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerProcess;
 import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
-import org.apache.flink.runtime.util.ZooKeeperUtils;
 import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
 import org.apache.flink.runtime.state.CheckpointListener;
 import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
@@ -170,6 +172,10 @@ public void testCheckpointedStreamingSumProgram() throws Exception {
 		final JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
 		LeaderRetrievalService leaderRetrievalService = null;
 		ActorSystem taskManagerSystem = null;
+		final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			TestingUtils.defaultExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
 		try {
 			final Deadline deadline = TestTimeOut.fromNow();
@@ -187,16 +193,21 @@ public void testCheckpointedStreamingSumProgram() throws Exception {
 
 			// Leader listener
 			TestingListener leaderListener = new TestingListener();
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
+			leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 			leaderRetrievalService.start(leaderListener);
 
 			// The task manager
 			taskManagerSystem = AkkaUtils.createActorSystem(
 					config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
 			TaskManager.startTaskManagerComponentsAndActor(
-					config, ResourceID.generate(), taskManagerSystem, "localhost",
-					Option.<String>empty(), Option.<LeaderRetrievalService>empty(),
-					false, TaskManager.class);
+				config,
+				ResourceID.generate(),
+				taskManagerSystem,
+				highAvailabilityServices,
+				"localhost",
+				Option.<String>empty(),
+				false,
+				TaskManager.class);
 
 			{
 				// Initial submission
@@ -298,6 +309,8 @@ public void testCheckpointedStreamingSumProgram() throws Exception {
 			if (testSystem != null) {
 				testSystem.shutdown();
 			}
+
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
@@ -323,6 +336,10 @@ public void testCheckpointRecoveryFailure() throws Exception {
 		LeaderRetrievalService leaderRetrievalService = null;
 		ActorSystem taskManagerSystem = null;
 		ActorSystem testActorSystem = null;
+		final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			TestingUtils.defaultExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
 		try {
 			// Test actor system
@@ -338,16 +355,21 @@ public void testCheckpointRecoveryFailure() throws Exception {
 
 			// Leader listener
 			TestingListener leaderListener = new TestingListener();
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
+			leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 			leaderRetrievalService.start(leaderListener);
 
 			// The task manager
 			taskManagerSystem = AkkaUtils.createActorSystem(
 					config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
 			TaskManager.startTaskManagerComponentsAndActor(
-					config, ResourceID.generate(), taskManagerSystem, "localhost",
-					Option.<String>empty(), Option.<LeaderRetrievalService>empty(),
-					false, TaskManager.class);
+				config,
+				ResourceID.generate(),
+				taskManagerSystem,
+				highAvailabilityServices,
+				"localhost",
+				Option.<String>empty(),
+				false,
+				TaskManager.class);
 
 			// Get the leader
 			leaderListener.waitForNewLeader(testDeadline.timeLeft().toMillis());
@@ -453,6 +475,8 @@ public void testCheckpointRecoveryFailure() throws Exception {
 			if (testActorSystem != null) {
 				testActorSystem.shutdown();
 			}
+
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
index e4d0f65f99e8b..052195a0e4a65 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
@@ -32,6 +32,8 @@
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.akka.ListeningBehaviour;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -45,12 +47,12 @@
 import org.apache.flink.runtime.messages.JobManagerMessages.SubmitJob;
 import org.apache.flink.runtime.taskmanager.TaskManager;
 import org.apache.flink.runtime.testingUtils.TestingCluster;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerActorTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerProcess;
 import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
-import org.apache.flink.runtime.util.ZooKeeperUtils;
 import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
 import org.apache.flink.util.TestLogger;
 import org.apache.zookeeper.data.Stat;
@@ -183,6 +185,11 @@ public void testClientNonDetachedListeningBehaviour() throws Exception {
 
 		ActorSystem taskManagerSystem = null;
 
+		final HighAvailabilityServices highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+			config,
+			TestingUtils.defaultExecutor(),
+			HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
+
 		try {
 			final Deadline deadline = TestTimeOut.fromNow();
 
@@ -199,15 +206,20 @@ public void testClientNonDetachedListeningBehaviour() throws Exception {
 
 			// Leader listener
 			TestingListener leaderListener = new TestingListener();
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
+			leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 			leaderRetrievalService.start(leaderListener);
 
 			// The task manager
 			taskManagerSystem = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
 			TaskManager.startTaskManagerComponentsAndActor(
-					config, ResourceID.generate(), taskManagerSystem, "localhost",
-					Option.<String>empty(), Option.<LeaderRetrievalService>empty(),
-					false, TaskManager.class);
+				config,
+				ResourceID.generate(),
+				taskManagerSystem,
+				highAvailabilityServices,
+				"localhost",
+				Option.<String>empty(),
+				false,
+				TaskManager.class);
 
 			// Client test actor
 			TestActorRef<RecordingTestClient> clientRef = TestActorRef.create(
@@ -327,6 +339,8 @@ public void testClientNonDetachedListeningBehaviour() throws Exception {
 			if (testSystem != null) {
 				testSystem.shutdown();
 			}
+
+			highAvailabilityServices.closeAndCleanupAllData();
 		}
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java
index b6a1bd4ee63b2..5f9d1784ae0a1 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java
@@ -34,16 +34,18 @@
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.leaderelection.TestingListener;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerActorTestUtils;
 import org.apache.flink.runtime.testutils.JobManagerProcess;
 import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
-import org.apache.flink.runtime.util.ZooKeeperUtils;
 import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
@@ -238,7 +240,8 @@ public void testJobManagerProcessFailure() throws Exception {
 		// Task managers
 		final ActorSystem[] tmActorSystem = new ActorSystem[numberOfTaskManagers];
 
-		// Leader election service
+		HighAvailabilityServices highAvailabilityServices = null;
+
 		LeaderRetrievalService leaderRetrievalService = null;
 
 		// Coordination between the processes goes through a directory
@@ -263,13 +266,22 @@ public void testJobManagerProcessFailure() throws Exception {
 			config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 100);
 			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
 
+			highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+				config,
+				TestingUtils.defaultExecutor());
+
 			// Start the task manager process
 			for (int i = 0; i < numberOfTaskManagers; i++) {
 				tmActorSystem[i] = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
 				TaskManager.startTaskManagerComponentsAndActor(
-						config, ResourceID.generate(), tmActorSystem[i], "localhost",
-						Option.<String>empty(), Option.<LeaderRetrievalService>empty(),
-						false, TaskManager.class);
+					config,
+					ResourceID.generate(),
+					tmActorSystem[i],
+					highAvailabilityServices,
+					"localhost",
+					Option.<String>empty(),
+					false,
+					TaskManager.class);
 			}
 
 			// Test actor system
@@ -279,7 +291,7 @@ public void testJobManagerProcessFailure() throws Exception {
 
 			// Leader listener
 			TestingListener leaderListener = new TestingListener();
-			leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
+			leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
 			leaderRetrievalService.start(leaderListener);
 
 			// Initial submission
@@ -378,6 +390,10 @@ public void run() {
 				}
 			}
 
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
+			}
+
 			// Delete coordination directory
 			if (coordinateTempDir != null) {
 				try {
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
index 5ba651f6ce661..9d2806c5483a0 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
@@ -31,8 +31,11 @@
 import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.client.JobStatusMessage;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.messages.JobManagerMessages;
@@ -70,12 +73,13 @@
 public class ProcessFailureCancelingITCase extends TestLogger {
 	
 	@Test
-	public void testCancelingOnProcessFailure() {
+	public void testCancelingOnProcessFailure() throws Exception {
 		final StringWriter processOutput = new StringWriter();
 
 		ActorSystem jmActorSystem = null;
 		Process taskManagerProcess = null;
-		
+		HighAvailabilityServices highAvailabilityServices = null;
+
 		try {
 			// check that we run this test only if the java command
 			// is available on this machine
@@ -101,6 +105,13 @@ public void testCancelingOnProcessFailure() {
 			jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2000 s");
 			jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 10);
 			jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
+			jmConfig.setString(JobManagerOptions.ADDRESS, localAddress._1());
+			jmConfig.setInteger(JobManagerOptions.PORT, jobManagerPort);
+
+			highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				jmConfig,
+				TestingUtils.defaultExecutor(),
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
 			jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<>(localAddress));
 			ActorRef jmActor = JobManager.startJobManagerActors(
@@ -108,6 +119,7 @@ public void testCancelingOnProcessFailure() {
 				jmActorSystem,
 				TestingUtils.defaultExecutor(),
 				TestingUtils.defaultExecutor(),
+				highAvailabilityServices,
 				JobManager.class,
 				MemoryArchivist.class)._1();
 
@@ -193,12 +205,10 @@ public Long map(Long value) throws Exception {
 			// all seems well :-)
 		}
 		catch (Exception e) {
-			e.printStackTrace();
 			printProcessLog("TaskManager", processOutput.toString());
-			fail(e.getMessage());
+			throw e;
 		}
 		catch (Error e) {
-			e.printStackTrace();
 			printProcessLog("TaskManager 1", processOutput.toString());
 			throw e;
 		}
@@ -209,6 +219,10 @@ public Long map(Long value) throws Exception {
 			if (jmActorSystem != null) {
 				jmActorSystem.shutdown();
 			}
+
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
+			}
 		}
 	}
 	
@@ -250,7 +264,11 @@ private void cancelRunningJob(ActorRef jobManager) throws Exception {
 		}
 		
 		// tell the JobManager to cancel the job
-		jobManager.tell(new JobManagerMessages.CancelJob(jobId), ActorRef.noSender());
+		jobManager.tell(
+			new JobManagerMessages.LeaderSessionMessage(
+				HighAvailabilityServices.DEFAULT_LEADER_ID,
+				new JobManagerMessages.CancelJob(jobId)),
+			ActorRef.noSender());
 	}
 
 	private void waitUntilNumTaskManagersAreRegistered(ActorRef jobManager, int numExpected, long maxDelay)
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java
index fa7e071fa2a13..bafdd9f685d42 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java
@@ -32,6 +32,7 @@
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.messages.TaskManagerMessages;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 import scala.concurrent.Await;
@@ -87,8 +88,7 @@ public void testRestartWithFailingTaskManager() {
 			// for the result
 			List<Long> resultCollection = new ArrayList<Long>();
 
-			final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(
-					"localhost", cluster.getLeaderRPCPort());
+			final ExecutionEnvironment env = new TestEnvironment(cluster, PARALLELISM, false);
 
 			env.setParallelism(PARALLELISM);
 			env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
index 5eadba6203486..40a8f09d55fe3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
@@ -34,7 +34,6 @@
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobmanager.Tasks;
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.runtime.testingUtils.TestingCluster;
@@ -42,7 +41,6 @@
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.WaitForAllVerticesToBeRunningOrFinished;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -80,6 +78,7 @@ public static void setup() throws Exception {
 	public static void tearDown() throws Exception {
 		if (zkServer != null) {
 			zkServer.close();
+			zkServer = null;
 		}
 	}
 
@@ -112,8 +111,8 @@ public void testTaskManagerRegistrationAtReelectedLeader() throws Exception {
 				cluster.waitForTaskManagersToBeRegisteredAtJobManager(leadingJM.actor());
 
 				Future<Object> registeredTMs = leadingJM.ask(
-						JobManagerMessages.getRequestNumberRegisteredTaskManager(),
-						timeout);
+					JobManagerMessages.getRequestNumberRegisteredTaskManager(),
+					timeout);
 
 				int numRegisteredTMs = (Integer) Await.result(registeredTMs, timeout);
 
@@ -122,8 +121,7 @@ public void testTaskManagerRegistrationAtReelectedLeader() throws Exception {
 				cluster.clearLeader();
 				leadingJM.tell(PoisonPill.getInstance());
 			}
-		}
-		finally {
+		} finally {
 			cluster.stop();
 		}
 	}
@@ -273,15 +271,10 @@ public JobSubmitterRunnable(
 		@Override
 		public void run() {
 			try {
-				LeaderRetrievalService lrService =
-						LeaderRetrievalUtils.createLeaderRetrievalService(
-							cluster.configuration(),
-							false);
-
 				JobExecutionResult result = JobClient.submitJobAndWait(
 						clientActorSystem,
 						cluster.configuration(),
-						lrService,
+						cluster.highAvailabilityServices(),
 						graph,
 						timeout,
 						false,
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java
index 2eecf499417bd..82e8d94f5dc16 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java
@@ -27,14 +27,13 @@
 import org.apache.flink.runtime.akka.ListeningBehaviour;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
 import org.apache.flink.runtime.jobmanager.JobManager;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
 import org.apache.flink.runtime.taskmanager.TaskManager;
@@ -44,9 +43,11 @@
 import org.apache.flink.runtime.testingUtils.TestingTaskManager;
 import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.streaming.api.CheckpointingMode;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator;
+import org.apache.flink.util.TestLogger;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -71,36 +72,48 @@
  * Step 1: Migrate the job to 1.3 by submitting the same job used for the 1.2 savepoint, and create a new savepoint.
  * Step 2: Modify the job topology, and restore from the savepoint created in step 1.
  */
-public abstract class AbstractOperatorRestoreTestBase {
+public abstract class AbstractOperatorRestoreTestBase extends TestLogger {
 
 	@Rule
 	public final TemporaryFolder tmpFolder = new TemporaryFolder();
 
 	private static ActorSystem actorSystem = null;
+	private static HighAvailabilityServices highAvailabilityServices = null;
 	private static ActorGateway jobManager = null;
 	private static ActorGateway archiver = null;
 	private static ActorGateway taskManager = null;
 
-	private static final FiniteDuration timeout = new FiniteDuration(30, TimeUnit.SECONDS);
+	private static final FiniteDuration timeout = new FiniteDuration(30L, TimeUnit.SECONDS);
 
 	@BeforeClass
 	public static void setupCluster() throws Exception {
-		FiniteDuration timeout = new FiniteDuration(30, TimeUnit.SECONDS);
+		final Configuration configuration = new Configuration();
+
+		FiniteDuration timeout = new FiniteDuration(30L, TimeUnit.SECONDS);
 
 		actorSystem = AkkaUtils.createLocalActorSystem(new Configuration());
 
+		highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+			configuration,
+			TestingUtils.defaultExecutor());
+
 		Tuple2<ActorRef, ActorRef> master = JobManager.startJobManagerActors(
-			new Configuration(),
+			configuration,
 			actorSystem,
 			TestingUtils.defaultExecutor(),
 			TestingUtils.defaultExecutor(),
+			highAvailabilityServices,
 			Option.apply("jm"),
 			Option.apply("arch"),
 			TestingJobManager.class,
 			TestingMemoryArchivist.class);
 
-		jobManager = new AkkaActorGateway(master._1(), HighAvailabilityServices.DEFAULT_LEADER_ID);
-		archiver = new AkkaActorGateway(master._2(), HighAvailabilityServices.DEFAULT_LEADER_ID);
+		jobManager = LeaderRetrievalUtils.retrieveLeaderGateway(
+			highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+			actorSystem,
+			timeout);
+
+		archiver = new AkkaActorGateway(master._2(), jobManager.leaderSessionID());
 
 		Configuration tmConfig = new Configuration();
 		tmConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
@@ -109,13 +122,13 @@ public static void setupCluster() throws Exception {
 			tmConfig,
 			ResourceID.generate(),
 			actorSystem,
+			highAvailabilityServices,
 			"localhost",
 			Option.apply("tm"),
-			Option.<LeaderRetrievalService>apply(new StandaloneLeaderRetrievalService(jobManager.path(), HighAvailabilityServices.DEFAULT_LEADER_ID)),
 			true,
 			TestingTaskManager.class);
 
-		taskManager = new AkkaActorGateway(taskManagerRef, HighAvailabilityServices.DEFAULT_LEADER_ID);
+		taskManager = new AkkaActorGateway(taskManagerRef, jobManager.leaderSessionID());
 
 		// Wait until connected
 		Object msg = new TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager(jobManager.actor());
@@ -123,7 +136,11 @@ public static void setupCluster() throws Exception {
 	}
 
 	@AfterClass
-	public static void tearDownCluster() {
+	public static void tearDownCluster() throws Exception {
+		if (highAvailabilityServices != null) {
+			highAvailabilityServices.closeAndCleanupAllData();
+		}
+
 		if (actorSystem != null) {
 			actorSystem.shutdown();
 		}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java
index 229d3fdcce435..45686ef9bf50b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java
@@ -45,6 +45,7 @@
 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.util.TestLogger;
 
 import org.junit.AfterClass;
@@ -86,32 +87,24 @@ public void setupLatch() {
 
 	@BeforeClass
 	public static void startCluster() {
-		try {
-			Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS);
-			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS);
-			config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L);
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS);
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS);
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L);
 
-			cluster = new LocalFlinkMiniCluster(config, false);
+		cluster = new LocalFlinkMiniCluster(config, false);
 
-			cluster.start();
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to start test cluster: " + e.getMessage());
-		}
+		cluster.start();
+
+		TestStreamEnvironment.setAsContext(cluster, PARALLELISM);
 	}
 
 	@AfterClass
 	public static void shutdownCluster() {
-		try {
-			cluster.shutdown();
-			cluster = null;
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail("Failed to stop test cluster: " + e.getMessage());
-		}
+		cluster.shutdown();
+		cluster = null;
+
+		TestStreamEnvironment.unsetAsContext();
 	}
 
 	/**
@@ -132,8 +125,7 @@ public void testWatermarkPropagation() throws Exception {
 
 		long initialTime = 0L;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.setParallelism(PARALLELISM);
@@ -182,8 +174,7 @@ public void testWatermarkPropagationNoFinalWatermarkOnStop() throws Exception {
 
 		long initialTime = 0L;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.setParallelism(PARALLELISM);
@@ -270,8 +261,7 @@ public void testTimestampHandling() throws Exception {
 		final int NUM_ELEMENTS = 10;
 
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.setParallelism(PARALLELISM);
@@ -297,8 +287,7 @@ public void testTimestampHandling() throws Exception {
 	public void testDisabledTimestamps() throws Exception {
 		final int NUM_ELEMENTS = 10;
 		
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		
 		env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
 		env.setParallelism(PARALLELISM);
@@ -325,7 +314,7 @@ public void testDisabledTimestamps() throws Exception {
 	public void testTimestampExtractorWithAutoInterval() throws Exception {
 		final int NUM_ELEMENTS = 10;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.getConfig().setAutoWatermarkInterval(10);
@@ -389,7 +378,7 @@ public long extractAscendingTimestamp(Integer element) {
 	public void testTimestampExtractorWithCustomWatermarkEmit() throws Exception {
 		final int NUM_ELEMENTS = 10;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.getConfig().setAutoWatermarkInterval(10);
@@ -451,7 +440,7 @@ public Watermark checkAndGetNextWatermark(Integer element, long extractedTimesta
 	public void testTimestampExtractorWithDecreasingCustomWatermarkEmit() throws Exception {
 		final int NUM_ELEMENTS = 10;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.getConfig().setAutoWatermarkInterval(1);
@@ -513,7 +502,7 @@ public Watermark checkAndGetNextWatermark(Integer element, long extractedTimesta
 	public void testTimestampExtractorWithLongMaxWatermarkFromSource() throws Exception {
 		final int NUM_ELEMENTS = 10;
 
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.getConfig().setAutoWatermarkInterval(1);
@@ -574,7 +563,7 @@ public void testTimestampExtractorWithLongMaxWatermarkFromSource2() throws Excep
 		final int NUM_ELEMENTS = 10;
 
 		StreamExecutionEnvironment env = StreamExecutionEnvironment
-				.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+				.getExecutionEnvironment();
 
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 		env.getConfig().setAutoWatermarkInterval(10);
@@ -630,7 +619,7 @@ public Watermark getCurrentWatermark() {
 	@Test
 	public void testEventTimeSourceWithProcessingTime() throws Exception {
 		StreamExecutionEnvironment env = 
-				StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+				StreamExecutionEnvironment.getExecutionEnvironment();
 		
 		env.setParallelism(2);
 		env.getConfig().disableSysoutLogging();
@@ -651,8 +640,7 @@ public void testEventTimeSourceWithProcessingTime() throws Exception {
 	
 	@Test
 	public void testErrorOnEventTimeOverProcessingTime() {
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setParallelism(2);
 		env.getConfig().disableSysoutLogging();
@@ -682,8 +670,7 @@ public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<Str
 
 	@Test
 	public void testErrorOnEventTimeWithoutTimestamps() {
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
-				"localhost", cluster.getLeaderRPCPort());
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		env.setParallelism(2);
 		env.getConfig().disableSysoutLogging();
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala
index 5db02d17b2d28..e5f26c54e9676 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala
@@ -18,7 +18,7 @@
 
 package org.apache.flink.api.scala.runtime.jobmanager
 
-import akka.actor.{ActorSystem, PoisonPill}
+import akka.actor.ActorSystem
 import akka.testkit.{ImplicitSender, TestKit}
 import org.apache.flink.configuration.{ConfigConstants, Configuration}
 import org.apache.flink.runtime.akka.{AkkaUtils, ListeningBehaviour}
@@ -28,7 +28,6 @@ import org.apache.flink.runtime.messages.Acknowledge
 import org.apache.flink.runtime.messages.JobManagerMessages._
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered
 import org.apache.flink.runtime.testingUtils.TestingMessages.DisableDisconnect
-import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.{JobManagerTerminated, NotifyWhenJobManagerTerminated}
 import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingCluster, TestingUtils}
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner
@@ -66,11 +65,8 @@ class JobManagerFailsITCase(_system: ActorSystem)
           jmGateway.tell(RequestNumberRegisteredTaskManager, self)
           expectMsg(1)
 
-          tm ! NotifyWhenJobManagerTerminated(jmGateway.leaderSessionID())
-
-          jmGateway.tell(PoisonPill, self)
-
-          expectMsgClass(classOf[JobManagerTerminated])
+          // stop the current leader and make sure that he is gone
+          TestingUtils.stopActorGracefully(jmGateway)
 
           cluster.restartLeadingJobManager()
 
@@ -109,11 +105,8 @@ class JobManagerFailsITCase(_system: ActorSystem)
           jmGateway.tell(SubmitJob(jobGraph, ListeningBehaviour.DETACHED), self)
           expectMsg(JobSubmitSuccess(jobGraph.getJobID))
 
-          tm.tell(NotifyWhenJobManagerTerminated(jmGateway.leaderSessionID()), self)
-
-          jmGateway.tell(PoisonPill, self)
-
-          expectMsgClass(classOf[JobManagerTerminated])
+          // stop the current leader and make sure that he is gone
+          TestingUtils.stopActorGracefully(jmGateway)
 
           cluster.restartLeadingJobManager()
 
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala
index 424fc8bd29998..61cb8ccbc3f49 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala
@@ -226,7 +226,7 @@ class TaskManagerFailsITCase(_system: ActorSystem)
 
           tm ! NotifyWhenRegisteredAtJobManager
 
-          expectMsg(RegisteredAtJobManager)
+          expectMsgClass(classOf[RegisteredAtJobManager])
 
           jmGateway.tell(SubmitJob(jobGraph2, ListeningBehaviour.EXECUTION_RESULT), self)
 
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java
index a503115125570..264b6aaafc2b0 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.yarn;
 
-import akka.actor.ActorSystem;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.DefaultParser;
@@ -27,7 +26,6 @@
 import org.apache.flink.client.CliFrontend;
 import org.apache.flink.client.cli.CliFrontendParser;
 import org.apache.flink.client.cli.RunOptions;
-import org.apache.flink.client.deployment.ClusterDescriptor;
 import org.apache.flink.client.program.ClusterClient;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
@@ -36,7 +34,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -129,7 +126,7 @@ public void testCorrectSettingOfMaxSlots() throws Exception {
 		Assert.assertEquals(2, descriptor.getTaskManagerCount());
 
 		Configuration config = new Configuration();
-		CliFrontend.setJobManagerAddressInConfig(config, new InetSocketAddress("test", 9000));
+		CliFrontend.setJobManagerAddressInConfig(config, new InetSocketAddress("localhost", 9000));
 		ClusterClient client = new TestingYarnClusterClient(descriptor, config);
 		Assert.assertEquals(6, client.getMaxSlots());
 	}
@@ -175,7 +172,7 @@ protected AbstractYarnClusterDescriptor getClusterDescriptor() {
 
 	private static class TestingYarnClusterClient extends YarnClusterClient {
 
-		public TestingYarnClusterClient(AbstractYarnClusterDescriptor descriptor, Configuration config) throws IOException, YarnException {
+		public TestingYarnClusterClient(AbstractYarnClusterDescriptor descriptor, Configuration config) throws Exception {
 			super(descriptor,
 				Mockito.mock(YarnClient.class),
 				Mockito.mock(ApplicationReport.class),
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index bc1af65ebf8dd..1f043efaba217 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -24,14 +24,15 @@
 import org.apache.curator.test.TestingServer;
 import org.apache.flink.client.program.ClusterClient;
 import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.instance.AkkaActorGateway;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.state.filesystem.FsStateBackendFactory;
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
@@ -56,7 +57,7 @@ public class YARNHighAvailabilityITCase extends YarnTestBase {
 
 	protected static ActorSystem actorSystem;
 
-	protected static final int numberApplicationAttempts = 10;
+	protected static final int numberApplicationAttempts = 3;
 
 	@Rule
 	public TemporaryFolder temp = new TemporaryFolder();
@@ -128,9 +129,19 @@ public void testMultipleAMKill() throws Exception {
 
 		final FiniteDuration timeout = new FiniteDuration(2, TimeUnit.MINUTES);
 
+		HighAvailabilityServices highAvailabilityServices = null;
+
 		try {
 			yarnCluster = flinkYarnClient.deploy();
-			final Configuration config = yarnCluster.getFlinkConfiguration();
+
+			final ClusterClient finalYarnCluster = yarnCluster;
+
+			highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				yarnCluster.getFlinkConfiguration(),
+				Executors.directExecutor(),
+				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);
+
+			final HighAvailabilityServices finalHighAvailabilityServices = highAvailabilityServices;
 
 			new JavaTestKit(actorSystem) {{
 				for (int attempt = 0; attempt < numberKillingAttempts; attempt++) {
@@ -138,8 +149,10 @@ public void testMultipleAMKill() throws Exception {
 						@Override
 						protected void run() {
 							try {
-								LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
-								ActorGateway gateway = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, actorSystem, timeout);
+								ActorGateway gateway = LeaderRetrievalUtils.retrieveLeaderGateway(
+									finalHighAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+									actorSystem,
+									timeout);
 								ActorGateway selfGateway = new AkkaActorGateway(getRef(), gateway.leaderSessionID());
 
 								gateway.tell(new TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered(1), selfGateway);
@@ -158,10 +171,13 @@ protected void run() {
 					@Override
 					protected void run() {
 						try {
-							LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config, false);
-							ActorGateway gateway2 = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, actorSystem, timeout);
-							ActorGateway selfGateway = new AkkaActorGateway(getRef(), gateway2.leaderSessionID());
-							gateway2.tell(new TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered(1), selfGateway);
+							ActorGateway gateway = LeaderRetrievalUtils.retrieveLeaderGateway(
+								finalHighAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
+								actorSystem,
+								timeout);
+							ActorGateway selfGateway = new AkkaActorGateway(getRef(), gateway.leaderSessionID());
+
+							gateway.tell(new TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered(1), selfGateway);
 
 							expectMsgEquals(Acknowledge.get());
 						} catch (Exception e) {
@@ -175,6 +191,10 @@ protected void run() {
 			if (yarnCluster != null) {
 				yarnCluster.shutdown();
 			}
+
+			if (highAvailabilityServices != null) {
+				highAvailabilityServices.closeAndCleanupAllData();
+			}
 		}
 	}
 }
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index 68cc73dd3e291..f45fe82bc7afc 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -224,7 +224,7 @@ public void testfullAlloc() {
 	 * Test the YARN Java API
 	 */
 	@Test
-	public void testJavaAPI() {
+	public void testJavaAPI() throws Exception {
 		final int WAIT_TIME = 15;
 		LOG.info("Starting testJavaAPI()");
 
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index a5a6c364aab2b..65525f28b1b00 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -1307,7 +1307,7 @@ protected YarnClusterClient createYarnClusterClient(
 			ApplicationReport report,
 			org.apache.flink.configuration.Configuration flinkConfiguration,
 			Path sessionFilesDir,
-			boolean perJobCluster) throws IOException, YarnException {
+			boolean perJobCluster) throws Exception {
 		return new YarnClusterClient(
 			descriptor,
 			yarnClient,
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
index ee87cfe147a67..b62f957363bc0 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
@@ -26,14 +26,16 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.SecurityOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
 import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.jobmaster.JobMaster;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.net.SSLUtils;
 import org.apache.flink.runtime.process.ProcessReaper;
 import org.apache.flink.runtime.security.SecurityUtils;
@@ -42,7 +44,6 @@
 import org.apache.flink.runtime.util.ExecutorThreadFactory;
 import org.apache.flink.runtime.util.Hardware;
 import org.apache.flink.runtime.util.JvmShutdownSafeguard;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.runtime.util.SignalHandler;
 import org.apache.flink.runtime.webmonitor.WebMonitor;
 import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
@@ -215,6 +216,7 @@ public Integer call() {
 	protected int runApplicationMaster(Configuration config) {
 		ActorSystem actorSystem = null;
 		WebMonitor webMonitor = null;
+		HighAvailabilityServices highAvailabilityServices = null;
 
 		int numberProcessors = Hardware.getNumberCPUCores();
 
@@ -332,6 +334,16 @@ protected int runApplicationMaster(Configuration config) {
 			// 3) Resource Master for YARN
 			// 4) Process reapers for the JobManager and Resource Master
 
+			// 0: Start the JobManager services
+
+			// update the configuration used to create the high availability services
+			config.setString(JobManagerOptions.ADDRESS, akkaHostname);
+			config.setInteger(JobManagerOptions.PORT, akkaPort);
+
+			highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
+				config,
+				ioExecutor,
+				HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
 			// 1: the JobManager
 			LOG.debug("Starting JobManager actor");
@@ -342,6 +354,7 @@ protected int runApplicationMaster(Configuration config) {
 				actorSystem,
 				futureExecutor,
 				ioExecutor,
+				highAvailabilityServices,
 				new Some<>(JobMaster.JOB_MANAGER_NAME),
 				Option.<String>empty(),
 				getJobManagerClass(),
@@ -351,7 +364,12 @@ protected int runApplicationMaster(Configuration config) {
 			// 2: the web monitor
 			LOG.debug("Starting Web Frontend");
 
-			webMonitor = BootstrapTools.startWebMonitorIfConfigured(config, actorSystem, jobManager, LOG);
+			webMonitor = BootstrapTools.startWebMonitorIfConfigured(
+				config,
+				highAvailabilityServices,
+				actorSystem,
+				jobManager,
+				LOG);
 
 			String protocol = "http://";
 			if (config.getBoolean(ConfigConstants.JOB_MANAGER_WEB_SSL_ENABLED,
@@ -364,15 +382,11 @@ protected int runApplicationMaster(Configuration config) {
 			// 3: Flink's Yarn ResourceManager
 			LOG.debug("Starting YARN Flink Resource Manager");
 
-			// we need the leader retrieval service here to be informed of new leaders and session IDs
-			LeaderRetrievalService leaderRetriever = 
-				LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager);
-
 			Props resourceMasterProps = YarnFlinkResourceManager.createActorProps(
 				getResourceManagerClass(),
 				config,
 				yarnConfig,
-				leaderRetriever,
+				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
 				appMasterHostname,
 				webMonitorURL,
 				taskManagerParameters,
@@ -437,6 +451,14 @@ protected int runApplicationMaster(Configuration config) {
 			}
 		}
 
+		if (highAvailabilityServices != null) {
+			try {
+				highAvailabilityServices.close();
+			} catch (Throwable t) {
+				LOG.error("Failed to stop the high availability services.", t);
+			}
+		}
+
 		org.apache.flink.runtime.concurrent.Executors.gracefulShutdown(
 			AkkaUtils.getTimeout(config).toMillis(),
 			TimeUnit.MILLISECONDS,
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java
index f044cdd5198f5..e70af09d1f68d 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java
@@ -32,9 +32,8 @@
 import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse;
 import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
 import org.apache.flink.runtime.clusterframework.messages.ShutdownClusterAfterJob;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
 import org.apache.hadoop.conf.Configuration;
@@ -109,7 +108,7 @@ public YarnClusterClient(
 		final ApplicationReport appReport,
 		org.apache.flink.configuration.Configuration flinkConfig,
 		Path sessionFilesDir,
-		boolean newlyCreatedCluster) throws IOException, YarnException {
+		boolean newlyCreatedCluster) throws Exception {
 
 		super(flinkConfig);
 
@@ -123,7 +122,10 @@ public YarnClusterClient(
 		this.trackingURL = appReport.getTrackingUrl();
 		this.newlyCreatedCluster = newlyCreatedCluster;
 
-		this.applicationClient = new LazApplicationClientLoader(flinkConfig, actorSystemLoader);
+		this.applicationClient = new LazApplicationClientLoader(
+			flinkConfig,
+			actorSystemLoader,
+			highAvailabilityServices);
 
 		this.pollingRunner = new PollingThread(yarnClient, appId);
 		this.pollingRunner.setDaemon(true);
@@ -443,7 +445,12 @@ private class ClientShutdownHook extends Thread {
 		@Override
 		public void run() {
 			LOG.info("Shutting down YarnClusterClient from the client shutdown hook");
-			shutdown();
+
+			try {
+				shutdown();
+			} catch (Throwable t) {
+				LOG.warn("Could not properly shut down the yarn cluster client.", t);
+			}
 		}
 	}
 
@@ -545,14 +552,17 @@ private static class LazApplicationClientLoader {
 
 		private final org.apache.flink.configuration.Configuration flinkConfig;
 		private final LazyActorSystemLoader actorSystemLoader;
+		private final HighAvailabilityServices highAvailabilityServices;
 
 		private ActorRef applicationClient;
 
 		private LazApplicationClientLoader(
 				org.apache.flink.configuration.Configuration flinkConfig,
-				LazyActorSystemLoader actorSystemLoader) {
-			this.flinkConfig = flinkConfig;
-			this.actorSystemLoader = actorSystemLoader;
+				LazyActorSystemLoader actorSystemLoader,
+				HighAvailabilityServices highAvailabilityServices) {
+			this.flinkConfig = Preconditions.checkNotNull(flinkConfig, "flinkConfig");
+			this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader, "actorSystemLoader");
+			this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices, "highAvailabilityServices");
 		}
 
 		/**
@@ -561,14 +571,6 @@ private LazApplicationClientLoader(
 		 */
 		public ActorRef get() {
 			if (applicationClient == null) {
-				/* The leader retrieval service for connecting to the cluster and finding the active leader. */
-				LeaderRetrievalService leaderRetrievalService;
-				try {
-					leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(flinkConfig, true);
-				} catch (Exception e) {
-					throw new RuntimeException("Could not create the leader retrieval service.", e);
-				}
-
 				// start application client
 				LOG.info("Start application client.");
 
@@ -576,7 +578,7 @@ public ActorRef get() {
 					Props.create(
 						ApplicationClient.class,
 						flinkConfig,
-						leaderRetrievalService),
+						highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID)),
 					"applicationClient");
 			}
 
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientV2.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientV2.java
index 5a3a3c00e1cea..33d5987e3fdd8 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientV2.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClientV2.java
@@ -31,7 +31,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.URL;
 import java.util.List;
@@ -58,11 +57,11 @@ public class YarnClusterClientV2 extends ClusterClient {
 	 *
 	 * @param clusterDescriptor The descriptor used to create yarn job
 	 * @param flinkConfig Flink configuration
-	 * @throws java.io.IOException
+	 * @throws Exception if the cluster client could not be created
 	 */
 	public YarnClusterClientV2(
 			final AbstractYarnClusterDescriptor clusterDescriptor,
-			org.apache.flink.configuration.Configuration flinkConfig) throws IOException {
+			org.apache.flink.configuration.Configuration flinkConfig) throws Exception {
 
 		super(flinkConfig);
 
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
index 05b2be8cd2c64..3f4d4f645df08 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkApplicationMasterRunner.java
@@ -138,9 +138,10 @@ protected int runApplicationMaster(Configuration config) {
 				LOG.info("Starting High Availability Services");
 				commonRpcService = createRpcService(config, appMasterHostname, amPortRange);
 
-				haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
+				haServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
 					config,
-					commonRpcService.getExecutor());
+					commonRpcService.getExecutor(),
+					HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
 
 				heartbeatServices = HeartbeatServices.fromConfiguration(config);
 				
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java
index ca5049c444566..557fa389f8bf9 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java
@@ -33,7 +33,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URL;
 import java.net.URLDecoder;
@@ -224,22 +223,14 @@ public YarnClusterClientV2 createCluster(
 			String applicationName,
 			CommandLine cmdLine,
 			Configuration config,
-			List<URL> userJarFiles) {
+			List<URL> userJarFiles) throws Exception {
 		Preconditions.checkNotNull(userJarFiles, "User jar files should not be null.");
 
 		YarnClusterDescriptorV2 yarnClusterDescriptor = createDescriptor(applicationName, cmdLine);
 		yarnClusterDescriptor.setFlinkConfiguration(config);
 		yarnClusterDescriptor.setProvidedUserJarFiles(userJarFiles);
 
-		YarnClusterClientV2 client = null;
-		try {
-			client = new YarnClusterClientV2(yarnClusterDescriptor, config);
-		}
-		catch (IOException e) {
-			throw new RuntimeException("Fail to create YarnClusterClientV2", e.getCause());
-		}
-		return client;
-
+		return new YarnClusterClientV2(yarnClusterDescriptor, config);
 	}
 
 	/**
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
index f4557c98c2556..9277d21d35202 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
@@ -678,7 +678,12 @@ public int run(String[] args) {
 	public void stop() {
 		if (yarnCluster != null) {
 			LOG.info("Command line interface is shutting down the yarnCluster");
-			yarnCluster.shutdown();
+
+			try {
+				yarnCluster.shutdown();
+			} catch (Throwable t) {
+				LOG.warn("Could not properly shutdown the yarn cluster.", t);
+			}
 		}
 	}
 

From 3ab8298e13739aaa8a1f9703f8be6cf0b74f0034 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Wed, 3 May 2017 14:40:46 +0200
Subject: [PATCH 146/253] [FLINK-6435] [async] React to exceptionally completed
 StreamElementQueueEntry

The AsyncWaitOperator should not only react to orderly completed
StreamElementQueueEntries but also to those completed with a user exception
or those which timed out.

This PR fixes the problem by calling the onComplete function passed to
StreamElementQueueEntry#onComplete also in the exceptional case.

This closes #3814.
---
 .../async/queue/StreamElementQueueEntry.java  |   9 +-
 .../async/AsyncWaitOperatorTest.java          | 196 ++++++++++++++----
 2 files changed, 161 insertions(+), 44 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
index 66872df317355..4a50201f095a5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java
@@ -21,6 +21,7 @@
 import java.util.concurrent.Executor;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.runtime.concurrent.BiFunction;
 import org.apache.flink.runtime.concurrent.Future;
 import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
 import org.apache.flink.util.Preconditions;
@@ -65,10 +66,14 @@ public void onComplete(
 			Executor executor) {
 		final StreamElementQueueEntry<T> thisReference = this;
 
-		getFuture().thenAcceptAsync(new AcceptFunction<T>() {
+		getFuture().handleAsync(new BiFunction<T, Throwable, Void>() {
 			@Override
-			public void accept(T value) {
+			public Void apply(T t, Throwable throwable) {
+				// call the complete function for normal completion as well as exceptional completion
+				// see FLINK-6435
 				completeFunction.accept(thisReference);
+
+				return null;
 			}
 		}, executor);
 	}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java
index c4867ffb58f04..1d832292e838e 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java
@@ -71,6 +71,7 @@
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
+import javax.annotation.Nonnull;
 import java.util.ArrayDeque;
 import java.util.Collections;
 import java.util.Comparator;
@@ -90,13 +91,7 @@
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
 
 /**
  * Tests for {@link AsyncWaitOperator}. These test that:
@@ -173,7 +168,7 @@ public void run() {
 	}
 
 	/**
-	 * A special {@link org.apache.flink.streaming.api.functions.async.AsyncFunction} without issuing
+	 * A special {@link AsyncFunction} without issuing
 	 * {@link AsyncCollector#collect} until the latch counts to zero.
 	 * This function is used in the testStateSnapshotAndRestore, ensuring
 	 * that {@link StreamElementQueueEntry} can stay
@@ -637,20 +632,7 @@ public void testAsyncTimeout() throws Exception {
 			2,
 			AsyncDataStream.OutputMode.ORDERED);
 
-		final Environment mockEnvironment = mock(Environment.class);
-
-		final Configuration taskConfiguration = new Configuration();
-		final ExecutionConfig executionConfig = new ExecutionConfig();
-		final TaskMetricGroup metricGroup = new UnregisteredTaskMetricsGroup();
-		final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
-		final TaskInfo taskInfo = new TaskInfo("foobarTask", 1, 0, 1, 1);
-
-		when(mockEnvironment.getTaskConfiguration()).thenReturn(taskConfiguration);
-		when(mockEnvironment.getExecutionConfig()).thenReturn(executionConfig);
-		when(mockEnvironment.getMetricGroup()).thenReturn(metricGroup);
-		when(mockEnvironment.getTaskManagerInfo()).thenReturn(taskManagerRuntimeInfo);
-		when(mockEnvironment.getTaskInfo()).thenReturn(taskInfo);
-		when(mockEnvironment.getUserClassLoader()).thenReturn(AsyncWaitOperatorTest.class.getClassLoader());
+		final Environment mockEnvironment = createMockEnvironment();
 
 		final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
 			new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);
@@ -696,6 +678,25 @@ public void testAsyncTimeout() throws Exception {
 		Assert.assertTrue(failureCause.getCause().getCause() instanceof TimeoutException);
 	}
 
+	@Nonnull
+	private Environment createMockEnvironment() {
+		final Environment mockEnvironment = mock(Environment.class);
+
+		final Configuration taskConfiguration = new Configuration();
+		final ExecutionConfig executionConfig = new ExecutionConfig();
+		final TaskMetricGroup metricGroup = new UnregisteredTaskMetricsGroup();
+		final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
+		final TaskInfo taskInfo = new TaskInfo("foobarTask", 1, 0, 1, 1);
+
+		when(mockEnvironment.getTaskConfiguration()).thenReturn(taskConfiguration);
+		when(mockEnvironment.getExecutionConfig()).thenReturn(executionConfig);
+		when(mockEnvironment.getMetricGroup()).thenReturn(metricGroup);
+		when(mockEnvironment.getTaskManagerInfo()).thenReturn(taskManagerRuntimeInfo);
+		when(mockEnvironment.getTaskInfo()).thenReturn(taskInfo);
+		when(mockEnvironment.getUserClassLoader()).thenReturn(Thread.currentThread().getContextClassLoader());
+		return mockEnvironment;
+	}
+
 	/**
 	 * Test case for FLINK-5638: Tests that the async wait operator can be closed even if the
 	 * emitter is currently waiting on the checkpoint lock (e.g. in the case of two chained async
@@ -710,16 +711,7 @@ public void testClosingWithBlockedEmitter() throws Exception {
 
 		ArgumentCaptor<Throwable> failureReason = ArgumentCaptor.forClass(Throwable.class);
 
-		Environment environment = mock(Environment.class);
-		when(environment.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
-		when(environment.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
-		when(environment.getUserClassLoader()).thenReturn(getClass().getClassLoader());
-		when(environment.getTaskInfo()).thenReturn(new TaskInfo(
-			"testTask",
-			1,
-			0,
-			1,
-			0));
+		Environment environment = createMockEnvironment();
 		doNothing().when(environment).failExternally(failureReason.capture());
 
 		StreamTask<?, ?> containingTask = mock(StreamTask.class);
@@ -827,16 +819,7 @@ public void testTimeoutCleanup() throws Exception {
 		final long timeout = 100000L;
 		final long timestamp = 1L;
 
-		Environment environment = mock(Environment.class);
-		when(environment.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
-		when(environment.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
-		when(environment.getUserClassLoader()).thenReturn(getClass().getClassLoader());
-		when(environment.getTaskInfo()).thenReturn(new TaskInfo(
-			"testTask",
-			1,
-			0,
-			1,
-			0));
+		Environment environment = createMockEnvironment();
 
 		ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);
 
@@ -893,4 +876,133 @@ public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws
 		// check that we have cancelled our registered timeout
 		verify(scheduledFuture).cancel(eq(true));
 	}
+
+	/**
+	 * FLINK-6435
+	 *
+	 * Tests that a user exception triggers the completion of a StreamElementQueueEntry and does not wait to until
+	 * another StreamElementQueueEntry is properly completed before it is collected.
+	 */
+	@Test(timeout = 2000)
+	public void testOrderedWaitUserExceptionHandling() throws Exception {
+		testUserExceptionHandling(AsyncDataStream.OutputMode.ORDERED);
+	}
+
+	/**
+	 * FLINK-6435
+	 *
+	 * Tests that a user exception triggers the completion of a StreamElementQueueEntry and does not wait to until
+	 * another StreamElementQueueEntry is properly completed before it is collected.
+	 */
+	@Test(timeout = 2000)
+	public void testUnorderedWaitUserExceptionHandling() throws Exception {
+		testUserExceptionHandling(AsyncDataStream.OutputMode.UNORDERED);
+	}
+
+	private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
+		UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction();
+		long timeout = 2000L;
+
+		AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
+			asyncWaitFunction,
+			TIMEOUT,
+			2,
+			outputMode);
+
+		final Environment mockEnvironment = createMockEnvironment();
+
+		OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
+			asyncWaitOperator,
+			IntSerializer.INSTANCE,
+			mockEnvironment);
+
+		harness.open();
+
+		synchronized (harness.getCheckpointLock()) {
+			harness.processElement(1, 1L);
+		}
+
+		verify(harness.getEnvironment(), timeout(timeout)).failExternally(any(Exception.class));
+
+		synchronized (harness.getCheckpointLock()) {
+			harness.close();
+		}
+	}
+
+	/**
+	 * AsyncFunction which completes the result with an {@link Exception}.
+	 */
+	private static class UserExceptionAsyncFunction implements AsyncFunction<Integer, Integer> {
+
+		private static final long serialVersionUID = 6326568632967110990L;
+
+		@Override
+		public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws Exception {
+			collector.collect(new Exception("Test exception"));
+		}
+	}
+
+	/**
+	 * FLINK-6435
+	 *
+	 * Tests that timeout exceptions are properly handled in ordered output mode. The proper handling means that
+	 * a StreamElementQueueEntry is completed in case of a timeout exception.
+	 */
+	@Test
+	public void testOrderedWaitTimeoutHandling() throws Exception {
+		testTimeoutExceptionHandling(AsyncDataStream.OutputMode.ORDERED);
+	}
+
+	/**
+	 * FLINK-6435
+	 *
+	 * Tests that timeout exceptions are properly handled in ordered output mode. The proper handling means that
+	 * a StreamElementQueueEntry is completed in case of a timeout exception.
+	 */
+	@Test
+	public void testUnorderedWaitTimeoutHandling() throws Exception {
+		testTimeoutExceptionHandling(AsyncDataStream.OutputMode.UNORDERED);
+	}
+
+	private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
+		AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>();
+		long timeout = 10L; // 1 milli second
+
+		AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
+			asyncFunction,
+			timeout,
+			2,
+			outputMode);
+
+		final Environment mockenvironment = createMockEnvironment();
+
+		OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
+			asyncWaitOperator,
+			IntSerializer.INSTANCE,
+			mockenvironment);
+
+		harness.open();
+
+		synchronized (harness.getCheckpointLock()) {
+			harness.processElement(1, 1L);
+		}
+
+		harness.setProcessingTime(10L);
+
+		verify(harness.getEnvironment(), timeout(100L * timeout)).failExternally(any(Exception.class));
+
+		synchronized (harness.getCheckpointLock()) {
+			harness.close();
+		}
+	}
+
+	private static class NoOpAsyncFunction<IN, OUT> implements AsyncFunction<IN, OUT> {
+		private static final long serialVersionUID = -3060481953330480694L;
+
+		@Override
+		public void asyncInvoke(IN input, AsyncCollector<OUT> collector) throws Exception {
+			// no op
+		}
+	}
+
 }

From da94476caa67c64deee16847f67a150e3c759fdb Mon Sep 17 00:00:00 2001
From: Haohui Mai <wheat9@apache.org>
Date: Wed, 29 Mar 2017 16:57:52 -0700
Subject: [PATCH 147/253] [FLINK-6217] ContaineredTaskManagerParameters sets
 off-heap memory size incorrectly.

This closes #3648.
---
 .../LaunchableMesosWorker.java                |  4 +-
 .../clusterframework/BootstrapTools.java      | 18 +++++--
 .../ContaineredTaskManagerParameters.java     |  5 +-
 .../ContaineredTaskManagerParametersTest.java | 49 +++++++++++++++++++
 4 files changed, 69 insertions(+), 7 deletions(-)
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java

diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
index 3d5350adfe062..2408ac67e1f86 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
@@ -223,7 +223,9 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assig
 		// finalize the memory parameters
 		jvmArgs.append(" -Xms").append(tmParams.taskManagerHeapSizeMB()).append("m");
 		jvmArgs.append(" -Xmx").append(tmParams.taskManagerHeapSizeMB()).append("m");
-		jvmArgs.append(" -XX:MaxDirectMemorySize=").append(tmParams.taskManagerDirectMemoryLimitMB()).append("m");
+		if (tmParams.taskManagerDirectMemoryLimitMB() >= 0) {
+			jvmArgs.append(" -XX:MaxDirectMemorySize=").append(tmParams.taskManagerDirectMemoryLimitMB()).append("m");
+		}
 
 		// pass dynamic system properties
 		jvmArgs.append(' ').append(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
index 9bcaa18cf74dd..e9d3cbd36007a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
@@ -25,6 +25,7 @@
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.CoreOptions;
@@ -46,6 +47,7 @@
 import java.io.PrintWriter;
 import java.net.BindException;
 import java.net.ServerSocket;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -358,10 +360,18 @@ public static String getTaskManagerShellCommand(
 
 		final Map<String, String> startCommandValues = new HashMap<>();
 		startCommandValues.put("java", "$JAVA_HOME/bin/java");
-		startCommandValues
-			.put("jvmmem", 	"-Xms" + tmParams.taskManagerHeapSizeMB() + "m " +
-							"-Xmx" + tmParams.taskManagerHeapSizeMB() + "m " +
-							"-XX:MaxDirectMemorySize=" + tmParams.taskManagerDirectMemoryLimitMB() + "m");
+
+		ArrayList<String> params = new ArrayList<>();
+		params.add(String.format("-Xms%dm", tmParams.taskManagerHeapSizeMB()));
+		params.add(String.format("-Xmx%dm", tmParams.taskManagerHeapSizeMB()));
+
+		if (tmParams.taskManagerDirectMemoryLimitMB() >= 0) {
+			params.add(String.format("-XX:MaxDirectMemorySize=%dm",
+				tmParams.taskManagerDirectMemoryLimitMB()));
+		}
+
+		startCommandValues.put("jvmmem", StringUtils.join(params, ' '));
+
 		String javaOpts = flinkConfig.getString(CoreOptions.FLINK_JVM_OPTIONS);
 		if (flinkConfig.getString(CoreOptions.FLINK_TM_JVM_OPTIONS).length() > 0) {
 			javaOpts += " " + flinkConfig.getString(CoreOptions.FLINK_TM_JVM_OPTIONS);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java
index 0fc087032cde4..8ff3c257aa270 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java
@@ -146,8 +146,9 @@ public static ContaineredTaskManagerParameters create(
 		final boolean useOffHeap = config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP);
 
 		final long heapSizeMB;
+		long offHeapSize = -1;
 		if (useOffHeap) {
-			long offHeapSize = config.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE);
+			offHeapSize = config.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE);
 
 			if (offHeapSize <= 0) {
 				double fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
@@ -174,6 +175,6 @@ public static ContaineredTaskManagerParameters create(
 		
 		// done
 		return new ContaineredTaskManagerParameters(
-			containerMemoryMB, heapSizeMB, javaMemorySizeMB, numSlots, envVars);
+			containerMemoryMB, heapSizeMB, offHeapSize, numSlots, envVars);
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java
new file mode 100644
index 0000000000000..c0c48f9120dd0
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.clusterframework;
+
+import org.apache.flink.configuration.Configuration;
+import org.junit.Test;
+
+import static org.apache.flink.configuration.ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY;
+import static org.apache.flink.configuration.TaskManagerOptions.MANAGED_MEMORY_SIZE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ContaineredTaskManagerParametersTest {
+	private static final long CONTAINER_MEMORY = 8192;
+
+	@Test
+	public void testDefaultOffHeapMemory() {
+		Configuration conf = new Configuration();
+		ContaineredTaskManagerParameters params =
+			ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);
+		assertEquals(-1, params.taskManagerDirectMemoryLimitMB());
+	}
+
+	@Test
+	public void testTotalMemoryDoesNotExceedContainerMemory() {
+		Configuration conf = new Configuration();
+		conf.setBoolean(MANAGED_MEMORY_SIZE.key(), true);
+		ContaineredTaskManagerParameters params =
+			ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);
+		assertTrue(params.taskManagerHeapSizeMB() +
+			params.taskManagerDirectMemoryLimitMB() <= CONTAINER_MEMORY);
+	}
+}

From 8456fccef562e1880fb57b6fde7d580179ae5f88 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Wed, 3 May 2017 17:33:01 +0200
Subject: [PATCH 148/253] [FLINK-6217] Fix ContaineredTaskManagerParametersTest
 to properly test off heap settings

---
 .../ContaineredTaskManagerParametersTest.java | 26 ++++++++++++++-----
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java
index c0c48f9120dd0..ad11f70c8473f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java
@@ -19,30 +19,42 @@
 package org.apache.flink.runtime.clusterframework;
 
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
-import static org.apache.flink.configuration.ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY;
-import static org.apache.flink.configuration.TaskManagerOptions.MANAGED_MEMORY_SIZE;
+import static org.apache.flink.configuration.TaskManagerOptions.MEMORY_OFF_HEAP;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class ContaineredTaskManagerParametersTest {
-	private static final long CONTAINER_MEMORY = 8192;
+public class ContaineredTaskManagerParametersTest extends TestLogger {
+	private static final long CONTAINER_MEMORY = 8192L;
 
+	/**
+	 * This tests that per default the off heap memory is set to -1.
+	 */
 	@Test
-	public void testDefaultOffHeapMemory() {
+	public void testOffHeapMemoryWithDefaultConfiguration() {
 		Configuration conf = new Configuration();
+
 		ContaineredTaskManagerParameters params =
 			ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);
-		assertEquals(-1, params.taskManagerDirectMemoryLimitMB());
+		assertEquals(-1L, params.taskManagerDirectMemoryLimitMB());
 	}
 
+	/**
+	 * This tests that when using off heap memory the sum of on and off heap memory does not exceeds the container
+	 * maximum.
+	 */
 	@Test
 	public void testTotalMemoryDoesNotExceedContainerMemory() {
 		Configuration conf = new Configuration();
-		conf.setBoolean(MANAGED_MEMORY_SIZE.key(), true);
+		conf.setBoolean(MEMORY_OFF_HEAP, true);
+
 		ContaineredTaskManagerParameters params =
 			ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);
+
+		assertTrue(params.taskManagerDirectMemoryLimitMB() > 0L);
+
 		assertTrue(params.taskManagerHeapSizeMB() +
 			params.taskManagerDirectMemoryLimitMB() <= CONTAINER_MEMORY);
 	}

From cdcc4e3de4c787bd3d3400496736fecb09419d76 Mon Sep 17 00:00:00 2001
From: Vijay Srinivasaraghavan <vijayaraghavan.srinivasaraghavan@emc.com>
Date: Thu, 6 Apr 2017 09:48:39 -0700
Subject: [PATCH 149/253] [FLINK-5974] [mesos] Added configurations to support
 mesos-dns hostname resolution

This closes #3692.
---
 .../flink/configuration/ConfigConstants.java  |  4 ++
 .../LaunchableMesosWorker.java                | 26 ++++++++++--
 .../MesosApplicationMasterRunner.java         | 17 ++++++--
 .../clusterframework/MesosConfigKeys.java     | 10 +++++
 .../MesosFlinkResourceManager.java            |  3 +-
 .../MesosTaskManagerParameters.java           | 40 ++++++++++++++++++-
 .../MesosFlinkResourceManagerTest.java        |  9 ++++-
 7 files changed, 98 insertions(+), 11 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 975a3d49cf052..cafef2e528f97 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -1286,6 +1286,10 @@ public final class ConfigConstants {
 
 	public static final String DEFAULT_MESOS_RESOURCEMANAGER_FRAMEWORK_USER = "";
 
+	public static final String MESOS_RESOURCEMANAGER_TASKS_HOSTNAME = "mesos.resourcemanager.tasks.hostname";
+
+	public static final String MESOS_RESOURCEMANAGER_TASKS_BOOTSTRAP_CMD = "mesos.resourcemanager.tasks.cmd-prefix";
+
 	/** Default value to override SSL support for the Artifact Server */
 	public static final boolean DEFAULT_MESOS_ARTIFACT_SERVER_SSL_ENABLED = true;
 
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
index 2408ac67e1f86..3e66a5d6fa352 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
@@ -22,10 +22,12 @@
 import com.netflix.fenzo.TaskAssignmentResult;
 import com.netflix.fenzo.TaskRequest;
 import com.netflix.fenzo.VMTaskFitnessCalculator;
+import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.mesos.Utils;
 import org.apache.flink.mesos.scheduler.LaunchableTask;
 import org.apache.flink.mesos.util.MesosArtifactResolver;
+import org.apache.flink.mesos.util.MesosConfiguration;
 import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
 import org.apache.flink.runtime.clusterframework.ContainerSpecification;
 import org.apache.flink.util.Preconditions;
@@ -64,6 +66,7 @@ public class LaunchableMesosWorker implements LaunchableTask {
 	private final MesosTaskManagerParameters params;
 	private final Protos.TaskID taskID;
 	private final Request taskRequest;
+	private final MesosConfiguration mesosConfiguration;
 
 	/**
 	 * Construct a launchable Mesos worker.
@@ -76,11 +79,14 @@ public LaunchableMesosWorker(
 			MesosArtifactResolver resolver,
 			MesosTaskManagerParameters params,
 			ContainerSpecification containerSpec,
-			Protos.TaskID taskID) {
+			Protos.TaskID taskID,
+			MesosConfiguration mesosConfiguration) {
 		this.resolver = Preconditions.checkNotNull(resolver);
-		this.params = Preconditions.checkNotNull(params);
 		this.containerSpec = Preconditions.checkNotNull(containerSpec);
+		this.params = Preconditions.checkNotNull(params);
 		this.taskID = Preconditions.checkNotNull(taskID);
+		this.mesosConfiguration = Preconditions.checkNotNull(mesosConfiguration);
+		
 		this.taskRequest = new Request();
 	}
 
@@ -193,6 +199,12 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assig
 		final Protos.Environment.Builder env = cmd.getEnvironmentBuilder();
 		final StringBuilder jvmArgs = new StringBuilder();
 
+		//configure task manager hostname property if hostname override property is supplied
+		if(params.getTaskManagerHostname().isDefined()) {
+			final String taskManagerHostName = params.getTaskManagerHostname().get().replace("_TASK",taskID.getValue());
+			dynamicProperties.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, taskManagerHostName);
+		}
+
 		// use the assigned ports for the TM
 		if (assignment.getAssignedPorts().size() < TM_PORT_KEYS.length) {
 			throw new IllegalArgumentException("unsufficient # of ports assigned");
@@ -234,8 +246,16 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assig
 		// finalize JVM args
 		env.addVariables(variable(MesosConfigKeys.ENV_JVM_ARGS, jvmArgs.toString()));
 
+		// populate TASK_NAME and FRAMEWORK_NAME environment variables to the TM container
+		env.addVariables(variable(MesosConfigKeys.ENV_TASK_NAME, taskInfo.getTaskId().getValue()));
+		env.addVariables(variable(MesosConfigKeys.ENV_FRAMEWORK_NAME, mesosConfiguration.frameworkInfo().getName()));
+
 		// build the launch command w/ dynamic application properties
-		StringBuilder launchCommand = new StringBuilder("$FLINK_HOME/bin/mesos-taskmanager.sh ");
+		StringBuilder launchCommand = new StringBuilder();
+		if(params.bootstrapCommand().isDefined()) {
+			launchCommand.append(params.bootstrapCommand().get()).append(" && ");
+		}
+		launchCommand.append("$FLINK_HOME/bin/mesos-taskmanager.sh ");
 		launchCommand.append(ContainerSpecification.formatSystemProperties(dynamicProperties));
 		cmd.setValue(launchCommand.toString());
 
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
index 5513df43cc88f..1cedcc38b5030 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
@@ -210,10 +210,19 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 		try {
 			// ------- (1) load and parse / validate all configurations -------
 
-			// Note that we use the "appMasterHostname" given by the system, to make sure
-			// we use the hostnames consistently throughout akka.
-			// for akka "localhost" and "localhost.localdomain" are different actors.
-			final String appMasterHostname = InetAddress.getLocalHost().getHostName();
+			final String appMasterHostname;
+			//We will use JM RPC address property if it is supplied through configuration
+			final String jmRpcAddress = config.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
+			if(jmRpcAddress != null) {
+				LOG.info("JM RPC address from Flink configuration file: {} ", jmRpcAddress);
+				appMasterHostname = jmRpcAddress;
+			} else {
+				// Note that we use the "appMasterHostname" given by the system, to make sure
+				// we use the hostnames consistently throughout akka.
+				// for akka "localhost" and "localhost.localdomain" are different actors.
+				appMasterHostname = InetAddress.getLocalHost().getHostName();
+			}
+			LOG.info("App Master Hostname to use: {}", appMasterHostname);
 
 			// Mesos configuration
 			final MesosConfiguration mesosConfig = createMesosConfig(config, appMasterHostname);
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosConfigKeys.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosConfigKeys.java
index ebd9af5709fbf..35da95f8d53dc 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosConfigKeys.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosConfigKeys.java
@@ -41,6 +41,16 @@ public class MesosConfigKeys {
 	 */
 	public static final String ENV_JVM_ARGS = "JVM_ARGS";
 
+	/**
+	 * Standard environment variables used in DCOS environment
+	 */
+	public static final String ENV_TASK_NAME = "TASK_NAME";
+
+	/**
+ 	 * Standard environment variables used in DCOS environment
+ 	 */
+	public static final String ENV_FRAMEWORK_NAME = "FRAMEWORK_NAME";
+
 	/** Private constructor to prevent instantiation */
 	private MesosConfigKeys() {}
 }
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
index a7321a3a20371..a9ff6cb21adc6 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
@@ -669,7 +669,8 @@ private void taskTerminated(Protos.TaskID taskID, Protos.TaskStatus status) {
 
 	private LaunchableMesosWorker createLaunchableMesosWorker(Protos.TaskID taskID) {
 		LaunchableMesosWorker launchable =
-			new LaunchableMesosWorker(artifactResolver, taskManagerParameters, taskManagerContainerSpec, taskID);
+			new LaunchableMesosWorker(artifactResolver, taskManagerParameters, taskManagerContainerSpec,
+					taskID, mesosConfig);
 		return launchable;
 	}
 
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
index aaab027faaf55..b3b81627c206e 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
@@ -64,6 +64,14 @@ public class MesosTaskManagerParameters {
 		key("mesos.resourcemanager.tasks.container.image.name")
 			.noDefaultValue();
 
+	public static final ConfigOption<String> MESOS_TM_HOSTNAME =
+			key(ConfigConstants.MESOS_RESOURCEMANAGER_TASKS_HOSTNAME)
+			.noDefaultValue();
+
+	public static final ConfigOption<String> MESOS_TM_BOOTSTRAP_CMD =
+			key(ConfigConstants.MESOS_RESOURCEMANAGER_TASKS_BOOTSTRAP_CMD)
+			.noDefaultValue();
+	
 	public static final ConfigOption<String> MESOS_RM_CONTAINER_VOLUMES =
 		key("mesos.resourcemanager.tasks.container.volumes")
 		.noDefaultValue();
@@ -92,6 +100,10 @@ public class MesosTaskManagerParameters {
 	private final List<Protos.Volume> containerVolumes;
 	
 	private final List<ConstraintEvaluator> constraints;
+	
+	private final Option<String> bootstrapCommand;
+
+	private final Option<String> taskManagerHostname;
 
 	public MesosTaskManagerParameters(
 			double cpus,
@@ -99,7 +111,9 @@ public MesosTaskManagerParameters(
 			Option<String> containerImageName,
 			ContaineredTaskManagerParameters containeredParameters,
 			List<Protos.Volume> containerVolumes,
-			List<ConstraintEvaluator> constraints) {
+			List<ConstraintEvaluator> constraints,
+			Option<String> bootstrapCommand,
+			Option<String> taskManagerHostname) {
 
 		this.cpus = cpus;
 		this.containerType = Preconditions.checkNotNull(containerType);
@@ -107,6 +121,8 @@ public MesosTaskManagerParameters(
 		this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
 		this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
 		this.constraints = Preconditions.checkNotNull(constraints);
+		this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
+		this.taskManagerHostname = Preconditions.checkNotNull(taskManagerHostname);
 	}
 
 
@@ -154,6 +170,16 @@ public List<ConstraintEvaluator> constraints() {
 		return constraints;
 	}
 
+	/**
+ 	 * Get the taskManager hostname.
+ 	 */
+	public Option<String> getTaskManagerHostname() { return taskManagerHostname; }
+
+	/**
+ 	 * Get the bootstrap command.
+ 	 */
+	public Option<String> bootstrapCommand() { return bootstrapCommand;	}	
+
 	@Override
 	public String toString() {
 		return "MesosTaskManagerParameters{" +
@@ -163,6 +189,8 @@ public String toString() {
 			", containeredParameters=" + containeredParameters +
 			", containerVolumes=" + containerVolumes +
 			", constraints=" + constraints +
+			", taskManagerHostName=" + taskManagerHostname +
+			", bootstrapCommand=" + bootstrapCommand +
 			'}';
 	}
 
@@ -208,13 +236,21 @@ public static MesosTaskManagerParameters create(Configuration flinkConfig) {
 
 		List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);
 
+		//obtain Task Manager Host Name from the configuration
+		Option<String> tmHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));
+
+		//obtain bootstrap command from the configuration
+		Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));
+
 		return new MesosTaskManagerParameters(
 			cpus,
 			containerType,
 			Option.apply(imageName),
 			containeredParameters,			
 			containerVolumes,
-			constraints);
+			constraints,
+			tmBootstrapCommand,
+			tmHostname);
 	}
 
 	private static List<ConstraintEvaluator> parseConstraints(String mesosConstraints) {
diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
index 7fe5db5531563..7ab4e40d2c6f3 100644
--- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
+++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java
@@ -214,7 +214,14 @@ public void initialize() {
 			ContaineredTaskManagerParameters containeredParams =
 				new ContaineredTaskManagerParameters(1024, 768, 256, 4, new HashMap<String, String>());
 			MesosTaskManagerParameters tmParams = new MesosTaskManagerParameters(
-				1.0, MesosTaskManagerParameters.ContainerType.MESOS, Option.<String>empty(), containeredParams, Collections.<Protos.Volume>emptyList(), Collections.<ConstraintEvaluator>emptyList());
+				1.0, 
+				MesosTaskManagerParameters.ContainerType.MESOS, 
+				Option.<String>empty(), 
+				containeredParams, 
+				Collections.<Protos.Volume>emptyList(), 
+				Collections.<ConstraintEvaluator>emptyList(),
+				Option.<String>empty(),
+				Option.<String>empty());
 
 			TestActorRef<TestingMesosFlinkResourceManager> resourceManagerRef =
 				TestActorRef.create(system, MesosFlinkResourceManager.createActorProps(

From f4d9dd373621caaf62f6d95b81c5e660b4df69bd Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Wed, 3 May 2017 16:48:06 +0200
Subject: [PATCH 150/253] [FLINK-5974] [mesos] Make mesos service name
 configurable for dns lookups

---
 docs/setup/mesos.md                           |  4 +++
 .../flink/configuration/ConfigConstants.java  |  4 ---
 .../LaunchableMesosWorker.java                | 27 +++++++++------
 .../MesosApplicationMasterRunner.java         | 17 +++-------
 .../MesosFlinkResourceManager.java            |  9 +++--
 .../MesosTaskManagerParameters.java           | 34 +++++++++++--------
 6 files changed, 52 insertions(+), 43 deletions(-)

diff --git a/docs/setup/mesos.md b/docs/setup/mesos.md
index 4c72fbcc461bd..674aa92c10437 100644
--- a/docs/setup/mesos.md
+++ b/docs/setup/mesos.md
@@ -263,3 +263,7 @@ May be set to -1 to disable this feature.
 `mesos.resourcemanager.tasks.container.image.name`: Image name to use for the container (**NO DEFAULT**)
 
 `mesos.resourcemanager.tasks.container.volumes`: A comma seperated list of [host_path:]container_path[:RO|RW]. This allows for mounting additional volumes into your container. (**NO DEFAULT**)
+
+`mesos.resourcemanager.tasks.hostname`: Optional value to define the TaskManager's hostname. The pattern `_TASK_` is replaced by the actual id of the Mesos task. This can be used to configure the TaskManager to use Mesos DNS (e.g. `_TASK_.flink-service.mesos`) for name lookups. (**NO DEFAULT**)
+
+`mesos.resourcemanager.tasks.bootstrap-cmd`: A command which is executed before the TaskManager is started (**NO DEFAULT**).
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index cafef2e528f97..975a3d49cf052 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -1286,10 +1286,6 @@ public final class ConfigConstants {
 
 	public static final String DEFAULT_MESOS_RESOURCEMANAGER_FRAMEWORK_USER = "";
 
-	public static final String MESOS_RESOURCEMANAGER_TASKS_HOSTNAME = "mesos.resourcemanager.tasks.hostname";
-
-	public static final String MESOS_RESOURCEMANAGER_TASKS_BOOTSTRAP_CMD = "mesos.resourcemanager.tasks.cmd-prefix";
-
 	/** Default value to override SSL support for the Artifact Server */
 	public static final boolean DEFAULT_MESOS_ARTIFACT_SERVER_SSL_ENABLED = true;
 
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
index 3e66a5d6fa352..04a406fa4bff2 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
@@ -34,11 +34,13 @@
 import org.apache.mesos.Protos;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import scala.Option;
 
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
 
 import static org.apache.flink.mesos.Utils.variable;
 import static org.apache.flink.mesos.Utils.range;
@@ -200,9 +202,15 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assig
 		final StringBuilder jvmArgs = new StringBuilder();
 
 		//configure task manager hostname property if hostname override property is supplied
-		if(params.getTaskManagerHostname().isDefined()) {
-			final String taskManagerHostName = params.getTaskManagerHostname().get().replace("_TASK",taskID.getValue());
-			dynamicProperties.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, taskManagerHostName);
+		Option<String> taskManagerHostnameOption = params.getTaskManagerHostname();
+
+		if(taskManagerHostnameOption.isDefined()) {
+			// replace the TASK_ID pattern by the actual task id value of the Mesos task
+			final String taskManagerHostname = MesosTaskManagerParameters.TASK_ID_PATTERN
+				.matcher(taskManagerHostnameOption.get())
+				.replaceAll(Matcher.quoteReplacement(taskID.getValue()));
+
+			dynamicProperties.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, taskManagerHostname);
 		}
 
 		// use the assigned ports for the TM
@@ -251,13 +259,12 @@ public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assig
 		env.addVariables(variable(MesosConfigKeys.ENV_FRAMEWORK_NAME, mesosConfiguration.frameworkInfo().getName()));
 
 		// build the launch command w/ dynamic application properties
-		StringBuilder launchCommand = new StringBuilder();
-		if(params.bootstrapCommand().isDefined()) {
-			launchCommand.append(params.bootstrapCommand().get()).append(" && ");
-		}
-		launchCommand.append("$FLINK_HOME/bin/mesos-taskmanager.sh ");
-		launchCommand.append(ContainerSpecification.formatSystemProperties(dynamicProperties));
-		cmd.setValue(launchCommand.toString());
+		Option<String> bootstrapCmdOption = params.bootstrapCommand();
+
+		final String bootstrapCommand = bootstrapCmdOption.isDefined() ? bootstrapCmdOption.get() + " && " : "";
+		final String launchCommand = bootstrapCommand + "$FLINK_HOME/bin/mesos-taskmanager.sh " + ContainerSpecification.formatSystemProperties(dynamicProperties);
+
+		cmd.setValue(launchCommand);
 
 		// build the container info
 		Protos.ContainerInfo.Builder containerInfo = Protos.ContainerInfo.newBuilder();
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
index 1cedcc38b5030..09ef3806f194d 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
@@ -32,6 +32,7 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.mesos.runtime.clusterframework.services.MesosServices;
 import org.apache.flink.mesos.runtime.clusterframework.services.MesosServicesUtils;
@@ -210,18 +211,10 @@ protected int runPrivileged(Configuration config, Configuration dynamicPropertie
 		try {
 			// ------- (1) load and parse / validate all configurations -------
 
-			final String appMasterHostname;
-			//We will use JM RPC address property if it is supplied through configuration
-			final String jmRpcAddress = config.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
-			if(jmRpcAddress != null) {
-				LOG.info("JM RPC address from Flink configuration file: {} ", jmRpcAddress);
-				appMasterHostname = jmRpcAddress;
-			} else {
-				// Note that we use the "appMasterHostname" given by the system, to make sure
-				// we use the hostnames consistently throughout akka.
-				// for akka "localhost" and "localhost.localdomain" are different actors.
-				appMasterHostname = InetAddress.getLocalHost().getHostName();
-			}
+			final String appMasterHostname = config.getString(
+				JobManagerOptions.ADDRESS,
+				InetAddress.getLocalHost().getHostName());
+
 			LOG.info("App Master Hostname to use: {}", appMasterHostname);
 
 			// Mesos configuration
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
index a9ff6cb21adc6..17ffef7577a56 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
@@ -669,8 +669,13 @@ private void taskTerminated(Protos.TaskID taskID, Protos.TaskStatus status) {
 
 	private LaunchableMesosWorker createLaunchableMesosWorker(Protos.TaskID taskID) {
 		LaunchableMesosWorker launchable =
-			new LaunchableMesosWorker(artifactResolver, taskManagerParameters, taskManagerContainerSpec,
-					taskID, mesosConfig);
+			new LaunchableMesosWorker(
+				artifactResolver,
+				taskManagerParameters,
+				taskManagerContainerSpec,
+				taskID,
+				mesosConfig);
+
 		return launchable;
 	}
 
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
index b3b81627c206e..4324469ead4fc 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
@@ -33,6 +33,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import static org.apache.flink.configuration.ConfigOptions.key;
 
@@ -44,33 +45,36 @@
  */
 public class MesosTaskManagerParameters {
 
+	/** Pattern replaced in the {@link #MESOS_TM_HOSTNAME} by the actual task id of the Mesos task */
+	public static final Pattern TASK_ID_PATTERN = Pattern.compile("_TASK_", Pattern.LITERAL);
+
 	public static final ConfigOption<Integer> MESOS_RM_TASKS_SLOTS =
-			key(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS)
-			.defaultValue(1);
+		key(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS)
+		.defaultValue(1);
 
 	public static final ConfigOption<Integer> MESOS_RM_TASKS_MEMORY_MB =
-			key("mesos.resourcemanager.tasks.mem")
-			.defaultValue(1024);
+		key("mesos.resourcemanager.tasks.mem")
+		.defaultValue(1024);
 
 	public static final ConfigOption<Double> MESOS_RM_TASKS_CPUS =
-			key("mesos.resourcemanager.tasks.cpus")
-			.defaultValue(0.0);
+		key("mesos.resourcemanager.tasks.cpus")
+		.defaultValue(0.0);
 
 	public static final ConfigOption<String> MESOS_RM_CONTAINER_TYPE =
 		key("mesos.resourcemanager.tasks.container.type")
-			.defaultValue("mesos");
+		.defaultValue("mesos");
 
 	public static final ConfigOption<String> MESOS_RM_CONTAINER_IMAGE_NAME =
 		key("mesos.resourcemanager.tasks.container.image.name")
-			.noDefaultValue();
+		.noDefaultValue();
 
 	public static final ConfigOption<String> MESOS_TM_HOSTNAME =
-			key(ConfigConstants.MESOS_RESOURCEMANAGER_TASKS_HOSTNAME)
-			.noDefaultValue();
+		key("mesos.resourcemanager.tasks.hostname")
+		.noDefaultValue();
 
 	public static final ConfigOption<String> MESOS_TM_BOOTSTRAP_CMD =
-			key(ConfigConstants.MESOS_RESOURCEMANAGER_TASKS_BOOTSTRAP_CMD)
-			.noDefaultValue();
+		key("mesos.resourcemanager.tasks.bootstrap-cmd")
+		.noDefaultValue();
 	
 	public static final ConfigOption<String> MESOS_RM_CONTAINER_VOLUMES =
 		key("mesos.resourcemanager.tasks.container.volumes")
@@ -78,7 +82,7 @@ public class MesosTaskManagerParameters {
 	
 	public static final ConfigOption<String> MESOS_CONSTRAINTS_HARD_HOSTATTR =
 		key("mesos.constraints.hard.hostattribute")
-			.noDefaultValue();
+		.noDefaultValue();
 
 	/**
 	 * Value for {@code MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE} setting. Tells to use the Mesos containerizer.
@@ -237,7 +241,7 @@ public static MesosTaskManagerParameters create(Configuration flinkConfig) {
 		List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);
 
 		//obtain Task Manager Host Name from the configuration
-		Option<String> tmHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));
+		Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));
 
 		//obtain bootstrap command from the configuration
 		Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));
@@ -250,7 +254,7 @@ public static MesosTaskManagerParameters create(Configuration flinkConfig) {
 			containerVolumes,
 			constraints,
 			tmBootstrapCommand,
-			tmHostname);
+			taskManagerHostname);
 	}
 
 	private static List<ConstraintEvaluator> parseConstraints(String mesosConstraints) {

From da19ceba34c3e0b0d8549fdcb39027ce92136c29 Mon Sep 17 00:00:00 2001
From: Dawid Wysakowicz <dawid@getindata.com>
Date: Fri, 28 Apr 2017 15:24:37 +0200
Subject: [PATCH 151/253] [FLINK-3320] Add NOT pattern support to CEP's pattern
 API

NOT patterns are not yet supported when an OPTIONAL
pattern directly preceeds a NOT. In these cases, an
exception is thrown that proposes an alternative
(but not the most efficient) way to support these
patterns.
---
 .../flink/cep/scala/pattern/Pattern.scala     |   24 +
 .../flink/cep/nfa/ComputationState.java       |    4 +
 .../java/org/apache/flink/cep/nfa/NFA.java    |   62 +-
 .../java/org/apache/flink/cep/nfa/State.java  |   13 +-
 .../flink/cep/nfa/compiler/NFACompiler.java   |  321 +++--
 .../AbstractKeyedCEPPatternOperator.java      |    1 -
 .../org/apache/flink/cep/pattern/Pattern.java |   42 +-
 .../apache/flink/cep/pattern/Quantifier.java  |   12 +-
 .../org/apache/flink/cep/nfa/NFAITCase.java   | 1056 ++++++++++++++++-
 .../cep/nfa/compiler/NFACompilerTest.java     |   35 +
 .../apache/flink/cep/pattern/PatternTest.java |   76 +-
 11 files changed, 1497 insertions(+), 149 deletions(-)

diff --git a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
index 65b7ab0a31ce0..fe7a30ccdd45e 100644
--- a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
+++ b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/pattern/Pattern.scala
@@ -200,6 +200,17 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
     Pattern[T, T](jPattern.next(name))
   }
 
+  /**
+    * Appends a new pattern to the existing one. The new pattern enforces that there is no event
+    * matching this pattern right after the preceding matched event.
+    *
+    * @param name Name of the new pattern
+    * @return A new pattern which is appended to this one
+    */
+  def notNext(name: String): Pattern[T, T] = {
+    Pattern[T, T](jPattern.notNext(name))
+  }
+
   /**
     * Appends a new pattern to the existing one. The new pattern enforces non-strict
     * temporal contiguity. This means that a matching event of this pattern and the
@@ -212,6 +223,19 @@ class Pattern[T , F <: T](jPattern: JPattern[T, F]) {
     Pattern[T, T](jPattern.followedBy(name))
   }
 
+  /**
+    * Appends a new pattern to the existing one. The new pattern enforces that there is no event
+    * matching this pattern between the preceding pattern and succeeding this one.
+    *
+    * NOTE: There has to be other pattern after this one.
+    *
+    * @param name Name of the new pattern
+    * @return A new pattern which is appended to this one
+    */
+  def notFollowedBy(name : String) {
+    Pattern[T, T](jPattern.notFollowedBy(name))
+  }
+
   /**
     * Appends a new pattern to the existing one. The new pattern enforces non-strict
     * temporal contiguity. This means that a matching event of this pattern and the
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/ComputationState.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/ComputationState.java
index 80227fc99f77b..08b9b78f44018 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/ComputationState.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/ComputationState.java
@@ -127,6 +127,10 @@ public static <T> ComputationState<T> createState(
 		return new ComputationState<>(nfa, currentState, previousState, event, timestamp, version, startTimestamp);
 	}
 
+	public boolean isStopState() {
+		return state.isStop();
+	}
+
 	/**
 	 * The context used when evaluating this computation state.
 	 */
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java
index 98c1fc9541ed9..b8c4e65c223c2 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java
@@ -21,21 +21,6 @@
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.LinkedHashMultimap;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
-import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
-import org.apache.flink.cep.NonDuplicatingTypeSerializer;
-import org.apache.flink.cep.nfa.compiler.NFACompiler;
-import org.apache.flink.cep.pattern.conditions.IterativeCondition;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.util.Preconditions;
-
-import javax.annotation.Nullable;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -58,6 +43,20 @@
 import java.util.Stack;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
+import org.apache.flink.cep.NonDuplicatingTypeSerializer;
+import org.apache.flink.cep.nfa.compiler.NFACompiler;
+import org.apache.flink.cep.pattern.conditions.IterativeCondition;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.util.Preconditions;
 
 /**
  * Non-deterministic finite automaton implementation.
@@ -180,6 +179,9 @@ public boolean isEmpty() {
 	 * resulting event sequences are returned. If computations time out and timeout handling is
 	 * activated, then the timed out event patterns are returned.
 	 *
+	 * <p>If computations reach a stop state, the path forward is discarded and currently constructed path is returned
+	 * with the element that resulted in the stop state.
+	 *
 	 * @param event The current event to be processed or null if only pruning shall be done
 	 * @param timestamp The timestamp of the current event
 	 * @return Tuple of the collection of matched patterns (e.g. the result of computations which have
@@ -222,7 +224,12 @@ public Tuple2<Collection<Map<String, T>>, Collection<Tuple2<Map<String, T>, Long
 				newComputationStates = Collections.singleton(computationState);
 			}
 
-			for (ComputationState<T> newComputationState: newComputationStates) {
+
+			//delay adding new computation states in case a stop state is reached and we discard the path.
+			final Collection<ComputationState<T>> statesToRetain = new ArrayList<>();
+			//if stop state reached in this path
+			boolean shouldDiscardPath = false;
+			for (final ComputationState<T> newComputationState: newComputationStates) {
 				if (newComputationState.isFinalState()) {
 					// we've reached a final state and can thus retrieve the matching event sequence
 					Collection<Map<String, T>> matches = extractPatternMatches(newComputationState);
@@ -233,11 +240,32 @@ public Tuple2<Collection<Map<String, T>>, Collection<Tuple2<Map<String, T>, Long
 							newComputationState.getPreviousState().getName(),
 							newComputationState.getEvent(),
 							newComputationState.getTimestamp());
+				} else if (newComputationState.isStopState()) {
+					//reached stop state. release entry for the stop state
+					shouldDiscardPath = true;
+					stringSharedBuffer.release(
+						newComputationState.getPreviousState().getName(),
+						newComputationState.getEvent(),
+						newComputationState.getTimestamp());
 				} else {
 					// add new computation state; it will be processed once the next event arrives
-					computationStates.add(newComputationState);
+					statesToRetain.add(newComputationState);
 				}
 			}
+
+			if (shouldDiscardPath) {
+				// a stop state was reached in this branch. release branch which results in removing previous event from
+				// the buffer
+				for (final ComputationState<T> state : statesToRetain) {
+					stringSharedBuffer.release(
+						state.getPreviousState().getName(),
+						state.getEvent(),
+						state.getTimestamp());
+				}
+			} else {
+				computationStates.addAll(statesToRetain);
+			}
+
 		}
 
 		// prune shared buffer based on window length
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/State.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/State.java
index 2503ffdc89cea..275266b1e4fa9 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/State.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/State.java
@@ -51,6 +51,10 @@ public State(final String name, final StateType stateType) {
 		stateTransitions = new ArrayList<>();
 	}
 
+	public StateType getStateType() {
+		return stateType;
+	}
+
 	public boolean isFinal() {
 		return stateType == StateType.Final;
 	}
@@ -69,7 +73,7 @@ public void makeStart() {
 		this.stateType = StateType.Start;
 	}
 
-	private void addStateTransition(
+	public void addStateTransition(
 			final StateTransitionAction action,
 			final State<T> targetState,
 			final IterativeCondition<T> condition) {
@@ -130,13 +134,18 @@ public int hashCode() {
 		return Objects.hash(name, stateType, stateTransitions);
 	}
 
+	public boolean isStop() {
+		return stateType == StateType.Stop;
+	}
+
 	/**
 	 * Set of valid state types.
 	 */
 	public enum StateType {
 		Start, // the state is a starting state for the NFA
 		Final, // the state is a final state for the NFA
-		Normal // the state is neither a start nor a final state
+		Normal, // the state is neither a start nor a final state
+		Stop
 	}
 
 	private void readObject(ObjectInputStream ois) throws IOException, ClassNotFoundException {
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
index 0ca0e14c52e79..39c18b93fe279 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
@@ -32,6 +32,7 @@
 import javax.annotation.Nullable;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.cep.nfa.NFA;
 import org.apache.flink.cep.nfa.State;
 import org.apache.flink.cep.nfa.StateTransition;
@@ -101,13 +102,15 @@ public static <T> NFAFactory<T> compileFactory(
 	 *
 	 * @param <T>
 	 */
-	private static class NFAFactoryCompiler<T> {
+	static class NFAFactoryCompiler<T> {
 
 		private final Set<String> usedNames = new HashSet<>();
+		private final Map<String, State<T>> stopStates = new HashMap<>();
 		private final List<State<T>> states = new ArrayList<>();
 
 		private long windowTime = 0;
 		private Pattern<T, ?> currentPattern;
+		private Pattern<T, ?> followingPattern;
 
 		NFAFactoryCompiler(final Pattern<T, ?> pattern) {
 			this.currentPattern = pattern;
@@ -118,6 +121,10 @@ private static class NFAFactoryCompiler<T> {
 		 * multiple NFAs.
 		 */
 		void compileFactory() {
+			if (currentPattern.getQuantifier().getConsumingStrategy() == Quantifier.ConsumingStrategy.NOT_FOLLOW) {
+				throw new MalformedPatternException("NotFollowedBy is not supported as a last part of a Pattern!");
+			}
+
 			// we're traversing the pattern from the end to the beginning --> the first state is the final state
 			State<T> sinkState = createEndingState();
 			// add all the normal states
@@ -134,14 +141,46 @@ long getWindowTime() {
 			return windowTime;
 		}
 
+		/**
+		 * Retrieves list of conditions resulting in Stop state and names of the corresponding NOT patterns.
+		 *
+		 * <p>A current not condition can be produced in two cases:
+		 * <ol>
+		 *     <li>the previous pattern is a {@link Quantifier.ConsumingStrategy#NOT_FOLLOW}</li>
+		 *     <li>exists a backward path of {@link Quantifier.QuantifierProperty#OPTIONAL} patterns to
+		 *       {@link Quantifier.ConsumingStrategy#NOT_FOLLOW}</li>
+		 * </ol>
+		 *
+		 * <p><b>WARNING:</b> for more info on the second case see: {@link NFAFactoryCompiler#copyWithoutTransitiveNots(State)}
+		 *
+		 * @return list of not conditions with corresponding names
+		 */
+		private List<Tuple2<IterativeCondition<T>, String>> getCurrentNotCondition() {
+			List<Tuple2<IterativeCondition<T>, String>> notConditions = new ArrayList<>();
+
+			Pattern<T, ? extends T> previousPattern = currentPattern;
+			while (previousPattern.getPrevious() != null && (
+				previousPattern.getPrevious().getQuantifier().hasProperty(Quantifier.QuantifierProperty.OPTIONAL) ||
+				previousPattern.getPrevious().getQuantifier().getConsumingStrategy() == Quantifier.ConsumingStrategy.NOT_FOLLOW)) {
+
+				previousPattern = previousPattern.getPrevious();
+
+				if (previousPattern.getQuantifier().getConsumingStrategy() == Quantifier.ConsumingStrategy.NOT_FOLLOW) {
+					final IterativeCondition<T> notCondition = (IterativeCondition<T>) previousPattern.getCondition();
+					notConditions.add(Tuple2.of(notCondition, previousPattern.getName()));
+				}
+			}
+			return notConditions;
+		}
+
 		/**
 		 * Creates the dummy Final {@link State} of the NFA graph.
 		 * @return dummy Final state
 		 */
 		private State<T> createEndingState() {
+			checkPatternNameUniqueness(ENDING_STATE_NAME);
 			State<T> endState = new State<>(ENDING_STATE_NAME, State.StateType.Final);
 			states.add(endState);
-			usedNames.add(ENDING_STATE_NAME);
 
 			windowTime = currentPattern.getWindowTime() != null ? currentPattern.getWindowTime().toMilliseconds() : 0L;
 			return endState;
@@ -154,10 +193,30 @@ private State<T> createEndingState() {
 		 * @return the next state after Start in the resulting graph
 		 */
 		private State<T> createMiddleStates(final State<T> sinkState) {
-
 			State<T> lastSink = sinkState;
 			while (currentPattern.getPrevious() != null) {
-				lastSink = convertPattern(lastSink);
+
+				if (currentPattern.getQuantifier().getConsumingStrategy() == Quantifier.ConsumingStrategy.NOT_FOLLOW) {
+					//skip notFollow patterns, they are converted into edge conditions
+				} else if (currentPattern.getQuantifier().getConsumingStrategy() == Quantifier.ConsumingStrategy.NOT_NEXT) {
+					final State<T> notNext = createNormalState();
+					final IterativeCondition<T> notCondition = (IterativeCondition<T>) currentPattern.getCondition();
+					final State<T> stopState = createStopState(notCondition, currentPattern.getName());
+
+					if (lastSink.isFinal()) {
+						//so that the proceed to final is not fired
+						notNext.addIgnore(lastSink, new NotCondition<>(notCondition));
+					} else {
+						notNext.addProceed(lastSink, new NotCondition<>(notCondition));
+					}
+					notNext.addProceed(stopState, notCondition);
+					lastSink = notNext;
+				} else {
+					lastSink = convertPattern(lastSink);
+				}
+
+				// we traverse the pattern graph backwards
+				followingPattern = currentPattern;
 				currentPattern = currentPattern.getPrevious();
 
 				final Time currentWindowTime = currentPattern.getWindowTime();
@@ -166,79 +225,152 @@ private State<T> createMiddleStates(final State<T> sinkState) {
 					windowTime = currentWindowTime.toMilliseconds();
 				}
 			}
-
 			return lastSink;
 		}
 
+		/**
+		 * Creates the Start {@link State} of the resulting NFA graph.
+		 *
+		 * @param sinkState the state that Start state should point to (always first state of middle states)
+		 * @return created state
+		 */
+		@SuppressWarnings("unchecked")
+		private State<T> createStartState(State<T> sinkState) {
+			final State<T> beginningState = convertPattern(sinkState);
+			beginningState.makeStart();
+			return beginningState;
+		}
+
 		private State<T> convertPattern(final State<T> sinkState) {
 			final State<T> lastSink;
-			checkPatternNameUniqueness();
-			usedNames.add(currentPattern.getName());
+			checkPatternNameUniqueness(currentPattern.getName());
 
 			final Quantifier quantifier = currentPattern.getQuantifier();
 			if (quantifier.hasProperty(Quantifier.QuantifierProperty.LOOPING)) {
-				final State<T> looping = createLooping(sinkState);
+
+				// if loop has started then all notPatterns previous to the optional states are no longer valid
+				final State<T> sink = copyWithoutTransitiveNots(sinkState);
+				final State<T> looping = createLooping(sink);
 
 				if (!quantifier.hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
-					lastSink = createFirstMandatoryStateOfLoop(looping);
+					lastSink = createInitMandatoryStateOfOneOrMore(looping);
 				} else {
-					lastSink = createWaitingStateForZeroOrMore(looping, sinkState);
+					lastSink = createInitOptionalStateOfZeroOrMore(looping, sinkState);
 				}
 			} else if (quantifier.hasProperty(Quantifier.QuantifierProperty.TIMES)) {
 				lastSink = createTimesState(sinkState, currentPattern.getTimes());
 			} else {
 				lastSink = createSingletonState(sinkState);
 			}
+			addStopStates(lastSink);
 
 			return lastSink;
 		}
 
 		/**
-		 * Creates a pair of states that enables relaxed strictness before a zeroOrMore looping state.
+		 * Creates a state with {@link State.StateType#Normal} and adds it to the collection of created states.
+		 * Should be used instead of instantiating with new operator.
 		 *
-		 * @param loopingState the first state of zeroOrMore complex state
-		 * @param lastSink     the state that the looping one points to
-		 * @return the newly created state
+		 * @return the created state
 		 */
-		@SuppressWarnings("unchecked")
-		private State<T> createWaitingStateForZeroOrMore(final State<T> loopingState, final State<T> lastSink) {
-			final IterativeCondition<T> currentFunction = (IterativeCondition<T>)currentPattern.getCondition();
+		private State<T> createNormalState() {
+			final State<T> state = new State<>(currentPattern.getName(), State.StateType.Normal);
+			states.add(state);
+			return state;
+		}
 
-			final State<T> followByState = createNormalState();
-			followByState.addProceed(lastSink, BooleanConditions.<T>trueFunction());
-			followByState.addTake(loopingState, currentFunction);
+		private State<T> createStopState(final IterativeCondition<T> notCondition, final String name) {
+			// We should not duplicate the notStates. All states from which we can stop should point to the same one.
+			State<T> stopState = stopStates.get(name);
+			if (stopState == null) {
+				stopState = new State<>(name, State.StateType.Stop);
+				states.add(stopState);
+				stopState.addTake(notCondition);
+				stopStates.put(name, stopState);
+			}
+			return stopState;
+		}
 
-			final IterativeCondition<T> ignoreFunction = getIgnoreCondition(currentPattern);
-			if (ignoreFunction != null) {
-				final State<T> followByStateWithoutProceed = createNormalState();
-				followByState.addIgnore(followByStateWithoutProceed, ignoreFunction);
-				followByStateWithoutProceed.addIgnore(ignoreFunction);
-				followByStateWithoutProceed.addTake(loopingState, currentFunction);
+		/**
+		 * This method creates an alternative state that is target for TAKE transition from an optional State.
+		 * Accepting an event in optional State discards all not Patterns that were present before it.
+		 *
+		 * <p>E.g for a Pattern begin("a").notFollowedBy("b").followedByAny("c").optional().followedByAny("d")
+		 * a sequence like : {a c b d} is a valid match, but {a b d} is not.
+		 *
+		 * <p><b>NOTICE:</b> This method creates copy only if it necessary.
+		 *
+		 * @param sinkState a state to create copy without transitive nots
+		 * @return the copy of the state itself if no modifications were needed
+		 */
+		private State<T> copyWithoutTransitiveNots(final State<T> sinkState) {
+			final List<Tuple2<IterativeCondition<T>, String>> currentNotCondition = getCurrentNotCondition();
+
+			if (currentNotCondition.isEmpty() ||
+				!currentPattern.getQuantifier().hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
+				//we do not create an alternative path if we are NOT in an OPTIONAL state or there is no NOTs prior to
+				//the optional state
+				return sinkState;
 			}
 
-			return followByState;
+			final State<T> copyOfSink = new State<>(sinkState.getName(), sinkState.getStateType());
+			states.add(copyOfSink);
+
+			for (StateTransition<T> tStateTransition : sinkState.getStateTransitions()) {
+
+				if (tStateTransition.getAction() == StateTransitionAction.PROCEED) {
+					State<T> targetState = tStateTransition.getTargetState();
+					boolean remove = false;
+					if (targetState.isStop()) {
+						for (Tuple2<IterativeCondition<T>, String> notCondition : currentNotCondition) {
+							if (targetState.getName().equals(notCondition.f1)) {
+								remove = true;
+							}
+						}
+					} else {
+						targetState = copyWithoutTransitiveNots(tStateTransition.getTargetState());
+					}
+
+					if (!remove) {
+						copyOfSink.addStateTransition(tStateTransition.getAction(), targetState, tStateTransition.getCondition());
+					}
+				} else {
+					copyOfSink.addStateTransition(
+							tStateTransition.getAction(),
+							tStateTransition.getTargetState().equals(tStateTransition.getSourceState())
+									? copyOfSink
+									: tStateTransition.getTargetState(),
+							tStateTransition.getCondition()
+					);
+				}
+
+			}
+			return copyOfSink;
 		}
 
-		private void checkPatternNameUniqueness() {
-			if (usedNames.contains(currentPattern.getName())) {
-				throw new MalformedPatternException(
-					"Duplicate pattern name: " + currentPattern.getName() + ". " +
-					"Pattern names must be unique.");
+		private void addStopStates(final State<T> state) {
+			for (Tuple2<IterativeCondition<T>, String> notCondition: getCurrentNotCondition()) {
+				final State<T> stopState = createStopState(notCondition.f0, notCondition.f1);
+				state.addProceed(stopState, notCondition.f0);
 			}
 		}
 
-		/**
-		 * Creates the Start {@link State} of the resulting NFA graph.
-		 *
-		 * @param sinkState the state that Start state should point to (always first state of middle states)
-		 * @return created state
-		 */
-		@SuppressWarnings("unchecked")
-		private State<T> createStartState(State<T> sinkState) {
-			final State<T> beginningState = convertPattern(sinkState);
-			beginningState.makeStart();
+		private void addStopStateToLooping(final State<T> loopingState) {
+			if (followingPattern != null &&
+					followingPattern.getQuantifier().getConsumingStrategy() == Quantifier.ConsumingStrategy.NOT_FOLLOW) {
+				final IterativeCondition<T> notCondition = (IterativeCondition<T>) followingPattern.getCondition();
+				final State<T> stopState = createStopState(notCondition, followingPattern.getName());
+				loopingState.addProceed(stopState, notCondition);
+			}
+		}
 
-			return beginningState;
+		private void checkPatternNameUniqueness(String patternName) {
+			if (usedNames.contains(currentPattern.getName())) {
+				throw new MalformedPatternException(
+					"Duplicate pattern name: " + patternName + ". " +
+					"Pattern names must be unique.");
+			}
+			usedNames.add(patternName);
 		}
 
 		/**
@@ -250,12 +382,13 @@ private State<T> createStartState(State<T> sinkState) {
 		 * @return the first state of the "complex" state, next state should point to it
 		 */
 		private State<T> createTimesState(final State<T> sinkState, int times) {
-			State<T> lastSink = sinkState;
+			State<T> lastSink = copyWithoutTransitiveNots(sinkState);
 			for (int i = 0; i < times - 1; i++) {
 				lastSink = createSingletonState(lastSink, getInnerIgnoreCondition(currentPattern), false);
+				addStopStateToLooping(lastSink);
 			}
 
-			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
+			final IterativeCondition<T> currentCondition = (IterativeCondition<T>) currentPattern.getCondition();
 			final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
 
 			// we created the intermediate states in the loop, now we create the start of the loop.
@@ -264,14 +397,15 @@ private State<T> createTimesState(final State<T> sinkState, int times) {
 			}
 
 			final State<T> singletonState = createNormalState();
-			singletonState.addTake(lastSink, currentFilterFunction);
+			singletonState.addTake(lastSink, currentCondition);
 			singletonState.addProceed(sinkState, BooleanConditions.<T>trueFunction());
 
 			if (ignoreCondition != null) {
 				State<T> ignoreState = createNormalState();
-				ignoreState.addTake(lastSink, currentFilterFunction);
+				ignoreState.addTake(lastSink, currentCondition);
 				ignoreState.addIgnore(ignoreCondition);
 				singletonState.addIgnore(ignoreState, ignoreCondition);
+				addStopStates(ignoreState);
 			}
 			return singletonState;
 		}
@@ -303,13 +437,16 @@ private State<T> createSingletonState(final State<T> sinkState) {
 		 */
 		@SuppressWarnings("unchecked")
 		private State<T> createSingletonState(final State<T> sinkState, final IterativeCondition<T> ignoreCondition, final boolean isOptional) {
-			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
+			final IterativeCondition<T> currentCondition = (IterativeCondition<T>) currentPattern.getCondition();
 			final IterativeCondition<T> trueFunction = BooleanConditions.trueFunction();
 
 			final State<T> singletonState = createNormalState();
-			singletonState.addTake(sinkState, currentFilterFunction);
+			// if event is accepted then all notPatterns previous to the optional states are no longer valid
+			final State<T> sink = copyWithoutTransitiveNots(sinkState);
+			singletonState.addTake(sink, currentCondition);
 
 			if (isOptional) {
+				// if no element accepted the previous nots are still valid.
 				singletonState.addProceed(sinkState, trueFunction);
 			}
 
@@ -317,7 +454,9 @@ private State<T> createSingletonState(final State<T> sinkState, final IterativeC
 				final State<T> ignoreState;
 				if (isOptional) {
 					ignoreState = createNormalState();
-					ignoreState.addTake(sinkState, currentFilterFunction);
+					ignoreState.addTake(sink, currentCondition);
+					ignoreState.addIgnore(ignoreCondition);
+					addStopStates(ignoreState);
 				} else {
 					ignoreState = singletonState;
 				}
@@ -326,27 +465,6 @@ private State<T> createSingletonState(final State<T> sinkState, final IterativeC
 			return singletonState;
 		}
 
-		/**
-		 * Patterns with quantifiers AT_LEAST_ONE_* are created as a pair of states: a singleton state and
-		 * looping state. This method creates the first of the two.
-		 *
-		 * @param sinkState the state the newly created state should point to, it should be a looping state
-		 * @return the newly created state
-		 */
-		@SuppressWarnings("unchecked")
-		private State<T> createFirstMandatoryStateOfLoop(final State<T> sinkState) {
-
-			final IterativeCondition<T> currentFilterFunction = (IterativeCondition<T>) currentPattern.getCondition();
-			final State<T> firstState = createNormalState();
-
-			firstState.addTake(sinkState, currentFilterFunction);
-			final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
-			if (ignoreCondition != null) {
-				firstState.addIgnore(ignoreCondition);
-			}
-			return firstState;
-		}
-
 		/**
 		 * Creates the given state as a looping one. Looping state is one with TAKE edge to itself and
 		 * PROCEED edge to the sinkState. It also consists of a similar state without the PROCEED edge, so that
@@ -357,34 +475,73 @@ private State<T> createFirstMandatoryStateOfLoop(final State<T> sinkState) {
 		 */
 		@SuppressWarnings("unchecked")
 		private State<T> createLooping(final State<T> sinkState) {
-			final IterativeCondition<T> filterFunction = (IterativeCondition<T>) currentPattern.getCondition();
+			final IterativeCondition<T> currentCondition = (IterativeCondition<T>) currentPattern.getCondition();
 			final IterativeCondition<T> ignoreCondition = getInnerIgnoreCondition(currentPattern);
 			final IterativeCondition<T> trueFunction = BooleanConditions.trueFunction();
 
 			final State<T> loopingState = createNormalState();
 			loopingState.addProceed(sinkState, trueFunction);
-			loopingState.addTake(filterFunction);
+			loopingState.addTake(currentCondition);
+
+			addStopStateToLooping(loopingState);
 
 			if (ignoreCondition != null) {
 				final State<T> ignoreState = createNormalState();
-				ignoreState.addTake(loopingState, filterFunction);
+				ignoreState.addTake(loopingState, currentCondition);
 				ignoreState.addIgnore(ignoreCondition);
 				loopingState.addIgnore(ignoreState, ignoreCondition);
-			}
 
+				addStopStateToLooping(ignoreState);
+			}
 			return loopingState;
 		}
 
 		/**
-		 * Creates a state with {@link State.StateType#Normal} and adds it to the collection of created states.
-		 * Should be used instead of instantiating with new operator.
+		 * Patterns with quantifiers AT_LEAST_ONE_* are created as a pair of states: a singleton state and
+		 * looping state. This method creates the first of the two.
 		 *
-		 * @return the created state
+		 * @param sinkState the state the newly created state should point to, it should be a looping state
+		 * @return the newly created state
 		 */
-		private State<T> createNormalState() {
-			final State<T> state = new State<>(currentPattern.getName(), State.StateType.Normal);
-			states.add(state);
-			return state;
+		@SuppressWarnings("unchecked")
+		private State<T> createInitMandatoryStateOfOneOrMore(final State<T> sinkState) {
+			final IterativeCondition<T> currentCondition = (IterativeCondition<T>) currentPattern.getCondition();
+
+			final State<T> firstState = createNormalState();
+			firstState.addTake(sinkState, currentCondition);
+
+			final IterativeCondition<T> ignoreCondition = getIgnoreCondition(currentPattern);
+			if (ignoreCondition != null) {
+				firstState.addIgnore(ignoreCondition);
+			}
+			return firstState;
+		}
+
+		/**
+		 * Creates a pair of states that enables relaxed strictness before a zeroOrMore looping state.
+		 *
+		 * @param loopingState the first state of zeroOrMore complex state
+		 * @param lastSink     the state that the looping one points to
+		 * @return the newly created state
+		 */
+		@SuppressWarnings("unchecked")
+		private State<T> createInitOptionalStateOfZeroOrMore(final State<T> loopingState, final State<T> lastSink) {
+			final IterativeCondition<T> currentCondition = (IterativeCondition<T>) currentPattern.getCondition();
+
+			final State<T> firstState = createNormalState();
+			firstState.addProceed(lastSink, BooleanConditions.<T>trueFunction());
+			firstState.addTake(loopingState, currentCondition);
+
+			final IterativeCondition<T> ignoreFunction = getIgnoreCondition(currentPattern);
+			if (ignoreFunction != null) {
+				final State<T> firstStateWithoutProceed = createNormalState();
+				firstState.addIgnore(firstStateWithoutProceed, ignoreFunction);
+				firstStateWithoutProceed.addIgnore(ignoreFunction);
+				firstStateWithoutProceed.addTake(loopingState, currentCondition);
+
+				addStopStates(firstStateWithoutProceed);
+			}
+			return firstState;
 		}
 
 		/**
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
index cd62c0d4de029..b6374cdc441cc 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
@@ -218,7 +218,6 @@ private void saveRegisterWatermarkTimer() {
 		}
 	}
 
-	long count = 0;
 	@Override
 	public void onEventTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {
 
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
index b100bc5f80135..3cf25ef783f9d 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
@@ -210,7 +210,18 @@ public Pattern<T, F> within(Time windowTime) {
 	 * @return A new pattern which is appended to this one
 	 */
 	public Pattern<T, T> next(final String name) {
-		return new Pattern<T, T>(name, this, ConsumingStrategy.STRICT);
+		return new Pattern<>(name, this, ConsumingStrategy.STRICT);
+	}
+
+	/**
+	 * Appends a new pattern to the existing one. The new pattern enforces that there is no event matching this pattern
+	 * right after the preceding matched event.
+	 *
+	 * @param name Name of the new pattern
+	 * @return A new pattern which is appended to this one
+	 */
+	public Pattern<T, T> notNext(final String name) {
+		return new Pattern<>(name, this, ConsumingStrategy.NOT_NEXT);
 	}
 
 	/**
@@ -225,6 +236,26 @@ public Pattern<T, T> followedBy(final String name) {
 		return new Pattern<>(name, this, ConsumingStrategy.SKIP_TILL_NEXT);
 	}
 
+	/**
+	 * Appends a new pattern to the existing one. The new pattern enforces that there is no event matching this pattern
+	 * between the preceding pattern and succeeding this one.
+	 *
+	 * <p><b>NOTE:</b> There has to be other pattern after this one.
+	 *
+	 * @param name Name of the new pattern
+	 * @return A new pattern which is appended to this one
+	 */
+	public Pattern<T, T> notFollowedBy(final String name) {
+		if (quantifier.hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
+			throw new MalformedPatternException(
+				"Specifying a pattern with an optional path to NOT condition is not supported yet. " +
+				"You can simulate such pattern with two independent patterns, one with and the other without " +
+				"the optional part.");
+		}
+
+		return new Pattern<>(name, this, ConsumingStrategy.NOT_FOLLOW);
+	}
+
 	/**
 	 * Appends a new pattern to the existing one. The new pattern enforces non-strict
 	 * temporal contiguity. This means that a matching event of this pattern and the
@@ -263,6 +294,7 @@ public Pattern<T, F> optional() {
 	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
 	public Pattern<T, F> oneOrMore() {
+		checkIfNoNotPattern();
 		checkIfQuantifierApplied();
 		this.quantifier = Quantifier.ONE_OR_MORE(quantifier.getConsumingStrategy());
 		return this;
@@ -277,6 +309,7 @@ public Pattern<T, F> oneOrMore() {
 	 * @throws MalformedPatternException if the quantifier is not applicable to this pattern.
 	 */
 	public Pattern<T, F> times(int times) {
+		checkIfNoNotPattern();
 		checkIfQuantifierApplied();
 		Preconditions.checkArgument(times > 0, "You should give a positive number greater than 0.");
 		this.quantifier = Quantifier.TIMES(quantifier.getConsumingStrategy());
@@ -341,6 +374,13 @@ public Pattern<T, F> consecutive() {
 		return this;
 	}
 
+	private void checkIfNoNotPattern() {
+		if (quantifier.getConsumingStrategy() == ConsumingStrategy.NOT_FOLLOW ||
+				quantifier.getConsumingStrategy() == ConsumingStrategy.NOT_NEXT) {
+			throw new MalformedPatternException("Option not applicable to NOT pattern");
+		}
+	}
+
 	private void checkIfQuantifierApplied() {
 		if (!quantifier.hasProperty(Quantifier.QuantifierProperty.SINGLE)) {
 			throw new MalformedPatternException("Already applied quantifier to this Pattern. " +
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
index b0f882c4ef7d5..382c3ba9eeb92 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Quantifier.java
@@ -83,9 +83,10 @@ public void consecutive() {
 	}
 
 	public void optional() {
-		if (hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
-			throw new MalformedPatternException("Optional already applied!");
-		}
+		checkPattern(!hasProperty(QuantifierProperty.OPTIONAL), "Optional already applied!");
+		checkPattern(!(consumingStrategy == ConsumingStrategy.NOT_NEXT ||
+					consumingStrategy == ConsumingStrategy.NOT_FOLLOW), "NOT pattern cannot be optional");
+
 		properties.add(Quantifier.QuantifierProperty.OPTIONAL);
 	}
 
@@ -120,7 +121,10 @@ public enum QuantifierProperty {
 	public enum ConsumingStrategy {
 		STRICT,
 		SKIP_TILL_NEXT,
-		SKIP_TILL_ANY
+		SKIP_TILL_ANY,
+
+		NOT_FOLLOW,
+		NOT_NEXT
 	}
 
 }
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index fe31564af30af..ab6ff82bf0d65 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -393,8 +393,7 @@ public boolean filter(Event value) throws Exception {
 		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), true);
 
 		for (StreamRecord<Event> event: events) {
-			Tuple2<Collection<Map<String, Event>>, Collection<Tuple2<Map<String, Event>, Long>>> patterns =
-				nfa.process(event.getValue(), event.getTimestamp());
+			final Tuple2<Collection<Map<String, Event>>, Collection<Tuple2<Map<String, Event>, Long>>> patterns = nfa.process(event.getValue(), event.getTimestamp());
 
 			Collection<Map<String, Event>> matchedPatterns = patterns.f0;
 			Collection<Tuple2<Map<String, Event>, Long>> timeoutPatterns = patterns.f1;
@@ -786,14 +785,16 @@ public boolean filter(Event value) throws Exception {
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("a");
 			}
-		}).oneOrMore().allowCombinations().optional().followedByAny("middle-second").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional()
+			.followedBy("middle-second").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
 			public boolean filter(Event value) throws Exception {
 				return value.getName().equals("d");
 			}
-		}).oneOrMore().allowCombinations().optional().followedBy("end").where(new SimpleCondition<Event>() {
+		}).oneOrMore().allowCombinations().optional()
+			.followedBy("end").where(new SimpleCondition<Event>() {
 			private static final long serialVersionUID = 5726188262756267490L;
 
 			@Override
@@ -818,13 +819,11 @@ public boolean filter(Event value) throws Exception {
 			}
 		}
 
-		assertEquals(8, allPatterns.size());
+		assertEquals(6, allPatterns.size());
 		assertEquals(Sets.newHashSet(
 			Sets.newHashSet(startEvent, middleEvent1, middleEvent2, middleEvent3, end),
-			Sets.newHashSet(startEvent, middleEvent1, middleEvent3, end),
 			Sets.newHashSet(startEvent, middleEvent1, middleEvent2, end),
 			Sets.newHashSet(startEvent, middleEvent2, middleEvent3, end),
-			Sets.newHashSet(startEvent, middleEvent3, end),
 			Sets.newHashSet(startEvent, middleEvent2, end),
 			Sets.newHashSet(startEvent, middleEvent1, end),
 			Sets.newHashSet(startEvent, end)
@@ -3147,7 +3146,1048 @@ public boolean filter(Event value) throws Exception {
 	}
 
 
-	/////////////////////////////////////////    Utility      ////////////////////////////////
+	/////////////////////////////////////////       Not pattern    /////////////////////////////////////////////////
+
+	@Test
+	public void testNotNext() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event b1 = new Event(42, "b", 3.0);
+		Event c2 = new Event(43, "c", 4.0);
+		Event d = new Event(43, "d", 4.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(b1, 3));
+		inputEvents.add(new StreamRecord<>(c2, 4));
+		inputEvents.add(new StreamRecord<>(d, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notNext("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(a1, c1, d),
+			Lists.newArrayList(a1, c2, d)
+		));
+	}
+
+	@Test
+	public void testNotNextNoMatches() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event b1 = new Event(42, "b", 3.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event c2 = new Event(43, "c", 4.0);
+		Event d = new Event(43, "d", 4.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(b1, 2));
+		inputEvents.add(new StreamRecord<>(c1, 3));
+		inputEvents.add(new StreamRecord<>(c2, 4));
+		inputEvents.add(new StreamRecord<>(d, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notNext("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		assertEquals(0, matches.size());
+	}
+
+	@Test
+	public void testNotNextNoMatchesAtTheEnd() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event c2 = new Event(43, "c", 4.0);
+		Event d = new Event(43, "d", 4.0);
+		Event b1 = new Event(42, "b", 3.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(c2, 3));
+		inputEvents.add(new StreamRecord<>(d, 4));
+		inputEvents.add(new StreamRecord<>(b1, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		}).notNext("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		assertEquals(0, matches.size());
+	}
+
+	@Test
+	public void testNotFollowedBy() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event b1 = new Event(42, "b", 3.0);
+		Event c2 = new Event(43, "c", 4.0);
+		Event d = new Event(43, "d", 4.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(b1, 3));
+		inputEvents.add(new StreamRecord<>(c2, 4));
+		inputEvents.add(new StreamRecord<>(d, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches,Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(a1, c1, d)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByBeforeOptional() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event b1 = new Event(42, "b", 3.0);
+		Event c2 = new Event(43, "c", 4.0);
+		Event d = new Event(43, "d", 4.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(b1, 3));
+		inputEvents.add(new StreamRecord<>(c2, 4));
+		inputEvents.add(new StreamRecord<>(d, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).optional().followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches,Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(a1, c1, d)
+		));
+	}
+
+	@Test
+	public void testTimesWithNotFollowedBy() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event b1 = new Event(41, "b", 2.0);
+		Event c = new Event(42, "c", 3.0);
+		Event b2 = new Event(43, "b", 4.0);
+		Event d = new Event(43, "d", 4.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(b1, 2));
+		inputEvents.add(new StreamRecord<>(c, 3));
+		inputEvents.add(new StreamRecord<>(b2, 4));
+		inputEvents.add(new StreamRecord<>(d, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).times(2).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches,Lists.<List<Event>>newArrayList());
+	}
+
+	@Test
+	public void testIgnoreStateOfTimesWithNotFollowedBy() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event e = new Event(41, "e", 2.0);
+		Event c1 = new Event(42, "c", 3.0);
+		Event b1 = new Event(43, "b", 4.0);
+		Event c2 = new Event(44, "c", 5.0);
+		Event d1 = new Event(45, "d", 6.0);
+		Event d2 = new Event(46, "d", 7.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(d1, 2));
+		inputEvents.add(new StreamRecord<>(e, 1));
+		inputEvents.add(new StreamRecord<>(b1, 3));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(c2, 4));
+		inputEvents.add(new StreamRecord<>(d2, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).times(2).optional().followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(a1, d1)
+		));
+	}
+
+	@Test
+	public void testTimesWithNotFollowedByAfter() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event e = new Event(41, "e", 2.0);
+		Event c1 = new Event(42, "c", 3.0);
+		Event b1 = new Event(43, "b", 4.0);
+		Event b2 = new Event(44, "b", 5.0);
+		Event d1 = new Event(46, "d", 7.0);
+		Event d2 = new Event(47, "d", 8.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(d1, 2));
+		inputEvents.add(new StreamRecord<>(e, 1));
+		inputEvents.add(new StreamRecord<>(b1, 3));
+		inputEvents.add(new StreamRecord<>(b2, 3));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(d2, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).times(2).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList());
+	}
+
+	@Test
+	public void testNotFollowedByBeforeOptionalAtTheEnd() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event b1 = new Event(42, "b", 3.0);
+		Event c2 = new Event(43, "c", 4.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(b1, 3));
+		inputEvents.add(new StreamRecord<>(c2, 4));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedByAny("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).optional();
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches,Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(a1, c1),
+			Lists.newArrayList(a1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByBeforeOptionalTimes() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event b1 = new Event(42, "b", 3.0);
+		Event c2 = new Event(43, "c", 4.0);
+		Event d = new Event(43, "d", 4.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(c1, 2));
+		inputEvents.add(new StreamRecord<>(b1, 3));
+		inputEvents.add(new StreamRecord<>(c2, 4));
+		inputEvents.add(new StreamRecord<>(d, 5));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedByAny("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).times(2).optional().followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches,Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(a1, c1, c2, d)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByWithBranchingAtStart() {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		Event a1 = new Event(40, "a", 1.0);
+		Event b1 = new Event(42, "b", 3.0);
+		Event c1 = new Event(41, "c", 2.0);
+		Event a2 = new Event(41, "a", 4.0);
+		Event c2 = new Event(43, "c", 5.0);
+		Event d = new Event(43, "d", 6.0);
+
+		inputEvents.add(new StreamRecord<>(a1, 1));
+		inputEvents.add(new StreamRecord<>(b1, 2));
+		inputEvents.add(new StreamRecord<>(c1, 3));
+		inputEvents.add(new StreamRecord<>(a2, 4));
+		inputEvents.add(new StreamRecord<>(c2, 5));
+		inputEvents.add(new StreamRecord<>(d, 6));
+
+		Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("a");
+			}
+		}).notFollowedBy("notPattern").where(new SimpleCondition<Event>() {
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).followedBy("middle").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("c");
+			}
+		}).followedBy("end").where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("d");
+			}
+		});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		final List<List<Event>> matches = feedNFA(inputEvents, nfa);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(a2, c2, d)
+		));
+	}
+
+	private static class NotFollowByData {
+		static final Event a1 = new Event(40, "a", 1.0);
+		static final Event b1 = new Event(41, "b", 2.0);
+		static final Event b2 = new Event(42, "b", 3.0);
+		static final Event b3 = new Event(42, "b", 4.0);
+		static final Event c1 = new Event(43, "c", 5.0);
+		static final Event b4 = new Event(42, "b", 6.0);
+		static final Event b5 = new Event(42, "b", 7.0);
+		static final Event b6 = new Event(42, "b", 8.0);
+		static final Event d1 = new Event(43, "d", 9.0);
+
+		private NotFollowByData() {
+		}
+	}
+
+	@Test
+	public void testNotNextAfterZeroOrMoreSkipTillNext() {
+		final List<List<Event>> matches = testNotNextAfterZeroOrMore(false);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotNextAfterZeroOrMoreSkipTillAny() {
+		final List<List<Event>> matches = testNotNextAfterZeroOrMore(true);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b2, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.d1)
+		));
+	}
+
+	private List<List<Event>> testNotNextAfterZeroOrMore(boolean allMatches) {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		int i = 0;
+		inputEvents.add(new StreamRecord<>(NotFollowByData.a1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.c1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b2, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.d1, i++));
+
+		Pattern<Event, ?> pattern = Pattern
+			.<Event>begin("a").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("a");
+				}
+			});
+
+		pattern = (allMatches ? pattern.followedByAny("b*") : pattern.followedBy("b*")).where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).oneOrMore().optional()
+			.notNext("not c").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("c");
+				}
+			})
+			.followedBy("d").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("d");
+				}
+			});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		return feedNFA(inputEvents, nfa);
+	}
+
+	@Test
+	public void testNotNextAfterOneOrMoreSkipTillNext() {
+		final List<List<Event>> matches = testNotNextAfterOneOrMore(false);
+		assertEquals(0, matches.size());
+	}
+
+	@Test
+	public void testNotNextAfterOneOrMoreSkipTillAny() {
+		final List<List<Event>> matches = testNotNextAfterOneOrMore(true);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b2, NotFollowByData.d1)
+		));
+	}
+
+	private List<List<Event>> testNotNextAfterOneOrMore(boolean allMatches) {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		int i = 0;
+		inputEvents.add(new StreamRecord<>(NotFollowByData.a1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.c1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b2, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.d1, i++));
+
+		Pattern<Event, ?> pattern = Pattern
+			.<Event>begin("a").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("a");
+				}
+			});
+
+		pattern = (allMatches ? pattern.followedByAny("b*") : pattern.followedBy("b*")).where(new SimpleCondition<Event>() {
+			private static final long serialVersionUID = 5726188262756267490L;
+
+			@Override
+			public boolean filter(Event value) throws Exception {
+				return value.getName().equals("b");
+			}
+		}).oneOrMore()
+			.notNext("not c").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("c");
+				}
+			})
+			.followedBy("d").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("d");
+				}
+			});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		return feedNFA(inputEvents, nfa);
+	}
+
+	@Test
+	public void testNotFollowedByNextAfterOneOrMoreEager() {
+		final List<List<Event>> matches = testNotFollowedByAfterOneOrMore(true, false);
+		assertEquals(0, matches.size());
+	}
+
+	@Test
+	public void testNotFollowedByAnyAfterOneOrMoreEager() {
+		final List<List<Event>> matches = testNotFollowedByAfterOneOrMore(true, true);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b6, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByNextAfterOneOrMoreCombinations() {
+		final List<List<Event>> matches = testNotFollowedByAfterOneOrMore(false, false);
+		assertEquals(0, matches.size());
+	}
+
+	@Test
+	public void testNotFollowedByAnyAfterOneOrMoreCombinations() {
+		final List<List<Event>> matches = testNotFollowedByAfterOneOrMore(false, true);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b4, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b6, NotFollowByData.d1)
+		));
+	}
+
+	private List<List<Event>> testNotFollowedByAfterOneOrMore(boolean eager, boolean allMatches) {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		int i = 0;
+		inputEvents.add(new StreamRecord<>(NotFollowByData.a1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b2, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b3, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.c1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b4, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b5, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b6, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.d1, i));
+
+		Pattern<Event, ?> pattern = Pattern
+			.<Event>begin("a").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("a");
+				}
+			});
+
+		pattern = (allMatches ? pattern.followedByAny("b*") : pattern.followedBy("b*"))
+			.where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("b");
+				}
+			});
+
+		pattern = (eager ? pattern.oneOrMore() : pattern.oneOrMore().allowCombinations())
+			.notFollowedBy("not c").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("c");
+				}
+			})
+			.followedBy("d").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("d");
+				}
+			});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		return feedNFA(inputEvents, nfa);
+	}
+
+	@Test
+	public void testNotFollowedByAnyBeforeOneOrMoreEager() {
+		final List<List<Event>> matches = testNotFollowedByBeforeOneOrMore(true, true);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByAnyBeforeOneOrMoreCombinations() {
+		final List<List<Event>> matches = testNotFollowedByBeforeOneOrMore(false, true);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByBeforeOneOrMoreEager() {
+		final List<List<Event>> matches = testNotFollowedByBeforeOneOrMore(true, false);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByBeforeOneOrMoreCombinations() {
+		final List<List<Event>> matches = testNotFollowedByBeforeOneOrMore(false, false);
+
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1)
+		));
+	}
+
+	private List<List<Event>> testNotFollowedByBeforeOneOrMore(boolean eager, boolean allMatches) {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		int i = 0;
+		inputEvents.add(new StreamRecord<>(NotFollowByData.a1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.c1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b4, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b5, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b6, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.d1, i));
+
+		Pattern<Event, ?> pattern = Pattern
+			.<Event>begin("a").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("a");
+				}
+			})
+			.notFollowedBy("not c").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("c");
+				}
+			});
+
+		pattern = (allMatches ? pattern.followedByAny("b*") : pattern.followedBy("b*"))
+			.where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("b");
+				}
+			}).oneOrMore();
+
+		pattern = (eager ? pattern : pattern.allowCombinations())
+			.followedBy("d").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("d");
+				}
+			});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		return feedNFA(inputEvents, nfa);
+	}
+
+	@Test
+	public void testNotFollowedByBeforeZeroOrMoreEagerSkipTillNext() {
+		final List<List<Event>> matches = testNotFollowedByBeforeZeroOrMore(true, false);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByBeforeZeroOrMoreCombinationsSkipTillNext() {
+		final List<List<Event>> matches = testNotFollowedByBeforeZeroOrMore(false, false);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b6, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByBeforeZeroOrMoreEagerSkipTillAny() {
+		final List<List<Event>> matches = testNotFollowedByBeforeZeroOrMore(true, true);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1)
+		));
+	}
+
+	@Test
+	public void testNotFollowedByBeforeZeroOrMoreCombinationsSkipTillAny() {
+		final List<List<Event>> matches = testNotFollowedByBeforeZeroOrMore(false, true);
+		compareMaps(matches, Lists.<List<Event>>newArrayList(
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b4, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.b6, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b5, NotFollowByData.d1),
+			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b1, NotFollowByData.b6, NotFollowByData.d1)
+		));
+	}
+
+	private List<List<Event>> testNotFollowedByBeforeZeroOrMore(boolean eager, boolean allMatches) {
+		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
+
+		int i = 0;
+		inputEvents.add(new StreamRecord<>(NotFollowByData.a1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.c1, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b4, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b5, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.b6, i++));
+		inputEvents.add(new StreamRecord<>(NotFollowByData.d1, i));
+
+		Pattern<Event, ?> pattern = Pattern
+			.<Event>begin("a").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("a");
+				}
+			})
+			.notFollowedBy("not c").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("c");
+				}
+			});
+
+		pattern = (allMatches ? pattern.followedByAny("b*") : pattern.followedBy("b*"))
+			.where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("b");
+				}
+			}).oneOrMore().optional();
+
+		pattern = (eager ? pattern : pattern.allowCombinations())
+			.followedBy("d").where(new SimpleCondition<Event>() {
+				private static final long serialVersionUID = 5726188262756267490L;
+
+				@Override
+				public boolean filter(Event value) throws Exception {
+					return value.getName().equals("d");
+				}
+			});
+
+		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
+
+		return feedNFA(inputEvents, nfa);
+	}
+
+	/////////////////////////////////////////       Utility        /////////////////////////////////////////////////
+
 	private List<List<Event>> feedNFA(List<StreamRecord<Event>> inputEvents, NFA<Event> nfa) {
 		List<List<Event>> resultingPatterns = new ArrayList<>();
 
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
index ced9efe93db10..90a6321948681 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/compiler/NFACompilerTest.java
@@ -87,6 +87,21 @@ public void testNFACompilerUniquePatternName() {
 		NFACompiler.compile(invalidPattern, Event.createTypeSerializer(), false);
 	}
 
+	@Test
+	public void testNFACompilerPatternEndsWithNotFollowedBy() {
+
+		// adjust the rule
+		expectedException.expect(MalformedPatternException.class);
+		expectedException.expectMessage("NotFollowedBy is not supported as a last part of a Pattern!");
+
+		Pattern<Event, ?> invalidPattern = Pattern.<Event>begin("start").where(new TestFilter())
+			.followedBy("middle").where(new TestFilter())
+			.notFollowedBy("end").where(new TestFilter());
+
+		// here we must have an exception because of the two "start" patterns with the same name.
+		NFACompiler.compile(invalidPattern, Event.createTypeSerializer(), false);
+	}
+
 	/**
 	 * A filter implementation to test invalid pattern specification with
 	 * duplicate pattern names. Check {@link #testNFACompilerUniquePatternName()}.
@@ -149,6 +164,26 @@ public void testNFACompilerWithSimplePattern() {
 		assertEquals(0, endingState.getStateTransitions().size());
 	}
 
+	@Test
+	public void testNoUnnecessaryStateCopiesCreated() {
+		final Pattern<Event, Event> pattern = Pattern.<Event>begin("start").where(startFilter)
+			.notFollowedBy("not").where(startFilter)
+			.followedBy("oneOrMore").where(startFilter).oneOrMore()
+			.followedBy("end").where(endFilter);
+
+		final NFACompiler.NFAFactoryCompiler<Event> nfaFactoryCompiler = new NFACompiler.NFAFactoryCompiler<>(pattern);
+		nfaFactoryCompiler.compileFactory();
+
+		int endStateCount = 0;
+		for (State<Event> state : nfaFactoryCompiler.getStates()) {
+			if (state.getName().equals("end")) {
+				endStateCount++;
+			}
+		}
+
+		assertEquals(1, endStateCount);
+	}
+
 	private <T> Set<Tuple2<String, StateTransitionAction>> unfoldTransitions(final State<T> state) {
 		final Set<Tuple2<String, StateTransitionAction>> transitions = new HashSet<>();
 		for (StateTransition<T> transition : state.getStateTransitions()) {
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
index 3c1da2ed3b7b8..b6fb48418d905 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/pattern/PatternTest.java
@@ -190,65 +190,73 @@ public boolean filter(Event value) throws Exception {
 	@Test(expected = MalformedPatternException.class)
 	public void testPatternCanHaveQuantifierSpecifiedOnce1() throws Exception {
 
-		Pattern.begin("start").where(new SimpleCondition<Object>() {
-			private static final long serialVersionUID = 8876425689668531458L;
-
-			@Override
-			public boolean filter(Object value) throws Exception {
-				return true;
-			}
-		}).oneOrMore().oneOrMore().optional();
+		Pattern.begin("start").where(dummyCondition()).oneOrMore().oneOrMore().optional();
 	}
 
 	@Test(expected = MalformedPatternException.class)
 	public void testPatternCanHaveQuantifierSpecifiedOnce2() throws Exception {
 
-		Pattern.begin("start").where(new SimpleCondition<Object>() {
-			private static final long serialVersionUID = 8311890695733430258L;
-
-			@Override
-			public boolean filter(Object value) throws Exception {
-				return true;
-			}
-		}).oneOrMore().optional().times(1);
+		Pattern.begin("start").where(dummyCondition()).oneOrMore().optional().times(1);
 	}
 
 	@Test(expected = MalformedPatternException.class)
 	public void testPatternCanHaveQuantifierSpecifiedOnce3() throws Exception {
 
-		Pattern.begin("start").where(new SimpleCondition<Object>() {
-			private static final long serialVersionUID = 8093713196099078214L;
-
-			@Override
-			public boolean filter(Object value) throws Exception {
-				return true;
-			}
-		}).times(1).oneOrMore();
+		Pattern.begin("start").where(dummyCondition()).times(1).oneOrMore();
 	}
 
 	@Test(expected = MalformedPatternException.class)
 	public void testPatternCanHaveQuantifierSpecifiedOnce4() throws Exception {
 
-		Pattern.begin("start").where(new SimpleCondition<Object>() {
-			private static final long serialVersionUID = -2995187062849334113L;
-
-			@Override
-			public boolean filter(Object value) throws Exception {
-				return true;
-			}
-		}).oneOrMore().oneOrMore();
+		Pattern.begin("start").where(dummyCondition()).oneOrMore().oneOrMore();
 	}
 
 	@Test(expected = MalformedPatternException.class)
 	public void testPatternCanHaveQuantifierSpecifiedOnce5() throws Exception {
 
-		Pattern.begin("start").where(new SimpleCondition<Object>() {
+		Pattern.begin("start").where(dummyCondition()).oneOrMore().oneOrMore().optional();
+	}
+
+	@Test(expected = MalformedPatternException.class)
+	public void testNotNextCannotBeOneOrMore() throws Exception {
+		Pattern.begin("start").where(dummyCondition()).notNext("not").where(dummyCondition()).oneOrMore();
+	}
+
+	@Test(expected = MalformedPatternException.class)
+	public void testNotNextCannotBeTimes() throws Exception {
+		Pattern.begin("start").where(dummyCondition()).notNext("not").where(dummyCondition()).times(3);
+	}
+
+	@Test(expected = MalformedPatternException.class)
+	public void testNotNextCannotBeOptional() throws Exception {
+
+		Pattern.begin("start").where(dummyCondition()).notNext("not").where(dummyCondition()).optional();
+	}
+
+	@Test(expected = MalformedPatternException.class)
+	public void testNotFollowedCannotBeOneOrMore() throws Exception {
+		Pattern.begin("start").where(dummyCondition()).notFollowedBy("not").where(dummyCondition()).oneOrMore();
+	}
+
+	@Test(expected = MalformedPatternException.class)
+	public void testNotFollowedCannotBeTimes() throws Exception {
+		Pattern.begin("start").where(dummyCondition()).notFollowedBy("not").where(dummyCondition()).times(3);
+	}
+
+	@Test(expected = MalformedPatternException.class)
+	public void testNotFollowedCannotBeOptional() throws Exception {
+
+		Pattern.begin("start").where(dummyCondition()).notFollowedBy("not").where(dummyCondition()).optional();
+	}
+
+	private SimpleCondition<Object> dummyCondition() {
+		return new SimpleCondition<Object>() {
 			private static final long serialVersionUID = -2205071036073867531L;
 
 			@Override
 			public boolean filter(Object value) throws Exception {
 				return true;
 			}
-		}).oneOrMore().oneOrMore().optional();
+		};
 	}
 }

From 68ccdb2bf87d8c06ac5761c7e4ede9fc2c3a6ad4 Mon Sep 17 00:00:00 2001
From: "xiaogang.sxg" <xiaogang.sxg@alibaba-inc.com>
Date: Sat, 29 Apr 2017 23:44:36 +0800
Subject: [PATCH 152/253] [FLINK-6364] [checkpoint] Incremental checkpointing
 in RocksDBKeyedStateBackend

---
 .../RocksDBIncrementalKeyedStateHandle.java   | 248 ++++++
 .../state/RocksDBKeyedStateBackend.java       | 711 +++++++++++++++++-
 .../streaming/state/RocksDBStateBackend.java  |  60 +-
 .../state/RocksDBAggregatingStateTest.java    |   6 +-
 .../state/RocksDBAsyncSnapshotTest.java       |  19 +-
 .../streaming/state/RocksDBListStateTest.java |   6 +-
 .../state/RocksDBReducingStateTest.java       |   6 +-
 .../state/RocksDBStateBackendTest.java        |  41 +-
 .../runtime/checkpoint/SubtaskState.java      |  16 +-
 .../state/AbstractKeyedStateBackend.java      |  10 +
 .../runtime/state/KeyGroupsStateHandle.java   |  10 +
 .../flink/runtime/state/KeyedStateHandle.java |   2 +-
 .../apache/flink/runtime/state/StateUtil.java |   9 +
 .../state/heap/HeapKeyedStateBackend.java     |   9 +
 .../runtime/state/StateBackendTestBase.java   |  15 +-
 .../api/operators/AbstractStreamOperator.java |   6 +-
 .../streaming/runtime/tasks/StreamTask.java   |   7 +-
 ...eyedOneInputStreamOperatorTestHarness.java |   4 +-
 .../PartitionedStateCheckpointingITCase.java  |  45 ++
 .../KVStateRequestSerializerRocksDBTest.java  |   8 +-
 20 files changed, 1163 insertions(+), 75 deletions(-)
 create mode 100644 flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
new file mode 100644
index 0000000000000..5ac9e4618c0b1
--- /dev/null
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.state.CompositeStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.SharedStateHandle;
+import org.apache.flink.runtime.state.SharedStateRegistry;
+import org.apache.flink.runtime.state.StateUtil;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * The handle to states in incremental snapshots taken by {@link RocksDBKeyedStateBackend}.
+ *
+ * The states contained in an incremental snapshot include
+ * <ul>
+ * <li> New SST state which includes the sst files produced since the last completed
+ *   checkpoint. These files can be referenced by succeeding checkpoints if the
+ *   checkpoint succeeds to complete. </li>
+ * <li> Old SST state which includes the sst files materialized in previous
+ *   checkpoints. </li>
+ * <li> MISC state which include the other files in the RocksDB instance, e.g. the
+ *   LOG and MANIFEST files. These files are mutable, hence cannot be shared by
+ *   other checkpoints. </li>
+ * <li> Meta state which includes the information of existing states. </li>
+ * </ul>
+ */
+public class RocksDBIncrementalKeyedStateHandle implements KeyedStateHandle, CompositeStateHandle {
+
+	private static final Logger LOG = LoggerFactory.getLogger(RocksDBIncrementalKeyedStateHandle.class);
+
+	private static final long serialVersionUID = -8328808513197388231L;
+
+	private final JobID jobId;
+
+	private final String operatorIdentifier;
+
+	private final KeyGroupRange keyGroupRange;
+
+	private final long checkpointId;
+
+	private final Map<String, StreamStateHandle> newSstFiles;
+
+	private final Map<String, StreamStateHandle> oldSstFiles;
+
+	private final Map<String, StreamStateHandle> miscFiles;
+
+	private final StreamStateHandle metaStateHandle;
+
+	/**
+	 * True if the state handle has already registered shared states.
+	 *
+	 * Once the shared states are registered, it's the {@link SharedStateRegistry}'s
+	 * responsibility to maintain the shared states. But in the cases where the
+	 * state handle is discarded before performing the registration, the handle
+	 * should delete all the shared states created by it.
+	 */
+	private boolean registered;
+
+	RocksDBIncrementalKeyedStateHandle(
+			JobID jobId,
+			String operatorIdentifier,
+			KeyGroupRange keyGroupRange,
+			long checkpointId,
+			Map<String, StreamStateHandle> newSstFiles,
+			Map<String, StreamStateHandle> oldSstFiles,
+			Map<String, StreamStateHandle> miscFiles,
+			StreamStateHandle metaStateHandle) {
+
+		this.jobId = Preconditions.checkNotNull(jobId);
+		this.operatorIdentifier = Preconditions.checkNotNull(operatorIdentifier);
+		this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange);
+		this.checkpointId = checkpointId;
+		this.newSstFiles = Preconditions.checkNotNull(newSstFiles);
+		this.oldSstFiles = Preconditions.checkNotNull(oldSstFiles);
+		this.miscFiles = Preconditions.checkNotNull(miscFiles);
+		this.metaStateHandle = Preconditions.checkNotNull(metaStateHandle);
+		this.registered = false;
+	}
+
+	@Override
+	public KeyGroupRange getKeyGroupRange() {
+		return keyGroupRange;
+	}
+
+	long getCheckpointId() {
+		return checkpointId;
+	}
+
+	Map<String, StreamStateHandle> getNewSstFiles() {
+		return newSstFiles;
+	}
+
+	Map<String, StreamStateHandle> getOldSstFiles() {
+		return oldSstFiles;
+	}
+
+	Map<String, StreamStateHandle> getMiscFiles() {
+		return miscFiles;
+	}
+
+	StreamStateHandle getMetaStateHandle() {
+		return metaStateHandle;
+	}
+
+	@Override
+	public KeyedStateHandle getIntersection(KeyGroupRange keyGroupRange) {
+		if (this.keyGroupRange.getIntersection(keyGroupRange) != KeyGroupRange.EMPTY_KEY_GROUP_RANGE) {
+			return this;
+		} else {
+			return null;
+		}
+	}
+
+	@Override
+	public void discardState() throws Exception {
+
+		try {
+			metaStateHandle.discardState();
+		} catch (Exception e) {
+			LOG.warn("Could not properly discard meta data.", e);
+		}
+
+		try {
+			StateUtil.bestEffortDiscardAllStateObjects(miscFiles.values());
+		} catch (Exception e) {
+			LOG.warn("Could not properly discard misc file states.", e);
+		}
+
+		if (!registered) {
+			try {
+				StateUtil.bestEffortDiscardAllStateObjects(newSstFiles.values());
+			} catch (Exception e) {
+				LOG.warn("Could not properly discard new sst file states.", e);
+			}
+		}
+	}
+
+	@Override
+	public long getStateSize() {
+		long size = StateUtil.getStateSize(metaStateHandle);
+
+		for (StreamStateHandle newSstFileHandle : newSstFiles.values()) {
+			size += newSstFileHandle.getStateSize();
+		}
+
+		for (StreamStateHandle oldSstFileHandle : oldSstFiles.values()) {
+			size += oldSstFileHandle.getStateSize();
+		}
+
+		for (StreamStateHandle miscFileHandle : miscFiles.values()) {
+			size += miscFileHandle.getStateSize();
+		}
+
+		return size;
+	}
+
+	@Override
+	public void registerSharedStates(SharedStateRegistry stateRegistry) {
+		Preconditions.checkState(!registered, "The state handle has already registered its shared states.");
+
+		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : newSstFiles.entrySet()) {
+			SstFileStateHandle stateHandle = new SstFileStateHandle(newSstFileEntry.getKey(), newSstFileEntry.getValue());
+
+			int referenceCount = stateRegistry.register(stateHandle);
+			Preconditions.checkState(referenceCount == 1);
+		}
+
+		for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : oldSstFiles.entrySet()) {
+			SstFileStateHandle stateHandle = new SstFileStateHandle(oldSstFileEntry.getKey(), oldSstFileEntry.getValue());
+
+			int referenceCount = stateRegistry.register(stateHandle);
+			Preconditions.checkState(referenceCount > 1);
+		}
+
+		registered = true;
+	}
+
+	@Override
+	public void unregisterSharedStates(SharedStateRegistry stateRegistry) {
+		Preconditions.checkState(registered, "The state handle has not registered its shared states yet.");
+
+		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : newSstFiles.entrySet()) {
+			stateRegistry.unregister(new SstFileStateHandle(newSstFileEntry.getKey(), newSstFileEntry.getValue()));
+		}
+
+		for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : oldSstFiles.entrySet()) {
+			stateRegistry.unregister(new SstFileStateHandle(oldSstFileEntry.getKey(), oldSstFileEntry.getValue()));
+		}
+
+		registered = false;
+	}
+
+	private class SstFileStateHandle implements SharedStateHandle {
+
+		private static final long serialVersionUID = 9092049285789170669L;
+
+		private final String fileName;
+
+		private final StreamStateHandle delegateStateHandle;
+
+		private SstFileStateHandle(
+				String fileName,
+				StreamStateHandle delegateStateHandle) {
+			this.fileName = fileName;
+			this.delegateStateHandle = delegateStateHandle;
+		}
+
+		@Override
+		public String getRegistrationKey() {
+			return jobId + "-" + operatorIdentifier + "-" + keyGroupRange + "-" + fileName;
+		}
+
+		@Override
+		public void discardState() throws Exception {
+			delegateStateHandle.discardState();
+		}
+
+		@Override
+		public long getStateSize() {
+			return delegateStateHandle.getStateSize();
+		}
+	}
+}
+
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 199a5a4a0ef2d..ee5f956128e56 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -31,7 +31,12 @@
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
 import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.core.fs.CloseableRegistry;
 import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FSDataOutputStream;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
 import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
@@ -55,6 +60,8 @@
 import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.StateObject;
+import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.internal.InternalAggregatingState;
 import org.apache.flink.runtime.state.internal.InternalFoldingState;
@@ -67,6 +74,7 @@
 import org.apache.flink.util.IOUtils;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.Preconditions;
+import org.rocksdb.Checkpoint;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.ColumnFamilyOptions;
@@ -83,13 +91,20 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.ObjectInputStream;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
 import java.util.concurrent.RunnableFuture;
 
 /**
@@ -102,6 +117,10 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
 
 	private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyedStateBackend.class);
 
+	private final JobID jobId;
+
+	private final String operatorIdentifier;
+
 	/** The column family options from the options factory */
 	private final ColumnFamilyOptions columnOptions;
 
@@ -137,6 +156,17 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
 	/** Number of bytes required to prefix the key groups. */
 	private final int keyGroupPrefixBytes;
 
+	/** True if incremental checkpointing is enabled */
+	private final boolean enableIncrementalCheckpointing;
+
+	/** The sst files materialized in pending checkpoints */
+	private final SortedMap<Long, Map<String, StreamStateHandle>> materializedSstFiles = new TreeMap<>();
+
+	/** The identifier of the last completed checkpoint */
+	private long lastCompletedCheckpointId = -1;
+
+	private static final String SST_FILE_SUFFIX = ".sst";
+
 	public RocksDBKeyedStateBackend(
 			JobID jobId,
 			String operatorIdentifier,
@@ -148,10 +178,17 @@ public RocksDBKeyedStateBackend(
 			TypeSerializer<K> keySerializer,
 			int numberOfKeyGroups,
 			KeyGroupRange keyGroupRange,
-			ExecutionConfig executionConfig
+			ExecutionConfig executionConfig,
+			boolean enableIncrementalCheckpointing
 	) throws IOException {
 
 		super(kvStateRegistry, keySerializer, userCodeClassLoader, numberOfKeyGroups, keyGroupRange, executionConfig);
+
+		this.jobId = Preconditions.checkNotNull(jobId);
+		this.operatorIdentifier = Preconditions.checkNotNull(operatorIdentifier);
+
+		this.enableIncrementalCheckpointing = enableIncrementalCheckpointing;
+
 		this.columnOptions = Preconditions.checkNotNull(columnFamilyOptions);
 		this.dbOptions = Preconditions.checkNotNull(dbOptions);
 
@@ -174,21 +211,6 @@ public RocksDBKeyedStateBackend(
 			throw new IOException("Error cleaning RocksDB data directory.", e);
 		}
 
-		List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(1);
-		// RocksDB seems to need this...
-		columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes(ConfigConstants.DEFAULT_CHARSET)));
-		List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(1);
-		try {
-
-			db = RocksDB.open(
-					Preconditions.checkNotNull(dbOptions),
-					instanceRocksDBPath.getAbsolutePath(),
-					columnFamilyDescriptors,
-					columnFamilyHandles);
-
-		} catch (RocksDBException e) {
-			throw new IOException("Error while opening RocksDB instance.", e);
-		}
 		keyGroupPrefixBytes = getNumberOfKeyGroups() > (Byte.MAX_VALUE + 1) ? 2 : 1;
 		kvStateInformation = new HashMap<>();
 	}
@@ -265,9 +287,71 @@ public RunnableFuture<KeyedStateHandle> snapshot(
 			final CheckpointStreamFactory streamFactory,
 			CheckpointOptions checkpointOptions) throws Exception {
 
+		if (checkpointOptions.getCheckpointType() != CheckpointOptions.CheckpointType.SAVEPOINT &&
+			enableIncrementalCheckpointing) {
+			return snapshotIncrementally(checkpointId, timestamp, streamFactory);
+		} else {
+			return snapshotFully(checkpointId, timestamp, streamFactory);
+		}
+	}
+
+	private RunnableFuture<KeyedStateHandle> snapshotIncrementally(
+			final long checkpointId,
+			final long checkpointTimestamp,
+			final CheckpointStreamFactory checkpointStreamFactory) throws Exception {
+
+		final RocksDBIncrementalSnapshotOperation snapshotOperation =
+			new RocksDBIncrementalSnapshotOperation(
+				this,
+				checkpointStreamFactory,
+				checkpointId,
+				checkpointTimestamp);
+
+		synchronized (asyncSnapshotLock) {
+			if (db == null) {
+				throw new IOException("RocksDB closed.");
+			}
+
+			if (!hasRegisteredState()) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Asynchronous RocksDB snapshot performed on empty keyed state at " +
+							checkpointTimestamp + " . Returning null.");
+				}
+				return DoneFuture.nullValue();
+			}
+
+			snapshotOperation.takeSnapshot();
+		}
+
+		return new FutureTask<KeyedStateHandle>(
+			new Callable<KeyedStateHandle>() {
+				@Override
+				public KeyedStateHandle call() throws Exception {
+					return snapshotOperation.materializeSnapshot();
+				}
+			}
+		) {
+			@Override
+			public boolean cancel(boolean mayInterruptIfRunning) {
+				snapshotOperation.stop();
+				return super.cancel(mayInterruptIfRunning);
+			}
+
+			@Override
+			protected void done() {
+				snapshotOperation.releaseResources(isCancelled());
+			}
+		};
+	}
+
+	private RunnableFuture<KeyedStateHandle> snapshotFully(
+			final long checkpointId,
+			final long timestamp,
+			final CheckpointStreamFactory streamFactory) throws Exception {
+
 		long startTime = System.currentTimeMillis();
 
-		final RocksDBSnapshotOperation snapshotOperation = new RocksDBSnapshotOperation(this, streamFactory);
+		final RocksDBFullSnapshotOperation snapshotOperation = new RocksDBFullSnapshotOperation(this, streamFactory);
 		// hold the db lock while operation on the db to guard us against async db disposal
 		synchronized (asyncSnapshotLock) {
 
@@ -342,7 +426,7 @@ public void done(boolean canceled) {
 	/**
 	 * Encapsulates the process to perform a snapshot of a RocksDBKeyedStateBackend.
 	 */
-	static final class RocksDBSnapshotOperation {
+	static final class RocksDBFullSnapshotOperation {
 
 		static final int FIRST_BIT_IN_BYTE_MASK = 0x80;
 		static final int END_OF_KEY_GROUP_MARK = 0xFFFF;
@@ -362,7 +446,7 @@ static final class RocksDBSnapshotOperation {
 		private DataOutputView outputView;
 		private KeyGroupsStateHandle snapshotResultStateHandle;
 
-		RocksDBSnapshotOperation(
+		RocksDBFullSnapshotOperation(
 				RocksDBKeyedStateBackend<?> stateBackend,
 				CheckpointStreamFactory checkpointStreamFactory) {
 
@@ -607,11 +691,11 @@ static void setMetaDataFollowsFlagInKey(byte[] key) {
 		}
 
 		static void clearMetaDataFollowsFlag(byte[] key) {
-			key[0] &= (~RocksDBSnapshotOperation.FIRST_BIT_IN_BYTE_MASK);
+			key[0] &= (~RocksDBFullSnapshotOperation.FIRST_BIT_IN_BYTE_MASK);
 		}
 
 		static boolean hasMetaDataFollowsFlag(byte[] key) {
-			return 0 != (key[0] & RocksDBSnapshotOperation.FIRST_BIT_IN_BYTE_MASK);
+			return 0 != (key[0] & RocksDBFullSnapshotOperation.FIRST_BIT_IN_BYTE_MASK);
 		}
 
 		private static void checkInterrupted() throws InterruptedException {
@@ -621,6 +705,239 @@ private static void checkInterrupted() throws InterruptedException {
 		}
 	}
 
+	private static class RocksDBIncrementalSnapshotOperation {
+
+		private final RocksDBKeyedStateBackend<?> stateBackend;
+
+		private final CheckpointStreamFactory checkpointStreamFactory;
+
+		private final long checkpointId;
+
+		private final long checkpointTimestamp;
+
+		private Map<String, StreamStateHandle> baseSstFiles;
+
+		private List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfos = new ArrayList<>();
+
+		private FileSystem backupFileSystem;
+		private Path backupPath;
+
+		// Registry for all opened i/o streams
+		private CloseableRegistry closeableRegistry = new CloseableRegistry();
+
+		// new sst files since the last completed checkpoint
+		private Map<String, StreamStateHandle> newSstFiles = new HashMap<>();
+
+		// old sst files which have been materialized in previous completed checkpoints
+		private Map<String, StreamStateHandle> oldSstFiles = new HashMap<>();
+
+		// handles to the misc files in the current snapshot
+		private Map<String, StreamStateHandle> miscFiles = new HashMap<>();
+
+		private StreamStateHandle metaStateHandle = null;
+
+		private RocksDBIncrementalSnapshotOperation(
+				RocksDBKeyedStateBackend<?> stateBackend,
+				CheckpointStreamFactory checkpointStreamFactory,
+				long checkpointId,
+				long checkpointTimestamp) {
+
+			this.stateBackend = stateBackend;
+			this.checkpointStreamFactory = checkpointStreamFactory;
+			this.checkpointId = checkpointId;
+			this.checkpointTimestamp = checkpointTimestamp;
+		}
+
+		private StreamStateHandle materializeStateData(Path filePath) throws Exception {
+			FSDataInputStream inputStream = null;
+			CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null;
+
+			try {
+				final byte[] buffer = new byte[1024];
+
+				FileSystem backupFileSystem = backupPath.getFileSystem();
+				inputStream = backupFileSystem.open(filePath);
+				closeableRegistry.registerClosable(inputStream);
+
+				outputStream = checkpointStreamFactory
+					.createCheckpointStateOutputStream(checkpointId, checkpointTimestamp);
+				closeableRegistry.registerClosable(outputStream);
+
+				while (true) {
+					int numBytes = inputStream.read(buffer);
+
+					if (numBytes == -1) {
+						break;
+					}
+
+					outputStream.write(buffer, 0, numBytes);
+				}
+
+				closeableRegistry.unregisterClosable(outputStream);
+				StreamStateHandle result = outputStream.closeAndGetHandle();
+				outputStream = null;
+
+				return result;
+			} finally {
+				if (inputStream != null) {
+					closeableRegistry.unregisterClosable(inputStream);
+					inputStream.close();
+				}
+
+				if (outputStream != null) {
+					closeableRegistry.unregisterClosable(outputStream);
+					outputStream.close();
+				}
+			}
+		}
+
+		private StreamStateHandle materializeMetaData() throws Exception {
+			CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null;
+
+			try {
+				outputStream = checkpointStreamFactory
+					.createCheckpointStateOutputStream(checkpointId, checkpointTimestamp);
+				stateBackend.cancelStreamRegistry.registerClosable(outputStream);
+
+				KeyedBackendSerializationProxy serializationProxy =
+					new KeyedBackendSerializationProxy(stateBackend.keySerializer, stateMetaInfos);
+				DataOutputView out = new DataOutputViewStreamWrapper(outputStream);
+
+				serializationProxy.write(out);
+
+				stateBackend.cancelStreamRegistry.unregisterClosable(outputStream);
+				StreamStateHandle result = outputStream.closeAndGetHandle();
+				outputStream = null;
+
+				return result;
+			} finally {
+				if (outputStream != null) {
+					stateBackend.cancelStreamRegistry.unregisterClosable(outputStream);
+					outputStream.close();
+				}
+			}
+		}
+
+		void takeSnapshot() throws Exception {
+			// use the last completed checkpoint as the comparison base.
+			baseSstFiles = stateBackend.materializedSstFiles.get(stateBackend.lastCompletedCheckpointId);
+
+			// save meta data
+			for (Map.Entry<String, Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>> stateMetaInfoEntry : stateBackend.kvStateInformation.entrySet()) {
+
+				RegisteredBackendStateMetaInfo<?, ?> metaInfo = stateMetaInfoEntry.getValue().f1;
+
+				KeyedBackendSerializationProxy.StateMetaInfo<?, ?> metaInfoProxy =
+					new KeyedBackendSerializationProxy.StateMetaInfo<>(
+						metaInfo.getStateType(),
+						metaInfo.getName(),
+						metaInfo.getNamespaceSerializer(),
+						metaInfo.getStateSerializer());
+
+				stateMetaInfos.add(metaInfoProxy);
+			}
+
+			// save state data
+			backupPath = new Path(stateBackend.instanceBasePath.getAbsolutePath(), "chk-" + checkpointId);
+			backupFileSystem = backupPath.getFileSystem();
+			if (backupFileSystem.exists(backupPath)) {
+				throw new IllegalStateException("Unexpected existence of the backup directory.");
+			}
+
+			// create hard links of living files in the checkpoint path
+			Checkpoint checkpoint = Checkpoint.create(stateBackend.db);
+			checkpoint.createCheckpoint(backupPath.getPath());
+		}
+
+		KeyedStateHandle materializeSnapshot() throws Exception {
+
+			synchronized (stateBackend.asyncSnapshotLock) {
+
+				if (stateBackend.db == null) {
+					throw new IOException("RocksDB closed.");
+				}
+
+				stateBackend.cancelStreamRegistry.registerClosable(closeableRegistry);
+
+				// write meta data
+				metaStateHandle = materializeMetaData();
+
+				// write state data
+				Preconditions.checkState(backupFileSystem.exists(backupPath));
+
+				FileStatus[] fileStatuses = backupFileSystem.listStatus(backupPath);
+				if (fileStatuses != null) {
+					for (FileStatus fileStatus : fileStatuses) {
+						Path filePath = fileStatus.getPath();
+						String fileName = filePath.getName();
+
+						if (fileName.endsWith(SST_FILE_SUFFIX)) {
+							StreamStateHandle fileHandle =
+								baseSstFiles == null ? null : baseSstFiles.get(fileName);
+
+							if (fileHandle == null) {
+								fileHandle = materializeStateData(filePath);
+
+								newSstFiles.put(fileName, fileHandle);
+							} else {
+								oldSstFiles.put(fileName, fileHandle);
+							}
+						} else {
+							StreamStateHandle fileHandle = materializeStateData(filePath);
+							miscFiles.put(fileName, fileHandle);
+						}
+					}
+				}
+
+				Map<String, StreamStateHandle> sstFiles = new HashMap<>(newSstFiles.size() + oldSstFiles.size());
+				sstFiles.putAll(newSstFiles);
+				sstFiles.putAll(oldSstFiles);
+
+				stateBackend.materializedSstFiles.put(checkpointId, sstFiles);
+
+				return new RocksDBIncrementalKeyedStateHandle(stateBackend.jobId,
+					stateBackend.operatorIdentifier, stateBackend.keyGroupRange,
+					checkpointId, newSstFiles, oldSstFiles, miscFiles, metaStateHandle);
+			}
+		}
+
+		void stop() {
+			try {
+				closeableRegistry.close();
+			} catch (IOException e) {
+				LOG.warn("Could not properly close io streams.", e);
+			}
+		}
+
+		void releaseResources(boolean canceled) {
+			stateBackend.cancelStreamRegistry.unregisterClosable(closeableRegistry);
+
+			if (backupPath != null) {
+				try {
+					if (backupFileSystem.exists(backupPath)) {
+						backupFileSystem.delete(backupPath, true);
+					}
+				} catch (Exception e) {
+					LOG.warn("Could not properly delete the checkpoint directory.", e);
+				}
+			}
+
+			if (canceled) {
+				List<StateObject> statesToDiscard = new ArrayList<>();
+
+				statesToDiscard.add(metaStateHandle);
+				statesToDiscard.addAll(miscFiles.values());
+				statesToDiscard.addAll(newSstFiles.values());
+
+				try {
+					StateUtil.bestEffortDiscardAllStateObjects(statesToDiscard);
+				} catch (Exception e) {
+					LOG.warn("Could not properly discard states.", e);
+				}
+			}
+		}
+	}
+
 	@Override
 	public void restore(Collection<KeyedStateHandle> restoreState) throws Exception {
 		LOG.info("Initializing RocksDB keyed state backend from snapshot.");
@@ -630,11 +947,16 @@ public void restore(Collection<KeyedStateHandle> restoreState) throws Exception
 		}
 
 		try {
-			if (MigrationUtil.isOldSavepointKeyedState(restoreState)) {
+			if (restoreState == null || restoreState.isEmpty()) {
+				createDB();
+			} else if (MigrationUtil.isOldSavepointKeyedState(restoreState)) {
 				LOG.info("Converting RocksDB state from old savepoint.");
 				restoreOldSavepointKeyedState(restoreState);
+			} else if (restoreState.iterator().next() instanceof RocksDBIncrementalKeyedStateHandle) {
+				RocksDBIncrementalRestoreOperation restoreOperation = new RocksDBIncrementalRestoreOperation(this);
+				restoreOperation.restore(restoreState);
 			} else {
-				RocksDBRestoreOperation restoreOperation = new RocksDBRestoreOperation(this);
+				RocksDBFullRestoreOperation restoreOperation = new RocksDBFullRestoreOperation(this);
 				restoreOperation.doRestore(restoreState);
 			}
 		} catch (Exception ex) {
@@ -643,10 +965,68 @@ public void restore(Collection<KeyedStateHandle> restoreState) throws Exception
 		}
 	}
 
+	@Override
+	public void notifyOfCompletedCheckpoint(long completedCheckpointId) {
+		synchronized (asyncSnapshotLock) {
+			if (completedCheckpointId < lastCompletedCheckpointId) {
+				return;
+			}
+
+			Iterator<Long> materializedCheckpointIterator = materializedSstFiles.keySet().iterator();
+			while (materializedCheckpointIterator.hasNext()) {
+				long materializedCheckpointId = materializedCheckpointIterator.next();
+
+				if (materializedCheckpointId < completedCheckpointId) {
+					materializedCheckpointIterator.remove();
+				}
+			}
+
+			lastCompletedCheckpointId = completedCheckpointId;
+		}
+	}
+
+	private void createDB() throws IOException {
+		db = openDB(instanceRocksDBPath.getAbsolutePath(),
+			new ArrayList<ColumnFamilyDescriptor>(),
+			null);
+	}
+
+	private RocksDB openDB(
+			String path,
+			List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
+			List<ColumnFamilyHandle> stateColumnFamilyHandles) throws IOException {
+
+		List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(stateColumnFamilyDescriptors);
+		columnFamilyDescriptors.add(
+			new ColumnFamilyDescriptor(
+				"default".getBytes(ConfigConstants.DEFAULT_CHARSET), columnOptions));
+
+		List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());
+
+		RocksDB db;
+
+		try {
+			db = RocksDB.open(
+					Preconditions.checkNotNull(dbOptions),
+					Preconditions.checkNotNull(path),
+					columnFamilyDescriptors,
+					columnFamilyHandles);
+		} catch (RocksDBException e) {
+			throw new IOException("Error while opening RocksDB instance.", e);
+		}
+
+		if (stateColumnFamilyHandles != null) {
+			stateColumnFamilyHandles.addAll(
+				columnFamilyHandles.subList(0, columnFamilyHandles.size() - 1));
+		}
+
+		return db;
+	}
+
 	/**
 	 * Encapsulates the process of restoring a RocksDBKeyedStateBackend from a snapshot.
 	 */
-	static final class RocksDBRestoreOperation {
+	static final class RocksDBFullRestoreOperation {
 
 		private final RocksDBKeyedStateBackend<?> rocksDBKeyedStateBackend;
 
@@ -664,7 +1044,7 @@ static final class RocksDBRestoreOperation {
 		 *
 		 * @param rocksDBKeyedStateBackend the state backend into which we restore
 		 */
-		public RocksDBRestoreOperation(RocksDBKeyedStateBackend<?> rocksDBKeyedStateBackend) {
+		public RocksDBFullRestoreOperation(RocksDBKeyedStateBackend<?> rocksDBKeyedStateBackend) {
 			this.rocksDBKeyedStateBackend = Preconditions.checkNotNull(rocksDBKeyedStateBackend);
 		}
 
@@ -679,6 +1059,8 @@ public RocksDBRestoreOperation(RocksDBKeyedStateBackend<?> rocksDBKeyedStateBack
 		public void doRestore(Collection<KeyedStateHandle> keyedStateHandles)
 				throws IOException, ClassNotFoundException, RocksDBException {
 
+			rocksDBKeyedStateBackend.createDB();
+
 			for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {
 				if (keyedStateHandle != null) {
 
@@ -787,14 +1169,14 @@ private void restoreKVStateData() throws IOException, RocksDBException {
 					while (keyGroupHasMoreKeys) {
 						byte[] key = BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView);
 						byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView);
-						if (RocksDBSnapshotOperation.hasMetaDataFollowsFlag(key)) {
+						if (RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(key)) {
 							//clear the signal bit in the key to make it ready for insertion again
-							RocksDBSnapshotOperation.clearMetaDataFollowsFlag(key);
+							RocksDBFullSnapshotOperation.clearMetaDataFollowsFlag(key);
 							rocksDBKeyedStateBackend.db.put(handle, key, value);
 							//TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
-							kvStateId = RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK
+							kvStateId = RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK
 									& currentStateHandleInView.readShort();
-							if (RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK == kvStateId) {
+							if (RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK == kvStateId) {
 								keyGroupHasMoreKeys = false;
 							} else {
 								handle = currentStateHandleKVStateColumnFamilies.get(kvStateId);
@@ -808,6 +1190,272 @@ private void restoreKVStateData() throws IOException, RocksDBException {
 		}
 	}
 
+	private static class RocksDBIncrementalRestoreOperation {
+
+		private final RocksDBKeyedStateBackend<?> stateBackend;
+
+		private RocksDBIncrementalRestoreOperation(RocksDBKeyedStateBackend<?> stateBackend) {
+			this.stateBackend = stateBackend;
+		}
+
+		private List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> readMetaData(
+				StreamStateHandle metaStateHandle) throws Exception {
+
+			FSDataInputStream inputStream = null;
+
+			try {
+				inputStream = metaStateHandle.openInputStream();
+				stateBackend.cancelStreamRegistry.registerClosable(inputStream);
+
+				KeyedBackendSerializationProxy serializationProxy =
+					new KeyedBackendSerializationProxy(stateBackend.userCodeClassLoader);
+				DataInputView in = new DataInputViewStreamWrapper(inputStream);
+				serializationProxy.read(in);
+
+				return serializationProxy.getNamedStateSerializationProxies();
+			} finally {
+				if (inputStream != null) {
+					stateBackend.cancelStreamRegistry.unregisterClosable(inputStream);
+					inputStream.close();
+				}
+			}
+		}
+
+		private void readStateData(
+				Path restoreFilePath,
+				StreamStateHandle remoteFileHandle) throws IOException {
+
+			FileSystem restoreFileSystem = restoreFilePath.getFileSystem();
+
+			FSDataInputStream inputStream = null;
+			FSDataOutputStream outputStream = null;
+
+			try {
+				inputStream = remoteFileHandle.openInputStream();
+				stateBackend.cancelStreamRegistry.registerClosable(inputStream);
+
+				outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
+				stateBackend.cancelStreamRegistry.registerClosable(outputStream);
+
+				byte[] buffer = new byte[1024];
+				while (true) {
+					int numBytes = inputStream.read(buffer);
+					if (numBytes == -1) {
+						break;
+					}
+
+					outputStream.write(buffer, 0, numBytes);
+				}
+			} finally {
+				if (inputStream != null) {
+					stateBackend.cancelStreamRegistry.unregisterClosable(inputStream);
+					inputStream.close();
+				}
+
+				if (outputStream != null) {
+					stateBackend.cancelStreamRegistry.unregisterClosable(outputStream);
+					outputStream.close();
+				}
+			}
+		}
+
+		private void restoreInstance(
+				RocksDBIncrementalKeyedStateHandle restoreStateHandle,
+				boolean hasExtraKeys) throws Exception {
+
+			// read state data
+			Path restoreInstancePath = new Path(
+				stateBackend.instanceBasePath.getAbsolutePath(),
+				UUID.randomUUID().toString());
+
+			try {
+				Map<String, StreamStateHandle> newSstFiles = restoreStateHandle.getNewSstFiles();
+				for (Map.Entry<String, StreamStateHandle> newSstFileEntry : newSstFiles.entrySet()) {
+					String fileName = newSstFileEntry.getKey();
+					StreamStateHandle remoteFileHandle = newSstFileEntry.getValue();
+
+					readStateData(new Path(restoreInstancePath, fileName), remoteFileHandle);
+				}
+
+				Map<String, StreamStateHandle> oldSstFiles = restoreStateHandle.getOldSstFiles();
+				for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : oldSstFiles.entrySet()) {
+					String fileName = oldSstFileEntry.getKey();
+					StreamStateHandle remoteFileHandle = oldSstFileEntry.getValue();
+
+					readStateData(new Path(restoreInstancePath, fileName), remoteFileHandle);
+				}
+
+				Map<String, StreamStateHandle> miscFiles = restoreStateHandle.getMiscFiles();
+				for (Map.Entry<String, StreamStateHandle> miscFileEntry : miscFiles.entrySet()) {
+					String fileName = miscFileEntry.getKey();
+					StreamStateHandle remoteFileHandle = miscFileEntry.getValue();
+
+					readStateData(new Path(restoreInstancePath, fileName), remoteFileHandle);
+				}
+
+				// read meta data
+				List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfoProxies =
+					readMetaData(restoreStateHandle.getMetaStateHandle());
+
+				List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
+
+				for (KeyedBackendSerializationProxy.StateMetaInfo<?, ?> stateMetaInfoProxy : stateMetaInfoProxies) {
+
+					ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(
+						stateMetaInfoProxy.getStateName().getBytes(ConfigConstants.DEFAULT_CHARSET),
+						stateBackend.columnOptions);
+
+					columnFamilyDescriptors.add(columnFamilyDescriptor);
+				}
+
+				if (hasExtraKeys) {
+
+					List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
+
+					try (RocksDB restoreDb = stateBackend.openDB(
+							restoreInstancePath.getPath(),
+							columnFamilyDescriptors,
+							columnFamilyHandles)) {
+
+						for (int i = 0; i < columnFamilyHandles.size(); ++i) {
+							ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i);
+							ColumnFamilyDescriptor columnFamilyDescriptor = columnFamilyDescriptors.get(i);
+							KeyedBackendSerializationProxy.StateMetaInfo<?, ?> stateMetaInfoProxy = stateMetaInfoProxies.get(i);
+
+							Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>> registeredStateMetaInfoEntry =
+								stateBackend.kvStateInformation.get(stateMetaInfoProxy.getStateName());
+
+							if (null == registeredStateMetaInfoEntry) {
+
+								RegisteredBackendStateMetaInfo<?, ?> stateMetaInfo =
+									new RegisteredBackendStateMetaInfo<>(stateMetaInfoProxy);
+
+								registeredStateMetaInfoEntry =
+									new Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>(
+										stateBackend.db.createColumnFamily(columnFamilyDescriptor),
+										stateMetaInfo);
+
+								stateBackend.kvStateInformation.put(
+									stateMetaInfoProxy.getStateName(),
+									registeredStateMetaInfoEntry);
+							}
+
+							ColumnFamilyHandle targetColumnFamilyHandle = registeredStateMetaInfoEntry.f0;
+
+							try (RocksIterator iterator = restoreDb.newIterator(columnFamilyHandle)) {
+
+								int startKeyGroup = stateBackend.getKeyGroupRange().getStartKeyGroup();
+								byte[] startKeyGroupPrefixBytes = new byte[stateBackend.keyGroupPrefixBytes];
+								for (int j = 0; j < stateBackend.keyGroupPrefixBytes; ++j) {
+									startKeyGroupPrefixBytes[j] = (byte)(startKeyGroup >>> ((stateBackend.keyGroupPrefixBytes - j - 1) * Byte.SIZE));
+								}
+
+								iterator.seek(startKeyGroupPrefixBytes);
+
+								while (iterator.isValid()) {
+
+									int keyGroup = 0;
+									for (int j = 0; j < stateBackend.keyGroupPrefixBytes; ++j) {
+										keyGroup = (keyGroup << Byte.SIZE) + iterator.key()[j];
+									}
+
+									if (stateBackend.keyGroupRange.contains(keyGroup)) {
+										stateBackend.db.put(targetColumnFamilyHandle,
+											iterator.key(), iterator.value());
+									}
+
+									iterator.next();
+								}
+							}
+						}
+					}
+				} else {
+
+					// create hard links in the instance directory
+					if (!stateBackend.instanceRocksDBPath.mkdirs()) {
+						throw new IOException("Could not create RocksDB data directory.");
+					}
+
+					for (String newSstFileName : newSstFiles.keySet()) {
+						File restoreFile = new File(restoreInstancePath.getPath(), newSstFileName);
+						File targetFile = new File(stateBackend.instanceRocksDBPath, newSstFileName);
+
+						Files.createLink(targetFile.toPath(), restoreFile.toPath());
+					}
+
+					for (String oldSstFileName : oldSstFiles.keySet()) {
+						File restoreFile = new File(restoreInstancePath.getPath(), oldSstFileName);
+						File targetFile = new File(stateBackend.instanceRocksDBPath, oldSstFileName);
+
+						Files.createLink(targetFile.toPath(), restoreFile.toPath());
+					}
+
+					for (String miscFileName : miscFiles.keySet()) {
+						File restoreFile = new File(restoreInstancePath.getPath(), miscFileName);
+						File targetFile = new File(stateBackend.instanceRocksDBPath, miscFileName);
+
+						Files.createLink(targetFile.toPath(), restoreFile.toPath());
+					}
+
+					List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
+					stateBackend.db = stateBackend.openDB(
+						stateBackend.instanceRocksDBPath.getAbsolutePath(),
+						columnFamilyDescriptors, columnFamilyHandles);
+
+					for (int i = 0; i < columnFamilyDescriptors.size(); ++i) {
+						KeyedBackendSerializationProxy.StateMetaInfo<?, ?> stateMetaInfoProxy = stateMetaInfoProxies.get(i);
+
+						ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i);
+						RegisteredBackendStateMetaInfo<?, ?> stateMetaInfo =
+							new RegisteredBackendStateMetaInfo<>(stateMetaInfoProxy);
+
+						stateBackend.kvStateInformation.put(
+							stateMetaInfoProxy.getStateName(),
+							new Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>(
+								columnFamilyHandle, stateMetaInfo));
+					}
+
+
+					// use the restore sst files as the base for succeeding checkpoints
+					Map<String, StreamStateHandle> sstFiles = new HashMap<>();
+					sstFiles.putAll(newSstFiles);
+					sstFiles.putAll(oldSstFiles);
+					stateBackend.materializedSstFiles.put(restoreStateHandle.getCheckpointId(), sstFiles);
+
+					stateBackend.lastCompletedCheckpointId = restoreStateHandle.getCheckpointId();
+				}
+			} finally {
+				FileSystem restoreFileSystem = restoreInstancePath.getFileSystem();
+				if (restoreFileSystem.exists(restoreInstancePath)) {
+					restoreFileSystem.delete(restoreInstancePath, true);
+				}
+			}
+		}
+
+		void restore(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {
+
+			boolean hasExtraKeys = (restoreStateHandles.size() > 1 ||
+				!restoreStateHandles.iterator().next().getKeyGroupRange().equals(stateBackend.keyGroupRange));
+
+			if (hasExtraKeys) {
+				stateBackend.createDB();
+			}
+
+			for (KeyedStateHandle rawStateHandle : restoreStateHandles) {
+
+				if (! (rawStateHandle instanceof RocksDBIncrementalKeyedStateHandle)) {
+					throw new IllegalStateException("Unexpected state handle type, " +
+						"expected " + RocksDBIncrementalKeyedStateHandle.class +
+						", but found " + rawStateHandle.getClass());
+				}
+
+				RocksDBIncrementalKeyedStateHandle keyedStateHandle = (RocksDBIncrementalKeyedStateHandle) rawStateHandle;
+
+				restoreInstance(keyedStateHandle, hasExtraKeys);
+			}
+		}
+	}
+
 	// ------------------------------------------------------------------------
 	//  State factories
 	// ------------------------------------------------------------------------
@@ -1160,10 +1808,7 @@ public File getInstanceBasePath() {
 	 */
 	@Deprecated
 	private void restoreOldSavepointKeyedState(Collection<KeyedStateHandle> restoreState) throws Exception {
-
-		if (restoreState.isEmpty()) {
-			return;
-		}
+		createDB();
 
 		Preconditions.checkState(1 == restoreState.size(), "Only one element expected here.");
 
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
index 80c9a291ddef5..55b8be231718e 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
@@ -109,6 +109,9 @@ public class RocksDBStateBackend extends AbstractStateBackend {
 	/** Whether we already lazily initialized our local storage directories. */
 	private transient boolean isInitialized = false;
 
+	/** True if incremental checkpointing is enabled */
+	private boolean enableIncrementalCheckpointing;
+
 
 	/**
 	 * Creates a new {@code RocksDBStateBackend} that stores its checkpoint data in the
@@ -123,7 +126,24 @@ public class RocksDBStateBackend extends AbstractStateBackend {
 	 * @throws IOException Thrown, if no file system can be found for the scheme in the URI.
 	 */
 	public RocksDBStateBackend(String checkpointDataUri) throws IOException {
-		this(new Path(checkpointDataUri).toUri());
+		this(new Path(checkpointDataUri).toUri(), false);
+	}
+
+	/**
+	 * Creates a new {@code RocksDBStateBackend} that stores its checkpoint data in the
+	 * file system and location defined by the given URI.
+	 *
+	 * <p>A state backend that stores checkpoints in HDFS or S3 must specify the file system
+	 * host and port in the URI, or have the Hadoop configuration that describes the file system
+	 * (host / high-availability group / possibly credentials) either referenced from the Flink
+	 * config, or included in the classpath.
+	 *
+	 * @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory.
+	 * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled.
+	 * @throws IOException Thrown, if no file system can be found for the scheme in the URI.
+	 */
+	public RocksDBStateBackend(String checkpointDataUri, boolean enableIncrementalCheckpointing) throws IOException {
+		this(new Path(checkpointDataUri).toUri(), enableIncrementalCheckpointing);
 	}
 
 	/**
@@ -139,7 +159,24 @@ public RocksDBStateBackend(String checkpointDataUri) throws IOException {
 	 * @throws IOException Thrown, if no file system can be found for the scheme in the URI.
 	 */
 	public RocksDBStateBackend(URI checkpointDataUri) throws IOException {
-		this(new FsStateBackend(checkpointDataUri));
+		this(new FsStateBackend(checkpointDataUri), false);
+	}
+
+	/**
+	 * Creates a new {@code RocksDBStateBackend} that stores its checkpoint data in the
+	 * file system and location defined by the given URI.
+	 *
+	 * <p>A state backend that stores checkpoints in HDFS or S3 must specify the file system
+	 * host and port in the URI, or have the Hadoop configuration that describes the file system
+	 * (host / high-availability group / possibly credentials) either referenced from the Flink
+	 * config, or included in the classpath.
+	 *
+	 * @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory.
+	 * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled.
+	 * @throws IOException Thrown, if no file system can be found for the scheme in the URI.
+	 */
+	public RocksDBStateBackend(URI checkpointDataUri, boolean enableIncrementalCheckpointing) throws IOException {
+		this(new FsStateBackend(checkpointDataUri), enableIncrementalCheckpointing);
 	}
 
 	/**
@@ -156,6 +193,22 @@ public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) {
 		this.checkpointStreamBackend = requireNonNull(checkpointStreamBackend);
 	}
 
+	/**
+	 * Creates a new {@code RocksDBStateBackend} that uses the given state backend to store its
+	 * checkpoint data streams. Typically, one would supply a filesystem or database state backend
+	 * here where the snapshots from RocksDB would be stored.
+	 *
+	 * <p>The snapshots of the RocksDB state will be stored using the given backend's
+	 * {@link AbstractStateBackend#createStreamFactory(JobID, String) checkpoint stream}.
+	 *
+	 * @param checkpointStreamBackend The backend to store the
+	 * @param enableIncrementalCheckpointing True if incremental checkponting is enabled
+	 */
+	public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend, boolean enableIncrementalCheckpointing) {
+		this.checkpointStreamBackend = requireNonNull(checkpointStreamBackend);
+		this.enableIncrementalCheckpointing = enableIncrementalCheckpointing;
+	}
+
 	// ------------------------------------------------------------------------
 	//  State backend methods
 	// ------------------------------------------------------------------------
@@ -260,7 +313,8 @@ public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
 				keySerializer,
 				numberOfKeyGroups,
 				keyGroupRange,
-				env.getExecutionConfig());
+				env.getExecutionConfig(),
+				enableIncrementalCheckpointing);
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java
index 983e569da6b9f..1b6546612dbc7 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java
@@ -204,7 +204,7 @@ public void testMerging() throws Exception {
 	}
 
 	private static RocksDBKeyedStateBackend<String> createKeyedBackend(RocksDBStateBackend backend) throws Exception {
-		return (RocksDBKeyedStateBackend<String>) backend.createKeyedStateBackend(
+		RocksDBKeyedStateBackend<String> keyedBackend = (RocksDBKeyedStateBackend<String>) backend.createKeyedStateBackend(
 						new DummyEnvironment("TestTask", 1, 0),
 						new JobID(),
 						"test-op",
@@ -212,6 +212,10 @@ private static RocksDBKeyedStateBackend<String> createKeyedBackend(RocksDBStateB
 						16,
 						new KeyGroupRange(2, 3),
 						mock(TaskKvStateRegistry.class));
+
+		keyedBackend.restore(null);
+
+		return keyedBackend;
 	}
 
 	//  test functions
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java
index ffe2ce209164a..812babbb46c6c 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java
@@ -41,7 +41,6 @@
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
 import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.VoidNamespace;
 import org.apache.flink.runtime.state.VoidNamespaceSerializer;
@@ -338,6 +337,8 @@ public void testCleanupOfSnapshotsInFailureCase() throws Exception {
 			new KeyGroupRange(0, 0),
 			null);
 
+		keyedStateBackend.restore(null);
+
 		// register a state so that the state backend has to checkpoint something
 		keyedStateBackend.getPartitionedState(
 			"namespace",
@@ -360,19 +361,21 @@ public void testCleanupOfSnapshotsInFailureCase() throws Exception {
 	@Test
 	public void testConsistentSnapshotSerializationFlagsAndMasks() {
 
-		Assert.assertEquals(0xFFFF, RocksDBKeyedStateBackend.RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK);
-		Assert.assertEquals(0x80, RocksDBKeyedStateBackend.RocksDBSnapshotOperation.FIRST_BIT_IN_BYTE_MASK);
+		Assert.assertEquals(0xFFFF, RocksDBKeyedStateBackend.RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK);
+		Assert.assertEquals(0x80, RocksDBKeyedStateBackend.RocksDBFullSnapshotOperation.FIRST_BIT_IN_BYTE_MASK);
 
 		byte[] expectedKey = new byte[] {42, 42};
 		byte[] modKey = expectedKey.clone();
 
-		Assert.assertFalse(RocksDBKeyedStateBackend.RocksDBSnapshotOperation.hasMetaDataFollowsFlag(modKey));
+		Assert.assertFalse(
+			RocksDBKeyedStateBackend.RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(modKey));
 
-		RocksDBKeyedStateBackend.RocksDBSnapshotOperation.setMetaDataFollowsFlagInKey(modKey);
-		Assert.assertTrue(RocksDBKeyedStateBackend.RocksDBSnapshotOperation.hasMetaDataFollowsFlag(modKey));
+		RocksDBKeyedStateBackend.RocksDBFullSnapshotOperation.setMetaDataFollowsFlagInKey(modKey);
+		Assert.assertTrue(RocksDBKeyedStateBackend.RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(modKey));
 
-		RocksDBKeyedStateBackend.RocksDBSnapshotOperation.clearMetaDataFollowsFlag(modKey);
-		Assert.assertFalse(RocksDBKeyedStateBackend.RocksDBSnapshotOperation.hasMetaDataFollowsFlag(modKey));
+		RocksDBKeyedStateBackend.RocksDBFullSnapshotOperation.clearMetaDataFollowsFlag(modKey);
+		Assert.assertFalse(
+			RocksDBKeyedStateBackend.RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(modKey));
 
 		Assert.assertTrue(Arrays.equals(expectedKey, modKey));
 	}
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java
index d8d0308b1f601..e7efcfaf076bb 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java
@@ -210,7 +210,7 @@ public void testMerging() throws Exception {
 	// ------------------------------------------------------------------------
 
 	private static RocksDBKeyedStateBackend<String> createKeyedBackend(RocksDBStateBackend backend) throws Exception {
-		return (RocksDBKeyedStateBackend<String>) backend.createKeyedStateBackend(
+		RocksDBKeyedStateBackend<String> keyedBackend = (RocksDBKeyedStateBackend<String>) backend.createKeyedStateBackend(
 				new DummyEnvironment("TestTask", 1, 0),
 				new JobID(),
 				"test-op",
@@ -218,6 +218,10 @@ private static RocksDBKeyedStateBackend<String> createKeyedBackend(RocksDBStateB
 				16,
 				new KeyGroupRange(2, 3),
 				mock(TaskKvStateRegistry.class));
+
+		keyedBackend.restore(null);
+
+		return keyedBackend;
 	}
 
 	private static <T> void validateResult(Iterable<T> values, Set<T> expected) {
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java
index fb854f2d17729..a8b45353637a3 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java
@@ -210,7 +210,7 @@ public void testMerging() throws Exception {
 	// ------------------------------------------------------------------------
 
 	private static RocksDBKeyedStateBackend<String> createKeyedBackend(RocksDBStateBackend backend) throws Exception {
-		return (RocksDBKeyedStateBackend<String>) backend.createKeyedStateBackend(
+		RocksDBKeyedStateBackend<String> keyedBackend = (RocksDBKeyedStateBackend<String>) backend.createKeyedStateBackend(
 				new DummyEnvironment("TestTask", 1, 0),
 				new JobID(),
 				"test-op",
@@ -218,6 +218,10 @@ private static RocksDBKeyedStateBackend<String> createKeyedBackend(RocksDBStateB
 				16,
 				new KeyGroupRange(2, 3),
 				mock(TaskKvStateRegistry.class));
+
+		keyedBackend.restore(null);
+
+		return keyedBackend;
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
index b5f18a4f30c45..fad1559079e46 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
@@ -33,7 +33,6 @@
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.StateBackendTestBase;
 import org.apache.flink.runtime.state.VoidNamespace;
@@ -42,8 +41,11 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
+import org.rocksdb.Checkpoint;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.ReadOptions;
@@ -55,6 +57,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.RunnableFuture;
@@ -73,6 +76,7 @@
 /**
  * Tests for the partitioned state part of {@link RocksDBStateBackend}.
  */
+@RunWith(Parameterized.class)
 public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBackend> {
 
 	private OneShotLatch blocker;
@@ -83,17 +87,25 @@ public class RocksDBStateBackendTest extends StateBackendTestBase<RocksDBStateBa
 	private ValueState<Integer> testState1;
 	private ValueState<String> testState2;
 
+	@Parameterized.Parameters
+	public static Collection<Boolean> parameters() {
+		return Arrays.asList(false, true);
+	}
+
+	@Parameterized.Parameter
+	public boolean enableIncrementalCheckpointing;
+
 	@Rule
 	public TemporaryFolder tempFolder = new TemporaryFolder();
 
 	// Store it because we need it for the cleanup test.
-	private String dbPath;
+	String dbPath;
 
 	@Override
 	protected RocksDBStateBackend getStateBackend() throws IOException {
 		dbPath = tempFolder.newFolder().getAbsolutePath();
 		String checkpointPath = tempFolder.newFolder().toURI().toString();
-		RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath));
+		RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
 		backend.setDbStoragePath(dbPath);
 		return backend;
 	}
@@ -105,7 +117,7 @@ public void setupRocksKeyedStateBackend() throws Exception {
 		testStreamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
 		testStreamFactory.setBlockerLatch(blocker);
 		testStreamFactory.setWaiterLatch(waiter);
-		testStreamFactory.setAfterNumberInvocations(100);
+		testStreamFactory.setAfterNumberInvocations(10);
 
 		RocksDBStateBackend backend = getStateBackend();
 		Environment env = new DummyEnvironment("TestTask", 1, 0);
@@ -119,6 +131,8 @@ public void setupRocksKeyedStateBackend() throws Exception {
 				new KeyGroupRange(0, 1),
 				mock(TaskKvStateRegistry.class));
 
+		keyedStateBackend.restore(null);
+
 		testState1 = keyedStateBackend.getPartitionedState(
 				VoidNamespace.INSTANCE,
 				VoidNamespaceSerializer.INSTANCE,
@@ -178,8 +192,10 @@ public void testRunningSnapshotAfterBackendClosed() throws Exception {
 
 		RocksDB spyDB = keyedStateBackend.db;
 
-		verify(spyDB, times(1)).getSnapshot();
-		verify(spyDB, times(0)).releaseSnapshot(any(Snapshot.class));
+		if (!enableIncrementalCheckpointing) {
+			verify(spyDB, times(1)).getSnapshot();
+			verify(spyDB, times(0)).releaseSnapshot(any(Snapshot.class));
+		}
 
 		this.keyedStateBackend.dispose();
 		verify(spyDB, times(1)).close();
@@ -216,8 +232,10 @@ public void testReleasingSnapshotAfterBackendClosed() throws Exception {
 
 		RocksDB spyDB = keyedStateBackend.db;
 
-		verify(spyDB, times(1)).getSnapshot();
-		verify(spyDB, times(0)).releaseSnapshot(any(Snapshot.class));
+		if (!enableIncrementalCheckpointing) {
+			verify(spyDB, times(1)).getSnapshot();
+			verify(spyDB, times(0)).releaseSnapshot(any(Snapshot.class));
+		}
 
 		this.keyedStateBackend.dispose();
 		verify(spyDB, times(1)).close();
@@ -319,7 +337,6 @@ public void testDisposeDeletesAllDirectories() throws Exception {
 		backend.setCurrentKey(1);
 		state.update("Hello");
 
-
 		Collection<File> allFilesInDbDir =
 				FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
 
@@ -356,8 +373,10 @@ private void verifyRocksObjectsReleased() {
 		assertNotNull(null, keyedStateBackend.db);
 		RocksDB spyDB = keyedStateBackend.db;
 
-		verify(spyDB, times(1)).getSnapshot();
-		verify(spyDB, times(1)).releaseSnapshot(any(Snapshot.class));
+		if (!enableIncrementalCheckpointing) {
+			verify(spyDB, times(1)).getSnapshot();
+			verify(spyDB, times(1)).releaseSnapshot(any(Snapshot.class));
+		}
 
 		keyedStateBackend.dispose();
 		verify(spyDB, times(1)).close();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
index 121ac57f1efbd..a77baf3bdb3b4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
@@ -152,12 +152,24 @@ public void discardState() {
 
 	@Override
 	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
-		// No shared states
+		if (managedKeyedState != null) {
+			managedKeyedState.registerSharedStates(sharedStateRegistry);
+		}
+
+		if (rawKeyedState != null) {
+			rawKeyedState.registerSharedStates(sharedStateRegistry);
+		}
 	}
 
 	@Override
 	public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
-		// No shared states
+		if (managedKeyedState != null) {
+			managedKeyedState.unregisterSharedStates(sharedStateRegistry);
+		}
+
+		if (rawKeyedState != null) {
+			rawKeyedState.unregisterSharedStates(sharedStateRegistry);
+		}
 	}
 
 	@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
index e86f1f8ff67f5..61f397c38b255 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
@@ -211,6 +211,16 @@ protected abstract <N, UK, UV> InternalMapState<N, UK, UV> createMapState(
 			TypeSerializer<N> namespaceSerializer,
 			MapStateDescriptor<UK, UV> stateDesc) throws Exception;
 
+	/**
+	 * Called when the checkpoint with the given ID is completed and acknowledged on the JobManager.
+	 *
+	 * @param checkpointId The ID of the checkpoint that has been completed.
+	 *
+	 * @throws Exception Exceptions during checkpoint acknowledgement may be forwarded and will cause
+	 *                   the program to fail and enter recovery.
+	 */
+	public abstract void notifyOfCompletedCheckpoint(long checkpointId) throws Exception;
+
 	/**
 	 * @see KeyedStateBackend
 	 */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyGroupsStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyGroupsStateHandle.java
index bad7fd424df5b..82804601f042a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyGroupsStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyGroupsStateHandle.java
@@ -92,6 +92,16 @@ public KeyGroupRange getKeyGroupRange() {
 		return groupRangeOffsets.getKeyGroupRange();
 	}
 
+	@Override
+	public void registerSharedStates(SharedStateRegistry stateRegistry) {
+		// No shared states
+	}
+
+	@Override
+	public void unregisterSharedStates(SharedStateRegistry stateRegistry) {
+		// No shared states
+	}
+
 	@Override
 	public void discardState() throws Exception {
 		stateHandle.discardState();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedStateHandle.java
index dc9c97df5053a..704ec145b8677 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedStateHandle.java
@@ -23,7 +23,7 @@
  * recovering from failures, the handle will be passed to all tasks whose key
  * group ranges overlap with it.
  */
-public interface KeyedStateHandle extends StateObject {
+public interface KeyedStateHandle extends CompositeStateHandle {
 
 	/**
 	 * Returns the range of the key groups contained in the state.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtil.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtil.java
index b2508310e8e61..6f231e42c5151 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtil.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtil.java
@@ -32,6 +32,15 @@ private StateUtil() {
 		throw new AssertionError();
 	}
 
+	/**
+	 * Returns the size of a state object
+	 *
+	 * @param handle The handle to the retrieved state
+	 */
+	public static long getStateSize(StateObject handle) {
+		return handle == null ? 0 : handle.getStateSize();
+	}
+
 	/**
 	 * Iterates through the passed state handles and calls discardState() on each handle that is not null. All
 	 * occurring exceptions are suppressed and collected until the iteration is over and emitted as a single exception.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index 38817cdb4bd57..ead89b3146859 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -328,6 +328,10 @@ public KeyGroupsStateHandle performOperation() throws Exception {
 	@SuppressWarnings("deprecation")
 	@Override
 	public void restore(Collection<KeyedStateHandle> restoredState) throws Exception {
+		if (restoredState == null || restoredState.isEmpty()) {
+			return;
+		}
+
 		LOG.info("Initializing heap keyed state backend from snapshot.");
 
 		if (LOG.isDebugEnabled()) {
@@ -425,6 +429,11 @@ private void restorePartitionedState(Collection<KeyedStateHandle> state) throws
 		}
 	}
 
+	@Override
+	public void notifyOfCompletedCheckpoint(long checkpointId) {
+		//Nothing to do
+	}
+
 	@Override
 	public String toString() {
 		return "HeapKeyedStateBackend";
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
index ccc1eaee3ea0a..60f9c819429a9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
@@ -133,7 +133,8 @@ protected <K> AbstractKeyedStateBackend<K> createKeyedBackend(
 			int numberOfKeyGroups,
 			KeyGroupRange keyGroupRange,
 			Environment env) throws Exception {
-		return getStateBackend().createKeyedStateBackend(
+
+		AbstractKeyedStateBackend<K> backend = getStateBackend().createKeyedStateBackend(
 				env,
 				new JobID(),
 				"test_op",
@@ -141,6 +142,10 @@ protected <K> AbstractKeyedStateBackend<K> createKeyedBackend(
 				numberOfKeyGroups,
 				keyGroupRange,
 				env.getTaskKvStateRegistry());
+
+		backend.restore(null);
+
+		return backend;
 	}
 
 	protected <K> AbstractKeyedStateBackend<K> restoreKeyedBackend(TypeSerializer<K> keySerializer, KeyedStateHandle state) throws Exception {
@@ -2197,9 +2202,11 @@ public void testAsyncSnapshot() throws Exception {
 
 		Assert.assertNotNull(stateHandle);
 
-		backend = createKeyedBackend(IntSerializer.INSTANCE);
+		backend = null;
+
 		try {
-			backend.restore(Collections.singleton(stateHandle));
+			backend = restoreKeyedBackend(IntSerializer.INSTANCE, stateHandle);
+
 			InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(
 					VoidNamespaceSerializer.INSTANCE,
 					new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
@@ -2297,7 +2304,7 @@ public String fold(String acc, Integer value) throws Exception {
 	 * Returns the value by getting the serialized value and deserializing it
 	 * if it is not null.
 	 */
-	private static <V, K, N> V getSerializedValue(
+	protected static <V, K, N> V getSerializedValue(
 			InternalKvState<N> kvState,
 			K key,
 			TypeSerializer<K> keySerializer,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 18500073d52fd..d45ad42f790f0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -504,7 +504,11 @@ public void initializeState(StateInitializationContext context) throws Exception
 	}
 
 	@Override
-	public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception {}
+	public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception {
+		if (keyedStateBackend != null) {
+			keyedStateBackend.notifyOfCompletedCheckpoint(checkpointId);
+		}
+	}
 
 	/**
 	 * Returns a checkpoint stream factory for the provided options.
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 57e43de7f508b..bc66751f22d59 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -765,9 +765,10 @@ public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
 		cancelables.registerClosable(keyedStateBackend);
 
 		// restore if we have some old state
-		if (null != restoreStateHandles && null != restoreStateHandles.getManagedKeyedState()) {
-			keyedStateBackend.restore(restoreStateHandles.getManagedKeyedState());
-		}
+		Collection<KeyedStateHandle> restoreKeyedStateHandles =
+			restoreStateHandles == null ? null : restoreStateHandles.getManagedKeyedState();
+
+		keyedStateBackend.restore(restoreKeyedStateHandles);
 
 		@SuppressWarnings("unchecked")
 		AbstractKeyedStateBackend<K> typedBackend = (AbstractKeyedStateBackend<K>) keyedStateBackend;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/KeyedOneInputStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/KeyedOneInputStreamOperatorTestHarness.java
index d9c73874def4d..c6d0bce8cebc8 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/KeyedOneInputStreamOperatorTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/KeyedOneInputStreamOperatorTestHarness.java
@@ -116,9 +116,7 @@ public KeyedStateBackend answer(InvocationOnMock invocationOnMock) throws Throwa
 							keyGroupRange,
 							mockTask.getEnvironment().getTaskKvStateRegistry());
 
-					if (restoredKeyedState != null) {
-						keyedStateBackend.restore(restoredKeyedState);
-					}
+					keyedStateBackend.restore(restoredKeyedState);
 
 					return keyedStateBackend;
 				}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java
index 4761d70abe8ce..517c82b404146 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java
@@ -22,6 +22,8 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -35,11 +37,18 @@
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
+import org.apache.flink.runtime.state.AbstractStateBackend;
+import org.apache.flink.runtime.state.filesystem.FsStateBackend;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
 import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
 import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /**
  * A simple test that runs a streaming topology with checkpointing enabled.
@@ -50,15 +59,49 @@
  * It is designed to check partitioned states.
  */
 @SuppressWarnings("serial")
+@RunWith(Parameterized.class)
 public class PartitionedStateCheckpointingITCase extends StreamFaultToleranceTestBase {
 
+	private static final int MAX_MEM_STATE_SIZE = 10 * 1024 * 1024;
+
 	final long NUM_STRINGS = 10_000_000L;
 	final static int NUM_KEYS = 40;
 
+	@Parameterized.Parameters
+	public static Collection<AbstractStateBackend> parameters() throws IOException {
+		TemporaryFolder tempFolder = new TemporaryFolder();
+		tempFolder.create();
+
+		MemoryStateBackend syncMemBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE, false);
+		MemoryStateBackend asyncMemBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE, true);
+
+		FsStateBackend syncFsBackend = new FsStateBackend("file://" + tempFolder.newFolder().getAbsolutePath(), false);
+		FsStateBackend asyncFsBackend = new FsStateBackend("file://" + tempFolder.newFolder().getAbsolutePath(), true);
+
+		RocksDBStateBackend fullRocksDbBackend = new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), false);
+		fullRocksDbBackend.setDbStoragePath(tempFolder.newFolder().getAbsolutePath());
+
+		RocksDBStateBackend incRocksDbBackend = new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), true);
+		incRocksDbBackend.setDbStoragePath(tempFolder.newFolder().getAbsolutePath());
+
+		return Arrays.asList(
+			syncMemBackend,
+			asyncMemBackend,
+			syncFsBackend,
+			asyncFsBackend,
+			fullRocksDbBackend,
+			incRocksDbBackend);
+	}
+
+	@Parameterized.Parameter
+	public AbstractStateBackend stateBackend;
+
 	@Override
 	public void testProgram(StreamExecutionEnvironment env) {
 		assertTrue("Broken test setup", (NUM_STRINGS/2) % NUM_KEYS == 0);
 
+		env.setStateBackend(stateBackend);
+
 		DataStream<Integer> stream1 = env.addSource(new IntGeneratingSourceFunction(NUM_STRINGS / 2));
 		DataStream<Integer> stream2 = env.addSource(new IntGeneratingSourceFunction(NUM_STRINGS / 2));
 
@@ -163,6 +206,7 @@ private static class OnceFailingPartitionedSum extends RichMapFunction<Integer,
 
 		OnceFailingPartitionedSum(long numElements) {
 			this.numElements = numElements;
+			this.hasFailed = false;
 		}
 
 		@Override
@@ -181,6 +225,7 @@ public void open(Configuration parameters) throws IOException {
 		@Override
 		public Tuple2<Integer, Long> map(Integer value) throws Exception {
 			count++;
+
 			if (!hasFailed && count >= failurePos) {
 				hasFailed = true;
 				throw new Exception("Test Failure");
diff --git a/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java b/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java
index 3c86f9014dc1b..05f72c2677d6d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java
@@ -81,7 +81,7 @@ final static class RocksDBKeyedStateBackend2<K> extends RocksDBKeyedStateBackend
 			super(jobId, operatorIdentifier, userCodeClassLoader,
 				instanceBasePath,
 				dbOptions, columnFamilyOptions, kvStateRegistry, keySerializer,
-				numberOfKeyGroups, keyGroupRange, executionConfig);
+				numberOfKeyGroups, keyGroupRange, executionConfig, false);
 		}
 
 		@Override
@@ -120,6 +120,7 @@ public void testListSerialization() throws Exception {
 				1, new KeyGroupRange(0, 0),
 				new ExecutionConfig()
 			);
+		longHeapKeyedStateBackend.restore(null);
 		longHeapKeyedStateBackend.setCurrentKey(key);
 
 		final InternalListState<VoidNamespace, Long> listState = longHeapKeyedStateBackend
@@ -154,8 +155,9 @@ public void testMapSerialization() throws Exception {
 				mock(TaskKvStateRegistry.class),
 				LongSerializer.INSTANCE,
 				1, new KeyGroupRange(0, 0),
-				new ExecutionConfig()
-			);
+				new ExecutionConfig(),
+				false);
+		longHeapKeyedStateBackend.restore(null);
 		longHeapKeyedStateBackend.setCurrentKey(key);
 
 		final InternalMapState<VoidNamespace, Long, String> mapState = (InternalMapState<VoidNamespace, Long, String>) 

From b9e75cacdbf9a466fba18d36fb10cb21570fed0e Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Fri, 5 May 2017 13:09:46 +0200
Subject: [PATCH 153/253] [FLINK-6364] [checkpoint] Additional minor review
 changes

---
 .../state/RocksDBKeyedStateBackend.java       | 18 ++++----
 .../state/RocksDBStateBackendTest.java        |  1 -
 .../state/AbstractKeyedStateBackend.java      | 12 +----
 .../state/heap/HeapKeyedStateBackend.java     |  2 +-
 .../runtime/state/StateBackendTestBase.java   |  6 +--
 .../util/BlockerCheckpointStreamFactory.java  | 10 +++-
 .../api/operators/AbstractStreamOperator.java |  2 +-
 ...actEventTimeWindowCheckpointingITCase.java | 10 +++-
 ...endEventTimeWindowCheckpointingITCase.java | 46 +++++++++++++++++++
 9 files changed, 79 insertions(+), 28 deletions(-)
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index ee5f956128e56..b8e60cd9961f5 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -705,7 +705,7 @@ private static void checkInterrupted() throws InterruptedException {
 		}
 	}
 
-	private static class RocksDBIncrementalSnapshotOperation {
+	private static final class RocksDBIncrementalSnapshotOperation {
 
 		private final RocksDBKeyedStateBackend<?> stateBackend;
 
@@ -717,22 +717,22 @@ private static class RocksDBIncrementalSnapshotOperation {
 
 		private Map<String, StreamStateHandle> baseSstFiles;
 
-		private List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfos = new ArrayList<>();
+		private final List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfos = new ArrayList<>();
 
 		private FileSystem backupFileSystem;
 		private Path backupPath;
 
 		// Registry for all opened i/o streams
-		private CloseableRegistry closeableRegistry = new CloseableRegistry();
+		private final CloseableRegistry closeableRegistry = new CloseableRegistry();
 
 		// new sst files since the last completed checkpoint
-		private Map<String, StreamStateHandle> newSstFiles = new HashMap<>();
+		private final Map<String, StreamStateHandle> newSstFiles = new HashMap<>();
 
 		// old sst files which have been materialized in previous completed checkpoints
-		private Map<String, StreamStateHandle> oldSstFiles = new HashMap<>();
+		private final Map<String, StreamStateHandle> oldSstFiles = new HashMap<>();
 
 		// handles to the misc files in the current snapshot
-		private Map<String, StreamStateHandle> miscFiles = new HashMap<>();
+		private final Map<String, StreamStateHandle> miscFiles = new HashMap<>();
 
 		private StreamStateHandle metaStateHandle = null;
 
@@ -753,7 +753,7 @@ private StreamStateHandle materializeStateData(Path filePath) throws Exception {
 			CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null;
 
 			try {
-				final byte[] buffer = new byte[1024];
+				final byte[] buffer = new byte[8 * 1024];
 
 				FileSystem backupFileSystem = backupPath.getFileSystem();
 				inputStream = backupFileSystem.open(filePath);
@@ -966,7 +966,7 @@ public void restore(Collection<KeyedStateHandle> restoreState) throws Exception
 	}
 
 	@Override
-	public void notifyOfCompletedCheckpoint(long completedCheckpointId) {
+	public void notifyCheckpointComplete(long completedCheckpointId) {
 		synchronized (asyncSnapshotLock) {
 			if (completedCheckpointId < lastCompletedCheckpointId) {
 				return;
@@ -1237,7 +1237,7 @@ private void readStateData(
 				outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
 				stateBackend.cancelStreamRegistry.registerClosable(outputStream);
 
-				byte[] buffer = new byte[1024];
+				byte[] buffer = new byte[8 * 1024];
 				while (true) {
 					int numBytes = inputStream.read(buffer);
 					if (numBytes == -1) {
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
index fad1559079e46..99b71c52386ca 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
@@ -45,7 +45,6 @@
 import org.junit.runners.Parameterized;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
-import org.rocksdb.Checkpoint;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.ReadOptions;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
index 61f397c38b255..4f3ed016ac036 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
@@ -61,7 +61,7 @@
  * @param <K> Type of the key by which state is keyed.
  */
 public abstract class AbstractKeyedStateBackend<K>
-		implements KeyedStateBackend<K>, Snapshotable<KeyedStateHandle>, Closeable {
+		implements KeyedStateBackend<K>, Snapshotable<KeyedStateHandle>, Closeable, CheckpointListener {
 
 	/** {@link TypeSerializer} for our key. */
 	protected final TypeSerializer<K> keySerializer;
@@ -211,16 +211,6 @@ protected abstract <N, UK, UV> InternalMapState<N, UK, UV> createMapState(
 			TypeSerializer<N> namespaceSerializer,
 			MapStateDescriptor<UK, UV> stateDesc) throws Exception;
 
-	/**
-	 * Called when the checkpoint with the given ID is completed and acknowledged on the JobManager.
-	 *
-	 * @param checkpointId The ID of the checkpoint that has been completed.
-	 *
-	 * @throws Exception Exceptions during checkpoint acknowledgement may be forwarded and will cause
-	 *                   the program to fail and enter recovery.
-	 */
-	public abstract void notifyOfCompletedCheckpoint(long checkpointId) throws Exception;
-
 	/**
 	 * @see KeyedStateBackend
 	 */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index ead89b3146859..aecc72e3bf869 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -430,7 +430,7 @@ private void restorePartitionedState(Collection<KeyedStateHandle> state) throws
 	}
 
 	@Override
-	public void notifyOfCompletedCheckpoint(long checkpointId) {
+	public void notifyCheckpointComplete(long checkpointId) {
 		//Nothing to do
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
index 60f9c819429a9..7152bfc9b8809 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
@@ -43,8 +43,8 @@
 import org.apache.flink.api.common.typeutils.base.LongSerializer;
 import org.apache.flink.api.common.typeutils.base.StringSerializer;
 import org.apache.flink.api.java.typeutils.GenericTypeInfo;
-import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
 import org.apache.flink.api.java.typeutils.runtime.kryo.JavaSerializer;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.core.testutils.CheckedThread;
 import org.apache.flink.core.testutils.OneShotLatch;
@@ -69,6 +69,7 @@
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
 import java.io.Serializable;
@@ -98,7 +99,6 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import org.junit.rules.ExpectedException;
 
 
 /**
@@ -2235,7 +2235,7 @@ public void testAsyncSnapshotCancellation() throws Exception {
 		BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
 		streamFactory.setWaiterLatch(waiter);
 		streamFactory.setBlockerLatch(blocker);
-		streamFactory.setAfterNumberInvocations(100);
+		streamFactory.setAfterNumberInvocations(10);
 
 		AbstractKeyedStateBackend<Integer> backend = null;
 		try {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java
index 291f3ed024104..1e31490a38501 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java
@@ -99,6 +99,14 @@ public void write(int b) throws IOException {
 				}
 			}
 
+			//We override this to ensure that writes go through the blocking #write(int) method!
+			@Override
+			public void write(byte[] b, int off, int len) throws IOException {
+				for (int i = 0; i < len; i++) {
+					write(b[off + i]);
+				}
+			}
+
 			@Override
 			public void close() {
 				super.close();
@@ -115,4 +123,4 @@ public void close() {
 	public void close() throws Exception {
 
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index d45ad42f790f0..8c1caee4dd2c3 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -506,7 +506,7 @@ public void initializeState(StateInitializationContext context) throws Exception
 	@Override
 	public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception {
 		if (keyedStateBackend != null) {
-			keyedStateBackend.notifyOfCompletedCheckpoint(checkpointId);
+			keyedStateBackend.notifyCheckpointComplete(checkpointId);
 		}
 	}
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
index d91c57f0f619c..dbef01f879aec 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
@@ -94,7 +94,7 @@ public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLog
 	}
 
 	enum StateBackendEnum {
-		MEM, FILE, ROCKSDB_FULLY_ASYNC, MEM_ASYNC, FILE_ASYNC
+		MEM, FILE, ROCKSDB_FULLY_ASYNC, ROCKSDB_INCREMENTAL, MEM_ASYNC, FILE_ASYNC
 	}
 
 	@BeforeClass
@@ -143,6 +143,14 @@ public void initStateBackend() throws IOException {
 				this.stateBackend = rdb;
 				break;
 			}
+			case ROCKSDB_INCREMENTAL: {
+				String rocksDb = tempFolder.newFolder().getAbsolutePath();
+				RocksDBStateBackend rdb =
+					new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), true);
+				rdb.setDbStoragePath(rocksDb);
+				this.stateBackend = rdb;
+				break;
+			}
 
 		}
 	}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java
new file mode 100644
index 0000000000000..352f9f7397e83
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.checkpointing;
+
+public class IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase {
+
+	public IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase() {
+		super(StateBackendEnum.ROCKSDB_INCREMENTAL);
+	}
+
+	@Override
+	protected int numElementsPerKey() {
+		return 3000;
+	}
+
+	@Override
+	protected int windowSize() {
+		return 1000;
+	}
+
+	@Override
+	protected int windowSlide() {
+		return 100;
+	}
+
+	@Override
+	protected int numKeys() {
+		return 100;
+	}
+}

From e2a7082e5b5b53c0d3359fb8114369db074b51f2 Mon Sep 17 00:00:00 2001
From: kl0u <kkloudas@gmail.com>
Date: Fri, 5 May 2017 15:05:52 +0200
Subject: [PATCH 154/253] [FLINK-6463] [cep] Throw exception when NOT-NEXT is
 after OPTIONAL.

---
 .../org/apache/flink/cep/pattern/Pattern.java | 15 +++--
 .../org/apache/flink/cep/nfa/NFAITCase.java   | 67 -------------------
 2 files changed, 10 insertions(+), 72 deletions(-)

diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
index 3cf25ef783f9d..2d10b41a388d5 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/Pattern.java
@@ -221,6 +221,12 @@ public Pattern<T, T> next(final String name) {
 	 * @return A new pattern which is appended to this one
 	 */
 	public Pattern<T, T> notNext(final String name) {
+		if (quantifier.hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
+			throw new UnsupportedOperationException(
+					"Specifying a pattern with an optional path to NOT condition is not supported yet. " +
+					"You can simulate such pattern with two independent patterns, one with and the other without " +
+					"the optional part.");
+		}
 		return new Pattern<>(name, this, ConsumingStrategy.NOT_NEXT);
 	}
 
@@ -247,12 +253,11 @@ public Pattern<T, T> followedBy(final String name) {
 	 */
 	public Pattern<T, T> notFollowedBy(final String name) {
 		if (quantifier.hasProperty(Quantifier.QuantifierProperty.OPTIONAL)) {
-			throw new MalformedPatternException(
-				"Specifying a pattern with an optional path to NOT condition is not supported yet. " +
-				"You can simulate such pattern with two independent patterns, one with and the other without " +
-				"the optional part.");
+			throw new UnsupportedOperationException(
+					"Specifying a pattern with an optional path to NOT condition is not supported yet. " +
+					"You can simulate such pattern with two independent patterns, one with and the other without " +
+					"the optional part.");
 		}
-
 		return new Pattern<>(name, this, ConsumingStrategy.NOT_FOLLOW);
 	}
 
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index ab6ff82bf0d65..2cc67e5d7e63a 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -3741,73 +3741,6 @@ private NotFollowByData() {
 		}
 	}
 
-	@Test
-	public void testNotNextAfterZeroOrMoreSkipTillNext() {
-		final List<List<Event>> matches = testNotNextAfterZeroOrMore(false);
-		compareMaps(matches, Lists.<List<Event>>newArrayList(
-			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.d1)
-		));
-	}
-
-	@Test
-	public void testNotNextAfterZeroOrMoreSkipTillAny() {
-		final List<List<Event>> matches = testNotNextAfterZeroOrMore(true);
-		compareMaps(matches, Lists.<List<Event>>newArrayList(
-			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.b2, NotFollowByData.d1),
-			Lists.newArrayList(NotFollowByData.a1, NotFollowByData.d1)
-		));
-	}
-
-	private List<List<Event>> testNotNextAfterZeroOrMore(boolean allMatches) {
-		List<StreamRecord<Event>> inputEvents = new ArrayList<>();
-
-		int i = 0;
-		inputEvents.add(new StreamRecord<>(NotFollowByData.a1, i++));
-		inputEvents.add(new StreamRecord<>(NotFollowByData.b1, i++));
-		inputEvents.add(new StreamRecord<>(NotFollowByData.c1, i++));
-		inputEvents.add(new StreamRecord<>(NotFollowByData.b2, i++));
-		inputEvents.add(new StreamRecord<>(NotFollowByData.d1, i++));
-
-		Pattern<Event, ?> pattern = Pattern
-			.<Event>begin("a").where(new SimpleCondition<Event>() {
-				private static final long serialVersionUID = 5726188262756267490L;
-
-				@Override
-				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("a");
-				}
-			});
-
-		pattern = (allMatches ? pattern.followedByAny("b*") : pattern.followedBy("b*")).where(new SimpleCondition<Event>() {
-			private static final long serialVersionUID = 5726188262756267490L;
-
-			@Override
-			public boolean filter(Event value) throws Exception {
-				return value.getName().equals("b");
-			}
-		}).oneOrMore().optional()
-			.notNext("not c").where(new SimpleCondition<Event>() {
-				private static final long serialVersionUID = 5726188262756267490L;
-
-				@Override
-				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("c");
-				}
-			})
-			.followedBy("d").where(new SimpleCondition<Event>() {
-				private static final long serialVersionUID = 5726188262756267490L;
-
-				@Override
-				public boolean filter(Event value) throws Exception {
-					return value.getName().equals("d");
-				}
-			});
-
-		NFA<Event> nfa = NFACompiler.compile(pattern, Event.createTypeSerializer(), false);
-
-		return feedNFA(inputEvents, nfa);
-	}
-
 	@Test
 	public void testNotNextAfterOneOrMoreSkipTillNext() {
 		final List<List<Event>> matches = testNotNextAfterOneOrMore(false);

From 00b4692a3f5056b935e2dc96cf1cb39f13dc6560 Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Thu, 2 Mar 2017 16:06:55 +0100
Subject: [PATCH 155/253] [FLINK-5884] [table] Integrate time indicators for
 Table API & SQL

This closes #3808.
---
 .../table/api/BatchTableEnvironment.scala     |  25 +-
 .../table/api/StreamTableEnvironment.scala    |  79 ++--
 .../flink/table/api/TableEnvironment.scala    | 101 +++-
 .../flink/table/api/scala/expressionDsl.scala |  18 +-
 .../flink/table/api/scala/windows.scala       |   2 +-
 .../org/apache/flink/table/api/table.scala    |  14 +-
 .../org/apache/flink/table/api/windows.scala  | 357 +++++++-------
 .../calcite/FlinkCalciteSqlValidator.scala    |  12 +-
 .../table/calcite/FlinkPlannerImpl.scala      |   6 +-
 .../table/calcite/FlinkTypeFactory.scala      | 132 +++++-
 .../calcite/RelTimeIndicatorConverter.scala   | 222 +++++++++
 .../flink/table/codegen/CodeGenerator.scala   |  99 ++--
 .../codegen/calls/FunctionGenerator.scala     |  10 -
 .../table/expressions/ExpressionUtils.scala   |  39 ++
 .../apache/flink/table/expressions/call.scala |  16 +-
 .../table/expressions/fieldExpression.scala   |  48 +-
 .../TimeMaterializationSqlFunction.scala      |  41 ++
 .../TimeModeIndicatorFunctions.scala          | 137 ------
 .../table/plan/ProjectionTranslator.scala     |  31 +-
 .../table/plan/logical/LogicalWindow.scala    |  14 +-
 .../table/plan/logical/groupWindows.scala     | 280 ++++-------
 .../flink/table/plan/logical/operators.scala  |  40 +-
 .../flink/table/plan/nodes/CommonCalc.scala   |  37 +-
 .../table/plan/nodes/CommonCorrelate.scala    |  32 +-
 .../flink/table/plan/nodes/CommonScan.scala   |  24 +-
 .../table/plan/nodes/OverAggregate.scala      |  35 +-
 .../plan/nodes/PhysicalTableSourceScan.scala  |   6 +-
 .../table/plan/nodes/dataset/BatchScan.scala  |  12 +-
 .../nodes/dataset/BatchTableSourceScan.scala  |  14 +-
 .../plan/nodes/dataset/DataSetAggregate.scala |   8 +-
 .../plan/nodes/dataset/DataSetCalc.scala      |  21 +-
 .../plan/nodes/dataset/DataSetCorrelate.scala |   9 +-
 .../dataset/DataSetWindowAggregate.scala      |  68 ++-
 .../datastream/DataStreamAggregate.scala      | 178 +++----
 .../nodes/datastream/DataStreamCalc.scala     |  22 +-
 .../datastream/DataStreamCorrelate.scala      |  22 +-
 .../datastream/DataStreamOverAggregate.scala  | 112 +++--
 .../plan/nodes/datastream/DataStreamRel.scala |   1 -
 .../nodes/datastream/DataStreamScan.scala     |  10 +-
 .../nodes/datastream/DataStreamUnion.scala    |  14 +-
 .../nodes/datastream/DataStreamValues.scala   |  21 +-
 .../plan/nodes/datastream/StreamScan.scala    |  32 +-
 .../datastream/StreamTableSourceScan.scala    |  58 ++-
 .../logical/FlinkLogicalOverWindow.scala      |   2 +-
 .../logical/FlinkLogicalTableSourceScan.scala |   6 +-
 .../common/WindowStartEndPropertiesRule.scala |   4 +-
 .../datastream/DataStreamAggregateRule.scala  |   5 +-
 .../rules/datastream/DataStreamCalcRule.scala |   4 +-
 .../datastream/DataStreamCorrelateRule.scala  |   6 +-
 ...DataStreamLogicalWindowAggregateRule.scala |  56 ++-
 .../DataStreamOverAggregateRule.scala         |   5 +-
 .../rules/datastream/DataStreamScanRule.scala |   4 +-
 .../datastream/DataStreamUnionRule.scala      |   3 +-
 .../datastream/DataStreamValuesRule.scala     |   3 +-
 .../table/plan/schema/DataStreamTable.scala   |  14 +
 .../flink/table/plan/schema/FlinkTable.scala  |   7 +-
 .../flink/table/plan/schema/RowSchema.scala   | 152 ++++++
 .../schema/TimeIndicatorRelDataType.scala     |  49 ++
 .../flink/table/runtime/MapRunner.scala       |   2 +-
 .../runtime/aggregate/AggregateUtil.scala     | 190 ++++----
 .../aggregate/RowTimeBoundedRangeOver.scala   |   6 +-
 .../aggregate/RowTimeBoundedRowsOver.scala    |   2 +-
 .../table/sources/DefinedTimeAttributes.scala |  47 ++
 .../typeutils/TimeIndicatorTypeInfo.scala     |  45 ++
 .../table/typeutils/TypeCheckUtils.scala      |   5 +-
 .../table/validate/FunctionCatalog.scala      |  10 +-
 .../java/batch/TableEnvironmentITCase.java    |   9 -
 .../flink/table/TableEnvironmentTest.scala    |  55 ++-
 .../scala/batch/TableEnvironmentITCase.scala  |  10 -
 .../scala/batch/sql/WindowAggregateTest.scala |  18 +-
 .../batch/table/FieldProjectionTest.scala     |  36 +-
 .../scala/batch/table/GroupWindowTest.scala   | 121 ++---
 .../api/scala/stream/sql/SqlITCase.scala      | 150 +++---
 .../stream/sql/WindowAggregateTest.scala      | 179 ++++---
 .../stream/table/AggregationsITCase.scala     |  12 +-
 .../api/scala/stream/table/CalcITCase.scala   |  16 -
 .../scala/stream/table/GroupWindowTest.scala  | 440 ++++++------------
 .../scala/stream/table/OverWindowITCase.scala |  12 +-
 .../scala/stream/table/OverWindowTest.scala   | 101 ++--
 .../GroupWindowStringExpressionTest.scala     |   6 +-
 .../OverWindowStringExpressionTest.scala      |  16 +-
 .../DataStreamAggregateITCase.scala           |  22 +-
 82 files changed, 2368 insertions(+), 1921 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeMaterializationSqlFunction.scala
 delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeModeIndicatorFunctions.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/TimeIndicatorRelDataType.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TimeIndicatorTypeInfo.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index 00cf11cf61220..3eb2ffc269826 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -194,7 +194,30 @@ abstract class BatchTableEnvironment(
   protected def registerDataSetInternal[T](
       name: String, dataSet: DataSet[T], fields: Array[Expression]): Unit = {
 
-    val (fieldNames, fieldIndexes) = getFieldInfo[T](dataSet.getType, fields)
+    val (fieldNames, fieldIndexes) = getFieldInfo[T](
+      dataSet.getType,
+      fields,
+      ignoreTimeAttributes = true)
+
+    // validate and extract time attributes
+    val (rowtime, proctime) = validateAndExtractTimeAttributes(fieldNames, fieldIndexes, fields)
+
+    // don't allow proctime on batch
+    proctime match {
+      case Some(_) =>
+        throw new ValidationException(
+          "A proctime attribute is not allowed in a batch environment. " +
+            "Working with processing-time on batch would lead to non-deterministic results.")
+      case _ => // ok
+    }
+    // rowtime must not extend the schema of a batch table
+    rowtime match {
+      case Some((idx, _)) if idx >= dataSet.getType.getArity =>
+        throw new ValidationException(
+          "A rowtime attribute must be defined on an existing field in a batch environment.")
+      case _ => // ok
+    }
+
     val dataSetTable = new DataSetTable[T](dataSet, fieldIndexes, fieldNames)
     registerTableInternal(name, dataSetTable)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index f532c5b97386f..d1f2fb544da0b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -30,6 +30,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+import org.apache.flink.table.calcite.RelTimeIndicatorConverter
 import org.apache.flink.table.explain.PlanJsonParser
 import org.apache.flink.table.expressions.Expression
 import org.apache.flink.table.plan.nodes.FlinkConventions
@@ -86,47 +87,6 @@ abstract class StreamTableEnvironment(
   /** Returns a unique table name according to the internal naming pattern. */
   protected def createUniqueTableName(): String = "_DataStreamTable_" + nameCntr.getAndIncrement()
 
-  /**
-    * Returns field names and field positions for a given [[TypeInformation]].
-    *
-    * Field names are automatically extracted for
-    * [[org.apache.flink.api.common.typeutils.CompositeType]].
-    * The method fails if inputType is not a
-    * [[org.apache.flink.api.common.typeutils.CompositeType]].
-    *
-    * @param inputType The TypeInformation extract the field names and positions from.
-    * @tparam A The type of the TypeInformation.
-    * @return A tuple of two arrays holding the field names and corresponding field positions.
-    */
-  override protected[flink] def getFieldInfo[A](inputType: TypeInformation[A])
-    : (Array[String], Array[Int]) = {
-    val fieldInfo = super.getFieldInfo(inputType)
-    if (fieldInfo._1.contains("rowtime")) {
-      throw new TableException("'rowtime' ia a reserved field name in stream environment.")
-    }
-    fieldInfo
-  }
-
-  /**
-    * Returns field names and field positions for a given [[TypeInformation]] and [[Array]] of
-    * [[Expression]].
-    *
-    * @param inputType The [[TypeInformation]] against which the [[Expression]]s are evaluated.
-    * @param exprs     The expressions that define the field names.
-    * @tparam A The type of the TypeInformation.
-    * @return A tuple of two arrays holding the field names and corresponding field positions.
-    */
-  override protected[flink] def getFieldInfo[A](
-      inputType: TypeInformation[A],
-      exprs: Array[Expression])
-    : (Array[String], Array[Int]) = {
-    val fieldInfo = super.getFieldInfo(inputType, exprs)
-    if (fieldInfo._1.contains("rowtime")) {
-      throw new TableException("'rowtime' is a reserved field name in stream environment.")
-    }
-    fieldInfo
-  }
-
   /**
     * Registers an external [[StreamTableSource]] in this [[TableEnvironment]]'s catalog.
     * Registered tables can be referenced in SQL queries.
@@ -145,6 +105,7 @@ abstract class StreamTableEnvironment(
             "StreamTableEnvironment")
     }
   }
+
   /**
     * Writes a [[Table]] to a [[TableSink]].
     *
@@ -185,7 +146,9 @@ abstract class StreamTableEnvironment(
     val dataStreamTable = new DataStreamTable[T](
       dataStream,
       fieldIndexes,
-      fieldNames
+      fieldNames,
+      None,
+      None
     )
     registerTableInternal(name, dataStreamTable)
   }
@@ -200,15 +163,26 @@ abstract class StreamTableEnvironment(
     * @tparam T The type of the [[DataStream]].
     */
   protected def registerDataStreamInternal[T](
-    name: String,
-    dataStream: DataStream[T],
-    fields: Array[Expression]): Unit = {
+      name: String,
+      dataStream: DataStream[T],
+      fields: Array[Expression])
+    : Unit = {
+
+    val (fieldNames, fieldIndexes) = getFieldInfo[T](
+      dataStream.getType,
+      fields,
+      ignoreTimeAttributes = false)
+
+    // validate and extract time attributes
+    val (rowtime, proctime) = validateAndExtractTimeAttributes(fieldNames, fieldIndexes, fields)
+
 
-    val (fieldNames, fieldIndexes) = getFieldInfo[T](dataStream.getType, fields)
     val dataStreamTable = new DataStreamTable[T](
       dataStream,
       fieldIndexes,
-      fieldNames
+      fieldNames,
+      rowtime,
+      proctime
     )
     registerTableInternal(name, dataStreamTable)
   }
@@ -259,7 +233,10 @@ abstract class StreamTableEnvironment(
     // 1. decorrelate
     val decorPlan = RelDecorrelator.decorrelateQuery(relNode)
 
-    // 2. normalize the logical plan
+    // 2. convert time indicators
+    val convPlan = RelTimeIndicatorConverter.convert(decorPlan, getRelBuilder.getRexBuilder)
+
+    // 3. normalize the logical plan
     val normRuleSet = getNormRuleSet
     val normalizedPlan = if (normRuleSet.iterator().hasNext) {
       runHepPlanner(HepMatchOrder.BOTTOM_UP, normRuleSet, decorPlan, decorPlan.getTraitSet)
@@ -267,7 +244,7 @@ abstract class StreamTableEnvironment(
       decorPlan
     }
 
-    // 3. optimize the logical Flink plan
+    // 4. optimize the logical Flink plan
     val logicalOptRuleSet = getLogicalOptRuleSet
     val logicalOutputProps = relNode.getTraitSet.replace(FlinkConventions.LOGICAL).simplify()
     val logicalPlan = if (logicalOptRuleSet.iterator().hasNext) {
@@ -276,7 +253,7 @@ abstract class StreamTableEnvironment(
       normalizedPlan
     }
 
-    // 4. optimize the physical Flink plan
+    // 5. optimize the physical Flink plan
     val physicalOptRuleSet = getPhysicalOptRuleSet
     val physicalOutputProps = relNode.getTraitSet.replace(FlinkConventions.DATASTREAM).simplify()
     val physicalPlan = if (physicalOptRuleSet.iterator().hasNext) {
@@ -285,7 +262,7 @@ abstract class StreamTableEnvironment(
       logicalPlan
     }
 
-    // 5. decorate the optimized plan
+    // 6. decorate the optimized plan
     val decoRuleSet = getDecoRuleSet
     val decoratedPlan = if (decoRuleSet.iterator().hasNext) {
       runHepPlanner(HepMatchOrder.BOTTOM_UP, decoRuleSet, physicalPlan, physicalPlan.getTraitSet)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 06c405eccff38..4c72e8ff6d27d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -52,6 +52,9 @@ import org.apache.flink.table.codegen.{CodeGenerator, ExpressionReducer}
 import org.apache.flink.table.expressions.{Alias, Expression, UnresolvedFieldReference}
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
 import org.apache.flink.table.functions.{ScalarFunction, TableFunction, AggregateFunction}
+import org.apache.flink.table.expressions._
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{checkForInstantiation, checkNotSingleton, createScalarSqlFunction, createTableSqlFunctions}
+import org.apache.flink.table.functions.{ScalarFunction, TableFunction}
 import org.apache.flink.table.plan.cost.DataSetCostFactory
 import org.apache.flink.table.plan.logical.{CatalogNode, LogicalRelNode}
 import org.apache.flink.table.plan.rules.FlinkRuleSets
@@ -598,70 +601,94 @@ abstract class TableEnvironment(val config: TableConfig) {
 
   /**
     * Returns field names and field positions for a given [[TypeInformation]] and [[Array]] of
-    * [[Expression]].
+    * [[Expression]]. It does not handle time attributes but considers them in indices, if
+    * ignore flag is not false.
     *
     * @param inputType The [[TypeInformation]] against which the [[Expression]]s are evaluated.
     * @param exprs The expressions that define the field names.
+    * @param ignoreTimeAttributes ignore time attributes and handle them as regular expressions.
     * @tparam A The type of the TypeInformation.
     * @return A tuple of two arrays holding the field names and corresponding field positions.
     */
   protected[flink] def getFieldInfo[A](
-    inputType: TypeInformation[A],
-    exprs: Array[Expression]): (Array[String], Array[Int]) = {
+      inputType: TypeInformation[A],
+      exprs: Array[Expression],
+      ignoreTimeAttributes: Boolean)
+    : (Array[String], Array[Int]) = {
 
     TableEnvironment.validateType(inputType)
 
+    val filteredExprs = if (ignoreTimeAttributes) {
+        exprs.map {
+          case ta: TimeAttribute => ta.expression
+          case e@_ => e
+        }
+    } else {
+      exprs
+    }
+
     val indexedNames: Array[(Int, String)] = inputType match {
       case g: GenericTypeInfo[A] if g.getTypeClass == classOf[Row] =>
         throw new TableException(
           "An input of GenericTypeInfo<Row> cannot be converted to Table. " +
             "Please specify the type of the input with a RowTypeInfo.")
       case a: AtomicType[A] =>
-        if (exprs.length != 1) {
-          throw new TableException("Table of atomic type can only have a single field.")
-        }
-        exprs.map {
-          case UnresolvedFieldReference(name) => (0, name)
+        filteredExprs.zipWithIndex flatMap {
+          case (UnresolvedFieldReference(name), idx) =>
+            if (idx > 0) {
+              throw new TableException("Table of atomic type can only have a single field.")
+            }
+            Some((0, name))
+          case (_: TimeAttribute, _) if ignoreTimeAttributes =>
+            None
           case _ => throw new TableException("Field reference expression requested.")
         }
       case t: TupleTypeInfo[A] =>
-        exprs.zipWithIndex.map {
-          case (UnresolvedFieldReference(name), idx) => (idx, name)
+        filteredExprs.zipWithIndex flatMap {
+          case (UnresolvedFieldReference(name), idx) =>
+            Some((idx, name))
           case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
             val idx = t.getFieldIndex(origName)
             if (idx < 0) {
               throw new TableException(s"$origName is not a field of type $t")
             }
-            (idx, name)
+            Some((idx, name))
+          case (_: TimeAttribute, _) =>
+            None
           case _ => throw new TableException(
             "Field reference expression or alias on field expression expected.")
         }
       case c: CaseClassTypeInfo[A] =>
-        exprs.zipWithIndex.map {
-          case (UnresolvedFieldReference(name), idx) => (idx, name)
+        filteredExprs.zipWithIndex flatMap {
+          case (UnresolvedFieldReference(name), idx) =>
+            Some((idx, name))
           case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
             val idx = c.getFieldIndex(origName)
             if (idx < 0) {
               throw new TableException(s"$origName is not a field of type $c")
             }
-            (idx, name)
+            Some((idx, name))
+          case (_: TimeAttribute, _) =>
+            None
           case _ => throw new TableException(
             "Field reference expression or alias on field expression expected.")
         }
       case p: PojoTypeInfo[A] =>
-        exprs.map {
+        filteredExprs flatMap {
           case (UnresolvedFieldReference(name)) =>
             val idx = p.getFieldIndex(name)
             if (idx < 0) {
               throw new TableException(s"$name is not a field of type $p")
             }
-            (idx, name)
+            Some((idx, name))
           case Alias(UnresolvedFieldReference(origName), name, _) =>
             val idx = p.getFieldIndex(origName)
             if (idx < 0) {
               throw new TableException(s"$origName is not a field of type $p")
             }
-            (idx, name)
+            Some((idx, name))
+          case _: TimeAttribute =>
+            None
           case _ => throw new TableException(
             "Field reference expression or alias on field expression expected.")
         }
@@ -795,6 +822,42 @@ abstract class TableEnvironment(val config: TableConfig) {
     Some(mapFunction)
   }
 
+  /**
+    * Checks for at most one rowtime and proctime attribute.
+    * Returns the time attributes.
+    *
+    * @return rowtime attribute and proctime attribute
+    */
+  protected def validateAndExtractTimeAttributes(
+      fieldNames: Seq[String],
+      fieldIndices: Seq[Int],
+      exprs: Array[Expression])
+    : (Option[(Int, String)], Option[(Int, String)]) = {
+
+    var rowtime: Option[(Int, String)] = None
+    var proctime: Option[(Int, String)] = None
+
+    exprs.zipWithIndex.foreach {
+      case (RowtimeAttribute(reference@UnresolvedFieldReference(name)), idx) =>
+        if (rowtime.isDefined) {
+          throw new TableException(
+            "The rowtime attribute can only be defined once in a table schema.")
+        } else {
+          rowtime = Some(idx, name)
+        }
+      case (ProctimeAttribute(reference@UnresolvedFieldReference(name)), idx) =>
+        if (proctime.isDefined) {
+          throw new TableException(
+            "The proctime attribute can only be defined once in a table schema.")
+        } else {
+          proctime = Some(idx, name)
+        }
+      case _ =>
+        // do nothing
+    }
+
+    (rowtime, proctime)
+  }
 }
 
 /**
@@ -803,6 +866,10 @@ abstract class TableEnvironment(val config: TableConfig) {
   */
 object TableEnvironment {
 
+  // default names that can be used in in TableSources etc.
+  val DEFAULT_ROWTIME_ATTRIBUTE = "rowtime"
+  val DEFAULT_PROCTIME_ATTRIBUTE = "proctime"
+
   /**
     * Returns a [[JavaBatchTableEnv]] for a Java [[JavaBatchExecEnv]].
     *
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
index cc58ff54fbf10..6d15212ec0649 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala
@@ -625,7 +625,7 @@ trait ImplicitExpressionOperations {
     */
   def millis = milli
 
-  // row interval type
+  // Row interval type
 
   /**
     * Creates an interval of rows.
@@ -634,6 +634,8 @@ trait ImplicitExpressionOperations {
     */
   def rows = toRowInterval(expr)
 
+  // Advanced type helper functions
+
   /**
     * Accesses the field of a Flink composite type (such as Tuple, POJO, etc.) by name and
     * returns it's value.
@@ -680,6 +682,20 @@ trait ImplicitExpressionOperations {
     * @return the first and only element of an array with a single element
     */
   def element() = ArrayElement(expr)
+
+  // Schema definition
+
+  /**
+    * Declares a field as the rowtime attribute for indicating, accessing, and working in
+    * Flink's event time.
+    */
+  def rowtime = RowtimeAttribute(expr)
+
+  /**
+    * Declares a field as the proctime attribute for indicating, accessing, and working in
+    * Flink's processing time.
+    */
+  def proctime = ProctimeAttribute(expr)
 }
 
 /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
index 5e7044053f38b..d0430c2340bde 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
@@ -18,8 +18,8 @@
 
 package org.apache.flink.table.api.scala
 
+import org.apache.flink.table.api.{OverWindowWithOrderBy, SessionWithGap, SlideWithSize, TumbleWithSize}
 import org.apache.flink.table.expressions.Expression
-import org.apache.flink.table.api.{TumbleWithSize, OverWindowWithOrderBy, SlideWithSize, SessionWithGap}
 
 /**
   * Helper object for creating a tumbling window. Tumbling windows are consecutive, non-overlapping
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 87dde0aa68cf0..dd8265b6f01c5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -20,12 +20,12 @@ package org.apache.flink.table.api
 import org.apache.calcite.rel.RelNode
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.operators.join.JoinType
-import org.apache.flink.table.calcite.{FlinkRelBuilder, FlinkTypeFactory}
+import org.apache.flink.table.calcite.FlinkRelBuilder
+import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.expressions.{Alias, Asc, Expression, ExpressionParser, Ordering, UnresolvedAlias, UnresolvedFieldReference}
-import org.apache.flink.table.plan.logical.Minus
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
 import org.apache.flink.table.plan.ProjectionTranslator._
-import org.apache.flink.table.plan.logical._
+import org.apache.flink.table.plan.logical.{Minus, _}
 import org.apache.flink.table.sinks.TableSink
 
 import _root_.scala.collection.JavaConverters._
@@ -1015,13 +1015,7 @@ class WindowGroupedTable(
     val projectsOnAgg = replaceAggregationsAndProperties(
       fields, table.tableEnv, aggNames, propNames)
 
-    val projectFields = (table.tableEnv, window) match {
-      // event time can be arbitrary field in batch environment
-      case (_: BatchTableEnvironment, w: EventTimeWindow) =>
-        extractFieldReferences(fields ++ groupKeys ++ Seq(w.timeField))
-      case (_, _) =>
-        extractFieldReferences(fields ++ groupKeys)
-    }
+    val projectFields = extractFieldReferences(fields ++ groupKeys :+ window.timeField)
 
     new Table(table.tableEnv,
       Project(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
index 80260f7d80e22..11ef3602d502a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/windows.scala
@@ -149,7 +149,7 @@ class OverWindowWithOrderBy(
   * A window specification.
   *
   * Window groups rows based on time or row-count intervals. It is a general way to group the
-  * elements, which is very helpful for both groupby-aggregations and over-aggregations to
+  * elements, which is very helpful for both groupBy-aggregations and over-aggregations to
   * compute aggregates on groups of elements.
   *
   * Infinite streaming tables can only be grouped into time or row intervals. Hence window grouping
@@ -157,111 +157,73 @@ class OverWindowWithOrderBy(
   *
   * For finite batch tables, window provides shortcuts for time-based groupBy.
   *
-  * @param alias The expression of alias for this Window
   */
-abstract class Window(val alias: Expression) {
+abstract class Window(val alias: Expression, val timeField: Expression) {
 
   /**
     * Converts an API class to a logical window for planning.
     */
   private[flink] def toLogicalWindow: LogicalWindow
+
 }
 
+// ------------------------------------------------------------------------------------------------
+// Tumbling windows
+// ------------------------------------------------------------------------------------------------
+
 /**
-  * A window specification without alias.
+  * Tumbling window.
+  *
+  * For streaming tables you can specify grouping by a event-time or processing-time attribute.
+  *
+  * For batch tables you can specify grouping on a timestamp or long attribute.
+  *
+  * @param size the size of the window either as time or row-count interval.
   */
-abstract class WindowWithoutAlias {
+class TumbleWithSize(size: Expression) {
 
   /**
-    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
-    * refer to. `select()` statement can access window properties such as window start or end time.
+    * Tumbling window.
     *
-    * @param alias alias for this window
-    * @return this window
-    */
-  def as(alias: Expression): Window
-
-  /**
-    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
-    * refer to. `select()` statement can access window properties such as window start or end time.
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
     *
-    * @param alias alias for this window
-    * @return this window
+    * For batch tables you can specify grouping on a timestamp or long attribute.
+    *
+    * @param size the size of the window either as time or row-count interval.
     */
-  def as(alias: String): Window = as(ExpressionParser.parseExpression(alias))
-}
-
-/**
-  * A predefined specification of window on processing-time
-  */
-abstract class ProcTimeWindowWithoutAlias extends WindowWithoutAlias {
+  def this(size: String) = this(ExpressionParser.parseExpression(size))
 
   /**
     * Specifies the time attribute on which rows are grouped.
     *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
     *
-    * For batch tables, refer to a timestamp or long attribute.
+    * For batch tables you can specify grouping on a timestamp or long attribute.
     *
-    * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a predefined window on event-time
+    * @param timeField time attribute for streaming and batch tables
+    * @return a tumbling window on event-time
     */
-  def on(timeField: Expression): WindowWithoutAlias
+  def on(timeField: Expression): TumbleWithSizeOnTime =
+    new TumbleWithSizeOnTime(timeField, size)
 
   /**
     * Specifies the time attribute on which rows are grouped.
     *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
     *
-    * For batch tables, refer to a timestamp or long attribute.
+    * For batch tables you can specify grouping on a timestamp or long attribute.
     *
-    * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a predefined window on event-time
+    * @param timeField time attribute for streaming and batch tables
+    * @return a tumbling window on event-time
     */
-  def on(timeField: String): WindowWithoutAlias =
+  def on(timeField: String): TumbleWithSizeOnTime =
     on(ExpressionParser.parseExpression(timeField))
 }
 
 /**
-  * A window operating on event-time.
-  *
-  * For streaming tables call on('rowtime) to specify grouping by event-time.
-  * Otherwise rows are grouped by processing-time.
-  *
-  * For batch tables, refer to a timestamp or long attribute.
-  *
-  * @param timeField time mode for streaming tables and time attribute for batch tables
+  * Tumbling window on time.
   */
-abstract class EventTimeWindow(alias: Expression, val timeField: Expression) extends Window(alias)
-
-// ------------------------------------------------------------------------------------------------
-// Tumbling windows
-// ------------------------------------------------------------------------------------------------
-
-/**
-  * A partial specification of a tumbling window.
-  *
-  * @param size the size of the window either a time or a row-count interval.
-  */
-class TumbleWithSize(size: Expression) extends ProcTimeWindowWithoutAlias {
-
-  def this(size: String) = this(ExpressionParser.parseExpression(size))
-
-  /**
-    * Specifies the time attribute on which rows are grouped.
-    *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time.
-    * Otherwise rows are grouped by processing-time.
-    *
-    * For batch tables, refer to a timestamp or long attribute.
-    *
-    * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a predefined window on event-time
-    */
-  override def on(timeField: Expression): WindowWithoutAlias =
-    new TumbleWithoutAlias(timeField, size)
+class TumbleWithSizeOnTime(time: Expression, size: Expression) {
 
   /**
     * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
@@ -270,15 +232,9 @@ class TumbleWithSize(size: Expression) extends ProcTimeWindowWithoutAlias {
     * @param alias alias for this window
     * @return this window
     */
-  override def as(alias: Expression) = new TumblingWindow(alias, size)
-}
-
-/**
-  * A tumbling window on event-time without alias.
-  */
-class TumbleWithoutAlias(
-    time: Expression,
-    size: Expression) extends WindowWithoutAlias {
+  def as(alias: Expression): TumbleWithSizeOnTimeWithAlias = {
+    new TumbleWithSizeOnTimeWithAlias(alias, time, size)
+  }
 
   /**
     * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
@@ -287,31 +243,28 @@ class TumbleWithoutAlias(
     * @param alias alias for this window
     * @return this window
     */
-  override def as(alias: Expression): Window = new TumblingEventTimeWindow(alias, time, size)
-}
-
-/**
-  * Tumbling window on processing-time.
-  *
-  * @param alias the alias of the window.
-  * @param size the size of the window either a time or a row-count interval.
-  */
-class TumblingWindow(alias: Expression, size: Expression) extends Window(alias) {
-
-  override private[flink] def toLogicalWindow: LogicalWindow =
-    ProcessingTimeTumblingGroupWindow(alias, size)
+  def as(alias: String): TumbleWithSizeOnTimeWithAlias = {
+    as(ExpressionParser.parseExpression(alias))
+  }
 }
 
 /**
-  * Tumbling window on event-time.
+  * Tumbling window on time with alias. Fully specifies a window.
   */
-class TumblingEventTimeWindow(
+class TumbleWithSizeOnTimeWithAlias(
     alias: Expression,
-    time: Expression,
-    size: Expression) extends EventTimeWindow(alias, time) {
+    timeField: Expression,
+    size: Expression)
+  extends Window(
+    alias,
+    timeField) {
 
-  override private[flink] def toLogicalWindow: LogicalWindow =
-    EventTimeTumblingGroupWindow(alias, time, size)
+  /**
+    * Converts an API class to a logical window for planning.
+    */
+  override private[flink] def toLogicalWindow: LogicalWindow = {
+    TumblingGroupWindow(alias, timeField, size)
+  }
 }
 
 // ------------------------------------------------------------------------------------------------
@@ -319,16 +272,16 @@ class TumblingEventTimeWindow(
 // ------------------------------------------------------------------------------------------------
 
 /**
-  * A partially specified sliding window.
+  * Partially specified sliding window.
   *
-  * @param size the size of the window either a time or a row-count interval.
+  * @param size the size of the window either as time or row-count interval.
   */
 class SlideWithSize(size: Expression) {
 
   /**
-    * A partially specified sliding window.
+    * Partially specified sliding window.
     *
-    * @param size the size of the window either a time or a row-count interval.
+    * @param size the size of the window either as time or row-count interval.
     */
   def this(size: String) = this(ExpressionParser.parseExpression(size))
 
@@ -343,9 +296,9 @@ class SlideWithSize(size: Expression) {
     * windows.
     *
     * @param slide the slide of the window either as time or row-count interval.
-    * @return a predefined sliding window.
+    * @return a sliding window
     */
-  def every(slide: Expression): SlideWithSlide = new SlideWithSlide(size, slide)
+  def every(slide: Expression): SlideWithSizeAndSlide = new SlideWithSizeAndSlide(size, slide)
 
   /**
     * Specifies the window's slide as time or row-count interval.
@@ -358,48 +311,54 @@ class SlideWithSize(size: Expression) {
     * windows.
     *
     * @param slide the slide of the window either as time or row-count interval.
-    * @return a predefined sliding window.
+    * @return a sliding window
     */
-  def every(slide: String): WindowWithoutAlias = every(ExpressionParser.parseExpression(slide))
+  def every(slide: String): SlideWithSizeAndSlide = every(ExpressionParser.parseExpression(slide))
 }
 
 /**
-  * A partially defined sliding window.
+  * Sliding window.
+  *
+  * For streaming tables you can specify grouping by a event-time or processing-time attribute.
+  *
+  * For batch tables you can specify grouping on a timestamp or long attribute.
+  *
+  * @param size the size of the window either as time or row-count interval.
   */
-class SlideWithSlide(
-    size: Expression,
-    slide: Expression) extends ProcTimeWindowWithoutAlias {
+class SlideWithSizeAndSlide(size: Expression, slide: Expression) {
+
   /**
     * Specifies the time attribute on which rows are grouped.
     *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
     *
-    * For batch tables, refer to a timestamp or long attribute.
+    * For batch tables you can specify grouping on a timestamp or long attribute.
     *
-    * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return a predefined Sliding window on event-time.
+    * @param timeField time attribute for streaming and batch tables
+    * @return a tumbling window on event-time
     */
-  override def on(timeField: Expression): SlideWithoutAlias =
-    new SlideWithoutAlias(timeField, size, slide)
+  def on(timeField: Expression): SlideWithSizeAndSlideOnTime =
+    new SlideWithSizeAndSlideOnTime(timeField, size, slide)
 
   /**
-    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
-    * refer to. `select()` statement can access window properties such as window start or end time.
+    * Specifies the time attribute on which rows are grouped.
     *
-    * @param alias alias for this window
-    * @return this window
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
+    *
+    * For batch tables you can specify grouping on a timestamp or long attribute.
+    *
+    * @param timeField time attribute for streaming and batch tables
+    * @return a tumbling window on event-time
     */
-  override def as(alias: Expression): Window = new SlidingWindow(alias, size, slide)
+  def on(timeField: String): SlideWithSizeAndSlideOnTime =
+    on(ExpressionParser.parseExpression(timeField))
 }
 
 /**
-  * A partially defined sliding window on event-time without alias.
+  * Sliding window on time.
   */
-class SlideWithoutAlias(
-    timeField: Expression,
-    size: Expression,
-    slide: Expression) extends WindowWithoutAlias {
+class SlideWithSizeAndSlideOnTime(timeField: Expression, size: Expression, slide: Expression) {
+
   /**
     * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
     * refer to. `select()` statement can access window properties such as window start or end time.
@@ -407,39 +366,40 @@ class SlideWithoutAlias(
     * @param alias alias for this window
     * @return this window
     */
-  override def as(alias: Expression): Window =
-    new SlidingEventTimeWindow(alias, timeField, size, slide)
-}
+  def as(alias: Expression): SlideWithSizeAndSlideOnTimeWithAlias = {
+    new SlideWithSizeAndSlideOnTimeWithAlias(alias, timeField, size, slide)
+  }
 
-/**
-  * A sliding window on processing-time.
-  *
-  * @param alias the alias of the window.
-  * @param size the size of the window either a time or a row-count interval.
-  * @param slide the interval by which the window slides.
-  */
-class SlidingWindow(
-  alias: Expression,
-  size: Expression,
-  slide: Expression)
-  extends Window(alias) {
-
-  override private[flink] def toLogicalWindow: LogicalWindow =
-    ProcessingTimeSlidingGroupWindow(alias, size, slide)
+  /**
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
+    *
+    * @param alias alias for this window
+    * @return this window
+    */
+  def as(alias: String): SlideWithSizeAndSlideOnTimeWithAlias = {
+    as(ExpressionParser.parseExpression(alias))
+  }
 }
 
 /**
-  * A sliding window on event-time.
+  * Sliding window on time with alias. Fully specifies a window.
   */
-class SlidingEventTimeWindow(
+class SlideWithSizeAndSlideOnTimeWithAlias(
     alias: Expression,
     timeField: Expression,
     size: Expression,
     slide: Expression)
-  extends EventTimeWindow(alias, timeField) {
+  extends Window(
+    alias,
+    timeField) {
 
-  override private[flink] def toLogicalWindow: LogicalWindow =
-    EventTimeSlidingGroupWindow(alias, timeField, size, slide)
+  /**
+    * Converts an API class to a logical window for planning.
+    */
+  override private[flink] def toLogicalWindow: LogicalWindow = {
+    SlidingGroupWindow(alias, timeField, size, slide)
+  }
 }
 
 // ------------------------------------------------------------------------------------------------
@@ -447,42 +407,59 @@ class SlidingEventTimeWindow(
 // ------------------------------------------------------------------------------------------------
 
 /**
-  * A partially defined session window.
+  * Session window.
+  *
+  * For streaming tables you can specify grouping by a event-time or processing-time attribute.
+  *
+  * For batch tables you can specify grouping on a timestamp or long attribute.
+  *
+  * @param gap the time interval of inactivity before a window is closed.
   */
-class SessionWithGap(gap: Expression) extends ProcTimeWindowWithoutAlias {
+class SessionWithGap(gap: Expression) {
 
+  /**
+    * Session window.
+    *
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
+    *
+    * For batch tables you can specify grouping on a timestamp or long attribute.
+    *
+    * @param gap the time interval of inactivity before a window is closed.
+    */
   def this(gap: String) = this(ExpressionParser.parseExpression(gap))
 
   /**
     * Specifies the time attribute on which rows are grouped.
     *
-    * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
-    * are grouped by processing-time.
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
     *
-    * For batch tables, refer to a timestamp or long attribute.
+    * For batch tables you can specify grouping on a timestamp or long attribute.
     *
-    * @param timeField time mode for streaming tables and time attribute for batch tables
-    * @return an on event-time session window on event-time
+    * @param timeField time attribute for streaming and batch tables
+    * @return a tumbling window on event-time
     */
-  override def on(timeField: Expression): SessionWithoutAlias =
-    new SessionWithoutAlias(timeField, gap)
+  def on(timeField: Expression): SessionWithGapOnTime =
+    new SessionWithGapOnTime(timeField, gap)
 
   /**
-    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
-    * refer to. `select()` statement can access window properties such as window start or end time.
+    * Specifies the time attribute on which rows are grouped.
     *
-    * @param alias alias for this window
-    * @return this window
+    * For streaming tables you can specify grouping by a event-time or processing-time attribute.
+    *
+    * For batch tables you can specify grouping on a timestamp or long attribute.
+    *
+    * @param timeField time attribute for streaming and batch tables
+    * @return a tumbling window on event-time
     */
-  override def as(alias: Expression): Window = new SessionWindow(alias, gap)
+  def on(timeField: String): SessionWithGapOnTime =
+    on(ExpressionParser.parseExpression(timeField))
 }
 
 /**
-  * A partially defined session window on event-time without alias.
+  * Session window on time.
   */
-class SessionWithoutAlias(
-    timeField: Expression,
-    gap: Expression) extends WindowWithoutAlias {
+class SessionWithGapOnTime(timeField: Expression, gap: Expression) {
+
   /**
     * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
     * refer to. `select()` statement can access window properties such as window start or end time.
@@ -490,29 +467,37 @@ class SessionWithoutAlias(
     * @param alias alias for this window
     * @return this window
     */
-  override def as(alias: Expression): Window = new SessionEventTimeWindow(alias, timeField, gap)
-}
-
-/**
-  * A session window on processing-time.
-  *
-  * @param gap the time interval of inactivity before a window is closed.
-  */
-class SessionWindow(alias: Expression, gap: Expression) extends Window(alias) {
+  def as(alias: Expression): SessionWithGapOnTimeWithAlias = {
+    new SessionWithGapOnTimeWithAlias(alias, timeField, gap)
+  }
 
-  override private[flink] def toLogicalWindow: LogicalWindow =
-    ProcessingTimeSessionGroupWindow(alias, gap)
+  /**
+    * Assigns an alias for this window that the following `groupBy()` and `select()` clause can
+    * refer to. `select()` statement can access window properties such as window start or end time.
+    *
+    * @param alias alias for this window
+    * @return this window
+    */
+  def as(alias: String): SessionWithGapOnTimeWithAlias = {
+    as(ExpressionParser.parseExpression(alias))
+  }
 }
 
 /**
-  * A session window on event-time.
+  * Session window on time with alias. Fully specifies a window.
   */
-class SessionEventTimeWindow(
+class SessionWithGapOnTimeWithAlias(
     alias: Expression,
     timeField: Expression,
     gap: Expression)
-  extends EventTimeWindow(alias, timeField) {
+  extends Window(
+    alias,
+    timeField) {
 
-  override private[flink] def toLogicalWindow: LogicalWindow =
-    EventTimeSessionGroupWindow(alias, timeField, gap)
+  /**
+    * Converts an API class to a logical window for planning.
+    */
+  override private[flink] def toLogicalWindow: LogicalWindow = {
+    SessionGroupWindow(alias, timeField, gap)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkCalciteSqlValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkCalciteSqlValidator.scala
index b4a3c422c3849..2bdf360649877 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkCalciteSqlValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkCalciteSqlValidator.scala
@@ -21,8 +21,8 @@ package org.apache.flink.table.calcite
 import org.apache.calcite.adapter.java.JavaTypeFactory
 import org.apache.calcite.prepare.CalciteCatalogReader
 import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.sql.validate.{SqlConformance, SqlValidatorImpl}
-import org.apache.calcite.sql.{SqlInsert, SqlOperatorTable}
+import org.apache.calcite.sql._
+import org.apache.calcite.sql.validate.{SqlConformanceEnum, SqlValidatorImpl}
 
 /**
  * This is a copy of Calcite's CalciteSqlValidator to use with [[FlinkPlannerImpl]].
@@ -30,8 +30,12 @@ import org.apache.calcite.sql.{SqlInsert, SqlOperatorTable}
 class FlinkCalciteSqlValidator(
     opTab: SqlOperatorTable,
     catalogReader: CalciteCatalogReader,
-    typeFactory: JavaTypeFactory) extends SqlValidatorImpl(
-        opTab, catalogReader, typeFactory, SqlConformance.DEFAULT) {
+    factory: JavaTypeFactory)
+  extends SqlValidatorImpl(
+    opTab,
+    catalogReader,
+    factory,
+    SqlConformanceEnum.DEFAULT) {
 
   override def getLogicalSourceRowType(
       sourceRowType: RelDataType,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala
index 09e32776b12d7..beb24361d7a4e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala
@@ -107,7 +107,11 @@ class FlinkPlannerImpl(
       // we disable automatic flattening in order to let composite types pass without modification
       // we might enable it again once Calcite has better support for structured types
       // root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true))
-      root = root.withRel(RelDecorrelator.decorrelateQuery(root.rel))
+
+      // TableEnvironment.optimize will execute the following
+      // root = root.withRel(RelDecorrelator.decorrelateQuery(root.rel))
+      // convert time indicators
+      // root = root.withRel(RelTimeIndicatorConverter.convert(root.rel, rexBuilder))
       root
     } catch {
       case e: RelConversionException => throw TableException(e.getMessage)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
index 7762ff853482a..001011b866c80 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
@@ -20,25 +20,25 @@ package org.apache.flink.table.calcite
 
 import org.apache.calcite.avatica.util.TimeUnit
 import org.apache.calcite.jdbc.JavaTypeFactoryImpl
-import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeSystem}
+import org.apache.calcite.rel.`type`._
 import org.apache.calcite.sql.SqlIntervalQualifier
-import org.apache.calcite.sql.`type`.SqlTypeName
 import org.apache.calcite.sql.`type`.SqlTypeName._
+import org.apache.calcite.sql.`type`.{BasicSqlType, SqlTypeName}
 import org.apache.calcite.sql.parser.SqlParserPos
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.{NothingTypeInfo, PrimitiveArrayTypeInfo, SqlTimeTypeInfo, TypeInformation}
 import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo, RowTypeInfo}
 import org.apache.flink.api.java.typeutils.ValueTypeInfo._
+import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo, RowTypeInfo}
 import org.apache.flink.table.api.TableException
-import org.apache.flink.table.plan.schema.{ArrayRelDataType, CompositeRelDataType, GenericRelDataType, MapRelDataType}
-import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
-import org.apache.flink.table.typeutils.TypeCheckUtils.isSimple
 import org.apache.flink.table.calcite.FlinkTypeFactory.typeInfoToSqlTypeName
+import org.apache.flink.table.plan.schema._
+import org.apache.flink.table.typeutils.TypeCheckUtils.isSimple
+import org.apache.flink.table.typeutils.{TimeIndicatorTypeInfo, TimeIntervalTypeInfo}
 import org.apache.flink.types.Row
 
-import scala.collection.mutable
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 /**
   * Flink specific type factory that represents the interface between Flink's [[TypeInformation]]
@@ -65,6 +65,12 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
           createSqlIntervalType(
             new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.SECOND, SqlParserPos.ZERO))
 
+        case TimeIndicatorTypeInfo.ROWTIME_INDICATOR =>
+          createRowtimeIndicatorType()
+
+        case TimeIndicatorTypeInfo.PROCTIME_INDICATOR =>
+          createProctimeIndicatorType()
+
         case _ =>
           createSqlType(sqlType)
       }
@@ -76,24 +82,76 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
     }
   }
 
+  /**
+    * Creates a indicator type for processing-time, but with similar properties as SQL timestamp.
+    */
+  def createProctimeIndicatorType(): RelDataType = {
+    val originalType = createTypeFromTypeInfo(SqlTimeTypeInfo.TIMESTAMP)
+    canonize(
+      new TimeIndicatorRelDataType(
+        getTypeSystem,
+        originalType.asInstanceOf[BasicSqlType],
+        isEventTime = false)
+    )
+  }
+
+  /**
+    * Creates a indicator type for event-time, but with similar properties as SQL timestamp.
+    */
+  def createRowtimeIndicatorType(): RelDataType = {
+    val originalType = createTypeFromTypeInfo(SqlTimeTypeInfo.TIMESTAMP)
+    canonize(
+      new TimeIndicatorRelDataType(
+        getTypeSystem,
+        originalType.asInstanceOf[BasicSqlType],
+        isEventTime = true)
+    )
+  }
+
   /**
     * Creates a struct type with the input fieldNames and input fieldTypes using FlinkTypeFactory
     *
     * @param fieldNames field names
     * @param fieldTypes field types, every element is Flink's [[TypeInformation]]
-    * @return a struct type with the input fieldNames and input fieldTypes
+    * @param rowtime optional system field to indicate event-time; the index determines the index
+    *                in the final record and might replace an existing field
+    * @param proctime optional system field to indicate processing-time; the index determines the
+    *                 index in the final record and might replace an existing field
+    * @return a struct type with the input fieldNames, input fieldTypes, and system fields
     */
-  def buildRowDataType(
-      fieldNames: Array[String],
-      fieldTypes: Array[TypeInformation[_]])
+  def buildLogicalRowType(
+      fieldNames: Seq[String],
+      fieldTypes: Seq[TypeInformation[_]],
+      rowtime: Option[(Int, String)],
+      proctime: Option[(Int, String)])
     : RelDataType = {
-    val rowDataTypeBuilder = builder
-    fieldNames
-      .zip(fieldTypes)
-      .foreach { f =>
-        rowDataTypeBuilder.add(f._1, createTypeFromTypeInfo(f._2)).nullable(true)
+    val logicalRowTypeBuilder = builder
+
+    val fields = fieldNames.zip(fieldTypes)
+
+    var totalNumberOfFields = fields.length
+    if (rowtime.isDefined) {
+      totalNumberOfFields += 1
+    }
+    if (proctime.isDefined) {
+      totalNumberOfFields += 1
+    }
+
+    var addedTimeAttributes = 0
+    for (i <- 0 until totalNumberOfFields) {
+      if (rowtime.isDefined && rowtime.get._1 == i) {
+        logicalRowTypeBuilder.add(rowtime.get._2, createRowtimeIndicatorType())
+        addedTimeAttributes += 1
+      } else if (proctime.isDefined && proctime.get._1 == i) {
+        logicalRowTypeBuilder.add(proctime.get._2, createProctimeIndicatorType())
+        addedTimeAttributes += 1
+      } else {
+        val field = fields(i - addedTimeAttributes)
+        logicalRowTypeBuilder.add(field._1, createTypeFromTypeInfo(field._2)).nullable(true)
       }
-    rowDataTypeBuilder.build
+    }
+
+    logicalRowTypeBuilder.build
   }
 
   override def createSqlType(typeName: SqlTypeName, precision: Int): RelDataType = {
@@ -178,6 +236,7 @@ object FlinkTypeFactory {
   /**
     * Converts a Calcite logical record into a Flink type information.
     */
+  @deprecated("Use the RowSchema class instead because it handles both logical and physical rows.")
   def toInternalRowTypeInfo(logicalRowType: RelDataType): TypeInformation[Row] = {
     // convert to type information
     val logicalFieldTypes = logicalRowType.getFieldList.asScala map { relDataType =>
@@ -188,6 +247,36 @@ object FlinkTypeFactory {
     new RowTypeInfo(logicalFieldTypes.toArray, logicalFieldNames.toArray)
   }
 
+  def isProctimeIndicatorType(relDataType: RelDataType): Boolean = relDataType match {
+    case ti: TimeIndicatorRelDataType if !ti.isEventTime => true
+    case _ => false
+  }
+
+  def isProctimeIndicatorType(typeInfo: TypeInformation[_]): Boolean = typeInfo match {
+    case ti: TimeIndicatorTypeInfo if !ti.isEventTime => true
+    case _ => false
+  }
+
+  def isRowtimeIndicatorType(relDataType: RelDataType): Boolean = relDataType match {
+    case ti: TimeIndicatorRelDataType if ti.isEventTime => true
+    case _ => false
+  }
+
+  def isRowtimeIndicatorType(typeInfo: TypeInformation[_]): Boolean = typeInfo match {
+    case ti: TimeIndicatorTypeInfo if ti.isEventTime => true
+    case _ => false
+  }
+
+  def isTimeIndicatorType(relDataType: RelDataType): Boolean = relDataType match {
+    case ti: TimeIndicatorRelDataType => true
+    case _ => false
+  }
+
+  def isTimeIndicatorType(typeInfo: TypeInformation[_]): Boolean = typeInfo match {
+    case ti: TimeIndicatorTypeInfo => true
+    case _ => false
+  }
+
   def toTypeInfo(relDataType: RelDataType): TypeInformation[_] = relDataType.getSqlTypeName match {
     case BOOLEAN => BOOLEAN_TYPE_INFO
     case TINYINT => BYTE_TYPE_INFO
@@ -199,6 +288,15 @@ object FlinkTypeFactory {
     case VARCHAR | CHAR => STRING_TYPE_INFO
     case DECIMAL => BIG_DEC_TYPE_INFO
 
+    // time indicators
+    case TIMESTAMP if relDataType.isInstanceOf[TimeIndicatorRelDataType] =>
+      val indicator = relDataType.asInstanceOf[TimeIndicatorRelDataType]
+      if (indicator.isEventTime) {
+        TimeIndicatorTypeInfo.ROWTIME_INDICATOR
+      } else {
+        TimeIndicatorTypeInfo.PROCTIME_INDICATOR
+      }
+
     // temporal types
     case DATE => SqlTimeTypeInfo.DATE
     case TIME => SqlTimeTypeInfo.TIME
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala
new file mode 100644
index 0000000000000..fa2e3eeb0a01d
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.calcite
+
+import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFieldImpl, RelRecordType, StructKind}
+import org.apache.calcite.rel.logical._
+import org.apache.calcite.rel.{RelNode, RelShuttleImpl}
+import org.apache.calcite.rex._
+import org.apache.calcite.sql.fun.SqlStdOperatorTable
+import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo
+import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.calcite.FlinkTypeFactory.isTimeIndicatorType
+import org.apache.flink.table.functions.TimeMaterializationSqlFunction
+import org.apache.flink.table.plan.schema.TimeIndicatorRelDataType
+
+import scala.collection.JavaConversions._
+
+/**
+  * Traverses a [[RelNode]] tree and converts fields with [[TimeIndicatorRelDataType]] type. If a
+  * time attribute is accessed for a calculation, it will be materialized. Forwarding is allowed in
+  * some cases, but not all.
+  */
+class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttleImpl {
+
+  override def visit(project: LogicalProject): RelNode = {
+    // visit children and update inputs
+    val updatedProject = super.visit(project).asInstanceOf[LogicalProject]
+
+    // check if input field contains time indicator type
+    // materialize field if no time indicator is present anymore
+    // if input field is already materialized, change to timestamp type
+    val materializer = new RexTimeIndicatorMaterializer(
+      rexBuilder,
+      updatedProject.getInput.getRowType.getFieldList.map(_.getType))
+    val newProjects = updatedProject.getProjects.map(_.accept(materializer))
+
+    // copy project
+    updatedProject.copy(
+      updatedProject.getTraitSet,
+      updatedProject.getInput,
+      newProjects,
+      buildRowType(updatedProject.getRowType.getFieldNames, newProjects.map(_.getType))
+    )
+  }
+
+  override def visit(filter: LogicalFilter): RelNode = {
+    // visit children and update inputs
+    val updatedFilter = super.visit(filter).asInstanceOf[LogicalFilter]
+
+    // check if input field contains time indicator type
+    // materialize field if no time indicator is present anymore
+    // if input field is already materialized, change to timestamp type
+    val materializer = new RexTimeIndicatorMaterializer(
+      rexBuilder,
+      updatedFilter.getInput.getRowType.getFieldList.map(_.getType))
+    val newCondition = updatedFilter.getCondition.accept(materializer)
+
+    // copy filter
+    updatedFilter.copy(
+      updatedFilter.getTraitSet,
+      updatedFilter.getInput,
+      newCondition
+    )
+  }
+
+  override def visit(union: LogicalUnion): RelNode = {
+    // visit children and update inputs
+    val updatedUnion = super.visit(union).asInstanceOf[LogicalUnion]
+
+    // make sure that time indicator types match
+    val inputTypes = updatedUnion.getInputs.map(_.getRowType)
+
+    val head = inputTypes.head.getFieldList.map(_.getType)
+
+    val isValid = inputTypes.forall { t =>
+      val fieldTypes = t.getFieldList.map(_.getType)
+
+      fieldTypes.zip(head).forall { case (l, r) =>
+        // check if time indicators match
+        if (isTimeIndicatorType(l) && isTimeIndicatorType(r)) {
+          val leftTime = l.asInstanceOf[TimeIndicatorRelDataType].isEventTime
+          val rightTime = r.asInstanceOf[TimeIndicatorRelDataType].isEventTime
+          leftTime == rightTime
+        }
+        // one side is not an indicator
+        else if (isTimeIndicatorType(l) || isTimeIndicatorType(r)) {
+          false
+        }
+        // uninteresting types
+        else {
+          true
+        }
+      }
+    }
+
+    if (!isValid) {
+      throw new ValidationException(
+        "Union fields with time attributes have different types.")
+    }
+
+    updatedUnion
+  }
+
+  override def visit(other: RelNode): RelNode = other match {
+    case scan: LogicalTableFunctionScan if
+        stack.size() > 0 && stack.peek().isInstanceOf[LogicalCorrelate] =>
+      // visit children and update inputs
+      val updatedScan = super.visit(scan).asInstanceOf[LogicalTableFunctionScan]
+
+      val correlate = stack.peek().asInstanceOf[LogicalCorrelate]
+
+      // check if input field contains time indicator type
+      // materialize field if no time indicator is present anymore
+      // if input field is already materialized, change to timestamp type
+      val materializer = new RexTimeIndicatorMaterializer(
+        rexBuilder,
+        correlate.getInputs.get(0).getRowType.getFieldList.map(_.getType))
+      val newCall = updatedScan.getCall.accept(materializer)
+
+      // copy scan
+      updatedScan.copy(
+        updatedScan.getTraitSet,
+        updatedScan.getInputs,
+        newCall,
+        updatedScan.getElementType,
+        updatedScan.getRowType,
+        updatedScan.getColumnMappings
+      )
+
+    case _ =>
+      super.visit(other)
+  }
+
+  private def buildRowType(names: Seq[String], types: Seq[RelDataType]): RelDataType = {
+    val fields = names.zipWithIndex.map { case (name, idx) =>
+      new RelDataTypeFieldImpl(name, idx, types(idx))
+    }
+    new RelRecordType(StructKind.FULLY_QUALIFIED, fields)
+  }
+}
+
+class RexTimeIndicatorMaterializer(
+    private val rexBuilder: RexBuilder,
+    private val input: Seq[RelDataType])
+  extends RexShuttle {
+
+  val timestamp = rexBuilder
+    .getTypeFactory
+    .asInstanceOf[FlinkTypeFactory]
+    .createTypeFromTypeInfo(SqlTimeTypeInfo.TIMESTAMP)
+
+  override def visitInputRef(inputRef: RexInputRef): RexNode = {
+    // reference is interesting
+    if (isTimeIndicatorType(inputRef.getType)) {
+      val resolvedRefType = input(inputRef.getIndex)
+      // input is a valid time indicator
+      if (isTimeIndicatorType(resolvedRefType)) {
+        inputRef
+      }
+      // input has been materialized
+      else {
+        new RexInputRef(inputRef.getIndex, resolvedRefType)
+      }
+    }
+    // reference is a regular field
+    else {
+      super.visitInputRef(inputRef)
+    }
+  }
+
+  override def visitCall(call: RexCall): RexNode = {
+    val updatedCall = super.visitCall(call).asInstanceOf[RexCall]
+
+    // skip materialization for special operators
+    updatedCall.getOperator match {
+      case SqlStdOperatorTable.SESSION | SqlStdOperatorTable.HOP | SqlStdOperatorTable.TUMBLE =>
+        return updatedCall
+
+      case _ => // do nothing
+    }
+
+    // materialize operands with time indicators
+    val materializedOperands = updatedCall.getOperands.map { o =>
+      if (isTimeIndicatorType(o.getType)) {
+        rexBuilder.makeCall(TimeMaterializationSqlFunction, o)
+      } else {
+        o
+      }
+    }
+
+    // remove time indicator return type
+    if (isTimeIndicatorType(updatedCall.getType)) {
+      updatedCall.clone(timestamp, materializedOperands)
+    } else {
+      updatedCall.clone(updatedCall.getType, materializedOperands)
+    }
+  }
+}
+
+object RelTimeIndicatorConverter {
+
+  def convert(rootRel: RelNode, rexBuilder: RexBuilder): RelNode = {
+    val converter = new RelTimeIndicatorConverter(rexBuilder)
+    rootRel.accept(converter)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 5bb3b0e4bffd7..25addbc5e923b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -23,7 +23,6 @@ import java.lang.{Iterable => JIterable}
 import java.math.{BigDecimal => JBigDecimal}
 
 import org.apache.calcite.avatica.util.DateTimeUtils
-import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rex._
 import org.apache.calcite.sql.SqlOperator
 import org.apache.calcite.sql.`type`.SqlTypeName._
@@ -42,8 +41,8 @@ import org.apache.flink.table.codegen.GeneratedExpression.{NEVER_NULL, NO_CODE}
 import org.apache.flink.table.codegen.Indenter.toISC
 import org.apache.flink.table.codegen.calls.FunctionGenerator
 import org.apache.flink.table.codegen.calls.ScalarOperators._
-import org.apache.flink.table.functions.{AggregateFunction, FunctionContext, UserDefinedFunction}
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
+import org.apache.flink.table.functions.{AggregateFunction, FunctionContext, TimeMaterializationSqlFunction, UserDefinedFunction}
 import org.apache.flink.table.runtime.TableFunctionCollector
 import org.apache.flink.table.typeutils.TypeCheckUtils._
 import org.apache.flink.types.Row
@@ -59,19 +58,18 @@ import scala.collection.mutable
   * @param nullableInput input(s) can be null.
   * @param input1 type information about the first input of the Function
   * @param input2 type information about the second input if the Function is binary
-  * @param input1PojoFieldMapping additional mapping information if input1 is a POJO (POJO types
-  *                              have no deterministic field order).
-  * @param input2PojoFieldMapping additional mapping information if input2 is a POJO (POJO types
-  *                              have no deterministic field order).
-  *
+  * @param input1FieldMapping additional mapping information for input1
+  *   (e.g. POJO types have no deterministic field order and some input fields might not be read)
+  * @param input2FieldMapping additional mapping information for input2
+  *   (e.g. POJO types have no deterministic field order and some input fields might not be read)
   */
 class CodeGenerator(
-   config: TableConfig,
-   nullableInput: Boolean,
-   input1: TypeInformation[_ <: Any],
-   input2: Option[TypeInformation[_ <: Any]] = None,
-   input1PojoFieldMapping: Option[Array[Int]] = None,
-   input2PojoFieldMapping: Option[Array[Int]] = None)
+    config: TableConfig,
+    nullableInput: Boolean,
+    input1: TypeInformation[_ <: Any],
+    input2: Option[TypeInformation[_ <: Any]] = None,
+    input1FieldMapping: Option[Array[Int]] = None,
+    input2FieldMapping: Option[Array[Int]] = None)
   extends RexVisitor[GeneratedExpression] {
 
   // check if nullCheck is enabled when inputs can be null
@@ -82,7 +80,7 @@ class CodeGenerator(
   // check for POJO input1 mapping
   input1 match {
     case pt: PojoTypeInfo[_] =>
-      input1PojoFieldMapping.getOrElse(
+      input1FieldMapping.getOrElse(
         throw new CodeGenException("No input mapping is specified for input1 of type POJO."))
     case _ => // ok
   }
@@ -90,11 +88,24 @@ class CodeGenerator(
   // check for POJO input2 mapping
   input2 match {
     case Some(pt: PojoTypeInfo[_]) =>
-      input2PojoFieldMapping.getOrElse(
+      input2FieldMapping.getOrElse(
         throw new CodeGenException("No input mapping is specified for input2 of type POJO."))
     case _ => // ok
   }
 
+  private val input1Mapping = input1FieldMapping match {
+    case Some(mapping) => mapping
+    case _ => (0 until input1.getArity).toArray
+  }
+
+  private val input2Mapping = input2FieldMapping match {
+    case Some(mapping) => mapping
+    case _ => input2 match {
+      case Some(input) => (0 until input.getArity).toArray
+      case _ => Array[Int]()
+    }
+  }
+
   /**
     * A code generator for generating unary Flink
     * [[org.apache.flink.api.common.functions.Function]]s with one input.
@@ -102,15 +113,15 @@ class CodeGenerator(
     * @param config configuration that determines runtime behavior
     * @param nullableInput input(s) can be null.
     * @param input type information about the input of the Function
-    * @param inputPojoFieldMapping additional mapping information necessary if input is a
-    *                              POJO (POJO types have no deterministic field order).
+    * @param inputFieldMapping additional mapping information necessary for input
+    *   (e.g. POJO types have no deterministic field order and some input fields might not be read)
     */
   def this(
       config: TableConfig,
       nullableInput: Boolean,
       input: TypeInformation[Any],
-      inputPojoFieldMapping: Array[Int]) =
-    this(config, nullableInput, input, None, Some(inputPojoFieldMapping))
+      inputFieldMapping: Array[Int]) =
+    this(config, nullableInput, input, None, Some(inputFieldMapping))
 
   /**
     * A code generator for generating Flink input formats.
@@ -249,7 +260,7 @@ class CodeGenerator(
     * @param name        Class name of the function.
     *                    Does not need to be unique but has to be a valid Java class identifier.
     * @param generator   The code generator instance
-    * @param inputType   Input row type
+    * @param physicalInputTypes Physical input row types
     * @param aggregates  All aggregate functions
     * @param aggFields   Indexes of the input fields for all aggregate functions
     * @param aggMapping  The mapping of aggregates to output fields
@@ -270,7 +281,7 @@ class CodeGenerator(
   def generateAggregations(
       name: String,
       generator: CodeGenerator,
-      inputType: RelDataType,
+      physicalInputTypes: Seq[TypeInformation[_]],
       aggregates: Array[AggregateFunction[_ <: Any, _ <: Any]],
       aggFields: Array[Array[Int]],
       aggMapping: Array[Int],
@@ -295,8 +306,7 @@ class CodeGenerator(
     val accTypes = accTypeClasses.map(_.getCanonicalName)
 
     // get java classes of input fields
-    val javaClasses = inputType.getFieldList
-      .map(f => FlinkTypeFactory.toTypeInfo(f.getType).getTypeClass)
+    val javaClasses = physicalInputTypes.map(t => t.getTypeClass)
     // get parameter lists for aggregation functions
     val parameters = aggFields.map { inFields =>
       val fields = for (f <- inFields) yield
@@ -844,12 +854,12 @@ class CodeGenerator(
       returnType: TypeInformation[_ <: Any],
       resultFieldNames: Seq[String])
     : GeneratedExpression = {
-    val input1AccessExprs = for (i <- 0 until input1.getArity)
-      yield generateInputAccess(input1, input1Term, i, input1PojoFieldMapping)
+    val input1AccessExprs = for (i <- 0 until input1.getArity if input1Mapping.contains(i))
+      yield generateInputAccess(input1, input1Term, i, input1Mapping)
 
     val input2AccessExprs = input2 match {
-      case Some(ti) => for (i <- 0 until ti.getArity)
-        yield generateInputAccess(ti, input2Term, i, input2PojoFieldMapping)
+      case Some(ti) => for (i <- 0 until ti.getArity if input2Mapping.contains(i))
+        yield generateInputAccess(ti, input2Term, i, input2Mapping)
       case None => Seq() // add nothing
     }
 
@@ -861,14 +871,14 @@ class CodeGenerator(
     */
   def generateCorrelateAccessExprs: (Seq[GeneratedExpression], Seq[GeneratedExpression]) = {
     val input1AccessExprs = for (i <- 0 until input1.getArity)
-      yield generateInputAccess(input1, input1Term, i, input1PojoFieldMapping)
+      yield generateInputAccess(input1, input1Term, i, input1Mapping)
 
     val input2AccessExprs = input2 match {
-      case Some(ti) => for (i <- 0 until ti.getArity)
+      case Some(ti) => for (i <- 0 until ti.getArity if input2Mapping.contains(i))
         // use generateFieldAccess instead of generateInputAccess to avoid the generated table
         // function's field access code is put on the top of function body rather than
         // the while loop
-        yield generateFieldAccess(ti, input2Term, i, input2PojoFieldMapping)
+        yield generateFieldAccess(ti, input2Term, i, input2Mapping)
       case None => throw new CodeGenException("Type information of input2 must not be null.")
     }
     (input1AccessExprs, input2AccessExprs)
@@ -1123,11 +1133,11 @@ class CodeGenerator(
   override def visitInputRef(inputRef: RexInputRef): GeneratedExpression = {
     // if inputRef index is within size of input1 we work with input1, input2 otherwise
     val input = if (inputRef.getIndex < input1.getArity) {
-      (input1, input1Term, input1PojoFieldMapping)
+      (input1, input1Term, input1Mapping)
     } else {
       (input2.getOrElse(throw new CodeGenException("Invalid input access.")),
         input2Term,
-        input2PojoFieldMapping)
+        input2Mapping)
     }
 
     val index = if (input._2 == input1Term) {
@@ -1146,7 +1156,7 @@ class CodeGenerator(
       refExpr.resultType,
       refExpr.resultTerm,
       index,
-      input1PojoFieldMapping)
+      input1Mapping)
 
     val resultTerm = newName("result")
     val nullTerm = newName("isNull")
@@ -1302,6 +1312,11 @@ class CodeGenerator(
     throw new CodeGenException("Dynamic parameter references are not supported yet.")
 
   override def visitCall(call: RexCall): GeneratedExpression = {
+    // time materialization is not implemented yet
+    if (call.getOperator == TimeMaterializationSqlFunction) {
+      throw new CodeGenException("Access to time attributes is not possible yet.")
+    }
+
     val operands = call.getOperands.map(_.accept(this))
     val resultType = FlinkTypeFactory.toTypeInfo(call.getType)
 
@@ -1546,7 +1561,7 @@ class CodeGenerator(
       inputType: TypeInformation[_ <: Any],
       inputTerm: String,
       index: Int,
-      pojoFieldMapping: Option[Array[Int]])
+      fieldMapping: Array[Int])
     : GeneratedExpression = {
     // if input has been used before, we can reuse the code that
     // has already been generated
@@ -1558,9 +1573,9 @@ class CodeGenerator(
       // generate input access and unboxing if necessary
       case None =>
         val expr = if (nullableInput) {
-          generateNullableInputFieldAccess(inputType, inputTerm, index, pojoFieldMapping)
+          generateNullableInputFieldAccess(inputType, inputTerm, index, fieldMapping)
         } else {
-          generateFieldAccess(inputType, inputTerm, index, pojoFieldMapping)
+          generateFieldAccess(inputType, inputTerm, index, fieldMapping)
         }
 
         reusableInputUnboxingExprs((inputTerm, index)) = expr
@@ -1574,7 +1589,7 @@ class CodeGenerator(
       inputType: TypeInformation[_ <: Any],
       inputTerm: String,
       index: Int,
-      pojoFieldMapping: Option[Array[Int]])
+      fieldMapping: Array[Int])
     : GeneratedExpression = {
     val resultTerm = newName("result")
     val nullTerm = newName("isNull")
@@ -1582,7 +1597,7 @@ class CodeGenerator(
     val fieldType = inputType match {
       case ct: CompositeType[_] =>
         val fieldIndex = if (ct.isInstanceOf[PojoTypeInfo[_]]) {
-          pojoFieldMapping.get(index)
+          fieldMapping(index)
         }
         else {
           index
@@ -1593,7 +1608,7 @@ class CodeGenerator(
     }
     val resultTypeTerm = primitiveTypeTermForTypeInfo(fieldType)
     val defaultValue = primitiveDefaultValue(fieldType)
-    val fieldAccessExpr = generateFieldAccess(inputType, inputTerm, index, pojoFieldMapping)
+    val fieldAccessExpr = generateFieldAccess(inputType, inputTerm, index, fieldMapping)
 
     val inputCheckCode =
       s"""
@@ -1617,12 +1632,12 @@ class CodeGenerator(
       inputType: TypeInformation[_],
       inputTerm: String,
       index: Int,
-      pojoFieldMapping: Option[Array[Int]])
+      fieldMapping: Array[Int])
     : GeneratedExpression = {
     inputType match {
       case ct: CompositeType[_] =>
-        val fieldIndex = if (ct.isInstanceOf[PojoTypeInfo[_]] && pojoFieldMapping.nonEmpty) {
-          pojoFieldMapping.get(index)
+        val fieldIndex = if (ct.isInstanceOf[PojoTypeInfo[_]]) {
+          fieldMapping(index)
         }
         else {
           index
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
index d9f394bc2bfd3..64280c8f5d978 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala
@@ -29,7 +29,6 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, SqlTimeTypeInfo, TypeInformation}
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.table.codegen.{CodeGenerator, GeneratedExpression}
-import org.apache.flink.table.functions.{EventTimeExtractor, ProcTimeExtractor}
 import org.apache.flink.table.functions.utils.{ScalarSqlFunction, TableSqlFunction}
 
 import scala.collection.mutable
@@ -496,15 +495,6 @@ object FunctionGenerator {
         )
       )
 
-    // generate a constant for time indicator functions.
-    // this is a temporary solution and will be removed when FLINK-5884 is implemented.
-    case ProcTimeExtractor | EventTimeExtractor =>
-      Some(new CallGenerator {
-        override def generate(codeGenerator: CodeGenerator, operands: Seq[GeneratedExpression]) = {
-          GeneratedExpression("0L", "false", "", SqlTimeTypeInfo.TIMESTAMP)
-        }
-      })
-
     // built-in scalar function
     case _ =>
       sqlFunctions.get((sqlOperator, operandTypes))
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala
index 4b5781ff77000..08abc8f2f1da9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala
@@ -28,10 +28,49 @@ import org.apache.calcite.rex.{RexBuilder, RexNode}
 import org.apache.calcite.sql.fun.SqlStdOperatorTable
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
 import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
+import org.apache.flink.streaming.api.windowing.time.{Time => FlinkTime}
 
 object ExpressionUtils {
 
+  private[flink] def isTimeIntervalLiteral(expr: Expression): Boolean = expr match {
+    case Literal(_, TimeIntervalTypeInfo.INTERVAL_MILLIS) => true
+    case _ => false
+  }
+
+  private[flink] def isRowCountLiteral(expr: Expression): Boolean = expr match {
+    case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) => true
+    case _ => false
+  }
+
+  private[flink] def isTimeAttribute(expr: Expression): Boolean = expr match {
+    case r: ResolvedFieldReference if FlinkTypeFactory.isTimeIndicatorType(r.resultType) => true
+    case _ => false
+  }
+
+  private[flink] def isRowtimeAttribute(expr: Expression): Boolean = expr match {
+    case r: ResolvedFieldReference if FlinkTypeFactory.isRowtimeIndicatorType(r.resultType) => true
+    case _ => false
+  }
+
+  private[flink] def isProctimeAttribute(expr: Expression): Boolean = expr match {
+    case r: ResolvedFieldReference if FlinkTypeFactory.isProctimeIndicatorType(r.resultType) =>
+      true
+    case _ => false
+  }
+
+  private[flink] def toTime(expr: Expression): FlinkTime = expr match {
+    case Literal(value: Long, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
+      FlinkTime.milliseconds(value)
+    case _ => throw new IllegalArgumentException()
+  }
+
+  private[flink] def toLong(expr: Expression): Long = expr match {
+    case Literal(value: Long, RowIntervalTypeInfo.INTERVAL_ROWS) => value
+    case _ => throw new IllegalArgumentException()
+  }
+
   private[flink] def toMonthInterval(expr: Expression, multiplier: Int): Expression = expr match {
     case Literal(value: Int, BasicTypeInfo.INT_TYPE_INFO) =>
       Literal(value * multiplier, TimeIntervalTypeInfo.INTERVAL_MONTHS)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
index 5f7204a29ae71..13f8a11af770e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
@@ -110,18 +110,11 @@ case class OverCall(
     val aggExprs = agg.asInstanceOf[Aggregation].children.map(_.toRexNode(relBuilder)).asJava
 
     // assemble order by key
-    val orderKey = orderBy match {
-      case _: RowTime =>
-        new RexFieldCollation(relBuilder.call(EventTimeExtractor), Set[SqlKind]().asJava)
-      case _: ProcTime =>
-        new RexFieldCollation(relBuilder.call(ProcTimeExtractor), Set[SqlKind]().asJava)
-      case _ =>
-        throw new ValidationException("Invalid OrderBy expression.")
-    }
+    val orderKey = new RexFieldCollation(orderBy.toRexNode, Set[SqlKind]().asJava)
     val orderKeys = ImmutableList.of(orderKey)
 
     // assemble partition by keys
-    val partitionKeys = partitionBy.map(_.toRexNode(relBuilder)).asJava
+    val partitionKeys = partitionBy.map(_.toRexNode).asJava
 
     // assemble bounds
     val isPhysical: Boolean = preceding.resultType.isInstanceOf[RowIntervalTypeInfo]
@@ -249,6 +242,11 @@ case class OverCall(
         return ValidationFailure("Preceding and following must be of same interval type.")
     }
 
+    // check time field
+    if (!ExpressionUtils.isTimeAttribute(orderBy)) {
+      return ValidationFailure("Ordering must be defined on a time attribute.")
+    }
+
     ValidationSuccess
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala
index 08859298c038a..362d84641d711 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala
@@ -19,8 +19,9 @@ package org.apache.flink.table.expressions
 
 import org.apache.calcite.rex.RexNode
 import org.apache.calcite.tools.RelBuilder
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeinfo.{SqlTimeTypeInfo, TypeInformation}
 import org.apache.flink.table.api.{UnresolvedException, ValidationException}
+import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
 trait NamedExpression extends Expression {
@@ -116,24 +117,6 @@ case class UnresolvedAlias(child: Expression) extends UnaryExpression with Named
   override private[flink] lazy val valid = false
 }
 
-case class RowtimeAttribute() extends Attribute {
-  override private[flink] def withName(newName: String): Attribute = {
-    if (newName == "rowtime") {
-      this
-    } else {
-      throw new ValidationException("Cannot rename streaming rowtime attribute.")
-    }
-  }
-
-  override private[flink] def name: String = "rowtime"
-
-  override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
-    throw new UnsupportedOperationException("A rowtime attribute can not be used solely.")
-  }
-
-  override private[flink] def resultType: TypeInformation[_] = BasicTypeInfo.LONG_TYPE_INFO
-}
-
 case class WindowReference(name: String) extends Attribute {
 
   override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode =
@@ -150,3 +133,30 @@ case class WindowReference(name: String) extends Attribute {
     }
   }
 }
+
+abstract class TimeAttribute(val expression: Expression)
+  extends UnaryExpression
+  with NamedExpression {
+
+  override private[flink] def child: Expression = expression
+
+  override private[flink] def name: String = expression match {
+    case UnresolvedFieldReference(name) => name
+    case _ => throw new ValidationException("Unresolved field reference expected.")
+  }
+
+  override private[flink] def toAttribute: Attribute =
+    throw new UnsupportedOperationException("Time attribute can not be used solely.")
+}
+
+case class RowtimeAttribute(expr: Expression) extends TimeAttribute(expr) {
+
+  override private[flink] def resultType: TypeInformation[_] =
+    TimeIndicatorTypeInfo.ROWTIME_INDICATOR
+}
+
+case class ProctimeAttribute(expr: Expression) extends TimeAttribute(expr) {
+
+  override private[flink] def resultType: TypeInformation[_] =
+    TimeIndicatorTypeInfo.PROCTIME_INDICATOR
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeMaterializationSqlFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeMaterializationSqlFunction.scala
new file mode 100644
index 0000000000000..d87502650edb9
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeMaterializationSqlFunction.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.functions
+
+import org.apache.calcite.sql._
+import org.apache.calcite.sql.`type`._
+import org.apache.calcite.sql.validate.SqlMonotonicity
+
+/**
+  * Function that materializes a time attribute to the metadata timestamp. After materialization
+  * the result can be used in regular arithmetical calculations.
+  */
+object TimeMaterializationSqlFunction
+  extends SqlFunction(
+    "TIME_MATERIALIZATION",
+    SqlKind.OTHER_FUNCTION,
+    ReturnTypes.explicit(SqlTypeName.TIMESTAMP),
+    InferTypes.RETURN_TYPE,
+    OperandTypes.family(SqlTypeFamily.TIMESTAMP),
+    SqlFunctionCategory.SYSTEM) {
+
+  override def getSyntax: SqlSyntax = SqlSyntax.FUNCTION
+
+  override def getMonotonicity(call: SqlOperatorBinding): SqlMonotonicity =
+    SqlMonotonicity.INCREASING
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeModeIndicatorFunctions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeModeIndicatorFunctions.scala
deleted file mode 100644
index 3ddcbdc046bcc..0000000000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TimeModeIndicatorFunctions.scala
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.functions
-
-import java.nio.charset.Charset
-import java.util
-
-import org.apache.calcite.rel.`type`._
-import org.apache.calcite.sql._
-import org.apache.calcite.sql.`type`.{OperandTypes, ReturnTypes, SqlTypeFamily, SqlTypeName}
-import org.apache.calcite.sql.validate.SqlMonotonicity
-import org.apache.calcite.tools.RelBuilder
-import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo
-import org.apache.flink.table.api.TableException
-import org.apache.flink.table.expressions.LeafExpression
-
-object EventTimeExtractor extends SqlFunction("ROWTIME", SqlKind.OTHER_FUNCTION,
-  ReturnTypes.explicit(TimeModeTypes.ROWTIME), null, OperandTypes.NILADIC,
-  SqlFunctionCategory.SYSTEM) {
-  override def getSyntax: SqlSyntax = SqlSyntax.FUNCTION
-
-  override def getMonotonicity(call: SqlOperatorBinding): SqlMonotonicity =
-    SqlMonotonicity.INCREASING
-}
-
-object ProcTimeExtractor extends SqlFunction("PROCTIME", SqlKind.OTHER_FUNCTION,
-  ReturnTypes.explicit(TimeModeTypes.PROCTIME), null, OperandTypes.NILADIC,
-  SqlFunctionCategory.SYSTEM) {
-  override def getSyntax: SqlSyntax = SqlSyntax.FUNCTION
-
-  override def getMonotonicity(call: SqlOperatorBinding): SqlMonotonicity =
-    SqlMonotonicity.INCREASING
-}
-
-abstract class TimeIndicator extends LeafExpression {
-  /**
-    * Returns the [[org.apache.flink.api.common.typeinfo.TypeInformation]]
-    * for evaluating this expression.
-    * It is sometimes not available until the expression is valid.
-    */
-  override private[flink] def resultType = SqlTimeTypeInfo.TIMESTAMP
-
-  /**
-    * Convert Expression to its counterpart in Calcite, i.e. RexNode
-    */
-  override private[flink] def toRexNode(implicit relBuilder: RelBuilder) =
-    throw new TableException("indicator functions (e.g. proctime() and rowtime()" +
-      " are not executable. Please check your expressions.")
-}
-
-case class RowTime() extends TimeIndicator
-case class ProcTime() extends TimeIndicator
-
-object TimeModeTypes {
-
-  // indicator data type for row time (event time)
-  val ROWTIME = new RowTimeType
-  // indicator data type for processing time
-  val PROCTIME = new ProcTimeType
-
-}
-
-class RowTimeType extends TimeModeType {
-
-  override def toString(): String = "ROWTIME"
-  override def getFullTypeString: String = "ROWTIME_INDICATOR"
-}
-
-class ProcTimeType extends TimeModeType {
-
-  override def toString(): String = "PROCTIME"
-  override def getFullTypeString: String = "PROCTIME_INDICATOR"
-}
-
-abstract class TimeModeType extends RelDataType {
-
-  override def getComparability: RelDataTypeComparability = RelDataTypeComparability.NONE
-
-  override def isStruct: Boolean = false
-
-  override def getFieldList: util.List[RelDataTypeField] = null
-
-  override def getFieldNames: util.List[String] = null
-
-  override def getFieldCount: Int = 0
-
-  override def getStructKind: StructKind = StructKind.NONE
-
-  override def getField(
-     fieldName: String,
-     caseSensitive: Boolean,
-     elideRecord: Boolean): RelDataTypeField = null
-
-  override def isNullable: Boolean = false
-
-  override def getComponentType: RelDataType = null
-
-  override def getKeyType: RelDataType = null
-
-  override def getValueType: RelDataType = null
-
-  override def getCharset: Charset = null
-
-  override def getCollation: SqlCollation = null
-
-  override def getIntervalQualifier: SqlIntervalQualifier = null
-
-  override def getPrecision: Int = -1
-
-  override def getScale: Int = -1
-
-  override def getSqlTypeName: SqlTypeName = SqlTypeName.TIMESTAMP
-
-  override def getSqlIdentifier: SqlIdentifier = null
-
-  override def getFamily: RelDataTypeFamily = SqlTypeFamily.NUMERIC
-
-  override def getPrecedenceList: RelDataTypePrecedenceList = ???
-
-  override def isDynamicStruct: Boolean = false
-
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
index d26cdcf208260..98a7e6310242c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/ProjectionTranslator.scala
@@ -19,9 +19,8 @@
 package org.apache.flink.table.plan
 
 import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.table.api.{OverWindow, StreamTableEnvironment, TableEnvironment}
+import org.apache.flink.table.api.{OverWindow, TableEnvironment}
 import org.apache.flink.table.expressions._
-import org.apache.flink.table.functions.{ProcTime, RowTime}
 import org.apache.flink.table.plan.logical.{LogicalNode, Project}
 
 import scala.collection.mutable
@@ -231,28 +230,12 @@ object ProjectionTranslator {
 
       val overWindow = overWindows.find(_.alias.equals(unresolvedCall.alias))
       if (overWindow.isDefined) {
-        if (tEnv.isInstanceOf[StreamTableEnvironment]) {
-          val timeIndicator = overWindow.get.orderBy match {
-            case u: UnresolvedFieldReference if u.name.toLowerCase == "rowtime" =>
-              RowTime()
-            case u: UnresolvedFieldReference if u.name.toLowerCase == "proctime" =>
-              ProcTime()
-            case e: Expression => e
-          }
-          OverCall(
-            unresolvedCall.agg,
-            overWindow.get.partitionBy,
-            timeIndicator,
-            overWindow.get.preceding,
-            overWindow.get.following)
-        } else {
-          OverCall(
-            unresolvedCall.agg,
-            overWindow.get.partitionBy,
-            overWindow.get.orderBy,
-            overWindow.get.preceding,
-            overWindow.get.following)
-        }
+        OverCall(
+          unresolvedCall.agg,
+          overWindow.get.partitionBy,
+          overWindow.get.orderBy,
+          overWindow.get.preceding,
+          overWindow.get.following)
       } else {
         unresolvedCall
       }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
index 1884e543077a5..92dc50177bcc4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
@@ -22,14 +22,24 @@ import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.expressions.{Expression, WindowReference}
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
-abstract class LogicalWindow(val alias: Expression) extends Resolvable[LogicalWindow] {
+/**
+  * Logical super class for all types of windows (group-windows and row-windows).
+  *
+  * @param aliasAttribute window alias
+  * @param timeAttribute time field indicating event-time or processing-time
+  */
+abstract class LogicalWindow(
+    val aliasAttribute: Expression,
+    val timeAttribute: Expression)
+  extends Resolvable[LogicalWindow] {
 
   def resolveExpressions(resolver: (Expression) => Expression): LogicalWindow = this
 
-  def validate(tableEnv: TableEnvironment): ValidationResult = alias match {
+  def validate(tableEnv: TableEnvironment): ValidationResult = aliasAttribute match {
     case WindowReference(_) => ValidationSuccess
     case _ => ValidationFailure("Window reference for window expected.")
   }
 
   override def toString: String = getClass.getSimpleName
+
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
index 576756d174c0f..3e5de283ab03d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
@@ -18,259 +18,165 @@
 
 package org.apache.flink.table.plan.logical
 
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
 import org.apache.flink.table.api.{BatchTableEnvironment, StreamTableEnvironment, TableEnvironment}
+import org.apache.flink.table.expressions.ExpressionUtils.{isRowCountLiteral, isRowtimeAttribute, isTimeAttribute, isTimeIntervalLiteral}
 import org.apache.flink.table.expressions._
-import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo, TypeCoercion}
+import org.apache.flink.table.typeutils.TypeCheckUtils.isTimePoint
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
-abstract class EventTimeGroupWindow(
-    alias: Expression,
-    time: Expression)
-  extends LogicalWindow(alias) {
-
-  override def validate(tableEnv: TableEnvironment): ValidationResult = {
-    val valid = super.validate(tableEnv)
-    if (valid.isFailure) {
-        return valid
-    }
-
-    tableEnv match {
-      case _: StreamTableEnvironment =>
-        time match {
-          case RowtimeAttribute() =>
-            ValidationSuccess
-          case _ =>
-            ValidationFailure("Event-time window expects a 'rowtime' time field.")
-      }
-      case _: BatchTableEnvironment =>
-        if (!TypeCoercion.canCast(time.resultType, BasicTypeInfo.LONG_TYPE_INFO)) {
-          ValidationFailure(s"Event-time window expects a time field that can be safely cast " +
-            s"to Long, but is ${time.resultType}")
-        } else {
-          ValidationSuccess
-        }
-    }
-
-  }
-}
-
-abstract class ProcessingTimeGroupWindow(alias: Expression) extends LogicalWindow(alias) {
-  override def validate(tableEnv: TableEnvironment): ValidationResult = {
-    val valid = super.validate(tableEnv)
-    if (valid.isFailure) {
-      return valid
-    }
-
-    tableEnv match {
-      case b: BatchTableEnvironment => ValidationFailure(
-        "Window on batch must declare a time attribute over which the query is evaluated.")
-      case _ =>
-        ValidationSuccess
-    }
-  }
-}
-
 // ------------------------------------------------------------------------------------------------
 // Tumbling group windows
 // ------------------------------------------------------------------------------------------------
 
-object TumblingGroupWindow {
-  def validate(tableEnv: TableEnvironment, size: Expression): ValidationResult = size match {
-    case Literal(_, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
-      ValidationSuccess
-    case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
-      ValidationSuccess
-    case _ =>
-      ValidationFailure("Tumbling window expects size literal of type Interval of Milliseconds " +
-        "or Interval of Rows.")
-  }
-}
-
-case class ProcessingTimeTumblingGroupWindow(
-    override val alias: Expression,
-    size: Expression)
-  extends ProcessingTimeGroupWindow(alias) {
-
-  override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
-    ProcessingTimeTumblingGroupWindow(
-      resolve(alias),
-      resolve(size))
-
-  override def validate(tableEnv: TableEnvironment): ValidationResult =
-    super.validate(tableEnv).orElse(TumblingGroupWindow.validate(tableEnv, size))
-
-  override def toString: String = s"ProcessingTimeTumblingGroupWindow($alias, $size)"
-}
-
-case class EventTimeTumblingGroupWindow(
-    override val alias: Expression,
+case class TumblingGroupWindow(
+    alias: Expression,
     timeField: Expression,
     size: Expression)
-  extends EventTimeGroupWindow(
+  extends LogicalWindow(
     alias,
     timeField) {
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
-    EventTimeTumblingGroupWindow(
+    TumblingGroupWindow(
       resolve(alias),
       resolve(timeField),
       resolve(size))
 
   override def validate(tableEnv: TableEnvironment): ValidationResult =
-    super.validate(tableEnv)
-      .orElse(TumblingGroupWindow.validate(tableEnv, size))
-      .orElse(size match {
-        case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS)
-          if tableEnv.isInstanceOf[StreamTableEnvironment] =>
+    super.validate(tableEnv).orElse(
+      tableEnv match {
+
+        // check size
+        case _ if !isTimeIntervalLiteral(size) && !isRowCountLiteral(size) =>
+          ValidationFailure(
+            "Tumbling window expects size literal of type Interval of Milliseconds " +
+              "or Interval of Rows.")
+
+        // check time attribute
+        case _: StreamTableEnvironment if !isTimeAttribute(timeField) =>
+          ValidationFailure(
+            "Tumbling window expects a time attribute for grouping in a stream environment.")
+        case _: BatchTableEnvironment if isTimePoint(size.resultType) =>
+          ValidationFailure(
+            "Tumbling window expects a time attribute for grouping in a stream environment.")
+
+        // check row intervals on event-time
+        case _: StreamTableEnvironment
+            if isRowCountLiteral(size) && isRowtimeAttribute(timeField) =>
           ValidationFailure(
             "Event-time grouping windows on row intervals in a stream environment " +
               "are currently not supported.")
+
         case _ =>
           ValidationSuccess
-      })
+      }
+    )
 
-  override def toString: String = s"EventTimeTumblingGroupWindow($alias, $timeField, $size)"
+  override def toString: String = s"TumblingGroupWindow($alias, $timeField, $size)"
 }
 
 // ------------------------------------------------------------------------------------------------
 // Sliding group windows
 // ------------------------------------------------------------------------------------------------
 
-object SlidingGroupWindow {
-  def validate(
-      tableEnv: TableEnvironment,
-      size: Expression,
-      slide: Expression)
-    : ValidationResult = {
-
-    val checkedSize = size match {
-      case Literal(_, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
-        ValidationSuccess
-      case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
-        ValidationSuccess
-      case _ =>
-        ValidationFailure("Sliding window expects size literal of type Interval of " +
-          "Milliseconds or Interval of Rows.")
-    }
-
-    val checkedSlide = slide match {
-      case Literal(_, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
-        ValidationSuccess
-      case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
-        ValidationSuccess
-      case _ =>
-        ValidationFailure("Sliding window expects slide literal of type Interval of " +
-          "Milliseconds or Interval of Rows.")
-    }
-
-    checkedSize
-      .orElse(checkedSlide)
-      .orElse {
-        if (size.resultType != slide.resultType) {
-          ValidationFailure("Sliding window expects same type of size and slide.")
-        } else {
-          ValidationSuccess
-        }
-      }
-  }
-}
-
-case class ProcessingTimeSlidingGroupWindow(
-    override val alias: Expression,
+case class SlidingGroupWindow(
+    alias: Expression,
+    timeField: Expression,
     size: Expression,
     slide: Expression)
-  extends ProcessingTimeGroupWindow(alias) {
+  extends LogicalWindow(
+    alias,
+    timeField) {
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
-    ProcessingTimeSlidingGroupWindow(
+    SlidingGroupWindow(
       resolve(alias),
+      resolve(timeField),
       resolve(size),
       resolve(slide))
 
   override def validate(tableEnv: TableEnvironment): ValidationResult =
-    super.validate(tableEnv).orElse(SlidingGroupWindow.validate(tableEnv, size, slide))
+    super.validate(tableEnv).orElse(
+      tableEnv match {
 
-  override def toString: String = s"ProcessingTimeSlidingGroupWindow($alias, $size, $slide)"
-}
+        // check size
+        case _ if !isTimeIntervalLiteral(size) && !isRowCountLiteral(size) =>
+          ValidationFailure(
+            "Sliding window expects size literal of type Interval of Milliseconds " +
+              "or Interval of Rows.")
 
-case class EventTimeSlidingGroupWindow(
-    override val alias: Expression,
-    timeField: Expression,
-    size: Expression,
-    slide: Expression)
-  extends EventTimeGroupWindow(alias, timeField) {
+        // check slide
+        case _ if !isTimeIntervalLiteral(slide) && !isRowCountLiteral(slide) =>
+          ValidationFailure(
+            "Sliding window expects slide literal of type Interval of Milliseconds " +
+              "or Interval of Rows.")
 
-  override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
-    EventTimeSlidingGroupWindow(
-      resolve(alias),
-      resolve(timeField),
-      resolve(size),
-      resolve(slide))
+        // check same type of intervals
+        case _ if isTimeIntervalLiteral(size) != isTimeIntervalLiteral(slide) =>
+          ValidationFailure("Sliding window expects same type of size and slide.")
 
-  override def validate(tableEnv: TableEnvironment): ValidationResult =
-    super.validate(tableEnv)
-      .orElse(SlidingGroupWindow.validate(tableEnv, size, slide))
-      .orElse(size match {
-        case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS)
-          if tableEnv.isInstanceOf[StreamTableEnvironment] =>
+        // check time attribute
+        case _: StreamTableEnvironment if !isTimeAttribute(timeField) =>
+          ValidationFailure(
+            "Sliding window expects a time attribute for grouping in a stream environment.")
+        case _: BatchTableEnvironment if isTimePoint(size.resultType) =>
+          ValidationFailure(
+            "Sliding window expects a time attribute for grouping in a stream environment.")
+
+        // check row intervals on event-time
+        case _: StreamTableEnvironment
+            if isRowCountLiteral(size) && isRowtimeAttribute(timeField) =>
           ValidationFailure(
             "Event-time grouping windows on row intervals in a stream environment " +
               "are currently not supported.")
+
         case _ =>
           ValidationSuccess
-      })
+      }
+    )
 
-  override def toString: String = s"EventTimeSlidingGroupWindow($alias, $timeField, $size, $slide)"
+  override def toString: String = s"SlidingGroupWindow($alias, $timeField, $size, $slide)"
 }
 
 // ------------------------------------------------------------------------------------------------
 // Session group windows
 // ------------------------------------------------------------------------------------------------
 
-object SessionGroupWindow {
-
-  def validate(tableEnv: TableEnvironment, gap: Expression): ValidationResult = gap match {
-    case Literal(timeInterval: Long, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
-      ValidationSuccess
-    case _ =>
-      ValidationFailure(
-        "Session window expects gap literal of type Interval of Milliseconds.")
-  }
-}
-
-case class ProcessingTimeSessionGroupWindow(
-    override val alias: Expression,
-    gap: Expression)
-  extends ProcessingTimeGroupWindow(alias) {
-
-  override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
-    ProcessingTimeSessionGroupWindow(
-      resolve(alias),
-      resolve(gap))
-
-  override def validate(tableEnv: TableEnvironment): ValidationResult =
-    super.validate(tableEnv).orElse(SessionGroupWindow.validate(tableEnv, gap))
-
-  override def toString: String = s"ProcessingTimeSessionGroupWindow($alias, $gap)"
-}
-
-case class EventTimeSessionGroupWindow(
-    override val alias: Expression,
+case class SessionGroupWindow(
+    alias: Expression,
     timeField: Expression,
     gap: Expression)
-  extends EventTimeGroupWindow(
+  extends LogicalWindow(
     alias,
     timeField) {
 
   override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
-    EventTimeSessionGroupWindow(
+    SessionGroupWindow(
       resolve(alias),
       resolve(timeField),
       resolve(gap))
 
   override def validate(tableEnv: TableEnvironment): ValidationResult =
-    super.validate(tableEnv).orElse(SessionGroupWindow.validate(tableEnv, gap))
+    super.validate(tableEnv).orElse(
+      tableEnv match {
+
+        // check size
+        case _ if !isTimeIntervalLiteral(gap) =>
+          ValidationFailure(
+            "Session window expects size literal of type Interval of Milliseconds.")
+
+        // check time attribute
+        case _: StreamTableEnvironment if !isTimeAttribute(timeField) =>
+          ValidationFailure(
+            "Session window expects a time attribute for grouping in a stream environment.")
+        case _: BatchTableEnvironment if isTimePoint(gap.resultType) =>
+          ValidationFailure(
+            "Session window expects a time attribute for grouping in a stream environment.")
+
+        case _ =>
+          ValidationSuccess
+      }
+    )
 
-  override def toString: String = s"EventTimeSessionGroupWindow($alias, $timeField, $gap)"
+  override def toString: String = s"SessionGroupWindow($alias, $timeField, $gap)"
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index 5f2394cd26a3a..383914526f095 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -70,8 +70,6 @@ case class Project(projectList: Seq[NamedExpression], child: LogicalNode) extend
     def checkName(name: String): Unit = {
       if (names.contains(name)) {
         failValidation(s"Duplicate field name $name.")
-      } else if (tableEnv.isInstanceOf[StreamTableEnvironment] && name == "rowtime") {
-        failValidation("'rowtime' cannot be used as field name in a streaming environment.")
       } else {
         names.add(name)
       }
@@ -112,10 +110,6 @@ case class AliasNode(aliasList: Seq[Expression], child: LogicalNode) extends Una
       failValidation("Alias only accept name expressions as arguments")
     } else if (!aliasList.forall(_.asInstanceOf[UnresolvedFieldReference].name != "*")) {
       failValidation("Alias can not accept '*' as name")
-    } else if (tableEnv.isInstanceOf[StreamTableEnvironment] && !aliasList.forall {
-          case UnresolvedFieldReference(name) => name != "rowtime"
-        }) {
-      failValidation("'rowtime' cannot be used as field name in a streaming environment.")
     } else {
       val names = aliasList.map(_.asInstanceOf[UnresolvedFieldReference].name)
       val input = child.output
@@ -561,26 +555,20 @@ case class WindowAggregate(
   override def resolveReference(
       tableEnv: TableEnvironment,
       name: String)
-    : Option[NamedExpression] = tableEnv match {
-    // resolve reference to rowtime attribute in a streaming environment
-    case _: StreamTableEnvironment if name == "rowtime" =>
-      Some(RowtimeAttribute())
-    case _ =>
-      window.alias match {
-        // resolve reference to this window's alias
-        case UnresolvedFieldReference(alias) if name == alias =>
-          // check if reference can already be resolved by input fields
-          val found = super.resolveReference(tableEnv, name)
-          if (found.isDefined) {
-            failValidation(s"Reference $name is ambiguous.")
-          } else {
-            Some(WindowReference(name))
-          }
-        case _ =>
-          // resolve references as usual
-          super.resolveReference(tableEnv, name)
-      }
-  }
+    : Option[NamedExpression] = window.aliasAttribute match {
+      // resolve reference to this window's name
+      case UnresolvedFieldReference(alias) if name == alias =>
+        // check if reference can already be resolved by input fields
+        val found = super.resolveReference(tableEnv, name)
+        if (found.isDefined) {
+          failValidation(s"Reference $name is ambiguous.")
+        } else {
+          Some(WindowReference(name))
+        }
+      case _ =>
+        // resolve references as usual
+        super.resolveReference(tableEnv, name)
+    }
 
   override protected[logical] def construct(relBuilder: RelBuilder): RelBuilder = {
     val flinkRelBuilder = relBuilder.asInstanceOf[FlinkRelBuilder]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
index 96a74704e6edb..5c35129aa35e1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
@@ -19,13 +19,12 @@
 package org.apache.flink.table.plan.nodes
 
 import org.apache.calcite.plan.{RelOptCost, RelOptPlanner}
-import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rex._
 import org.apache.flink.api.common.functions.{FlatMapFunction, RichFlatMapFunction}
-import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.TableConfig
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.{CodeGenerator, GeneratedFunction}
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.runtime.FlatMapRunner
 import org.apache.flink.types.Row
 
@@ -35,21 +34,30 @@ import scala.collection.JavaConverters._
 trait CommonCalc {
 
   private[flink] def functionBody(
-      generator: CodeGenerator,
-      inputType: TypeInformation[Row],
-      rowType: RelDataType,
-      calcProgram: RexProgram,
-      config: TableConfig)
+     generator: CodeGenerator,
+     inputSchema: RowSchema,
+     returnSchema: RowSchema,
+     calcProgram: RexProgram,
+     config: TableConfig)
     : String = {
 
-    val returnType = FlinkTypeFactory.toInternalRowTypeInfo(rowType)
+    val expandedExpressions = calcProgram
+      .getProjectList
+      .map(expr => calcProgram.expandLocalRef(expr))
+      // time indicator fields must not be part of the code generation
+      .filter(expr => !FlinkTypeFactory.isTimeIndicatorType(expr.getType))
+      // update indices
+      .map(expr => inputSchema.mapRexNode(expr))
+
+    val condition = if (calcProgram.getCondition != null) {
+      inputSchema.mapRexNode(calcProgram.expandLocalRef(calcProgram.getCondition))
+    } else {
+      null
+    }
 
-    val condition = calcProgram.getCondition
-    val expandedExpressions = calcProgram.getProjectList.map(
-      expr => calcProgram.expandLocalRef(expr))
     val projection = generator.generateResultExpression(
-      returnType,
-      rowType.getFieldNames,
+      returnSchema.physicalTypeInfo,
+      returnSchema.physicalFieldNames,
       expandedExpressions)
 
     // only projection
@@ -60,8 +68,7 @@ trait CommonCalc {
         |""".stripMargin
     }
     else {
-      val filterCondition = generator.generateExpression(
-        calcProgram.expandLocalRef(calcProgram.getCondition))
+      val filterCondition = generator.generateExpression(condition)
       // only filter
       if (projection == null) {
         s"""
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
index 6c4066b7af9d4..02305ee8495c0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
@@ -23,11 +23,11 @@ import org.apache.calcite.sql.SemiJoinType
 import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.{TableConfig, TableException}
-import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenUtils.primitiveDefaultValue
 import org.apache.flink.table.codegen.GeneratedExpression.{ALWAYS_NULL, NO_CODE}
 import org.apache.flink.table.codegen.{CodeGenerator, GeneratedCollector, GeneratedExpression, GeneratedFunction}
 import org.apache.flink.table.functions.utils.TableSqlFunction
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.runtime.{CorrelateFlatMapRunner, TableFunctionCollector}
 import org.apache.flink.types.Row
 
@@ -44,9 +44,9 @@ trait CommonCorrelate {
     */
   private[flink] def correlateMapFunction(
       config: TableConfig,
-      inputTypeInfo: TypeInformation[Row],
+      inputSchema: RowSchema,
       udtfTypeInfo: TypeInformation[Any],
-      rowType: RelDataType,
+      returnSchema: RowSchema,
       joinType: SemiJoinType,
       rexCall: RexCall,
       condition: Option[RexNode],
@@ -54,26 +54,24 @@ trait CommonCorrelate {
       ruleDescription: String)
     : CorrelateFlatMapRunner[Row, Row] = {
 
-    val returnType = FlinkTypeFactory.toInternalRowTypeInfo(rowType)
-
     val flatMap = generateFunction(
       config,
-      inputTypeInfo,
+      inputSchema.physicalTypeInfo,
       udtfTypeInfo,
-      returnType,
-      rowType,
+      returnSchema.physicalTypeInfo,
+      returnSchema.logicalFieldNames,
       joinType,
-      rexCall,
+      inputSchema.mapRexNode(rexCall).asInstanceOf[RexCall],
       pojoFieldMapping,
       ruleDescription)
 
     val collector = generateCollector(
       config,
-      inputTypeInfo,
+      inputSchema.physicalTypeInfo,
       udtfTypeInfo,
-      returnType,
-      rowType,
-      condition,
+      returnSchema.physicalTypeInfo,
+      returnSchema.logicalFieldNames,
+      condition.map(inputSchema.mapRexNode),
       pojoFieldMapping)
 
     new CorrelateFlatMapRunner[Row, Row](
@@ -93,7 +91,7 @@ trait CommonCorrelate {
       inputTypeInfo: TypeInformation[Row],
       udtfTypeInfo: TypeInformation[Any],
       returnType: TypeInformation[Row],
-      rowType: RelDataType,
+      resultFieldNames: Seq[String],
       joinType: SemiJoinType,
       rexCall: RexCall,
       pojoFieldMapping: Option[Array[Int]],
@@ -134,7 +132,7 @@ trait CommonCorrelate {
           x.resultType)
       }
       val outerResultExpr = functionGenerator.generateResultExpression(
-        input1AccessExprs ++ input2NullExprs, returnType, rowType.getFieldNames.asScala)
+        input1AccessExprs ++ input2NullExprs, returnType, resultFieldNames)
       body +=
         s"""
           |boolean hasOutput = $collectorTerm.isCollected();
@@ -162,7 +160,7 @@ trait CommonCorrelate {
       inputTypeInfo: TypeInformation[Row],
       udtfTypeInfo: TypeInformation[Any],
       returnType: TypeInformation[Row],
-      rowType: RelDataType,
+      resultFieldNames: Seq[String],
       condition: Option[RexNode],
       pojoFieldMapping: Option[Array[Int]])
     : GeneratedCollector = {
@@ -180,7 +178,7 @@ trait CommonCorrelate {
     val crossResultExpr = generator.generateResultExpression(
       input1AccessExprs ++ input2AccessExprs,
       returnType,
-      rowType.getFieldNames.asScala)
+      resultFieldNames)
 
     val collectorCode = if (condition.isEmpty) {
       s"""
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala
index 0a0d20441db37..091a1ea298705 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala
@@ -18,11 +18,10 @@
 
 package org.apache.flink.table.plan.nodes
 
-import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.api.common.functions.Function
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.TableConfig
-import org.apache.flink.table.codegen.CodeGenerator
-import org.apache.flink.table.runtime.MapRunner
+import org.apache.flink.table.codegen.{CodeGenerator, GeneratedFunction}
 import org.apache.flink.types.Row
 
 /**
@@ -42,21 +41,22 @@ trait CommonScan {
     externalTypeInfo != internalTypeInfo
   }
 
-  private[flink] def getConversionMapper(
+  private[flink] def generatedConversionFunction[F <: Function](
       config: TableConfig,
+      functionClass: Class[F],
       inputType: TypeInformation[Any],
       expectedType: TypeInformation[Row],
       conversionOperatorName: String,
       fieldNames: Seq[String],
-      inputPojoFieldMapping: Option[Array[Int]] = None)
-    : MapFunction[Any, Row] = {
+      inputFieldMapping: Option[Array[Int]] = None)
+    : GeneratedFunction[F, Row] = {
 
     val generator = new CodeGenerator(
       config,
       false,
       inputType,
       None,
-      inputPojoFieldMapping)
+      inputFieldMapping)
     val conversion = generator.generateConverterResultExpression(expectedType, fieldNames)
 
     val body =
@@ -65,17 +65,11 @@ trait CommonScan {
          |return ${conversion.resultTerm};
          |""".stripMargin
 
-    val genFunction = generator.generateFunction(
+    generator.generateFunction(
       conversionOperatorName,
-      classOf[MapFunction[Any, Row]],
+      functionClass,
       body,
       expectedType)
-
-    new MapRunner[Any, Row](
-      genFunction.name,
-      genFunction.code,
-      genFunction.returnType)
-
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala
index 6878473e0f4b0..1048549e2da4a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/OverAggregate.scala
@@ -18,14 +18,13 @@
 
 package org.apache.flink.table.plan.nodes
 
-import org.apache.calcite.rel.{RelFieldCollation, RelNode}
-import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFieldImpl}
-import org.apache.calcite.rel.core.AggregateCall
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.core.{AggregateCall, Window}
 import org.apache.calcite.rel.core.Window.Group
-import org.apache.calcite.rel.core.Window
-import org.apache.calcite.rex.{RexInputRef}
+import org.apache.calcite.rel.{RelFieldCollation, RelNode}
+import org.apache.calcite.rex.RexInputRef
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.runtime.aggregate.AggregateUtil._
-import org.apache.flink.table.functions.{ProcTimeType, RowTimeType}
 
 import scala.collection.JavaConverters._
 
@@ -43,7 +42,7 @@ trait OverAggregate {
     val inFields = inputType.getFieldList.asScala
 
     val orderingString = orderFields.asScala.map {
-      x => inFields(x.getFieldIndex).getValue
+      x => inFields(x.getFieldIndex).getName
     }.mkString(", ")
 
     orderingString
@@ -66,24 +65,8 @@ trait OverAggregate {
     rowType: RelDataType,
     namedAggregates: Seq[CalcitePair[AggregateCall, String]]): String = {
 
-    val inFields = inputType.getFieldList.asScala.map {
-      x =>
-        x.asInstanceOf[RelDataTypeFieldImpl].getType
-        match {
-          case proceTime: ProcTimeType => "PROCTIME"
-          case rowTime: RowTimeType => "ROWTIME"
-          case _ => x.asInstanceOf[RelDataTypeFieldImpl].getName
-        }
-    }
-    val outFields = rowType.getFieldList.asScala.map {
-      x =>
-        x.asInstanceOf[RelDataTypeFieldImpl].getType
-        match {
-          case proceTime: ProcTimeType => "PROCTIME"
-          case rowTime: RowTimeType => "ROWTIME"
-          case _ => x.asInstanceOf[RelDataTypeFieldImpl].getName
-        }
-    }
+    val inFields = inputType.getFieldNames.asScala
+    val outFields = rowType.getFieldNames.asScala
 
     val aggStrings = namedAggregates.map(_.getKey).map(
       a => s"${a.getAggregation}(${
@@ -109,7 +92,7 @@ trait OverAggregate {
     input: RelNode): Long = {
 
     val ref: RexInputRef = overWindow.lowerBound.getOffset.asInstanceOf[RexInputRef]
-    val lowerBoundIndex = input.getRowType.getFieldCount - ref.getIndex;
+    val lowerBoundIndex = input.getRowType.getFieldCount - ref.getIndex
     val lowerBound = logicWindow.constants.get(lowerBoundIndex).getValue2
     lowerBound match {
       case x: java.math.BigDecimal => x.asInstanceOf[java.math.BigDecimal].longValue()
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala
index d9244505b499b..c18c3d1544ff6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala
@@ -37,9 +37,11 @@ abstract class PhysicalTableSourceScan(
 
   override def deriveRowType(): RelDataType = {
     val flinkTypeFactory = cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
-    flinkTypeFactory.buildRowDataType(
+    flinkTypeFactory.buildLogicalRowType(
       TableEnvironment.getFieldNames(tableSource),
-      TableEnvironment.getFieldTypes(tableSource.getReturnType))
+      TableEnvironment.getFieldTypes(tableSource.getReturnType),
+      None,
+      None)
   }
 
   override def explainTerms(pw: RelWriter): RelWriter = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala
index b39b8ed8896be..cc5d9fb2d7f52 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala
@@ -18,11 +18,13 @@
 
 package org.apache.flink.table.plan.nodes.dataset
 
+import org.apache.flink.api.common.functions.MapFunction
 import org.apache.flink.api.java.DataSet
 import org.apache.flink.table.api.TableConfig
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.CommonScan
 import org.apache.flink.table.plan.schema.FlinkTable
+import org.apache.flink.table.runtime.MapRunner
 import org.apache.flink.types.Row
 
 import scala.collection.JavaConversions._
@@ -43,17 +45,23 @@ trait BatchScan extends CommonScan with DataSetRel {
     // conversion
     if (needsConversion(inputType, internalType)) {
 
-      val mapFunc = getConversionMapper(
+      val function = generatedConversionFunction(
         config,
+        classOf[MapFunction[Any, Row]],
         inputType,
         internalType,
         "DataSetSourceConversion",
         getRowType.getFieldNames,
         Some(flinkTable.fieldIndexes))
 
+      val runner = new MapRunner[Any, Row](
+        function.name,
+        function.code,
+        function.returnType)
+
       val opName = s"from: (${getRowType.getFieldNames.asScala.toList.mkString(", ")})"
 
-      input.map(mapFunc).name(opName)
+      input.map(runner).name(opName)
     }
     // no conversion necessary, forward
     else {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchTableSourceScan.scala
index bf4291a41675d..fb291e407a5a7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchTableSourceScan.scala
@@ -22,7 +22,8 @@ import org.apache.calcite.plan._
 import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.flink.api.java.DataSet
-import org.apache.flink.table.api.BatchTableEnvironment
+import org.apache.flink.table.api.{BatchTableEnvironment, TableEnvironment}
+import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.PhysicalTableSourceScan
 import org.apache.flink.table.plan.schema.TableSourceTable
 import org.apache.flink.table.sources.{BatchTableSource, TableSource}
@@ -37,7 +38,16 @@ class BatchTableSourceScan(
   extends PhysicalTableSourceScan(cluster, traitSet, table, tableSource)
   with BatchScan {
 
-  override def computeSelfCost(planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = {
+  override def deriveRowType() = {
+    val flinkTypeFactory = cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+    flinkTypeFactory.buildLogicalRowType(
+      TableEnvironment.getFieldNames(tableSource),
+      TableEnvironment.getFieldTypes(tableSource.getReturnType),
+      None,
+      None)
+  }
+
+  override def computeSelfCost (planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = {
     val rowCnt = metadata.getRowCount(this)
     planner.getCostFactory.makeCost(rowCnt, rowCnt, rowCnt * estimateRowSize(getRowType))
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
index b92775c72485f..c22dc54d99f17 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
@@ -91,6 +91,9 @@ class DataSetAggregate(
   override def translateToPlan(tableEnv: BatchTableEnvironment): DataSet[Row] = {
 
     val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(tableEnv)
+    val input = inputNode.asInstanceOf[DataSetRel]
+
+    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
 
     val generator = new CodeGenerator(
       tableEnv.getConfig,
@@ -104,15 +107,14 @@ class DataSetAggregate(
       ) = AggregateUtil.createDataSetAggregateFunctions(
         generator,
         namedAggregates,
-        inputType,
+        input.getRowType,
+        inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
         rowRelDataType,
         grouping,
         inGroupingSet)
 
     val aggString = aggregationToString(inputType, grouping, getRowType, namedAggregates, Nil)
 
-    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
-
     if (grouping.length > 0) {
       // grouped aggregation
       val aggOpName = s"groupBy: (${groupingToString(inputType, grouping)}), " +
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
index e05b5a83e69cb..9e180825fc7ca 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
@@ -26,10 +26,13 @@ import org.apache.calcite.rel.{RelNode, RelWriter}
 import org.apache.calcite.rex._
 import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.api.java.DataSet
+import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.table.api.BatchTableEnvironment
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.CommonCalc
+import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.runtime.FlatMapRunner
 import org.apache.flink.types.Row
 
 /**
@@ -83,14 +86,14 @@ class DataSetCalc(
 
     val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(tableEnv)
 
-    val returnType = FlinkTypeFactory.toInternalRowTypeInfo(getRowType)
-
     val generator = new CodeGenerator(config, false, inputDS.getType)
 
+    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
+
     val body = functionBody(
       generator,
-      inputDS.getType,
-      getRowType,
+      new RowSchema(getInput.getRowType),
+      new RowSchema(getRowType),
       calcProgram,
       config)
 
@@ -98,9 +101,13 @@ class DataSetCalc(
       ruleDescription,
       classOf[FlatMapFunction[Row, Row]],
       body,
-      returnType)
+      rowTypeInfo)
+
+    val runner = new FlatMapRunner[Row, Row](
+      genFunction.name,
+      genFunction.code,
+      genFunction.returnType)
 
-    val mapFunc = calcMapFunction(genFunction)
-    inputDS.flatMap(mapFunc).name(calcOpName(calcProgram, getExpressionString))
+    inputDS.flatMap(runner).name(calcOpName(calcProgram, getExpressionString))
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
index 2a62e21ab7ef9..6c79b45d8667a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
@@ -25,10 +25,13 @@ import org.apache.calcite.rex.{RexCall, RexNode}
 import org.apache.calcite.sql.SemiJoinType
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.DataSet
+import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.table.api.BatchTableEnvironment
+import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.functions.utils.TableSqlFunction
 import org.apache.flink.table.plan.nodes.CommonCorrelate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalTableFunctionScan
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.types.Row
 
 /**
@@ -98,11 +101,13 @@ class DataSetCorrelate(
     val pojoFieldMapping = sqlFunction.getPojoFieldMapping
     val udtfTypeInfo = sqlFunction.getRowTypeInfo.asInstanceOf[TypeInformation[Any]]
 
+    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
+
     val mapFunc = correlateMapFunction(
       config,
-      inputDS.getType,
+      new RowSchema(getInput.getRowType),
       udtfTypeInfo,
-      getRowType,
+      new RowSchema(getRowType),
       joinType,
       rexCall,
       condition,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala
index 96c427eed60f6..3cb872aeb2f86 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala
@@ -24,15 +24,16 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
 import org.apache.flink.api.common.operators.Order
 import org.apache.flink.api.java.DataSet
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.api.java.typeutils.{ResultTypeQueryable, RowTypeInfo}
 import org.apache.flink.table.api.BatchTableEnvironment
 import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
+import org.apache.flink.table.expressions.ExpressionUtils._
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.plan.nodes.CommonAggregate
 import org.apache.flink.table.runtime.aggregate.AggregateUtil.{CalcitePair, _}
-import org.apache.flink.table.typeutils.TypeCheckUtils.isTimeInterval
+import org.apache.flink.table.typeutils.TypeCheckUtils.{isLong, isTimePoint}
 import org.apache.flink.types.Row
 
 /**
@@ -106,8 +107,6 @@ class DataSetWindowAggregate(
 
   override def translateToPlan(tableEnv: BatchTableEnvironment): DataSet[Row] = {
 
-    val config = tableEnv.getConfig
-
     val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(tableEnv)
 
     val generator = new CodeGenerator(
@@ -119,30 +118,31 @@ class DataSetWindowAggregate(
     val caseSensitive = tableEnv.getFrameworkConfig.getParserConfig.caseSensitive()
 
     window match {
-      case EventTimeTumblingGroupWindow(_, _, size) =>
+      case TumblingGroupWindow(_, timeField, size)
+          if isTimePoint(timeField.resultType) || isLong(timeField.resultType) =>
         createEventTimeTumblingWindowDataSet(
           generator,
           inputDS,
-          isTimeInterval(size.resultType),
+          isTimeIntervalLiteral(size),
           caseSensitive)
 
-      case EventTimeSessionGroupWindow(_, _, gap) =>
+      case SessionGroupWindow(_, timeField, gap)
+          if isTimePoint(timeField.resultType) || isLong(timeField.resultType) =>
         createEventTimeSessionWindowDataSet(generator, inputDS, caseSensitive)
 
-      case EventTimeSlidingGroupWindow(_, _, size, slide) =>
+      case SlidingGroupWindow(_, timeField, size, slide)
+          if isTimePoint(timeField.resultType) || isLong(timeField.resultType) =>
         createEventTimeSlidingWindowDataSet(
           generator,
           inputDS,
-          isTimeInterval(size.resultType),
+          isTimeIntervalLiteral(size),
           asLong(size),
           asLong(slide),
           caseSensitive)
 
-      case _: ProcessingTimeGroupWindow =>
+      case _ =>
         throw new UnsupportedOperationException(
-          "Processing-time tumbling windows are not supported in a batch environment, " +
-            "windows in a batch environment must declare a time attribute over which " +
-            "the query is evaluated.")
+          s"Window $window is not supported in a batch environment.")
     }
   }
 
@@ -152,18 +152,22 @@ class DataSetWindowAggregate(
       isTimeWindow: Boolean,
       isParserCaseSensitive: Boolean): DataSet[Row] = {
 
+    val input = inputNode.asInstanceOf[DataSetRel]
+
     val mapFunction = createDataSetWindowPrepareMapFunction(
       generator,
       window,
       namedAggregates,
       grouping,
-      inputType,
+      input.getRowType,
+      inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
       isParserCaseSensitive)
     val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction(
       generator,
       window,
       namedAggregates,
-      inputType,
+      input.getRowType,
+      inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
       getRowType,
       grouping,
       namedProperties)
@@ -210,6 +214,8 @@ class DataSetWindowAggregate(
       inputDS: DataSet[Row],
       isParserCaseSensitive: Boolean): DataSet[Row] = {
 
+    val input = inputNode.asInstanceOf[DataSetRel]
+
     val groupingKeys = grouping.indices.toArray
     val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType)
 
@@ -219,7 +225,8 @@ class DataSetWindowAggregate(
       window,
       namedAggregates,
       grouping,
-      inputType,
+      input.getRowType,
+      inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
       isParserCaseSensitive)
 
     val mappedInput = inputDS.map(mapFunction).name(prepareOperatorName)
@@ -245,7 +252,8 @@ class DataSetWindowAggregate(
           generator,
           window,
           namedAggregates,
-          inputType,
+          input.getRowType,
+          inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
           grouping)
 
         // create groupReduceFunction for calculating the aggregations
@@ -253,7 +261,8 @@ class DataSetWindowAggregate(
           generator,
           window,
           namedAggregates,
-          inputType,
+          input.getRowType,
+          inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
           rowRelDataType,
           grouping,
           namedProperties,
@@ -275,7 +284,8 @@ class DataSetWindowAggregate(
           generator,
           window,
           namedAggregates,
-          inputType,
+          input.getRowType,
+          inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
           grouping)
 
         // create groupReduceFunction for calculating the aggregations
@@ -283,7 +293,8 @@ class DataSetWindowAggregate(
           generator,
           window,
           namedAggregates,
-          inputType,
+          input.getRowType,
+          inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
           rowRelDataType,
           grouping,
           namedProperties,
@@ -308,7 +319,8 @@ class DataSetWindowAggregate(
           generator,
           window,
           namedAggregates,
-          inputType,
+          input.getRowType,
+          inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
           rowRelDataType,
           grouping,
           namedProperties)
@@ -324,7 +336,8 @@ class DataSetWindowAggregate(
           generator,
           window,
           namedAggregates,
-          inputType,
+          input.getRowType,
+          inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
           rowRelDataType,
           grouping,
           namedProperties)
@@ -347,6 +360,8 @@ class DataSetWindowAggregate(
       isParserCaseSensitive: Boolean)
     : DataSet[Row] = {
 
+    val input = inputNode.asInstanceOf[DataSetRel]
+
     // create MapFunction for initializing the aggregations
     // it aligns the rowtime for pre-tumbling in case of a time-window for partial aggregates
     val mapFunction = createDataSetWindowPrepareMapFunction(
@@ -354,7 +369,8 @@ class DataSetWindowAggregate(
       window,
       namedAggregates,
       grouping,
-      inputType,
+      input.getRowType,
+      inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
       isParserCaseSensitive)
 
     val mappedDataSet = inputDS
@@ -390,7 +406,8 @@ class DataSetWindowAggregate(
           window,
           namedAggregates,
           grouping,
-          inputType,
+          input.getRowType,
+          inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
           isParserCaseSensitive)
 
         mappedDataSet.asInstanceOf[DataSet[Row]]
@@ -426,7 +443,8 @@ class DataSetWindowAggregate(
       generator,
       window,
       namedAggregates,
-      inputType,
+      input.getRowType,
+      inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes,
       rowRelDataType,
       grouping,
       namedProperties,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
index c232a71f02631..56974498bc418 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
@@ -25,20 +25,18 @@ import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
 import org.apache.flink.api.java.tuple.Tuple
 import org.apache.flink.streaming.api.datastream.{AllWindowedStream, DataStream, KeyedStream, WindowedStream}
 import org.apache.flink.streaming.api.windowing.assigners._
-import org.apache.flink.streaming.api.windowing.time.Time
 import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
 import org.apache.flink.table.api.StreamTableEnvironment
 import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
-import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
-import org.apache.flink.table.expressions._
+import org.apache.flink.table.expressions.ExpressionUtils._
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.plan.nodes.CommonAggregate
 import org.apache.flink.table.plan.nodes.datastream.DataStreamAggregate._
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.runtime.aggregate.AggregateUtil._
 import org.apache.flink.table.runtime.aggregate._
 import org.apache.flink.table.typeutils.TypeCheckUtils.isTimeInterval
-import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
 import org.apache.flink.types.Row
 
 class DataStreamAggregate(
@@ -48,12 +46,12 @@ class DataStreamAggregate(
     traitSet: RelTraitSet,
     inputNode: RelNode,
     namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-    rowRelDataType: RelDataType,
-    inputType: RelDataType,
+    schema: RowSchema,
+    inputSchema: RowSchema,
     grouping: Array[Int])
   extends SingleRel(cluster, traitSet, inputNode) with CommonAggregate with DataStreamRel {
 
-  override def deriveRowType(): RelDataType = rowRelDataType
+  override def deriveRowType(): RelDataType = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamAggregate(
@@ -63,22 +61,22 @@ class DataStreamAggregate(
       traitSet,
       inputs.get(0),
       namedAggregates,
-      getRowType,
-      inputType,
+      schema,
+      inputSchema,
       grouping)
   }
 
   override def toString: String = {
     s"Aggregate(${
       if (!grouping.isEmpty) {
-        s"groupBy: (${groupingToString(inputType, grouping)}), "
+        s"groupBy: (${groupingToString(inputSchema.logicalType, grouping)}), "
       } else {
         ""
       }
     }window: ($window), " +
       s"select: (${
         aggregationToString(
-          inputType,
+          inputSchema.logicalType,
           grouping,
           getRowType,
           namedAggregates,
@@ -88,13 +86,13 @@ class DataStreamAggregate(
 
   override def explainTerms(pw: RelWriter): RelWriter = {
     super.explainTerms(pw)
-      .itemIf("groupBy", groupingToString(inputType, grouping), !grouping.isEmpty)
+      .itemIf("groupBy", groupingToString(inputSchema.logicalType, grouping), !grouping.isEmpty)
       .item("window", window)
       .item(
         "select", aggregationToString(
-          inputType,
+          inputSchema.logicalType,
           grouping,
-          getRowType,
+          schema.logicalType,
           namedAggregates,
           namedProperties))
   }
@@ -102,17 +100,20 @@ class DataStreamAggregate(
   override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
 
     val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
-
-    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType)
+    val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
+      new CalcitePair[AggregateCall, String](
+        inputSchema.mapAggregateCall(namedAggregate.left),
+        namedAggregate.right)
+    }
 
     val aggString = aggregationToString(
-      inputType,
+      inputSchema.logicalType,
       grouping,
-      getRowType,
+      schema.logicalType,
       namedAggregates,
       namedProperties)
 
-    val keyedAggOpName = s"groupBy: (${groupingToString(inputType, grouping)}), " +
+    val keyedAggOpName = s"groupBy: (${groupingToString(schema.logicalType, grouping)}), " +
       s"window: ($window), " +
       s"select: ($aggString)"
     val nonKeyedAggOpName = s"window: ($window), select: ($aggString)"
@@ -123,21 +124,21 @@ class DataStreamAggregate(
       inputDS.getType)
 
     val needMerge = window match {
-      case ProcessingTimeSessionGroupWindow(_, _) => true
-      case EventTimeSessionGroupWindow(_, _, _) => true
+      case SessionGroupWindow(_, _, _) => true
       case _ => false
     }
+    val physicalGrouping = grouping.map(inputSchema.mapIndex)
 
     // grouped / keyed aggregation
-    if (grouping.length > 0) {
+    if (physicalGrouping.length > 0) {
       val windowFunction = AggregateUtil.createAggregationGroupWindowFunction(
         window,
-        grouping.length,
-        namedAggregates.size,
-        rowRelDataType.getFieldCount,
+        physicalGrouping.length,
+        physicalNamedAggregates.size,
+        schema.physicalArity,
         namedProperties)
 
-      val keyedStream = inputDS.keyBy(grouping: _*)
+      val keyedStream = inputDS.keyBy(physicalGrouping: _*)
       val windowedStream =
         createKeyedWindowedStream(window, keyedStream)
           .asInstanceOf[WindowedStream[Row, Tuple, DataStreamWindow]]
@@ -145,20 +146,26 @@ class DataStreamAggregate(
       val (aggFunction, accumulatorRowType, aggResultRowType) =
         AggregateUtil.createDataStreamAggregateFunction(
           generator,
-          namedAggregates,
-          inputType,
-          rowRelDataType,
+          physicalNamedAggregates,
+          inputSchema.physicalType,
+          inputSchema.physicalFieldTypeInfo,
+          schema.physicalType,
           needMerge)
 
       windowedStream
-        .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, rowTypeInfo)
+        .aggregate(
+          aggFunction,
+          windowFunction,
+          accumulatorRowType,
+          aggResultRowType,
+          schema.physicalTypeInfo)
         .name(keyedAggOpName)
     }
     // global / non-keyed aggregation
     else {
       val windowFunction = AggregateUtil.createAggregationAllWindowFunction(
         window,
-        rowRelDataType.getFieldCount,
+        schema.physicalArity,
         namedProperties)
 
       val windowedStream =
@@ -168,13 +175,19 @@ class DataStreamAggregate(
       val (aggFunction, accumulatorRowType, aggResultRowType) =
         AggregateUtil.createDataStreamAggregateFunction(
           generator,
-          namedAggregates,
-          inputType,
-          rowRelDataType,
+          physicalNamedAggregates,
+          inputSchema.physicalType,
+          inputSchema.physicalFieldTypeInfo,
+          schema.physicalType,
           needMerge)
 
       windowedStream
-        .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, rowTypeInfo)
+        .aggregate(
+          aggFunction,
+          windowFunction,
+          accumulatorRowType,
+          aggResultRowType,
+          schema.physicalTypeInfo)
         .name(nonKeyedAggOpName)
     }
   }
@@ -186,95 +199,102 @@ object DataStreamAggregate {
   private def createKeyedWindowedStream(groupWindow: LogicalWindow, stream: KeyedStream[Row, Tuple])
     : WindowedStream[Row, Tuple, _ <: DataStreamWindow] = groupWindow match {
 
-    case ProcessingTimeTumblingGroupWindow(_, size) if isTimeInterval(size.resultType) =>
-      stream.window(TumblingProcessingTimeWindows.of(asTime(size)))
+    case TumblingGroupWindow(_, timeField, size)
+        if isProctimeAttribute(timeField) && isTimeIntervalLiteral(size)=>
+      stream.window(TumblingProcessingTimeWindows.of(toTime(size)))
 
-    case ProcessingTimeTumblingGroupWindow(_, size) =>
-      stream.countWindow(asCount(size))
+    case TumblingGroupWindow(_, timeField, size)
+        if isProctimeAttribute(timeField) && isRowCountLiteral(size)=>
+      stream.countWindow(toLong(size))
 
-    case EventTimeTumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
-      stream.window(TumblingEventTimeWindows.of(asTime(size)))
+    case TumblingGroupWindow(_, timeField, size)
+        if isRowtimeAttribute(timeField) && isTimeIntervalLiteral(size) =>
+      stream.window(TumblingEventTimeWindows.of(toTime(size)))
 
-    case EventTimeTumblingGroupWindow(_, _, size) =>
+    case TumblingGroupWindow(_, _, size) =>
       // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
       // before applying the  windowing logic. Otherwise, this would be the same as a
       // ProcessingTimeTumblingGroupWindow
       throw new UnsupportedOperationException(
         "Event-time grouping windows on row intervals are currently not supported.")
 
-    case ProcessingTimeSlidingGroupWindow(_, size, slide) if isTimeInterval(size.resultType) =>
-      stream.window(SlidingProcessingTimeWindows.of(asTime(size), asTime(slide)))
+    case SlidingGroupWindow(_, timeField, size, slide)
+        if isProctimeAttribute(timeField) && isTimeIntervalLiteral(slide) =>
+      stream.window(SlidingProcessingTimeWindows.of(toTime(size), toTime(slide)))
 
-    case ProcessingTimeSlidingGroupWindow(_, size, slide) =>
-      stream.countWindow(asCount(size), asCount(slide))
+    case SlidingGroupWindow(_, timeField, size, slide)
+        if isProctimeAttribute(timeField) && isRowCountLiteral(size) =>
+      stream.countWindow(toLong(size), toLong(slide))
 
-    case EventTimeSlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
-      stream.window(SlidingEventTimeWindows.of(asTime(size), asTime(slide)))
+    case SlidingGroupWindow(_, timeField, size, slide)
+        if isRowtimeAttribute(timeField) && isTimeIntervalLiteral(size)=>
+      stream.window(SlidingEventTimeWindows.of(toTime(size), toTime(slide)))
 
-    case EventTimeSlidingGroupWindow(_, _, size, slide) =>
+    case SlidingGroupWindow(_, _, size, slide) =>
       // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
       // before applying the  windowing logic. Otherwise, this would be the same as a
       // ProcessingTimeTumblingGroupWindow
       throw new UnsupportedOperationException(
         "Event-time grouping windows on row intervals are currently not supported.")
 
-    case ProcessingTimeSessionGroupWindow(_, gap: Expression) =>
-      stream.window(ProcessingTimeSessionWindows.withGap(asTime(gap)))
+    case SessionGroupWindow(_, timeField, gap)
+        if isProctimeAttribute(timeField) =>
+      stream.window(ProcessingTimeSessionWindows.withGap(toTime(gap)))
 
-    case EventTimeSessionGroupWindow(_, _, gap) =>
-      stream.window(EventTimeSessionWindows.withGap(asTime(gap)))
+    case SessionGroupWindow(_, timeField, gap)
+        if isRowtimeAttribute(timeField) =>
+      stream.window(EventTimeSessionWindows.withGap(toTime(gap)))
   }
 
   private def createNonKeyedWindowedStream(groupWindow: LogicalWindow, stream: DataStream[Row])
     : AllWindowedStream[Row, _ <: DataStreamWindow] = groupWindow match {
 
-    case ProcessingTimeTumblingGroupWindow(_, size) if isTimeInterval(size.resultType) =>
-      stream.windowAll(TumblingProcessingTimeWindows.of(asTime(size)))
+    case TumblingGroupWindow(_, timeField, size)
+        if isProctimeAttribute(timeField) && isTimeIntervalLiteral(size) =>
+      stream.windowAll(TumblingProcessingTimeWindows.of(toTime(size)))
 
-    case ProcessingTimeTumblingGroupWindow(_, size) =>
-      stream.countWindowAll(asCount(size))
+    case TumblingGroupWindow(_, timeField, size)
+        if isProctimeAttribute(timeField) && isRowCountLiteral(size)=>
+      stream.countWindowAll(toLong(size))
 
-    case EventTimeTumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
-      stream.windowAll(TumblingEventTimeWindows.of(asTime(size)))
+    case TumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
+      stream.windowAll(TumblingEventTimeWindows.of(toTime(size)))
 
-    case EventTimeTumblingGroupWindow(_, _, size) =>
+    case TumblingGroupWindow(_, _, size) =>
       // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
       // before applying the  windowing logic. Otherwise, this would be the same as a
       // ProcessingTimeTumblingGroupWindow
       throw new UnsupportedOperationException(
         "Event-time grouping windows on row intervals are currently not supported.")
 
-    case ProcessingTimeSlidingGroupWindow(_, size, slide) if isTimeInterval(size.resultType) =>
-      stream.windowAll(SlidingProcessingTimeWindows.of(asTime(size), asTime(slide)))
+    case SlidingGroupWindow(_, timeField, size, slide)
+        if isProctimeAttribute(timeField) && isTimeIntervalLiteral(size) =>
+      stream.windowAll(SlidingProcessingTimeWindows.of(toTime(size), toTime(slide)))
 
-    case ProcessingTimeSlidingGroupWindow(_, size, slide) =>
-      stream.countWindowAll(asCount(size), asCount(slide))
+    case SlidingGroupWindow(_, timeField, size, slide)
+        if isProctimeAttribute(timeField) && isRowCountLiteral(size)=>
+      stream.countWindowAll(toLong(size), toLong(slide))
 
-    case EventTimeSlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
-      stream.windowAll(SlidingEventTimeWindows.of(asTime(size), asTime(slide)))
+    case SlidingGroupWindow(_, timeField, size, slide)
+        if isRowtimeAttribute(timeField) && isTimeIntervalLiteral(size)=>
+      stream.windowAll(SlidingEventTimeWindows.of(toTime(size), toTime(slide)))
 
-    case EventTimeSlidingGroupWindow(_, _, size, slide) =>
+    case SlidingGroupWindow(_, _, size, slide) =>
       // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
       // before applying the  windowing logic. Otherwise, this would be the same as a
       // ProcessingTimeTumblingGroupWindow
       throw new UnsupportedOperationException(
         "Event-time grouping windows on row intervals are currently not supported.")
 
-    case ProcessingTimeSessionGroupWindow(_, gap) =>
-      stream.windowAll(ProcessingTimeSessionWindows.withGap(asTime(gap)))
+    case SessionGroupWindow(_, timeField, gap)
+        if isProctimeAttribute(timeField) && isTimeIntervalLiteral(gap) =>
+      stream.windowAll(ProcessingTimeSessionWindows.withGap(toTime(gap)))
 
-    case EventTimeSessionGroupWindow(_, _, gap) =>
-      stream.windowAll(EventTimeSessionWindows.withGap(asTime(gap)))
+    case SessionGroupWindow(_, timeField, gap)
+        if isRowtimeAttribute(timeField) && isTimeIntervalLiteral(gap) =>
+      stream.windowAll(EventTimeSessionWindows.withGap(toTime(gap)))
   }
 
-  def asTime(expr: Expression): Time = expr match {
-    case Literal(value: Long, TimeIntervalTypeInfo.INTERVAL_MILLIS) => Time.milliseconds(value)
-    case _ => throw new IllegalArgumentException()
-  }
 
-  def asCount(expr: Expression): Long = expr match {
-    case Literal(value: Long, RowIntervalTypeInfo.INTERVAL_ROWS) => value
-    case _ => throw new IllegalArgumentException()
-  }
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
index b015a1dd7aa8f..c6c25c09fb471 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
@@ -27,9 +27,9 @@ import org.apache.calcite.rex.RexProgram
 import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
-import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.CommonCalc
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.types.Row
 
 /**
@@ -40,17 +40,25 @@ class DataStreamCalc(
     cluster: RelOptCluster,
     traitSet: RelTraitSet,
     input: RelNode,
-    rowRelDataType: RelDataType,
+    inputSchema: RowSchema,
+    schema: RowSchema,
     calcProgram: RexProgram,
     ruleDescription: String)
   extends Calc(cluster, traitSet, input, calcProgram)
   with CommonCalc
   with DataStreamRel {
 
-  override def deriveRowType(): RelDataType = rowRelDataType
+  override def deriveRowType(): RelDataType = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, child: RelNode, program: RexProgram): Calc = {
-    new DataStreamCalc(cluster, traitSet, child, getRowType, program, ruleDescription)
+    new DataStreamCalc(
+      cluster,
+      traitSet,
+      child,
+      inputSchema,
+      schema,
+      program,
+      ruleDescription)
   }
 
   override def toString: String = calcToString(calcProgram, getExpressionString)
@@ -85,8 +93,8 @@ class DataStreamCalc(
 
     val body = functionBody(
       generator,
-      inputDataStream.getType,
-      getRowType,
+      inputSchema,
+      schema,
       calcProgram,
       config)
 
@@ -94,7 +102,7 @@ class DataStreamCalc(
       ruleDescription,
       classOf[FlatMapFunction[Row, Row]],
       body,
-      FlinkTypeFactory.toInternalRowTypeInfo(getRowType))
+      schema.physicalTypeInfo)
 
     val mapFunc = calcMapFunction(genFunction)
     inputDataStream.flatMap(mapFunc).name(calcOpName(calcProgram, getExpressionString))
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
index 342920a0a5b51..8955110f96d27 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
@@ -18,7 +18,6 @@
 package org.apache.flink.table.plan.nodes.datastream
 
 import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
 import org.apache.calcite.rex.{RexCall, RexNode}
 import org.apache.calcite.sql.SemiJoinType
@@ -28,6 +27,7 @@ import org.apache.flink.table.api.StreamTableEnvironment
 import org.apache.flink.table.functions.utils.TableSqlFunction
 import org.apache.flink.table.plan.nodes.CommonCorrelate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalTableFunctionScan
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.types.Row
 
 /**
@@ -36,28 +36,30 @@ import org.apache.flink.types.Row
 class DataStreamCorrelate(
     cluster: RelOptCluster,
     traitSet: RelTraitSet,
+    inputSchema: RowSchema,
     inputNode: RelNode,
     scan: FlinkLogicalTableFunctionScan,
     condition: Option[RexNode],
-    relRowType: RelDataType,
-    joinRowType: RelDataType,
+    schema: RowSchema,
+    joinSchema: RowSchema,
     joinType: SemiJoinType,
     ruleDescription: String)
   extends SingleRel(cluster, traitSet, inputNode)
   with CommonCorrelate
   with DataStreamRel {
 
-  override def deriveRowType() = relRowType
+  override def deriveRowType() = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamCorrelate(
       cluster,
       traitSet,
+      inputSchema,
       inputs.get(0),
       scan,
       condition,
-      relRowType,
-      joinRowType,
+      schema,
+      joinSchema,
       joinType,
       ruleDescription)
   }
@@ -74,7 +76,7 @@ class DataStreamCorrelate(
     super.explainTerms(pw)
       .item("invocation", scan.getCall)
       .item("function", sqlFunction.getTableFunction.getClass.getCanonicalName)
-      .item("rowType", relRowType)
+      .item("rowType", schema.logicalType)
       .item("joinType", joinType)
       .itemIf("condition", condition.orNull, condition.isDefined)
   }
@@ -94,16 +96,16 @@ class DataStreamCorrelate(
 
     val mapFunc = correlateMapFunction(
       config,
-      inputDS.getType,
+      inputSchema,
       udtfTypeInfo,
-      getRowType,
+      schema,
       joinType,
       rexCall,
       condition,
       Some(pojoFieldMapping),
       ruleDescription)
 
-    inputDS.flatMap(mapFunc).name(correlateOpName(rexCall, sqlFunction, relRowType))
+    inputDS.flatMap(mapFunc).name(correlateOpName(rexCall, sqlFunction, schema.logicalType))
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 2224752c0018b..8eb9d40846b61 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -20,36 +20,34 @@ package org.apache.flink.table.plan.nodes.datastream
 import java.util.{List => JList}
 
 import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
+import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
 import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.{AggregateCall, Window}
 import org.apache.calcite.rel.core.Window.Group
+import org.apache.calcite.rel.core.{AggregateCall, Window}
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
-import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.java.functions.NullByteKeySelector
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.{StreamTableEnvironment, TableException}
 import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.OverAggregate
+import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
 import org.apache.flink.table.runtime.aggregate._
 import org.apache.flink.types.Row
 
-import org.apache.flink.api.java.functions.NullByteKeySelector
-import org.apache.flink.table.codegen.CodeGenerator
-import org.apache.flink.table.functions.{ProcTimeType, RowTimeType}
-import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
-
 class DataStreamOverAggregate(
     logicWindow: Window,
     cluster: RelOptCluster,
     traitSet: RelTraitSet,
     inputNode: RelNode,
-    rowRelDataType: RelDataType,
-    inputType: RelDataType)
+    schema: RowSchema,
+    inputSchema: RowSchema)
   extends SingleRel(cluster, traitSet, inputNode)
   with OverAggregate
   with DataStreamRel {
 
-  override def deriveRowType(): RelDataType = rowRelDataType
+  override def deriveRowType(): RelDataType = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, inputs: JList[RelNode]): RelNode = {
     new DataStreamOverAggregate(
@@ -57,8 +55,8 @@ class DataStreamOverAggregate(
       cluster,
       traitSet,
       inputs.get(0),
-      getRowType,
-      inputType)
+      schema,
+      inputSchema)
   }
 
   override def toString: String = {
@@ -72,14 +70,16 @@ class DataStreamOverAggregate(
     val namedAggregates: Seq[CalcitePair[AggregateCall, String]] = generateNamedAggregates
 
     super.explainTerms(pw)
-      .itemIf("partitionBy", partitionToString(inputType, partitionKeys), partitionKeys.nonEmpty)
-      .item("orderBy",orderingToString(inputType, overWindow.orderKeys.getFieldCollations))
-      .itemIf("rows", windowRange(logicWindow, overWindow, getInput), overWindow.isRows)
-      .itemIf("range", windowRange(logicWindow, overWindow, getInput), !overWindow.isRows)
+      .itemIf("partitionBy",
+        partitionToString(schema.logicalType, partitionKeys), partitionKeys.nonEmpty)
+      .item("orderBy",
+        orderingToString(schema.logicalType, overWindow.orderKeys.getFieldCollations))
+      .itemIf("rows", windowRange(logicWindow, overWindow, inputNode), overWindow.isRows)
+      .itemIf("range", windowRange(logicWindow, overWindow, inputNode), !overWindow.isRows)
       .item(
         "select", aggregationToString(
-          inputType,
-          getRowType,
+          inputSchema.logicalType,
+          schema.logicalType,
           namedAggregates))
   }
 
@@ -111,13 +111,13 @@ class DataStreamOverAggregate(
       false,
       inputDS.getType)
 
-    val timeType = inputType
+    val timeType = schema.logicalType
       .getFieldList
       .get(orderKey.getFieldIndex)
-      .getValue
+      .getType
 
     timeType match {
-      case _: ProcTimeType =>
+      case _ if FlinkTypeFactory.isProctimeIndicatorType(timeType)  =>
         // proc-time OVER window
         if (overWindow.lowerBound.isUnbounded && overWindow.upperBound.isCurrentRow) {
           // unbounded OVER window
@@ -140,7 +140,8 @@ class DataStreamOverAggregate(
           throw new TableException(
             "OVER RANGE FOLLOWING windows are not supported yet.")
         }
-      case _: RowTimeType =>
+
+      case _ if FlinkTypeFactory.isRowtimeIndicatorType(timeType) =>
         // row-time OVER window
         if (overWindow.lowerBound.isPreceding &&
           overWindow.lowerBound.isUnbounded && overWindow.upperBound.isCurrentRow) {
@@ -158,17 +159,16 @@ class DataStreamOverAggregate(
             inputDS,
             isRowTimeType = true,
             isRowsClause = overWindow.isRows
-            )
+          )
         } else {
           throw new TableException(
             "OVER RANGE FOLLOWING windows are not supported yet.")
         }
+
       case _ =>
         throw new TableException(
-          "Unsupported time type {$timeType}. " +
-            "OVER windows do only support RowTimeType and ProcTimeType.")
+          s"OVER windows can only be applied on time attributes.")
     }
-
   }
 
   def createUnboundedAndCurrentRowOverWindow(
@@ -178,16 +178,20 @@ class DataStreamOverAggregate(
     isRowsClause: Boolean): DataStream[Row] = {
 
     val overWindow: Group = logicWindow.groups.get(0)
-    val partitionKeys: Array[Int] = overWindow.keys.toArray
-    val namedAggregates: Seq[CalcitePair[AggregateCall, String]] = generateNamedAggregates
-
-    // get the output types
-    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
+    val partitionKeys: Array[Int] = overWindow.keys.toArray.map(schema.mapIndex)
+    val namedAggregates: Seq[CalcitePair[AggregateCall, String]] = generateNamedAggregates.map {
+      namedAggregate =>
+        new CalcitePair[AggregateCall, String](
+          schema.mapAggregateCall(namedAggregate.left),
+          namedAggregate.right)
+    }
 
     val processFunction = AggregateUtil.createUnboundedOverProcessFunction(
       generator,
       namedAggregates,
-      inputType,
+      inputSchema.physicalType,
+      inputSchema.physicalTypeInfo,
+      inputSchema.physicalFieldTypeInfo,
       isRowTimeType,
       partitionKeys.nonEmpty,
       isRowsClause)
@@ -198,7 +202,7 @@ class DataStreamOverAggregate(
         inputDS
           .keyBy(partitionKeys: _*)
           .process(processFunction)
-          .returns(rowTypeInfo)
+          .returns(schema.physicalTypeInfo)
           .name(aggOpName)
           .asInstanceOf[DataStream[Row]]
       }
@@ -207,13 +211,13 @@ class DataStreamOverAggregate(
         if (isRowTimeType) {
           inputDS.keyBy(new NullByteKeySelector[Row])
             .process(processFunction).setParallelism(1).setMaxParallelism(1)
-            .returns(rowTypeInfo)
+            .returns(schema.physicalTypeInfo)
             .name(aggOpName)
             .asInstanceOf[DataStream[Row]]
         } else {
           inputDS
             .process(processFunction).setParallelism(1).setMaxParallelism(1)
-            .returns(rowTypeInfo)
+            .returns(schema.physicalTypeInfo)
             .name(aggOpName)
             .asInstanceOf[DataStream[Row]]
         }
@@ -228,19 +232,26 @@ class DataStreamOverAggregate(
     isRowsClause: Boolean): DataStream[Row] = {
 
     val overWindow: Group = logicWindow.groups.get(0)
-    val partitionKeys: Array[Int] = overWindow.keys.toArray
-    val namedAggregates: Seq[CalcitePair[AggregateCall, String]] = generateNamedAggregates
+    val partitionKeys: Array[Int] = overWindow.keys.toArray.map(schema.mapIndex)
+    val namedAggregates: Seq[CalcitePair[AggregateCall, String]] = generateNamedAggregates.map {
+      namedAggregate =>
+        new CalcitePair[AggregateCall, String](
+          schema.mapAggregateCall(namedAggregate.left),
+          namedAggregate.right)
+    }
 
     val precedingOffset =
-      getLowerBoundary(logicWindow, overWindow, getInput()) + (if (isRowsClause) 1 else 0)
-
-    // get the output types
-    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
+      getLowerBoundary(
+        logicWindow,
+        overWindow,
+        input) + (if (isRowsClause) 1 else 0)
 
     val processFunction = AggregateUtil.createBoundedOverProcessFunction(
       generator,
       namedAggregates,
-      inputType,
+      inputSchema.physicalType,
+      inputSchema.physicalTypeInfo,
+      inputSchema.physicalFieldTypeInfo,
       precedingOffset,
       isRowsClause,
       isRowTimeType
@@ -251,7 +262,7 @@ class DataStreamOverAggregate(
         inputDS
           .keyBy(partitionKeys: _*)
           .process(processFunction)
-          .returns(rowTypeInfo)
+          .returns(schema.physicalTypeInfo)
           .name(aggOpName)
           .asInstanceOf[DataStream[Row]]
       }
@@ -260,7 +271,7 @@ class DataStreamOverAggregate(
         inputDS
           .keyBy(new NullByteKeySelector[Row])
           .process(processFunction).setParallelism(1).setMaxParallelism(1)
-          .returns(rowTypeInfo)
+          .returns(schema.physicalTypeInfo)
           .name(aggOpName)
           .asInstanceOf[DataStream[Row]]
       }
@@ -282,17 +293,18 @@ class DataStreamOverAggregate(
 
     s"over: (${
       if (!partitionKeys.isEmpty) {
-        s"PARTITION BY: ${partitionToString(inputType, partitionKeys)}, "
+        s"PARTITION BY: ${partitionToString(inputSchema.logicalType, partitionKeys)}, "
       } else {
         ""
       }
-    }ORDER BY: ${orderingToString(inputType, overWindow.orderKeys.getFieldCollations)}, " +
+    }ORDER BY: ${orderingToString(inputSchema.logicalType,
+        overWindow.orderKeys.getFieldCollations)}, " +
       s"${if (overWindow.isRows) "ROWS" else "RANGE"}" +
-      s"${windowRange(logicWindow, overWindow, getInput)}, " +
+      s"${windowRange(logicWindow, overWindow, inputNode.asInstanceOf[DataStreamRel])}, " +
       s"select: (${
         aggregationToString(
-          inputType,
-          getRowType,
+          inputSchema.logicalType,
+          schema.logicalType,
           namedAggregates)
       }))"
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
index ae172a5e3bd33..03938f3b4d91b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
@@ -34,4 +34,3 @@ trait DataStreamRel extends FlinkRelNode {
   def translateToPlan(tableEnv: StreamTableEnvironment) : DataStream[Row]
 
 }
-
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
index c187ae88c17d4..05f60ba035632 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
@@ -24,7 +24,7 @@ import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.core.TableScan
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
-import org.apache.flink.table.plan.schema.DataStreamTable
+import org.apache.flink.table.plan.schema.{DataStreamTable, RowSchema}
 import org.apache.flink.types.Row
 
 /**
@@ -36,27 +36,27 @@ class DataStreamScan(
     cluster: RelOptCluster,
     traitSet: RelTraitSet,
     table: RelOptTable,
-    rowRelDataType: RelDataType)
+    schema: RowSchema)
   extends TableScan(cluster, traitSet, table)
   with StreamScan {
 
   val dataStreamTable: DataStreamTable[Any] = getTable.unwrap(classOf[DataStreamTable[Any]])
 
-  override def deriveRowType(): RelDataType = rowRelDataType
+  override def deriveRowType(): RelDataType = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamScan(
       cluster,
       traitSet,
       getTable,
-      getRowType
+      schema
     )
   }
 
   override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
     val config = tableEnv.getConfig
     val inputDataStream: DataStream[Any] = dataStreamTable.dataStream
-    convertToInternalRow(inputDataStream, dataStreamTable, config)
+    convertToInternalRow(schema, inputDataStream, dataStreamTable, config)
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
index f340ac75348b9..47b4946168024 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
@@ -19,14 +19,12 @@
 package org.apache.flink.table.plan.nodes.datastream
 
 import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.{BiRel, RelNode, RelWriter}
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.types.Row
 
-import scala.collection.JavaConverters._
-
 /**
   * Flink RelNode which matches along with Union.
   *
@@ -36,11 +34,11 @@ class DataStreamUnion(
     traitSet: RelTraitSet,
     leftNode: RelNode,
     rightNode: RelNode,
-    rowRelDataType: RelDataType)
+    schema: RowSchema)
   extends BiRel(cluster, traitSet, leftNode, rightNode)
   with DataStreamRel {
 
-  override def deriveRowType() = rowRelDataType
+  override def deriveRowType() = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamUnion(
@@ -48,7 +46,7 @@ class DataStreamUnion(
       traitSet,
       inputs.get(0),
       inputs.get(1),
-      getRowType
+      schema
     )
   }
 
@@ -57,7 +55,7 @@ class DataStreamUnion(
   }
 
   override def toString = {
-    s"Union All(union: (${getRowType.getFieldNames.asScala.toList.mkString(", ")}))"
+    s"Union All(union: (${schema.logicalFieldNames.mkString(", ")}))"
   }
 
   override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
@@ -68,6 +66,6 @@ class DataStreamUnion(
   }
 
   private def unionSelectionToString: String = {
-    getRowType.getFieldNames.asScala.toList.mkString(", ")
+    schema.logicalFieldNames.mkString(", ")
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
index 0ab4a489e2c6d..c964e03471d4e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
@@ -21,13 +21,12 @@ package org.apache.flink.table.plan.nodes.datastream
 import com.google.common.collect.ImmutableList
 import org.apache.calcite.plan._
 import org.apache.calcite.rel.RelNode
-import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.core.Values
 import org.apache.calcite.rex.RexLiteral
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
-import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.runtime.io.ValuesInputFormat
 import org.apache.flink.types.Row
 
@@ -39,19 +38,19 @@ import scala.collection.JavaConverters._
 class DataStreamValues(
     cluster: RelOptCluster,
     traitSet: RelTraitSet,
-    rowRelDataType: RelDataType,
+    schema: RowSchema,
     tuples: ImmutableList[ImmutableList[RexLiteral]],
     ruleDescription: String)
-  extends Values(cluster, rowRelDataType, tuples, traitSet)
+  extends Values(cluster, schema.logicalType, tuples, traitSet)
   with DataStreamRel {
 
-  override def deriveRowType() = rowRelDataType
+  override def deriveRowType() = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamValues(
       cluster,
       traitSet,
-      getRowType,
+      schema,
       getTuples,
       ruleDescription
     )
@@ -61,15 +60,13 @@ class DataStreamValues(
 
     val config = tableEnv.getConfig
 
-    val returnType = FlinkTypeFactory.toInternalRowTypeInfo(getRowType)
-
     val generator = new CodeGenerator(config)
 
     // generate code for every record
     val generatedRecords = getTuples.asScala.map { r =>
       generator.generateResultExpression(
-        returnType,
-        getRowType.getFieldNames.asScala,
+        schema.physicalTypeInfo,
+        schema.physicalFieldNames,
         r.asScala)
     }
 
@@ -77,14 +74,14 @@ class DataStreamValues(
     val generatedFunction = generator.generateValuesInputFormat(
       ruleDescription,
       generatedRecords.map(_.code),
-      returnType)
+      schema.physicalTypeInfo)
 
     val inputFormat = new ValuesInputFormat[Row](
       generatedFunction.name,
       generatedFunction.code,
       generatedFunction.returnType)
 
-    tableEnv.execEnv.createInput(inputFormat, returnType)
+    tableEnv.execEnv.createInput(inputFormat, schema.physicalTypeInfo)
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala
index 6d083025801d3..dd828196e7a32 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala
@@ -18,42 +18,46 @@
 
 package org.apache.flink.table.plan.nodes.datastream
 
+import org.apache.flink.api.common.functions.MapFunction
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.TableConfig
-import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.CommonScan
-import org.apache.flink.table.plan.schema.FlinkTable
+import org.apache.flink.table.plan.schema.{FlinkTable, RowSchema}
+import org.apache.flink.table.runtime.MapRunner
 import org.apache.flink.types.Row
 
-import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 
 trait StreamScan extends CommonScan with DataStreamRel {
 
   protected def convertToInternalRow(
+      schema: RowSchema,
       input: DataStream[Any],
       flinkTable: FlinkTable[_],
       config: TableConfig)
     : DataStream[Row] = {
 
-    val inputType = input.getType
-
-    val internalType = FlinkTypeFactory.toInternalRowTypeInfo(getRowType)
-
     // conversion
-    if (needsConversion(inputType, internalType)) {
+    if (needsConversion(input.getType, schema.physicalTypeInfo)) {
 
-      val mapFunc = getConversionMapper(
+      val function = generatedConversionFunction(
         config,
-        inputType,
-        internalType,
+        classOf[MapFunction[Any, Row]],
+        input.getType,
+        schema.physicalTypeInfo,
         "DataStreamSourceConversion",
-        getRowType.getFieldNames,
+        schema.physicalFieldNames,
         Some(flinkTable.fieldIndexes))
 
-      val opName = s"from: (${getRowType.getFieldNames.asScala.toList.mkString(", ")})"
+      val runner = new MapRunner[Any, Row](
+        function.name,
+        function.code,
+        function.returnType)
+
+      val opName = s"from: (${schema.logicalFieldNames.mkString(", ")})"
 
-      input.map(mapFunc).name(opName)
+      // TODO we need a ProcessFunction here
+      input.map(runner).name(opName)
     }
     // no conversion necessary, forward
     else {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
index 0a466a31b6b4c..5dc3da866d392 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
@@ -22,10 +22,11 @@ import org.apache.calcite.plan._
 import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamTableEnvironment, TableEnvironment}
+import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.PhysicalTableSourceScan
-import org.apache.flink.table.plan.schema.TableSourceTable
-import org.apache.flink.table.sources.{StreamTableSource, TableSource}
+import org.apache.flink.table.plan.schema.{RowSchema, TableSourceTable}
+import org.apache.flink.table.sources.{DefinedTimeAttributes, StreamTableSource, TableSource}
 import org.apache.flink.types.Row
 
 /** Flink RelNode to read data from an external source defined by a [[StreamTableSource]]. */
@@ -37,7 +38,50 @@ class StreamTableSourceScan(
   extends PhysicalTableSourceScan(cluster, traitSet, table, tableSource)
   with StreamScan {
 
-  override def computeSelfCost(planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = {
+  override def deriveRowType() = {
+    val flinkTypeFactory = cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+
+    def removeIndex[T](idx: Int, l: List[T]): List[T] = {
+      if (l.size < idx) {
+        l
+      } else {
+        l.take(idx) ++ l.drop(idx + 1)
+      }
+    }
+
+    var fieldNames = TableEnvironment.getFieldNames(tableSource).toList
+    var fieldTypes = TableEnvironment.getFieldTypes(tableSource.getReturnType).toList
+
+    val rowtime = tableSource match {
+      case timeSource: DefinedTimeAttributes if timeSource.getRowtimeAttribute != null =>
+        val rowtimeAttribute = timeSource.getRowtimeAttribute
+        // remove physical field if it is overwritten by time attribute
+        fieldNames = removeIndex(rowtimeAttribute.f0, fieldNames)
+        fieldTypes = removeIndex(rowtimeAttribute.f0, fieldTypes)
+        Some((rowtimeAttribute.f0, rowtimeAttribute.f1))
+      case _ =>
+        None
+    }
+
+    val proctime = tableSource match {
+      case timeSource: DefinedTimeAttributes if timeSource.getProctimeAttribute != null =>
+        val proctimeAttribute = timeSource.getProctimeAttribute
+        // remove physical field if it is overwritten by time attribute
+        fieldNames = removeIndex(proctimeAttribute.f0, fieldNames)
+        fieldTypes = removeIndex(proctimeAttribute.f0, fieldTypes)
+        Some((proctimeAttribute.f0, proctimeAttribute.f1))
+      case _ =>
+        None
+    }
+
+    flinkTypeFactory.buildLogicalRowType(
+      fieldNames,
+      fieldTypes,
+      rowtime,
+      proctime)
+  }
+
+  override def computeSelfCost (planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = {
     val rowCnt = metadata.getRowCount(this)
     planner.getCostFactory.makeCost(rowCnt, rowCnt, rowCnt * estimateRowSize(getRowType))
   }
@@ -67,6 +111,10 @@ class StreamTableSourceScan(
   override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
     val config = tableEnv.getConfig
     val inputDataStream = tableSource.getDataStream(tableEnv.execEnv).asInstanceOf[DataStream[Any]]
-    convertToInternalRow(inputDataStream, new TableSourceTable(tableSource), config)
+    convertToInternalRow(
+      new RowSchema(getRowType),
+      inputDataStream,
+      new TableSourceTable(tableSource),
+      config)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalOverWindow.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalOverWindow.scala
index b1f991e777998..11b227f537f74 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalOverWindow.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalOverWindow.scala
@@ -45,7 +45,7 @@ class FlinkLogicalOverWindow(
       traitSet,
       inputs.get(0),
       windowConstants,
-      rowType,
+      getRowType,
       windowGroups)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
index eacbafa8547b8..53e7b31446f27 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
@@ -47,9 +47,11 @@ class FlinkLogicalTableSourceScan(
 
   override def deriveRowType(): RelDataType = {
     val flinkTypeFactory = cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
-    flinkTypeFactory.buildRowDataType(
+    flinkTypeFactory.buildLogicalRowType(
       TableEnvironment.getFieldNames(tableSource),
-      TableEnvironment.getFieldTypes(tableSource.getReturnType))
+      TableEnvironment.getFieldTypes(tableSource.getReturnType),
+      None,
+      None)
   }
 
   override def computeSelfCost(planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala
index 4da2da995eb2b..7577debec342f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowStartEndPropertiesRule.scala
@@ -53,8 +53,8 @@ class WindowStartEndPropertiesRule
     transformed.push(LogicalWindowAggregate.create(
       agg.getWindow,
       Seq(
-        NamedWindowProperty("w$start", WindowStart(agg.getWindow.alias)),
-        NamedWindowProperty("w$end", WindowEnd(agg.getWindow.alias))
+        NamedWindowProperty("w$start", WindowStart(agg.getWindow.aliasAttribute)),
+        NamedWindowProperty("w$end", WindowEnd(agg.getWindow.aliasAttribute))
       ), agg)
     )
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamAggregateRule.scala
index f011b66942ccf..fc654036cefb2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamAggregateRule.scala
@@ -25,6 +25,7 @@ import org.apache.flink.table.api.TableException
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamAggregate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalWindowAggregate
+import org.apache.flink.table.plan.schema.RowSchema
 
 import scala.collection.JavaConversions._
 
@@ -65,8 +66,8 @@ class DataStreamAggregateRule
       traitSet,
       convInput,
       agg.getNamedAggCalls,
-      rel.getRowType,
-      agg.getInput.getRowType,
+      new RowSchema(rel.getRowType),
+      new RowSchema(agg.getInput.getRowType),
       agg.getGroupSet.toArray)
     }
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala
index 177726437f22a..0a1a31a7a5f3e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala
@@ -24,6 +24,7 @@ import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamCalc
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalCalc
+import org.apache.flink.table.plan.schema.RowSchema
 
 class DataStreamCalcRule
   extends ConverterRule(
@@ -42,7 +43,8 @@ class DataStreamCalcRule
       rel.getCluster,
       traitSet,
       convInput,
-      rel.getRowType,
+      new RowSchema(convInput.getRowType),
+      new RowSchema(rel.getRowType),
       calc.getProgram,
       description)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala
index ae39d4009f620..cd0663e6ca870 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala
@@ -25,6 +25,7 @@ import org.apache.calcite.rex.RexNode
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamCorrelate
 import org.apache.flink.table.plan.nodes.logical.{FlinkLogicalCalc, FlinkLogicalCorrelate, FlinkLogicalTableFunctionScan}
+import org.apache.flink.table.plan.schema.RowSchema
 
 class DataStreamCorrelateRule
   extends ConverterRule(
@@ -68,11 +69,12 @@ class DataStreamCorrelateRule
           new DataStreamCorrelate(
             rel.getCluster,
             traitSet,
+            new RowSchema(convInput.getRowType),
             convInput,
             scan,
             condition,
-            rel.getRowType,
-            join.getRowType,
+            new RowSchema(rel.getRowType),
+            new RowSchema(join.getRowType),
             join.getJoinType,
             description)
       }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
index 175a202ff8125..28efcf53152c0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
@@ -18,15 +18,15 @@
 
 package org.apache.flink.table.plan.rules.datastream
 
-import java.math.BigDecimal
+import java.math.{BigDecimal => JBigDecimal}
 
 import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rex.{RexBuilder, RexCall, RexLiteral, RexNode}
+import org.apache.calcite.rex._
 import org.apache.calcite.sql.fun.SqlStdOperatorTable
 import org.apache.flink.table.api.{TableException, Window}
 import org.apache.flink.table.api.scala.{Session, Slide, Tumble}
-import org.apache.flink.table.expressions.Literal
-import org.apache.flink.table.functions.TimeModeTypes
+import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.expressions.{Literal, UnresolvedFieldReference}
 import org.apache.flink.table.plan.rules.common.LogicalWindowAggregateRule
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
 
@@ -49,16 +49,12 @@ class DataStreamLogicalWindowAggregateRule
 
     val timeType = windowExpression.operands.get(0).getType
     timeType match {
-      case TimeModeTypes.ROWTIME =>
-        rexBuilder.makeAbstractCast(
-          TimeModeTypes.ROWTIME,
-          rexBuilder.makeLiteral(0L, TimeModeTypes.ROWTIME, true))
-      case TimeModeTypes.PROCTIME =>
-        rexBuilder.makeAbstractCast(
-          TimeModeTypes.PROCTIME,
-          rexBuilder.makeLiteral(0L, TimeModeTypes.PROCTIME, true))
+
+      case _ if FlinkTypeFactory.isTimeIndicatorType(timeType) =>
+        rexBuilder.makeLiteral(0L, timeType, true)
+
       case _ =>
-        throw TableException(s"""Unexpected time type $timeType encountered""")
+        throw TableException(s"""Time attribute expected but $timeType encountered.""")
     }
   }
 
@@ -68,41 +64,41 @@ class DataStreamLogicalWindowAggregateRule
 
     def getOperandAsLong(call: RexCall, idx: Int): Long =
       call.getOperands.get(idx) match {
-        case v : RexLiteral => v.getValue.asInstanceOf[BigDecimal].longValue()
-        case _ => throw new TableException("Only constant window descriptors are supported")
+        case v: RexLiteral => v.getValue.asInstanceOf[JBigDecimal].longValue()
+        case _ => throw new TableException("Only constant window descriptors are supported.")
+      }
+
+    def getOperandAsTimeIndicator(call: RexCall, idx: Int): String =
+      call.getOperands.get(idx) match {
+        case v: RexInputRef if FlinkTypeFactory.isTimeIndicatorType(v.getType) =>
+          rowType.getFieldList.get(v.getIndex).getName
+        case _ =>
+          throw new TableException("Window can only be defined over a time attribute column.")
       }
 
     windowExpr.getOperator match {
       case SqlStdOperatorTable.TUMBLE =>
+        val time = getOperandAsTimeIndicator(windowExpr, 0)
         val interval = getOperandAsLong(windowExpr, 1)
         val w = Tumble.over(Literal(interval, TimeIntervalTypeInfo.INTERVAL_MILLIS))
 
-        val window = windowExpr.getType match {
-          case TimeModeTypes.PROCTIME => w
-          case TimeModeTypes.ROWTIME => w.on("rowtime")
-        }
-        window.as("w$")
+        w.on(UnresolvedFieldReference(time)).as("w$")
 
       case SqlStdOperatorTable.HOP =>
+        val time = getOperandAsTimeIndicator(windowExpr, 0)
         val (slide, size) = (getOperandAsLong(windowExpr, 1), getOperandAsLong(windowExpr, 2))
         val w = Slide
           .over(Literal(size, TimeIntervalTypeInfo.INTERVAL_MILLIS))
           .every(Literal(slide, TimeIntervalTypeInfo.INTERVAL_MILLIS))
 
-        val window = windowExpr.getType match {
-          case TimeModeTypes.PROCTIME => w
-          case TimeModeTypes.ROWTIME => w.on("rowtime")
-        }
-        window.as("w$")
+        w.on(UnresolvedFieldReference(time)).as("w$")
+
       case SqlStdOperatorTable.SESSION =>
+        val time = getOperandAsTimeIndicator(windowExpr, 0)
         val gap = getOperandAsLong(windowExpr, 1)
         val w = Session.withGap(Literal(gap, TimeIntervalTypeInfo.INTERVAL_MILLIS))
 
-        val window = windowExpr.getType match {
-          case TimeModeTypes.PROCTIME => w
-          case TimeModeTypes.ROWTIME => w.on("rowtime")
-        }
-        window.as("w$")
+        w.on(UnresolvedFieldReference(time)).as("w$")
     }
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamOverAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamOverAggregateRule.scala
index 8e9697050619f..b3d7603eaa2f6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamOverAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamOverAggregateRule.scala
@@ -25,6 +25,7 @@ import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamOverAggregate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalOverWindow
+import org.apache.flink.table.plan.schema.RowSchema
 
 class DataStreamOverAggregateRule
   extends ConverterRule(
@@ -46,8 +47,8 @@ class DataStreamOverAggregateRule
       rel.getCluster,
       traitSet,
       convertInput,
-      rel.getRowType,
-      inputRowType)
+      new RowSchema(rel.getRowType),
+      new RowSchema(inputRowType))
   }
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamScanRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamScanRule.scala
index 5bf60a7adc7e3..d8dda80bf9853 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamScanRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamScanRule.scala
@@ -23,7 +23,7 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamScan
-import org.apache.flink.table.plan.schema.DataStreamTable
+import org.apache.flink.table.plan.schema.{DataStreamTable, RowSchema}
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalNativeTableScan
 
 class DataStreamScanRule
@@ -53,7 +53,7 @@ class DataStreamScanRule
       rel.getCluster,
       traitSet,
       scan.getTable,
-      rel.getRowType
+      new RowSchema(rel.getRowType)
     )
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamUnionRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamUnionRule.scala
index 4241f53b973a4..8402f6df0351a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamUnionRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamUnionRule.scala
@@ -24,6 +24,7 @@ import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamUnion
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalUnion
+import org.apache.flink.table.plan.schema.RowSchema
 
 class DataStreamUnionRule
   extends ConverterRule(
@@ -44,7 +45,7 @@ class DataStreamUnionRule
       traitSet,
       convLeft,
       convRight,
-      rel.getRowType)
+      new RowSchema(rel.getRowType))
   }
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala
index fbad21f9049bb..a1453a8d1b52a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala
@@ -24,6 +24,7 @@ import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamValues
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalValues
+import org.apache.flink.table.plan.schema.RowSchema
 
 class DataStreamValuesRule
   extends ConverterRule(
@@ -40,7 +41,7 @@ class DataStreamValuesRule
     new DataStreamValues(
       rel.getCluster,
       traitSet,
-      rel.getRowType,
+      new RowSchema(rel.getRowType),
       values.getTuples,
       description)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/DataStreamTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/DataStreamTable.scala
index 6ce65700eb137..70054b4a0d2c9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/DataStreamTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/DataStreamTable.scala
@@ -18,13 +18,27 @@
 
 package org.apache.flink.table.plan.schema
 
+import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFactory}
 import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.stats.FlinkStatistic
 
 class DataStreamTable[T](
     val dataStream: DataStream[T],
     override val fieldIndexes: Array[Int],
     override val fieldNames: Array[String],
+    val rowtime: Option[(Int, String)],
+    val proctime: Option[(Int, String)],
     override val statistic: FlinkStatistic = FlinkStatistic.UNKNOWN)
   extends FlinkTable[T](dataStream.getType, fieldIndexes, fieldNames, statistic) {
+
+  override def getRowType(typeFactory: RelDataTypeFactory): RelDataType = {
+    val flinkTypeFactory = typeFactory.asInstanceOf[FlinkTypeFactory]
+
+    flinkTypeFactory.buildLogicalRowType(
+      fieldNames,
+      fieldTypes,
+      rowtime,
+      proctime)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/FlinkTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/FlinkTable.scala
index ea770614cb61c..752b00e9747dd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/FlinkTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/FlinkTable.scala
@@ -48,10 +48,11 @@ abstract class FlinkTable[T](
   val fieldTypes: Array[TypeInformation[_]] =
     typeInfo match {
       case cType: CompositeType[_] =>
-        if (fieldNames.length != cType.getArity) {
+        // it is ok to leave out fields
+        if (fieldNames.length > cType.getArity) {
           throw new TableException(
           s"Arity of type (" + cType.getFieldNames.deep + ") " +
-            "not equal to number of field names " + fieldNames.deep + ".")
+            "must not be greater than number of field names " + fieldNames.deep + ".")
         }
         fieldIndexes.map(cType.getTypeAt(_).asInstanceOf[TypeInformation[_]])
       case aType: AtomicType[_] =>
@@ -64,7 +65,7 @@ abstract class FlinkTable[T](
 
   override def getRowType(typeFactory: RelDataTypeFactory): RelDataType = {
     val flinkTypeFactory = typeFactory.asInstanceOf[FlinkTypeFactory]
-    flinkTypeFactory.buildRowDataType(fieldNames, fieldTypes)
+    flinkTypeFactory.buildLogicalRowType(fieldNames, fieldTypes, None, None)
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala
new file mode 100644
index 0000000000000..b42be827da99b
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.schema
+
+import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeField, RelRecordType}
+import org.apache.calcite.rel.core.AggregateCall
+import org.apache.calcite.rex.{RexInputRef, RexNode, RexShuttle}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.table.api.TableException
+import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.types.Row
+
+import scala.collection.JavaConversions._
+
+/**
+  * Schema that describes both a logical and physical row.
+  */
+class RowSchema(private val logicalRowType: RelDataType) {
+
+  private lazy val physicalRowFields: Seq[RelDataTypeField] = logicalRowType.getFieldList filter {
+    field => !FlinkTypeFactory.isTimeIndicatorType(field.getType)
+  }
+
+  private lazy val physicalRowType: RelDataType = new RelRecordType(physicalRowFields)
+
+  private lazy val physicalRowFieldTypes: Seq[TypeInformation[_]] = physicalRowFields map { f =>
+    FlinkTypeFactory.toTypeInfo(f.getType)
+  }
+
+  private lazy val physicalRowFieldNames: Seq[String] = physicalRowFields.map(_.getName)
+
+  private lazy val physicalRowTypeInfo: TypeInformation[Row] = new RowTypeInfo(
+    physicalRowFieldTypes.toArray, physicalRowFieldNames.toArray)
+
+  private lazy val indexMapping: Array[Int] = generateIndexMapping
+
+  private lazy val inputRefUpdater = new RexInputRefUpdater()
+
+  private def generateIndexMapping: Array[Int] = {
+    val mapping = new Array[Int](logicalRowType.getFieldCount)
+    var countTimeIndicators = 0
+    var i = 0
+    while (i < logicalRowType.getFieldCount) {
+      val t = logicalRowType.getFieldList.get(i).getType
+      if (FlinkTypeFactory.isTimeIndicatorType(t)) {
+        countTimeIndicators += 1
+        // no mapping
+        mapping(i) = -1
+      } else {
+        mapping(i) = i - countTimeIndicators
+      }
+      i += 1
+    }
+    mapping
+  }
+
+  private class RexInputRefUpdater extends RexShuttle {
+
+    override def visitInputRef(inputRef: RexInputRef): RexNode = {
+      new RexInputRef(mapIndex(inputRef.getIndex), inputRef.getType)
+    }
+  }
+
+  /**
+    * Returns the arity of the logical record.
+    */
+  def logicalArity: Int = logicalRowType.getFieldCount
+
+  /**
+    * Returns the arity of the physical record.
+    */
+  def physicalArity: Int = physicalTypeInfo.getArity
+
+  /**
+    * Returns a logical [[RelDataType]] including logical fields (i.e. time indicators).
+    */
+  def logicalType: RelDataType = logicalRowType
+
+  /**
+    * Returns a physical [[RelDataType]] with no logical fields (i.e. time indicators).
+    */
+  def physicalType: RelDataType = physicalRowType
+
+  /**
+    * Returns a physical [[TypeInformation]] of row with no logical fields (i.e. time indicators).
+    */
+  def physicalTypeInfo: TypeInformation[Row] = physicalRowTypeInfo
+
+  /**
+    * Returns [[TypeInformation]] of the row's fields with no logical fields (i.e. time indicators).
+    */
+  def physicalFieldTypeInfo: Seq[TypeInformation[_]] = physicalRowFieldTypes
+
+  /**
+    * Returns the logical fields names including logical fields (i.e. time indicators).
+    */
+  def logicalFieldNames: Seq[String] = logicalRowType.getFieldNames
+
+  /**
+    * Returns the physical fields names with no logical fields (i.e. time indicators).
+    */
+  def physicalFieldNames: Seq[String] = physicalRowFieldNames
+
+  /**
+    * Converts logical indices to physical indices based on this schema.
+    */
+  def mapIndex(logicalIndex: Int): Int = {
+    val mappedIndex = indexMapping(logicalIndex)
+    if (mappedIndex < 0) {
+      throw new TableException("Invalid access to a logical field.")
+    } else {
+      mappedIndex
+    }
+  }
+
+  /**
+    * Converts logical indices of a aggregate call to physical ones.
+    */
+  def mapAggregateCall(logicalAggCall: AggregateCall): AggregateCall = {
+    logicalAggCall.copy(
+      logicalAggCall.getArgList.map(mapIndex(_).asInstanceOf[Integer]),
+      if (logicalAggCall.filterArg < 0) {
+        logicalAggCall.filterArg
+      } else {
+        mapIndex(logicalAggCall.filterArg)
+      }
+    )
+  }
+
+  /**
+    * Converts logical field references of a [[RexNode]] to physical ones.
+    */
+  def mapRexNode(logicalRexNode: RexNode): RexNode = logicalRexNode.accept(inputRefUpdater)
+
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/TimeIndicatorRelDataType.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/TimeIndicatorRelDataType.scala
new file mode 100644
index 0000000000000..5e27061166b4a
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/TimeIndicatorRelDataType.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.schema
+
+import org.apache.calcite.rel.`type`.RelDataTypeSystem
+import org.apache.calcite.sql.`type`.BasicSqlType
+
+/**
+  * Creates a time indicator type for event-time or processing-time, but with similar properties
+  * as a basic SQL type.
+  */
+class TimeIndicatorRelDataType(
+    typeSystem: RelDataTypeSystem,
+    originalType: BasicSqlType,
+    val isEventTime: Boolean)
+  extends BasicSqlType(
+    typeSystem,
+    originalType.getSqlTypeName,
+    originalType.getPrecision) {
+
+  override def equals(other: Any): Boolean = other match {
+    case that: TimeIndicatorRelDataType =>
+      super.equals(that) &&
+        isEventTime == that.isEventTime
+    case that: BasicSqlType =>
+      super.equals(that)
+    case _ => false
+  }
+
+  override def hashCode(): Int = {
+    super.hashCode() + 42 // we change the hash code to differentiate from regular timestamps
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
index 51e2fc59a4540..32562c7e3ec42 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
@@ -35,7 +35,7 @@ class MapRunner[IN, OUT](
 
   val LOG = LoggerFactory.getLogger(this.getClass)
 
-  private var function: MapFunction[IN, OUT] = null
+  private var function: MapFunction[IN, OUT] = _
 
   override def open(parameters: Configuration): Unit = {
     LOG.debug(s"Compiling MapFunction: $name \n\n Code:\n$code")
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index e38207df87fb1..07992cd032dab 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -26,17 +26,18 @@ import org.apache.calcite.sql.`type`.SqlTypeName
 import org.apache.calcite.sql.`type`.SqlTypeName._
 import org.apache.calcite.sql.fun._
 import org.apache.calcite.sql.{SqlAggFunction, SqlKind}
-import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.api.common.functions.{MapFunction, RichGroupReduceFunction, AggregateFunction => DataStreamAggFunction, _}
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.tuple.Tuple
 import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.api.common.functions.{AggregateFunction => DataStreamAggFunction, _}
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction, WindowFunction}
 import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
 import org.apache.flink.table.api.TableException
 import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
+import org.apache.flink.table.expressions.ExpressionUtils.isTimeIntervalLiteral
 import org.apache.flink.table.expressions._
 import org.apache.flink.table.functions.aggfunctions._
 import org.apache.flink.table.functions.utils.AggSqlFunction
@@ -61,26 +62,31 @@ object AggregateUtil {
     * window to evaluate final aggregate value.
     *
     * @param generator       code generator instance
-    * @param namedAggregates List of calls to aggregate functions and their output field names
-    * @param inputType Input row type
+    * @param namedAggregates Physical calls to aggregate functions and their output field names
+    * @param inputType Physical type of the row.
+    * @param inputTypeInfo Physical type information of the row.
+    * @param inputFieldTypeInfo Physical type information of the row's fields.
     * @param isRowTimeType It is a tag that indicates whether the time type is rowTimeType
     * @param isPartitioned It is a tag that indicate whether the input is partitioned
     * @param isRowsClause It is a tag that indicates whether the OVER clause is ROWS clause
     */
   private[flink] def createUnboundedOverProcessFunction(
-    generator: CodeGenerator,
-    namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-    inputType: RelDataType,
-    isRowTimeType: Boolean,
-    isPartitioned: Boolean,
-    isRowsClause: Boolean): ProcessFunction[Row, Row] = {
+      generator: CodeGenerator,
+      namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+      inputType: RelDataType,
+      inputTypeInfo: TypeInformation[Row],
+      inputFieldTypeInfo: Seq[TypeInformation[_]],
+      isRowTimeType: Boolean,
+      isPartitioned: Boolean,
+      isRowsClause: Boolean)
+    : ProcessFunction[Row, Row] = {
 
     val needRetract = false
     val (aggFields, aggregates) =
       transformToAggregateFunctions(
         namedAggregates.map(_.getKey),
         inputType,
-        needRetract)
+        needRetraction = false)
 
     val aggregationStateType: RowTypeInfo =
       createDataSetAggregateBufferDataType(Array(), aggregates, inputType)
@@ -92,7 +98,7 @@ object AggregateUtil {
     val genFunction = generator.generateAggregations(
       "UnboundedProcessingOverAggregateHelper",
       generator,
-      inputType,
+      inputFieldTypeInfo,
       aggregates,
       aggFields,
       aggMapping,
@@ -112,13 +118,13 @@ object AggregateUtil {
         new RowTimeUnboundedRowsOver(
           genFunction,
           aggregationStateType,
-          FlinkTypeFactory.toInternalRowTypeInfo(inputType))
+          inputTypeInfo)
       } else {
         // RANGE unbounded over process function
         new RowTimeUnboundedRangeOver(
           genFunction,
           aggregationStateType,
-          FlinkTypeFactory.toInternalRowTypeInfo(inputType))
+          inputTypeInfo)
       }
     } else {
       if (isPartitioned) {
@@ -138,20 +144,25 @@ object AggregateUtil {
     * bounded OVER window to evaluate final aggregate value.
     *
     * @param generator       code generator instance
-    * @param namedAggregates List of calls to aggregate functions and their output field names
-    * @param inputType       Input row type
+    * @param namedAggregates Physical calls to aggregate functions and their output field names
+    * @param inputType Physical type of the row.
+    * @param inputTypeInfo Physical type information of the row.
+    * @param inputFieldTypeInfo Physical type information of the row's fields.
     * @param precedingOffset the preceding offset
     * @param isRowsClause   It is a tag that indicates whether the OVER clause is ROWS clause
     * @param isRowTimeType   It is a tag that indicates whether the time type is rowTimeType
     * @return [[org.apache.flink.streaming.api.functions.ProcessFunction]]
     */
   private[flink] def createBoundedOverProcessFunction(
-    generator: CodeGenerator,
-    namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-    inputType: RelDataType,
-    precedingOffset: Long,
-    isRowsClause: Boolean,
-    isRowTimeType: Boolean): ProcessFunction[Row, Row] = {
+      generator: CodeGenerator,
+      namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+      inputType: RelDataType,
+      inputTypeInfo: TypeInformation[Row],
+      inputFieldTypeInfo: Seq[TypeInformation[_]],
+      precedingOffset: Long,
+      isRowsClause: Boolean,
+      isRowTimeType: Boolean)
+    : ProcessFunction[Row, Row] = {
 
     val needRetract = true
     val (aggFields, aggregates) =
@@ -161,7 +172,6 @@ object AggregateUtil {
         needRetract)
 
     val aggregationStateType: RowTypeInfo = createAccumulatorRowType(aggregates)
-    val inputRowType = FlinkTypeFactory.toInternalRowTypeInfo(inputType).asInstanceOf[RowTypeInfo]
 
     val forwardMapping = (0 until inputType.getFieldCount).toArray
     val aggMapping = aggregates.indices.map(x => x + inputType.getFieldCount).toArray
@@ -170,7 +180,7 @@ object AggregateUtil {
     val genFunction = generator.generateAggregations(
       "BoundedOverAggregateHelper",
       generator,
-      inputType,
+      inputFieldTypeInfo,
       aggregates,
       aggFields,
       aggMapping,
@@ -189,14 +199,14 @@ object AggregateUtil {
         new RowTimeBoundedRowsOver(
           genFunction,
           aggregationStateType,
-          inputRowType,
+          inputTypeInfo,
           precedingOffset
         )
       } else {
         new RowTimeBoundedRangeOver(
           genFunction,
           aggregationStateType,
-          inputRowType,
+          inputTypeInfo,
           precedingOffset
         )
       }
@@ -206,13 +216,13 @@ object AggregateUtil {
           genFunction,
           precedingOffset,
           aggregationStateType,
-          inputRowType)
+          inputTypeInfo)
       } else {
         new ProcTimeBoundedRangeOver(
           genFunction,
           precedingOffset,
           aggregationStateType,
-          inputRowType)
+          inputTypeInfo)
       }
     }
   }
@@ -241,12 +251,13 @@ object AggregateUtil {
     * NOTE: this function is only used for time based window on batch tables.
     */
   def createDataSetWindowPrepareMapFunction(
-      generator: CodeGenerator,
-      window: LogicalWindow,
-      namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-      groupings: Array[Int],
-      inputType: RelDataType,
-      isParserCaseSensitive: Boolean)
+    generator: CodeGenerator,
+    window: LogicalWindow,
+    namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+    groupings: Array[Int],
+    inputType: RelDataType,
+    inputFieldTypeInfo: Seq[TypeInformation[_]],
+    isParserCaseSensitive: Boolean)
   : MapFunction[Row, Row] = {
 
     val needRetract = false
@@ -263,28 +274,28 @@ object AggregateUtil {
         Some(Array(BasicTypeInfo.LONG_TYPE_INFO)))
 
     val (timeFieldPos, tumbleTimeWindowSize) = window match {
-      case EventTimeTumblingGroupWindow(_, time, size) if isTimeInterval(size.resultType) =>
-        val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
-        (timeFieldPos, Some(asLong(size)))
 
-      case EventTimeTumblingGroupWindow(_, time, _) =>
+      case TumblingGroupWindow(_, time, size) =>
         val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
-        (timeFieldPos, None)
+        size match {
+          case Literal(value: Long, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
+            (timeFieldPos, Some(value))
+          case _ => (timeFieldPos, None)
+        }
 
-      case EventTimeSessionGroupWindow(_, time, _) =>
-        val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
-        (timeFieldPos, None)
+      case SessionGroupWindow(_, time, _) =>
+        (getTimeFieldPosition(time, inputType, isParserCaseSensitive), None)
 
-      case EventTimeSlidingGroupWindow(_, time, size, slide)
-          if isTimeInterval(time.resultType) && doAllSupportPartialMerge(aggregates) =>
-        // pre-tumble incremental aggregates on time-windows
+      case SlidingGroupWindow(_, time, size, slide) =>
         val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
-        val preTumblingSize = determineLargestTumblingSize(asLong(size), asLong(slide))
-        (timeFieldPos, Some(preTumblingSize))
-
-      case EventTimeSlidingGroupWindow(_, time, _, _) =>
-        val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
-        (timeFieldPos, None)
+        size match {
+          case Literal(value: Long, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
+            // pre-tumble incremental aggregates on time-windows
+            val timeFieldPos = getTimeFieldPosition(time, inputType, isParserCaseSensitive)
+            val preTumblingSize = determineLargestTumblingSize(asLong(size), asLong(slide))
+            (timeFieldPos, Some(preTumblingSize))
+          case _ => (timeFieldPos, None)
+        }
 
       case _ =>
         throw new UnsupportedOperationException(s"$window is currently not supported on batch")
@@ -296,7 +307,7 @@ object AggregateUtil {
     val genFunction = generator.generateAggregations(
       "DataSetAggregatePrepareMapHelper",
       generator,
-      inputType,
+      inputFieldTypeInfo,
       aggregates,
       aggFieldIndexes,
       aggMapping,
@@ -349,31 +360,32 @@ object AggregateUtil {
       window: LogicalWindow,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       groupings: Array[Int],
-      inputType: RelDataType,
+      physicalInputRowType: RelDataType,
+      physicalInputTypes: Seq[TypeInformation[_]],
       isParserCaseSensitive: Boolean)
     : RichGroupReduceFunction[Row, Row] = {
 
     val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
-      inputType,
+      physicalInputRowType,
       needRetract)
 
     val returnType: RowTypeInfo = createDataSetAggregateBufferDataType(
       groupings,
       aggregates,
-      inputType,
+      physicalInputRowType,
       Some(Array(BasicTypeInfo.LONG_TYPE_INFO)))
 
     val keysAndAggregatesArity = groupings.length + namedAggregates.length
 
     window match {
-      case EventTimeSlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
+      case SlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
         // sliding time-window for partial aggregations
         val genFunction = generator.generateAggregations(
           "DataSetAggregatePrepareMapHelper",
           generator,
-          inputType,
+          physicalInputTypes,
           aggregates,
           aggFieldIndexes,
           aggregates.indices.map(_ + groupings.length).toArray,
@@ -433,7 +445,7 @@ object AggregateUtil {
     : FlatMapFunction[Row, Row] = {
 
     window match {
-      case EventTimeSlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
+      case SlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
         new DataSetSlideTimeWindowAggFlatMapFunction(
           inputType.getArity - 1,
           asLong(size),
@@ -458,7 +470,8 @@ object AggregateUtil {
       generator: CodeGenerator,
       window: LogicalWindow,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-      inputType: RelDataType,
+      physicalInputRowType: RelDataType,
+      physicalInputTypes: Seq[TypeInformation[_]],
       outputType: RelDataType,
       groupings: Array[Int],
       properties: Seq[NamedWindowProperty],
@@ -468,7 +481,7 @@ object AggregateUtil {
     val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
-      inputType,
+      physicalInputRowType,
       needRetract)
 
     val aggMapping = aggregates.indices.toArray.map(_ + groupings.length)
@@ -476,7 +489,7 @@ object AggregateUtil {
     val genPreAggFunction = generator.generateAggregations(
       "GroupingWindowAggregateHelper",
       generator,
-      inputType,
+      physicalInputTypes,
       aggregates,
       aggFieldIndexes,
       aggMapping,
@@ -493,7 +506,7 @@ object AggregateUtil {
     val genFinalAggFunction = generator.generateAggregations(
       "GroupingWindowAggregateHelper",
       generator,
-      inputType,
+      physicalInputTypes,
       aggregates,
       aggFieldIndexes,
       aggMapping,
@@ -510,7 +523,7 @@ object AggregateUtil {
     val keysAndAggregatesArity = groupings.length + namedAggregates.length
 
     window match {
-      case EventTimeTumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
+      case TumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
         // tumbling time window
         val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
         if (doAllSupportPartialMerge(aggregates)) {
@@ -532,13 +545,13 @@ object AggregateUtil {
             endPos,
             outputType.getFieldCount)
         }
-      case EventTimeTumblingGroupWindow(_, _, size) =>
+      case TumblingGroupWindow(_, _, size) =>
         // tumbling count window
         new DataSetTumbleCountWindowAggReduceGroupFunction(
           genFinalAggFunction,
           asLong(size))
 
-      case EventTimeSessionGroupWindow(_, _, gap) =>
+      case SessionGroupWindow(_, _, gap) =>
         val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
         new DataSetSessionWindowAggReduceGroupFunction(
           genFinalAggFunction,
@@ -548,7 +561,7 @@ object AggregateUtil {
           asLong(gap),
           isInputCombined)
 
-      case EventTimeSlidingGroupWindow(_, _, size, _) if isTimeInterval(size.resultType) =>
+      case SlidingGroupWindow(_, _, size, _) if isTimeInterval(size.resultType) =>
         val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
         if (doAllSupportPartialMerge(aggregates)) {
           // for partial aggregations
@@ -570,7 +583,7 @@ object AggregateUtil {
             asLong(size))
         }
 
-      case EventTimeSlidingGroupWindow(_, _, size, _) =>
+      case SlidingGroupWindow(_, _, size, _) =>
         new DataSetSlideWindowAggReduceGroupFunction(
             genFinalAggFunction,
             keysAndAggregatesArity,
@@ -608,13 +621,14 @@ object AggregateUtil {
     generator: CodeGenerator,
     window: LogicalWindow,
     namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-    inputType: RelDataType,
+    physicalInputRowType: RelDataType,
+    physicalInputTypes: Seq[TypeInformation[_]],
     groupings: Array[Int]): MapPartitionFunction[Row, Row] = {
 
     val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
-      inputType,
+      physicalInputRowType,
       needRetract)
 
     val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
@@ -622,18 +636,18 @@ object AggregateUtil {
     val keysAndAggregatesArity = groupings.length + namedAggregates.length
 
     window match {
-      case EventTimeSessionGroupWindow(_, _, gap) =>
+      case SessionGroupWindow(_, _, gap) =>
         val combineReturnType: RowTypeInfo =
           createDataSetAggregateBufferDataType(
             groupings,
             aggregates,
-            inputType,
+            physicalInputRowType,
             Option(Array(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO)))
 
         val genFunction = generator.generateAggregations(
           "GroupingWindowAggregateHelper",
           generator,
-          inputType,
+          physicalInputTypes,
           aggregates,
           aggFieldIndexes,
           aggMapping,
@@ -679,14 +693,15 @@ object AggregateUtil {
       generator: CodeGenerator,
       window: LogicalWindow,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
-      inputType: RelDataType,
+      physicalInputRowType: RelDataType,
+      physicalInputTypes: Seq[TypeInformation[_]],
       groupings: Array[Int])
     : GroupCombineFunction[Row, Row] = {
 
     val needRetract = false
     val (aggFieldIndexes, aggregates) = transformToAggregateFunctions(
       namedAggregates.map(_.getKey),
-      inputType,
+      physicalInputRowType,
       needRetract)
 
     val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
@@ -695,18 +710,18 @@ object AggregateUtil {
 
     window match {
 
-      case EventTimeSessionGroupWindow(_, _, gap) =>
+      case SessionGroupWindow(_, _, gap) =>
         val combineReturnType: RowTypeInfo =
           createDataSetAggregateBufferDataType(
             groupings,
             aggregates,
-            inputType,
+            physicalInputRowType,
             Option(Array(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO)))
 
         val genFunction = generator.generateAggregations(
           "GroupingWindowAggregateHelper",
           generator,
-          inputType,
+          physicalInputTypes,
           aggregates,
           aggFieldIndexes,
           aggMapping,
@@ -742,6 +757,7 @@ object AggregateUtil {
       generator: CodeGenerator,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       inputType: RelDataType,
+      inputFieldTypeInfo: Seq[TypeInformation[_]],
       outputType: RelDataType,
       groupings: Array[Int],
       inGroupingSet: Boolean): (Option[DataSetPreAggFunction],
@@ -786,7 +802,7 @@ object AggregateUtil {
       val genPreAggFunction = generator.generateAggregations(
         "DataSetAggregatePrepareMapHelper",
         generator,
-        inputType,
+        inputFieldTypeInfo,
         aggregates,
         aggInFields,
         aggregates.indices.map(_ + groupings.length).toArray,
@@ -813,7 +829,7 @@ object AggregateUtil {
       val genFinalAggFunction = generator.generateAggregations(
         "DataSetAggregateFinalHelper",
         generator,
-        inputType,
+        inputFieldTypeInfo,
         aggregates,
         aggInFields,
         aggOutFields,
@@ -837,7 +853,7 @@ object AggregateUtil {
       val genFunction = generator.generateAggregations(
         "DataSetAggregateHelper",
         generator,
-        inputType,
+        inputFieldTypeInfo,
         aggregates,
         aggInFields,
         aggOutFields,
@@ -914,6 +930,7 @@ object AggregateUtil {
       generator: CodeGenerator,
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       inputType: RelDataType,
+      inputFieldTypeInfo: Seq[TypeInformation[_]],
       outputType: RelDataType,
       needMerge: Boolean)
     : (DataStreamAggFunction[Row, Row, Row], RowTypeInfo, RowTypeInfo) = {
@@ -931,7 +948,7 @@ object AggregateUtil {
     val genFunction = generator.generateAggregations(
       "GroupingWindowAggregateHelper",
       generator,
-      inputType,
+      inputFieldTypeInfo,
       aggregates,
       aggFields,
       aggMapping,
@@ -1047,12 +1064,9 @@ object AggregateUtil {
 
   private def isTimeWindow(window: LogicalWindow) = {
     window match {
-      case ProcessingTimeTumblingGroupWindow(_, size) => isTimeInterval(size.resultType)
-      case ProcessingTimeSlidingGroupWindow(_, size, _) => isTimeInterval(size.resultType)
-      case ProcessingTimeSessionGroupWindow(_, _) => true
-      case EventTimeTumblingGroupWindow(_, _, size) => isTimeInterval(size.resultType)
-      case EventTimeSlidingGroupWindow(_, _, size, _) => isTimeInterval(size.resultType)
-      case EventTimeSessionGroupWindow(_, _, _) => true
+      case TumblingGroupWindow(_, _, size) => isTimeIntervalLiteral(size)
+      case SlidingGroupWindow(_, _, size, _) => isTimeIntervalLiteral(size)
+      case SessionGroupWindow(_, _, _) => true
     }
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index 03ca02c80ac6a..ef97e716efb0a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -21,7 +21,7 @@ import java.util.{List => JList, ArrayList => JArrayList}
 
 import org.apache.flink.api.common.state._
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
-import org.apache.flink.api.java.typeutils.{ListTypeInfo, RowTypeInfo}
+import org.apache.flink.api.java.typeutils.ListTypeInfo
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
@@ -39,8 +39,8 @@ import org.slf4j.LoggerFactory
  */
 class RowTimeBoundedRangeOver(
     genAggregations: GeneratedAggregationsFunction,
-    aggregationStateType: RowTypeInfo,
-    inputRowType: RowTypeInfo,
+    aggregationStateType: TypeInformation[Row],
+    inputRowType: TypeInformation[Row],
     precedingOffset: Long)
   extends ProcessFunction[Row, Row]
     with Compiler[GeneratedAggregations] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index 4a9a14cbc9aa4..7169cf723f0e3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -41,7 +41,7 @@ import org.slf4j.LoggerFactory
 class RowTimeBoundedRowsOver(
     genAggregations: GeneratedAggregationsFunction,
     aggregationStateType: RowTypeInfo,
-    inputRowType: RowTypeInfo,
+    inputRowType: TypeInformation[Row],
     precedingOffset: Long)
   extends ProcessFunction[Row, Row]
     with Compiler[GeneratedAggregations] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
new file mode 100644
index 0000000000000..8466cdf025042
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.sources
+
+import org.apache.flink.api.java.tuple.Tuple2
+
+/**
+  * Defines logical time attributes for a [[TableSource]]. Time attributes can be used for
+  * indicating, accessing, and working with Flink's event-time or processing-time. A
+  * [[TableSource]] that implements this interface can define names and positions of rowtime
+  * and proctime attributes in the rows it produces.
+  */
+trait DefinedTimeAttributes {
+
+  /**
+    * Defines a name and position (starting at 0) of rowtime attribute that represents Flink's
+    * event-time. Null if no rowtime should be available. If the position is within the arity of
+    * the result row, the logical attribute will overwrite the physical attribute. If the position
+    * is higher than the result row, the time attribute will be appended logically.
+    */
+  def getRowtimeAttribute: Tuple2[Int, String]
+
+  /**
+    * Defines a name and position (starting at 0) of proctime attribute that represents Flink's
+    * processing-time. Null if no proctime should be available. If the position is within the arity
+    * of the result row, the logical attribute will overwrite the physical attribute. If the
+    * position is higher than the result row, the time attribute will be appended logically.
+    */
+  def getProctimeAttribute: Tuple2[Int, String]
+
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TimeIndicatorTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TimeIndicatorTypeInfo.scala
new file mode 100644
index 0000000000000..31dcb5c1e7a4e
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TimeIndicatorTypeInfo.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.typeutils
+
+import java.sql.Timestamp
+
+import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo
+import org.apache.flink.api.common.typeutils.TypeComparator
+import org.apache.flink.api.common.typeutils.base.{SqlTimestampComparator, SqlTimestampSerializer}
+
+/**
+  * Type information for indicating event or processing time. However, it behaves like a
+  * regular SQL timestamp.
+  */
+class TimeIndicatorTypeInfo(val isEventTime: Boolean)
+  extends SqlTimeTypeInfo[Timestamp](
+    classOf[Timestamp],
+    SqlTimestampSerializer.INSTANCE,
+    classOf[SqlTimestampComparator].asInstanceOf[Class[TypeComparator[Timestamp]]]) {
+
+  override def toString: String = s"TimeIndicatorTypeInfo"
+}
+
+object TimeIndicatorTypeInfo {
+
+  val ROWTIME_INDICATOR = new TimeIndicatorTypeInfo(true)
+  val PROCTIME_INDICATOR = new TimeIndicatorTypeInfo(false)
+
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala
index 40f0cf298b90f..9896a8c0440f8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala
@@ -17,7 +17,7 @@
  */
 package org.apache.flink.table.typeutils
 
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{BIG_DEC_TYPE_INFO, BOOLEAN_TYPE_INFO, INT_TYPE_INFO, STRING_TYPE_INFO}
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo._
 import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo
 import org.apache.flink.table.validate._
@@ -29,6 +29,7 @@ object TypeCheckUtils {
     * SQL type but NOT vice versa.
     */
   def isAdvanced(dataType: TypeInformation[_]): Boolean = dataType match {
+    case _: TimeIndicatorTypeInfo => false
     case _: BasicTypeInfo[_] => false
     case _: SqlTimeTypeInfo[_] => false
     case _: TimeIntervalTypeInfo[_] => false
@@ -64,6 +65,8 @@ object TypeCheckUtils {
 
   def isInteger(dataType: TypeInformation[_]): Boolean = dataType == INT_TYPE_INFO
 
+  def isLong(dataType: TypeInformation[_]): Boolean = dataType == LONG_TYPE_INFO
+
   def isArray(dataType: TypeInformation[_]): Boolean = dataType match {
     case _: ObjectArrayTypeInfo[_, _] | _: PrimitiveArrayTypeInfo[_] => true
     case _ => false
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
index 63dc1aedc337e..31ad55831edb4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
@@ -24,7 +24,7 @@ import org.apache.calcite.sql.{SqlFunction, SqlOperator, SqlOperatorTable}
 import org.apache.flink.table.api.ValidationException
 import org.apache.flink.table.expressions._
 import org.apache.flink.table.functions.utils.{AggSqlFunction, ScalarSqlFunction, TableSqlFunction}
-import org.apache.flink.table.functions.{AggregateFunction, EventTimeExtractor, RowTime, ScalarFunction, TableFunction, _}
+import org.apache.flink.table.functions.{AggregateFunction, ScalarFunction, TableFunction}
 
 import scala.collection.JavaConversions._
 import scala.collection.mutable
@@ -242,15 +242,11 @@ object FunctionCatalog {
     // array
     "cardinality" -> classOf[ArrayCardinality],
     "at" -> classOf[ArrayElementAt],
-    "element" -> classOf[ArrayElement],
+    "element" -> classOf[ArrayElement]
 
     // TODO implement function overloading here
     // "floor" -> classOf[TemporalFloor]
     // "ceil" -> classOf[TemporalCeil]
-
-    // extensions to support streaming query
-    "rowtime" -> classOf[RowTime],
-    "proctime" -> classOf[ProcTime]
   )
 
   /**
@@ -392,8 +388,6 @@ class BasicOperatorTable extends ReflectiveSqlOperatorTable {
     SqlStdOperatorTable.ROUND,
     SqlStdOperatorTable.PI,
     // EXTENSIONS
-    EventTimeExtractor,
-    ProcTimeExtractor,
     SqlStdOperatorTable.TUMBLE,
     SqlStdOperatorTable.TUMBLE_START,
     SqlStdOperatorTable.TUMBLE_END,
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
index cab3855af1c19..81c60b48b8a49 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
@@ -405,15 +405,6 @@ public void testGenericRowWithAlias() throws Exception {
 		tableEnv.fromDataSet(dataSet, "nullField");
 	}
 
-	@Test(expected = TableException.class)
-	public void testAsWithToFewFields() throws Exception {
-		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-		BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
-
-		// Must fail. Not enough field names specified.
-		tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b");
-	}
-
 	@Test(expected = TableException.class)
 	public void testAsWithToManyFields() throws Exception {
 		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index 9939a9c17db4c..faacc549bc104 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -93,7 +93,8 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2"),
         UnresolvedFieldReference("name3")
-    ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -107,7 +108,8 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2"),
         UnresolvedFieldReference("name3")
-    ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -121,7 +123,8 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2"),
         UnresolvedFieldReference("name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
   }
 
   @Test
@@ -132,7 +135,8 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("pf3"),
         UnresolvedFieldReference("pf1"),
         UnresolvedFieldReference("pf2")
-      ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("pf3", "pf1", "pf2")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -142,7 +146,8 @@ class TableEnvironmentTest extends TableTestBase {
   def testGetFieldInfoAtomicName1(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(
       atomicType,
-      Array(UnresolvedFieldReference("name"))
+      Array(UnresolvedFieldReference("name")),
+      ignoreTimeAttributes = true
     )
 
     fieldInfo._1.zip(Array("name")).foreach(x => assertEquals(x._2, x._1))
@@ -156,7 +161,8 @@ class TableEnvironmentTest extends TableTestBase {
       Array(
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2")
-      ))
+      ),
+      ignoreTimeAttributes = true)
   }
 
   @Test
@@ -167,7 +173,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("f0"), "name1"),
         Alias(UnresolvedFieldReference("f1"), "name2"),
         Alias(UnresolvedFieldReference("f2"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -181,7 +188,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("f2"), "name1"),
         Alias(UnresolvedFieldReference("f0"), "name2"),
         Alias(UnresolvedFieldReference("f1"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -195,7 +203,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("xxx"), "name1"),
         Alias(UnresolvedFieldReference("yyy"), "name2"),
         Alias(UnresolvedFieldReference("zzz"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
   }
 
   @Test
@@ -206,7 +215,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("cf1"), "name1"),
         Alias(UnresolvedFieldReference("cf2"), "name2"),
         Alias(UnresolvedFieldReference("cf3"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -220,7 +230,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("cf3"), "name1"),
         Alias(UnresolvedFieldReference("cf1"), "name2"),
         Alias(UnresolvedFieldReference("cf2"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -234,7 +245,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("xxx"), "name1"),
         Alias(UnresolvedFieldReference("yyy"), "name2"),
         Alias(UnresolvedFieldReference("zzz"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
   }
 
   @Test
@@ -245,7 +257,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("pf1"), "name1"),
         Alias(UnresolvedFieldReference("pf2"), "name2"),
         Alias(UnresolvedFieldReference("pf3"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -259,7 +272,8 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("pf3"), "name1"),
         Alias(UnresolvedFieldReference("pf1"), "name2"),
         Alias(UnresolvedFieldReference("pf2"), "name3")
-      ))
+      ),
+      ignoreTimeAttributes = true)
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -272,8 +286,9 @@ class TableEnvironmentTest extends TableTestBase {
       Array(
         Alias(UnresolvedFieldReference("xxx"), "name1"),
         Alias(UnresolvedFieldReference("yyy"), "name2"),
-        Alias( UnresolvedFieldReference("zzz"), "name3")
-      ))
+        Alias(UnresolvedFieldReference("zzz"), "name3")
+      ),
+      ignoreTimeAttributes = true)
   }
 
   @Test(expected = classOf[TableException])
@@ -282,12 +297,16 @@ class TableEnvironmentTest extends TableTestBase {
       atomicType,
       Array(
         Alias(UnresolvedFieldReference("name1"), "name2")
-      ))
+      ),
+      ignoreTimeAttributes = true)
   }
 
   @Test(expected = classOf[TableException])
   def testGetFieldInfoGenericRowAlias(): Unit = {
-    tEnv.getFieldInfo(genericRowType, Array(UnresolvedFieldReference("first")))
+    tEnv.getFieldInfo(
+      genericRowType,
+      Array(UnresolvedFieldReference("first")),
+      ignoreTimeAttributes = true)
   }
 
   @Test
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
index e61e1907e76c4..57ee3b3fca2cb 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
@@ -207,16 +207,6 @@ class TableEnvironmentITCase(
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
-  @Test(expected = classOf[TableException])
-  def testToTableWithToFewFields(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    CollectionDataSets.get3TupleDataSet(env)
-      // Must fail. Number of fields does not match.
-      .toTable(tEnv, 'a, 'b)
-  }
-
   @Test(expected = classOf[TableException])
   def testToTableWithToManyFields(): Unit = {
     val env = ExecutionEnvironment.getExecutionEnvironment
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
index 0ccb55723a397..71d0002272c19 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/WindowAggregateTest.scala
@@ -47,7 +47,7 @@ class WindowAggregateTest extends TableTestBase {
           batchTableNode(0),
           term("select", "ts, a, b")
         ),
-        term("window", EventTimeTumblingGroupWindow('w$, 'ts, 7200000.millis)),
+        term("window", TumblingGroupWindow('w$, 'ts, 7200000.millis)),
         term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
       )
 
@@ -76,7 +76,7 @@ class WindowAggregateTest extends TableTestBase {
           "DataSetWindowAggregate",
           batchTableNode(0),
           term("groupBy", "c"),
-          term("window", EventTimeTumblingGroupWindow('w$, 'ts, 240000.millis)),
+          term("window", TumblingGroupWindow('w$, 'ts, 240000.millis)),
           term("select", "c, SUM(a) AS sumA, MIN(b) AS minB, " +
             "start('w$) AS w$start, end('w$) AS w$end")
         ),
@@ -106,7 +106,7 @@ class WindowAggregateTest extends TableTestBase {
           batchTableNode(0),
           term("select", "ts, b, a")
         ),
-        term("window", EventTimeTumblingGroupWindow('w$, 'ts, 240000.millis)),
+        term("window", TumblingGroupWindow('w$, 'ts, 240000.millis)),
         term("select", "weightedAvg(b, a) AS wAvg")
       )
 
@@ -132,7 +132,7 @@ class WindowAggregateTest extends TableTestBase {
           term("select", "ts, a, b")
         ),
         term("window",
-          EventTimeSlidingGroupWindow('w$, 'ts, 5400000.millis, 900000.millis)),
+          SlidingGroupWindow('w$, 'ts, 5400000.millis, 900000.millis)),
         term("select", "SUM(a) AS sumA, COUNT(b) AS cntB")
       )
 
@@ -162,7 +162,7 @@ class WindowAggregateTest extends TableTestBase {
           batchTableNode(0),
           term("groupBy", "c, d"),
           term("window",
-            EventTimeSlidingGroupWindow('w$, 'ts, 10800000.millis, 3600000.millis)),
+            SlidingGroupWindow('w$, 'ts, 10800000.millis, 3600000.millis)),
           term("select", "c, d, SUM(a) AS sumA, AVG(b) AS avgB, " +
             "start('w$) AS w$start, end('w$) AS w$end")
         ),
@@ -188,7 +188,7 @@ class WindowAggregateTest extends TableTestBase {
           batchTableNode(0),
           term("select", "ts")
         ),
-        term("window", EventTimeSessionGroupWindow('w$, 'ts, 1800000.millis)),
+        term("window", SessionGroupWindow('w$, 'ts, 1800000.millis)),
         term("select", "COUNT(*) AS cnt")
       )
 
@@ -217,7 +217,7 @@ class WindowAggregateTest extends TableTestBase {
           "DataSetWindowAggregate",
           batchTableNode(0),
           term("groupBy", "c, d"),
-          term("window", EventTimeSessionGroupWindow('w$, 'ts, 43200000.millis)),
+          term("window", SessionGroupWindow('w$, 'ts, 43200000.millis)),
           term("select", "c, d, SUM(a) AS sumA, MIN(b) AS minB, " +
             "start('w$) AS w$start, end('w$) AS w$end")
         ),
@@ -249,7 +249,7 @@ class WindowAggregateTest extends TableTestBase {
             term("select", "ts, c")
           ),
           term("groupBy", "c"),
-          term("window", EventTimeTumblingGroupWindow('w$, 'ts, 240000.millis)),
+          term("window", TumblingGroupWindow('w$, 'ts, 240000.millis)),
           term("select", "c, start('w$) AS w$start, end('w$) AS w$end")
         ),
         term("select", "CAST(w$end) AS w$end")
@@ -304,7 +304,7 @@ class WindowAggregateTest extends TableTestBase {
 
     val sql = "SELECT COUNT(*) " +
       "FROM T " +
-      "GROUP BY TUMBLE(proctime(), b * INTERVAL '1' MINUTE)"
+      "GROUP BY TUMBLE(ts, b * INTERVAL '1' MINUTE)"
     util.verifySql(sql, "n/a")
   }
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
index 6ebfec073bd76..b484293132f98 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
@@ -19,14 +19,12 @@ package org.apache.flink.table.api.scala.batch.table
 
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.expressions.{RowtimeAttribute, Upper, WindowReference}
-import org.apache.flink.table.functions.ScalarFunction
 import org.apache.flink.table.api.scala.batch.table.FieldProjectionTest._
-import org.apache.flink.table.plan.logical.EventTimeTumblingGroupWindow
-import org.apache.flink.table.utils._
-import org.apache.flink.table.utils.TableTestBase
+import org.apache.flink.table.expressions.{Upper, WindowReference}
+import org.apache.flink.table.functions.ScalarFunction
+import org.apache.flink.table.plan.logical.TumblingGroupWindow
 import org.apache.flink.table.utils.TableTestUtil._
+import org.apache.flink.table.utils.{TableTestBase, _}
 import org.junit.Test
 
 /**
@@ -223,7 +221,8 @@ class FieldProjectionTest extends TableTestBase {
 
   @Test
   def testSelectFromStreamingWindow(): Unit = {
-    val sourceTable = streamUtil.addTable[(Int, Long, String, Double)]("MyTable", 'a, 'b, 'c, 'd)
+    val sourceTable = streamUtil
+      .addTable[(Int, Long, String, Double)]("MyTable", 'a, 'b, 'c, 'd, 'rowtime.rowtime)
     val resultTable = sourceTable
         .window(Tumble over 5.millis on 'rowtime as 'w)
         .groupBy('w)
@@ -235,14 +234,14 @@ class FieldProjectionTest extends TableTestBase {
         unaryNode(
           "DataStreamCalc",
           streamTableNode(0),
-          term("select", "c", "a", "UPPER(c) AS $f2")
+          term("select", "c", "a", "rowtime", "UPPER(c) AS $f3")
         ),
         term("window",
-          EventTimeTumblingGroupWindow(
-           WindowReference("w"),
-            RowtimeAttribute(),
+          TumblingGroupWindow(
+            WindowReference("w"),
+            'rowtime,
             5.millis)),
-        term("select", "COUNT($f2) AS TMP_0", "SUM(a) AS TMP_1")
+        term("select", "COUNT($f3) AS TMP_0", "SUM(a) AS TMP_1")
       )
 
     streamUtil.verifyTable(resultTable, expected)
@@ -250,7 +249,8 @@ class FieldProjectionTest extends TableTestBase {
 
   @Test
   def testSelectFromStreamingGroupedWindow(): Unit = {
-    val sourceTable = streamUtil.addTable[(Int, Long, String, Double)]("MyTable", 'a, 'b, 'c, 'd)
+    val sourceTable = streamUtil
+      .addTable[(Int, Long, String, Double)]("MyTable", 'a, 'b, 'c, 'd, 'rowtime.rowtime)
     val resultTable = sourceTable
         .window(Tumble over 5.millis on 'rowtime as 'w)
         .groupBy('w, 'b)
@@ -263,15 +263,15 @@ class FieldProjectionTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "c", "a", "b", "UPPER(c) AS $f3")
+            term("select", "c", "a", "b", "rowtime", "UPPER(c) AS $f4")
           ),
           term("groupBy", "b"),
           term("window",
-            EventTimeTumblingGroupWindow(
-             WindowReference("w"),
-              RowtimeAttribute(),
+            TumblingGroupWindow(
+              WindowReference("w"),
+              'rowtime,
               5.millis)),
-          term("select", "b", "COUNT($f3) AS TMP_0", "SUM(a) AS TMP_1")
+          term("select", "b", "COUNT($f4) AS TMP_0", "SUM(a) AS TMP_1")
         ),
         term("select", "TMP_0", "TMP_1", "b")
     )
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
index 8a20f6d0d11e7..c4811050a926c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
@@ -63,25 +63,24 @@ class GroupWindowTest extends TableTestBase {
   //===============================================================================================
 
   @Test(expected = classOf[ValidationException])
-  def testProcessingTimeTumblingGroupWindowOverTime(): Unit = {
+  def testInvalidProcessingTimeDefinition(): Unit = {
     val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-      .window(Tumble over 50.milli as 'w)   // require a time attribute
-      .groupBy('w, 'string)
-      .select('string, 'int.count)
+    // proctime is not allowed
+    util.addTable[(Long, Int, String)]('long.proctime, 'int, 'string)
   }
 
   @Test(expected = classOf[ValidationException])
-  def testProcessingTimeTumblingGroupWindowOverCount(): Unit = {
+  def testInvalidProcessingTimeDefinition2(): Unit = {
     val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    // proctime is not allowed
+    util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
+  }
 
-    table
-      .window(Tumble over 2.rows as 'w)   // require a time attribute
-      .groupBy('w, 'string)
-      .select('string, 'int.count)
+  @Test(expected = classOf[ValidationException])
+  def testInvalidEventTimeDefinition(): Unit = {
+    val util = batchTestUtil()
+    // definition must not extend schema
+    util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -101,7 +100,7 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeTumblingGroupWindowOverCount(): Unit = {
     val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
       .window(Tumble over 2.rows on 'long as 'w)
@@ -112,7 +111,7 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 2.rows)),
+      term("window", TumblingGroupWindow(WindowReference("w"), 'long, 2.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -135,7 +134,7 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
+      term("window", TumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
 
@@ -145,7 +144,7 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeTumblingGroupWindowOverTime(): Unit = {
     val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
       .window(Tumble over 5.milli on 'long as 'w)
@@ -156,35 +155,13 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
+      term("window", TumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
     util.verifyTable(windowedTable, expected)
   }
 
-  @Test(expected = classOf[ValidationException])
-  def testAllProcessingTimeTumblingGroupWindowOverTime(): Unit = {
-    val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-      .window(Tumble over 50.milli as 'w) // require a time attribute
-      .groupBy('w)
-      .select('string, 'int.count)
-  }
-
-  @Test(expected = classOf[ValidationException])
-  def testAllProcessingTimeTumblingGroupWindowOverCount(): Unit = {
-    val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-      .window(Tumble over 2.rows as 'w) // require a time attribute
-      .groupBy('w)
-      .select('int.count)
-  }
-
   @Test(expected = classOf[ValidationException])
   def testAllTumblingGroupWindowWithInvalidUdAggArgs(): Unit = {
     val util = batchTestUtil()
@@ -216,7 +193,7 @@ class GroupWindowTest extends TableTestBase {
         batchTableNode(0),
         term("select", "int", "long")
       ),
-      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
+      term("window", TumblingGroupWindow(WindowReference("w"), 'long, 5.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -240,7 +217,7 @@ class GroupWindowTest extends TableTestBase {
         batchTableNode(0),
         term("select", "int", "long")
       ),
-      term("window", EventTimeTumblingGroupWindow(WindowReference("w"), 'long, 2.rows)),
+      term("window", TumblingGroupWindow(WindowReference("w"), 'long, 2.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -251,28 +228,6 @@ class GroupWindowTest extends TableTestBase {
   // Sliding Windows
   //===============================================================================================
 
-  @Test(expected = classOf[ValidationException])
-  def testProcessingTimeSlidingGroupWindowOverTime(): Unit = {
-    val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-      .window(Slide over 50.milli every 50.milli as 'w) // require on a time attribute
-      .groupBy('w, 'string)
-      .select('string, 'int.count)
-  }
-
-  @Test(expected = classOf[ValidationException])
-  def testProcessingTimeSlidingGroupWindowOverCount(): Unit = {
-    val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-      .window(Slide over 10.rows every 5.rows as 'w) // require on a time attribute
-      .groupBy('w, 'string)
-      .select('string, 'int.count)
-  }
-
   @Test(expected = classOf[ValidationException])
   def testSlidingGroupWindowWithInvalidUdAggArgs(): Unit = {
     val util = batchTestUtil()
@@ -302,7 +257,7 @@ class GroupWindowTest extends TableTestBase {
       batchTableNode(0),
       term("groupBy", "string"),
       term("window",
-        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
+        SlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -324,7 +279,7 @@ class GroupWindowTest extends TableTestBase {
       batchTableNode(0),
       term("groupBy", "string"),
       term("window",
-        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 2.rows, 1.rows)),
+        SlidingGroupWindow(WindowReference("w"), 'long, 2.rows, 1.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -348,24 +303,13 @@ class GroupWindowTest extends TableTestBase {
       batchTableNode(0),
       term("groupBy", "string"),
       term("window",
-           EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
+           SlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
 
     util.verifyTable(windowedTable, expected)
   }
 
-  @Test(expected = classOf[ValidationException])
-  def testAllProcessingTimeSlidingGroupWindowOverCount(): Unit = {
-    val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-      .window(Slide over 2.rows every 1.rows as 'w) // require on a time attribute
-      .groupBy('w)
-      .select('int.count)
-  }
-
   @Test(expected = classOf[ValidationException])
   def testAllSlidingGroupWindowWithInvalidUdAggArgs(): Unit = {
     val util = batchTestUtil()
@@ -374,7 +318,7 @@ class GroupWindowTest extends TableTestBase {
     val myWeightedAvg = new WeightedAvgWithMerge
 
     table
-      .window(Slide over 2.minutes every 1.minute on 'rowtime as 'w)
+      .window(Slide over 2.minutes every 1.minute on 'long as 'w)
       .groupBy('w)
       // invalid function arguments
       .select(myWeightedAvg('int, 'string))
@@ -398,7 +342,7 @@ class GroupWindowTest extends TableTestBase {
         term("select", "int", "long")
       ),
       term("window",
-        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
+        SlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -423,7 +367,7 @@ class GroupWindowTest extends TableTestBase {
         term("select", "int", "long")
       ),
       term("window",
-        EventTimeSlidingGroupWindow(WindowReference("w"), 'long, 2.rows, 1.rows)),
+        SlidingGroupWindow(WindowReference("w"), 'long, 2.rows, 1.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -448,7 +392,7 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeSessionGroupWindow(WindowReference("w"), 'long, 7.milli)),
+      term("window", SessionGroupWindow(WindowReference("w"), 'long, 7.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -471,24 +415,13 @@ class GroupWindowTest extends TableTestBase {
       "DataSetWindowAggregate",
       batchTableNode(0),
       term("groupBy", "string"),
-      term("window", EventTimeSessionGroupWindow(WindowReference("w"), 'long, 7.milli)),
+      term("window", SessionGroupWindow(WindowReference("w"), 'long, 7.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
 
     util.verifyTable(windowedTable, expected)
   }
 
-  @Test(expected = classOf[ValidationException])
-  def testProcessingTimeSessionGroupWindowOverTime(): Unit = {
-    val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    val windowedTable = table
-      .window(Session withGap 7.milli as 'w) // require on a time attribute
-      .groupBy('string, 'w)
-      .select('string, 'int.count)
-  }
-
   @Test(expected = classOf[ValidationException])
   def testSessionGroupWindowWithInvalidUdAggArgs(): Unit = {
     val util = batchTestUtil()
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
index 67d13b0f455a5..6bab4b33f6dfd 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
@@ -197,14 +197,14 @@ class SqlITCase extends StreamingWithStateTestBase {
     // for sum aggregation ensure that every time the order of each element is consistent
     env.setParallelism(1)
 
-    val t1 = env.fromCollection(data).toTable(tEnv).as('a, 'b, 'c)
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY ProcTime()  RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY c ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt2 " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime  RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
       "from T1"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -224,13 +224,13 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.testResults = mutable.MutableList()
 
-    val t1 = env.fromCollection(data).toTable(tEnv).as('a, 'b, 'c)
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY ProcTime() ROWS BETWEEN UNBOUNDED preceding AND " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
       "CURRENT ROW)" +
       "from T1"
 
@@ -254,14 +254,14 @@ class SqlITCase extends StreamingWithStateTestBase {
     // for sum aggregation ensure that every time the order of each element is consistent
     env.setParallelism(1)
 
-    val t1 = env.fromCollection(data).toTable(tEnv).as('a, 'b, 'c)
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, " +
-      "count(a) OVER (ORDER BY ProcTime()  RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt2 " +
+      "count(a) OVER (ORDER BY proctime  RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
       "from T1"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -281,12 +281,12 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.testResults = mutable.MutableList()
 
-    val t1 = env.fromCollection(data).toTable(tEnv).as('a, 'b, 'c)
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
-      "count(a) OVER (ORDER BY ProcTime() ROWS BETWEEN UNBOUNDED preceding AND CURRENT ROW)" +
+      "count(a) OVER (ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND CURRENT ROW)" +
       "from T1"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -328,14 +328,14 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val t1 = env
       .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, a, " +
-      "count(a) OVER (PARTITION BY c ORDER BY RowTime() ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
-      ", sum(a) OVER (PARTITION BY c ORDER BY RowTime() ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
+      ", sum(a) OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
       " from T1"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -385,14 +385,14 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val t1 = env
       .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, a, " +
-      "count(a) OVER (ORDER BY RowTime() ROWS BETWEEN 2 preceding AND CURRENT ROW)," +
-      "sum(a) OVER (ORDER BY RowTime() ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
+      "count(a) OVER (ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)," +
+      "sum(a) OVER (ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
       "from T1"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -453,15 +453,15 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val t1 = env
       .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, b, " +
-      "count(a) OVER (PARTITION BY c ORDER BY RowTime() RANGE BETWEEN INTERVAL '1' SECOND " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
       "preceding AND CURRENT ROW)" +
-      ", sum(a) OVER (PARTITION BY c ORDER BY RowTime() RANGE BETWEEN INTERVAL '1' SECOND " +
+      ", sum(a) OVER (PARTITION BY c ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
       " preceding AND CURRENT ROW)" +
       " from T1"
 
@@ -525,15 +525,15 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val t1 = env
       .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, b, " +
-      "count(a) OVER (ORDER BY RowTime() RANGE BETWEEN INTERVAL '1' SECOND " +
+      "count(a) OVER (ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
       "preceding AND CURRENT ROW)" +
-      ", sum(a) OVER (ORDER BY RowTime() RANGE BETWEEN INTERVAL '1' SECOND " +
+      ", sum(a) OVER (ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
       " preceding AND CURRENT ROW)" +
       " from T1"
 
@@ -565,14 +565,14 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.testResults = mutable.MutableList()
 
-    val t1 = env.fromCollection(data).toTable(tEnv).as('a, 'b, 'c)
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
     tEnv.registerTable("T1", t1)
 
     val sqlQuery = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY b ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt2 " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY b ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
       "from T1"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -592,15 +592,15 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val sqlQuery = "SELECT a, b, c, " +
       "SUM(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "count(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "avg(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "max(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "min(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row) " +
+      "partition by a order by rowtime rows between unbounded preceding and current row) " +
       "from T1"
 
     val data = Seq(
@@ -632,7 +632,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     )
 
     val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
@@ -670,15 +670,15 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val sqlQuery = "SELECT a, b, c, " +
       "SUM(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "count(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "avg(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "max(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row), " +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
       "min(b) over (" +
-      "partition by a order by rowtime() rows between unbounded preceding and current row) " +
+      "partition by a order by rowtime rows between unbounded preceding and current row) " +
       "from T1"
 
     val data = Seq(
@@ -702,7 +702,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     )
 
     val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
@@ -740,11 +740,11 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "count(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "avg(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "max(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "min(b) over (order by rowtime() rows between unbounded preceding and current row) " +
+      "SUM(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "count(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "avg(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "max(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "min(b) over (order by rowtime rows between unbounded preceding and current row) " +
       "from T1"
 
     val data = Seq(
@@ -764,7 +764,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     )
 
     val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
@@ -795,11 +795,11 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "count(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "avg(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "max(b) over (order by rowtime() rows between unbounded preceding and current row), " +
-      "min(b) over (order by rowtime() rows between unbounded preceding and current row) " +
+      "SUM(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "count(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "avg(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "max(b) over (order by rowtime rows between unbounded preceding and current row), " +
+      "min(b) over (order by rowtime rows between unbounded preceding and current row) " +
       "from T1"
 
     val data = Seq(
@@ -820,7 +820,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     )
 
     val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
@@ -852,11 +852,11 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (order by rowtime() range between unbounded preceding and current row), " +
-      "count(b) over (order by rowtime() range between unbounded preceding and current row), " +
-      "avg(b) over (order by rowtime() range between unbounded preceding and current row), " +
-      "max(b) over (order by rowtime() range between unbounded preceding and current row), " +
-      "min(b) over (order by rowtime() range between unbounded preceding and current row) " +
+      "SUM(b) over (order by rowtime range between unbounded preceding and current row), " +
+      "count(b) over (order by rowtime range between unbounded preceding and current row), " +
+      "avg(b) over (order by rowtime range between unbounded preceding and current row), " +
+      "max(b) over (order by rowtime range between unbounded preceding and current row), " +
+      "min(b) over (order by rowtime range between unbounded preceding and current row) " +
       "from T1"
 
     val data = Seq(
@@ -878,7 +878,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     )
 
     val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
@@ -916,15 +916,15 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val sqlQuery = "SELECT a, b, c, " +
       "SUM(b) over (" +
-      "partition by a order by rowtime() range between unbounded preceding and current row), " +
+      "partition by a order by rowtime range between unbounded preceding and current row), " +
       "count(b) over (" +
-      "partition by a order by rowtime() range between unbounded preceding and current row), " +
+      "partition by a order by rowtime range between unbounded preceding and current row), " +
       "avg(b) over (" +
-      "partition by a order by rowtime() range between unbounded preceding and current row), " +
+      "partition by a order by rowtime range between unbounded preceding and current row), " +
       "max(b) over (" +
-      "partition by a order by rowtime() range between unbounded preceding and current row), " +
+      "partition by a order by rowtime range between unbounded preceding and current row), " +
       "min(b) over (" +
-      "partition by a order by rowtime() range between unbounded preceding and current row) " +
+      "partition by a order by rowtime range between unbounded preceding and current row) " +
       "from T1"
 
     val data = Seq(
@@ -946,7 +946,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     )
 
     val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     tEnv.registerTable("T1", t1)
 
@@ -981,14 +981,15 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setParallelism(1)
     StreamITCase.testResults = mutable.MutableList()
 
-    val t = StreamTestData.get5TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val t = StreamTestData.get5TupleDataStream(env)
+      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
     tEnv.registerTable("MyTable", t)
 
     val sqlQuery = "SELECT a,  " +
       " SUM(c) OVER (" +
-      " PARTITION BY a ORDER BY procTime() ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS sumC , " +
+      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS sumC , " +
       " MIN(c) OVER (" +
-      " PARTITION BY a ORDER BY procTime() ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS minC " +
+      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS minC " +
       " FROM MyTable"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -1023,14 +1024,15 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setParallelism(1)
     StreamITCase.testResults = mutable.MutableList()
 
-    val t = StreamTestData.get5TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val t = StreamTestData.get5TupleDataStream(env)
+      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
     tEnv.registerTable("MyTable", t)
 
     val sqlQuery = "SELECT a,  " +
       " SUM(c) OVER (" +
-      " PARTITION BY a ORDER BY procTime() ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS sumC , " +
+      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS sumC , " +
       " MIN(c) OVER (" +
-      " PARTITION BY a ORDER BY procTime() ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS minC " +
+      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS minC " +
       " FROM MyTable"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -1066,14 +1068,15 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setParallelism(1)
     StreamITCase.testResults = mutable.MutableList()
 
-    val t = StreamTestData.get5TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val t = StreamTestData.get5TupleDataStream(env)
+      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
     tEnv.registerTable("MyTable", t)
 
     val sqlQuery = "SELECT a,  " +
       " SUM(c) OVER (" +
-      " ORDER BY procTime() ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS sumC , " +
+      " ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS sumC , " +
       " MIN(c) OVER (" +
-      " ORDER BY procTime() ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS minC " +
+      " ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS minC " +
       " FROM MyTable"
 
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
@@ -1108,14 +1111,15 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setParallelism(1)
     StreamITCase.testResults = mutable.MutableList()
 
-    val t = StreamTestData.get5TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val t = StreamTestData.get5TupleDataStream(env)
+      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
     tEnv.registerTable("MyTable", t)
 
     val sqlQuery = "SELECT a,  " +
       " SUM(c) OVER (" +
-      " ORDER BY procTime() ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS sumC , " +
+      " ORDER BY proctime ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS sumC , " +
       " MIN(c) OVER (" +
-      " ORDER BY procTime() ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS minC " +
+      " ORDER BY proctime ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS minC " +
       " FROM MyTable"
     val result = tEnv.sql(sqlQuery).toDataStream[Row]
     result.addSink(new StreamITCase.StringSink)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
index cef2665692da7..edf7b1d0218d7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
@@ -21,22 +21,23 @@ import org.apache.flink.api.scala._
 import org.apache.flink.table.api.{TableException, ValidationException}
 import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.plan.logical.{EventTimeTumblingGroupWindow, ProcessingTimeSessionGroupWindow, ProcessingTimeSlidingGroupWindow}
+import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.utils.TableTestUtil._
 import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase}
-import org.junit.Test
+import org.junit.{Ignore, Test}
 
 class WindowAggregateTest extends TableTestBase {
   private val streamUtil: StreamTableTestUtil = streamTestUtil()
-  streamUtil.addTable[(Int, String, Long)]("MyTable", 'a, 'b, 'c)
+  streamUtil.addTable[(Int, String, Long)](
+    "MyTable", 'a, 'b, 'c, 'proctime.proctime, 'rowtime.rowtime)
 
   @Test
   def testNonPartitionedProcessingTimeBoundedWindow() = {
 
-    val sqlQuery = "SELECT a, Count(c) OVER (ORDER BY procTime()" +
-      "RANGE BETWEEN INTERVAL '10' SECOND PRECEDING AND CURRENT ROW) AS countA FROM MyTable"
-
-    val expected =
+    val sqlQuery = "SELECT a, Count(c) OVER (ORDER BY proctime  " +
+      "RANGE BETWEEN INTERVAL '10' SECOND PRECEDING AND CURRENT ROW) AS countA " +
+      "FROM MyTable"
+      val expected =
       unaryNode(
         "DataStreamCalc",
         unaryNode(
@@ -44,11 +45,11 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN 10000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(c) AS w0$o0")
+          term("select", "a", "c", "proctime", "COUNT(c) AS w0$o0")
         ),
         term("select", "a", "w0$o0 AS $1")
       )
@@ -59,7 +60,7 @@ class WindowAggregateTest extends TableTestBase {
   @Test
   def testPartitionedProcessingTimeBoundedWindow() = {
 
-    val sqlQuery = "SELECT a, AVG(c) OVER (PARTITION BY a ORDER BY procTime()" +
+    val sqlQuery = "SELECT a, AVG(c) OVER (PARTITION BY a ORDER BY proctime " +
       "RANGE BETWEEN INTERVAL '2' HOUR PRECEDING AND CURRENT ROW) AS avgA " +
       "FROM MyTable"
       val expected =
@@ -70,12 +71,12 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
           term("partitionBy","a"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(c) AS w0$o0", "$SUM0(c) AS w0$o1")
+          term("select", "a", "c", "proctime", "COUNT(c) AS w0$o0", "$SUM0(c) AS w0$o1")
         ),
         term("select", "a", "/(CASE(>(w0$o0, 0)", "CAST(w0$o1), null), w0$o0) AS avgA")
       )
@@ -84,23 +85,24 @@ class WindowAggregateTest extends TableTestBase {
   }
 
   @Test
+  @Ignore // TODO enable once CALCITE-1761 is fixed
   def testTumbleFunction() = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
 
     val sql =
       "SELECT " +
         "  COUNT(*), weightedAvg(c, a) AS wAvg, " +
-        "  TUMBLE_START(rowtime(), INTERVAL '15' MINUTE), " +
-        "  TUMBLE_END(rowtime(), INTERVAL '15' MINUTE)" +
+        "  TUMBLE_START(rowtime, INTERVAL '15' MINUTE), " +
+        "  TUMBLE_END(rowtime, INTERVAL '15' MINUTE)" +
         "FROM MyTable " +
-        "GROUP BY TUMBLE(rowtime(), INTERVAL '15' MINUTE)"
+        "GROUP BY TUMBLE(rowtime, INTERVAL '15' MINUTE)"
     val expected =
       unaryNode(
         "DataStreamCalc",
         unaryNode(
           "DataStreamAggregate",
           streamTableNode(0),
-          term("window", EventTimeTumblingGroupWindow('w$, 'rowtime, 900000.millis)),
+          term("window", TumblingGroupWindow('w$, 'rowtime, 900000.millis)),
           term("select",
             "COUNT(*) AS EXPR$0, " +
               "weightedAvg(c, a) AS wAvg, " +
@@ -113,23 +115,23 @@ class WindowAggregateTest extends TableTestBase {
   }
 
   @Test
+  @Ignore // TODO enable once CALCITE-1761 is fixed
   def testHoppingFunction() = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
 
     val sql =
       "SELECT COUNT(*), weightedAvg(c, a) AS wAvg, " +
-        "  HOP_START(proctime(), INTERVAL '15' MINUTE, INTERVAL '1' HOUR), " +
-        "  HOP_END(proctime(), INTERVAL '15' MINUTE, INTERVAL '1' HOUR) " +
+        "  HOP_START(proctime, INTERVAL '15' MINUTE, INTERVAL '1' HOUR), " +
+        "  HOP_END(proctime, INTERVAL '15' MINUTE, INTERVAL '1' HOUR) " +
         "FROM MyTable " +
-        "GROUP BY HOP(proctime(), INTERVAL '15' MINUTE, INTERVAL '1' HOUR)"
+        "GROUP BY HOP(proctime, INTERVAL '15' MINUTE, INTERVAL '1' HOUR)"
     val expected =
       unaryNode(
         "DataStreamCalc",
         unaryNode(
           "DataStreamAggregate",
           streamTableNode(0),
-          term("window", ProcessingTimeSlidingGroupWindow('w$,
-            3600000.millis, 900000.millis)),
+          term("window", SlidingGroupWindow('w$, 'rowtime, 3600000.millis, 900000.millis)),
           term("select",
             "COUNT(*) AS EXPR$0, " +
               "weightedAvg(c, a) AS wAvg, " +
@@ -142,23 +144,24 @@ class WindowAggregateTest extends TableTestBase {
   }
 
   @Test
+  @Ignore // TODO enable once CALCITE-1761 is fixed
   def testSessionFunction() = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
 
     val sql =
       "SELECT " +
         "  COUNT(*), weightedAvg(c, a) AS wAvg, " +
-        "  SESSION_START(proctime(), INTERVAL '15' MINUTE), " +
-        "  SESSION_END(proctime(), INTERVAL '15' MINUTE) " +
+        "  SESSION_START(proctime, INTERVAL '15' MINUTE), " +
+        "  SESSION_END(proctime, INTERVAL '15' MINUTE) " +
         "FROM MyTable " +
-        "GROUP BY SESSION(proctime(), INTERVAL '15' MINUTE)"
+        "GROUP BY SESSION(proctime, INTERVAL '15' MINUTE)"
     val expected =
       unaryNode(
         "DataStreamCalc",
         unaryNode(
           "DataStreamAggregate",
           streamTableNode(0),
-          term("window", ProcessingTimeSessionGroupWindow('w$, 900000.millis)),
+          term("window", SessionGroupWindow('w$, 'rowtime, 900000.millis)),
           term("select",
             "COUNT(*) AS EXPR$0, " +
               "weightedAvg(c, a) AS wAvg, " +
@@ -175,7 +178,7 @@ class WindowAggregateTest extends TableTestBase {
     val sqlQuery =
       "SELECT SUM(a) AS sumA, COUNT(b) AS cntB " +
         "FROM MyTable " +
-        "GROUP BY TUMBLE(proctime(), INTERVAL '2' HOUR, TIME '10:00:00')"
+        "GROUP BY TUMBLE(proctime, INTERVAL '2' HOUR, TIME '10:00:00')"
 
     streamUtil.verifySql(sqlQuery, "n/a")
   }
@@ -185,7 +188,7 @@ class WindowAggregateTest extends TableTestBase {
     val sqlQuery =
       "SELECT SUM(a) AS sumA, COUNT(b) AS cntB " +
         "FROM MyTable " +
-        "GROUP BY HOP(proctime(), INTERVAL '1' HOUR, INTERVAL '2' HOUR, TIME '10:00:00')"
+        "GROUP BY HOP(proctime, INTERVAL '1' HOUR, INTERVAL '2' HOUR, TIME '10:00:00')"
 
     streamUtil.verifySql(sqlQuery, "n/a")
   }
@@ -195,21 +198,21 @@ class WindowAggregateTest extends TableTestBase {
     val sqlQuery =
       "SELECT SUM(a) AS sumA, COUNT(b) AS cntB " +
         "FROM MyTable " +
-        "GROUP BY SESSION(proctime(), INTERVAL '2' HOUR, TIME '10:00:00')"
+        "GROUP BY SESSION(proctime, INTERVAL '2' HOUR, TIME '10:00:00')"
 
     streamUtil.verifySql(sqlQuery, "n/a")
   }
 
   @Test(expected = classOf[TableException])
   def testVariableWindowSize() = {
-    val sql = "SELECT COUNT(*) FROM MyTable GROUP BY TUMBLE(proctime(), c * INTERVAL '1' MINUTE)"
+    val sql = "SELECT COUNT(*) FROM MyTable GROUP BY TUMBLE(proctime, c * INTERVAL '1' MINUTE)"
     streamUtil.verifySql(sql, "n/a")
   }
 
   @Test(expected = classOf[TableException])
   def testMultiWindow() = {
     val sql = "SELECT COUNT(*) FROM MyTable GROUP BY " +
-      "FLOOR(rowtime() TO HOUR), FLOOR(rowtime() TO MINUTE)"
+      "FLOOR(rowtime TO HOUR), FLOOR(rowtime TO MINUTE)"
     val expected = ""
     streamUtil.verifySql(sql, expected)
   }
@@ -237,8 +240,8 @@ class WindowAggregateTest extends TableTestBase {
   def testUnboundPartitionedProcessingWindowWithRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY c ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt2 " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
       "from MyTable"
 
     val expected =
@@ -249,12 +252,12 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
         ),
         term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
       )
@@ -265,7 +268,7 @@ class WindowAggregateTest extends TableTestBase {
   def testUnboundPartitionedProcessingWindowWithRow() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY ProcTime() ROWS BETWEEN UNBOUNDED preceding AND " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
       "CURRENT ROW) as cnt1 " +
       "from MyTable"
 
@@ -274,15 +277,11 @@ class WindowAggregateTest extends TableTestBase {
         "DataStreamCalc",
         unaryNode(
           "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
-          ),
+          streamTableNode(0),
           term("partitionBy", "c"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "b", "c", "proctime", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
@@ -293,8 +292,8 @@ class WindowAggregateTest extends TableTestBase {
   def testUnboundNonPartitionedProcessingWindowWithRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (ORDER BY ProcTime() RANGE UNBOUNDED preceding) as cnt2 " +
+      "count(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
       "from MyTable"
 
     val expected =
@@ -305,11 +304,11 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
         ),
         term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
       )
@@ -320,7 +319,7 @@ class WindowAggregateTest extends TableTestBase {
   def testUnboundNonPartitionedProcessingWindowWithRow() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (ORDER BY ProcTime() ROWS BETWEEN UNBOUNDED preceding AND " +
+      "count(a) OVER (ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
       "CURRENT ROW) as cnt1 " +
       "from MyTable"
 
@@ -329,14 +328,10 @@ class WindowAggregateTest extends TableTestBase {
         "DataStreamCalc",
         unaryNode(
           "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
-          ),
-          term("orderBy", "PROCTIME"),
+          streamTableNode(0),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "b", "c", "proctime", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
@@ -347,8 +342,8 @@ class WindowAggregateTest extends TableTestBase {
   def testUnboundNonPartitionedEventTimeWindowWithRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (ORDER BY RowTime() RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (ORDER BY RowTime() RANGE UNBOUNDED preceding) as cnt2 " +
+      "count(a) OVER (ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt2 " +
       "from MyTable"
 
     val expected =
@@ -359,11 +354,11 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
         ),
         term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
       )
@@ -374,8 +369,8 @@ class WindowAggregateTest extends TableTestBase {
   def testUnboundPartitionedEventTimeWindowWithRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY RowTime() RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY c ORDER BY RowTime() RANGE UNBOUNDED preceding) as cnt2 " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY c ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt2 " +
       "from MyTable"
 
     val expected =
@@ -386,12 +381,12 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
         ),
         term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
       )
@@ -402,7 +397,7 @@ class WindowAggregateTest extends TableTestBase {
   def testBoundPartitionedRowTimeWindowWithRow() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY RowTime() ROWS BETWEEN 5 preceding AND " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 5 preceding AND " +
       "CURRENT ROW) as cnt1 " +
       "from MyTable"
 
@@ -414,12 +409,12 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("rows", "BETWEEN 5 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
@@ -430,7 +425,7 @@ class WindowAggregateTest extends TableTestBase {
   def testBoundNonPartitionedRowTimeWindowWithRow() = {
     val sql = "SELECT " +
         "c, " +
-        "count(a) OVER (ORDER BY RowTime() ROWS BETWEEN 5 preceding AND " +
+        "count(a) OVER (ORDER BY rowtime ROWS BETWEEN 5 preceding AND " +
         "CURRENT ROW) as cnt1 " +
         "from MyTable"
 
@@ -442,11 +437,11 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("rows", "BETWEEN 5 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
@@ -457,7 +452,7 @@ class WindowAggregateTest extends TableTestBase {
   def testBoundPartitionedRowTimeWindowWithRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY RowTime() " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime " +
       "RANGE BETWEEN INTERVAL '1' SECOND  preceding AND CURRENT ROW) as cnt1 " +
       "from MyTable"
 
@@ -469,12 +464,12 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN 1000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
@@ -485,7 +480,7 @@ class WindowAggregateTest extends TableTestBase {
   def testBoundNonPartitionedRowTimeWindowWithRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (ORDER BY RowTime() " +
+      "count(a) OVER (ORDER BY rowtime " +
       "RANGE BETWEEN INTERVAL '1' SECOND  preceding AND CURRENT ROW) as cnt1 " +
       "from MyTable"
 
@@ -497,11 +492,11 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN 1000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
@@ -512,7 +507,7 @@ class WindowAggregateTest extends TableTestBase {
   def testBoundNonPartitionedProcTimeWindowWithRowRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (ORDER BY procTime() ROWS BETWEEN 2 preceding AND " +
+      "count(a) OVER (ORDER BY proctime ROWS BETWEEN 2 preceding AND " +
       "CURRENT ROW) as cnt1 " +
       "from MyTable"
 
@@ -524,11 +519,11 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
@@ -539,7 +534,7 @@ class WindowAggregateTest extends TableTestBase {
   def testBoundPartitionedProcTimeWindowWithRowRange() = {
     val sql = "SELECT " +
       "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY procTime() ROWS BETWEEN 2 preceding AND " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN 2 preceding AND " +
       "CURRENT ROW) as cnt1 " +
       "from MyTable"
 
@@ -551,12 +546,12 @@ class WindowAggregateTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS $1")
       )
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
index 36517498d7817..4a6a616dc6b42 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
@@ -57,13 +57,13 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
     StreamITCase.testResults = mutable.MutableList()
 
     val stream = env.fromCollection(data)
-    val table = stream.toTable(tEnv, 'long, 'int, 'string)
+    val table = stream.toTable(tEnv, 'long, 'int, 'string, 'proctime.proctime)
 
     val countFun = new CountAggFunction
     val weightAvgFun = new WeightedAvg
 
     val windowedTable = table
-      .window(Slide over 2.rows every 1.rows as 'w)
+      .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
       .groupBy('w, 'string)
       .select('string, countFun('int), 'int.avg,
               weightAvgFun('long, 'int), weightAvgFun('int, 'int))
@@ -102,7 +102,7 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
     val stream = env
       .fromCollection(sessionWindowTestdata)
       .assignTimestampsAndWatermarks(new TimestampAndWatermarkWithOffset(10L))
-    val table = stream.toTable(tEnv, 'long, 'int, 'string)
+    val table = stream.toTable(tEnv, 'long, 'int, 'string, 'rowtime.rowtime)
 
     val windowedTable = table
       .window(Session withGap 5.milli on 'rowtime as 'w)
@@ -126,12 +126,12 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
     StreamITCase.testResults = mutable.MutableList()
 
     val stream = env.fromCollection(data)
-    val table = stream.toTable(tEnv, 'long, 'int, 'string)
+    val table = stream.toTable(tEnv, 'long, 'int, 'string, 'proctime.proctime)
     val countFun = new CountAggFunction
     val weightAvgFun = new WeightedAvg
 
     val windowedTable = table
-      .window(Tumble over 2.rows as 'w)
+      .window(Tumble over 2.rows on 'proctime as 'w)
       .groupBy('w)
       .select(countFun('string), 'int.avg,
               weightAvgFun('long, 'int), weightAvgFun('int, 'int))
@@ -154,7 +154,7 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
     val stream = env
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampAndWatermarkWithOffset(0L))
-    val table = stream.toTable(tEnv, 'long, 'int, 'string)
+    val table = stream.toTable(tEnv, 'long, 'int, 'string, 'rowtime.rowtime)
     val countFun = new CountAggFunction
     val weightAvgFun = new WeightedAvg
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/CalcITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/CalcITCase.scala
index 5969e911f0749..1114cf06472a2 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/CalcITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/CalcITCase.scala
@@ -110,22 +110,6 @@ class CalcITCase extends StreamingMultipleProgramsTestBase {
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
-  @Test(expected = classOf[TableException])
-  def testAsWithToFewFields(): Unit = {
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
-    val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b)
-
-    val results = ds.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = mutable.MutableList("no")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
   @Test(expected = classOf[TableException])
   def testAsWithToManyFields(): Unit = {
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index 084ee1457e559..de6cbfac48564 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -20,9 +20,9 @@ package org.apache.flink.table.api.scala.stream.table
 
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
+import org.apache.flink.table.api.ValidationException
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.{TableException, ValidationException}
-import org.apache.flink.table.expressions.{RowtimeAttribute, WindowReference}
+import org.apache.flink.table.expressions.WindowReference
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.utils.TableTestBase
 import org.apache.flink.table.utils.TableTestUtil.{streamTableNode, term, unaryNode}
@@ -40,46 +40,10 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'string.start) // property in non windowed table
   }
 
-  @Test(expected = classOf[TableException])
-  def testInvalidRowtime1(): Unit = {
-    val util = streamTestUtil()
-    // rowtime attribute must not be a field name
-    util.addTable[(Long, Int, String)]('rowtime, 'long, 'int, 'string)
-  }
-
-  @Test(expected = classOf[ValidationException])
-  def testInvalidRowtime2(): Unit = {
-    val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-      .select('string, 'int as 'rowtime) // rowtime attribute must not be an alias
-  }
-
-  @Test(expected = classOf[ValidationException])
-  def testInvalidRowtime3(): Unit = {
-    val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table.as('rowtime, 'myint, 'mystring) // rowtime attribute must not be an alias
-  }
-
-  @Test(expected = classOf[ValidationException])
-  def testInvalidRowtime4(): Unit = {
-    val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    table
-    // only rowtime is a valid time attribute in a stream environment
-      .window(Tumble over 50.milli on 'string as 'w)
-      .groupBy('w, 'string)
-      .select('string, 'int.count)
-  }
-
   @Test(expected = classOf[ValidationException])
   def testGroupByWithoutWindowAlias(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
       .window(Tumble over 5.milli on 'long as 'w)
@@ -90,7 +54,7 @@ class GroupWindowTest extends TableTestBase {
   @Test(expected = classOf[ValidationException])
   def testInvalidRowTimeRef(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
       .window(Tumble over 5.milli on 'long as 'w)
@@ -104,10 +68,10 @@ class GroupWindowTest extends TableTestBase {
   @Test(expected = classOf[ValidationException])
   def testInvalidTumblingSize(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
-      .window(Tumble over "WRONG" as 'w) // string is not a valid interval
+      .window(Tumble over "WRONG" on 'long as 'w) // string is not a valid interval
       .groupBy('w, 'string)
       .select('string, 'int.count)
   }
@@ -127,10 +91,10 @@ class GroupWindowTest extends TableTestBase {
   @Test(expected = classOf[ValidationException])
   def testInvalidSlidingSize(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
-      .window(Slide over "WRONG" every "WRONG" as 'w) // string is not a valid interval
+      .window(Slide over "WRONG" every "WRONG" on 'long as 'w) // string is not a valid interval
       .groupBy('w, 'string)
       .select('string, 'int.count)
   }
@@ -138,10 +102,11 @@ class GroupWindowTest extends TableTestBase {
   @Test(expected = classOf[ValidationException])
   def testInvalidSlidingSlide(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
-      .window(Slide over 12.rows every 1.minute as 'w) // row and time intervals may not be mixed
+      // row and time intervals may not be mixed
+      .window(Slide over 12.rows every 1.minute on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
   }
@@ -161,10 +126,11 @@ class GroupWindowTest extends TableTestBase {
   @Test(expected = classOf[ValidationException])
   def testInvalidSessionGap(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
-      .window(Session withGap 10.rows as 'w) // row interval is not valid for session windows
+      // row interval is not valid for session windows
+      .window(Session withGap 10.rows on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
   }
@@ -172,10 +138,10 @@ class GroupWindowTest extends TableTestBase {
   @Test(expected = classOf[ValidationException])
   def testInvalidWindowAlias1(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
-      .window(Session withGap 100.milli as 1 + 1) // expression instead of a symbol
+      .window(Session withGap 100.milli on 'long as 1 + 1) // expression instead of a symbol
       .groupBy('string)
       .select('string, 'int.count)
   }
@@ -183,10 +149,11 @@ class GroupWindowTest extends TableTestBase {
   @Test(expected = classOf[ValidationException])
   def testInvalidWindowAlias2(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     table
-      .window(Session withGap 100.milli as 'string) // field name "string" is already present
+      // field name "string" is already present
+      .window(Session withGap 100.milli on 'long as 'string)
       .groupBy('string)
       .select('string, 'int.count)
   }
@@ -195,7 +162,7 @@ class GroupWindowTest extends TableTestBase {
   def testSessionUdAggWithInvalidArgs(): Unit = {
     val util = streamTestUtil()
     val weightedAvg = new WeightedAvgWithMerge
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
 
     table
       .window(Session withGap 2.hours on 'rowtime as 'w)
@@ -203,16 +170,17 @@ class GroupWindowTest extends TableTestBase {
       .select('string, weightedAvg('string, 'int)) // invalid UDAGG args
   }
 
+  @Ignore // TODO
   @Test
   def testMultiWindow(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Tumble over 50.milli as 'w1)
+      .window(Tumble over 50.milli on 'proctime as 'w1)
       .groupBy('w1, 'string)
-      .select('string, 'int.count)
-      .window(Slide over 20.milli every 10.milli as 'w2)
+      .select('w.end as 'proctime, 'string, 'int.count)
+      .window(Slide over 20.milli every 10.milli on 'proctime as 'w2)
       .groupBy('w2)
       .select('string.count)
 
@@ -230,8 +198,9 @@ class GroupWindowTest extends TableTestBase {
           term("groupBy", "string"),
           term(
             "window",
-            ProcessingTimeTumblingGroupWindow(
+            TumblingGroupWindow(
               WindowReference("w1"),
+              'proctime,
               50.milli)),
           term("select", "string", "COUNT(int) AS TMP_0")
         ),
@@ -239,9 +208,11 @@ class GroupWindowTest extends TableTestBase {
       ),
       term(
         "window",
-        ProcessingTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w2"),
-          20.milli, 10.milli)),
+          'proctime,
+          20.milli,
+          10.milli)),
       term("select", "COUNT(string) AS TMP_1")
     )
     util.verifyTable(windowedTable, expected)
@@ -250,10 +221,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testProcessingTimeTumblingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Tumble over 50.milli as 'w)
+      .window(Tumble over 50.milli on 'proctime as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
@@ -262,13 +233,14 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "string", "int")
+        term("select", "string", "int", "proctime")
       ),
       term("groupBy", "string"),
       term(
         "window",
-        ProcessingTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
+          'proctime,
           50.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -279,38 +251,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testProcessingTimeTumblingGroupWindowOverCount(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    val windowedTable = table
-      .window(Tumble over 2.rows as 'w)
-      .groupBy('w, 'string)
-      .select('string, 'int.count)
-
-    val expected = unaryNode(
-      "DataStreamAggregate",
-      unaryNode(
-        "DataStreamCalc",
-        streamTableNode(0),
-        term("select", "string", "int")
-      ),
-      term("groupBy", "string"),
-      term(
-        "window",
-        ProcessingTimeTumblingGroupWindow(
-          WindowReference("w"), 2.rows)),
-      term("select", "string", "COUNT(int) AS TMP_0")
-    )
-
-    util.verifyTable(windowedTable, expected)
-  }
-
-  @Test
-  def testEventTimeTumblingGroupWindowOverTime(): Unit = {
-    val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Tumble over 5.milli on 'rowtime as 'w)
+      .window(Tumble over 2.rows on 'proctime as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
@@ -319,15 +263,15 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "string", "int")
+        term("select", "string", "int", "proctime")
       ),
       term("groupBy", "string"),
       term(
         "window",
-        EventTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(),
-          5.milli)),
+          'proctime,
+          2.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -335,13 +279,12 @@ class GroupWindowTest extends TableTestBase {
   }
 
   @Test
-  @Ignore // see comments in DataStreamAggregate
-  def testEventTimeTumblingGroupWindowOverCount(): Unit = {
+  def testEventTimeTumblingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Tumble over 2.rows on 'rowtime as 'w)
+      .window(Tumble over 5.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
@@ -351,9 +294,10 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term(
         "window",
-        EventTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(), 2.rows)),
+          'long,
+          5.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -363,7 +307,7 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeTumblingGroupWindowWithUdAgg(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
 
     val weightedAvg = new WeightedAvgWithMerge
 
@@ -378,9 +322,9 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term(
         "window",
-        EventTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(),
+          'rowtime,
           5.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
@@ -391,10 +335,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testProcessingTimeSlidingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Slide over 50.milli every 50.milli as 'w)
+      .window(Slide over 50.milli every 50.milli on 'proctime as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
@@ -403,14 +347,16 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "string", "int")
+        term("select", "string", "int", "proctime")
       ),
       term("groupBy", "string"),
       term(
         "window",
-        ProcessingTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w"),
-          50.milli, 50.milli)),
+          'proctime,
+          50.milli,
+          50.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -420,10 +366,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testProcessingTimeSlidingGroupWindowOverCount(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Slide over 2.rows every 1.rows as 'w)
+      .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
@@ -432,14 +378,16 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "string", "int")
+        term("select", "string", "int", "proctime")
       ),
       term("groupBy", "string"),
       term(
         "window",
-        ProcessingTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w"),
-          2.rows, 1.rows)),
+          'proctime,
+          2.rows,
+          1.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -449,40 +397,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeSlidingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    val windowedTable = table
-      .window(Slide over 8.milli every 10.milli on 'rowtime as 'w)
-      .groupBy('w, 'string)
-      .select('string, 'int.count)
-
-    val expected = unaryNode(
-      "DataStreamAggregate",
-      unaryNode(
-        "DataStreamCalc",
-        streamTableNode(0),
-        term("select", "string", "int")
-      ),
-      term("groupBy", "string"),
-      term(
-        "window",
-        EventTimeSlidingGroupWindow(
-          WindowReference("w"),
-          RowtimeAttribute(), 8.milli, 10.milli)),
-      term("select", "string", "COUNT(int) AS TMP_0")
-    )
-
-    util.verifyTable(windowedTable, expected)
-  }
-
-  @Test
-  @Ignore // see comments in DataStreamAggregate
-  def testEventTimeSlidingGroupWindowOverCount(): Unit = {
-    val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Slide over 2.rows every 1.rows on 'rowtime as 'w)
+      .window(Slide over 8.milli every 10.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
@@ -492,9 +410,11 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term(
         "window",
-        EventTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(), 2.rows, 1.rows)),
+          'long,
+          8.milli,
+          10.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -504,7 +424,7 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeSlidingGroupWindowWithUdAgg(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
 
     val weightedAvg = new WeightedAvgWithMerge
 
@@ -519,9 +439,7 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term(
         "window",
-        EventTimeSlidingGroupWindow(
-          WindowReference("w"),
-          RowtimeAttribute(), 8.milli, 10.milli)),
+        SlidingGroupWindow(WindowReference("w"), 'rowtime, 8.milli, 10.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
 
@@ -531,26 +449,23 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeSessionGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Session withGap 7.milli on 'rowtime as 'w)
+      .window(Session withGap 7.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
     val expected = unaryNode(
       "DataStreamAggregate",
-      unaryNode(
-        "DataStreamCalc",
-        streamTableNode(0),
-        term("select", "string", "int")
-      ),
+      streamTableNode(0),
       term("groupBy", "string"),
       term(
         "window",
-        EventTimeSessionGroupWindow(
+        SessionGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(), 7.milli)),
+          'long,
+          7.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -560,7 +475,7 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeSessionGroupWindowWithUdAgg(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
 
     val weightedAvg = new WeightedAvgWithMerge
 
@@ -575,9 +490,7 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term(
         "window",
-        EventTimeSessionGroupWindow(
-          WindowReference("w"),
-          RowtimeAttribute(), 7.milli)),
+        SessionGroupWindow(WindowReference("w"), 'rowtime, 7.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
 
@@ -587,10 +500,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testAllProcessingTimeTumblingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Tumble over 50.milli as 'w)
+      .window(Tumble over 50.milli on 'proctime as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count)
 
@@ -599,13 +512,14 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "string", "int")
+        term("select", "string", "int", "proctime")
       ),
       term("groupBy", "string"),
       term(
         "window",
-        ProcessingTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
+          'proctime,
           50.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
@@ -616,10 +530,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testAllProcessingTimeTumblingGroupWindowOverCount(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Tumble over 2.rows as 'w)
+      .window(Tumble over 2.rows on 'proctime as 'w)
       .groupBy('w)
       .select('int.count)
 
@@ -628,12 +542,13 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "int")
+        term("select", "int", "proctime")
       ),
       term(
         "window",
-        ProcessingTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
+          'proctime,
           2.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
@@ -644,39 +559,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testAllEventTimeTumblingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    val windowedTable = table
-      .window(Tumble over 5.milli on 'rowtime as 'w)
-      .groupBy('w)
-      .select('int.count)
-
-    val expected = unaryNode(
-      "DataStreamAggregate",
-      unaryNode(
-        "DataStreamCalc",
-        streamTableNode(0),
-        term("select", "int")
-      ),
-      term(
-        "window",
-        EventTimeTumblingGroupWindow(
-          WindowReference("w"),
-          RowtimeAttribute(), 5.milli)),
-      term("select", "COUNT(int) AS TMP_0")
-    )
-
-    util.verifyTable(windowedTable, expected)
-  }
-
-  @Test
-  @Ignore // see comments in DataStreamAggregate
-  def testAllEventTimeTumblingGroupWindowOverCount(): Unit = {
-    val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Tumble over 2.rows on 'rowtime as 'w)
+      .window(Tumble over 5.milli on 'long as 'w)
       .groupBy('w)
       .select('int.count)
 
@@ -685,27 +571,27 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "int")
+        term("select", "int", "long")
       ),
       term(
         "window",
-        EventTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(), 2.rows)),
+          'long,
+          5.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
     util.verifyTable(windowedTable, expected)
   }
 
-
   @Test
   def testAllProcessingTimeSlidingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Slide over 50.milli every 50.milli as 'w)
+      .window(Slide over 50.milli every 50.milli on 'proctime as 'w)
       .groupBy('w)
       .select('int.count)
 
@@ -714,13 +600,15 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "int")
+        term("select", "int", "proctime")
       ),
       term(
         "window",
-        ProcessingTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w"),
-          50.milli, 50.milli)),
+          'proctime,
+          50.milli,
+          50.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -730,10 +618,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testAllProcessingTimeSlidingGroupWindowOverCount(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'proctime.proctime)
 
     val windowedTable = table
-      .window(Slide over 2.rows every 1.rows as 'w)
+      .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
       .groupBy('w)
       .select('int.count)
 
@@ -742,13 +630,15 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "int")
+        term("select", "int", "proctime")
       ),
       term(
         "window",
-        ProcessingTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w"),
-          2.rows, 1.rows)),
+          'proctime,
+          2.rows,
+          1.rows)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -758,39 +648,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testAllEventTimeSlidingGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
-
-    val windowedTable = table
-      .window(Slide over 8.milli every 10.milli on 'rowtime as 'w)
-      .groupBy('w)
-      .select('int.count)
-
-    val expected = unaryNode(
-      "DataStreamAggregate",
-      unaryNode(
-        "DataStreamCalc",
-        streamTableNode(0),
-        term("select", "int")
-      ),
-      term(
-        "window",
-        EventTimeSlidingGroupWindow(
-          WindowReference("w"),
-          RowtimeAttribute(), 8.milli, 10.milli)),
-      term("select", "COUNT(int) AS TMP_0")
-    )
-
-    util.verifyTable(windowedTable, expected)
-  }
-
-  @Test
-  @Ignore // see comments in DataStreamAggregate
-  def testAllEventTimeSlidingGroupWindowOverCount(): Unit = {
-    val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Slide over 2.rows every 1.rows on 'rowtime as 'w)
+      .window(Slide over 8.milli every 10.milli on 'long as 'w)
       .groupBy('w)
       .select('int.count)
 
@@ -799,13 +660,15 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "int")
+        term("select", "int", "long")
       ),
       term(
         "window",
-        EventTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(), 2.rows, 1.rows)),
+          'long,
+          8.milli,
+          10.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -815,10 +678,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testAllEventTimeSessionGroupWindowOverTime(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Session withGap 7.milli on 'rowtime as 'w)
+      .window(Session withGap 7.milli on 'long as 'w)
       .groupBy('w)
       .select('int.count)
 
@@ -827,13 +690,14 @@ class GroupWindowTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
-        term("select", "int")
+        term("select", "int", "long")
       ),
       term(
         "window",
-        EventTimeSessionGroupWindow(
+        SessionGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(), 7.milli)),
+          'long,
+          7.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -843,25 +707,21 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testTumbleWindowStartEnd(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Tumble over 5.milli on 'rowtime as 'w)
+      .window(Tumble over 5.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
     val expected = unaryNode(
       "DataStreamAggregate",
-      unaryNode(
-        "DataStreamCalc",
-        streamTableNode(0),
-        term("select", "string", "int")
-      ),
+      streamTableNode(0),
       term("groupBy", "string"),
       term("window",
-        EventTimeTumblingGroupWindow(
+        TumblingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(),
+          'long,
           5.milli)),
       term("select",
         "string",
@@ -876,25 +736,21 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testSlideWindowStartEnd(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Slide over 10.milli every 5.milli on 'rowtime as 'w)
+      .window(Slide over 10.milli every 5.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
     val expected = unaryNode(
       "DataStreamAggregate",
-      unaryNode(
-        "DataStreamCalc",
-        streamTableNode(0),
-        term("select", "string", "int")
-      ),
+      streamTableNode(0),
       term("groupBy", "string"),
       term("window",
-        EventTimeSlidingGroupWindow(
+        SlidingGroupWindow(
           WindowReference("w"),
-          RowtimeAttribute(),
+          'long,
           10.milli,
           5.milli)),
       term("select",
@@ -910,10 +766,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testSessionWindowStartWithTwoEnd(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Session withGap 3.milli on 'rowtime as 'w)
+      .window(Session withGap 3.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('w.end as 'we1, 'string, 'int.count as 'cnt, 'w.start as 'ws, 'w.end as 'we2)
 
@@ -921,16 +777,12 @@ class GroupWindowTest extends TableTestBase {
       "DataStreamCalc",
       unaryNode(
         "DataStreamAggregate",
-        unaryNode(
-          "DataStreamCalc",
-          streamTableNode(0),
-          term("select", "string", "int")
-        ),
+        streamTableNode(0),
         term("groupBy", "string"),
         term("window",
-          EventTimeSessionGroupWindow(
+          SessionGroupWindow(
             WindowReference("w"),
-            RowtimeAttribute(),
+            'long,
             3.milli)),
         term("select",
           "string",
@@ -947,10 +799,10 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testTumbleWindowWithDuplicateAggsAndProps(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
     val windowedTable = table
-      .window(Tumble over 5.millis on 'rowtime as 'w)
+      .window(Tumble over 5.millis on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.sum + 1 as 's1, 'int.sum + 3 as 's2, 'w.start as 'x, 'w.start as 'x2,
         'w.end as 'x3, 'w.end)
@@ -959,16 +811,12 @@ class GroupWindowTest extends TableTestBase {
       "DataStreamCalc",
       unaryNode(
         "DataStreamAggregate",
-        unaryNode(
-          "DataStreamCalc",
-          streamTableNode(0),
-          term("select", "string", "int")
-        ),
+        streamTableNode(0),
         term("groupBy", "string"),
         term("window",
-          EventTimeTumblingGroupWindow(
+          TumblingGroupWindow(
             WindowReference("w"),
-            RowtimeAttribute(),
+            'long,
             5.millis)),
         term("select",
           "string",
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
index 4c0fea7c3ddb6..b097767202165 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowITCase.scala
@@ -58,7 +58,7 @@ class OverWindowITCase extends StreamingWithStateTestBase {
     StreamITCase.testResults = mutable.MutableList()
     StreamITCase.clear
     val stream = env.fromCollection(data)
-    val table = stream.toTable(tEnv, 'a, 'b, 'c)
+    val table = stream.toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
     val countFun = new CountAggFunction
     val weightAvgFun = new WeightedAvg
 
@@ -107,7 +107,7 @@ class OverWindowITCase extends StreamingWithStateTestBase {
     )
     val table = env
       .addSource(new RowTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv).as('a, 'b, 'c)
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
     val countFun = new CountAggFunction
     val weightAvgFun = new WeightedAvg
 
@@ -173,7 +173,7 @@ class OverWindowITCase extends StreamingWithStateTestBase {
     StreamITCase.testResults = mutable.MutableList()
 
     val stream = env.fromCollection(data)
-    val table = stream.toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val table = stream.toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
 
     val windowedTable = table
       .window(Over partitionBy 'a orderBy 'proctime preceding 4.rows following CURRENT_ROW as 'w)
@@ -234,7 +234,8 @@ class OverWindowITCase extends StreamingWithStateTestBase {
     StreamITCase.clear
 
     val table = env.addSource[(Long, Int, String)](
-      new RowTimeSourceFunction[(Long, Int, String)](data)).toTable(tEnv).as('a, 'b, 'c)
+      new RowTimeSourceFunction[(Long, Int, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     val windowedTable = table
       .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
@@ -295,7 +296,8 @@ class OverWindowITCase extends StreamingWithStateTestBase {
     StreamITCase.clear
 
     val table = env.addSource[(Long, Int, String)](
-      new RowTimeSourceFunction[(Long, Int, String)](data)).toTable(tEnv).as('a, 'b, 'c)
+      new RowTimeSourceFunction[(Long, Int, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
 
     val windowedTable = table
       .window(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
index 7dea521462725..ea3ab221a60e2 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
@@ -18,7 +18,7 @@
 package org.apache.flink.table.api.scala.stream.table
 
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.{WeightedAvgWithRetract}
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithRetract
 import org.apache.flink.table.api.{Table, ValidationException}
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.utils.TableTestUtil._
@@ -27,7 +27,8 @@ import org.junit.Test
 
 class OverWindowTest extends TableTestBase {
   private val streamUtil: StreamTableTestUtil = streamTestUtil()
-  val table: Table = streamUtil.addTable[(Int, String, Long)]("MyTable", 'a, 'b, 'c)
+  val table: Table = streamUtil.addTable[(Int, String, Long)]("MyTable",
+    'a, 'b, 'c, 'proctime.proctime, 'rowtime.rowtime)
 
   @Test(expected = classOf[ValidationException])
   def testInvalidWindowAlias(): Unit = {
@@ -121,12 +122,12 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "b", "c", "PROCTIME() AS $3")
+            term("select", "a", "b", "c", "proctime")
           ),
           term("partitionBy", "b"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "b", "c", "PROCTIME", "WeightedAvgWithRetract(c, a) AS w0$o0")
+          term("select", "a", "b", "c", "proctime", "WeightedAvgWithRetract(c, a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS _c1")
       )
@@ -150,16 +151,16 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
           term("partitionBy", "a"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
           term(
             "select",
             "a",
             "c",
-            "PROCTIME",
+            "proctime",
             "WeightedAvgWithRetract(c, a) AS w0$o0"
           )
         ),
@@ -183,11 +184,11 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN 10000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(c) AS w0$o0")
+          term("select", "a", "c", "proctime", "COUNT(c) AS w0$o0")
         ),
         term("select", "a", "w0$o0 AS _c1")
       )
@@ -209,11 +210,11 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS _c1")
       )
@@ -238,16 +239,16 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
           term(
             "select",
             "a",
             "c",
-            "PROCTIME",
+            "proctime",
             "COUNT(a) AS w0$o0",
             "WeightedAvgWithRetract(c, a) AS w0$o1"
           )
@@ -280,12 +281,12 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME",
+          term("select", "a", "c", "proctime",
                "COUNT(a) AS w0$o0",
                "WeightedAvgWithRetract(c, a) AS w0$o1")
         ),
@@ -310,15 +311,15 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
           term(
             "select",
             "a",
             "c",
-            "PROCTIME",
+            "proctime",
             "COUNT(a) AS w0$o0",
             "SUM(a) AS w0$o1"
           )
@@ -349,11 +350,11 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "PROCTIME() AS $2")
+            term("select", "a", "c", "proctime")
           ),
-          term("orderBy", "PROCTIME"),
+          term("orderBy", "proctime"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "PROCTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS _c1")
       )
@@ -378,12 +379,12 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "b", "c", "ROWTIME() AS $3")
+            term("select", "a", "b", "c", "rowtime")
           ),
           term("partitionBy", "b"),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "b", "c", "ROWTIME",
+          term("select", "a", "b", "c", "rowtime",
                "COUNT(b) AS w0$o0",
                "WeightedAvgWithRetract(c, a) AS w0$o1")
         ),
@@ -410,16 +411,16 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
           term("partitionBy", "a"),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
           term(
             "select",
             "a",
             "c",
-            "ROWTIME",
+            "rowtime",
             "AVG(c) AS w0$o0",
             "WeightedAvgWithRetract(c, a) AS w0$o1"
           )
@@ -444,11 +445,11 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN 10000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(c) AS w0$o0")
+          term("select", "a", "c", "rowtime", "COUNT(c) AS w0$o0")
         ),
         term("select", "a", "w0$o0 AS _c1")
       )
@@ -470,11 +471,11 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS _c1")
       )
@@ -499,16 +500,16 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
           term(
             "select",
             "a",
             "c",
-            "ROWTIME",
+            "rowtime",
             "COUNT(a) AS w0$o0",
             "WeightedAvgWithRetract(c, a) AS w0$o1"
           )
@@ -542,12 +543,12 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
           term("partitionBy", "c"),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME",
+          term("select", "a", "c", "rowtime",
                "COUNT(a) AS w0$o0",
                "WeightedAvgWithRetract(c, a) AS w0$o1")
         ),
@@ -572,15 +573,15 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
           term(
             "select",
             "a",
             "c",
-            "ROWTIME",
+            "rowtime",
             "COUNT(a) AS w0$o0",
             "SUM(a) AS w0$o1"
           )
@@ -611,11 +612,11 @@ class OverWindowTest extends TableTestBase {
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
-            term("select", "a", "c", "ROWTIME() AS $2")
+            term("select", "a", "c", "rowtime")
           ),
-          term("orderBy", "ROWTIME"),
+          term("orderBy", "rowtime"),
           term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "ROWTIME", "COUNT(a) AS w0$o0")
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
         ),
         term("select", "c", "w0$o0 AS _c1")
       )
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala
index d314c9ae6b022..d261e36104373 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/GroupWindowStringExpressionTest.scala
@@ -31,7 +31,7 @@ class GroupWindowStringExpressionTest extends TableTestBase {
   @Test
   def testJavaScalaTableAPIEquality(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Int, Long, String)]('int, 'long, 'string)
+    val t = util.addTable[(Int, Long, String)]('int, 'long, 'string, 'rowtime.rowtime)
 
     val myCountFun = new CountAggFunction
     util.tEnv.registerFunction("myCountFun", myCountFun)
@@ -40,7 +40,7 @@ class GroupWindowStringExpressionTest extends TableTestBase {
 
     // Expression / Scala API
     val resScala = t
-      .window(Slide over 4.rows every 2.rows as 'w)
+      .window(Slide over 4.hours every 2.hours on 'rowtime as 'w)
       .groupBy('w, 'string)
       .select(
         'string,
@@ -51,7 +51,7 @@ class GroupWindowStringExpressionTest extends TableTestBase {
 
     // String / Java API
     val resJava = t
-      .window(JSlide.over("4.rows").every("2.rows").as("w"))
+      .window(JSlide.over("4.hours").every("2.hours").on("rowtime").as("w"))
       .groupBy("w, string")
       .select(
         "string, " +
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
index 0a5e0019b1101..04016f1cce9d6 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
@@ -29,7 +29,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testPartitionedUnboundedOverRow(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_ROW as 'w)
@@ -44,7 +44,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testUnboundedOverRow(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver orderBy 'rowtime preceding UNBOUNDED_ROW following CURRENT_ROW as 'w)
@@ -59,7 +59,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testPartitionedBoundedOverRow(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver partitionBy('a, 'd) orderBy 'rowtime preceding 10.rows as 'w)
@@ -74,7 +74,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testBoundedOverRow(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver orderBy 'rowtime preceding 10.rows following CURRENT_ROW as 'w)
@@ -89,7 +89,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testPartitionedUnboundedOverRange(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE as 'w)
@@ -104,7 +104,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testUnboundedOverRange(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver orderBy 'rowtime preceding UNBOUNDED_RANGE following CURRENT_RANGE as 'w)
@@ -120,7 +120,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testPartitionedBoundedOverRange(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver partitionBy('a, 'c) orderBy 'rowtime preceding 10.minutes as 'w)
@@ -135,7 +135,7 @@ class OverWindowStringExpressionTest extends TableTestBase {
   @Test
   def testBoundedOverRange(): Unit = {
     val util = streamTestUtil()
-    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+    val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rowtime.rowtime)
 
     val resScala = t
       .window(SOver orderBy 'rowtime preceding 4.hours following CURRENT_RANGE as 'w)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala
index dcd3c6cfce1d5..05e189255abf8 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamAggregateITCase.scala
@@ -62,10 +62,10 @@ class DataStreamAggregateITCase extends StreamingMultipleProgramsTestBase {
     val stream = env
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
-    val table = stream.toTable(tEnv, 'long, 'int, 'double, 'float, 'bigdec, 'string)
+    val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
 
     val windowedTable = table
-      .window(Slide over 5.milli every 2.milli on 'rowtime as 'w)
+      .window(Slide over 5.milli every 2.milli on 'long as 'w)
       .groupBy('w)
       .select('int.count, 'w.start, 'w.end)
 
@@ -97,10 +97,10 @@ class DataStreamAggregateITCase extends StreamingMultipleProgramsTestBase {
     val stream = env
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
-    val table = stream.toTable(tEnv, 'long, 'int, 'double, 'float, 'bigdec, 'string)
+    val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
 
     val windowedTable = table
-      .window(Slide over 10.milli every 5.milli on 'rowtime as 'w)
+      .window(Slide over 10.milli every 5.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
@@ -134,10 +134,10 @@ class DataStreamAggregateITCase extends StreamingMultipleProgramsTestBase {
     val stream = env
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
-    val table = stream.toTable(tEnv, 'long, 'int, 'double, 'float, 'bigdec, 'string)
+    val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
 
     val windowedTable = table
-      .window(Slide over 5.milli every 4.milli on 'rowtime as 'w)
+      .window(Slide over 5.milli every 4.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
@@ -168,10 +168,10 @@ class DataStreamAggregateITCase extends StreamingMultipleProgramsTestBase {
     val stream = env
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
-    val table = stream.toTable(tEnv, 'long, 'int, 'double, 'float, 'bigdec, 'string)
+    val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
 
     val windowedTable = table
-      .window(Slide over 5.milli every 10.milli on 'rowtime as 'w)
+      .window(Slide over 5.milli every 10.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
@@ -197,10 +197,10 @@ class DataStreamAggregateITCase extends StreamingMultipleProgramsTestBase {
     val stream = env
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
-    val table = stream.toTable(tEnv, 'long, 'int, 'double, 'float, 'bigdec, 'string)
+    val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
 
     val windowedTable = table
-      .window(Slide over 3.milli every 10.milli on 'rowtime as 'w)
+      .window(Slide over 3.milli every 10.milli on 'long as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
@@ -225,7 +225,7 @@ class DataStreamAggregateITCase extends StreamingMultipleProgramsTestBase {
       .fromCollection(data)
       .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
       .map(t => (t._2, t._6))
-    val table = stream.toTable(tEnv, 'int, 'string)
+    val table = stream.toTable(tEnv, 'int, 'string, 'rowtime.rowtime)
 
     val windowedTable = table
       .window(Slide over 3.milli every 10.milli on 'rowtime as 'w)

From 4b610681edbdb5bde7977cac79a59ed459d55f2f Mon Sep 17 00:00:00 2001
From: Fabian Hueske <fhueske@apache.org>
Date: Thu, 4 May 2017 18:05:27 +0200
Subject: [PATCH 156/253] [FLINK-5884] [table] Integrate time indicators for
 Table API & SQL. Continued

---
 .../table/api/BatchTableEnvironment.scala     |  25 +--
 .../table/api/StreamTableEnvironment.scala    |  86 +++++++--
 .../flink/table/api/TableEnvironment.scala    |  64 +------
 .../table/calcite/FlinkTypeFactory.scala      |  17 +-
 .../table/expressions/ExpressionParser.scala  |  18 +-
 .../table/plan/logical/LogicalWindow.scala    |   2 +-
 .../table/plan/logical/groupWindows.scala     |  11 +-
 .../flink/table/plan/nodes/CommonCalc.scala   |  10 +-
 .../plan/nodes/PhysicalTableSourceScan.scala  |   2 +-
 .../plan/nodes/dataset/DataSetAggregate.scala |   2 +-
 .../plan/nodes/dataset/DataSetCalc.scala      |   5 +-
 .../datastream/DataStreamOverAggregate.scala  |   5 +-
 .../datastream/StreamTableSourceScan.scala    |  28 +--
 .../logical/FlinkLogicalTableSourceScan.scala |  32 +++-
 .../plan/schema/StreamTableSourceTable.scala  |  65 +++++++
 .../runtime/aggregate/AggregateUtil.scala     |   3 +-
 .../table/sources/DefinedTimeAttributes.scala |  47 +++--
 .../flink/table/TableEnvironmentTest.scala    |  52 ++----
 .../scala/batch/table/GroupWindowTest.scala   |   4 +-
 .../stream/StreamTableEnvironmentTest.scala   | 164 ++++++++++++++++++
 .../api/scala/stream/TableSourceTest.scala    | 154 ++++++++++++++++
 .../stream/sql/WindowAggregateTest.scala      |  59 +++----
 22 files changed, 628 insertions(+), 227 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index 3eb2ffc269826..02c6063f524c2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -31,7 +31,7 @@ import org.apache.flink.api.java.io.DiscardingOutputFormat
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.api.java.{DataSet, ExecutionEnvironment}
 import org.apache.flink.table.explain.PlanJsonParser
-import org.apache.flink.table.expressions.Expression
+import org.apache.flink.table.expressions.{Expression, RowtimeAttribute, TimeAttribute}
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.DataSetRel
 import org.apache.flink.table.plan.rules.FlinkRuleSets
@@ -196,26 +196,11 @@ abstract class BatchTableEnvironment(
 
     val (fieldNames, fieldIndexes) = getFieldInfo[T](
       dataSet.getType,
-      fields,
-      ignoreTimeAttributes = true)
+      fields)
 
-    // validate and extract time attributes
-    val (rowtime, proctime) = validateAndExtractTimeAttributes(fieldNames, fieldIndexes, fields)
-
-    // don't allow proctime on batch
-    proctime match {
-      case Some(_) =>
-        throw new ValidationException(
-          "A proctime attribute is not allowed in a batch environment. " +
-            "Working with processing-time on batch would lead to non-deterministic results.")
-      case _ => // ok
-    }
-    // rowtime must not extend the schema of a batch table
-    rowtime match {
-      case Some((idx, _)) if idx >= dataSet.getType.getArity =>
-        throw new ValidationException(
-          "A rowtime attribute must be defined on an existing field in a batch environment.")
-      case _ => // ok
+    if (fields.exists(_.isInstanceOf[TimeAttribute])) {
+      throw new ValidationException(
+        ".rowtime and .proctime time indicators are not allowed in a batch environment.")
     }
 
     val dataSetTable = new DataSetTable[T](dataSet, fieldIndexes, fieldNames)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index d1f2fb544da0b..dd2c09d5eab18 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -26,19 +26,21 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.sql2rel.RelDecorrelator
 import org.apache.calcite.tools.{RuleSet, RuleSets}
-import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
+import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.apache.flink.table.calcite.RelTimeIndicatorConverter
 import org.apache.flink.table.explain.PlanJsonParser
-import org.apache.flink.table.expressions.Expression
+import org.apache.flink.table.expressions.{Expression, ProctimeAttribute, RowtimeAttribute, UnresolvedFieldReference}
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamRel
 import org.apache.flink.table.plan.rules.FlinkRuleSets
-import org.apache.flink.table.plan.schema.{DataStreamTable, TableSourceTable}
+import org.apache.flink.table.plan.schema.{DataStreamTable, StreamTableSourceTable, TableSourceTable}
 import org.apache.flink.table.sinks.{StreamTableSink, TableSink}
 import org.apache.flink.table.sources.{StreamTableSource, TableSource}
+import org.apache.flink.table.typeutils.TypeCheckUtils
 import org.apache.flink.types.Row
 
 import _root_.scala.collection.JavaConverters._
@@ -99,7 +101,7 @@ abstract class StreamTableEnvironment(
 
     tableSource match {
       case streamTableSource: StreamTableSource[_] =>
-        registerTableInternal(name, new TableSourceTable(streamTableSource))
+        registerTableInternal(name, new StreamTableSourceTable(streamTableSource))
       case _ =>
         throw new TableException("Only StreamTableSource can be registered in " +
             "StreamTableEnvironment")
@@ -168,14 +170,13 @@ abstract class StreamTableEnvironment(
       fields: Array[Expression])
     : Unit = {
 
-    val (fieldNames, fieldIndexes) = getFieldInfo[T](
-      dataStream.getType,
-      fields,
-      ignoreTimeAttributes = false)
+    val streamType = dataStream.getType
 
-    // validate and extract time attributes
-    val (rowtime, proctime) = validateAndExtractTimeAttributes(fieldNames, fieldIndexes, fields)
+    // get field names and types for all non-replaced fields
+    val (fieldNames, fieldIndexes) = getFieldInfo[T](streamType, fields)
 
+    // validate and extract time attributes
+    val (rowtime, proctime) = validateAndExtractTimeAttributes(streamType, fields)
 
     val dataStreamTable = new DataStreamTable[T](
       dataStream,
@@ -187,6 +188,71 @@ abstract class StreamTableEnvironment(
     registerTableInternal(name, dataStreamTable)
   }
 
+  /**
+    * Checks for at most one rowtime and proctime attribute.
+    * Returns the time attributes.
+    *
+    * @return rowtime attribute and proctime attribute
+    */
+  private def validateAndExtractTimeAttributes(
+    streamType: TypeInformation[_],
+    exprs: Array[Expression])
+  : (Option[(Int, String)], Option[(Int, String)]) = {
+
+    val fieldTypes: Array[TypeInformation[_]] = streamType match {
+      case c: CompositeType[_] => (0 until c.getArity).map(i => c.getTypeAt(i)).toArray
+      case a: AtomicType[_] => Array(a)
+    }
+
+    var fieldNames: List[String] = Nil
+    var rowtime: Option[(Int, String)] = None
+    var proctime: Option[(Int, String)] = None
+
+    exprs.zipWithIndex.foreach {
+      case (RowtimeAttribute(reference@UnresolvedFieldReference(name)), idx) =>
+        if (rowtime.isDefined) {
+          throw new TableException(
+            "The rowtime attribute can only be defined once in a table schema.")
+        } else {
+          // check type of field that is replaced
+          if (idx < fieldTypes.length &&
+            !(TypeCheckUtils.isLong(fieldTypes(idx)) ||
+              TypeCheckUtils.isTimePoint(fieldTypes(idx)))) {
+            throw new TableException(
+              "The rowtime attribute can only be replace a field with a valid time type, such as " +
+                "Timestamp or Long.")
+          }
+          rowtime = Some(idx, name)
+        }
+      case (ProctimeAttribute(reference@UnresolvedFieldReference(name)), idx) =>
+        if (proctime.isDefined) {
+          throw new TableException(
+            "The proctime attribute can only be defined once in a table schema.")
+        } else {
+          // check that proctime is only appended
+          if (idx < fieldTypes.length) {
+            throw new TableException(
+              "The proctime attribute can only be appended to the table schema and not replace " +
+                "an existing field. Please move it to the end of the schema.")
+          }
+          proctime = Some(idx, name)
+        }
+      case (u: UnresolvedFieldReference, _) => fieldNames = u.name :: fieldNames
+    }
+
+    if (rowtime.isDefined && fieldNames.contains(rowtime.get._2)) {
+      throw new TableException(
+        "The rowtime attribute may not have the same name as an another field.")
+    }
+
+    if (proctime.isDefined && fieldNames.contains(proctime.get._2)) {
+      throw new TableException(
+        "The proctime attribute may not have the same name as an another field.")
+    }
+
+    (rowtime, proctime)
+  }
+
   /**
     * Returns the decoration rule set for this environment
     * including a custom RuleSet configuration.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 4c72e8ff6d27d..9ed5000a379fc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -601,50 +601,36 @@ abstract class TableEnvironment(val config: TableConfig) {
 
   /**
     * Returns field names and field positions for a given [[TypeInformation]] and [[Array]] of
-    * [[Expression]]. It does not handle time attributes but considers them in indices, if
-    * ignore flag is not false.
+    * [[Expression]]. It does not handle time attributes but considers them in indices.
     *
     * @param inputType The [[TypeInformation]] against which the [[Expression]]s are evaluated.
-    * @param exprs The expressions that define the field names.
-    * @param ignoreTimeAttributes ignore time attributes and handle them as regular expressions.
+    * @param exprs     The expressions that define the field names.
     * @tparam A The type of the TypeInformation.
     * @return A tuple of two arrays holding the field names and corresponding field positions.
     */
   protected[flink] def getFieldInfo[A](
       inputType: TypeInformation[A],
-      exprs: Array[Expression],
-      ignoreTimeAttributes: Boolean)
+      exprs: Array[Expression])
     : (Array[String], Array[Int]) = {
 
     TableEnvironment.validateType(inputType)
 
-    val filteredExprs = if (ignoreTimeAttributes) {
-        exprs.map {
-          case ta: TimeAttribute => ta.expression
-          case e@_ => e
-        }
-    } else {
-      exprs
-    }
-
     val indexedNames: Array[(Int, String)] = inputType match {
       case g: GenericTypeInfo[A] if g.getTypeClass == classOf[Row] =>
         throw new TableException(
           "An input of GenericTypeInfo<Row> cannot be converted to Table. " +
             "Please specify the type of the input with a RowTypeInfo.")
       case a: AtomicType[A] =>
-        filteredExprs.zipWithIndex flatMap {
+        exprs.zipWithIndex flatMap {
           case (UnresolvedFieldReference(name), idx) =>
             if (idx > 0) {
               throw new TableException("Table of atomic type can only have a single field.")
             }
             Some((0, name))
-          case (_: TimeAttribute, _) if ignoreTimeAttributes =>
-            None
           case _ => throw new TableException("Field reference expression requested.")
         }
       case t: TupleTypeInfo[A] =>
-        filteredExprs.zipWithIndex flatMap {
+        exprs.zipWithIndex flatMap {
           case (UnresolvedFieldReference(name), idx) =>
             Some((idx, name))
           case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
@@ -659,7 +645,7 @@ abstract class TableEnvironment(val config: TableConfig) {
             "Field reference expression or alias on field expression expected.")
         }
       case c: CaseClassTypeInfo[A] =>
-        filteredExprs.zipWithIndex flatMap {
+        exprs.zipWithIndex flatMap {
           case (UnresolvedFieldReference(name), idx) =>
             Some((idx, name))
           case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
@@ -674,7 +660,7 @@ abstract class TableEnvironment(val config: TableConfig) {
             "Field reference expression or alias on field expression expected.")
         }
       case p: PojoTypeInfo[A] =>
-        filteredExprs flatMap {
+        exprs flatMap {
           case (UnresolvedFieldReference(name)) =>
             val idx = p.getFieldIndex(name)
             if (idx < 0) {
@@ -822,42 +808,6 @@ abstract class TableEnvironment(val config: TableConfig) {
     Some(mapFunction)
   }
 
-  /**
-    * Checks for at most one rowtime and proctime attribute.
-    * Returns the time attributes.
-    *
-    * @return rowtime attribute and proctime attribute
-    */
-  protected def validateAndExtractTimeAttributes(
-      fieldNames: Seq[String],
-      fieldIndices: Seq[Int],
-      exprs: Array[Expression])
-    : (Option[(Int, String)], Option[(Int, String)]) = {
-
-    var rowtime: Option[(Int, String)] = None
-    var proctime: Option[(Int, String)] = None
-
-    exprs.zipWithIndex.foreach {
-      case (RowtimeAttribute(reference@UnresolvedFieldReference(name)), idx) =>
-        if (rowtime.isDefined) {
-          throw new TableException(
-            "The rowtime attribute can only be defined once in a table schema.")
-        } else {
-          rowtime = Some(idx, name)
-        }
-      case (ProctimeAttribute(reference@UnresolvedFieldReference(name)), idx) =>
-        if (proctime.isDefined) {
-          throw new TableException(
-            "The proctime attribute can only be defined once in a table schema.")
-        } else {
-          proctime = Some(idx, name)
-        }
-      case _ =>
-        // do nothing
-    }
-
-    (rowtime, proctime)
-  }
 }
 
 /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
index 001011b866c80..9281ad863b045 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
@@ -65,11 +65,12 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
           createSqlIntervalType(
             new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.SECOND, SqlParserPos.ZERO))
 
-        case TimeIndicatorTypeInfo.ROWTIME_INDICATOR =>
-          createRowtimeIndicatorType()
-
-        case TimeIndicatorTypeInfo.PROCTIME_INDICATOR =>
-          createProctimeIndicatorType()
+        case TIMESTAMP if typeInfo.isInstanceOf[TimeIndicatorTypeInfo] =>
+          if (typeInfo.asInstanceOf[TimeIndicatorTypeInfo].isEventTime) {
+            createRowtimeIndicatorType()
+          } else {
+            createProctimeIndicatorType()
+          }
 
         case _ =>
           createSqlType(sqlType)
@@ -114,9 +115,11 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
     * @param fieldNames field names
     * @param fieldTypes field types, every element is Flink's [[TypeInformation]]
     * @param rowtime optional system field to indicate event-time; the index determines the index
-    *                in the final record and might replace an existing field
+    *                in the final record. If the index is smaller than the number of specified
+    *                fields, it shifts all following fields.
     * @param proctime optional system field to indicate processing-time; the index determines the
-    *                 index in the final record and might replace an existing field
+    *                 index in the final record. If the index is smaller than the number of
+    *                 specified fields, it shifts all following fields.
     * @return a struct type with the input fieldNames, input fieldTypes, and system fields
     */
   def buildLogicalRowType(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
index c33f8fc456059..98580baf357e6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
@@ -98,11 +98,14 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
   lazy val UNBOUNDED_ROW: Keyword = Keyword("unbounded_row")
   lazy val UNBOUNDED_RANGE: Keyword = Keyword("unbounded_range")
   lazy val ASIN: Keyword = Keyword("asin")
+  lazy val ROWTIME: Keyword = Keyword("rowtime")
+  lazy val PROCTIME: Keyword = Keyword("proctime")
 
   def functionIdent: ExpressionParser.Parser[String] =
     not(ARRAY) ~ not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~
       not(SUM0) ~ not(STDDEV_POP) ~ not(STDDEV_SAMP) ~ not(VAR_POP) ~ not(VAR_SAMP) ~
       not(SUM) ~ not(START) ~ not(END)~ not(CAST) ~ not(NULL) ~ not(IF) ~
+      not(ROWTIME) ~ not(PROCTIME) ~
       not(CURRENT_ROW) ~ not(UNBOUNDED_ROW) ~ not(CURRENT_RANGE) ~ not(UNBOUNDED_RANGE) ~>
       super.ident
 
@@ -532,12 +535,25 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
 
   // alias
 
-  lazy val alias: PackratParser[Expression] = logic ~ AS ~ fieldReference ^^ {
+  lazy val alias: PackratParser[Expression] = timeIndicator |
+    logic ~ AS ~ fieldReference ^^ {
       case e ~ _ ~ name => Alias(e, name.name)
   } | logic ~ AS ~ "(" ~ rep1sep(fieldReference, ",") ~ ")" ^^ {
     case e ~ _ ~ _ ~ names ~ _ => Alias(e, names.head.name, names.tail.map(_.name))
   } | logic
 
+  // time indicators
+
+  lazy val timeIndicator: PackratParser[Expression] = procTime | rowTime
+
+  lazy val procTime: PackratParser[Expression] = fieldReference ~ "." ~ PROCTIME ^^ {
+    case f ~ _ ~ _ => ProctimeAttribute(f)
+  }
+
+  lazy val rowTime: PackratParser[Expression] = fieldReference ~ "." ~ ROWTIME ^^ {
+    case f ~ _ ~ _ => RowtimeAttribute(f)
+  }
+
   lazy val expression: PackratParser[Expression] = alias |
     failure("Invalid expression.")
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
index 92dc50177bcc4..6161ef075d3f8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/LogicalWindow.scala
@@ -23,7 +23,7 @@ import org.apache.flink.table.expressions.{Expression, WindowReference}
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
 /**
-  * Logical super class for all types of windows (group-windows and row-windows).
+  * Logical super class for group windows.
   *
   * @param aliasAttribute window alias
   * @param timeAttribute time field indicating event-time or processing-time
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
index 3e5de283ab03d..4a8fb52da605b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/groupWindows.scala
@@ -21,7 +21,7 @@ package org.apache.flink.table.plan.logical
 import org.apache.flink.table.api.{BatchTableEnvironment, StreamTableEnvironment, TableEnvironment}
 import org.apache.flink.table.expressions.ExpressionUtils.{isRowCountLiteral, isRowtimeAttribute, isTimeAttribute, isTimeIntervalLiteral}
 import org.apache.flink.table.expressions._
-import org.apache.flink.table.typeutils.TypeCheckUtils.isTimePoint
+import org.apache.flink.table.typeutils.TypeCheckUtils.{isTimePoint, isLong}
 import org.apache.flink.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
 
 // ------------------------------------------------------------------------------------------------
@@ -56,7 +56,8 @@ case class TumblingGroupWindow(
         case _: StreamTableEnvironment if !isTimeAttribute(timeField) =>
           ValidationFailure(
             "Tumbling window expects a time attribute for grouping in a stream environment.")
-        case _: BatchTableEnvironment if isTimePoint(size.resultType) =>
+        case _: BatchTableEnvironment
+          if !(isTimePoint(timeField.resultType) || isLong(timeField.resultType)) =>
           ValidationFailure(
             "Tumbling window expects a time attribute for grouping in a stream environment.")
 
@@ -119,7 +120,8 @@ case class SlidingGroupWindow(
         case _: StreamTableEnvironment if !isTimeAttribute(timeField) =>
           ValidationFailure(
             "Sliding window expects a time attribute for grouping in a stream environment.")
-        case _: BatchTableEnvironment if isTimePoint(size.resultType) =>
+        case _: BatchTableEnvironment
+          if !(isTimePoint(timeField.resultType) || isLong(timeField.resultType)) =>
           ValidationFailure(
             "Sliding window expects a time attribute for grouping in a stream environment.")
 
@@ -169,7 +171,8 @@ case class SessionGroupWindow(
         case _: StreamTableEnvironment if !isTimeAttribute(timeField) =>
           ValidationFailure(
             "Session window expects a time attribute for grouping in a stream environment.")
-        case _: BatchTableEnvironment if isTimePoint(gap.resultType) =>
+        case _: BatchTableEnvironment
+          if !(isTimePoint(timeField.resultType) || isLong(timeField.resultType)) =>
           ValidationFailure(
             "Session window expects a time attribute for grouping in a stream environment.")
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
index 5c35129aa35e1..ff5ffb202be8f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
@@ -34,11 +34,11 @@ import scala.collection.JavaConverters._
 trait CommonCalc {
 
   private[flink] def functionBody(
-     generator: CodeGenerator,
-     inputSchema: RowSchema,
-     returnSchema: RowSchema,
-     calcProgram: RexProgram,
-     config: TableConfig)
+      generator: CodeGenerator,
+      inputSchema: RowSchema,
+      returnSchema: RowSchema,
+      calcProgram: RexProgram,
+      config: TableConfig)
     : String = {
 
     val expandedExpressions = calcProgram
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala
index c18c3d1544ff6..dc7a0d6d3579e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/PhysicalTableSourceScan.scala
@@ -46,7 +46,7 @@ abstract class PhysicalTableSourceScan(
 
   override def explainTerms(pw: RelWriter): RelWriter = {
     val terms = super.explainTerms(pw)
-        .item("fields", TableEnvironment.getFieldNames(tableSource).mkString(", "))
+        .item("fields", deriveRowType().getFieldNames.asScala.mkString(", "))
 
     val sourceDesc = tableSource.explainSource()
     if (sourceDesc.nonEmpty) {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
index c22dc54d99f17..b53081c935035 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala
@@ -90,8 +90,8 @@ class DataSetAggregate(
 
   override def translateToPlan(tableEnv: BatchTableEnvironment): DataSet[Row] = {
 
-    val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(tableEnv)
     val input = inputNode.asInstanceOf[DataSetRel]
+    val inputDS = input.translateToPlan(tableEnv)
 
     val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
index 9e180825fc7ca..5274fa19a731e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
@@ -103,10 +103,7 @@ class DataSetCalc(
       body,
       rowTypeInfo)
 
-    val runner = new FlatMapRunner[Row, Row](
-      genFunction.name,
-      genFunction.code,
-      genFunction.returnType)
+    val runner = calcMapFunction(genFunction)
 
     inputDS.flatMap(runner).name(calcOpName(calcProgram, getExpressionString))
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 8eb9d40846b61..db31f3218f2d2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -241,10 +241,7 @@ class DataStreamOverAggregate(
     }
 
     val precedingOffset =
-      getLowerBoundary(
-        logicWindow,
-        overWindow,
-        input) + (if (isRowsClause) 1 else 0)
+      getLowerBoundary(logicWindow, overWindow, input) + (if (isRowsClause) 1 else 0)
 
     val processFunction = AggregateUtil.createBoundedOverProcessFunction(
       generator,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
index 5dc3da866d392..e34e416946f4f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
@@ -26,7 +26,7 @@ import org.apache.flink.table.api.{StreamTableEnvironment, TableEnvironment}
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.PhysicalTableSourceScan
 import org.apache.flink.table.plan.schema.{RowSchema, TableSourceTable}
-import org.apache.flink.table.sources.{DefinedTimeAttributes, StreamTableSource, TableSource}
+import org.apache.flink.table.sources._
 import org.apache.flink.types.Row
 
 /** Flink RelNode to read data from an external source defined by a [[StreamTableSource]]. */
@@ -41,35 +41,23 @@ class StreamTableSourceScan(
   override def deriveRowType() = {
     val flinkTypeFactory = cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
 
-    def removeIndex[T](idx: Int, l: List[T]): List[T] = {
-      if (l.size < idx) {
-        l
-      } else {
-        l.take(idx) ++ l.drop(idx + 1)
-      }
-    }
+    val fieldNames = TableEnvironment.getFieldNames(tableSource).toList
+    val fieldTypes = TableEnvironment.getFieldTypes(tableSource.getReturnType).toList
 
-    var fieldNames = TableEnvironment.getFieldNames(tableSource).toList
-    var fieldTypes = TableEnvironment.getFieldTypes(tableSource.getReturnType).toList
+    val fieldCnt = fieldNames.length
 
     val rowtime = tableSource match {
-      case timeSource: DefinedTimeAttributes if timeSource.getRowtimeAttribute != null =>
+      case timeSource: DefinedRowTimeAttribute if timeSource.getRowtimeAttribute != null =>
         val rowtimeAttribute = timeSource.getRowtimeAttribute
-        // remove physical field if it is overwritten by time attribute
-        fieldNames = removeIndex(rowtimeAttribute.f0, fieldNames)
-        fieldTypes = removeIndex(rowtimeAttribute.f0, fieldTypes)
-        Some((rowtimeAttribute.f0, rowtimeAttribute.f1))
+        Some((fieldCnt, rowtimeAttribute))
       case _ =>
         None
     }
 
     val proctime = tableSource match {
-      case timeSource: DefinedTimeAttributes if timeSource.getProctimeAttribute != null =>
+      case timeSource: DefinedProcTimeAttribute if timeSource.getProctimeAttribute != null =>
         val proctimeAttribute = timeSource.getProctimeAttribute
-        // remove physical field if it is overwritten by time attribute
-        fieldNames = removeIndex(proctimeAttribute.f0, fieldNames)
-        fieldTypes = removeIndex(proctimeAttribute.f0, fieldTypes)
-        Some((proctimeAttribute.f0, proctimeAttribute.f1))
+        Some((fieldCnt + (if (rowtime.isDefined) 1 else 0), proctimeAttribute))
       case _ =>
         None
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
index 53e7b31446f27..a2777ecde9b52 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
@@ -29,7 +29,7 @@ import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.schema.TableSourceTable
-import org.apache.flink.table.sources.TableSource
+import org.apache.flink.table.sources.{DefinedProcTimeAttribute, DefinedRowTimeAttribute, TableSource}
 
 import scala.collection.JavaConverters._
 
@@ -47,11 +47,33 @@ class FlinkLogicalTableSourceScan(
 
   override def deriveRowType(): RelDataType = {
     val flinkTypeFactory = cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+
+    val fieldNames = TableEnvironment.getFieldNames(tableSource).toList
+    val fieldTypes = TableEnvironment.getFieldTypes(tableSource.getReturnType).toList
+
+    val fieldCnt = fieldNames.length
+
+    val rowtime = tableSource match {
+      case timeSource: DefinedRowTimeAttribute if timeSource.getRowtimeAttribute != null =>
+        val rowtimeAttribute = timeSource.getRowtimeAttribute
+        Some((fieldCnt, rowtimeAttribute))
+      case _ =>
+        None
+    }
+
+    val proctime = tableSource match {
+      case timeSource: DefinedProcTimeAttribute if timeSource.getProctimeAttribute != null =>
+        val proctimeAttribute = timeSource.getProctimeAttribute
+        Some((fieldCnt + (if (rowtime.isDefined) 1 else 0), proctimeAttribute))
+      case _ =>
+        None
+    }
+
     flinkTypeFactory.buildLogicalRowType(
-      TableEnvironment.getFieldNames(tableSource),
-      TableEnvironment.getFieldTypes(tableSource.getReturnType),
-      None,
-      None)
+      fieldNames,
+      fieldTypes,
+      rowtime,
+      proctime)
   }
 
   override def computeSelfCost(planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala
new file mode 100644
index 0000000000000..75deca5d544ad
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.schema
+
+import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFactory}
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.plan.stats.FlinkStatistic
+import org.apache.flink.table.sources.{DefinedProcTimeAttribute, DefinedRowTimeAttribute, TableSource}
+
+class StreamTableSourceTable[T](
+    override val tableSource: TableSource[T],
+    override val statistic: FlinkStatistic = FlinkStatistic.UNKNOWN)
+  extends TableSourceTable[T](tableSource, statistic) {
+
+
+  override def getRowType(typeFactory: RelDataTypeFactory): RelDataType = {
+    val flinkTypeFactory = typeFactory.asInstanceOf[FlinkTypeFactory]
+
+    val fieldNames = TableEnvironment.getFieldNames(tableSource).toList
+    val fieldTypes = TableEnvironment.getFieldTypes(tableSource.getReturnType).toList
+
+    val fieldCnt = fieldNames.length
+
+    val rowtime = tableSource match {
+      case timeSource: DefinedRowTimeAttribute if timeSource.getRowtimeAttribute != null =>
+        val rowtimeAttribute = timeSource.getRowtimeAttribute
+        Some((fieldCnt, rowtimeAttribute))
+      case _ =>
+        None
+    }
+
+    val proctime = tableSource match {
+      case timeSource: DefinedProcTimeAttribute if timeSource.getProctimeAttribute != null =>
+        val proctimeAttribute = timeSource.getProctimeAttribute
+        Some((fieldCnt + (if (rowtime.isDefined) 1 else 0), proctimeAttribute))
+      case _ =>
+        None
+    }
+
+    flinkTypeFactory.buildLogicalRowType(
+      fieldNames,
+      fieldTypes,
+      rowtime,
+      proctime)
+
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 07992cd032dab..dfed34ab0a8e6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -81,7 +81,6 @@ object AggregateUtil {
       isRowsClause: Boolean)
     : ProcessFunction[Row, Row] = {
 
-    val needRetract = false
     val (aggFields, aggregates) =
       transformToAggregateFunctions(
         namedAggregates.map(_.getKey),
@@ -107,7 +106,7 @@ object AggregateUtil {
       None,
       None,
       outputArity,
-      needRetract,
+      needRetract = false,
       needMerge = false,
       needReset = false
     )
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
index 8466cdf025042..6d87663416aa2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
@@ -18,30 +18,43 @@
 
 package org.apache.flink.table.sources
 
-import org.apache.flink.api.java.tuple.Tuple2
-
 /**
-  * Defines logical time attributes for a [[TableSource]]. Time attributes can be used for
-  * indicating, accessing, and working with Flink's event-time or processing-time. A
-  * [[TableSource]] that implements this interface can define names and positions of rowtime
-  * and proctime attributes in the rows it produces.
+  * Defines a logical event-time attribute for a [[TableSource]].
+  * The event-time attribute can be used for indicating, accessing, and working with Flink's
+  * event-time.
+  *
+  * A [[TableSource]] that implements this interface defines the name of
+  * the event-time attribute. The attribute will be added to the schema of the
+  * [[org.apache.flink.table.api.Table]] produced by the [[TableSource]].
   */
-trait DefinedTimeAttributes {
+trait DefinedRowTimeAttribute {
 
   /**
-    * Defines a name and position (starting at 0) of rowtime attribute that represents Flink's
-    * event-time. Null if no rowtime should be available. If the position is within the arity of
-    * the result row, the logical attribute will overwrite the physical attribute. If the position
-    * is higher than the result row, the time attribute will be appended logically.
+    * Defines a name of the event-time attribute that represents Flink's
+    * event-time. Null if no rowtime should be available.
+    *
+    * The field will be appended to the schema provided by the [[TableSource]].
     */
-  def getRowtimeAttribute: Tuple2[Int, String]
+  def getRowtimeAttribute: String
+}
+
+/**
+  * Defines a logical processing-time attribute for a [[TableSource]].
+  * The processing-time attribute can be used for indicating, accessing, and working with Flink's
+  * processing-time.
+  *
+  * A [[TableSource]] that implements this interface defines the name of
+  * the processing-time attribute. The attribute will be added to the schema of the
+  * [[org.apache.flink.table.api.Table]] produced by the [[TableSource]].
+  */
+trait DefinedProcTimeAttribute {
 
   /**
-    * Defines a name and position (starting at 0) of proctime attribute that represents Flink's
-    * processing-time. Null if no proctime should be available. If the position is within the arity
-    * of the result row, the logical attribute will overwrite the physical attribute. If the
-    * position is higher than the result row, the time attribute will be appended logically.
+    * Defines a name of the processing-time attribute that represents Flink's
+    * processing-time. Null if no rowtime should be available.
+    *
+    * The field will be appended to the schema provided by the [[TableSource]].
     */
-  def getProctimeAttribute: Tuple2[Int, String]
+  def getProctimeAttribute: String
 
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index faacc549bc104..52476853e0e75 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -93,8 +93,7 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2"),
         UnresolvedFieldReference("name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -108,8 +107,7 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2"),
         UnresolvedFieldReference("name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -123,8 +121,7 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2"),
         UnresolvedFieldReference("name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
   }
 
   @Test
@@ -135,8 +132,7 @@ class TableEnvironmentTest extends TableTestBase {
         UnresolvedFieldReference("pf3"),
         UnresolvedFieldReference("pf1"),
         UnresolvedFieldReference("pf2")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("pf3", "pf1", "pf2")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -146,9 +142,7 @@ class TableEnvironmentTest extends TableTestBase {
   def testGetFieldInfoAtomicName1(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(
       atomicType,
-      Array(UnresolvedFieldReference("name")),
-      ignoreTimeAttributes = true
-    )
+      Array(UnresolvedFieldReference("name")))
 
     fieldInfo._1.zip(Array("name")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0)).foreach(x => assertEquals(x._2, x._1))
@@ -161,8 +155,7 @@ class TableEnvironmentTest extends TableTestBase {
       Array(
         UnresolvedFieldReference("name1"),
         UnresolvedFieldReference("name2")
-      ),
-      ignoreTimeAttributes = true)
+      ))
   }
 
   @Test
@@ -173,8 +166,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("f0"), "name1"),
         Alias(UnresolvedFieldReference("f1"), "name2"),
         Alias(UnresolvedFieldReference("f2"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -188,8 +180,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("f2"), "name1"),
         Alias(UnresolvedFieldReference("f0"), "name2"),
         Alias(UnresolvedFieldReference("f1"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -203,8 +194,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("xxx"), "name1"),
         Alias(UnresolvedFieldReference("yyy"), "name2"),
         Alias(UnresolvedFieldReference("zzz"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
   }
 
   @Test
@@ -215,8 +205,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("cf1"), "name1"),
         Alias(UnresolvedFieldReference("cf2"), "name2"),
         Alias(UnresolvedFieldReference("cf3"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -230,8 +219,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("cf3"), "name1"),
         Alias(UnresolvedFieldReference("cf1"), "name2"),
         Alias(UnresolvedFieldReference("cf2"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -245,8 +233,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("xxx"), "name1"),
         Alias(UnresolvedFieldReference("yyy"), "name2"),
         Alias(UnresolvedFieldReference("zzz"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
   }
 
   @Test
@@ -257,8 +244,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("pf1"), "name1"),
         Alias(UnresolvedFieldReference("pf2"), "name2"),
         Alias(UnresolvedFieldReference("pf3"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
@@ -272,8 +258,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("pf3"), "name1"),
         Alias(UnresolvedFieldReference("pf1"), "name2"),
         Alias(UnresolvedFieldReference("pf2"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
 
     fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
     fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
@@ -287,8 +272,7 @@ class TableEnvironmentTest extends TableTestBase {
         Alias(UnresolvedFieldReference("xxx"), "name1"),
         Alias(UnresolvedFieldReference("yyy"), "name2"),
         Alias(UnresolvedFieldReference("zzz"), "name3")
-      ),
-      ignoreTimeAttributes = true)
+      ))
   }
 
   @Test(expected = classOf[TableException])
@@ -297,16 +281,14 @@ class TableEnvironmentTest extends TableTestBase {
       atomicType,
       Array(
         Alias(UnresolvedFieldReference("name1"), "name2")
-      ),
-      ignoreTimeAttributes = true)
+      ))
   }
 
   @Test(expected = classOf[TableException])
   def testGetFieldInfoGenericRowAlias(): Unit = {
     tEnv.getFieldInfo(
       genericRowType,
-      Array(UnresolvedFieldReference("first")),
-      ignoreTimeAttributes = true)
+      Array(UnresolvedFieldReference("first")))
   }
 
   @Test
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
index c4811050a926c..aa6edd323e1ad 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/GroupWindowTest.scala
@@ -100,7 +100,7 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeTumblingGroupWindowOverCount(): Unit = {
     val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
 
     val windowedTable = table
       .window(Tumble over 2.rows on 'long as 'w)
@@ -144,7 +144,7 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testEventTimeTumblingGroupWindowOverTime(): Unit = {
     val util = batchTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
 
     val windowedTable = table
       .window(Tumble over 5.milli on 'long as 'w)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala
new file mode 100644
index 0000000000000..e9384c7ad6a36
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream
+
+import java.lang.{Integer => JInt, Long => JLong}
+import java.util.Collections
+import java.util.{List => JList}
+
+import org.apache.flink.api.java.tuple.{Tuple5 => JTuple5}
+import org.apache.flink.api.java.typeutils.TupleTypeInfo
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JStreamExecEnv}
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.table.api.{TableEnvironment, TableException, Types}
+import org.apache.flink.table.api.java.{StreamTableEnvironment => JStreamTableEnv}
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.utils.TableTestBase
+import org.junit.Test
+import org.mockito.Mockito.{mock, when}
+
+class StreamTableEnvironmentTest extends TableTestBase {
+
+  @Test(expected = classOf[TableException])
+  def testInvalidProctimeAttribute(): Unit = {
+    val util = streamTestUtil()
+    // cannot replace an attribute with proctime
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b.proctime, 'c, 'd, 'e)
+  }
+
+  @Test
+  def testProctimeAttribute(): Unit = {
+    val util = streamTestUtil()
+    // cannot replace an attribute with proctime
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'pt.proctime)
+  }
+
+  @Test(expected = classOf[TableException])
+  def testRowtimeAttributeReplaceFieldOfInvalidType(): Unit = {
+    val util = streamTestUtil()
+    // cannot replace a non-time attribute with rowtime
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c.rowtime, 'd, 'e)
+  }
+
+  @Test
+  def testReplacedRowtimeAttribute(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('rt.rowtime, 'b, 'c, 'd, 'e)
+  }
+
+  @Test
+  def testAppendedRowtimeAttribute(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rt.rowtime)
+  }
+
+  @Test
+  def testRowtimeAndProctimeAttribute1(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'rt.rowtime, 'pt.proctime)
+  }
+
+  @Test
+  def testRowtimeAndProctimeAttribute2(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'pt.proctime, 'rt.rowtime)
+  }
+
+  @Test
+  def testRowtimeAndProctimeAttribute3(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('rt.rowtime, 'b, 'c, 'd, 'e, 'pt.proctime)
+  }
+
+  @Test(expected = classOf[TableException])
+  def testRowtimeAndInvalidProctimeAttribute(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('rt.rowtime, 'b, 'c, 'd, 'pt.proctime)
+  }
+
+  @Test(expected = classOf[TableException])
+  def testOnlyOneRowtimeAttribute1(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('a.rowtime, 'b, 'c, 'd, 'e, 'rt.rowtime)
+  }
+
+  @Test(expected = classOf[TableException])
+  def testOnlyOneProctimeAttribute1(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'pt1.proctime, 'pt2.proctime)
+  }
+
+  @Test(expected = classOf[TableException])
+  def testRowtimeAttributeUsedName(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'a.rowtime)
+  }
+
+  @Test(expected = classOf[TableException])
+  def testProctimeAttributeUsedName(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e, 'b.proctime)
+  }
+
+  @Test
+  def testProctimeAttributeParsed(): Unit = {
+    val (jTEnv, ds) = prepareSchemaExpressionParser
+    jTEnv.fromDataStream(ds, "a, b, c, d, e, pt.proctime")
+  }
+
+  @Test
+  def testReplacingRowtimeAttributeParsed(): Unit = {
+    val (jTEnv, ds) = prepareSchemaExpressionParser
+    jTEnv.fromDataStream(ds, "a.rowtime, b, c, d, e")
+  }
+
+  @Test
+  def testAppedingRowtimeAttributeParsed(): Unit = {
+    val (jTEnv, ds) = prepareSchemaExpressionParser
+    jTEnv.fromDataStream(ds, "a, b, c, d, e, rt.rowtime")
+  }
+
+  @Test
+  def testRowtimeAndProctimeAttributeParsed1(): Unit = {
+    val (jTEnv, ds) = prepareSchemaExpressionParser
+    jTEnv.fromDataStream(ds, "a, b, c, d, e, pt.proctime, rt.rowtime")
+  }
+
+  @Test
+  def testRowtimeAndProctimeAttributeParsed2(): Unit = {
+    val (jTEnv, ds) = prepareSchemaExpressionParser
+    jTEnv.fromDataStream(ds, "rt.rowtime, b, c, d, e, pt.proctime")
+  }
+
+  private def prepareSchemaExpressionParser:
+    (JStreamTableEnv, DataStream[JTuple5[JLong, JInt, String, JInt, JLong]]) = {
+
+    val jTEnv = TableEnvironment.getTableEnvironment(mock(classOf[JStreamExecEnv]))
+
+    val sType = new TupleTypeInfo(Types.LONG, Types.INT, Types.STRING, Types.INT, Types.LONG)
+      .asInstanceOf[TupleTypeInfo[JTuple5[JLong, JInt, String, JInt, JLong]]]
+    val ds = mock(classOf[DataStream[JTuple5[JLong, JInt, String, JInt, JLong]]])
+    when(ds.getType).thenReturn(sType)
+
+    (jTEnv, ds)
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
new file mode 100644
index 0000000000000..76732664697cd
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+import org.apache.flink.table.api.Types
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.sources.{DefinedProcTimeAttribute, DefinedRowTimeAttribute, StreamTableSource}
+import org.apache.flink.table.utils.TableTestBase
+import org.apache.flink.table.utils.TableTestUtil.{term, unaryNode}
+import org.apache.flink.types.Row
+import org.junit.Test
+
+class TableSourceTest extends TableTestBase {
+
+  @Test
+  def testRowTimeTableSourceSimple(): Unit = {
+    val util = streamTestUtil()
+    util.tEnv.registerTableSource("rowTimeT", new TestRowTimeSource("addTime"))
+
+    val t = util.tEnv.scan("rowTimeT").select("addTime, id, name, val")
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        "StreamTableSourceScan(table=[[rowTimeT]], fields=[id, val, name, addTime])",
+        term("select", "addTime", "id", "name", "val")
+      )
+    util.verifyTable(t, expected)
+  }
+
+  @Test
+  def testRowTimeTableSourceGroupWindow(): Unit = {
+    val util = streamTestUtil()
+    util.tEnv.registerTableSource("rowTimeT", new TestRowTimeSource("addTime"))
+
+    val t = util.tEnv.scan("rowTimeT")
+      .filter("val > 100")
+      .window(Tumble over 10.minutes on 'addTime as 'w)
+      .groupBy('name, 'w)
+      .select('name, 'w.end, 'val.avg)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            "StreamTableSourceScan(table=[[rowTimeT]], fields=[id, val, name, addTime])",
+            term("select", "name", "val", "addTime"),
+            term("where", ">(val, 100)")
+          ),
+          term("groupBy", "name"),
+          term("window", "TumblingGroupWindow(WindowReference(w), 'addTime, 600000.millis)"),
+          term("select", "name", "AVG(val) AS TMP_1", "end(WindowReference(w)) AS TMP_0")
+        ),
+        term("select", "name", "TMP_0", "TMP_1")
+      )
+    util.verifyTable(t, expected)
+  }
+
+  @Test
+  def testProcTimeTableSourceSimple(): Unit = {
+    val util = streamTestUtil()
+    util.tEnv.registerTableSource("procTimeT", new TestProcTimeSource("pTime"))
+
+    val t = util.tEnv.scan("procTimeT").select("pTime, id, name, val")
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        "StreamTableSourceScan(table=[[procTimeT]], fields=[id, val, name, pTime])",
+        term("select", "pTime", "id", "name", "val")
+      )
+    util.verifyTable(t, expected)
+  }
+
+  @Test
+  def testProcTimeTableSourceOverWindow(): Unit = {
+    val util = streamTestUtil()
+    util.tEnv.registerTableSource("procTimeT", new TestProcTimeSource("pTime"))
+
+    val t = util.tEnv.scan("procTimeT")
+      .window(Over partitionBy 'id orderBy 'pTime preceding 2.hours as 'w)
+      .select('id, 'name, 'val.sum over 'w as 'valSum)
+      .filter('valSum > 100)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          "StreamTableSourceScan(table=[[procTimeT]], fields=[id, val, name, pTime])",
+          term("partitionBy", "id"),
+          term("orderBy", "pTime"),
+          term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
+          term("select", "id", "val", "name", "pTime", "SUM(val) AS w0$o0")
+        ),
+        term("select", "id", "name", "w0$o0 AS valSum"),
+        term("where", ">(w0$o0, 100)")
+      )
+    util.verifyTable(t, expected)
+  }
+}
+
+class TestRowTimeSource(timeField: String)
+    extends StreamTableSource[Row] with DefinedRowTimeAttribute {
+
+  override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = ???
+
+  override def getRowtimeAttribute: String = timeField
+
+  override def getReturnType: TypeInformation[Row] = {
+    new RowTypeInfo(
+      Array(Types.INT, Types.LONG, Types.STRING).asInstanceOf[Array[TypeInformation[_]]],
+      Array("id", "val", "name"))
+  }
+}
+
+class TestProcTimeSource(timeField: String)
+    extends StreamTableSource[Row] with DefinedProcTimeAttribute {
+
+  override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = ???
+
+  override def getProctimeAttribute: String = timeField
+
+  override def getReturnType: TypeInformation[Row] = {
+    new RowTypeInfo(
+      Array(Types.INT, Types.LONG, Types.STRING).asInstanceOf[Array[TypeInformation[_]]],
+      Array("id", "val", "name"))
+  }
+}
+
+
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
index edf7b1d0218d7..f84ae3d4eb1fc 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
@@ -24,7 +24,7 @@ import org.apache.flink.table.api.scala._
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.utils.TableTestUtil._
 import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase}
-import org.junit.{Ignore, Test}
+import org.junit.Test
 
 class WindowAggregateTest extends TableTestBase {
   private val streamUtil: StreamTableTestUtil = streamTestUtil()
@@ -85,7 +85,6 @@ class WindowAggregateTest extends TableTestBase {
   }
 
   @Test
-  @Ignore // TODO enable once CALCITE-1761 is fixed
   def testTumbleFunction() = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
 
@@ -98,24 +97,23 @@ class WindowAggregateTest extends TableTestBase {
         "GROUP BY TUMBLE(rowtime, INTERVAL '15' MINUTE)"
     val expected =
       unaryNode(
-        "DataStreamCalc",
+        "DataStreamAggregate",
         unaryNode(
-          "DataStreamAggregate",
+          "DataStreamCalc",
           streamTableNode(0),
-          term("window", TumblingGroupWindow('w$, 'rowtime, 900000.millis)),
-          term("select",
-            "COUNT(*) AS EXPR$0, " +
-              "weightedAvg(c, a) AS wAvg, " +
-              "start('w$) AS w$start, " +
-              "end('w$) AS w$end")
+          term("select", "1970-01-01 00:00:00 AS $f0", "c", "a")
         ),
-        term("select", "EXPR$0, wAvg, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
+        term("window", TumblingGroupWindow('w$, 'rowtime, 900000.millis)),
+        term("select",
+          "COUNT(*) AS EXPR$0, " +
+            "weightedAvg(c, a) AS wAvg, " +
+            "start('w$) AS w$start, " +
+            "end('w$) AS w$end")
       )
     streamUtil.verifySql(sql, expected)
   }
 
   @Test
-  @Ignore // TODO enable once CALCITE-1761 is fixed
   def testHoppingFunction() = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
 
@@ -127,24 +125,23 @@ class WindowAggregateTest extends TableTestBase {
         "GROUP BY HOP(proctime, INTERVAL '15' MINUTE, INTERVAL '1' HOUR)"
     val expected =
       unaryNode(
-        "DataStreamCalc",
+        "DataStreamAggregate",
         unaryNode(
-          "DataStreamAggregate",
+          "DataStreamCalc",
           streamTableNode(0),
-          term("window", SlidingGroupWindow('w$, 'rowtime, 3600000.millis, 900000.millis)),
-          term("select",
-            "COUNT(*) AS EXPR$0, " +
-              "weightedAvg(c, a) AS wAvg, " +
-              "start('w$) AS w$start, " +
-              "end('w$) AS w$end")
+          term("select", "1970-01-01 00:00:00 AS $f0", "c", "a")
         ),
-        term("select", "EXPR$0, wAvg, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
+        term("window", SlidingGroupWindow('w$, 'proctime, 3600000.millis, 900000.millis)),
+        term("select",
+          "COUNT(*) AS EXPR$0, " +
+            "weightedAvg(c, a) AS wAvg, " +
+            "start('w$) AS w$start, " +
+            "end('w$) AS w$end")
       )
     streamUtil.verifySql(sql, expected)
   }
 
   @Test
-  @Ignore // TODO enable once CALCITE-1761 is fixed
   def testSessionFunction() = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
 
@@ -157,18 +154,18 @@ class WindowAggregateTest extends TableTestBase {
         "GROUP BY SESSION(proctime, INTERVAL '15' MINUTE)"
     val expected =
       unaryNode(
-        "DataStreamCalc",
+        "DataStreamAggregate",
         unaryNode(
-          "DataStreamAggregate",
+          "DataStreamCalc",
           streamTableNode(0),
-          term("window", SessionGroupWindow('w$, 'rowtime, 900000.millis)),
-          term("select",
-            "COUNT(*) AS EXPR$0, " +
-              "weightedAvg(c, a) AS wAvg, " +
-              "start('w$) AS w$start, " +
-              "end('w$) AS w$end")
+          term("select", "1970-01-01 00:00:00 AS $f0", "c", "a")
         ),
-        term("select", "EXPR$0, wAvg, CAST(w$start) AS w$start, CAST(w$end) AS w$end")
+        term("window", SessionGroupWindow('w$, 'proctime, 900000.millis)),
+        term("select",
+          "COUNT(*) AS EXPR$0, " +
+            "weightedAvg(c, a) AS wAvg, " +
+            "start('w$) AS w$start, " +
+            "end('w$) AS w$end")
       )
     streamUtil.verifySql(sql, expected)
   }

From 03935ee63954f16206a684131f5c59ca833a6cf5 Mon Sep 17 00:00:00 2001
From: shaoxuan-wang <wshaoxuan@gmail.com>
Date: Thu, 30 Mar 2017 03:57:58 +0800
Subject: [PATCH 157/253] [FLINK-6216] [table] Add non-windowed GroupBy
 aggregation for streams.

This closes #3646.
---
 .../flink/table/plan/logical/operators.scala  |   3 -
 .../datastream/DataStreamGroupAggregate.scala | 152 +++++++++++++
 ...a => DataStreamGroupWindowAggregate.scala} |   8 +-
 .../table/plan/rules/FlinkRuleSets.scala      |   3 +-
 .../DataStreamGroupAggregateRule.scala        |  81 +++++++
 ... DataStreamGroupWindowAggregateRule.scala} |  12 +-
 .../runtime/aggregate/AggregateUtil.scala     |  66 +++++-
 .../aggregate/GroupAggProcessFunction.scala   |  90 ++++++++
 .../aggregate/ProcTimeBoundedRangeOver.scala  |   2 +-
 .../batch/table/FieldProjectionTest.scala     |   4 +-
 .../api/scala/stream/TableSourceTest.scala    |   2 +-
 .../api/scala/stream/sql/SqlITCase.scala      |  21 ++
 .../stream/sql/WindowAggregateTest.scala      |  51 +++--
 .../table/GroupAggregationsITCase.scala       | 132 +++++++++++
 .../stream/table/GroupAggregationsTest.scala  | 214 ++++++++++++++++++
 ...la => GroupWindowAggregationsITCase.scala} |   6 +-
 .../scala/stream/table/GroupWindowTest.scala  | 195 ++++++++++------
 .../stream/table/UnsupportedOpsTest.scala     |   7 -
 18 files changed, 919 insertions(+), 130 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/{DataStreamAggregate.scala => DataStreamGroupWindowAggregate.scala} (98%)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/{DataStreamAggregateRule.scala => DataStreamGroupWindowAggregateRule.scala} (89%)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsTest.scala
 rename flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/{AggregationsITCase.scala => GroupWindowAggregationsITCase.scala} (96%)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index 383914526f095..36067eb1f21e7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -215,9 +215,6 @@ case class Aggregate(
   }
 
   override def validate(tableEnv: TableEnvironment): LogicalNode = {
-    if (tableEnv.isInstanceOf[StreamTableEnvironment]) {
-      failValidation(s"Aggregate on stream tables is currently not supported.")
-    }
 
     val resolvedAggregate = super.validate(tableEnv).asInstanceOf[Aggregate]
     val groupingExprs = resolvedAggregate.groupingExpressions
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
new file mode 100644
index 0000000000000..19f90c7da2d34
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.plan.nodes.datastream
+
+import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.core.AggregateCall
+import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
+import org.apache.flink.api.java.functions.NullByteKeySelector
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.codegen.CodeGenerator
+import org.apache.flink.table.runtime.aggregate._
+import org.apache.flink.table.plan.nodes.CommonAggregate
+import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.types.Row
+import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
+
+/**
+  *
+  * Flink RelNode for data stream unbounded group aggregate
+  *
+  * @param cluster         Cluster of the RelNode, represent for an environment of related
+  *                        relational expressions during the optimization of a query.
+  * @param traitSet        Trait set of the RelNode
+  * @param inputNode       The input RelNode of aggregation
+  * @param namedAggregates List of calls to aggregate functions and their output field names
+  * @param rowRelDataType  The type of the rows of the RelNode
+  * @param inputSchema     The type of the rows consumed by this RelNode
+  * @param schema          The type of the rows emitted by this RelNode
+  * @param groupings       The position (in the input Row) of the grouping keys
+  */
+class DataStreamGroupAggregate(
+    cluster: RelOptCluster,
+    traitSet: RelTraitSet,
+    inputNode: RelNode,
+    namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+    rowRelDataType: RelDataType,
+    schema: RowSchema,
+    inputSchema: RowSchema,
+    groupings: Array[Int])
+  extends SingleRel(cluster, traitSet, inputNode)
+    with CommonAggregate
+    with DataStreamRel {
+
+  override def deriveRowType() = schema.logicalType
+
+  override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
+    new DataStreamGroupAggregate(
+      cluster,
+      traitSet,
+      inputs.get(0),
+      namedAggregates,
+      getRowType,
+      schema,
+      inputSchema,
+      groupings)
+  }
+
+  override def toString: String = {
+    s"Aggregate(${
+      if (!groupings.isEmpty) {
+        s"groupBy: (${groupingToString(inputSchema.logicalType, groupings)}), "
+      } else {
+        ""
+      }
+    }select:(${aggregationToString(
+      inputSchema.logicalType, groupings, getRowType, namedAggregates, Nil)}))"
+  }
+
+  override def explainTerms(pw: RelWriter): RelWriter = {
+    super.explainTerms(pw)
+      .itemIf("groupBy", groupingToString(
+        inputSchema.logicalType, groupings), !groupings.isEmpty)
+      .item("select", aggregationToString(
+        inputSchema.logicalType, groupings, getRowType, namedAggregates, Nil))
+  }
+
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+
+    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
+      new CalcitePair[AggregateCall, String](
+        inputSchema.mapAggregateCall(namedAggregate.left),
+        namedAggregate.right)
+    }
+
+    val generator = new CodeGenerator(
+      tableEnv.getConfig,
+      false,
+      inputDS.getType)
+
+    val aggString = aggregationToString(
+      inputSchema.logicalType,
+      groupings,
+      getRowType,
+      namedAggregates,
+      Nil)
+
+    val keyedAggOpName = s"groupBy: (${groupingToString(inputSchema.logicalType, groupings)}), " +
+      s"select: ($aggString)"
+    val nonKeyedAggOpName = s"select: ($aggString)"
+
+    val physicalGrouping = groupings.map(inputSchema.mapIndex)
+
+    val processFunction = AggregateUtil.createGroupAggregateFunction(
+      generator,
+      physicalNamedAggregates,
+      inputSchema.logicalType,
+      inputSchema.physicalFieldTypeInfo,
+      groupings)
+
+    val result: DataStream[Row] =
+    // grouped / keyed aggregation
+      if (physicalGrouping.nonEmpty) {
+        inputDS
+        .keyBy(groupings: _*)
+        .process(processFunction)
+        .returns(schema.physicalTypeInfo)
+        .name(keyedAggOpName)
+        .asInstanceOf[DataStream[Row]]
+      }
+      // global / non-keyed aggregation
+      else {
+        inputDS
+        .keyBy(new NullByteKeySelector[Row])
+        .process(processFunction)
+        .setParallelism(1)
+        .setMaxParallelism(1)
+        .returns(schema.physicalTypeInfo)
+        .name(nonKeyedAggOpName)
+        .asInstanceOf[DataStream[Row]]
+      }
+    result
+  }
+}
+
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
similarity index 98%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index 56974498bc418..5aced669ef6cc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -32,14 +32,14 @@ import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.expressions.ExpressionUtils._
 import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.plan.nodes.CommonAggregate
-import org.apache.flink.table.plan.nodes.datastream.DataStreamAggregate._
 import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.plan.nodes.datastream.DataStreamGroupWindowAggregate._
 import org.apache.flink.table.runtime.aggregate.AggregateUtil._
 import org.apache.flink.table.runtime.aggregate._
 import org.apache.flink.table.typeutils.TypeCheckUtils.isTimeInterval
 import org.apache.flink.types.Row
 
-class DataStreamAggregate(
+class DataStreamGroupWindowAggregate(
     window: LogicalWindow,
     namedProperties: Seq[NamedWindowProperty],
     cluster: RelOptCluster,
@@ -54,7 +54,7 @@ class DataStreamAggregate(
   override def deriveRowType(): RelDataType = schema.logicalType
 
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
-    new DataStreamAggregate(
+    new DataStreamGroupWindowAggregate(
       window,
       namedProperties,
       cluster,
@@ -193,7 +193,7 @@ class DataStreamAggregate(
   }
 }
 
-object DataStreamAggregate {
+object DataStreamGroupWindowAggregate {
 
 
   private def createKeyedWindowedStream(groupWindow: LogicalWindow, stream: KeyedStream[Row, Tuple])
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
index 838cb22b6d6a2..f4de651e9ab44 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
@@ -176,8 +176,9 @@ object FlinkRuleSets {
     */
   val DATASTREAM_OPT_RULES: RuleSet = RuleSets.ofList(
     // translate to DataStream nodes
+    DataStreamGroupAggregateRule.INSTANCE,
     DataStreamOverAggregateRule.INSTANCE,
-    DataStreamAggregateRule.INSTANCE,
+    DataStreamGroupWindowAggregateRule.INSTANCE,
     DataStreamCalcRule.INSTANCE,
     DataStreamScanRule.INSTANCE,
     DataStreamUnionRule.INSTANCE,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala
new file mode 100644
index 0000000000000..fd7619cf0b2c0
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.rules.datastream
+
+import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet}
+import org.apache.calcite.rel.RelNode
+import org.apache.calcite.rel.convert.ConverterRule
+import org.apache.calcite.rel.logical.LogicalAggregate
+import org.apache.flink.table.api.TableException
+import org.apache.flink.table.plan.nodes.FlinkConventions
+import org.apache.flink.table.plan.nodes.datastream.DataStreamGroupAggregate
+import org.apache.flink.table.plan.nodes.logical.FlinkLogicalAggregate
+import org.apache.flink.table.plan.schema.RowSchema
+
+import scala.collection.JavaConversions._
+
+/**
+  * Rule to convert a [[LogicalAggregate]] into a [[DataStreamGroupAggregate]].
+  */
+class DataStreamGroupAggregateRule
+  extends ConverterRule(
+    classOf[FlinkLogicalAggregate],
+    FlinkConventions.LOGICAL,
+    FlinkConventions.DATASTREAM,
+    "DataStreamGroupAggregateRule") {
+
+  override def matches(call: RelOptRuleCall): Boolean = {
+    val agg: FlinkLogicalAggregate = call.rel(0).asInstanceOf[FlinkLogicalAggregate]
+
+    // check if we have distinct aggregates
+    val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
+    if (distinctAggs) {
+      throw TableException("DISTINCT aggregates are currently not supported.")
+    }
+
+    // check if we have grouping sets
+    val groupSets = agg.getGroupSets.size() != 1 || agg.getGroupSets.get(0) != agg.getGroupSet
+    if (groupSets || agg.indicator) {
+      throw TableException("GROUPING SETS are currently not supported.")
+    }
+
+    !distinctAggs && !groupSets && !agg.indicator
+  }
+
+  override def convert(rel: RelNode): RelNode = {
+    val agg: FlinkLogicalAggregate = rel.asInstanceOf[FlinkLogicalAggregate]
+    val traitSet: RelTraitSet = rel.getTraitSet.replace(FlinkConventions.DATASTREAM)
+    val convInput: RelNode = RelOptRule.convert(agg.getInput, FlinkConventions.DATASTREAM)
+
+    new DataStreamGroupAggregate(
+      rel.getCluster,
+      traitSet,
+      convInput,
+      agg.getNamedAggCalls,
+      rel.getRowType,
+      new RowSchema(rel.getRowType),
+      new RowSchema(agg.getInput.getRowType),
+      agg.getGroupSet.toArray)
+  }
+}
+
+object DataStreamGroupAggregateRule {
+  val INSTANCE: RelOptRule = new DataStreamGroupAggregateRule
+}
+
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupWindowAggregateRule.scala
similarity index 89%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamAggregateRule.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupWindowAggregateRule.scala
index fc654036cefb2..3beeb47997899 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupWindowAggregateRule.scala
@@ -23,18 +23,18 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.convert.ConverterRule
 import org.apache.flink.table.api.TableException
 import org.apache.flink.table.plan.nodes.FlinkConventions
-import org.apache.flink.table.plan.nodes.datastream.DataStreamAggregate
+import org.apache.flink.table.plan.nodes.datastream.DataStreamGroupWindowAggregate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalWindowAggregate
 import org.apache.flink.table.plan.schema.RowSchema
 
 import scala.collection.JavaConversions._
 
-class DataStreamAggregateRule
+class DataStreamGroupWindowAggregateRule
   extends ConverterRule(
     classOf[FlinkLogicalWindowAggregate],
     FlinkConventions.LOGICAL,
     FlinkConventions.DATASTREAM,
-    "DataStreamAggregateRule") {
+    "DataStreamGroupWindowAggregateRule") {
 
   override def matches(call: RelOptRuleCall): Boolean = {
     val agg: FlinkLogicalWindowAggregate = call.rel(0).asInstanceOf[FlinkLogicalWindowAggregate]
@@ -59,7 +59,7 @@ class DataStreamAggregateRule
     val traitSet: RelTraitSet = rel.getTraitSet.replace(FlinkConventions.DATASTREAM)
     val convInput: RelNode = RelOptRule.convert(agg.getInput, FlinkConventions.DATASTREAM)
 
-    new DataStreamAggregate(
+    new DataStreamGroupWindowAggregate(
       agg.getWindow,
       agg.getNamedProperties,
       rel.getCluster,
@@ -72,6 +72,6 @@ class DataStreamAggregateRule
     }
   }
 
-object DataStreamAggregateRule {
-  val INSTANCE: RelOptRule = new DataStreamAggregateRule
+object DataStreamGroupWindowAggregateRule {
+  val INSTANCE: RelOptRule = new DataStreamGroupWindowAggregateRule
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index dfed34ab0a8e6..5e9efd0d61c89 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -87,8 +87,7 @@ object AggregateUtil {
         inputType,
         needRetraction = false)
 
-    val aggregationStateType: RowTypeInfo =
-      createDataSetAggregateBufferDataType(Array(), aggregates, inputType)
+    val aggregationStateType: RowTypeInfo = createAccumulatorRowType(aggregates)
 
     val forwardMapping = (0 until inputType.getFieldCount).toArray
     val aggMapping = aggregates.indices.map(x => x + inputType.getFieldCount).toArray
@@ -139,7 +138,58 @@ object AggregateUtil {
   }
 
   /**
-    * Create an [[org.apache.flink.streaming.api.functions.ProcessFunction]] for
+    * Create an [[org.apache.flink.streaming.api.functions.ProcessFunction]] for group (without
+    * window) aggregate to evaluate final aggregate value.
+    *
+    * @param generator       code generator instance
+    * @param namedAggregates List of calls to aggregate functions and their output field names
+    * @param inputRowType    Input row type
+    * @param inputFieldTypes Types of the physical input fields
+    * @param groupings       the position (in the input Row) of the grouping keys
+    * @return [[org.apache.flink.streaming.api.functions.ProcessFunction]]
+    */
+  private[flink] def createGroupAggregateFunction(
+      generator: CodeGenerator,
+      namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+      inputRowType: RelDataType,
+      inputFieldTypes: Seq[TypeInformation[_]],
+      groupings: Array[Int]): ProcessFunction[Row, Row] = {
+
+    val (aggFields, aggregates) =
+      transformToAggregateFunctions(
+        namedAggregates.map(_.getKey),
+        inputRowType,
+        needRetraction = false)
+    val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
+
+    val outputArity = groupings.length + aggregates.length
+
+    val aggregationStateType: RowTypeInfo = createAccumulatorRowType(aggregates)
+
+    val genFunction = generator.generateAggregations(
+      "NonWindowedAggregationHelper",
+      generator,
+      inputFieldTypes,
+      aggregates,
+      aggFields,
+      aggMapping,
+      partialResults = false,
+      groupings,
+      None,
+      None,
+      outputArity,
+      needRetract = false,
+      needMerge = false,
+      needReset = false
+    )
+
+    new GroupAggProcessFunction(
+      genFunction,
+      aggregationStateType)
+  }
+
+  /**
+    * Create an [[org.apache.flink.streaming.api.functions.ProcessFunction]] for ROWS clause
     * bounded OVER window to evaluate final aggregate value.
     *
     * @param generator       code generator instance
@@ -266,7 +316,7 @@ object AggregateUtil {
       needRetract)
 
     val mapReturnType: RowTypeInfo =
-      createDataSetAggregateBufferDataType(
+      createRowTypeForKeysAndAggregates(
         groupings,
         aggregates,
         inputType,
@@ -370,7 +420,7 @@ object AggregateUtil {
       physicalInputRowType,
       needRetract)
 
-    val returnType: RowTypeInfo = createDataSetAggregateBufferDataType(
+    val returnType: RowTypeInfo = createRowTypeForKeysAndAggregates(
       groupings,
       aggregates,
       physicalInputRowType,
@@ -637,7 +687,7 @@ object AggregateUtil {
     window match {
       case SessionGroupWindow(_, _, gap) =>
         val combineReturnType: RowTypeInfo =
-          createDataSetAggregateBufferDataType(
+          createRowTypeForKeysAndAggregates(
             groupings,
             aggregates,
             physicalInputRowType,
@@ -711,7 +761,7 @@ object AggregateUtil {
 
       case SessionGroupWindow(_, _, gap) =>
         val combineReturnType: RowTypeInfo =
-          createDataSetAggregateBufferDataType(
+          createRowTypeForKeysAndAggregates(
             groupings,
             aggregates,
             physicalInputRowType,
@@ -1365,7 +1415,7 @@ object AggregateUtil {
     aggTypes
   }
 
-  private def createDataSetAggregateBufferDataType(
+  private def createRowTypeForKeysAndAggregates(
       groupings: Array[Int],
       aggregates: Array[TableAggregateFunction[_, _]],
       inputType: RelDataType,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
new file mode 100644
index 0000000000000..81c900cec68c6
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.aggregate
+
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.types.Row
+import org.apache.flink.util.Collector
+import org.apache.flink.api.common.state.ValueStateDescriptor
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.common.state.ValueState
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.slf4j.LoggerFactory
+
+/**
+  * Aggregate Function used for the groupby (without window) aggregate
+  *
+  * @param genAggregations      Generated aggregate helper function
+  * @param aggregationStateType The row type info of aggregation
+  */
+class GroupAggProcessFunction(
+    private val genAggregations: GeneratedAggregationsFunction,
+    private val aggregationStateType: RowTypeInfo)
+  extends ProcessFunction[Row, Row]
+    with Compiler[GeneratedAggregations] {
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+  private var function: GeneratedAggregations = _
+
+  private var output: Row = _
+  private var state: ValueState[Row] = _
+
+  override def open(config: Configuration) {
+    LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+      s"Code:\n$genAggregations.code")
+    val clazz = compile(
+      getRuntimeContext.getUserCodeClassLoader,
+      genAggregations.name,
+      genAggregations.code)
+    LOG.debug("Instantiating AggregateHelper.")
+    function = clazz.newInstance()
+    output = function.createOutputRow()
+
+    val stateDescriptor: ValueStateDescriptor[Row] =
+      new ValueStateDescriptor[Row]("GroupAggregateState", aggregationStateType)
+    state = getRuntimeContext.getState(stateDescriptor)
+  }
+
+  override def processElement(
+      input: Row,
+      ctx: ProcessFunction[Row, Row]#Context,
+      out: Collector[Row]): Unit = {
+
+    // get accumulators
+    var accumulators = state.value()
+    if (null == accumulators) {
+      accumulators = function.createAccumulators()
+    }
+
+    // Set group keys value to the final output
+    function.setForwardedFields(input, output)
+
+    // accumulate new input row
+    function.accumulate(accumulators, input)
+
+    // set aggregation results to output
+    function.setAggregationResults(accumulators, output)
+
+    // update accumulators
+    state.update(accumulators)
+
+    out.collect(output)
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index 7f87e505f4bcf..b63eb81413e12 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory
   * Process Function used for the aggregate in bounded proc-time OVER window
   * [[org.apache.flink.streaming.api.datastream.DataStream]]
   *
-  * @param genAggregations Generated aggregate helper function
+  * @param genAggregations          Generated aggregate helper function
   * @param precedingTimeBoundary    Is used to indicate the processing time boundaries
   * @param aggregatesTypeInfo       row type info of aggregation
   * @param inputType                row type info of input row
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
index b484293132f98..93e25f8f297d3 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/table/FieldProjectionTest.scala
@@ -230,7 +230,7 @@ class FieldProjectionTest extends TableTestBase {
 
     val expected =
       unaryNode(
-        "DataStreamAggregate",
+        "DataStreamGroupWindowAggregate",
         unaryNode(
           "DataStreamCalc",
           streamTableNode(0),
@@ -259,7 +259,7 @@ class FieldProjectionTest extends TableTestBase {
     val expected = unaryNode(
         "DataStreamCalc",
         unaryNode(
-          "DataStreamAggregate",
+          "DataStreamGroupWindowAggregate",
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
index 76732664697cd..18066c95cc2b9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
@@ -63,7 +63,7 @@ class TableSourceTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         unaryNode(
-          "DataStreamAggregate",
+          "DataStreamGroupWindowAggregate",
           unaryNode(
             "DataStreamCalc",
             "StreamTableSourceScan(table=[[rowTimeT]], fields=[id, val, name, addTime])",
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
index 6bab4b33f6dfd..abbcbdd083cb6 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
@@ -47,6 +47,27 @@ class SqlITCase extends StreamingWithStateTestBase {
     (8L, 8, "Hello World"),
     (20L, 20, "Hello World"))
 
+  /** test unbounded groupby (without window) **/
+  @Test
+  def testUnboundedGroupby(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val sqlQuery = "SELECT b, COUNT(a) FROM MyTable GROUP BY b"
+
+    val t = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c)
+    tEnv.registerTable("MyTable", t)
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList("1,1", "2,1", "2,2")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
   /** test selection **/
   @Test
   def testSelectExpressionFromTable(): Unit = {
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
index f84ae3d4eb1fc..4c1d6e62ec371 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
@@ -60,10 +60,13 @@ class WindowAggregateTest extends TableTestBase {
   @Test
   def testPartitionedProcessingTimeBoundedWindow() = {
 
-    val sqlQuery = "SELECT a, AVG(c) OVER (PARTITION BY a ORDER BY proctime " +
-      "RANGE BETWEEN INTERVAL '2' HOUR PRECEDING AND CURRENT ROW) AS avgA " +
+    val sqlQuery =
+      "SELECT a, " +
+      "  AVG(c) OVER (PARTITION BY a ORDER BY proctime " +
+      "    RANGE BETWEEN INTERVAL '2' HOUR PRECEDING AND CURRENT ROW) AS avgA " +
       "FROM MyTable"
-      val expected =
+
+    val expected =
       unaryNode(
         "DataStreamCalc",
         unaryNode(
@@ -84,6 +87,27 @@ class WindowAggregateTest extends TableTestBase {
     streamUtil.verifySql(sqlQuery, expected)
   }
 
+  @Test
+  def testGroupbyWithoutWindow() = {
+    val sql = "SELECT COUNT(a) FROM MyTable GROUP BY b"
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "b", "a")
+          ),
+          term("groupBy", "b"),
+          term("select", "b", "COUNT(a) AS EXPR$0")
+        ),
+        term("select", "EXPR$0")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
   @Test
   def testTumbleFunction() = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
@@ -97,7 +121,7 @@ class WindowAggregateTest extends TableTestBase {
         "GROUP BY TUMBLE(rowtime, INTERVAL '15' MINUTE)"
     val expected =
       unaryNode(
-        "DataStreamAggregate",
+        "DataStreamGroupWindowAggregate",
         unaryNode(
           "DataStreamCalc",
           streamTableNode(0),
@@ -125,7 +149,7 @@ class WindowAggregateTest extends TableTestBase {
         "GROUP BY HOP(proctime, INTERVAL '15' MINUTE, INTERVAL '1' HOUR)"
     val expected =
       unaryNode(
-        "DataStreamAggregate",
+        "DataStreamGroupWindowAggregate",
         unaryNode(
           "DataStreamCalc",
           streamTableNode(0),
@@ -154,7 +178,7 @@ class WindowAggregateTest extends TableTestBase {
         "GROUP BY SESSION(proctime, INTERVAL '15' MINUTE)"
     val expected =
       unaryNode(
-        "DataStreamAggregate",
+        "DataStreamGroupWindowAggregate",
         unaryNode(
           "DataStreamCalc",
           streamTableNode(0),
@@ -206,21 +230,6 @@ class WindowAggregateTest extends TableTestBase {
     streamUtil.verifySql(sql, "n/a")
   }
 
-  @Test(expected = classOf[TableException])
-  def testMultiWindow() = {
-    val sql = "SELECT COUNT(*) FROM MyTable GROUP BY " +
-      "FLOOR(rowtime TO HOUR), FLOOR(rowtime TO MINUTE)"
-    val expected = ""
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test(expected = classOf[TableException])
-  def testInvalidWindowExpression() = {
-    val sql = "SELECT COUNT(*) FROM MyTable GROUP BY FLOOR(localTimestamp TO HOUR)"
-    val expected = ""
-    streamUtil.verifySql(sql, expected)
-  }
-
   @Test(expected = classOf[ValidationException])
   def testWindowUdAggInvalidArgs(): Unit = {
     streamUtil.tEnv.registerFunction("weightedAvg", new WeightedAvgWithMerge)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
new file mode 100644
index 0000000000000..271e90ba23bab
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.types.Row
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+import scala.collection.mutable
+
+/**
+  * Tests of groupby (without window) aggregations
+  */
+class GroupAggregationsITCase extends StreamingWithStateTestBase {
+
+  @Test
+  def testNonKeyedGroupAggregate(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+            .select('a.sum, 'b.sum)
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "1,1", "3,3", "6,5", "10,8", "15,11", "21,14", "28,18", "36,22", "45,26", "55,30", "66,35",
+      "78,40", "91,45", "105,50", "120,55", "136,61", "153,67", "171,73", "190,79", "210,85",
+      "231,91")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testGroupAggregate(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+      .groupBy('b)
+      .select('b, 'a.sum)
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "1,1", "2,2", "2,5", "3,4", "3,9", "3,15", "4,7", "4,15",
+      "4,24", "4,34", "5,11", "5,23", "5,36", "5,50", "5,65", "6,16", "6,33", "6,51", "6,70",
+      "6,90", "6,111")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testDoubleGroupAggregation(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+      .groupBy('b)
+      .select('a.sum as 'd, 'b)
+      .groupBy('b, 'd)
+      .select('b)
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "1",
+      "2", "2",
+      "3", "3", "3",
+      "4", "4", "4", "4",
+      "5", "5", "5", "5", "5",
+      "6", "6", "6", "6", "6", "6")
+
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testGroupAggregateWithExpression(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t = StreamTestData.get5TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c, 'd, 'e)
+      .groupBy('e, 'b % 3)
+      .select('c.min, 'e, 'a.avg, 'd.count)
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "0,1,1,1", "1,2,2,1", "2,1,2,1", "3,2,3,1", "1,2,2,2",
+      "5,3,3,1", "3,2,3,2", "7,1,4,1", "2,1,3,2", "3,2,3,3", "7,1,4,2", "5,3,4,2", "12,3,5,1",
+      "1,2,3,3", "14,2,5,1")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsTest.scala
new file mode 100644
index 0000000000000..520592c6db3b1
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsTest.scala
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream.table
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.utils.TableTestBase
+import org.junit.Test
+import org.apache.flink.table.api.scala._
+import org.apache.flink.api.scala._
+import org.apache.flink.table.utils.TableTestUtil._
+
+class GroupAggregationsTest extends TableTestBase {
+
+  @Test(expected = classOf[ValidationException])
+  def testGroupingOnNonExistentField(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val ds = table
+             // must fail. '_foo is not a valid field
+             .groupBy('_foo)
+             .select('a.avg)
+  }
+
+  @Test(expected = classOf[ValidationException])
+  def testGroupingInvalidSelection(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val ds = table
+             .groupBy('a, 'b)
+             // must fail. 'c is not a grouping key or aggregation
+             .select('c)
+  }
+
+  @Test
+  def testGroupbyWithoutWindow() = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val resultTable = table
+                      .groupBy('b)
+                      .select('a.count)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "b")
+          ),
+          term("groupBy", "b"),
+          term("select", "b", "COUNT(a) AS TMP_0")
+        ),
+        term("select", "TMP_0")
+      )
+    util.verifyTable(resultTable, expected)
+  }
+
+
+  @Test
+  def testGroupAggregateWithConstant1(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val resultTable = table
+            .select('a, 4 as 'four, 'b)
+            .groupBy('four, 'a)
+            .select('four, 'b.sum)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "4 AS four", "b", "a")
+          ),
+          term("groupBy", "four", "a"),
+          term("select", "four", "a", "SUM(b) AS TMP_0")
+        ),
+        term("select", "4 AS four", "TMP_0")
+      )
+    util.verifyTable(resultTable, expected)
+  }
+
+  @Test
+  def testGroupAggregateWithConstant2(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val resultTable = table
+            .select('b, 4 as 'four, 'a)
+            .groupBy('b, 'four)
+            .select('four, 'a.sum)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "4 AS four", "a", "b")
+          ),
+          term("groupBy", "four", "b"),
+          term("select", "four", "b", "SUM(a) AS TMP_0")
+        ),
+        term("select", "4 AS four", "TMP_0")
+      )
+    util.verifyTable(resultTable, expected)
+  }
+
+  @Test
+  def testGroupAggregateWithExpressionInSelect(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val resultTable = table
+            .select('a as 'a, 'b % 3 as 'd, 'c as 'c)
+            .groupBy('d)
+            .select('c.min, 'a.avg)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "MOD(b, 3) AS d", "c")
+          ),
+          term("groupBy", "d"),
+          term("select", "d", "MIN(c) AS TMP_0", "AVG(a) AS TMP_1")
+        ),
+        term("select", "TMP_0", "TMP_1")
+      )
+    util.verifyTable(resultTable, expected)
+  }
+
+  @Test
+  def testGroupAggregateWithFilter(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val resultTable = table
+            .groupBy('b)
+            .select('b, 'a.sum)
+            .where('b === 2)
+
+    val expected =
+      unaryNode(
+        "DataStreamGroupAggregate",
+        unaryNode(
+          "DataStreamCalc",
+          streamTableNode(0),
+          term("select", "b", "a"),
+          term("where", "=(b, 2)")
+        ),
+        term("groupBy", "b"),
+        term("select", "b", "SUM(a) AS TMP_0")
+      )
+    util.verifyTable(resultTable, expected)
+  }
+
+  @Test
+  def testGroupAggregateWithAverage(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('a, 'b, 'c)
+
+    val resultTable = table
+            .groupBy('b)
+            .select('b, 'a.cast(BasicTypeInfo.DOUBLE_TYPE_INFO).avg)
+
+    val expected =
+      unaryNode(
+        "DataStreamGroupAggregate",
+        unaryNode(
+          "DataStreamCalc",
+          streamTableNode(0),
+          term("select", "b", "a", "CAST(a) AS a0")
+        ),
+        term("groupBy", "b"),
+        term("select", "b", "AVG(a0) AS TMP_0")
+      )
+
+    util.verifyTable(resultTable, expected)
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
similarity index 96%
rename from flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
rename to flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
index 4a6a616dc6b42..2c027a9046d4d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
@@ -27,7 +27,7 @@ import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
 import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge}
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.scala.stream.table.AggregationsITCase.TimestampAndWatermarkWithOffset
+import org.apache.flink.table.api.scala.stream.table.GroupWindowAggregationsITCase.TimestampAndWatermarkWithOffset
 import org.apache.flink.table.api.scala.stream.utils.StreamITCase
 import org.apache.flink.table.functions.aggfunctions.CountAggFunction
 import org.apache.flink.types.Row
@@ -40,7 +40,7 @@ import scala.collection.mutable
   * We only test some aggregations until better testing of constructed DataStream
   * programs is possible.
   */
-class AggregationsITCase extends StreamingMultipleProgramsTestBase {
+class GroupWindowAggregationsITCase extends StreamingMultipleProgramsTestBase {
 
   val data = List(
     (1L, 1, "Hi"),
@@ -177,7 +177,7 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
   }
 }
 
-object AggregationsITCase {
+object GroupWindowAggregationsITCase {
   class TimestampAndWatermarkWithOffset(
     offset: Long) extends AssignerWithPunctuatedWatermarks[(Long, Int, String)] {
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index de6cbfac48564..0573ff3323159 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -185,11 +185,11 @@ class GroupWindowTest extends TableTestBase {
       .select('string.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         unaryNode(
-          "DataStreamAggregate",
+          "DataStreamGroupWindowAggregate",
           unaryNode(
             "DataStreamCalc",
             streamTableNode(0),
@@ -229,7 +229,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -259,7 +259,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -268,10 +268,7 @@ class GroupWindowTest extends TableTestBase {
       term("groupBy", "string"),
       term(
         "window",
-        TumblingGroupWindow(
-          WindowReference("w"),
-          'proctime,
-          2.rows)),
+        TumblingGroupWindow(WindowReference("w"), 'proctime, 2.rows)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -289,7 +286,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       streamTableNode(0),
       term("groupBy", "string"),
       term(
@@ -317,7 +314,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, weightedAvg('long, 'int))
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       streamTableNode(0),
       term("groupBy", "string"),
       term(
@@ -343,7 +340,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -374,7 +371,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -396,6 +393,32 @@ class GroupWindowTest extends TableTestBase {
 
   @Test
   def testEventTimeSlidingGroupWindowOverTime(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
+
+    val windowedTable = table
+      .window(Slide over 8.milli every 10.milli on 'rowtime as 'w)
+      .groupBy('w, 'string)
+      .select('string, 'int.count)
+
+    val expected = unaryNode(
+      "DataStreamGroupWindowAggregate",
+      unaryNode(
+        "DataStreamCalc",
+        streamTableNode(0),
+        term("select", "string", "int", "rowtime")
+      ),
+      term("groupBy", "string"),
+      term("window", SlidingGroupWindow(WindowReference("w"), 'rowtime, 8.milli, 10.milli)),
+      term("select", "string", "COUNT(int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
+  @Test
+  @Ignore // see comments in DataStreamGroupWindowAggregate
+  def testEventTimeSlidingGroupWindowOverCount(): Unit = {
     val util = streamTestUtil()
     val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
@@ -405,7 +428,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       streamTableNode(0),
       term("groupBy", "string"),
       term(
@@ -434,12 +457,10 @@ class GroupWindowTest extends TableTestBase {
       .select('string, weightedAvg('long, 'int))
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       streamTableNode(0),
       term("groupBy", "string"),
-      term(
-        "window",
-        SlidingGroupWindow(WindowReference("w"), 'rowtime, 8.milli, 10.milli)),
+      term("window", SlidingGroupWindow(WindowReference("w"), 'rowtime, 8.milli, 10.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
 
@@ -457,15 +478,10 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       streamTableNode(0),
       term("groupBy", "string"),
-      term(
-        "window",
-        SessionGroupWindow(
-          WindowReference("w"),
-          'long,
-          7.milli)),
+      term("window", SessionGroupWindow(WindowReference("w"), 'long, 7.milli)),
       term("select", "string", "COUNT(int) AS TMP_0")
     )
 
@@ -485,12 +501,10 @@ class GroupWindowTest extends TableTestBase {
       .select('string, weightedAvg('long, 'int))
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       streamTableNode(0),
       term("groupBy", "string"),
-      term(
-        "window",
-        SessionGroupWindow(WindowReference("w"), 'rowtime, 7.milli)),
+      term("window", SessionGroupWindow(WindowReference("w"), 'rowtime, 7.milli)),
       term("select", "string", "WeightedAvgWithMerge(long, int) AS TMP_0")
     )
 
@@ -508,7 +522,7 @@ class GroupWindowTest extends TableTestBase {
       .select('string, 'int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -538,7 +552,7 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -558,6 +572,31 @@ class GroupWindowTest extends TableTestBase {
 
   @Test
   def testAllEventTimeTumblingGroupWindowOverTime(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
+
+    val windowedTable = table
+      .window(Tumble over 5.milli on 'rowtime as 'w)
+      .groupBy('w)
+      .select('int.count)
+
+    val expected = unaryNode(
+      "DataStreamGroupWindowAggregate",
+      unaryNode(
+        "DataStreamCalc",
+        streamTableNode(0),
+        term("select", "int", "rowtime")
+      ),
+      term("window", TumblingGroupWindow(WindowReference("w"), 'rowtime, 5.milli)),
+      term("select", "COUNT(int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
+  @Test
+  @Ignore // see comments in DataStreamGroupWindowAggregate
+  def testAllEventTimeTumblingGroupWindowOverCount(): Unit = {
     val util = streamTestUtil()
     val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
@@ -567,7 +606,7 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -596,7 +635,7 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -626,7 +665,7 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -647,6 +686,31 @@ class GroupWindowTest extends TableTestBase {
 
   @Test
   def testAllEventTimeSlidingGroupWindowOverTime(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
+
+    val windowedTable = table
+      .window(Slide over 8.milli every 10.milli on 'rowtime as 'w)
+      .groupBy('w)
+      .select('int.count)
+
+    val expected = unaryNode(
+      "DataStreamGroupWindowAggregate",
+      unaryNode(
+        "DataStreamCalc",
+        streamTableNode(0),
+        term("select", "int", "rowtime")
+      ),
+      term("window", SlidingGroupWindow(WindowReference("w"), 'rowtime, 8.milli, 10.milli)),
+      term("select", "COUNT(int) AS TMP_0")
+    )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
+  @Test
+//  @Ignore // see comments in DataStreamGroupWindowAggregate
+  def testAllEventTimeSlidingGroupWindowOverCount(): Unit = {
     val util = streamTestUtil()
     val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
 
@@ -656,19 +720,13 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
         term("select", "int", "long")
       ),
-      term(
-        "window",
-        SlidingGroupWindow(
-          WindowReference("w"),
-          'long,
-          8.milli,
-          10.milli)),
+      term("window", SlidingGroupWindow(WindowReference("w"), 'long, 8.milli, 10.milli)),
       term("select", "COUNT(int) AS TMP_0")
     )
 
@@ -686,7 +744,7 @@ class GroupWindowTest extends TableTestBase {
       .select('int.count)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
+      "DataStreamGroupWindowAggregate",
       unaryNode(
         "DataStreamCalc",
         streamTableNode(0),
@@ -707,22 +765,22 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testTumbleWindowStartEnd(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
 
     val windowedTable = table
-      .window(Tumble over 5.milli on 'long as 'w)
+      .window(Tumble over 5.milli on 'rowtime as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
-      streamTableNode(0),
+      "DataStreamGroupWindowAggregate",
+      unaryNode(
+        "DataStreamCalc",
+        streamTableNode(0),
+        term("select", "string", "int", "rowtime")
+      ),
       term("groupBy", "string"),
-      term("window",
-        TumblingGroupWindow(
-          WindowReference("w"),
-          'long,
-          5.milli)),
+      term("window", TumblingGroupWindow(WindowReference("w"), 'rowtime, 5.milli)),
       term("select",
         "string",
         "COUNT(int) AS TMP_0",
@@ -736,23 +794,22 @@ class GroupWindowTest extends TableTestBase {
   @Test
   def testSlideWindowStartEnd(): Unit = {
     val util = streamTestUtil()
-    val table = util.addTable[(Long, Int, String)]('long.rowtime, 'int, 'string)
+    val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
 
     val windowedTable = table
-      .window(Slide over 10.milli every 5.milli on 'long as 'w)
+      .window(Slide over 10.milli every 5.milli on 'rowtime as 'w)
       .groupBy('w, 'string)
       .select('string, 'int.count, 'w.start, 'w.end)
 
     val expected = unaryNode(
-      "DataStreamAggregate",
-      streamTableNode(0),
+      "DataStreamGroupWindowAggregate",
+      unaryNode(
+        "DataStreamCalc",
+        streamTableNode(0),
+        term("select", "string", "int", "rowtime")
+      ),
       term("groupBy", "string"),
-      term("window",
-        SlidingGroupWindow(
-          WindowReference("w"),
-          'long,
-          10.milli,
-          5.milli)),
+      term("window", SlidingGroupWindow(WindowReference("w"), 'rowtime, 10.milli, 5.milli)),
       term("select",
         "string",
         "COUNT(int) AS TMP_0",
@@ -776,14 +833,10 @@ class GroupWindowTest extends TableTestBase {
     val expected = unaryNode(
       "DataStreamCalc",
       unaryNode(
-        "DataStreamAggregate",
+        "DataStreamGroupWindowAggregate",
         streamTableNode(0),
         term("groupBy", "string"),
-        term("window",
-          SessionGroupWindow(
-            WindowReference("w"),
-            'long,
-            3.milli)),
+        term("window", SessionGroupWindow(WindowReference("w"), 'long, 3.milli)),
         term("select",
           "string",
           "COUNT(int) AS TMP_1",
@@ -810,14 +863,10 @@ class GroupWindowTest extends TableTestBase {
     val expected = unaryNode(
       "DataStreamCalc",
       unaryNode(
-        "DataStreamAggregate",
+        "DataStreamGroupWindowAggregate",
         streamTableNode(0),
         term("groupBy", "string"),
-        term("window",
-          TumblingGroupWindow(
-            WindowReference("w"),
-            'long,
-            5.millis)),
+        term("window", TumblingGroupWindow(WindowReference("w"), 'long, 5.millis)),
         term("select",
           "string",
           "SUM(int) AS TMP_0",
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UnsupportedOpsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UnsupportedOpsTest.scala
index b6a66607d8d8c..c72249ae3c47c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UnsupportedOpsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/UnsupportedOpsTest.scala
@@ -27,13 +27,6 @@ import org.junit.Test
 
 class UnsupportedOpsTest extends StreamingMultipleProgramsTestBase {
 
-  @Test(expected = classOf[ValidationException])
-  def testSelectWithAggregation(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv).select('_1.min)
-  }
-
   @Test(expected = classOf[ValidationException])
   def testDistinct(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment

From 292633b5812e527675e11bb74af8cc5e9391b0cc Mon Sep 17 00:00:00 2001
From: "hequn.chq" <hequn.chq@alibaba-inc.com>
Date: Fri, 7 Apr 2017 13:12:04 +0800
Subject: [PATCH 158/253] [FLINK-6090] [table] Add RetractionRules for
 annotating AccMode to DataStreamRel nodes.

---
 .../datastream/DataStreamCorrelate.scala      |   6 +-
 .../datastream/DataStreamGroupAggregate.scala |   6 +
 .../DataStreamGroupWindowAggregate.scala      |   4 +
 .../datastream/DataStreamOverAggregate.scala  |   6 +-
 .../plan/nodes/datastream/DataStreamRel.scala |  17 +
 .../datastream/retractionTraitDefs.scala      |  81 +++++
 .../nodes/datastream/retractionTraits.scala   | 100 ++++++
 .../table/plan/rules/FlinkRuleSets.scala      |   7 +-
 .../DataStreamRetractionRules.scala           | 248 ++++++++++++++
 .../table/CalciteConfigBuilderTest.scala      |  39 +--
 .../plan/rules/RetractionRulesTest.scala      | 321 ++++++++++++++++++
 11 files changed, 810 insertions(+), 25 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraitDefs.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
index 8955110f96d27..899d8efed7dde 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
@@ -37,14 +37,14 @@ class DataStreamCorrelate(
     cluster: RelOptCluster,
     traitSet: RelTraitSet,
     inputSchema: RowSchema,
-    inputNode: RelNode,
+    input: RelNode,
     scan: FlinkLogicalTableFunctionScan,
     condition: Option[RexNode],
     schema: RowSchema,
     joinSchema: RowSchema,
     joinType: SemiJoinType,
     ruleDescription: String)
-  extends SingleRel(cluster, traitSet, inputNode)
+  extends SingleRel(cluster, traitSet, input)
   with CommonCorrelate
   with DataStreamRel {
 
@@ -86,7 +86,7 @@ class DataStreamCorrelate(
     val config = tableEnv.getConfig
 
     // we do not need to specify input type
-    val inputDS = inputNode.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputDS = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
 
     val funcRel = scan.asInstanceOf[FlinkLogicalTableFunctionScan]
     val rexCall = funcRel.getCall.asInstanceOf[RexCall]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index 19f90c7da2d34..3555c80a2ac65 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -60,6 +60,12 @@ class DataStreamGroupAggregate(
 
   override def deriveRowType() = schema.logicalType
 
+  override def needsUpdatesAsRetraction = true
+
+  override def producesUpdates = true
+
+  override def consumesRetractions = true
+
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamGroupAggregate(
       cluster,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index 5aced669ef6cc..ea4b0bf211a43 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -53,6 +53,10 @@ class DataStreamGroupWindowAggregate(
 
   override def deriveRowType(): RelDataType = schema.logicalType
 
+  override def needsUpdatesAsRetraction = true
+
+  override def consumesRetractions = true
+
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamGroupWindowAggregate(
       window,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index db31f3218f2d2..fb912c42dc3ee 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -49,6 +49,10 @@ class DataStreamOverAggregate(
 
   override def deriveRowType(): RelDataType = schema.logicalType
 
+  override def needsUpdatesAsRetraction = true
+
+  override def consumesRetractions = true
+
   override def copy(traitSet: RelTraitSet, inputs: JList[RelNode]): RelNode = {
     new DataStreamOverAggregate(
       logicWindow,
@@ -297,7 +301,7 @@ class DataStreamOverAggregate(
     }ORDER BY: ${orderingToString(inputSchema.logicalType,
         overWindow.orderKeys.getFieldCollations)}, " +
       s"${if (overWindow.isRows) "ROWS" else "RANGE"}" +
-      s"${windowRange(logicWindow, overWindow, inputNode.asInstanceOf[DataStreamRel])}, " +
+      s"${windowRange(logicWindow, overWindow, inputNode)}, " +
       s"select: (${
         aggregationToString(
           inputSchema.logicalType,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
index 03938f3b4d91b..128da81b8a2ed 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
@@ -33,4 +33,21 @@ trait DataStreamRel extends FlinkRelNode {
     */
   def translateToPlan(tableEnv: StreamTableEnvironment) : DataStream[Row]
 
+  /**
+    * Whether the [[DataStreamRel]] requires that update and delete changes are sent with retraction
+    * messages.
+    */
+  def needsUpdatesAsRetraction: Boolean = false
+
+  /**
+    * Whether the [[DataStreamRel]] produces update and delete changes.
+    */
+  def producesUpdates: Boolean = false
+
+  /**
+    * Wheter the [[DataStreamRel]] consumes retraction messages instead of forwarding them.
+    * The node might or might not produce new retraction messages.
+    */
+  def consumesRetractions: Boolean = false
+
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraitDefs.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraitDefs.scala
new file mode 100644
index 0000000000000..c43d951344d0b
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraitDefs.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.nodes.datastream
+
+import org.apache.calcite.plan.{RelOptPlanner, RelTraitDef}
+import org.apache.calcite.rel.RelNode
+
+/**
+  * Definition of the [[UpdateAsRetractionTrait]].
+  */
+class UpdateAsRetractionTraitDef extends RelTraitDef[UpdateAsRetractionTrait] {
+  override def convert(
+      planner: RelOptPlanner,
+      rel: RelNode,
+      toTrait: UpdateAsRetractionTrait,
+      allowInfiniteCostConverters: Boolean): RelNode = {
+
+    rel.copy(rel.getTraitSet.plus(toTrait), rel.getInputs)
+  }
+
+  override def canConvert(
+      planner: RelOptPlanner,
+      fromTrait: UpdateAsRetractionTrait,
+      toTrait: UpdateAsRetractionTrait): Boolean = true
+
+  override def getTraitClass: Class[UpdateAsRetractionTrait] = classOf[UpdateAsRetractionTrait]
+
+  override def getSimpleName: String = this.getClass.getSimpleName
+
+  override def getDefault: UpdateAsRetractionTrait = UpdateAsRetractionTrait.DEFAULT
+}
+
+object UpdateAsRetractionTraitDef {
+  val INSTANCE = new UpdateAsRetractionTraitDef
+}
+
+/**
+  * Definition of the [[AccModeTrait]].
+  */
+class AccModeTraitDef extends RelTraitDef[AccModeTrait] {
+
+  override def convert(
+      planner: RelOptPlanner,
+      rel: RelNode,
+      toTrait: AccModeTrait,
+      allowInfiniteCostConverters: Boolean): RelNode = {
+
+    rel.copy(rel.getTraitSet.plus(toTrait), rel.getInputs)
+  }
+
+  override def canConvert(
+      planner: RelOptPlanner,
+      fromTrait: AccModeTrait,
+      toTrait: AccModeTrait): Boolean = true
+
+  override def getTraitClass: Class[AccModeTrait] = classOf[AccModeTrait]
+
+  override def getSimpleName: String = this.getClass.getSimpleName
+
+  override def getDefault: AccModeTrait = AccModeTrait.DEFAULT
+}
+
+object AccModeTraitDef {
+  val INSTANCE = new AccModeTraitDef
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala
new file mode 100644
index 0000000000000..c3b43ba69f1f3
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.nodes.datastream
+
+import org.apache.calcite.plan.{RelOptPlanner, RelTrait, RelTraitDef}
+import org.apache.flink.table.plan.nodes.datastream.AccMode.AccMode
+
+/** Tracks if a [[org.apache.calcite.rel.RelNode]] needs to send update and delete changes as
+  * retraction messages.
+  */
+class UpdateAsRetractionTrait extends RelTrait {
+
+  /**
+    * Defines whether the [[org.apache.calcite.rel.RelNode]] needs to send update and delete
+    * changes as retraction messages.
+    */
+  private var updateAsRetraction: Boolean = false
+
+  def this(updateAsRetraction: Boolean) {
+    this()
+    this.updateAsRetraction = updateAsRetraction
+  }
+
+  def sendsUpdatesAsRetractions: Boolean = updateAsRetraction
+
+  override def register(planner: RelOptPlanner): Unit = { }
+
+  override def getTraitDef: RelTraitDef[_ <: RelTrait] = UpdateAsRetractionTraitDef.INSTANCE
+
+  override def satisfies(`trait`: RelTrait): Boolean = this.equals(`trait`)
+
+  override def toString: String = updateAsRetraction.toString
+
+}
+
+object UpdateAsRetractionTrait {
+  val DEFAULT = new UpdateAsRetractionTrait(false)
+}
+
+/**
+  * Tracks the AccMode of a [[org.apache.calcite.rel.RelNode]].
+  */
+class AccModeTrait extends RelTrait {
+
+  /** Defines the accumulating mode for a operator. */
+  private var accMode = AccMode.Acc
+
+  def this(accMode: AccMode) {
+    this()
+    this.accMode = accMode
+  }
+
+  def getAccMode: AccMode = accMode
+
+  override def register(planner: RelOptPlanner): Unit = { }
+
+  override def getTraitDef: RelTraitDef[_ <: RelTrait] = AccModeTraitDef.INSTANCE
+
+  override def satisfies(`trait`: RelTrait): Boolean = this.equals(`trait`)
+
+  override def toString: String = accMode.toString
+}
+
+object AccModeTrait {
+  val DEFAULT = new AccModeTrait(AccMode.Acc)
+}
+
+/**
+  * The AccMode indicates which kinds of messages a [[org.apache.calcite.rel.RelNode]] might
+  * produce.
+  * In [[AccMode.Acc]] the node only emit accumulate messages.
+  * In [[AccMode.AccRetract]], the node produces accumulate messages for insert changes,
+  * retraction messages for delete changes, and accumulate and retraction messages
+  * for update changes.
+  */
+object AccMode extends Enumeration {
+  type AccMode = Value
+
+  val Acc        = Value // Operator produces only accumulate (insert) messages
+  val AccRetract = Value // Operator produces accumulate (insert, update) and
+                         //   retraction (delete, update) messages
+}
+
+
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
index f4de651e9ab44..fad60fde6683c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
@@ -21,9 +21,9 @@ package org.apache.flink.table.plan.rules
 import org.apache.calcite.rel.rules._
 import org.apache.calcite.tools.{RuleSet, RuleSets}
 import org.apache.flink.table.plan.rules.common._
+import org.apache.flink.table.plan.rules.logical._
 import org.apache.flink.table.plan.rules.dataSet._
 import org.apache.flink.table.plan.rules.datastream._
-import org.apache.flink.table.plan.rules.logical._
 import org.apache.flink.table.plan.nodes.logical._
 
 object FlinkRuleSets {
@@ -191,7 +191,10 @@ object FlinkRuleSets {
     * RuleSet to decorate plans for stream / DataStream execution
     */
   val DATASTREAM_DECO_RULES: RuleSet = RuleSets.ofList(
-    // rules
+    // retraction rules
+    DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE,
+    DataStreamRetractionRules.UPDATES_AS_RETRACTION_INSTANCE,
+    DataStreamRetractionRules.ACCMODE_INSTANCE
   )
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
new file mode 100644
index 0000000000000..aeb67b6f73fd2
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.rules.datastream
+
+import org.apache.calcite.plan.RelOptRule.{operand, _}
+import org.apache.calcite.plan.hep.HepRelVertex
+import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall}
+import org.apache.calcite.rel.RelNode
+import org.apache.flink.table.plan.nodes.datastream._
+
+import scala.collection.JavaConverters._
+
+/**
+  * Collection of rules to annotate [[DataStreamRel]] nodes with retraction information.
+  *
+  * The rules have to be applied in the following order:
+  * - [[DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE]]
+  * - [[DataStreamRetractionRules.UPDATES_AS_RETRACTION_INSTANCE]]
+  * - [[DataStreamRetractionRules.ACCMODE_INSTANCE]]
+  *
+  * The rules will assign a [[AccModeTrait]] to each [[DataStreamRel]] node of the plan. The
+  * trait defines the [[AccMode]] a node.
+  * - [[AccMode.Acc]] defines that the node produces only accumulate messages, i.e., all types of
+  * modifications (insert, update, delete) are encoded as accumulate messages.
+  * - [[AccMode.AccRetract]] defines that the node produces accumulate and retraction messages.
+  * Insert modifications are encoded as accumulate message, delete modifications as retraction
+  * message, and update modifications as a pair of accumulate and retraction messages.
+  *
+  */
+object DataStreamRetractionRules {
+
+  /**
+    * Rule instance that assigns default retraction to [[DataStreamRel]] nodes.
+    */
+  val DEFAULT_RETRACTION_INSTANCE = new AssignDefaultRetractionRule()
+
+  /**
+    * Rule instance that checks if [[DataStreamRel]] nodes need to ship updates as retractions.
+    */
+  val UPDATES_AS_RETRACTION_INSTANCE = new SetUpdatesAsRetractionRule()
+
+  /**
+    * Rule instance that assigns the [[AccMode]] to [[DataStreamRel]] nodes.
+    */
+  val ACCMODE_INSTANCE = new SetAccModeRule()
+
+  /**
+    * Get all children RelNodes of a RelNode.
+    *
+    * @param parent The parent RelNode
+    * @return All child nodes
+    */
+  def getChildRelNodes(parent: RelNode): Seq[RelNode] = {
+    parent.getInputs.asScala.map(_.asInstanceOf[HepRelVertex].getCurrentRel)
+  }
+
+  /**
+    * Checks if a [[RelNode]] ships updates as retractions.
+    *
+    * @param node The node to check.
+    * @return True if the node ships updates as retractions, false otherwise.
+    */
+  def sendsUpdatesAsRetraction(node: RelNode): Boolean = {
+    val retractionTrait = node.getTraitSet.getTrait(UpdateAsRetractionTraitDef.INSTANCE)
+    retractionTrait != null && retractionTrait.sendsUpdatesAsRetractions
+  }
+
+  /**
+    * Rule that assigns the default retraction information to [[DataStreamRel]] nodes.
+    * The default is to not publish updates as retraction messages and [[AccMode.Acc]].
+    */
+  class AssignDefaultRetractionRule extends RelOptRule(
+    operand(
+      classOf[DataStreamRel], none()),
+    "AssignDefaultRetractionRule") {
+
+    override def onMatch(call: RelOptRuleCall): Unit = {
+      val rel = call.rel(0).asInstanceOf[DataStreamRel]
+      val traits = rel.getTraitSet
+
+      val traitsWithUpdateAsRetrac = if (!traits.contains(UpdateAsRetractionTraitDef.INSTANCE)) {
+        traits.plus(UpdateAsRetractionTrait.DEFAULT)
+      } else {
+        traits
+      }
+      val traitsWithAccMode =
+        if (!traitsWithUpdateAsRetrac.contains(AccModeTraitDef.INSTANCE)) {
+          traitsWithUpdateAsRetrac.plus(AccModeTrait.DEFAULT)
+      } else {
+        traitsWithUpdateAsRetrac
+      }
+
+      if (traits != traitsWithAccMode) {
+        call.transformTo(rel.copy(traitsWithAccMode, rel.getInputs))
+      }
+    }
+  }
+
+  /**
+    * Rule that annotates all [[DataStreamRel]] nodes that need to sent out update and delete
+    * changes as retraction messages.
+    */
+  class SetUpdatesAsRetractionRule extends RelOptRule(
+    operand(
+      classOf[DataStreamRel], none()),
+    "SetUpdatesAsRetractionRule") {
+
+    /**
+      * Checks if a [[RelNode]] requires that update and delete changes are sent with retraction
+      * messages.
+      */
+    def needsUpdatesAsRetraction(node: RelNode): Boolean = {
+      node match {
+        case _ if sendsUpdatesAsRetraction(node) => true
+        case dsr: DataStreamRel => dsr.needsUpdatesAsRetraction
+      }
+    }
+
+    /**
+      * Annotates a [[RelNode]] to send out update and delete changes as retraction messages.
+      */
+    def setUpdatesAsRetraction(relNode: RelNode): RelNode = {
+      val traitSet = relNode.getTraitSet
+      relNode.copy(traitSet.plus(new UpdateAsRetractionTrait(true)), relNode.getInputs)
+    }
+
+    /**
+      * Annotates the children of a parent node with the information that they need to forward
+      * update and delete modifications as retraction messages.
+      *
+      * A child needs to produce retraction messages, if
+      *
+      * 1. its parent requires retraction messages by itself because it is a certain type
+      *    of operator, such as a [[DataStreamGroupAggregate]] or [[DataStreamOverAggregate]], or
+      * 2. its parent requires retraction because its own parent requires retraction
+      *    (transitive requirement).
+      *
+      */
+    override def onMatch(call: RelOptRuleCall): Unit = {
+      val parent = call.rel(0).asInstanceOf[DataStreamRel]
+
+      val children = getChildRelNodes(parent)
+      // check if children need to sent out retraction messages
+      val newChildren = for (c <- children) yield {
+        if (needsUpdatesAsRetraction(parent) && !sendsUpdatesAsRetraction(c)) {
+          setUpdatesAsRetraction(c)
+        } else {
+          c
+        }
+      }
+
+      // update parent if a child was updated
+      if (children != newChildren) {
+        call.transformTo(parent.copy(parent.getTraitSet, newChildren.asJava))
+      }
+    }
+  }
+
+  /**
+    * Sets the [[AccMode]] of [[DataStreamRel]] nodes.
+    */
+  class SetAccModeRule extends RelOptRule(
+    operand(
+      classOf[DataStreamRel], none()),
+    "SetAccModeRule") {
+
+    /**
+      * Checks if a [[RelNode]] produces update and delete changes.
+      */
+    def producesUpdates(relNode: RelNode): Boolean = {
+      relNode match {
+        case dsr: DataStreamRel => dsr.producesUpdates
+      }
+    }
+
+    /**
+      * Checks if a [[RelNode]] is in [[AccMode.AccRetract]] mode.
+      */
+    def isAccRetract(node: RelNode): Boolean = {
+      val accModeTrait = node.getTraitSet.getTrait(AccModeTraitDef.INSTANCE)
+      null != accModeTrait && accModeTrait.getAccMode == AccMode.AccRetract
+    }
+
+    /**
+      * Set [[AccMode.AccRetract]] to a [[RelNode]].
+      */
+    def setAccRetract(relNode: RelNode): RelNode = {
+      val traitSet = relNode.getTraitSet
+      relNode.copy(traitSet.plus(new AccModeTrait(AccMode.AccRetract)), relNode.getInputs)
+    }
+
+    /**
+      * Checks if a [[RelNode]] consumes retraction messages instead of forwarding them.
+      * The node might or might not produce new retraction messages.
+      * This is checked by [[producesRetractions()]].
+      */
+    def consumesRetractions(relNode: RelNode): Boolean = {
+      relNode match {
+        case dsr: DataStreamRel => dsr.consumesRetractions
+      }
+    }
+
+    /**
+      * Checks if a [[RelNode]] produces retraction messages.
+      */
+    def producesRetractions(node: RelNode): Boolean = {
+      sendsUpdatesAsRetraction(node) && producesUpdates(node)
+    }
+
+    /**
+      * Checks if a [[RelNode]] forwards retraction messages from its children.
+      */
+    def forwardsRetractions(parent: RelNode, children: Seq[RelNode]): Boolean = {
+      children.exists(c => isAccRetract(c)) && !consumesRetractions(parent)
+    }
+
+    /**
+      * Updates the [[AccMode]] of a [[RelNode]] and its children if necessary.
+      */
+    override def onMatch(call: RelOptRuleCall): Unit = {
+      val parent = call.rel(0).asInstanceOf[DataStreamRel]
+      val children = getChildRelNodes(parent)
+
+      // check if the AccMode of the parent needs to be updated
+      if (!isAccRetract(parent) &&
+          (producesRetractions(parent) || forwardsRetractions(parent, children))) {
+        call.transformTo(setAccRetract(parent))
+      }
+    }
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/CalciteConfigBuilderTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/CalciteConfigBuilderTest.scala
index ed29f0f8dfb2a..0c337e072b4ec 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/CalciteConfigBuilderTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/CalciteConfigBuilderTest.scala
@@ -22,6 +22,7 @@ import org.apache.calcite.rel.rules._
 import org.apache.calcite.sql.fun.{OracleSqlOperatorTable, SqlStdOperatorTable}
 import org.apache.calcite.tools.RuleSets
 import org.apache.flink.table.calcite.{CalciteConfig, CalciteConfigBuilder}
+import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
 import org.junit.Assert._
 import org.junit.Test
 
@@ -51,11 +52,11 @@ class CalciteConfigBuilderTest {
   def testRules(): Unit = {
 
     val cc: CalciteConfig = new CalciteConfigBuilder()
-        .addNormRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE))
-        .replaceLogicalOptRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE))
-        .replacePhysicalOptRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE))
-        .replaceDecoRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE))
-        .build()
+      .addNormRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE))
+      .replaceLogicalOptRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE))
+      .replacePhysicalOptRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE))
+      .replaceDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
+      .build()
 
     assertFalse(cc.replacesNormRuleSet)
     assertTrue(cc.getNormRuleSet.isDefined)
@@ -232,62 +233,62 @@ class CalciteConfigBuilderTest {
   def testReplaceDecorationRules(): Unit = {
 
     val cc: CalciteConfig = new CalciteConfigBuilder()
-      .replaceDecoRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE))
+      .replaceDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
       .build()
 
     assertEquals(true, cc.replacesDecoRuleSet)
     assertTrue(cc.getDecoRuleSet.isDefined)
     val cSet = cc.getDecoRuleSet.get.iterator().asScala.toSet
     assertEquals(1, cSet.size)
-    assertTrue(cSet.contains(ReduceExpressionsRule.FILTER_INSTANCE))
+    assertTrue(cSet.contains(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
   }
 
   @Test
   def testReplaceDecorationAddRules(): Unit = {
 
     val cc: CalciteConfig = new CalciteConfigBuilder()
-      .replaceDecoRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE))
-      .addDecoRuleSet(RuleSets.ofList(ReduceExpressionsRule.PROJECT_INSTANCE))
+      .replaceDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
+      .addDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.UPDATES_AS_RETRACTION_INSTANCE))
       .build()
 
     assertEquals(true, cc.replacesDecoRuleSet)
     assertTrue(cc.getDecoRuleSet.isDefined)
     val cSet = cc.getDecoRuleSet.get.iterator().asScala.toSet
     assertEquals(2, cSet.size)
-    assertTrue(cSet.contains(ReduceExpressionsRule.FILTER_INSTANCE))
-    assertTrue(cSet.contains(ReduceExpressionsRule.PROJECT_INSTANCE))
+    assertTrue(cSet.contains(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
+    assertTrue(cSet.contains(DataStreamRetractionRules.UPDATES_AS_RETRACTION_INSTANCE))
   }
 
   @Test
   def testAddDecorationRules(): Unit = {
 
     val cc: CalciteConfig = new CalciteConfigBuilder()
-      .addDecoRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE))
+      .addDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
       .build()
 
     assertEquals(false, cc.replacesDecoRuleSet)
     assertTrue(cc.getDecoRuleSet.isDefined)
     val cSet = cc.getDecoRuleSet.get.iterator().asScala.toSet
     assertEquals(1, cSet.size)
-    assertTrue(cSet.contains(ReduceExpressionsRule.FILTER_INSTANCE))
+    assertTrue(cSet.contains(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
   }
 
   @Test
   def testAddAddDecorationRules(): Unit = {
 
     val cc: CalciteConfig = new CalciteConfigBuilder()
-      .addDecoRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE))
-      .addDecoRuleSet(RuleSets.ofList(ReduceExpressionsRule.PROJECT_INSTANCE,
-                                      ReduceExpressionsRule.CALC_INSTANCE))
+      .addDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE))
+      .addDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.UPDATES_AS_RETRACTION_INSTANCE,
+                                      DataStreamRetractionRules.ACCMODE_INSTANCE))
       .build()
 
     assertEquals(false, cc.replacesDecoRuleSet)
     assertTrue(cc.getDecoRuleSet.isDefined)
     val cList = cc.getDecoRuleSet.get.iterator().asScala.toList
     assertEquals(3, cList.size)
-    assertEquals(cList.head, ReduceExpressionsRule.FILTER_INSTANCE)
-    assertEquals(cList(1), ReduceExpressionsRule.PROJECT_INSTANCE)
-    assertEquals(cList(2), ReduceExpressionsRule.CALC_INSTANCE)
+    assertEquals(cList.head, DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE)
+    assertEquals(cList(1), DataStreamRetractionRules.UPDATES_AS_RETRACTION_INSTANCE)
+    assertEquals(cList(2), DataStreamRetractionRules.ACCMODE_INSTANCE)
   }
 
   @Test
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala
new file mode 100644
index 0000000000000..580029f5c2101
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.rules
+
+import org.apache.calcite.rel.RelNode
+import org.apache.flink.table.api.Table
+import org.apache.flink.table.plan.nodes.datastream._
+import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase}
+import org.apache.flink.table.utils.TableTestUtil._
+import org.junit.Assert._
+import org.junit.{Ignore, Test}
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.scala._
+
+
+class RetractionRulesTest extends TableTestBase {
+
+  def streamTestForRetractionUtil(): StreamTableTestForRetractionUtil = {
+    new StreamTableTestForRetractionUtil()
+  }
+
+  @Test
+  def testSelect(): Unit = {
+    val util = streamTestForRetractionUtil()
+    val table = util.addTable[(String, Int)]('word, 'number)
+
+    val resultTable = table.select('word, 'number)
+
+    val expected = s"DataStreamScan(false, Acc)"
+
+    util.verifyTableTrait(resultTable, expected)
+  }
+
+  // one level unbounded groupBy
+  @Test
+  def testGroupBy(): Unit = {
+    val util = streamTestForRetractionUtil()
+    val table = util.addTable[(String, Int)]('word, 'number)
+    val defaultStatus = "false, Acc"
+
+    val resultTable = table
+      .groupBy('word)
+      .select('number.count)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          "DataStreamScan(true, Acc)",
+          s"$defaultStatus"
+        ),
+        s"$defaultStatus"
+      )
+
+    util.verifyTableTrait(resultTable, expected)
+  }
+
+  // two level unbounded groupBy
+  @Test
+  def testTwoGroupBy(): Unit = {
+    val util = streamTestForRetractionUtil()
+    val table = util.addTable[(String, Int)]('word, 'number)
+    val defaultStatus = "false, Acc"
+
+    val resultTable = table
+      .groupBy('word)
+      .select('word, 'number.count as 'count)
+      .groupBy('count)
+      .select('count, 'count.count as 'frequency)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            unaryNode(
+              "DataStreamGroupAggregate",
+              "DataStreamScan(true, Acc)",
+              "true, AccRetract"
+            ),
+            "true, AccRetract"
+          ),
+          s"$defaultStatus"
+        ),
+        s"$defaultStatus"
+      )
+
+    util.verifyTableTrait(resultTable, expected)
+  }
+
+  // group window
+  @Test
+  def testGroupWindow(): Unit = {
+    val util = streamTestForRetractionUtil()
+    val table = util.addTable[(String, Int)]('word, 'number, 'rowtime.rowtime)
+    val defaultStatus = "false, Acc"
+
+    val resultTable = table
+      .window(Tumble over 50.milli on 'rowtime as 'w)
+      .groupBy('w, 'word)
+      .select('word, 'number.count as 'count)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupWindowAggregate",
+          "DataStreamScan(true, Acc)",
+          s"$defaultStatus"
+        ),
+        s"$defaultStatus"
+      )
+
+    util.verifyTableTrait(resultTable, expected)
+  }
+
+  // group window after unbounded groupBy
+  @Test
+  @Ignore // cannot pass rowtime through non-windowed aggregation
+  def testGroupWindowAfterGroupBy(): Unit = {
+    val util = streamTestForRetractionUtil()
+    val table = util.addTable[(String, Int)]('word, 'number, 'rowtime.rowtime)
+    val defaultStatus = "false, Acc"
+
+    val resultTable = table
+      .groupBy('word)
+      .select('word, 'number.count as 'count)
+      .window(Tumble over 50.milli on 'rowtime as 'w)
+      .groupBy('w, 'count)
+      .select('count, 'count.count as 'frequency)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupWindowAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            unaryNode(
+              "DataStreamGroupAggregate",
+              "DataStreamScan(true, Acc)",
+              "true, AccRetract"
+            ),
+            "true, AccRetract"
+          ),
+          s"$defaultStatus"
+        ),
+        s"$defaultStatus"
+      )
+
+    util.verifyTableTrait(resultTable, expected)
+  }
+
+  // over window
+  @Test
+  def testOverWindow(): Unit = {
+    val util = streamTestForRetractionUtil()
+    util.addTable[(String, Int)]("T1", 'word, 'number, 'proctime.proctime)
+    val defaultStatus = "false, Acc"
+
+    val sqlQuery =
+      "SELECT " +
+        "word, count(number) " +
+        "OVER (PARTITION BY word ORDER BY proctime " +
+        "ROWS BETWEEN UNBOUNDED preceding AND CURRENT ROW)" +
+        "FROM T1"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          "DataStreamScan(true, Acc)",
+          s"$defaultStatus"
+        ),
+        s"$defaultStatus"
+      )
+
+    util.verifySqlTrait(sqlQuery, expected)
+  }
+
+
+  // over window after unbounded groupBy
+  @Test
+  @Ignore // cannot pass rowtime through non-windowed aggregation
+  def testOverWindowAfterGroupBy(): Unit = {
+    val util = streamTestForRetractionUtil()
+    util.addTable[(String, Int)]("T1", 'word, 'number, 'proctime.proctime)
+    val defaultStatus = "false, Acc"
+
+    val sqlQuery =
+      "SELECT " +
+        "_count, count(word) " +
+        "OVER (PARTITION BY _count ORDER BY proctime " +
+        "ROWS BETWEEN UNBOUNDED preceding AND CURRENT ROW)" +
+        "FROM " +
+        "(SELECT word, count(number) as _count FROM T1 GROUP BY word) "
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            unaryNode(
+              "DataStreamGroupAggregate",
+              "DataStreamScan(true, Acc)",
+              "true, AccRetract"
+            ),
+            "true, AccRetract"
+          ),
+          s"$defaultStatus"
+        ),
+        s"$defaultStatus"
+      )
+
+    util.verifySqlTrait(sqlQuery, expected)
+  }
+
+  // test binaryNode
+  @Test
+  def testBinaryNode(): Unit = {
+    val util = streamTestForRetractionUtil()
+    val lTable = util.addTable[(String, Int)]('word, 'number)
+    val rTable = util.addTable[(String, Long)]('word_r, 'count_r)
+    val defaultStatus = "false, Acc"
+
+    val resultTable = lTable
+      .groupBy('word)
+      .select('word, 'number.count as 'count)
+      .unionAll(rTable)
+      .groupBy('count)
+      .select('count, 'count.count as 'frequency)
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            binaryNode(
+              "DataStreamUnion",
+              unaryNode(
+                "DataStreamCalc",
+                unaryNode(
+                  "DataStreamGroupAggregate",
+                  "DataStreamScan(true, Acc)",
+                  "true, AccRetract"
+                ),
+                "true, AccRetract"
+              ),
+              "DataStreamScan(true, Acc)",
+              "true, AccRetract"
+            ),
+            "true, AccRetract"
+          ),
+          s"$defaultStatus"
+        ),
+        s"$defaultStatus"
+      )
+
+    util.verifyTableTrait(resultTable, expected)
+  }
+}
+
+class StreamTableTestForRetractionUtil extends StreamTableTestUtil {
+
+  def verifySqlTrait(query: String, expected: String): Unit = {
+    verifyTableTrait(tEnv.sql(query), expected)
+  }
+
+  def verifyTableTrait(resultTable: Table, expected: String): Unit = {
+    val relNode = resultTable.getRelNode
+    val optimized = tEnv.optimize(relNode)
+    val actual = TraitUtil.toString(optimized)
+    assertEquals(
+      expected.split("\n").map(_.trim).mkString("\n"),
+      actual.split("\n").map(_.trim).mkString("\n"))
+  }
+}
+
+object TraitUtil {
+  def toString(rel: RelNode): String = {
+    val className = rel.getClass.getSimpleName
+    var childString: String = ""
+    var i = 0
+    while (i < rel.getInputs.size()) {
+      childString += TraitUtil.toString(rel.getInput(i))
+      i += 1
+    }
+
+    val retractString = rel.getTraitSet.getTrait(UpdateAsRetractionTraitDef.INSTANCE).toString
+    val accModetString = rel.getTraitSet.getTrait(AccModeTraitDef.INSTANCE).toString
+
+    s"""$className($retractString, $accModetString)
+       |$childString
+       |""".stripMargin.stripLineEnd
+  }
+}
+

From e419dc22041a595eecc74469280a357ffdee71fd Mon Sep 17 00:00:00 2001
From: Hequn Cheng <chenghequn@gmail.com>
Date: Tue, 18 Apr 2017 16:54:09 +0800
Subject: [PATCH 159/253] [FLINK-6091] [table] Implement and turn on retraction
 for non-windowed aggregates.

---
 .../table/api/BatchTableEnvironment.scala     |  63 +++++-
 .../table/api/StreamTableEnvironment.scala    |  64 +++++-
 .../flink/table/api/TableEnvironment.scala    |  65 +++---
 .../flink/table/plan/nodes/CommonCalc.scala   |  32 ++-
 .../table/plan/nodes/CommonCorrelate.scala    | 139 ++++--------
 .../flink/table/plan/nodes/CommonScan.scala   |   7 +-
 .../table/plan/nodes/dataset/BatchScan.scala  |   2 +-
 .../plan/nodes/dataset/DataSetCalc.scala      |  16 +-
 .../plan/nodes/dataset/DataSetCorrelate.scala |  29 ++-
 .../plan/nodes/dataset/DataSetValues.scala    |   2 +-
 .../nodes/datastream/DataStreamCalc.scala     |  26 +--
 .../datastream/DataStreamCorrelate.scala      |  40 +++-
 .../datastream/DataStreamGroupAggregate.scala |  24 +-
 .../DataStreamGroupWindowAggregate.scala      |  52 +++--
 .../datastream/DataStreamOverAggregate.scala  |  72 +++---
 .../plan/nodes/datastream/DataStreamRel.scala |   6 +-
 .../nodes/datastream/DataStreamScan.scala     |   7 +-
 .../nodes/datastream/DataStreamUnion.scala    |   4 +-
 .../nodes/datastream/DataStreamValues.scala   |  13 +-
 .../plan/nodes/datastream/StreamScan.scala    |  27 ++-
 .../datastream/StreamTableSourceScan.scala    |   8 +-
 .../plan/nodes/logical/FlinkLogicalCalc.scala |   2 +-
 .../DataStreamRetractionRules.scala           |  16 +-
 .../runtime/CRowCorrelateFlatMapRunner.scala  |  83 +++++++
 .../table/runtime/CRowFlatMapRunner.scala     |  72 ++++++
 .../table/runtime/CRowInputMapRunner.scala    |  57 +++++
 .../table/runtime/CRowOutputMapRunner.scala   |  60 +++++
 .../table/runtime/CRowWrappingCollector.scala |  41 ++++
 .../flink/table/runtime/FlatMapRunner.scala   |  17 +-
 .../aggregate/AggregateAggFunction.scala      |  15 +-
 .../runtime/aggregate/AggregateUtil.scala     |  48 ++--
 ...tSessionWindowAggReduceGroupFunction.scala |   4 +-
 ...SetSlideWindowAggReduceGroupFunction.scala |   4 +-
 ...mbleTimeWindowAggReduceGroupFunction.scala |   4 +-
 .../aggregate/GroupAggProcessFunction.scala   |  58 +++--
 ...mentalAggregateAllTimeWindowFunction.scala |   7 +-
 ...ncrementalAggregateAllWindowFunction.scala |  11 +-
 ...crementalAggregateTimeWindowFunction.scala |   7 +-
 .../IncrementalAggregateWindowFunction.scala  |  13 +-
 .../aggregate/ProcTimeBoundedRangeOver.scala  |  30 +--
 .../aggregate/ProcTimeBoundedRowsOver.scala   |  26 ++-
 .../ProcTimeUnboundedNonPartitionedOver.scala |  23 +-
 .../ProcTimeUnboundedPartitionedOver.scala    |  22 +-
 .../aggregate/RowTimeBoundedRangeOver.scala   |  36 +--
 .../aggregate/RowTimeBoundedRowsOver.scala    |  30 +--
 .../aggregate/RowTimeUnboundedOver.scala      |  50 +++--
 .../TimeWindowPropertyCollector.scala         |  34 ++-
 .../runtime/io/CRowValuesInputFormat.scala    |  59 +++++
 .../table/runtime/io/ValuesInputFormat.scala  |  17 +-
 .../flink/table/runtime/types/CRow.scala      |  55 +++++
 .../table/runtime/types/CRowComparator.scala  |  83 +++++++
 .../table/runtime/types/CRowSerializer.scala  |  78 +++++++
 .../table/runtime/types/CRowTypeInfo.scala    |  98 +++++++++
 .../flink/table/sinks/CsvTableSink.scala      |   2 +
 .../flink/table/TableEnvironmentTest.scala    |  72 +++++-
 .../scala/batch/TableEnvironmentITCase.scala  |  20 ++
 .../api/scala/stream/RetractionITCase.scala   | 205 ++++++++++++++++++
 .../api/scala/stream/TableSinkITCase.scala    |   2 +-
 .../api/scala/stream/utils/StreamITCase.scala |  11 +-
 ...ocessingOverRangeProcessFunctionTest.scala | 105 ++++-----
 .../runtime/types/CRowComparatorTest.scala    |  61 ++++++
 .../table/utils/MockTableEnvironment.scala    |   8 +
 62 files changed, 1813 insertions(+), 531 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRow.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowComparator.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowTypeInfo.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowComparatorTest.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index 02c6063f524c2..f7955f055f871 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -26,16 +26,20 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.sql2rel.RelDecorrelator
 import org.apache.calcite.tools.RuleSet
+import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.io.DiscardingOutputFormat
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.api.java.{DataSet, ExecutionEnvironment}
+import org.apache.flink.configuration.Configuration
 import org.apache.flink.table.explain.PlanJsonParser
 import org.apache.flink.table.expressions.{Expression, RowtimeAttribute, TimeAttribute}
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.DataSetRel
 import org.apache.flink.table.plan.rules.FlinkRuleSets
 import org.apache.flink.table.plan.schema.{DataSetTable, TableSourceTable}
+import org.apache.flink.table.runtime.MapRunner
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.table.sinks.{BatchTableSink, TableSink}
 import org.apache.flink.table.sources.{BatchTableSource, TableSource}
 import org.apache.flink.types.Row
@@ -127,6 +131,56 @@ abstract class BatchTableEnvironment(
     }
   }
 
+  /**
+    * Creates a final converter that maps the internal row type to external type.
+    *
+    * @param physicalTypeInfo the input of the sink
+    * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
+    * @param requestedTypeInfo the output type of the sink
+    * @param functionName name of the map function. Must not be unique but has to be a
+    *                     valid Java class identifier.
+    */
+  override protected def getConversionMapper[IN, OUT](
+      physicalTypeInfo: TypeInformation[IN],
+      logicalRowType: RelDataType,
+      requestedTypeInfo: TypeInformation[OUT],
+      functionName: String):
+    Option[MapFunction[IN, OUT]] = {
+
+    if (requestedTypeInfo.getTypeClass == classOf[Row]) {
+      // Row to Row, no conversion needed
+      None
+    } else if (requestedTypeInfo.getTypeClass == classOf[CRow]) {
+      // Row to CRow, only needs to be wrapped
+      Some(
+        new RichMapFunction[Row, CRow] {
+          private var outCRow: CRow = _
+          override def open(parameters: Configuration): Unit = outCRow = new CRow(null, true)
+          override def map(value: Row): CRow = {
+            outCRow.row = value
+            outCRow
+          }
+        }.asInstanceOf[MapFunction[IN, OUT]]
+      )
+    } else {
+      // some type that is neither Row or CRow
+
+      val converterFunction = generateRowConverterFunction[OUT](
+        physicalTypeInfo.asInstanceOf[TypeInformation[Row]],
+        logicalRowType,
+        requestedTypeInfo,
+        functionName
+      )
+
+      val mapFunction = new MapRunner[IN, OUT](
+        converterFunction.name,
+        converterFunction.code,
+        converterFunction.returnType)
+
+      Some(mapFunction)
+    }
+  }
+
   /**
     * Returns the AST of the specified Table API and SQL queries and the execution plan to compute
     * the result of the given [[Table]].
@@ -293,10 +347,15 @@ abstract class BatchTableEnvironment(
     logicalPlan match {
       case node: DataSetRel =>
         val plan = node.translateToPlan(this)
-        val conversion = sinkConversion(plan.getType, logicalType, tpe, "DataSetSinkConversion")
+        val conversion =
+          getConversionMapper(plan.getType, logicalType, tpe, "DataSetSinkConversion")
         conversion match {
           case None => plan.asInstanceOf[DataSet[A]] // no conversion necessary
-          case Some(mapFunction) => plan.map(mapFunction).name(s"to: $tpe")
+          case Some(mapFunction: MapFunction[Row, A]) =>
+            plan.map(mapFunction)
+              .returns(tpe)
+              .name(s"to: ${tpe.getTypeClass.getSimpleName}")
+              .asInstanceOf[DataSet[A]]
         }
 
       case _ =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index dd2c09d5eab18..0632a47c55aa0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -28,6 +28,7 @@ import org.apache.calcite.sql2rel.RelDecorrelator
 import org.apache.calcite.tools.{RuleSet, RuleSets}
 import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
 import org.apache.flink.api.common.typeutils.CompositeType
+import org.apache.flink.api.common.functions.MapFunction
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
@@ -37,7 +38,9 @@ import org.apache.flink.table.expressions.{Expression, ProctimeAttribute, Rowtim
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.DataStreamRel
 import org.apache.flink.table.plan.rules.FlinkRuleSets
-import org.apache.flink.table.plan.schema.{DataStreamTable, StreamTableSourceTable, TableSourceTable}
+import org.apache.flink.table.plan.schema.{DataStreamTable, StreamTableSourceTable}
+import org.apache.flink.table.runtime.CRowInputMapRunner
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.apache.flink.table.sinks.{StreamTableSink, TableSink}
 import org.apache.flink.table.sources.{StreamTableSource, TableSource}
 import org.apache.flink.table.typeutils.TypeCheckUtils
@@ -132,6 +135,52 @@ abstract class StreamTableEnvironment(
     }
   }
 
+
+  /**
+    * Creates a final converter that maps the internal row type to external type.
+    *
+    * @param physicalTypeInfo the input of the sink
+    * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
+    * @param requestedTypeInfo the output type of the sink
+    * @param functionName name of the map function. Must not be unique but has to be a
+    *                     valid Java class identifier.
+    */
+  override protected def getConversionMapper[IN, OUT](
+      physicalTypeInfo: TypeInformation[IN],
+      logicalRowType: RelDataType,
+      requestedTypeInfo: TypeInformation[OUT],
+      functionName: String):
+  Option[MapFunction[IN, OUT]] = {
+
+    if (requestedTypeInfo.getTypeClass == classOf[CRow]) {
+      // CRow to CRow, no conversion needed
+      None
+    } else if (requestedTypeInfo.getTypeClass == classOf[Row]) {
+      // CRow to Row, only needs to be unwrapped
+      Some(
+        new MapFunction[CRow, Row] {
+          override def map(value: CRow): Row = value.row
+        }.asInstanceOf[MapFunction[IN, OUT]]
+      )
+    } else {
+      // Some type that is neither CRow nor Row
+
+      val converterFunction = generateRowConverterFunction[OUT](
+        physicalTypeInfo.asInstanceOf[CRowTypeInfo].rowType,
+        logicalRowType,
+        requestedTypeInfo,
+        functionName
+      )
+
+      Some(new CRowInputMapRunner[OUT](
+        converterFunction.name,
+        converterFunction.code,
+        converterFunction.returnType)
+        .asInstanceOf[MapFunction[IN, OUT]])
+
+    }
+  }
+
   /**
     * Registers a [[DataStream]] as a table under a given name in the [[TableEnvironment]]'s
     * catalog.
@@ -377,10 +426,15 @@ abstract class StreamTableEnvironment(
     logicalPlan match {
       case node: DataStreamRel =>
         val plan = node.translateToPlan(this)
-        val conversion = sinkConversion(plan.getType, logicalType, tpe, "DataStreamSinkConversion")
+        val conversion =
+          getConversionMapper(plan.getType, logicalType, tpe, "DataStreamSinkConversion")
         conversion match {
           case None => plan.asInstanceOf[DataStream[A]] // no conversion necessary
-          case Some(mapFunction) => plan.map(mapFunction).name(s"to: $tpe")
+          case Some(mapFunction: MapFunction[CRow, A]) =>
+            plan.map(mapFunction)
+              .returns(tpe)
+              .name(s"to: ${tpe.getTypeClass.getSimpleName}")
+              .asInstanceOf[DataStream[A]]
         }
 
       case _ =>
@@ -398,9 +452,9 @@ abstract class StreamTableEnvironment(
   def explain(table: Table): String = {
     val ast = table.getRelNode
     val optimizedPlan = optimize(ast)
-    val dataStream = translate[Row](
+    val dataStream = translate[CRow](
       optimizedPlan,
-      ast.getRowType)(new GenericTypeInfo(classOf[Row]))
+      ast.getRowType)(new GenericTypeInfo(classOf[CRow]))
 
     val env = dataStream.getExecutionEnvironment
     val jsonSqlPlan = env.getExecutionPlan
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 9ed5000a379fc..d27db1e67eb58 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -48,7 +48,7 @@ import org.apache.flink.table.api.java.{BatchTableEnvironment => JavaBatchTableE
 import org.apache.flink.table.api.scala.{BatchTableEnvironment => ScalaBatchTableEnv, StreamTableEnvironment => ScalaStreamTableEnv}
 import org.apache.flink.table.calcite.{FlinkPlannerImpl, FlinkRelBuilder, FlinkTypeFactory, FlinkTypeSystem}
 import org.apache.flink.table.catalog.{ExternalCatalog, ExternalCatalogSchema}
-import org.apache.flink.table.codegen.{CodeGenerator, ExpressionReducer}
+import org.apache.flink.table.codegen.{CodeGenerator, ExpressionReducer, GeneratedFunction}
 import org.apache.flink.table.expressions.{Alias, Expression, UnresolvedFieldReference}
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
 import org.apache.flink.table.functions.{ScalarFunction, TableFunction, AggregateFunction}
@@ -59,7 +59,7 @@ import org.apache.flink.table.plan.cost.DataSetCostFactory
 import org.apache.flink.table.plan.logical.{CatalogNode, LogicalRelNode}
 import org.apache.flink.table.plan.rules.FlinkRuleSets
 import org.apache.flink.table.plan.schema.RelTable
-import org.apache.flink.table.runtime.MapRunner
+import org.apache.flink.table.runtime.types.CRowTypeInfo
 import org.apache.flink.table.sinks.TableSink
 import org.apache.flink.table.sources.{DefinedFieldNames, TableSource}
 import org.apache.flink.table.validate.FunctionCatalog
@@ -644,6 +644,18 @@ abstract class TableEnvironment(val config: TableConfig) {
           case _ => throw new TableException(
             "Field reference expression or alias on field expression expected.")
         }
+      case cr: CRowTypeInfo =>
+        exprs.zipWithIndex.map {
+          case (UnresolvedFieldReference(name), idx) => (idx, name)
+          case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
+            val idx = cr.getFieldIndex(origName)
+            if (idx < 0) {
+              throw new TableException(s"$origName is not a field of type $cr")
+            }
+            (idx, name)
+          case _ => throw new TableException(
+            "Field reference expression or alias on field expression expected.")
+        }
       case c: CaseClassTypeInfo[A] =>
         exprs.zipWithIndex flatMap {
           case (UnresolvedFieldReference(name), idx) =>
@@ -694,37 +706,38 @@ abstract class TableEnvironment(val config: TableConfig) {
   /**
     * Creates a final converter that maps the internal row type to external type.
     *
-    * @param physicalRowTypeInfo the input of the sink
+    * @param physicalTypeInfo the input of the sink
     * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
     * @param requestedTypeInfo the output type of the sink
     * @param functionName name of the map function. Must not be unique but has to be a
     *                     valid Java class identifier.
     */
-  protected def sinkConversion[T](
-      physicalRowTypeInfo: TypeInformation[Row],
+  protected def getConversionMapper[IN, OUT](
+      physicalTypeInfo: TypeInformation[IN],
+      logicalRowType: RelDataType,
+      requestedTypeInfo: TypeInformation[OUT],
+      functionName: String):
+    Option[MapFunction[IN, OUT]]
+
+  protected def generateRowConverterFunction[OUT](
+      inputTypeInfo: TypeInformation[Row],
       logicalRowType: RelDataType,
-      requestedTypeInfo: TypeInformation[T],
-      functionName: String)
-    : Option[MapFunction[Row, T]] = {
+      requestedTypeInfo: TypeInformation[OUT],
+      functionName: String):
+    GeneratedFunction[MapFunction[Row, OUT], OUT] = {
 
     // validate that at least the field types of physical and logical type match
     // we do that here to make sure that plan translation was correct
     val logicalRowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(logicalRowType)
-    if (physicalRowTypeInfo != logicalRowTypeInfo) {
+    if (logicalRowTypeInfo != inputTypeInfo) {
       throw TableException("The field types of physical and logical row types do not match." +
         "This is a bug and should not happen. Please file an issue.")
     }
 
-    // requested type is a generic Row, no conversion needed
-    if (requestedTypeInfo.isInstanceOf[GenericTypeInfo[_]] &&
-          requestedTypeInfo.getTypeClass == classOf[Row]) {
-      return None
-    }
-
     // convert to type information
-    val logicalFieldTypes = logicalRowType.getFieldList.asScala map { relDataType =>
-      FlinkTypeFactory.toTypeInfo(relDataType.getType)
-    }
+    val logicalFieldTypes = logicalRowType.getFieldList.asScala
+      .map(t => FlinkTypeFactory.toTypeInfo(t.getType))
+
     // field names
     val logicalFieldNames = logicalRowType.getFieldNames.asScala
 
@@ -732,8 +745,8 @@ abstract class TableEnvironment(val config: TableConfig) {
     if (requestedTypeInfo.getArity != logicalFieldTypes.length) {
       throw new TableException("Arity of result does not match requested type.")
     }
-    requestedTypeInfo match {
 
+    requestedTypeInfo match {
       // POJO type requested
       case pt: PojoTypeInfo[_] =>
         logicalFieldNames.zip(logicalFieldTypes) foreach {
@@ -780,7 +793,7 @@ abstract class TableEnvironment(val config: TableConfig) {
     val generator = new CodeGenerator(
       config,
       false,
-      physicalRowTypeInfo,
+      inputTypeInfo,
       None,
       None)
 
@@ -794,20 +807,12 @@ abstract class TableEnvironment(val config: TableConfig) {
          |return ${conversion.resultTerm};
          |""".stripMargin
 
-    val genFunction = generator.generateFunction(
+    generator.generateFunction(
       functionName,
-      classOf[MapFunction[Row, T]],
+      classOf[MapFunction[Row, OUT]],
       body,
       requestedTypeInfo)
-
-    val mapFunction = new MapRunner[Row, T](
-      genFunction.name,
-      genFunction.code,
-      genFunction.returnType)
-
-    Some(mapFunction)
   }
-
 }
 
 /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
index ff5ffb202be8f..e8755878a3997 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
@@ -20,26 +20,26 @@ package org.apache.flink.table.plan.nodes
 
 import org.apache.calcite.plan.{RelOptCost, RelOptPlanner}
 import org.apache.calcite.rex._
-import org.apache.flink.api.common.functions.{FlatMapFunction, RichFlatMapFunction}
+import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.table.api.TableConfig
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.{CodeGenerator, GeneratedFunction}
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.table.runtime.FlatMapRunner
 import org.apache.flink.types.Row
 
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 
-trait CommonCalc {
+trait CommonCalc[T] {
 
-  private[flink] def functionBody(
+  private[flink] def generateFunction(
       generator: CodeGenerator,
+      ruleDescription: String,
       inputSchema: RowSchema,
       returnSchema: RowSchema,
       calcProgram: RexProgram,
-      config: TableConfig)
-    : String = {
+      config: TableConfig):
+    GeneratedFunction[FlatMapFunction[Row, Row], Row] = {
 
     val expandedExpressions = calcProgram
       .getProjectList
@@ -61,7 +61,7 @@ trait CommonCalc {
       expandedExpressions)
 
     // only projection
-    if (condition == null) {
+    val body = if (condition == null) {
       s"""
         |${projection.code}
         |${generator.collectorTerm}.collect(${projection.resultTerm});
@@ -89,16 +89,12 @@ trait CommonCalc {
           |""".stripMargin
       }
     }
-  }
-
-  private[flink] def calcMapFunction(
-      genFunction: GeneratedFunction[FlatMapFunction[Row, Row], Row])
-    : RichFlatMapFunction[Row, Row] = {
 
-    new FlatMapRunner[Row, Row](
-      genFunction.name,
-      genFunction.code,
-      genFunction.returnType)
+    generator.generateFunction(
+      ruleDescription,
+      classOf[FlatMapFunction[Row, Row]],
+      body,
+      returnSchema.physicalTypeInfo)
   }
 
   private[flink] def conditionToString(
@@ -168,8 +164,8 @@ trait CommonCalc {
     // CASTs in RexProgram are reduced as far as possible by ReduceExpressionsRule
     // in normalization stage. So we should ignore CASTs here in optimization stage.
     val compCnt = calcProgram.getExprList.asScala.toList.count {
-      case i: RexInputRef => false
-      case l: RexLiteral => false
+      case _: RexInputRef => false
+      case _: RexLiteral => false
       case c: RexCall if c.getOperator.getName.equals("CAST") => false
       case _ => true
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
index 02305ee8495c0..44a109e3b2b02 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
@@ -28,7 +28,7 @@ import org.apache.flink.table.codegen.GeneratedExpression.{ALWAYS_NULL, NO_CODE}
 import org.apache.flink.table.codegen.{CodeGenerator, GeneratedCollector, GeneratedExpression, GeneratedFunction}
 import org.apache.flink.table.functions.utils.TableSqlFunction
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.table.runtime.{CorrelateFlatMapRunner, TableFunctionCollector}
+import org.apache.flink.table.runtime.TableFunctionCollector
 import org.apache.flink.types.Row
 
 import scala.collection.JavaConverters._
@@ -36,72 +36,27 @@ import scala.collection.JavaConverters._
 /**
   * Join a user-defined table function
   */
-trait CommonCorrelate {
-
-  /**
-    * Creates the [[CorrelateFlatMapRunner]] to execute the join of input table
-    * and user-defined table function.
-    */
-  private[flink] def correlateMapFunction(
-      config: TableConfig,
-      inputSchema: RowSchema,
-      udtfTypeInfo: TypeInformation[Any],
-      returnSchema: RowSchema,
-      joinType: SemiJoinType,
-      rexCall: RexCall,
-      condition: Option[RexNode],
-      pojoFieldMapping: Option[Array[Int]], // udtf return type pojo field mapping
-      ruleDescription: String)
-    : CorrelateFlatMapRunner[Row, Row] = {
-
-    val flatMap = generateFunction(
-      config,
-      inputSchema.physicalTypeInfo,
-      udtfTypeInfo,
-      returnSchema.physicalTypeInfo,
-      returnSchema.logicalFieldNames,
-      joinType,
-      inputSchema.mapRexNode(rexCall).asInstanceOf[RexCall],
-      pojoFieldMapping,
-      ruleDescription)
-
-    val collector = generateCollector(
-      config,
-      inputSchema.physicalTypeInfo,
-      udtfTypeInfo,
-      returnSchema.physicalTypeInfo,
-      returnSchema.logicalFieldNames,
-      condition.map(inputSchema.mapRexNode),
-      pojoFieldMapping)
-
-    new CorrelateFlatMapRunner[Row, Row](
-      flatMap.name,
-      flatMap.code,
-      collector.name,
-      collector.code,
-      flatMap.returnType)
-
-  }
+trait CommonCorrelate[T] {
 
   /**
     * Generates the flat map function to run the user-defined table function.
     */
-  private def generateFunction(
-      config: TableConfig,
-      inputTypeInfo: TypeInformation[Row],
-      udtfTypeInfo: TypeInformation[Any],
-      returnType: TypeInformation[Row],
-      resultFieldNames: Seq[String],
-      joinType: SemiJoinType,
-      rexCall: RexCall,
-      pojoFieldMapping: Option[Array[Int]],
-      ruleDescription: String)
-    : GeneratedFunction[FlatMapFunction[Row, Row], Row] = {
+  private[flink] def generateFunction(
+    config: TableConfig,
+    inputSchema: RowSchema,
+    udtfTypeInfo: TypeInformation[Any],
+    returnSchema: RowSchema,
+    rowType: RelDataType,
+    joinType: SemiJoinType,
+    rexCall: RexCall,
+    pojoFieldMapping: Option[Array[Int]],
+    ruleDescription: String):
+  GeneratedFunction[FlatMapFunction[Row, Row], Row] = {
 
     val functionGenerator = new CodeGenerator(
       config,
       false,
-      inputTypeInfo,
+      inputSchema.physicalTypeInfo,
       Some(udtfTypeInfo),
       None,
       pojoFieldMapping)
@@ -115,9 +70,9 @@ trait CommonCorrelate {
     val call = functionGenerator.generateExpression(rexCall)
     var body =
       s"""
-        |${call.resultTerm}.setCollector($collectorTerm);
-        |${call.code}
-        |""".stripMargin
+         |${call.resultTerm}.setCollector($collectorTerm);
+         |${call.code}
+         |""".stripMargin
 
     if (joinType == SemiJoinType.LEFT) {
       // left outer join
@@ -132,15 +87,17 @@ trait CommonCorrelate {
           x.resultType)
       }
       val outerResultExpr = functionGenerator.generateResultExpression(
-        input1AccessExprs ++ input2NullExprs, returnType, resultFieldNames)
+        input1AccessExprs ++ input2NullExprs,
+        returnSchema.physicalTypeInfo,
+        rowType.getFieldNames.asScala)
       body +=
         s"""
-          |boolean hasOutput = $collectorTerm.isCollected();
-          |if (!hasOutput) {
-          |  ${outerResultExpr.code}
-          |  ${functionGenerator.collectorTerm}.collect(${outerResultExpr.resultTerm});
-          |}
-          |""".stripMargin
+           |boolean hasOutput = $collectorTerm.isCollected();
+           |if (!hasOutput) {
+           |  ${outerResultExpr.code}
+           |  ${functionGenerator.collectorTerm}.collect(${outerResultExpr.resultTerm});
+           |}
+           |""".stripMargin
     } else if (joinType != SemiJoinType.INNER) {
       throw TableException(s"Unsupported SemiJoinType: $joinType for correlate join.")
     }
@@ -149,26 +106,26 @@ trait CommonCorrelate {
       ruleDescription,
       classOf[FlatMapFunction[Row, Row]],
       body,
-      returnType)
+      returnSchema.physicalTypeInfo)
   }
 
   /**
     * Generates table function collector.
     */
   private[flink] def generateCollector(
-      config: TableConfig,
-      inputTypeInfo: TypeInformation[Row],
-      udtfTypeInfo: TypeInformation[Any],
-      returnType: TypeInformation[Row],
-      resultFieldNames: Seq[String],
-      condition: Option[RexNode],
-      pojoFieldMapping: Option[Array[Int]])
-    : GeneratedCollector = {
+    config: TableConfig,
+    inputSchema: RowSchema,
+    udtfTypeInfo: TypeInformation[Any],
+    returnSchema: RowSchema,
+    rowType: RelDataType,
+    condition: Option[RexNode],
+    pojoFieldMapping: Option[Array[Int]])
+  : GeneratedCollector = {
 
     val generator = new CodeGenerator(
       config,
       false,
-      inputTypeInfo,
+      inputSchema.physicalTypeInfo,
       Some(udtfTypeInfo),
       None,
       pojoFieldMapping)
@@ -177,26 +134,26 @@ trait CommonCorrelate {
 
     val crossResultExpr = generator.generateResultExpression(
       input1AccessExprs ++ input2AccessExprs,
-      returnType,
-      resultFieldNames)
+      returnSchema.physicalTypeInfo,
+      rowType.getFieldNames.asScala)
 
     val collectorCode = if (condition.isEmpty) {
       s"""
-        |${crossResultExpr.code}
-        |getCollector().collect(${crossResultExpr.resultTerm});
-        |""".stripMargin
+         |${crossResultExpr.code}
+         |getCollector().collect(${crossResultExpr.resultTerm});
+         |""".stripMargin
     } else {
       val filterGenerator = new CodeGenerator(config, false, udtfTypeInfo, None, pojoFieldMapping)
       filterGenerator.input1Term = filterGenerator.input2Term
       val filterCondition = filterGenerator.generateExpression(condition.get)
       s"""
-        |${filterGenerator.reuseInputUnboxingCode()}
-        |${filterCondition.code}
-        |if (${filterCondition.resultTerm}) {
-        |  ${crossResultExpr.code}
-        |  getCollector().collect(${crossResultExpr.resultTerm});
-        |}
-        |""".stripMargin
+         |${filterGenerator.reuseInputUnboxingCode()}
+         |${filterCondition.code}
+         |if (${filterCondition.resultTerm}) {
+         |  ${crossResultExpr.code}
+         |  getCollector().collect(${crossResultExpr.resultTerm});
+         |}
+         |""".stripMargin
     }
 
     generator.generateTableFunctionCollector(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala
index 091a1ea298705..7ce73eebd0e0d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonScan.scala
@@ -27,7 +27,7 @@ import org.apache.flink.types.Row
 /**
   * Common class for batch and stream scans.
   */
-trait CommonScan {
+trait CommonScan[T] {
 
   /**
     * We check if the input type is exactly the same as the internal row type.
@@ -35,11 +35,8 @@ trait CommonScan {
     */
   private[flink] def needsConversion(
       externalTypeInfo: TypeInformation[Any],
-      internalTypeInfo: TypeInformation[Row])
-    : Boolean = {
-
+      internalTypeInfo: TypeInformation[T]): Boolean =
     externalTypeInfo != internalTypeInfo
-  }
 
   private[flink] def generatedConversionFunction[F <: Function](
       config: TableConfig,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala
index cc5d9fb2d7f52..95707b8bcbab8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/BatchScan.scala
@@ -30,7 +30,7 @@ import org.apache.flink.types.Row
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 
-trait BatchScan extends CommonScan with DataSetRel {
+trait BatchScan extends CommonScan[Row] with DataSetRel {
 
   protected def convertToInternalRow(
       input: DataSet[Any],
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
index 5274fa19a731e..e340a8cce1f28 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
@@ -24,7 +24,6 @@ import org.apache.calcite.rel.core.Calc
 import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{RelNode, RelWriter}
 import org.apache.calcite.rex._
-import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.api.java.DataSet
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.table.api.BatchTableEnvironment
@@ -47,7 +46,7 @@ class DataSetCalc(
     calcProgram: RexProgram,
     ruleDescription: String)
   extends Calc(cluster, traitSet, input, calcProgram)
-  with CommonCalc
+  with CommonCalc[Row]
   with DataSetRel {
 
   override def deriveRowType(): RelDataType = rowRelDataType
@@ -88,22 +87,17 @@ class DataSetCalc(
 
     val generator = new CodeGenerator(config, false, inputDS.getType)
 
-    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
+    val returnType = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
 
-    val body = functionBody(
+    val genFunction = generateFunction(
       generator,
+      ruleDescription,
       new RowSchema(getInput.getRowType),
       new RowSchema(getRowType),
       calcProgram,
       config)
 
-    val genFunction = generator.generateFunction(
-      ruleDescription,
-      classOf[FlatMapFunction[Row, Row]],
-      body,
-      rowTypeInfo)
-
-    val runner = calcMapFunction(genFunction)
+    val runner = new FlatMapRunner(genFunction.name, genFunction.code, returnType)
 
     inputDS.flatMap(runner).name(calcOpName(calcProgram, getExpressionString))
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
index 6c79b45d8667a..49ead26a155d6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
@@ -32,6 +32,7 @@ import org.apache.flink.table.functions.utils.TableSqlFunction
 import org.apache.flink.table.plan.nodes.CommonCorrelate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalTableFunctionScan
 import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.runtime.CorrelateFlatMapRunner
 import org.apache.flink.types.Row
 
 /**
@@ -48,7 +49,7 @@ class DataSetCorrelate(
     joinType: SemiJoinType,
     ruleDescription: String)
   extends SingleRel(cluster, traitSet, inputNode)
-  with CommonCorrelate
+  with CommonCorrelate[Row]
   with DataSetRel {
 
   override def deriveRowType() = relRowType
@@ -98,22 +99,38 @@ class DataSetCorrelate(
     val funcRel = scan.asInstanceOf[FlinkLogicalTableFunctionScan]
     val rexCall = funcRel.getCall.asInstanceOf[RexCall]
     val sqlFunction = rexCall.getOperator.asInstanceOf[TableSqlFunction]
-    val pojoFieldMapping = sqlFunction.getPojoFieldMapping
+    val pojoFieldMapping = Some(sqlFunction.getPojoFieldMapping)
     val udtfTypeInfo = sqlFunction.getRowTypeInfo.asInstanceOf[TypeInformation[Any]]
 
-    val rowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
+    val returnType = FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
 
-    val mapFunc = correlateMapFunction(
+    val flatMap = generateFunction(
       config,
       new RowSchema(getInput.getRowType),
       udtfTypeInfo,
       new RowSchema(getRowType),
+      rowType,
       joinType,
       rexCall,
-      condition,
-      Some(pojoFieldMapping),
+      pojoFieldMapping,
       ruleDescription)
 
+    val collector = generateCollector(
+      config,
+      new RowSchema(getInput.getRowType),
+      udtfTypeInfo,
+      new RowSchema(getRowType),
+      rowType,
+      condition,
+      pojoFieldMapping)
+
+    val mapFunc = new CorrelateFlatMapRunner[Row, Row](
+      flatMap.name,
+      flatMap.code,
+      collector.name,
+      collector.code,
+      flatMap.returnType)
+
     inputDS.flatMap(mapFunc).name(correlateOpName(rexCall, sqlFunction, relRowType))
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetValues.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetValues.scala
index 3ebee2c30e51d..948dd27db3328 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetValues.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetValues.scala
@@ -88,7 +88,7 @@ class DataSetValues(
       generatedRecords.map(_.code),
       returnType)
 
-    val inputFormat = new ValuesInputFormat[Row](
+    val inputFormat = new ValuesInputFormat(
       generatedFunction.name,
       generatedFunction.code,
       generatedFunction.returnType)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
index c6c25c09fb471..59f723acb27fc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
@@ -24,13 +24,13 @@ import org.apache.calcite.rel.core.Calc
 import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{RelNode, RelWriter}
 import org.apache.calcite.rex.RexProgram
-import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.CommonCalc
-import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.types.Row
+import org.apache.flink.table.runtime.CRowFlatMapRunner
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 /**
   * Flink RelNode which matches along with FlatMapOperator.
@@ -45,7 +45,7 @@ class DataStreamCalc(
     calcProgram: RexProgram,
     ruleDescription: String)
   extends Calc(cluster, traitSet, input, calcProgram)
-  with CommonCalc
+  with CommonCalc[CRow]
   with DataStreamRel {
 
   override def deriveRowType(): RelDataType = schema.logicalType
@@ -83,28 +83,28 @@ class DataStreamCalc(
     estimateRowCount(calcProgram, rowCnt)
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
     val inputDataStream = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputRowType = inputDataStream.getType.asInstanceOf[CRowTypeInfo].rowType
 
-    val generator = new CodeGenerator(config, false, inputDataStream.getType)
+    val generator = new CodeGenerator(config, false, inputRowType)
 
-    val body = functionBody(
+    val genFunction = generateFunction(
       generator,
+      ruleDescription,
       inputSchema,
       schema,
       calcProgram,
       config)
 
-    val genFunction = generator.generateFunction(
-      ruleDescription,
-      classOf[FlatMapFunction[Row, Row]],
-      body,
-      schema.physicalTypeInfo)
+    val mapFunc = new CRowFlatMapRunner(
+      genFunction.name,
+      genFunction.code,
+      CRowTypeInfo(schema.physicalTypeInfo))
 
-    val mapFunc = calcMapFunction(genFunction)
     inputDataStream.flatMap(mapFunc).name(calcOpName(calcProgram, getExpressionString))
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
index 899d8efed7dde..19ad89b387abd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
@@ -28,7 +28,8 @@ import org.apache.flink.table.functions.utils.TableSqlFunction
 import org.apache.flink.table.plan.nodes.CommonCorrelate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalTableFunctionScan
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.types.Row
+import org.apache.flink.table.runtime.CRowCorrelateFlatMapRunner
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 /**
   * Flink RelNode which matches along with join a user defined table function.
@@ -45,7 +46,7 @@ class DataStreamCorrelate(
     joinType: SemiJoinType,
     ruleDescription: String)
   extends SingleRel(cluster, traitSet, input)
-  with CommonCorrelate
+  with CommonCorrelate[CRow]
   with DataStreamRel {
 
   override def deriveRowType() = schema.logicalType
@@ -81,31 +82,54 @@ class DataStreamCorrelate(
       .itemIf("condition", condition.orNull, condition.isDefined)
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
     // we do not need to specify input type
     val inputDS = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputType = inputDS.getType.asInstanceOf[CRowTypeInfo]
 
     val funcRel = scan.asInstanceOf[FlinkLogicalTableFunctionScan]
     val rexCall = funcRel.getCall.asInstanceOf[RexCall]
     val sqlFunction = rexCall.getOperator.asInstanceOf[TableSqlFunction]
-    val pojoFieldMapping = sqlFunction.getPojoFieldMapping
+    val pojoFieldMapping = Some(sqlFunction.getPojoFieldMapping)
     val udtfTypeInfo = sqlFunction.getRowTypeInfo.asInstanceOf[TypeInformation[Any]]
 
-    val mapFunc = correlateMapFunction(
+    val flatMap = generateFunction(
       config,
       inputSchema,
       udtfTypeInfo,
       schema,
+      getRowType,
       joinType,
       rexCall,
-      condition,
-      Some(pojoFieldMapping),
+      pojoFieldMapping,
       ruleDescription)
 
-    inputDS.flatMap(mapFunc).name(correlateOpName(rexCall, sqlFunction, schema.logicalType))
+    val collector = generateCollector(
+      config,
+      inputSchema,
+      udtfTypeInfo,
+      schema,
+      getRowType,
+      condition,
+      pojoFieldMapping)
+
+    val mapFunc = new CRowCorrelateFlatMapRunner(
+      flatMap.name,
+      flatMap.code,
+      collector.name,
+      collector.code,
+      CRowTypeInfo(flatMap.returnType))
+
+    val inputParallelism = inputDS.getParallelism
+
+    inputDS
+      .flatMap(mapFunc)
+      // preserve input parallelism to ensure that acc and retract messages remain in order
+      .setParallelism(inputParallelism)
+      .name(correlateOpName(rexCall, sqlFunction, schema.logicalType))
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index 3555c80a2ac65..056cda9d24e94 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -28,8 +28,9 @@ import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.runtime.aggregate._
 import org.apache.flink.table.plan.nodes.CommonAggregate
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.types.Row
+import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
 import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 /**
   *
@@ -97,15 +98,18 @@ class DataStreamGroupAggregate(
         inputSchema.logicalType, groupings, getRowType, namedAggregates, Nil))
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
 
     val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+
     val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
       new CalcitePair[AggregateCall, String](
         inputSchema.mapAggregateCall(namedAggregate.left),
         namedAggregate.right)
     }
 
+    val outRowType = CRowTypeInfo(schema.physicalTypeInfo)
+
     val generator = new CodeGenerator(
       tableEnv.getConfig,
       false,
@@ -129,28 +133,30 @@ class DataStreamGroupAggregate(
       physicalNamedAggregates,
       inputSchema.logicalType,
       inputSchema.physicalFieldTypeInfo,
-      groupings)
+      groupings,
+      DataStreamRetractionRules.isAccRetract(this),
+      DataStreamRetractionRules.isAccRetract(getInput))
 
-    val result: DataStream[Row] =
+    val result: DataStream[CRow] =
     // grouped / keyed aggregation
       if (physicalGrouping.nonEmpty) {
         inputDS
         .keyBy(groupings: _*)
         .process(processFunction)
-        .returns(schema.physicalTypeInfo)
+        .returns(outRowType)
         .name(keyedAggOpName)
-        .asInstanceOf[DataStream[Row]]
+        .asInstanceOf[DataStream[CRow]]
       }
       // global / non-keyed aggregation
       else {
         inputDS
-        .keyBy(new NullByteKeySelector[Row])
+        .keyBy(new NullByteKeySelector[CRow])
         .process(processFunction)
         .setParallelism(1)
         .setMaxParallelism(1)
-        .returns(schema.physicalTypeInfo)
+        .returns(outRowType)
         .name(nonKeyedAggOpName)
-        .asInstanceOf[DataStream[Row]]
+        .asInstanceOf[DataStream[CRow]]
       }
     result
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index ea4b0bf211a43..f61828b628f66 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -26,7 +26,7 @@ import org.apache.flink.api.java.tuple.Tuple
 import org.apache.flink.streaming.api.datastream.{AllWindowedStream, DataStream, KeyedStream, WindowedStream}
 import org.apache.flink.streaming.api.windowing.assigners._
 import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamTableEnvironment, TableException}
 import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.expressions.ExpressionUtils._
@@ -34,10 +34,12 @@ import org.apache.flink.table.plan.logical._
 import org.apache.flink.table.plan.nodes.CommonAggregate
 import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.plan.nodes.datastream.DataStreamGroupWindowAggregate._
+import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
 import org.apache.flink.table.runtime.aggregate.AggregateUtil._
 import org.apache.flink.table.runtime.aggregate._
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.apache.flink.table.typeutils.TypeCheckUtils.isTimeInterval
-import org.apache.flink.types.Row
+import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
 
 class DataStreamGroupWindowAggregate(
     window: LogicalWindow,
@@ -101,14 +103,25 @@ class DataStreamGroupWindowAggregate(
           namedProperties))
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
 
     val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+
     val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
       new CalcitePair[AggregateCall, String](
         inputSchema.mapAggregateCall(namedAggregate.left),
         namedAggregate.right)
     }
+    val consumeRetraction = DataStreamRetractionRules.isAccRetract(input)
+
+    if (consumeRetraction) {
+      throw new TableException(
+        "Retraction on windowed GroupBy aggregation is not supported yet. " +
+          "Note: Windowed GroupBy aggregation should not follow a " +
+          "non-windowed GroupBy aggregation.")
+    }
+
+    val outRowType = CRowTypeInfo(schema.physicalTypeInfo)
 
     val aggString = aggregationToString(
       inputSchema.logicalType,
@@ -145,7 +158,7 @@ class DataStreamGroupWindowAggregate(
       val keyedStream = inputDS.keyBy(physicalGrouping: _*)
       val windowedStream =
         createKeyedWindowedStream(window, keyedStream)
-          .asInstanceOf[WindowedStream[Row, Tuple, DataStreamWindow]]
+          .asInstanceOf[WindowedStream[CRow, Tuple, DataStreamWindow]]
 
       val (aggFunction, accumulatorRowType, aggResultRowType) =
         AggregateUtil.createDataStreamAggregateFunction(
@@ -154,15 +167,11 @@ class DataStreamGroupWindowAggregate(
           inputSchema.physicalType,
           inputSchema.physicalFieldTypeInfo,
           schema.physicalType,
+          physicalGrouping,
           needMerge)
 
       windowedStream
-        .aggregate(
-          aggFunction,
-          windowFunction,
-          accumulatorRowType,
-          aggResultRowType,
-          schema.physicalTypeInfo)
+        .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, outRowType)
         .name(keyedAggOpName)
     }
     // global / non-keyed aggregation
@@ -174,7 +183,7 @@ class DataStreamGroupWindowAggregate(
 
       val windowedStream =
         createNonKeyedWindowedStream(window, inputDS)
-          .asInstanceOf[AllWindowedStream[Row, DataStreamWindow]]
+          .asInstanceOf[AllWindowedStream[CRow, DataStreamWindow]]
 
       val (aggFunction, accumulatorRowType, aggResultRowType) =
         AggregateUtil.createDataStreamAggregateFunction(
@@ -183,15 +192,11 @@ class DataStreamGroupWindowAggregate(
           inputSchema.physicalType,
           inputSchema.physicalFieldTypeInfo,
           schema.physicalType,
+          Array[Int](),
           needMerge)
 
       windowedStream
-        .aggregate(
-          aggFunction,
-          windowFunction,
-          accumulatorRowType,
-          aggResultRowType,
-          schema.physicalTypeInfo)
+        .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, outRowType)
         .name(nonKeyedAggOpName)
     }
   }
@@ -199,9 +204,10 @@ class DataStreamGroupWindowAggregate(
 
 object DataStreamGroupWindowAggregate {
 
-
-  private def createKeyedWindowedStream(groupWindow: LogicalWindow, stream: KeyedStream[Row, Tuple])
-    : WindowedStream[Row, Tuple, _ <: DataStreamWindow] = groupWindow match {
+  private def createKeyedWindowedStream(
+      groupWindow: LogicalWindow,
+      stream: KeyedStream[CRow, Tuple]):
+    WindowedStream[CRow, Tuple, _ <: DataStreamWindow] = groupWindow match {
 
     case TumblingGroupWindow(_, timeField, size)
         if isProctimeAttribute(timeField) && isTimeIntervalLiteral(size)=>
@@ -250,8 +256,10 @@ object DataStreamGroupWindowAggregate {
       stream.window(EventTimeSessionWindows.withGap(toTime(gap)))
   }
 
-  private def createNonKeyedWindowedStream(groupWindow: LogicalWindow, stream: DataStream[Row])
-    : AllWindowedStream[Row, _ <: DataStreamWindow] = groupWindow match {
+  private def createNonKeyedWindowedStream(
+      groupWindow: LogicalWindow,
+      stream: DataStream[CRow]):
+    AllWindowedStream[CRow, _ <: DataStreamWindow] = groupWindow match {
 
     case TumblingGroupWindow(_, timeField, size)
         if isProctimeAttribute(timeField) && isTimeIntervalLiteral(size) =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index fb912c42dc3ee..e823cd6a5b9b9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -20,21 +20,22 @@ package org.apache.flink.table.plan.nodes.datastream
 import java.util.{List => JList}
 
 import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.core.Window.Group
 import org.apache.calcite.rel.core.{AggregateCall, Window}
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-import org.apache.flink.api.java.functions.NullByteKeySelector
+import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.{StreamTableEnvironment, TableException}
 import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.OverAggregate
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
 import org.apache.flink.table.runtime.aggregate._
-import org.apache.flink.types.Row
+import org.apache.flink.api.java.functions.NullByteKeySelector
+import org.apache.flink.table.codegen.CodeGenerator
+import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
+import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 class DataStreamOverAggregate(
     logicWindow: Window,
@@ -87,7 +88,7 @@ class DataStreamOverAggregate(
           namedAggregates))
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
     if (logicWindow.groups.size > 1) {
       throw new TableException(
         "Unsupported use of OVER windows. All aggregates must be computed on the same window.")
@@ -110,6 +111,8 @@ class DataStreamOverAggregate(
 
     val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
 
+    val consumeRetraction = DataStreamRetractionRules.isAccRetract(input)
+
     val generator = new CodeGenerator(
       tableEnv.getConfig,
       false,
@@ -120,6 +123,12 @@ class DataStreamOverAggregate(
       .get(orderKey.getFieldIndex)
       .getType
 
+    if (consumeRetraction) {
+      throw new TableException(
+        "Retraction on Over window aggregation is not supported yet. " +
+          "Note: Over window aggregation should not follow a non-windowed GroupBy aggregation.")
+    }
+
     timeType match {
       case _ if FlinkTypeFactory.isProctimeIndicatorType(timeType)  =>
         // proc-time OVER window
@@ -138,8 +147,7 @@ class DataStreamOverAggregate(
             generator,
             inputDS,
             isRowTimeType = false,
-            isRowsClause = overWindow.isRows
-          )
+            isRowsClause = overWindow.isRows)
         } else {
           throw new TableException(
             "OVER RANGE FOLLOWING windows are not supported yet.")
@@ -154,16 +162,14 @@ class DataStreamOverAggregate(
             generator,
             inputDS,
             isRowTimeType = true,
-            isRowsClause = overWindow.isRows
-          )
+            isRowsClause = overWindow.isRows)
         } else if (overWindow.lowerBound.isPreceding && overWindow.upperBound.isCurrentRow) {
           // bounded OVER window
           createBoundedAndCurrentRowOverWindow(
             generator,
             inputDS,
             isRowTimeType = true,
-            isRowsClause = overWindow.isRows
-          )
+            isRowsClause = overWindow.isRows)
         } else {
           throw new TableException(
             "OVER RANGE FOLLOWING windows are not supported yet.")
@@ -177,12 +183,14 @@ class DataStreamOverAggregate(
 
   def createUnboundedAndCurrentRowOverWindow(
     generator: CodeGenerator,
-    inputDS: DataStream[Row],
+    inputDS: DataStream[CRow],
     isRowTimeType: Boolean,
-    isRowsClause: Boolean): DataStream[Row] = {
+    isRowsClause: Boolean): DataStream[CRow] = {
 
     val overWindow: Group = logicWindow.groups.get(0)
+
     val partitionKeys: Array[Int] = overWindow.keys.toArray.map(schema.mapIndex)
+
     val namedAggregates: Seq[CalcitePair[AggregateCall, String]] = generateNamedAggregates.map {
       namedAggregate =>
         new CalcitePair[AggregateCall, String](
@@ -190,6 +198,9 @@ class DataStreamOverAggregate(
           namedAggregate.right)
     }
 
+    // get the output types
+    val returnTypeInfo = CRowTypeInfo(schema.physicalTypeInfo)
+
     val processFunction = AggregateUtil.createUnboundedOverProcessFunction(
       generator,
       namedAggregates,
@@ -200,30 +211,28 @@ class DataStreamOverAggregate(
       partitionKeys.nonEmpty,
       isRowsClause)
 
-    val result: DataStream[Row] =
+    val result: DataStream[CRow] =
     // partitioned aggregation
       if (partitionKeys.nonEmpty) {
         inputDS
           .keyBy(partitionKeys: _*)
           .process(processFunction)
-          .returns(schema.physicalTypeInfo)
+          .returns(returnTypeInfo)
           .name(aggOpName)
-          .asInstanceOf[DataStream[Row]]
+          .asInstanceOf[DataStream[CRow]]
       }
       // non-partitioned aggregation
       else {
         if (isRowTimeType) {
-          inputDS.keyBy(new NullByteKeySelector[Row])
+          inputDS.keyBy(new NullByteKeySelector[CRow])
             .process(processFunction).setParallelism(1).setMaxParallelism(1)
-            .returns(schema.physicalTypeInfo)
+            .returns(returnTypeInfo)
             .name(aggOpName)
-            .asInstanceOf[DataStream[Row]]
         } else {
           inputDS
             .process(processFunction).setParallelism(1).setMaxParallelism(1)
-            .returns(schema.physicalTypeInfo)
+            .returns(returnTypeInfo)
             .name(aggOpName)
-            .asInstanceOf[DataStream[Row]]
         }
       }
     result
@@ -231,9 +240,9 @@ class DataStreamOverAggregate(
 
   def createBoundedAndCurrentRowOverWindow(
     generator: CodeGenerator,
-    inputDS: DataStream[Row],
+    inputDS: DataStream[CRow],
     isRowTimeType: Boolean,
-    isRowsClause: Boolean): DataStream[Row] = {
+    isRowsClause: Boolean): DataStream[CRow] = {
 
     val overWindow: Group = logicWindow.groups.get(0)
     val partitionKeys: Array[Int] = overWindow.keys.toArray.map(schema.mapIndex)
@@ -245,7 +254,10 @@ class DataStreamOverAggregate(
     }
 
     val precedingOffset =
-      getLowerBoundary(logicWindow, overWindow, input) + (if (isRowsClause) 1 else 0)
+      getLowerBoundary(logicWindow, overWindow, getInput()) + (if (isRowsClause) 1 else 0)
+
+    // get the output types
+    val returnTypeInfo = CRowTypeInfo(schema.physicalTypeInfo)
 
     val processFunction = AggregateUtil.createBoundedOverProcessFunction(
       generator,
@@ -257,24 +269,22 @@ class DataStreamOverAggregate(
       isRowsClause,
       isRowTimeType
     )
-    val result: DataStream[Row] =
+    val result: DataStream[CRow] =
     // partitioned aggregation
       if (partitionKeys.nonEmpty) {
         inputDS
           .keyBy(partitionKeys: _*)
           .process(processFunction)
-          .returns(schema.physicalTypeInfo)
+          .returns(returnTypeInfo)
           .name(aggOpName)
-          .asInstanceOf[DataStream[Row]]
       }
       // non-partitioned aggregation
       else {
         inputDS
-          .keyBy(new NullByteKeySelector[Row])
+          .keyBy(new NullByteKeySelector[CRow])
           .process(processFunction).setParallelism(1).setMaxParallelism(1)
-          .returns(schema.physicalTypeInfo)
+          .returns(returnTypeInfo)
           .name(aggOpName)
-          .asInstanceOf[DataStream[Row]]
       }
     result
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
index 128da81b8a2ed..9754de4461df8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
@@ -21,7 +21,7 @@ package org.apache.flink.table.plan.nodes.datastream
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
 import org.apache.flink.table.plan.nodes.FlinkRelNode
-import org.apache.flink.types.Row
+import org.apache.flink.table.runtime.types.CRow
 
 trait DataStreamRel extends FlinkRelNode {
 
@@ -29,9 +29,9 @@ trait DataStreamRel extends FlinkRelNode {
     * Translates the FlinkRelNode into a Flink operator.
     *
     * @param tableEnv The [[StreamTableEnvironment]] of the translated Table.
-    * @return DataStream of type [[Row]]
+    * @return DataStream of type [[CRow]]
     */
-  def translateToPlan(tableEnv: StreamTableEnvironment) : DataStream[Row]
+  def translateToPlan(tableEnv: StreamTableEnvironment) : DataStream[CRow]
 
   /**
     * Whether the [[DataStreamRel]] requires that update and delete changes are sent with retraction
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
index 05f60ba035632..c613646f9cff2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
@@ -24,8 +24,9 @@ import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.core.TableScan
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
-import org.apache.flink.table.plan.schema.{DataStreamTable, RowSchema}
-import org.apache.flink.types.Row
+import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.plan.schema.DataStreamTable
+import org.apache.flink.table.runtime.types.CRow
 
 /**
   * Flink RelNode which matches along with DataStreamSource.
@@ -53,7 +54,7 @@ class DataStreamScan(
     )
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
     val config = tableEnv.getConfig
     val inputDataStream: DataStream[Any] = dataStreamTable.dataStream
     convertToInternalRow(schema, inputDataStream, dataStreamTable, config)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
index 47b4946168024..654c259f637c9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
@@ -23,7 +23,7 @@ import org.apache.calcite.rel.{BiRel, RelNode, RelWriter}
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.types.Row
+import org.apache.flink.table.runtime.types.CRow
 
 /**
   * Flink RelNode which matches along with Union.
@@ -58,7 +58,7 @@ class DataStreamUnion(
     s"Union All(union: (${schema.logicalFieldNames.mkString(", ")}))"
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
 
     val leftDataSet = left.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
     val rightDataSet = right.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
index c964e03471d4e..32c9aaf0e3d20 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
@@ -27,8 +27,8 @@ import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.StreamTableEnvironment
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.table.runtime.io.ValuesInputFormat
-import org.apache.flink.types.Row
+import org.apache.flink.table.runtime.io.CRowValuesInputFormat
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 import scala.collection.JavaConverters._
 
@@ -56,10 +56,11 @@ class DataStreamValues(
     )
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
+    val returnType = CRowTypeInfo(schema.physicalTypeInfo)
     val generator = new CodeGenerator(config)
 
     // generate code for every record
@@ -76,12 +77,12 @@ class DataStreamValues(
       generatedRecords.map(_.code),
       schema.physicalTypeInfo)
 
-    val inputFormat = new ValuesInputFormat[Row](
+    val inputFormat = new CRowValuesInputFormat(
       generatedFunction.name,
       generatedFunction.code,
-      generatedFunction.returnType)
+      returnType)
 
-    tableEnv.execEnv.createInput(inputFormat, schema.physicalTypeInfo)
+    tableEnv.execEnv.createInput(inputFormat, returnType)
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala
index dd828196e7a32..25e72fa887ef0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamScan.scala
@@ -22,46 +22,51 @@ import org.apache.flink.api.common.functions.MapFunction
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.TableConfig
 import org.apache.flink.table.plan.nodes.CommonScan
-import org.apache.flink.table.plan.schema.{FlinkTable, RowSchema}
-import org.apache.flink.table.runtime.MapRunner
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.types.Row
+import org.apache.flink.table.plan.schema.FlinkTable
+import org.apache.flink.table.runtime.CRowOutputMapRunner
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 import scala.collection.JavaConverters._
 
-trait StreamScan extends CommonScan with DataStreamRel {
+trait StreamScan extends CommonScan[CRow] with DataStreamRel {
 
   protected def convertToInternalRow(
       schema: RowSchema,
       input: DataStream[Any],
       flinkTable: FlinkTable[_],
       config: TableConfig)
-    : DataStream[Row] = {
+    : DataStream[CRow] = {
+
+    val inputType = input.getType
+    val internalType = CRowTypeInfo(schema.physicalTypeInfo)
 
     // conversion
-    if (needsConversion(input.getType, schema.physicalTypeInfo)) {
+    if (needsConversion(input.getType, internalType)) {
 
       val function = generatedConversionFunction(
         config,
         classOf[MapFunction[Any, Row]],
-        input.getType,
+        inputType,
         schema.physicalTypeInfo,
         "DataStreamSourceConversion",
         schema.physicalFieldNames,
         Some(flinkTable.fieldIndexes))
 
-      val runner = new MapRunner[Any, Row](
+      val mapFunc = new CRowOutputMapRunner(
         function.name,
         function.code,
-        function.returnType)
+        internalType)
 
-      val opName = s"from: (${schema.logicalFieldNames.mkString(", ")})"
+      val opName = s"from: (${getRowType.getFieldNames.asScala.toList.mkString(", ")})"
 
       // TODO we need a ProcessFunction here
-      input.map(runner).name(opName)
+      input.map(mapFunc).name(opName).returns(internalType)
     }
     // no conversion necessary, forward
     else {
-      input.asInstanceOf[DataStream[Row]]
+      input.asInstanceOf[DataStream[CRow]]
     }
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
index e34e416946f4f..b2d7019ae0fd6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
@@ -25,9 +25,11 @@ import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.{StreamTableEnvironment, TableEnvironment}
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.PhysicalTableSourceScan
-import org.apache.flink.table.plan.schema.{RowSchema, TableSourceTable}
+import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.sources._
-import org.apache.flink.types.Row
+import org.apache.flink.table.plan.schema.TableSourceTable
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.sources.{StreamTableSource, TableSource}
 
 /** Flink RelNode to read data from an external source defined by a [[StreamTableSource]]. */
 class StreamTableSourceScan(
@@ -96,7 +98,7 @@ class StreamTableSourceScan(
     )
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[Row] = {
+  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
     val config = tableEnv.getConfig
     val inputDataStream = tableSource.getDataStream(tableEnv.execEnv).asInstanceOf[DataStream[Any]]
     convertToInternalRow(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala
index ec903926ff747..0ca079e7f03ef 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala
@@ -34,7 +34,7 @@ class FlinkLogicalCalc(
     calcProgram: RexProgram)
   extends Calc(cluster, traitSet, input, calcProgram)
   with FlinkLogicalRel
-  with CommonCalc {
+  with CommonCalc[Any] {
 
   override def copy(traitSet: RelTraitSet, child: RelNode, program: RexProgram): Calc = {
     new FlinkLogicalCalc(cluster, traitSet, child, program)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
index aeb67b6f73fd2..bd9a7ee8eaf70 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
@@ -81,6 +81,14 @@ object DataStreamRetractionRules {
     retractionTrait != null && retractionTrait.sendsUpdatesAsRetractions
   }
 
+  /**
+    * Checks if a [[RelNode]] is in [[AccMode.AccRetract]] mode.
+    */
+  def isAccRetract(node: RelNode): Boolean = {
+    val accModeTrait = node.getTraitSet.getTrait(AccModeTraitDef.INSTANCE)
+    null != accModeTrait && accModeTrait.getAccMode == AccMode.AccRetract
+  }
+
   /**
     * Rule that assigns the default retraction information to [[DataStreamRel]] nodes.
     * The default is to not publish updates as retraction messages and [[AccMode.Acc]].
@@ -189,14 +197,6 @@ object DataStreamRetractionRules {
       }
     }
 
-    /**
-      * Checks if a [[RelNode]] is in [[AccMode.AccRetract]] mode.
-      */
-    def isAccRetract(node: RelNode): Boolean = {
-      val accModeTrait = node.getTraitSet.getTrait(AccModeTraitDef.INSTANCE)
-      null != accModeTrait && accModeTrait.getAccMode == AccMode.AccRetract
-    }
-
     /**
       * Set [[AccMode.AccRetract]] to a [[RelNode]].
       */
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
new file mode 100644
index 0000000000000..66e51b1d4b06e
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime
+
+import org.apache.flink.api.common.functions.{FlatMapFunction, RichFlatMapFunction}
+import org.apache.flink.api.common.functions.util.FunctionUtils
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.apache.flink.util.Collector
+import org.slf4j.{Logger, LoggerFactory}
+
+/**
+  * A CorrelateFlatMapRunner with [[CRow]] input and [[CRow]] output.
+  */
+class CRowCorrelateFlatMapRunner(
+    flatMapName: String,
+    flatMapCode: String,
+    collectorName: String,
+    collectorCode: String,
+    @transient returnType: TypeInformation[CRow])
+  extends RichFlatMapFunction[CRow, CRow]
+  with ResultTypeQueryable[CRow]
+  with Compiler[Any] {
+
+  val LOG: Logger = LoggerFactory.getLogger(this.getClass)
+
+  private var function: FlatMapFunction[Row, Row] = _
+  private var collector: TableFunctionCollector[_] = _
+  private var cRowWrapper: CRowWrappingCollector = _
+
+  override def open(parameters: Configuration): Unit = {
+    LOG.debug(s"Compiling TableFunctionCollector: $collectorName \n\n Code:\n$collectorCode")
+    val clazz = compile(getRuntimeContext.getUserCodeClassLoader, collectorName, collectorCode)
+    LOG.debug("Instantiating TableFunctionCollector.")
+    collector = clazz.newInstance().asInstanceOf[TableFunctionCollector[_]]
+    this.cRowWrapper = new CRowWrappingCollector()
+
+    LOG.debug(s"Compiling FlatMapFunction: $flatMapName \n\n Code:\n$flatMapCode")
+    val flatMapClazz = compile(getRuntimeContext.getUserCodeClassLoader, flatMapName, flatMapCode)
+    val constructor = flatMapClazz.getConstructor(classOf[TableFunctionCollector[_]])
+    LOG.debug("Instantiating FlatMapFunction.")
+    function = constructor.newInstance(collector).asInstanceOf[FlatMapFunction[Row, Row]]
+    FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext)
+    FunctionUtils.openFunction(function, parameters)
+  }
+
+  override def flatMap(in: CRow, out: Collector[CRow]): Unit = {
+    cRowWrapper.out = out
+    cRowWrapper.setChange(in.change)
+
+    collector.setCollector(cRowWrapper)
+    collector.setInput(in.row)
+    collector.reset()
+
+    function.flatMap(in.row, cRowWrapper)
+  }
+
+  override def getProducedType: TypeInformation[CRow] = returnType
+
+  override def close(): Unit = {
+    FunctionUtils.closeFunction(function)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
new file mode 100644
index 0000000000000..9a4650b518ad9
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime
+
+import org.apache.flink.api.common.functions.util.FunctionUtils
+import org.apache.flink.api.common.functions.{FlatMapFunction, RichFlatMapFunction}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.apache.flink.util.Collector
+import org.slf4j.LoggerFactory
+
+/**
+  * FlatMapRunner with [[CRow]] input and [[CRow]] output.
+  */
+class CRowFlatMapRunner(
+    name: String,
+    code: String,
+    @transient returnType: TypeInformation[CRow])
+  extends RichFlatMapFunction[CRow, CRow]
+  with ResultTypeQueryable[CRow]
+  with Compiler[FlatMapFunction[Row, Row]] {
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+
+  private var function: FlatMapFunction[Row, Row] = _
+  private var cRowWrapper: CRowWrappingCollector = _
+
+  override def open(parameters: Configuration): Unit = {
+    LOG.debug(s"Compiling FlatMapFunction: $name \n\n Code:\n$code")
+    val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
+    LOG.debug("Instantiating FlatMapFunction.")
+    function = clazz.newInstance()
+    FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext)
+    FunctionUtils.openFunction(function, parameters)
+
+    this.cRowWrapper = new CRowWrappingCollector()
+  }
+
+  override def flatMap(in: CRow, out: Collector[CRow]): Unit = {
+    cRowWrapper.out = out
+    cRowWrapper.setChange(in.change)
+    function.flatMap(in.row, cRowWrapper)
+  }
+
+  override def getProducedType: TypeInformation[CRow] = returnType
+
+  override def close(): Unit = {
+    FunctionUtils.closeFunction(function)
+  }
+}
+
+
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala
new file mode 100644
index 0000000000000..8e95c935afe43
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime
+
+import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.slf4j.LoggerFactory
+
+/**
+  * MapRunner with [[CRow]] input.
+  */
+class CRowInputMapRunner[OUT](
+    name: String,
+    code: String,
+    @transient returnType: TypeInformation[OUT])
+  extends RichMapFunction[CRow, OUT]
+  with ResultTypeQueryable[OUT]
+  with Compiler[MapFunction[Row, OUT]] {
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+
+  private var function: MapFunction[Row, OUT] = _
+
+  override def open(parameters: Configuration): Unit = {
+    LOG.debug(s"Compiling MapFunction: $name \n\n Code:\n$code")
+    val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
+    LOG.debug("Instantiating MapFunction.")
+    function = clazz.newInstance()
+  }
+
+  override def map(in: CRow): OUT = {
+    function.map(in.row)
+  }
+
+  override def getProducedType: TypeInformation[OUT] = returnType
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala
new file mode 100644
index 0000000000000..966dea904e112
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime
+
+import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.slf4j.LoggerFactory
+
+/**
+  * MapRunner with [[CRow]] output.
+  */
+class CRowOutputMapRunner(
+    name: String,
+    code: String,
+    @transient returnType: TypeInformation[CRow])
+  extends RichMapFunction[Any, CRow]
+  with ResultTypeQueryable[CRow]
+  with Compiler[MapFunction[Any, Row]] {
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+
+  private var function: MapFunction[Any, Row] = _
+  private var outCRow: CRow = _
+
+  override def open(parameters: Configuration): Unit = {
+    LOG.debug(s"Compiling MapFunction: $name \n\n Code:\n$code")
+    val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
+    LOG.debug("Instantiating MapFunction.")
+    function = clazz.newInstance()
+    outCRow = new CRow(null, true)
+  }
+
+  override def map(in: Any): CRow = {
+    outCRow.row = function.map(in)
+    outCRow
+  }
+
+  override def getProducedType: TypeInformation[CRow] = returnType
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala
new file mode 100644
index 0000000000000..b2b062e31d027
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime
+
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.apache.flink.util.Collector
+
+/**
+  * The collector is used to wrap a [[Row]] to a [[CRow]]
+  */
+class CRowWrappingCollector() extends Collector[Row] {
+
+  var out: Collector[CRow] = _
+  val outCRow: CRow = new CRow()
+
+  def setChange(change: Boolean): Unit = this.outCRow.change = change
+
+  override def collect(record: Row): Unit = {
+    outCRow.row = record
+    out.collect(outCRow)
+  }
+
+  override def close(): Unit = out.close()
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
index b446306f0e6fb..2e37baf0a6f3e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
@@ -24,20 +24,21 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.table.codegen.Compiler
 import org.apache.flink.configuration.Configuration
+import org.apache.flink.types.Row
 import org.apache.flink.util.Collector
 import org.slf4j.LoggerFactory
 
-class FlatMapRunner[IN, OUT](
+class FlatMapRunner(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT])
-  extends RichFlatMapFunction[IN, OUT]
-  with ResultTypeQueryable[OUT]
-  with Compiler[FlatMapFunction[IN, OUT]] {
+    @transient returnType: TypeInformation[Row])
+  extends RichFlatMapFunction[Row, Row]
+  with ResultTypeQueryable[Row]
+  with Compiler[FlatMapFunction[Row, Row]] {
 
   val LOG = LoggerFactory.getLogger(this.getClass)
 
-  private var function: FlatMapFunction[IN, OUT] = _
+  private var function: FlatMapFunction[Row, Row] = _
 
   override def open(parameters: Configuration): Unit = {
     LOG.debug(s"Compiling FlatMapFunction: $name \n\n Code:\n$code")
@@ -48,10 +49,10 @@ class FlatMapRunner[IN, OUT](
     FunctionUtils.openFunction(function, parameters)
   }
 
-  override def flatMap(in: IN, out: Collector[OUT]): Unit =
+  override def flatMap(in: Row, out: Collector[Row]): Unit =
     function.flatMap(in, out)
 
-  override def getProducedType: TypeInformation[OUT] = returnType
+  override def getProducedType: TypeInformation[Row] = returnType
 
   override def close(): Unit = {
     FunctionUtils.closeFunction(function)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
index 377e0ff6ca335..dd9c015c2d98e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.runtime.aggregate
 
 import org.apache.flink.api.common.functions.AggregateFunction
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.types.Row
 import org.slf4j.LoggerFactory
 
@@ -30,28 +31,28 @@ import org.slf4j.LoggerFactory
   * @param genAggregations Generated aggregate helper function
   */
 class AggregateAggFunction(genAggregations: GeneratedAggregationsFunction)
-  extends AggregateFunction[Row, Row, Row] with Compiler[GeneratedAggregations] {
+  extends AggregateFunction[CRow, Row, Row] with Compiler[GeneratedAggregations] {
 
   val LOG = LoggerFactory.getLogger(this.getClass)
   private var function: GeneratedAggregations = _
 
   override def createAccumulator(): Row = {
     if (function == null) {
-      initFunction
+      initFunction()
     }
     function.createAccumulators()
   }
 
-  override def add(value: Row, accumulatorRow: Row): Unit = {
+  override def add(value: CRow, accumulatorRow: Row): Unit = {
     if (function == null) {
-      initFunction
+      initFunction()
     }
-    function.accumulate(accumulatorRow, value)
+    function.accumulate(accumulatorRow, value.row)
   }
 
   override def getResult(accumulatorRow: Row): Row = {
     if (function == null) {
-      initFunction
+      initFunction()
     }
     val output = function.createOutputRow()
     function.setAggregationResults(accumulatorRow, output)
@@ -60,7 +61,7 @@ class AggregateAggFunction(genAggregations: GeneratedAggregationsFunction)
 
   override def merge(aAccumulatorRow: Row, bAccumulatorRow: Row): Row = {
     if (function == null) {
-      initFunction
+      initFunction()
     }
     function.mergeAccumulatorsPair(aAccumulatorRow, bAccumulatorRow)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 5e9efd0d61c89..768c9cbb482f0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -44,6 +44,7 @@ import org.apache.flink.table.functions.utils.AggSqlFunction
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
 import org.apache.flink.table.functions.{AggregateFunction => TableAggregateFunction}
 import org.apache.flink.table.plan.logical._
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.apache.flink.table.typeutils.TypeCheckUtils._
 import org.apache.flink.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
 import org.apache.flink.types.Row
@@ -79,7 +80,7 @@ object AggregateUtil {
       isRowTimeType: Boolean,
       isPartitioned: Boolean,
       isRowsClause: Boolean)
-    : ProcessFunction[Row, Row] = {
+    : ProcessFunction[CRow, CRow] = {
 
     val (aggFields, aggregates) =
       transformToAggregateFunctions(
@@ -116,13 +117,13 @@ object AggregateUtil {
         new RowTimeUnboundedRowsOver(
           genFunction,
           aggregationStateType,
-          inputTypeInfo)
+          CRowTypeInfo(inputTypeInfo))
       } else {
         // RANGE unbounded over process function
         new RowTimeUnboundedRangeOver(
           genFunction,
           aggregationStateType,
-          inputTypeInfo)
+          CRowTypeInfo(inputTypeInfo))
       }
     } else {
       if (isPartitioned) {
@@ -153,13 +154,16 @@ object AggregateUtil {
       namedAggregates: Seq[CalcitePair[AggregateCall, String]],
       inputRowType: RelDataType,
       inputFieldTypes: Seq[TypeInformation[_]],
-      groupings: Array[Int]): ProcessFunction[Row, Row] = {
+      groupings: Array[Int],
+      generateRetraction: Boolean,
+      consumeRetraction: Boolean): ProcessFunction[CRow, CRow] = {
 
     val (aggFields, aggregates) =
       transformToAggregateFunctions(
         namedAggregates.map(_.getKey),
         inputRowType,
-        needRetraction = false)
+        consumeRetraction)
+
     val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
 
     val outputArity = groupings.length + aggregates.length
@@ -178,14 +182,16 @@ object AggregateUtil {
       None,
       None,
       outputArity,
-      needRetract = false,
+      consumeRetraction,
       needMerge = false,
       needReset = false
     )
 
     new GroupAggProcessFunction(
       genFunction,
-      aggregationStateType)
+      aggregationStateType,
+      generateRetraction)
+
   }
 
   /**
@@ -198,7 +204,7 @@ object AggregateUtil {
     * @param inputTypeInfo Physical type information of the row.
     * @param inputFieldTypeInfo Physical type information of the row's fields.
     * @param precedingOffset the preceding offset
-    * @param isRowsClause   It is a tag that indicates whether the OVER clause is ROWS clause
+    * @param isRowsClause    It is a tag that indicates whether the OVER clause is ROWS clause
     * @param isRowTimeType   It is a tag that indicates whether the time type is rowTimeType
     * @return [[org.apache.flink.streaming.api.functions.ProcessFunction]]
     */
@@ -211,7 +217,7 @@ object AggregateUtil {
       precedingOffset: Long,
       isRowsClause: Boolean,
       isRowTimeType: Boolean)
-    : ProcessFunction[Row, Row] = {
+    : ProcessFunction[CRow, CRow] = {
 
     val needRetract = true
     val (aggFields, aggregates) =
@@ -221,6 +227,7 @@ object AggregateUtil {
         needRetract)
 
     val aggregationStateType: RowTypeInfo = createAccumulatorRowType(aggregates)
+    val inputRowType = CRowTypeInfo(inputTypeInfo)
 
     val forwardMapping = (0 until inputType.getFieldCount).toArray
     val aggMapping = aggregates.indices.map(x => x + inputType.getFieldCount).toArray
@@ -248,14 +255,14 @@ object AggregateUtil {
         new RowTimeBoundedRowsOver(
           genFunction,
           aggregationStateType,
-          inputTypeInfo,
+          inputRowType,
           precedingOffset
         )
       } else {
         new RowTimeBoundedRangeOver(
           genFunction,
           aggregationStateType,
-          inputTypeInfo,
+          inputRowType,
           precedingOffset
         )
       }
@@ -265,13 +272,13 @@ object AggregateUtil {
           genFunction,
           precedingOffset,
           aggregationStateType,
-          inputTypeInfo)
+          inputRowType)
       } else {
         new ProcTimeBoundedRangeOver(
           genFunction,
           precedingOffset,
           aggregationStateType,
-          inputTypeInfo)
+          inputRowType)
       }
     }
   }
@@ -932,7 +939,7 @@ object AggregateUtil {
       window: LogicalWindow,
       finalRowArity: Int,
       properties: Seq[NamedWindowProperty])
-    : AllWindowFunction[Row, Row, DataStreamWindow] = {
+    : AllWindowFunction[Row, CRow, DataStreamWindow] = {
 
     if (isTimeWindow(window)) {
       val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
@@ -940,7 +947,7 @@ object AggregateUtil {
         startPos,
         endPos,
         finalRowArity)
-        .asInstanceOf[AllWindowFunction[Row, Row, DataStreamWindow]]
+        .asInstanceOf[AllWindowFunction[Row, CRow, DataStreamWindow]]
     } else {
       new IncrementalAggregateAllWindowFunction(
         finalRowArity)
@@ -955,8 +962,8 @@ object AggregateUtil {
       numGroupingKeys: Int,
       numAggregates: Int,
       finalRowArity: Int,
-      properties: Seq[NamedWindowProperty])
-    : WindowFunction[Row, Row, Tuple, DataStreamWindow] = {
+      properties: Seq[NamedWindowProperty]):
+    WindowFunction[Row, CRow, Tuple, DataStreamWindow] = {
 
     if (isTimeWindow(window)) {
       val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
@@ -966,7 +973,7 @@ object AggregateUtil {
         startPos,
         endPos,
         finalRowArity)
-        .asInstanceOf[WindowFunction[Row, Row, Tuple, DataStreamWindow]]
+        .asInstanceOf[WindowFunction[Row, CRow, Tuple, DataStreamWindow]]
     } else {
       new IncrementalAggregateWindowFunction(
         numGroupingKeys,
@@ -981,8 +988,9 @@ object AggregateUtil {
       inputType: RelDataType,
       inputFieldTypeInfo: Seq[TypeInformation[_]],
       outputType: RelDataType,
+      groupingKeys: Array[Int],
       needMerge: Boolean)
-    : (DataStreamAggFunction[Row, Row, Row], RowTypeInfo, RowTypeInfo) = {
+    : (DataStreamAggFunction[CRow, Row, Row], RowTypeInfo, RowTypeInfo) = {
 
     val needRetract = false
     val (aggFields, aggregates) =
@@ -1002,7 +1010,7 @@ object AggregateUtil {
       aggFields,
       aggMapping,
       partialResults = false,
-      Array(), // no fields are forwarded
+      groupingKeys,
       None,
       None,
       outputArity,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
index 95699a24849aa..fabf200add416 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
@@ -56,7 +56,7 @@ class DataSetSessionWindowAggReduceGroupFunction(
   extends RichGroupReduceFunction[Row, Row]
     with Compiler[GeneratedAggregations] {
 
-  private var collector: TimeWindowPropertyCollector = _
+  private var collector: RowTimeWindowPropertyCollector = _
   private val intermediateRowWindowStartPos = keysAndAggregatesArity
   private val intermediateRowWindowEndPos = keysAndAggregatesArity + 1
 
@@ -78,7 +78,7 @@ class DataSetSessionWindowAggReduceGroupFunction(
 
     output = function.createOutputRow()
     accumulators = function.createAccumulators()
-    collector = new TimeWindowPropertyCollector(finalRowWindowStartPos, finalRowWindowEndPos)
+    collector = new RowTimeWindowPropertyCollector(finalRowWindowStartPos, finalRowWindowEndPos)
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
index a221c53be6e82..56ed08ade4a80 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
@@ -47,7 +47,7 @@ class DataSetSlideWindowAggReduceGroupFunction(
   extends RichGroupReduceFunction[Row, Row]
     with Compiler[GeneratedAggregations] {
 
-  private var collector: TimeWindowPropertyCollector = _
+  private var collector: RowTimeWindowPropertyCollector = _
   protected val windowStartPos: Int = keysAndAggregatesArity
 
   private var output: Row = _
@@ -68,7 +68,7 @@ class DataSetSlideWindowAggReduceGroupFunction(
 
     output = function.createOutputRow()
     accumulators = function.createAccumulators()
-    collector = new TimeWindowPropertyCollector(finalRowWindowStartPos, finalRowWindowEndPos)
+    collector = new RowTimeWindowPropertyCollector(finalRowWindowStartPos, finalRowWindowEndPos)
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
index f4a1fc51c9361..8af2c2e8a8a69 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
@@ -46,7 +46,7 @@ class DataSetTumbleTimeWindowAggReduceGroupFunction(
   extends RichGroupReduceFunction[Row, Row]
     with Compiler[GeneratedAggregations] {
 
-  private var collector: TimeWindowPropertyCollector = _
+  private var collector: RowTimeWindowPropertyCollector = _
   protected var aggregateBuffer: Row = new Row(keysAndAggregatesArity + 1)
 
   private var output: Row = _
@@ -67,7 +67,7 @@ class DataSetTumbleTimeWindowAggReduceGroupFunction(
 
     output = function.createOutputRow()
     accumulators = function.createAccumulators()
-    collector = new TimeWindowPropertyCollector(windowStartPos, windowEndPos)
+    collector = new RowTimeWindowPropertyCollector(windowStartPos, windowEndPos)
   }
 
   override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index 81c900cec68c6..745f24d8ca858 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -26,6 +26,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.api.common.state.ValueState
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.slf4j.LoggerFactory
+import org.apache.flink.table.runtime.types.CRow
 
 /**
   * Aggregate Function used for the groupby (without window) aggregate
@@ -35,14 +36,17 @@ import org.slf4j.LoggerFactory
   */
 class GroupAggProcessFunction(
     private val genAggregations: GeneratedAggregationsFunction,
-    private val aggregationStateType: RowTypeInfo)
-  extends ProcessFunction[Row, Row]
+    private val aggregationStateType: RowTypeInfo,
+    private val generateRetraction: Boolean)
+  extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
 
   val LOG = LoggerFactory.getLogger(this.getClass)
   private var function: GeneratedAggregations = _
 
-  private var output: Row = _
+  private var newRow: CRow = _
+  private var prevRow: CRow = _
+  private var firstRow: Boolean = _
   private var state: ValueState[Row] = _
 
   override def open(config: Configuration) {
@@ -54,7 +58,9 @@ class GroupAggProcessFunction(
       genAggregations.code)
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
-    output = function.createOutputRow()
+
+    newRow = new CRow(function.createOutputRow(), true)
+    prevRow = new CRow(function.createOutputRow(), false)
 
     val stateDescriptor: ValueStateDescriptor[Row] =
       new ValueStateDescriptor[Row]("GroupAggregateState", aggregationStateType)
@@ -62,29 +68,53 @@ class GroupAggProcessFunction(
   }
 
   override def processElement(
-      input: Row,
-      ctx: ProcessFunction[Row, Row]#Context,
-      out: Collector[Row]): Unit = {
+      inputC: CRow,
+      ctx: ProcessFunction[CRow, CRow]#Context,
+      out: Collector[CRow]): Unit = {
+
+    val input = inputC.row
 
     // get accumulators
     var accumulators = state.value()
     if (null == accumulators) {
+      firstRow = true
       accumulators = function.createAccumulators()
+    } else {
+      firstRow = false
     }
 
     // Set group keys value to the final output
-    function.setForwardedFields(input, output)
+    function.setForwardedFields(input, newRow.row)
+    function.setForwardedFields(input, prevRow.row)
 
-    // accumulate new input row
-    function.accumulate(accumulators, input)
+    // Set previous aggregate result to the prevRow
+    function.setAggregationResults(accumulators, prevRow.row)
 
-    // set aggregation results to output
-    function.setAggregationResults(accumulators, output)
+    // update aggregate result and set to the newRow
+    if (inputC.change) {
+      // accumulate input
+      function.accumulate(accumulators, input)
+      function.setAggregationResults(accumulators, newRow.row)
+    } else {
+      // retract input
+      function.retract(accumulators, input)
+      function.setAggregationResults(accumulators, newRow.row)
+    }
 
     // update accumulators
     state.update(accumulators)
 
-    out.collect(output)
-  }
+    // if previousRow is not null, do retraction process
+    if (generateRetraction && !firstRow) {
+      if (prevRow.row.equals(newRow.row)) {
+        // ignore same newRow
+        return
+      } else {
+        // retract previous row
+        out.collect(prevRow)
+      }
+    }
 
+    out.collect(newRow)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllTimeWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllTimeWindowFunction.scala
index ec9b65465cc60..711cc0505bf5b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllTimeWindowFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllTimeWindowFunction.scala
@@ -22,6 +22,7 @@ import java.lang.Iterable
 import org.apache.flink.types.Row
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.util.Collector
 
 /**
@@ -39,17 +40,17 @@ class IncrementalAggregateAllTimeWindowFunction(
   extends IncrementalAggregateAllWindowFunction[TimeWindow](
     finalRowArity) {
 
-  private var collector: TimeWindowPropertyCollector = _
+  private var collector: CRowTimeWindowPropertyCollector = _
 
   override def open(parameters: Configuration): Unit = {
-    collector = new TimeWindowPropertyCollector(windowStartPos, windowEndPos)
+    collector = new CRowTimeWindowPropertyCollector(windowStartPos, windowEndPos)
     super.open(parameters)
   }
 
   override def apply(
       window: TimeWindow,
       records: Iterable[Row],
-      out: Collector[Row]): Unit = {
+      out: Collector[CRow]): Unit = {
 
     // set collector and window
     collector.wrappedCollector = out
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllWindowFunction.scala
index f92be9210fb41..c19078566fccf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllWindowFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateAllWindowFunction.scala
@@ -23,6 +23,7 @@ import org.apache.flink.types.Row
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.windowing.RichAllWindowFunction
 import org.apache.flink.streaming.api.windowing.windows.Window
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.util.Collector
 
 /**
@@ -32,12 +33,12 @@ import org.apache.flink.util.Collector
   */
 class IncrementalAggregateAllWindowFunction[W <: Window](
     private val finalRowArity: Int)
-  extends RichAllWindowFunction[Row, Row, W] {
+  extends RichAllWindowFunction[Row, CRow, W] {
 
-  private var output: Row = _
+  private var output: CRow = _
 
   override def open(parameters: Configuration): Unit = {
-    output = new Row(finalRowArity)
+    output = new CRow(new Row(finalRowArity), true)
   }
 
   /**
@@ -47,7 +48,7 @@ class IncrementalAggregateAllWindowFunction[W <: Window](
   override def apply(
       window: W,
       records: Iterable[Row],
-      out: Collector[Row]): Unit = {
+      out: Collector[CRow]): Unit = {
 
     val iterator = records.iterator
 
@@ -55,7 +56,7 @@ class IncrementalAggregateAllWindowFunction[W <: Window](
       val record = iterator.next()
       var i = 0
       while (i < record.getArity) {
-        output.setField(i, record.getField(i))
+        output.row.setField(i, record.getField(i))
         i += 1
       }
       out.collect(output)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala
index dccb4f6880b90..809bbfdd5676a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala
@@ -23,6 +23,7 @@ import org.apache.flink.api.java.tuple.Tuple
 import org.apache.flink.types.Row
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.util.Collector
 
 /**
@@ -43,10 +44,10 @@ class IncrementalAggregateTimeWindowFunction(
     numAggregates,
     finalRowArity) {
 
-  private var collector: TimeWindowPropertyCollector = _
+  private var collector: CRowTimeWindowPropertyCollector = _
 
   override def open(parameters: Configuration): Unit = {
-    collector = new TimeWindowPropertyCollector(windowStartPos, windowEndPos)
+    collector = new CRowTimeWindowPropertyCollector(windowStartPos, windowEndPos)
     super.open(parameters)
   }
 
@@ -54,7 +55,7 @@ class IncrementalAggregateTimeWindowFunction(
       key: Tuple,
       window: TimeWindow,
       records: Iterable[Row],
-      out: Collector[Row]): Unit = {
+      out: Collector[CRow]): Unit = {
 
     // set collector and window
     collector.wrappedCollector = out
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateWindowFunction.scala
index 983efb3bdad06..7e9d738bc0dba 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateWindowFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateWindowFunction.scala
@@ -24,6 +24,7 @@ import org.apache.flink.types.Row
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction
 import org.apache.flink.streaming.api.windowing.windows.Window
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.util.Collector
 
 /**
@@ -37,12 +38,12 @@ class IncrementalAggregateWindowFunction[W <: Window](
     private val numGroupingKey: Int,
     private val numAggregates: Int,
     private val finalRowArity: Int)
-  extends RichWindowFunction[Row, Row, Tuple, W] {
+  extends RichWindowFunction[Row, CRow, Tuple, W] {
 
-  private var output: Row = _
+  private var output: CRow = _
 
   override def open(parameters: Configuration): Unit = {
-    output = new Row(finalRowArity)
+    output = new CRow(new Row(finalRowArity), true)
   }
 
   /**
@@ -53,7 +54,7 @@ class IncrementalAggregateWindowFunction[W <: Window](
       key: Tuple,
       window: W,
       records: Iterable[Row],
-      out: Collector[Row]): Unit = {
+      out: Collector[CRow]): Unit = {
 
     val iterator = records.iterator
 
@@ -62,12 +63,12 @@ class IncrementalAggregateWindowFunction[W <: Window](
 
       var i = 0
       while (i < numGroupingKey) {
-        output.setField(i, key.getField(i))
+        output.row.setField(i, key.getField(i))
         i += 1
       }
       i = 0
       while (i < numAggregates) {
-        output.setField(numGroupingKey + i, record.getField(i))
+        output.row.setField(numGroupingKey + i, record.getField(i))
         i += 1
       }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index b63eb81413e12..3fb506fd1333c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -31,7 +31,8 @@ import org.apache.flink.api.java.typeutils.ListTypeInfo
 import java.util.{ArrayList, List => JList}
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.slf4j.LoggerFactory
 
 /**
@@ -47,10 +48,10 @@ class ProcTimeBoundedRangeOver(
     genAggregations: GeneratedAggregationsFunction,
     precedingTimeBoundary: Long,
     aggregatesTypeInfo: RowTypeInfo,
-    inputType: TypeInformation[Row])
-  extends ProcessFunction[Row, Row]
+    inputType: TypeInformation[CRow])
+  extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
-  private var output: Row = _
+  private var output: CRow = _
   private var accumulatorState: ValueState[Row] = _
   private var rowMapState: MapState[Long, JList[Row]] = _
 
@@ -66,11 +67,12 @@ class ProcTimeBoundedRangeOver(
       genAggregations.code)
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
-    output = function.createOutputRow()
+    output = new CRow(function.createOutputRow(), true)
 
     // We keep the elements received in a MapState indexed based on their ingestion time
     val rowListTypeInfo: TypeInformation[JList[Row]] =
-      new ListTypeInfo[Row](inputType).asInstanceOf[TypeInformation[JList[Row]]]
+      new ListTypeInfo[Row](inputType.asInstanceOf[CRowTypeInfo].rowType)
+        .asInstanceOf[TypeInformation[JList[Row]]]
     val mapStateDescriptor: MapStateDescriptor[Long, JList[Row]] =
       new MapStateDescriptor[Long, JList[Row]]("rowmapstate",
         BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]], rowListTypeInfo)
@@ -82,9 +84,9 @@ class ProcTimeBoundedRangeOver(
   }
 
   override def processElement(
-    input: Row,
-    ctx: ProcessFunction[Row, Row]#Context,
-    out: Collector[Row]): Unit = {
+    input: CRow,
+    ctx: ProcessFunction[CRow, CRow]#Context,
+    out: Collector[CRow]): Unit = {
 
     val currentTime = ctx.timerService.currentProcessingTime
     // buffer the event incoming event
@@ -97,15 +99,15 @@ class ProcTimeBoundedRangeOver(
       // register timer to process event once the current millisecond passed
       ctx.timerService.registerProcessingTimeTimer(currentTime + 1)
     }
-    rowList.add(input)
+    rowList.add(input.row)
     rowMapState.put(currentTime, rowList)
 
   }
 
   override def onTimer(
     timestamp: Long,
-    ctx: ProcessFunction[Row, Row]#OnTimerContext,
-    out: Collector[Row]): Unit = {
+    ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+    out: Collector[CRow]): Unit = {
 
     // we consider the original timestamp of events that have registered this time trigger 1 ms ago
     val currentTime = timestamp - 1
@@ -166,10 +168,10 @@ class ProcTimeBoundedRangeOver(
       val input = currentElements.get(iElemenets)
 
       // set the fields of the last event to carry on with the aggregates
-      function.setForwardedFields(input, output)
+      function.setForwardedFields(input, output.row)
 
       // add the accumulators values to result
-      function.setAggregationResults(accumulators, output)
+      function.setAggregationResults(accumulators, output.row)
       out.collect(output)
       iElemenets += 1
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
index 31cfd730eab89..0c7f44ed1531b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
@@ -33,7 +33,8 @@ import org.apache.flink.api.java.typeutils.ListTypeInfo
 import java.util.{List => JList}
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.slf4j.LoggerFactory
 
 /**
@@ -48,15 +49,15 @@ class ProcTimeBoundedRowsOver(
     genAggregations: GeneratedAggregationsFunction,
     precedingOffset: Long,
     aggregatesTypeInfo: RowTypeInfo,
-    inputType: TypeInformation[Row])
-  extends ProcessFunction[Row, Row]
+    inputType: TypeInformation[CRow])
+  extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
 
   Preconditions.checkArgument(precedingOffset > 0)
 
   private var accumulatorState: ValueState[Row] = _
   private var rowMapState: MapState[Long, JList[Row]] = _
-  private var output: Row = _
+  private var output: CRow = _
   private var counterState: ValueState[Long] = _
   private var smallestTsState: ValueState[Long] = _
 
@@ -73,13 +74,14 @@ class ProcTimeBoundedRowsOver(
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
 
-    output = function.createOutputRow()
+    output = new CRow(function.createOutputRow(), true)
     // We keep the elements received in a Map state keyed
     // by the ingestion time in the operator.
     // we also keep counter of processed elements
     // and timestamp of oldest element
     val rowListTypeInfo: TypeInformation[JList[Row]] =
-      new ListTypeInfo[Row](inputType).asInstanceOf[TypeInformation[JList[Row]]]
+      new ListTypeInfo[Row](inputType.asInstanceOf[CRowTypeInfo].rowType)
+        .asInstanceOf[TypeInformation[JList[Row]]]
 
     val mapStateDescriptor: MapStateDescriptor[Long, JList[Row]] =
       new MapStateDescriptor[Long, JList[Row]]("windowBufferMapState",
@@ -100,9 +102,11 @@ class ProcTimeBoundedRowsOver(
   }
 
   override def processElement(
-    input: Row,
-    ctx: ProcessFunction[Row, Row]#Context,
-    out: Collector[Row]): Unit = {
+    inputC: CRow,
+    ctx: ProcessFunction[CRow, CRow]#Context,
+    out: Collector[CRow]): Unit = {
+
+    val input = inputC.row
 
     val currentTime = ctx.timerService.currentProcessingTime
 
@@ -154,11 +158,11 @@ class ProcTimeBoundedRowsOver(
     }
 
     // copy forwarded fields in output row
-    function.setForwardedFields(input, output)
+    function.setForwardedFields(input, output.row)
 
     // accumulate current row and set aggregate in output row
     function.accumulate(accumulators, input)
-    function.setAggregationResults(accumulators, output)
+    function.setAggregationResults(accumulators, output.row)
 
     // update map state, accumulator state, counter and timestamp
     val currentTimeState = rowMapState.get(currentTime)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala
index 75209db2def80..8a231327e01c2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala
@@ -23,9 +23,10 @@ import org.apache.flink.configuration.Configuration
 import org.apache.flink.runtime.state.{FunctionInitializationContext, FunctionSnapshotContext}
 import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction
 import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.apache.flink.types.Row
 import org.apache.flink.util.Collector
-import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.types.Row
 import org.slf4j.LoggerFactory
 
 /**
@@ -37,12 +38,12 @@ import org.slf4j.LoggerFactory
 class ProcTimeUnboundedNonPartitionedOver(
     genAggregations: GeneratedAggregationsFunction,
     aggregationStateType: RowTypeInfo)
-  extends ProcessFunction[Row, Row]
+  extends ProcessFunction[CRow, CRow]
     with CheckpointedFunction
     with Compiler[GeneratedAggregations] {
 
   private var accumulators: Row = _
-  private var output: Row = _
+  private var output: CRow = _
   private var state: ListState[Row] = _
   val LOG = LoggerFactory.getLogger(this.getClass)
 
@@ -58,7 +59,7 @@ class ProcTimeUnboundedNonPartitionedOver(
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
 
-    output = function.createOutputRow()
+    output = new CRow(function.createOutputRow(), true)
     if (null == accumulators) {
       val it = state.get().iterator()
       if (it.hasNext) {
@@ -70,14 +71,16 @@ class ProcTimeUnboundedNonPartitionedOver(
   }
 
   override def processElement(
-    input: Row,
-    ctx: ProcessFunction[Row, Row]#Context,
-    out: Collector[Row]): Unit = {
+      inputC: CRow,
+      ctx: ProcessFunction[CRow, CRow]#Context,
+      out: Collector[CRow]): Unit = {
+
+    val input = inputC.row
 
-    function.setForwardedFields(input, output)
+    function.setForwardedFields(input, output.row)
 
     function.accumulate(accumulators, input)
-    function.setAggregationResults(accumulators, output)
+    function.setAggregationResults(accumulators, output.row)
 
     out.collect(output)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala
index 9baa6a37a3950..847c1bfb142f5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala
@@ -24,7 +24,8 @@ import org.apache.flink.util.Collector
 import org.apache.flink.api.common.state.ValueStateDescriptor
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.api.common.state.ValueState
-import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.CRow
 import org.slf4j.LoggerFactory
 
 /**
@@ -36,10 +37,10 @@ import org.slf4j.LoggerFactory
 class ProcTimeUnboundedPartitionedOver(
     genAggregations: GeneratedAggregationsFunction,
     aggregationStateType: RowTypeInfo)
-  extends ProcessFunction[Row, Row]
+  extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
 
-  private var output: Row = _
+  private var output: CRow = _
   private var state: ValueState[Row] = _
   val LOG = LoggerFactory.getLogger(this.getClass)
   private var function: GeneratedAggregations = _
@@ -54,16 +55,18 @@ class ProcTimeUnboundedPartitionedOver(
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
 
-    output = function.createOutputRow()
+    output = new CRow(function.createOutputRow(), true)
     val stateDescriptor: ValueStateDescriptor[Row] =
       new ValueStateDescriptor[Row]("overState", aggregationStateType)
     state = getRuntimeContext.getState(stateDescriptor)
   }
 
   override def processElement(
-    input: Row,
-    ctx: ProcessFunction[Row, Row]#Context,
-    out: Collector[Row]): Unit = {
+    inputC: CRow,
+    ctx: ProcessFunction[CRow, CRow]#Context,
+    out: Collector[CRow]): Unit = {
+
+    val input = inputC.row
 
     var accumulators = state.value()
 
@@ -71,13 +74,12 @@ class ProcTimeUnboundedPartitionedOver(
       accumulators = function.createAccumulators()
     }
 
-    function.setForwardedFields(input, output)
+    function.setForwardedFields(input, output.row)
 
     function.accumulate(accumulators, input)
-    function.setAggregationResults(accumulators, output)
+    function.setAggregationResults(accumulators, output.row)
 
     state.update(accumulators)
-
     out.collect(output)
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index ef97e716efb0a..4020d440a78fa 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -17,14 +17,15 @@
  */
 package org.apache.flink.table.runtime.aggregate
 
-import java.util.{List => JList, ArrayList => JArrayList}
+import java.util.{ArrayList => JArrayList, List => JList}
 
 import org.apache.flink.api.common.state._
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
-import org.apache.flink.api.java.typeutils.ListTypeInfo
+import org.apache.flink.api.java.typeutils.{ListTypeInfo, RowTypeInfo}
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.apache.flink.types.Row
 import org.apache.flink.util.{Collector, Preconditions}
 import org.slf4j.LoggerFactory
@@ -39,15 +40,15 @@ import org.slf4j.LoggerFactory
  */
 class RowTimeBoundedRangeOver(
     genAggregations: GeneratedAggregationsFunction,
-    aggregationStateType: TypeInformation[Row],
-    inputRowType: TypeInformation[Row],
+    aggregationStateType: RowTypeInfo,
+    inputRowType: CRowTypeInfo,
     precedingOffset: Long)
-  extends ProcessFunction[Row, Row]
+  extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
   Preconditions.checkNotNull(aggregationStateType)
   Preconditions.checkNotNull(precedingOffset)
 
-  private var output: Row = _
+  private var output: CRow = _
 
   // the state which keeps the last triggering timestamp
   private var lastTriggeringTsState: ValueState[Long] = _
@@ -74,7 +75,7 @@ class RowTimeBoundedRangeOver(
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
 
-    output = function.createOutputRow()
+    output = new CRow(function.createOutputRow(), true)
 
     val lastTriggeringTsDescriptor: ValueStateDescriptor[Long] =
       new ValueStateDescriptor[Long]("lastTriggeringTsState", classOf[Long])
@@ -86,7 +87,8 @@ class RowTimeBoundedRangeOver(
 
     val keyTypeInformation: TypeInformation[Long] =
       BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]]
-    val valueTypeInformation: TypeInformation[JList[Row]] = new ListTypeInfo[Row](inputRowType)
+    val valueTypeInformation: TypeInformation[JList[Row]] =
+      new ListTypeInfo[Row](inputRowType.asInstanceOf[CRowTypeInfo].rowType)
 
     val mapStateDescriptor: MapStateDescriptor[Long, JList[Row]] =
       new MapStateDescriptor[Long, JList[Row]](
@@ -98,9 +100,11 @@ class RowTimeBoundedRangeOver(
   }
 
   override def processElement(
-    input: Row,
-    ctx: ProcessFunction[Row, Row]#Context,
-    out: Collector[Row]): Unit = {
+    inputC: CRow,
+    ctx: ProcessFunction[CRow, CRow]#Context,
+    out: Collector[CRow]): Unit = {
+
+    val input = inputC.row
 
     // triggering timestamp for trigger calculation
     val triggeringTs = ctx.timestamp
@@ -125,8 +129,8 @@ class RowTimeBoundedRangeOver(
 
   override def onTimer(
     timestamp: Long,
-    ctx: ProcessFunction[Row, Row]#OnTimerContext,
-    out: Collector[Row]): Unit = {
+    ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+    out: Collector[CRow]): Unit = {
     // gets all window data from state for the calculation
     val inputs: JList[Row] = dataState.get(timestamp)
 
@@ -172,13 +176,13 @@ class RowTimeBoundedRangeOver(
       }
 
       // set aggregate in output row
-      function.setAggregationResults(accumulators, output)
+      function.setAggregationResults(accumulators, output.row)
 
       // copy forwarded fields to output row and emit output row
       dataListIndex = 0
       while (dataListIndex < inputs.size()) {
         aggregatesIndex = 0
-        function.setForwardedFields(inputs.get(dataListIndex), output)
+        function.setForwardedFields(inputs.get(dataListIndex), output.row)
         out.collect(output)
         dataListIndex += 1
       }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index 7169cf723f0e3..5ec6ec7ab1ead 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -27,7 +27,8 @@ import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.types.Row
 import org.apache.flink.util.{Collector, Preconditions}
-import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.slf4j.LoggerFactory
 
 /**
@@ -41,15 +42,15 @@ import org.slf4j.LoggerFactory
 class RowTimeBoundedRowsOver(
     genAggregations: GeneratedAggregationsFunction,
     aggregationStateType: RowTypeInfo,
-    inputRowType: TypeInformation[Row],
+    inputRowType: CRowTypeInfo,
     precedingOffset: Long)
-  extends ProcessFunction[Row, Row]
+  extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
 
   Preconditions.checkNotNull(aggregationStateType)
   Preconditions.checkNotNull(precedingOffset)
 
-  private var output: Row = _
+  private var output: CRow = _
 
   // the state which keeps the last triggering timestamp
   private var lastTriggeringTsState: ValueState[Long] = _
@@ -79,7 +80,7 @@ class RowTimeBoundedRowsOver(
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
 
-    output = function.createOutputRow()
+    output = new CRow(function.createOutputRow(), true)
 
     val lastTriggeringTsDescriptor: ValueStateDescriptor[Long] =
       new ValueStateDescriptor[Long]("lastTriggeringTsState", classOf[Long])
@@ -95,7 +96,8 @@ class RowTimeBoundedRowsOver(
 
     val keyTypeInformation: TypeInformation[Long] =
       BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]]
-    val valueTypeInformation: TypeInformation[JList[Row]] = new ListTypeInfo[Row](inputRowType)
+    val valueTypeInformation: TypeInformation[JList[Row]] =
+      new ListTypeInfo[Row](inputRowType.asInstanceOf[CRowTypeInfo].rowType)
 
     val mapStateDescriptor: MapStateDescriptor[Long, JList[Row]] =
       new MapStateDescriptor[Long, JList[Row]](
@@ -107,9 +109,11 @@ class RowTimeBoundedRowsOver(
   }
 
   override def processElement(
-    input: Row,
-    ctx: ProcessFunction[Row, Row]#Context,
-    out: Collector[Row]): Unit = {
+    inputC: CRow,
+    ctx: ProcessFunction[CRow, CRow]#Context,
+    out: Collector[CRow]): Unit = {
+
+    val input = inputC.row
 
     // triggering timestamp for trigger calculation
     val triggeringTs = ctx.timestamp
@@ -134,8 +138,8 @@ class RowTimeBoundedRowsOver(
 
   override def onTimer(
     timestamp: Long,
-    ctx: ProcessFunction[Row, Row]#OnTimerContext,
-    out: Collector[Row]): Unit = {
+    ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+    out: Collector[CRow]): Unit = {
 
     // gets all window data from state for the calculation
     val inputs: JList[Row] = dataState.get(timestamp)
@@ -189,7 +193,7 @@ class RowTimeBoundedRowsOver(
         }
 
         // copy forwarded fields to output row
-        function.setForwardedFields(input, output)
+        function.setForwardedFields(input, output.row)
 
         // retract old row from accumulators
         if (null != retractRow) {
@@ -198,7 +202,7 @@ class RowTimeBoundedRowsOver(
 
         // accumulate current row and set aggregate in output row
         function.accumulate(accumulators, input)
-        function.setAggregationResults(accumulators, output)
+        function.setAggregationResults(accumulators, output.row)
         i += 1
 
         out.collect(output)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
index 525d4d76e9d2e..3e2a8117b56e5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -28,7 +28,8 @@ import org.apache.flink.util.{Collector, Preconditions}
 import org.apache.flink.api.common.state._
 import org.apache.flink.api.java.typeutils.ListTypeInfo
 import org.apache.flink.streaming.api.operators.TimestampedCollector
-import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.slf4j.LoggerFactory
 
 
@@ -42,11 +43,11 @@ import org.slf4j.LoggerFactory
 abstract class RowTimeUnboundedOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType: TypeInformation[Row],
-    inputType: TypeInformation[Row])
-  extends ProcessFunction[Row, Row]
+    inputType: TypeInformation[CRow])
+  extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
 
-  protected var output: Row = _
+  protected var output: CRow = _
   // state to hold the accumulators of the aggregations
   private var accumulatorState: ValueState[Row] = _
   // state to hold rows until the next watermark arrives
@@ -67,7 +68,7 @@ abstract class RowTimeUnboundedOver(
     LOG.debug("Instantiating AggregateHelper.")
     function = clazz.newInstance()
 
-    output = function.createOutputRow()
+    output = new CRow(function.createOutputRow(), true)
     sortedTimestamps = new util.LinkedList[Long]()
 
     // initialize accumulator state
@@ -76,7 +77,8 @@ abstract class RowTimeUnboundedOver(
     accumulatorState = getRuntimeContext.getState[Row](accDescriptor)
 
     // initialize row state
-    val rowListTypeInfo: TypeInformation[JList[Row]] = new ListTypeInfo[Row](inputType)
+    val rowListTypeInfo: TypeInformation[JList[Row]] =
+      new ListTypeInfo[Row](inputType.asInstanceOf[CRowTypeInfo].rowType)
     val mapStateDescriptor: MapStateDescriptor[Long, JList[Row]] =
       new MapStateDescriptor[Long, JList[Row]]("rowmapstate",
         BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]], rowListTypeInfo)
@@ -87,15 +89,17 @@ abstract class RowTimeUnboundedOver(
     * Puts an element from the input stream into state if it is not late.
     * Registers a timer for the next watermark.
     *
-    * @param input The input value.
+    * @param inputC The input value.
     * @param ctx   The ctx to register timer or get current time
     * @param out   The collector for returning result values.
     *
     */
   override def processElement(
-     input: Row,
-     ctx:  ProcessFunction[Row, Row]#Context,
-     out: Collector[Row]): Unit = {
+     inputC: CRow,
+     ctx:  ProcessFunction[CRow, CRow]#Context,
+     out: Collector[CRow]): Unit = {
+
+    val input = inputC.row
 
     val timestamp = ctx.timestamp()
     val curWatermark = ctx.timerService().currentWatermark()
@@ -126,11 +130,11 @@ abstract class RowTimeUnboundedOver(
     */
   override def onTimer(
       timestamp: Long,
-      ctx: ProcessFunction[Row, Row]#OnTimerContext,
-      out: Collector[Row]): Unit = {
+      ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+      out: Collector[CRow]): Unit = {
 
-    Preconditions.checkArgument(out.isInstanceOf[TimestampedCollector[Row]])
-    val collector = out.asInstanceOf[TimestampedCollector[Row]]
+    Preconditions.checkArgument(out.isInstanceOf[TimestampedCollector[CRow]])
+    val collector = out.asInstanceOf[TimestampedCollector[CRow]]
 
     val keyIterator = rowMapState.keys.iterator
     if (keyIterator.hasNext) {
@@ -206,7 +210,7 @@ abstract class RowTimeUnboundedOver(
   def processElementsWithSameTimestamp(
     curRowList: JList[Row],
     lastAccumulator: Row,
-    out: Collector[Row]): Unit
+    out: Collector[CRow]): Unit
 
 }
 
@@ -217,7 +221,7 @@ abstract class RowTimeUnboundedOver(
 class RowTimeUnboundedRowsOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType: TypeInformation[Row],
-    inputType: TypeInformation[Row])
+    inputType: TypeInformation[CRow])
   extends RowTimeUnboundedOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType,
@@ -226,7 +230,7 @@ class RowTimeUnboundedRowsOver(
   override def processElementsWithSameTimestamp(
     curRowList: JList[Row],
     lastAccumulator: Row,
-    out: Collector[Row]): Unit = {
+    out: Collector[CRow]): Unit = {
 
     var i = 0
     while (i < curRowList.size) {
@@ -234,11 +238,11 @@ class RowTimeUnboundedRowsOver(
 
       var j = 0
       // copy forwarded fields to output row
-      function.setForwardedFields(curRow, output)
+      function.setForwardedFields(curRow, output.row)
 
       // update accumulators and copy aggregates to output row
       function.accumulate(lastAccumulator, curRow)
-      function.setAggregationResults(lastAccumulator, output)
+      function.setAggregationResults(lastAccumulator, output.row)
       // emit output row
       out.collect(output)
       i += 1
@@ -255,7 +259,7 @@ class RowTimeUnboundedRowsOver(
 class RowTimeUnboundedRangeOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType: TypeInformation[Row],
-    inputType: TypeInformation[Row])
+    inputType: TypeInformation[CRow])
   extends RowTimeUnboundedOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType,
@@ -264,7 +268,7 @@ class RowTimeUnboundedRangeOver(
   override def processElementsWithSameTimestamp(
     curRowList: JList[Row],
     lastAccumulator: Row,
-    out: Collector[Row]): Unit = {
+    out: Collector[CRow]): Unit = {
 
     var i = 0
     // all same timestamp data should have same aggregation value.
@@ -281,10 +285,10 @@ class RowTimeUnboundedRangeOver(
       val curRow = curRowList.get(i)
 
       // copy forwarded fields to output row
-      function.setForwardedFields(curRow, output)
+      function.setForwardedFields(curRow, output.row)
 
       //copy aggregates to output row
-      function.setAggregationResults(lastAccumulator, output)
+      function.setAggregationResults(lastAccumulator, output.row)
       out.collect(output)
       i += 1
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/TimeWindowPropertyCollector.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/TimeWindowPropertyCollector.scala
index 9502607622fc8..0c8ae007a3f9c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/TimeWindowPropertyCollector.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/TimeWindowPropertyCollector.scala
@@ -19,6 +19,7 @@
 package org.apache.flink.table.runtime.aggregate
 
 import org.apache.calcite.runtime.SqlFunctions
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.types.Row
 import org.apache.flink.util.Collector
 
@@ -26,29 +27,48 @@ import org.apache.flink.util.Collector
   * Adds TimeWindow properties to specified fields of a row before it emits the row to a wrapped
   * collector.
   */
-class TimeWindowPropertyCollector(windowStartOffset: Option[Int], windowEndOffset: Option[Int])
-    extends Collector[Row] {
+abstract class TimeWindowPropertyCollector[T](
+    windowStartOffset: Option[Int],
+    windowEndOffset: Option[Int])
+  extends Collector[T] {
 
-  var wrappedCollector: Collector[Row] = _
+  var wrappedCollector: Collector[T] = _
+  var output: Row = _
   var windowStart:Long = _
   var windowEnd:Long = _
 
-  override def collect(record: Row): Unit = {
+  def getRow(record: T): Row
 
-    val lastFieldPos = record.getArity - 1
+  override def collect(record: T): Unit = {
+
+    output = getRow(record)
+    val lastFieldPos = output.getArity - 1
 
     if (windowStartOffset.isDefined) {
-      record.setField(
+      output.setField(
         lastFieldPos + windowStartOffset.get,
         SqlFunctions.internalToTimestamp(windowStart))
     }
     if (windowEndOffset.isDefined) {
-      record.setField(
+      output.setField(
         lastFieldPos + windowEndOffset.get,
         SqlFunctions.internalToTimestamp(windowEnd))
     }
+
     wrappedCollector.collect(record)
   }
 
   override def close(): Unit = wrappedCollector.close()
 }
+
+class RowTimeWindowPropertyCollector(startOffset: Option[Int], endOffset: Option[Int])
+  extends TimeWindowPropertyCollector[Row](startOffset, endOffset) {
+
+  override def getRow(record: Row): Row = record
+}
+
+class CRowTimeWindowPropertyCollector(startOffset: Option[Int], endOffset: Option[Int])
+  extends TimeWindowPropertyCollector[CRow](startOffset, endOffset) {
+
+  override def getRow(record: CRow): Row = record.row
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
new file mode 100644
index 0000000000000..ec73fa6058b53
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.io
+
+import org.apache.flink.api.common.io.{GenericInputFormat, NonParallelInput}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.core.io.GenericInputSplit
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.slf4j.LoggerFactory
+
+class CRowValuesInputFormat(
+    name: String,
+    code: String,
+    @transient returnType: TypeInformation[CRow])
+  extends GenericInputFormat[CRow]
+  with NonParallelInput
+  with ResultTypeQueryable[CRow]
+  with Compiler[GenericInputFormat[Row]] {
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+
+  private var format: GenericInputFormat[Row] = _
+
+  override def open(split: GenericInputSplit): Unit = {
+    LOG.debug(s"Compiling GenericInputFormat: $name \n\n Code:\n$code")
+    val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
+    LOG.debug("Instantiating GenericInputFormat.")
+    format = clazz.newInstance()
+  }
+
+  override def reachedEnd(): Boolean = format.reachedEnd()
+
+  override def nextRecord(reuse: CRow): CRow = {
+    reuse.row = format.nextRecord(reuse.row)
+    reuse.change = true
+    reuse
+  }
+
+  override def getProducedType: TypeInformation[CRow] = returnType
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
index 1a339e67d221f..d536b39f68798 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
@@ -23,20 +23,21 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.table.codegen.Compiler
 import org.apache.flink.core.io.GenericInputSplit
+import org.apache.flink.types.Row
 import org.slf4j.LoggerFactory
 
-class ValuesInputFormat[OUT](
+class ValuesInputFormat(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT])
-  extends GenericInputFormat[OUT]
+    @transient returnType: TypeInformation[Row])
+  extends GenericInputFormat[Row]
   with NonParallelInput
-  with ResultTypeQueryable[OUT]
-  with Compiler[GenericInputFormat[OUT]] {
+  with ResultTypeQueryable[Row]
+  with Compiler[GenericInputFormat[Row]] {
 
   val LOG = LoggerFactory.getLogger(this.getClass)
 
-  private var format: GenericInputFormat[OUT] = _
+  private var format: GenericInputFormat[Row] = _
 
   override def open(split: GenericInputSplit): Unit = {
     LOG.debug(s"Compiling GenericInputFormat: $name \n\n Code:\n$code")
@@ -47,7 +48,7 @@ class ValuesInputFormat[OUT](
 
   override def reachedEnd(): Boolean = format.reachedEnd()
 
-  override def nextRecord(reuse: OUT): OUT = format.nextRecord(reuse)
+  override def nextRecord(reuse: Row): Row = format.nextRecord(reuse)
 
-  override def getProducedType: TypeInformation[OUT] = returnType
+  override def getProducedType: TypeInformation[Row] = returnType
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRow.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRow.scala
new file mode 100644
index 0000000000000..25ec8c4d6fbdd
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRow.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.types
+
+import org.apache.flink.types.Row
+
+/**
+  * Wrapper for a [[Row]] to add retraction information.
+  *
+  * If [[change]] is true, the [[CRow]] is an accumulate message, if it is false it is a
+  * retraction message.
+  *
+  * @param row The wrapped [[Row]].
+  * @param change true for an accumulate message, false for a retraction message.
+  */
+class CRow(var row: Row, var change: Boolean) {
+
+  def this() {
+    this(null, true)
+  }
+
+  override def toString: String = s"${if(change) "+" else "-"}$row"
+
+  override def equals(other: scala.Any): Boolean = {
+    val otherCRow = other.asInstanceOf[CRow]
+    row.equals(otherCRow.row) && change == otherCRow.change
+  }
+}
+
+object CRow {
+
+  def apply(): CRow = {
+    new CRow()
+  }
+
+  def apply(row: Row, change: Boolean): CRow = {
+    new CRow(row, change)
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowComparator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowComparator.scala
new file mode 100644
index 0000000000000..d848c6589efa4
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowComparator.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.types
+
+import org.apache.flink.api.common.typeutils.TypeComparator
+import org.apache.flink.core.memory.{DataInputView, DataOutputView, MemorySegment}
+import org.apache.flink.types.Row
+
+class CRowComparator(val rowComp: TypeComparator[Row]) extends TypeComparator[CRow] {
+
+  override def hash(record: CRow): Int = rowComp.hash(record.row)
+
+  override def setReference(toCompare: CRow): Unit = rowComp.setReference(toCompare.row)
+
+  override def equalToReference(candidate: CRow): Boolean = rowComp.equalToReference(candidate.row)
+
+  override def compareToReference(otherComp: TypeComparator[CRow]): Int = {
+    val otherCRowComp = otherComp.asInstanceOf[CRowComparator]
+    rowComp.compareToReference(otherCRowComp.rowComp)
+  }
+
+  override def compare(first: CRow, second: CRow): Int = {
+    rowComp.compare(first.row, second.row)
+  }
+
+  override def compareSerialized(firstSource: DataInputView, secondSource: DataInputView): Int = {
+    rowComp.compareSerialized(firstSource, secondSource)
+  }
+
+  override def supportsNormalizedKey(): Boolean = rowComp.supportsNormalizedKey()
+
+  override def supportsSerializationWithKeyNormalization(): Boolean =
+    rowComp.supportsSerializationWithKeyNormalization()
+
+  override def getNormalizeKeyLen: Int = rowComp.getNormalizeKeyLen
+
+  override def isNormalizedKeyPrefixOnly(keyBytes: Int): Boolean =
+    rowComp.isNormalizedKeyPrefixOnly(keyBytes)
+
+  override def putNormalizedKey(
+      record: CRow,
+      target: MemorySegment,
+      offset: Int,
+      numBytes: Int): Unit = rowComp.putNormalizedKey(record.row, target, offset, numBytes)
+
+  override def writeWithKeyNormalization(record: CRow, target: DataOutputView): Unit = {
+    rowComp.writeWithKeyNormalization(record.row, target)
+    target.writeBoolean(record.change)
+  }
+
+  override def readWithKeyDenormalization(reuse: CRow, source: DataInputView): CRow = {
+    val row = rowComp.readWithKeyDenormalization(reuse.row, source)
+    reuse.row = row
+    reuse.change = source.readBoolean()
+    reuse
+  }
+
+  override def invertNormalizedKey(): Boolean = rowComp.invertNormalizedKey()
+
+  override def duplicate(): TypeComparator[CRow] = new CRowComparator(rowComp.duplicate())
+
+  override def extractKeys(record: scala.Any, target: Array[AnyRef], index: Int): Int =
+    rowComp.extractKeys(record.asInstanceOf[CRow].row, target, index)
+
+  override def getFlatComparators: Array[TypeComparator[_]] =
+    rowComp.getFlatComparators
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
new file mode 100644
index 0000000000000..1f56a985966c6
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.types
+
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.core.memory.{DataInputView, DataOutputView}
+import org.apache.flink.types.Row
+
+class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSerializer[CRow] {
+
+  override def isImmutableType: Boolean = false
+
+  override def duplicate(): TypeSerializer[CRow] = new CRowSerializer(rowSerializer.duplicate())
+
+  override def createInstance(): CRow = new CRow(rowSerializer.createInstance(), true)
+
+  override def copy(from: CRow): CRow = new CRow(rowSerializer.copy(from.row), from.change)
+
+  override def copy(from: CRow, reuse: CRow): CRow = {
+    rowSerializer.copy(from.row, reuse.row)
+    reuse.change = from.change
+    reuse
+  }
+
+  override def getLength: Int = -1
+
+  override def serialize(record: CRow, target: DataOutputView): Unit = {
+    rowSerializer.serialize(record.row, target)
+    target.writeBoolean(record.change)
+  }
+
+  override def deserialize(source: DataInputView): CRow = {
+    val row = rowSerializer.deserialize(source)
+    val change = source.readBoolean()
+    new CRow(row, change)
+  }
+
+  override def deserialize(reuse: CRow, source: DataInputView): CRow = {
+    rowSerializer.deserialize(reuse.row, source)
+    reuse.change = source.readBoolean()
+    reuse
+  }
+
+  override def copy(source: DataInputView, target: DataOutputView): Unit = {
+    rowSerializer.copy(source, target)
+    target.writeBoolean(source.readBoolean())
+  }
+
+  override def canEqual(obj: Any): Boolean = obj.isInstanceOf[CRowSerializer]
+
+  override def equals(obj: Any): Boolean = {
+
+    if (canEqual(obj)) {
+      val other = obj.asInstanceOf[CRowSerializer]
+      rowSerializer.equals(other.rowSerializer)
+    } else {
+      false
+    }
+  }
+
+  override def hashCode: Int = rowSerializer.hashCode() * 13
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowTypeInfo.scala
new file mode 100644
index 0000000000000..456207a93d556
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowTypeInfo.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.types
+
+import java.util
+
+import org.apache.flink.api.common.ExecutionConfig
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.{CompositeType, TypeComparator, TypeSerializer}
+import org.apache.flink.api.common.typeutils.CompositeType.{FlatFieldDescriptor, TypeComparatorBuilder}
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.types.Row
+
+class CRowTypeInfo(val rowType: RowTypeInfo) extends CompositeType[CRow](classOf[CRow]) {
+
+  override def getFieldNames: Array[String] = rowType.getFieldNames
+
+  override def getFieldIndex(fieldName: String): Int = rowType.getFieldIndex(fieldName)
+
+  override def getTypeAt[X](fieldExpression: String): TypeInformation[X] =
+    rowType.getTypeAt(fieldExpression)
+
+  override def getTypeAt[X](pos: Int): TypeInformation[X] =
+    rowType.getTypeAt(pos)
+
+  override def getFlatFields(
+      fieldExpression: String,
+      offset: Int,
+      result: util.List[FlatFieldDescriptor]): Unit =
+    rowType.getFlatFields(fieldExpression, offset, result)
+
+  override def isBasicType: Boolean = rowType.isBasicType
+
+  override def isTupleType: Boolean = rowType.isTupleType
+
+  override def getArity: Int = rowType.getArity
+
+  override def getTotalFields: Int = rowType.getTotalFields
+
+  override def createSerializer(config: ExecutionConfig): TypeSerializer[CRow] =
+    new CRowSerializer(rowType.createSerializer(config))
+
+  // not implemented because we override createComparator
+  override protected def createTypeComparatorBuilder(): TypeComparatorBuilder[CRow] = null
+
+  override def createComparator(
+      logicalKeyFields: Array[Int],
+      orders: Array[Boolean],
+      logicalFieldOffset: Int,
+      config: ExecutionConfig): TypeComparator[CRow] = {
+
+    val rowComparator = rowType.createComparator(
+      logicalKeyFields,
+      orders,
+      logicalFieldOffset,
+      config)
+
+    new CRowComparator(rowComparator)
+  }
+
+  override def equals(obj: scala.Any): Boolean = {
+    if (this.canEqual(obj)) {
+      rowType.equals(obj.asInstanceOf[CRowTypeInfo].rowType)
+    } else {
+      false
+    }
+  }
+
+  override def canEqual(obj: scala.Any): Boolean = obj.isInstanceOf[CRowTypeInfo]
+
+}
+
+object CRowTypeInfo {
+
+  def apply(rowType: TypeInformation[Row]): CRowTypeInfo = {
+    rowType match {
+      case r: RowTypeInfo => new CRowTypeInfo(r)
+    }
+  }
+
+}
+
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
index c37ee742ce132..4a2fcdfee511a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
@@ -25,6 +25,7 @@ import org.apache.flink.types.Row
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.core.fs.FileSystem.WriteMode
 import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 /**
   * A simple [[TableSink]] to emit data as CSV files.
@@ -133,3 +134,4 @@ class CsvFormatter(fieldDelim: String) extends MapFunction[Row, String] {
     builder.mkString
   }
 }
+
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index 52476853e0e75..60de1f14519f4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -22,9 +22,10 @@ import org.apache.flink.api.scala._
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.scala._
-import org.apache.flink.api.java.typeutils.{GenericTypeInfo, TupleTypeInfo, TypeExtractor}
+import org.apache.flink.api.java.typeutils.{GenericTypeInfo, RowTypeInfo, TupleTypeInfo, TypeExtractor}
 import org.apache.flink.table.api.TableException
 import org.apache.flink.table.expressions.{Alias, UnresolvedFieldReference}
+import org.apache.flink.table.runtime.types.CRowTypeInfo
 import org.apache.flink.table.utils.{MockTableEnvironment, TableTestBase}
 import org.apache.flink.table.utils.TableTestUtil.{batchTableNode, binaryNode, streamTableNode, term, unaryNode}
 import org.apache.flink.types.Row
@@ -40,9 +41,13 @@ class TableEnvironmentTest extends TableTestBase {
     STRING_TYPE_INFO,
     DOUBLE_TYPE_INFO)
 
-  val caseClassType = implicitly[TypeInformation[CClass]]
+  val rowType = new RowTypeInfo(INT_TYPE_INFO, STRING_TYPE_INFO,DOUBLE_TYPE_INFO)
 
-  val pojoType = TypeExtractor.createTypeInfo(classOf[PojoClass])
+  val cRowType = new CRowTypeInfo(rowType)
+
+  val caseClassType: TypeInformation[CClass] = implicitly[TypeInformation[CClass]]
+
+  val pojoType: TypeInformation[PojoClass] = TypeExtractor.createTypeInfo(classOf[PojoClass])
 
   val atomicType = INT_TYPE_INFO
 
@@ -56,6 +61,14 @@ class TableEnvironmentTest extends TableTestBase {
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
   }
 
+  @Test
+  def testGetFieldInfoCRow(): Unit = {
+    val fieldInfo = tEnv.getFieldInfo(cRowType)
+
+    fieldInfo._1.zip(Array("f0", "f1", "f2")).foreach(x => assertEquals(x._2, x._1))
+    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
+  }
+
   @Test
   def testGetFieldInfoCClass(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(caseClassType)
@@ -99,6 +112,20 @@ class TableEnvironmentTest extends TableTestBase {
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
   }
 
+  @Test
+  def testGetFieldInfoCRowNames(): Unit = {
+    val fieldInfo = tEnv.getFieldInfo(
+      cRowType,
+      Array(
+        UnresolvedFieldReference("name1"),
+        UnresolvedFieldReference("name2"),
+        UnresolvedFieldReference("name3")
+      ))
+
+    fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
+    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
+  }
+
   @Test
   def testGetFieldInfoCClassNames(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(
@@ -197,6 +224,45 @@ class TableEnvironmentTest extends TableTestBase {
       ))
   }
 
+  @Test
+  def testGetFieldInfoCRowAlias1(): Unit = {
+    val fieldInfo = tEnv.getFieldInfo(
+      cRowType,
+      Array(
+        Alias(UnresolvedFieldReference("f0"), "name1"),
+        Alias(UnresolvedFieldReference("f1"), "name2"),
+        Alias(UnresolvedFieldReference("f2"), "name3")
+      ))
+
+    fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
+    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
+  }
+
+  @Test
+  def testGetFieldInfoCRowAlias2(): Unit = {
+    val fieldInfo = tEnv.getFieldInfo(
+      cRowType,
+      Array(
+        Alias(UnresolvedFieldReference("f2"), "name1"),
+        Alias(UnresolvedFieldReference("f0"), "name2"),
+        Alias(UnresolvedFieldReference("f1"), "name3")
+      ))
+
+    fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
+    fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
+  }
+
+  @Test(expected = classOf[TableException])
+  def testGetFieldInfoCRowAlias3(): Unit = {
+    tEnv.getFieldInfo(
+      cRowType,
+      Array(
+        Alias(UnresolvedFieldReference("xxx"), "name1"),
+        Alias(UnresolvedFieldReference("yyy"), "name2"),
+        Alias(UnresolvedFieldReference("zzz"), "name3")
+      ))
+  }
+
   @Test
   def testGetFieldInfoCClassAlias1(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
index 57ee3b3fca2cb..ebfac0a2caece 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.scala.batch.utils.{TableProgramsCollectionTest
 import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
 import org.apache.flink.types.Row
 import org.apache.flink.table.api.{TableEnvironment, TableException}
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.test.util.TestBaseUtils
 import org.junit._
 import org.junit.runner.RunWith
@@ -163,6 +164,25 @@ class TableEnvironmentITCase(
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
+  @Test
+  def testToDataSetWithTypeOfCRow(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val t = CollectionDataSets.get3TupleDataSet(env)
+      .toTable(tEnv, 'a, 'b, 'c)
+      .select('a, 'b, 'c)
+
+    val expected = "+1,1,Hi\n" + "+2,2,Hello\n" + "+3,2,Hello world\n" +
+      "+4,3,Hello world, how are you?\n" + "+5,3,I am fine.\n" + "+6,3,Luke Skywalker\n" +
+      "+7,4,Comment#1\n" + "+8,4,Comment#2\n" + "+9,4,Comment#3\n" + "+10,4,Comment#4\n" +
+      "+11,5,Comment#5\n" + "+12,5,Comment#6\n" + "+13,5,Comment#7\n" + "+14,5,Comment#8\n" +
+      "+15,5,Comment#9\n" + "+16,6,Comment#10\n" + "+17,6,Comment#11\n" + "+18,6,Comment#12\n" +
+      "+19,6,Comment#13\n" + "+20,6,Comment#14\n" + "+21,6,Comment#15\n"
+    val results = t.toDataSet[CRow].collect()
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
   @Test
   def testToTableFromCaseClass(): Unit = {
     val env = ExecutionEnvironment.getExecutionEnvironment
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
new file mode 100644
index 0000000000000..dde7f890bfbb3
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream
+
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.table.api.{TableEnvironment, TableException}
+import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamingWithStateTestBase}
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+import org.apache.flink.table.api.scala._
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.utils.TableFunc0
+
+import scala.collection.mutable
+
+/**
+  * tests for retraction
+  */
+class RetractionITCase extends StreamingWithStateTestBase {
+  // input data
+  val data = List(
+    ("Hello", 1),
+    ("word", 1),
+    ("Hello", 1),
+    ("bark", 1),
+    ("bark", 1),
+    ("bark", 1),
+    ("bark", 1),
+    ("bark", 1),
+    ("bark", 1),
+    ("flink", 1)
+  )
+
+  // keyed groupby + keyed groupby
+  @Test
+  def testWordCount(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+    env.setParallelism(1)
+    env.setStateBackend(getStateBackend)
+
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv, 'word, 'num)
+    val resultTable = table
+      .groupBy('word)
+      .select('word as 'word, 'num.sum as 'count)
+      .groupBy('count)
+      .select('count, 'word.count as 'frequency)
+
+    // to DataStream with CRow
+    val results = resultTable.toDataStream[CRow]
+    results.addSink(new StreamITCase.StringSinkWithCRow)
+    env.execute()
+
+    val expected = Seq("+1,1", "+1,2", "+1,1", "+2,1", "+1,2", "+1,1", "+2,2", "+2,1", "+3,1",
+      "+3,0", "+4,1", "+4,0", "+5,1", "+5,0", "+6,1", "+1,2")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  // keyed groupby + non-keyed groupby
+  @Test
+  def testGroupByAndNonKeyedGroupBy(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+    env.setParallelism(1)
+    env.setStateBackend(getStateBackend)
+
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv, 'word, 'num)
+    val resultTable = table
+      .groupBy('word)
+      .select('word as 'word, 'num.sum as 'count)
+      .select('count.sum)
+
+    val results = resultTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq("1", "2", "1", "3", "4", "3", "5", "3", "6", "3", "7", "3", "8", "3", "9",
+      "10")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  // non-keyed groupby + keyed groupby
+  @Test
+  def testNonKeyedGroupByAndGroupBy(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+    env.setParallelism(1)
+    env.setStateBackend(getStateBackend)
+
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv, 'word, 'num)
+    val resultTable = table
+      .select('num.sum as 'count)
+      .groupBy('count)
+      .select('count, 'count.count)
+
+    val results = resultTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq("1,1", "1,0", "2,1", "2,0", "3,1", "3,0", "4,1", "4,0", "5,1", "5,0", "6," +
+      "1", "6,0", "7,1", "7,0", "8,1", "8,0", "9,1", "9,0", "10,1")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  // test unique process, if the current output message of unbounded groupby equals the
+  // previous message, unbounded groupby will ignore the current one.
+  @Test
+  def testUniqueProcess(): Unit = {
+    // data input
+    val data = List(
+      (1, 1L),
+      (2, 2L),
+      (3, 3L),
+      (3, 3L),
+      (4, 1L),
+      (4, 0L),
+      (4, 0L),
+      (4, 0L),
+      (5, 1L),
+      (6, 6L),
+      (6, 6L),
+      (6, 6L),
+      (7, 8L)
+    )
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+    env.setParallelism(1)
+    env.setStateBackend(getStateBackend)
+
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv, 'pk, 'value)
+    val resultTable = table
+      .groupBy('pk)
+      .select('pk as 'pk, 'value.sum as 'sum)
+      .groupBy('sum)
+      .select('sum, 'pk.count as 'count)
+
+    val results = resultTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq("1,1", "2,1", "3,1", "3,0", "6,1", "1,2", "1,3", "6,2", "6,1", "12,1","12," +
+      "0", "18,1", "8,1")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  // correlate should handle retraction messages correctly
+  @Test
+  def testCorrelate(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+    env.setParallelism(1)
+    env.setStateBackend(getStateBackend)
+
+    val func0 = new TableFunc0
+
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv, 'word, 'num)
+    val resultTable = table
+      .groupBy('word)
+      .select('word as 'word, 'num.sum as 'count)
+      .leftOuterJoin(func0('word))
+      .groupBy('count)
+      .select('count, 'word.count as 'frequency)
+
+    val results = resultTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq(
+      "1,1", "1,2", "1,1", "2,1", "1,2", "1,1", "2,2", "2,1", "3,1", "3,0", "4,1", "4,0", "5,1",
+      "5,0", "6,1", "1,2")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
index cdc4329c3b0a1..c446d64f780ed 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
@@ -59,5 +59,5 @@ class TableSinkITCase extends StreamingMultipleProgramsTestBase {
 
     TestBaseUtils.compareResultsByLinesInMemory(expected, path)
   }
-
+  
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
index f826bbab2875f..6c75d53ca221c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
@@ -25,6 +25,7 @@ import org.junit.Assert._
 
 import scala.collection.mutable
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
+import org.apache.flink.table.runtime.types.CRow
 
 import scala.collection.JavaConverters._
 
@@ -44,7 +45,15 @@ object StreamITCase {
   final class StringSink extends RichSinkFunction[Row]() {
     def invoke(value: Row) {
       testResults.synchronized {
-        testResults += value.toString 
+        testResults += value.toString
+      }
+    }
+  }
+
+  final class StringSinkWithCRow extends RichSinkFunction[CRow]() {
+    def invoke(value: CRow) {
+      testResults.synchronized {
+        testResults += value.toString
       }
     }
   }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
index 5e3e995221ec5..eadcfc8c6d774 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
@@ -33,6 +33,7 @@ import org.apache.flink.table.codegen.GeneratedAggregationsFunction
 import org.apache.flink.table.functions.AggregateFunction
 import org.apache.flink.table.functions.aggfunctions.{LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction}
 import org.apache.flink.table.runtime.aggregate.BoundedProcessingOverRangeProcessFunctionTest._
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.apache.flink.types.Row
 import org.junit.Test
 
@@ -41,13 +42,13 @@ class BoundedProcessingOverRangeProcessFunctionTest {
   @Test
   def testProcTimePartitionedOverRange(): Unit = {
 
-    val rT =  new RowTypeInfo(Array[TypeInformation[_]](
+    val rT =  new CRowTypeInfo(new RowTypeInfo(Array[TypeInformation[_]](
       INT_TYPE_INFO,
       LONG_TYPE_INFO,
       INT_TYPE_INFO,
       STRING_TYPE_INFO,
       LONG_TYPE_INFO),
-      Array("a", "b", "c", "d", "e"))
+      Array("a", "b", "c", "d", "e")))
 
     val aggregates =
       Array(new LongMinWithRetractAggFunction,
@@ -183,14 +184,14 @@ class BoundedProcessingOverRangeProcessFunctionTest {
     val funcName = "BoundedOverAggregateHelper$33"
 
     val genAggFunction = GeneratedAggregationsFunction(funcName, funcCode)
-    val processFunction = new KeyedProcessOperator[String, Row, Row](
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new ProcTimeBoundedRangeOver(
         genAggFunction,
         1000,
         aggregationStateType,
         rT))
 
-    val testHarness = new KeyedOneInputStreamOperatorTestHarness[JInt, Row, Row](
+    val testHarness = new KeyedOneInputStreamOperatorTestHarness[JInt, CRow, CRow](
       processFunction,
       new TupleRowSelector(0),
       BasicTypeInfo.INT_TYPE_INFO)
@@ -201,26 +202,26 @@ class BoundedProcessingOverRangeProcessFunctionTest {
     testHarness.setProcessingTime(3)
     // key = 1
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 0))
     // key = 2
     testHarness.processElement(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), 0))
+      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 0))
 
     // Time = 4
     testHarness.setProcessingTime(4)
     // key = 1
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 0))
     // key = 2
     testHarness.processElement(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), 0))
+      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 0))
 
     // Time = 5
     testHarness.setProcessingTime(5)
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 0))
 
     // Time = 6
     testHarness.setProcessingTime(6)
@@ -229,33 +230,33 @@ class BoundedProcessingOverRangeProcessFunctionTest {
     testHarness.setProcessingTime(1002)
     // key = 1
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 0))
     // key = 2
     testHarness.processElement(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), 0))
+      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 0))
 
     // Time = 1003
     testHarness.setProcessingTime(1003)
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 0))
 
     // Time = 1004
     testHarness.setProcessingTime(1004)
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 0))
 
     // Time = 1005
     testHarness.setProcessingTime(1005)
     // key = 1
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), 0))
+      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 0))
     // key = 2
     testHarness.processElement(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), 0))
+      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 0))
 
     testHarness.setProcessingTime(1006)
 
@@ -264,34 +265,34 @@ class BoundedProcessingOverRangeProcessFunctionTest {
     val expectedOutput = new ConcurrentLinkedQueue[Object]()
 
     // all elements at the same proc timestamp have the same value
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), 4))
-    expectedOutput.add(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), 4))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 3L: JLong), 5))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), 5))
-    expectedOutput.add(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), 5))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), 6))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 6L: JLong), 1003))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), 1003))
-    expectedOutput.add(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), 1003))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), 1004))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 2L: JLong, 8L: JLong), 1005))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 4L: JLong, 10L: JLong), 1006))
-    expectedOutput.add(new StreamRecord(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 4L: JLong, 10L: JLong), 1006))
-    expectedOutput.add(new StreamRecord(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 30L: JLong, 40L: JLong), 1006))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 4))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 4))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 3L: JLong), true), 5))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 5))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 5))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), true), 6))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 6L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 1004))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 2L: JLong, 8L: JLong), true), 1005))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 4L: JLong, 10L: JLong), true), 1006))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 4L: JLong, 10L: JLong), true), 1006))
+    expectedOutput.add(new StreamRecord(new CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 30L: JLong, 40L: JLong), true), 1006))
 
     TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.",
         expectedOutput, result, new RowResultSortComparator(6))
@@ -304,7 +305,7 @@ class BoundedProcessingOverRangeProcessFunctionTest {
 object BoundedProcessingOverRangeProcessFunctionTest {
 
 /**
- * Return 0 for equal Rows and non zero for different rows
+ * Return 0 for equal CRows and non zero for different CRows
  */
 class RowResultSortComparator(indexCounter: Int) extends Comparator[Object] with Serializable {
 
@@ -314,8 +315,8 @@ class RowResultSortComparator(indexCounter: Int) extends Comparator[Object] with
         // watermark is not expected
          -1
        } else {
-        val row1 = o1.asInstanceOf[StreamRecord[Row]].getValue
-        val row2 = o2.asInstanceOf[StreamRecord[Row]].getValue
+        val row1 = o1.asInstanceOf[StreamRecord[CRow]].getValue
+        val row2 = o2.asInstanceOf[StreamRecord[CRow]].getValue
         row1.toString.compareTo(row2.toString)
       }
    }
@@ -325,10 +326,10 @@ class RowResultSortComparator(indexCounter: Int) extends Comparator[Object] with
  * Simple test class that returns a specified field as the selector function
  */
 class TupleRowSelector(
-    private val selectorField:Int) extends KeySelector[Row, Integer] {
+    private val selectorField:Int) extends KeySelector[CRow, Integer] {
 
-  override def getKey(value: Row): Integer = {
-    value.getField(selectorField).asInstanceOf[Integer]
+  override def getKey(value: CRow): Integer = {
+    value.row.getField(selectorField).asInstanceOf[Integer]
   }
 }
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowComparatorTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowComparatorTest.scala
new file mode 100644
index 0000000000000..e574084a9f781
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowComparatorTest.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.types
+
+import java.lang.{Integer => JInt, Long => JLong}
+
+import org.apache.flink.api.common.ExecutionConfig
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.common.typeutils.{ComparatorTestBase, TypeComparator, TypeSerializer}
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.types.Row
+
+class CRowComparatorTest extends ComparatorTestBase[CRow] {
+
+  val rowType = new RowTypeInfo(
+    BasicTypeInfo.INT_TYPE_INFO,
+    BasicTypeInfo.STRING_TYPE_INFO,
+    BasicTypeInfo.LONG_TYPE_INFO
+  )
+
+  val cRowType = new CRowTypeInfo(rowType)
+
+  override protected def createComparator(asc: Boolean): TypeComparator[CRow] = {
+    cRowType.createComparator(
+      Array[Int](0, 2),
+      Array[Boolean](asc, asc),
+      0,
+      new ExecutionConfig
+    )
+  }
+
+  override protected def createSerializer(): TypeSerializer[CRow] =
+    cRowType.createSerializer(new ExecutionConfig)
+
+  override protected def getSortedTestData: Array[CRow] = Array[CRow](
+    new CRow(Row.of(new JInt(1), "Hello", new JLong(1L)), true),
+    new CRow(Row.of(new JInt(1), "Hello", new JLong(2L)), true),
+    new CRow(Row.of(new JInt(2), "Hello", new JLong(2L)), false),
+    new CRow(Row.of(new JInt(2), "Hello", new JLong(3L)), true),
+    new CRow(Row.of(new JInt(3), "World", new JLong(0L)), false),
+    new CRow(Row.of(new JInt(4), "Hello", new JLong(0L)), true),
+    new CRow(Row.of(new JInt(5), "Hello", new JLong(1L)), true),
+    new CRow(Row.of(new JInt(5), "Hello", new JLong(4L)), false)
+  )
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
index c5e13a18d2bb3..79e957a677a00 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
@@ -18,7 +18,9 @@
 
 package org.apache.flink.table.utils
 
+import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.tools.RuleSet
+import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.{Table, TableConfig, TableEnvironment}
 import org.apache.flink.table.sinks.TableSink
 import org.apache.flink.table.sources.TableSource
@@ -36,4 +38,10 @@ class MockTableEnvironment extends TableEnvironment(new TableConfig) {
   override protected def getBuiltInNormRuleSet: RuleSet = ???
 
   override protected def getBuiltInPhysicalOptRuleSet: RuleSet = ???
+
+  override protected def getConversionMapper[IN, OUT](
+      physicalTypeInfo: TypeInformation[IN],
+      logicalRowType: RelDataType,
+      requestedTypeInfo: TypeInformation[OUT],
+      functionName: String) = ???
 }

From 475bd66e3db07563ddf3c985eb6fb0f267c11143 Mon Sep 17 00:00:00 2001
From: Hequn Cheng <chenghequn@gmail.com>
Date: Thu, 27 Apr 2017 23:03:44 +0800
Subject: [PATCH 160/253] [FLINK-6093] [table] Implement and turn on retraction
 for table sinks.

---
 .../table/api/BatchTableEnvironment.scala     |  16 +--
 .../table/api/StreamTableEnvironment.scala    | 122 ++++++++++++++++--
 .../apache/flink/table/api/TableConfig.scala  |  17 +++
 .../flink/table/api/TableEnvironment.scala    |  12 --
 .../DataStreamRetractionRules.scala           |   5 +-
 .../CRowInputTupleOutputMapRunner.scala       |  63 +++++++++
 .../flink/table/sinks/CsvTableSink.scala      |  98 +++++++++++++-
 .../flink/table/sinks/StreamRetractSink.scala |  35 +++++
 .../flink/table/TableEnvironmentTest.scala    |  62 +--------
 .../scala/batch/TableEnvironmentITCase.scala  |  19 ---
 .../api/scala/stream/RetractionITCase.scala   |   9 +-
 .../api/scala/stream/TableSinkITCase.scala    |  33 ++++-
 .../api/scala/stream/utils/StreamITCase.scala |   8 --
 13 files changed, 363 insertions(+), 136 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index f7955f055f871..c7bacfe19140c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -26,12 +26,11 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.sql2rel.RelDecorrelator
 import org.apache.calcite.tools.RuleSet
-import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
+import org.apache.flink.api.common.functions.MapFunction
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.io.DiscardingOutputFormat
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.api.java.{DataSet, ExecutionEnvironment}
-import org.apache.flink.configuration.Configuration
 import org.apache.flink.table.explain.PlanJsonParser
 import org.apache.flink.table.expressions.{Expression, RowtimeAttribute, TimeAttribute}
 import org.apache.flink.table.plan.nodes.FlinkConventions
@@ -39,7 +38,6 @@ import org.apache.flink.table.plan.nodes.dataset.DataSetRel
 import org.apache.flink.table.plan.rules.FlinkRuleSets
 import org.apache.flink.table.plan.schema.{DataSetTable, TableSourceTable}
 import org.apache.flink.table.runtime.MapRunner
-import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.table.sinks.{BatchTableSink, TableSink}
 import org.apache.flink.table.sources.{BatchTableSource, TableSource}
 import org.apache.flink.types.Row
@@ -150,18 +148,6 @@ abstract class BatchTableEnvironment(
     if (requestedTypeInfo.getTypeClass == classOf[Row]) {
       // Row to Row, no conversion needed
       None
-    } else if (requestedTypeInfo.getTypeClass == classOf[CRow]) {
-      // Row to CRow, only needs to be wrapped
-      Some(
-        new RichMapFunction[Row, CRow] {
-          private var outCRow: CRow = _
-          override def open(parameters: Configuration): Unit = outCRow = new CRow(null, true)
-          override def map(value: Row): CRow = {
-            outCRow.row = value
-            outCRow
-          }
-        }.asInstanceOf[MapFunction[IN, OUT]]
-      )
     } else {
       // some type that is neither Row or CRow
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index 0632a47c55aa0..bd063051bf8b8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -26,22 +26,25 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.sql2rel.RelDecorrelator
 import org.apache.calcite.tools.{RuleSet, RuleSets}
-import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
+import org.apache.flink.api.common.typeinfo.AtomicType
 import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.api.common.functions.MapFunction
-import org.apache.flink.api.java.typeutils.GenericTypeInfo
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.api.java.typeutils.{GenericTypeInfo, TupleTypeInfo}
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.apache.flink.table.calcite.RelTimeIndicatorConverter
 import org.apache.flink.table.explain.PlanJsonParser
 import org.apache.flink.table.expressions.{Expression, ProctimeAttribute, RowtimeAttribute, UnresolvedFieldReference}
 import org.apache.flink.table.plan.nodes.FlinkConventions
-import org.apache.flink.table.plan.nodes.datastream.DataStreamRel
+import org.apache.flink.table.plan.nodes.datastream.{DataStreamRel, UpdateAsRetractionTrait}
 import org.apache.flink.table.plan.rules.FlinkRuleSets
-import org.apache.flink.table.plan.schema.{DataStreamTable, StreamTableSourceTable}
-import org.apache.flink.table.runtime.CRowInputMapRunner
+import org.apache.flink.table.plan.schema.StreamTableSourceTable
+import org.apache.flink.table.plan.schema.DataStreamTable
+import org.apache.flink.table.runtime.{CRowInputMapRunner, CRowInputTupleOutputMapRunner}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.apache.flink.table.sinks.{StreamTableSink, TableSink}
+import org.apache.flink.table.sinks.{StreamRetractSink, StreamTableSink, TableSink}
 import org.apache.flink.table.sources.{StreamTableSource, TableSource}
 import org.apache.flink.table.typeutils.TypeCheckUtils
 import org.apache.flink.types.Row
@@ -130,6 +133,14 @@ abstract class StreamTableEnvironment(
         val result: DataStream[T] = translate(table)(outputType)
         // Give the DataSet to the TableSink to emit it.
         streamSink.emitDataStream(result)
+
+      case streamRetractSink: StreamRetractSink[T] =>
+        val outputType = sink.getOutputType
+        this.config.setNeedsUpdatesAsRetractionForSink(streamRetractSink.needsUpdatesAsRetraction)
+        // translate the Table into a DataStream and provide the type that the TableSink expects.
+        val result: DataStream[JTuple2[Boolean, T]] = translate(table, true)(outputType)
+        // Give the DataSet to the TableSink to emit it.
+        streamRetractSink.emitDataStreamWithChange(result)
       case _ =>
         throw new TableException("StreamTableSink required to emit streaming Table")
     }
@@ -153,7 +164,7 @@ abstract class StreamTableEnvironment(
   Option[MapFunction[IN, OUT]] = {
 
     if (requestedTypeInfo.getTypeClass == classOf[CRow]) {
-      // CRow to CRow, no conversion needed
+      // only used to explain table
       None
     } else if (requestedTypeInfo.getTypeClass == classOf[Row]) {
       // CRow to Row, only needs to be unwrapped
@@ -164,7 +175,6 @@ abstract class StreamTableEnvironment(
       )
     } else {
       // Some type that is neither CRow nor Row
-
       val converterFunction = generateRowConverterFunction[OUT](
         physicalTypeInfo.asInstanceOf[CRowTypeInfo].rowType,
         logicalRowType,
@@ -177,10 +187,39 @@ abstract class StreamTableEnvironment(
         converterFunction.code,
         converterFunction.returnType)
         .asInstanceOf[MapFunction[IN, OUT]])
-
     }
   }
 
+  /**
+    * Creates a final converter that maps the internal CRow type to external Tuple2 type.
+    *
+    * @param physicalTypeInfo the input of the sink
+    * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
+    * @param requestedTypeInfo the output type of the sink
+    * @param functionName name of the map function. Must not be unique but has to be a
+    *                     valid Java class identifier.
+    */
+  protected def getTupleConversionMapper[IN, OUT](
+      physicalTypeInfo: TypeInformation[IN],
+      logicalRowType: RelDataType,
+      requestedTypeInfo: TypeInformation[OUT],
+      functionName: String):
+  Option[MapFunction[IN, JTuple2[Boolean, OUT]]] = {
+
+    val converterFunction = generateRowConverterFunction(
+      physicalTypeInfo.asInstanceOf[CRowTypeInfo].rowType,
+      logicalRowType,
+      requestedTypeInfo,
+      functionName
+    )
+
+    Some(new CRowInputTupleOutputMapRunner[OUT](
+      converterFunction.name,
+      converterFunction.code,
+      new TupleTypeInfo(BasicTypeInfo.BOOLEAN_TYPE_INFO, requestedTypeInfo))
+           .asInstanceOf[MapFunction[IN, JTuple2[Boolean, OUT]]])
+  }
+
   /**
     * Registers a [[DataStream]] as a table under a given name in the [[TableEnvironment]]'s
     * catalog.
@@ -371,7 +410,7 @@ abstract class StreamTableEnvironment(
     // 5. optimize the physical Flink plan
     val physicalOptRuleSet = getPhysicalOptRuleSet
     val physicalOutputProps = relNode.getTraitSet.replace(FlinkConventions.DATASTREAM).simplify()
-    val physicalPlan = if (physicalOptRuleSet.iterator().hasNext) {
+    var physicalPlan = if (physicalOptRuleSet.iterator().hasNext) {
       runVolcanoPlanner(physicalOptRuleSet, logicalPlan, physicalOutputProps)
     } else {
       logicalPlan
@@ -380,6 +419,12 @@ abstract class StreamTableEnvironment(
     // 6. decorate the optimized plan
     val decoRuleSet = getDecoRuleSet
     val decoratedPlan = if (decoRuleSet.iterator().hasNext) {
+
+      if (this.config.getNeedsUpdatesAsRetractionForSink) {
+        physicalPlan = physicalPlan.copy(
+          physicalPlan.getTraitSet.plus(new UpdateAsRetractionTrait(true)),
+          physicalPlan.getInputs)
+      }
       runHepPlanner(HepMatchOrder.BOTTOM_UP, decoRuleSet, physicalPlan, physicalPlan.getTraitSet)
     } else {
       physicalPlan
@@ -388,7 +433,6 @@ abstract class StreamTableEnvironment(
     decoratedPlan
   }
 
-
   /**
     * Translates a [[Table]] into a [[DataStream]].
     *
@@ -443,6 +487,62 @@ abstract class StreamTableEnvironment(
     }
   }
 
+  /**
+    * Translates a [[Table]] into a [[DataStream]] with change information.
+    *
+    * The transformation involves optimizing the relational expression tree as defined by
+    * Table API calls and / or SQL queries and generating corresponding [[DataStream]] operators.
+    *
+    * @param table       The root node of the relational expression tree.
+    * @param wrapToTuple True, if want to output chang information
+    * @param tpe         The [[TypeInformation]] of the resulting [[DataStream]].
+    * @tparam A The type of the resulting [[DataStream]].
+    * @return The [[DataStream]] that corresponds to the translated [[Table]].
+    */
+  protected def translate[A](table: Table, wrapToTuple: Boolean)(implicit tpe: TypeInformation[A])
+  : DataStream[JTuple2[Boolean, A]] = {
+    val relNode = table.getRelNode
+    val dataStreamPlan = optimize(relNode)
+    translate(dataStreamPlan, relNode.getRowType, wrapToTuple)
+  }
+
+  /**
+    * Translates a logical [[RelNode]] into a [[DataStream]] with change information.
+    *
+    * @param logicalPlan The root node of the relational expression tree.
+    * @param logicalType The row type of the result. Since the logicalPlan can lose the
+    * @param wrapToTuple True, if want to output chang information
+    * @param tpe         The [[TypeInformation]] of the resulting [[DataStream]].
+    * @tparam A The type of the resulting [[DataStream]].
+    * @return The [[DataStream]] that corresponds to the translated [[Table]].
+    */
+  protected def translate[A](
+      logicalPlan: RelNode,
+      logicalType: RelDataType,
+      wrapToTuple: Boolean)(implicit tpe: TypeInformation[A]): DataStream[JTuple2[Boolean, A]] = {
+
+    TableEnvironment.validateType(tpe)
+
+    logicalPlan match {
+      case node: DataStreamRel =>
+        val plan = node.translateToPlan(this)
+        val conversion =
+          getTupleConversionMapper(plan.getType, logicalType, tpe, "DataStreamSinkConversion")
+        conversion match {
+          case None => plan.asInstanceOf[DataStream[JTuple2[Boolean, A]]] // no conversion necessary
+          case Some(mapFunction: MapFunction[CRow, JTuple2[Boolean, A]]) =>
+            plan.map(mapFunction)
+              .returns(new TupleTypeInfo[JTuple2[Boolean, A]](BasicTypeInfo.BOOLEAN_TYPE_INFO, tpe))
+              .name(s"to: ${tpe.getTypeClass.getSimpleName}")
+              .asInstanceOf[DataStream[JTuple2[Boolean, A]]]
+        }
+
+      case _ =>
+        throw TableException("Cannot generate DataStream due to an invalid logical plan. " +
+          "This is a bug and should not happen. Please file an issue.")
+    }
+  }
+
   /**
     * Returns the AST of the specified Table API and SQL queries and the execution plan to compute
     * the result of the given [[Table]].
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala
index 6448657c5111f..d2969786bd669 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala
@@ -36,6 +36,11 @@ class TableConfig {
    */
   private var nullCheck: Boolean = true
 
+  /**
+    * Defines whether sink table requires that update and delete changes are sent with retraction
+    */
+  private var needsUpdatesAsRetractionForSink: Boolean = false
+
   /**
     * Defines the configuration of Calcite for Table API and SQL queries.
     */
@@ -66,6 +71,18 @@ class TableConfig {
     this.nullCheck = nullCheck
   }
 
+  /**
+    * Returns the need retraction property for table sink.
+    */
+  def getNeedsUpdatesAsRetractionForSink = needsUpdatesAsRetractionForSink
+
+  /**
+    * Set the need retraction property for table sink.
+    */
+  def setNeedsUpdatesAsRetractionForSink(needsUpdatesAsRetraction: Boolean ): Unit = {
+    this.needsUpdatesAsRetractionForSink = needsUpdatesAsRetraction
+  }
+
   /**
     * Returns the current configuration of Calcite for Table API and SQL queries.
     */
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index d27db1e67eb58..5b752ab1612a5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -644,18 +644,6 @@ abstract class TableEnvironment(val config: TableConfig) {
           case _ => throw new TableException(
             "Field reference expression or alias on field expression expected.")
         }
-      case cr: CRowTypeInfo =>
-        exprs.zipWithIndex.map {
-          case (UnresolvedFieldReference(name), idx) => (idx, name)
-          case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
-            val idx = cr.getFieldIndex(origName)
-            if (idx < 0) {
-              throw new TableException(s"$origName is not a field of type $cr")
-            }
-            (idx, name)
-          case _ => throw new TableException(
-            "Field reference expression or alias on field expression expected.")
-        }
       case c: CaseClassTypeInfo[A] =>
         exprs.zipWithIndex flatMap {
           case (UnresolvedFieldReference(name), idx) =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
index bd9a7ee8eaf70..97c0dbbc23e0d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
@@ -102,13 +102,14 @@ object DataStreamRetractionRules {
       val rel = call.rel(0).asInstanceOf[DataStreamRel]
       val traits = rel.getTraitSet
 
-      val traitsWithUpdateAsRetrac = if (!traits.contains(UpdateAsRetractionTraitDef.INSTANCE)) {
+      val traitsWithUpdateAsRetrac =
+        if (null == traits.getTrait(UpdateAsRetractionTraitDef.INSTANCE)) {
         traits.plus(UpdateAsRetractionTrait.DEFAULT)
       } else {
         traits
       }
       val traitsWithAccMode =
-        if (!traitsWithUpdateAsRetrac.contains(AccModeTraitDef.INSTANCE)) {
+        if (null == traitsWithUpdateAsRetrac.getTrait(AccModeTraitDef.INSTANCE)) {
           traitsWithUpdateAsRetrac.plus(AccModeTrait.DEFAULT)
       } else {
         traitsWithUpdateAsRetrac
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala
new file mode 100644
index 0000000000000..54bbf7ee3127f
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime
+
+import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.slf4j.LoggerFactory
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+
+
+/**
+  * Convert [[CRow]] to a [[Tuple2]]
+  */
+class CRowInputTupleOutputMapRunner[OUT](
+    name: String,
+    code: String,
+    @transient returnType: TypeInformation[JTuple2[Boolean, OUT]])
+  extends RichMapFunction[CRow, JTuple2[Boolean, OUT]]
+          with ResultTypeQueryable[JTuple2[Boolean, OUT]]
+          with Compiler[MapFunction[Row, OUT]] {
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+
+  private var function: MapFunction[Row, OUT] = _
+  private var tupleWrapper: JTuple2[Boolean, OUT] = _
+
+  override def open(parameters: Configuration): Unit = {
+    LOG.debug(s"Compiling MapFunction: $name \n\n Code:\n$code")
+    val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
+    LOG.debug("Instantiating MapFunction.")
+    function = clazz.newInstance()
+    tupleWrapper = new JTuple2[Boolean, OUT]()
+  }
+
+  override def map(in: CRow): JTuple2[Boolean, OUT] = {
+    tupleWrapper.f0 = in.change
+    tupleWrapper.f1 = function.map(in.row)
+    tupleWrapper
+  }
+
+  override def getProducedType: TypeInformation[JTuple2[Boolean, OUT]] = returnType
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
index 4a2fcdfee511a..809afd2fb4ec4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
@@ -25,7 +25,7 @@ import org.apache.flink.types.Row
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.core.fs.FileSystem.WriteMode
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 
 /**
   * A simple [[TableSink]] to emit data as CSV files.
@@ -135,3 +135,99 @@ class CsvFormatter(fieldDelim: String) extends MapFunction[Row, String] {
   }
 }
 
+/**
+  * A simple [[TableSink]] to emit data as CSV files.
+  *
+  * @param path The output path to write the Table to.
+  * @param fieldDelim The field delimiter
+  * @param numFiles The number of files to write to
+  * @param writeMode The write mode to specify whether existing files are overwritten or not.
+  */
+class CsvRetractTableSink(
+    path: String,
+    fieldDelim: Option[String],
+    numFiles: Option[Int],
+    writeMode: Option[WriteMode])
+  extends TableSinkBase[Row] with StreamRetractSink[Row] {
+
+  override def needsUpdatesAsRetraction: Boolean = true
+
+  /**
+    * A simple [[TableSink]] to emit data as CSV files.
+    *
+    * @param path The output path to write the Table to.
+    * @param fieldDelim The field delimiter, ',' by default.
+    */
+  def this(path: String, fieldDelim: String = ",") {
+    this(path, Some(fieldDelim), None, None)
+  }
+
+  /**
+    * A simple [[TableSink]] to emit data as CSV files.
+    *
+    * @param path The output path to write the Table to.
+    * @param fieldDelim The field delimiter.
+    * @param numFiles The number of files to write to.
+    * @param writeMode The write mode to specify whether existing files are overwritten or not.
+    */
+  def this(path: String, fieldDelim: String, numFiles: Int, writeMode: WriteMode) {
+    this(path, Some(fieldDelim), Some(numFiles), Some(writeMode))
+  }
+
+
+  override def emitDataStreamWithChange(dataStream: DataStream[JTuple2[Boolean,Row]]): Unit = {
+    val csvRows = dataStream
+      .map(new CsvRetractFormatter(fieldDelim.getOrElse(",")))
+      .returns(TypeInformation.of(classOf[String]))
+
+
+    if (numFiles.isDefined) {
+      csvRows.setParallelism(numFiles.get)
+    }
+
+    val sink = writeMode match {
+      case None => csvRows.writeAsText(path)
+      case Some(wm) => csvRows.writeAsText(path, wm)
+    }
+
+    if (numFiles.isDefined) {
+      sink.setParallelism(numFiles.get)
+    }
+  }
+
+  override protected def copy: TableSinkBase[Row] = {
+    new CsvRetractTableSink(path, fieldDelim, numFiles, writeMode)
+  }
+
+  override def getOutputType: TypeInformation[Row] = {
+    new RowTypeInfo(getFieldTypes: _*)
+  }
+}
+
+/**
+  * Formats a [[Tuple2]] with change information into a [[String]] with fields separated by the
+  * field delimiter.
+  *
+  * @param fieldDelim The field delimiter.
+  */
+class CsvRetractFormatter(fieldDelim: String) extends MapFunction[JTuple2[Boolean,Row], String] {
+  override def map(rowT: JTuple2[Boolean,Row]): String = {
+
+    val row: Row = rowT.f1
+
+    val builder = new StringBuilder
+
+    builder.append(rowT.f0.toString)
+
+    // write following values
+    for (i <- 0 until row.getArity) {
+      builder.append(fieldDelim)
+      val v = row.getField(i)
+      if (v != null) {
+        builder.append(v.toString)
+      }
+    }
+    builder.mkString
+  }
+}
+
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala
new file mode 100644
index 0000000000000..7f7c9449441e2
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.sinks
+
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.streaming.api.datastream.DataStream
+
+trait StreamRetractSink[T] extends TableSink[T]{
+
+  /**
+    * Whether the [[StreamTableSink]] requires that update and delete changes are sent with
+    * retraction messages.
+    */
+  def needsUpdatesAsRetraction: Boolean = false
+
+  /** Emits the DataStream with change infomation. */
+  def emitDataStreamWithChange(dataStream: DataStream[JTuple2[Boolean,T]]): Unit
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index 60de1f14519f4..675e5d9287574 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -21,6 +21,7 @@ package org.apache.flink.table
 import org.apache.flink.api.scala._
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.{TupleTypeInfo, TypeExtractor}
 import org.apache.flink.table.api.scala._
 import org.apache.flink.api.java.typeutils.{GenericTypeInfo, RowTypeInfo, TupleTypeInfo, TypeExtractor}
 import org.apache.flink.table.api.TableException
@@ -61,14 +62,6 @@ class TableEnvironmentTest extends TableTestBase {
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
   }
 
-  @Test
-  def testGetFieldInfoCRow(): Unit = {
-    val fieldInfo = tEnv.getFieldInfo(cRowType)
-
-    fieldInfo._1.zip(Array("f0", "f1", "f2")).foreach(x => assertEquals(x._2, x._1))
-    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
-  }
-
   @Test
   def testGetFieldInfoCClass(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(caseClassType)
@@ -112,20 +105,6 @@ class TableEnvironmentTest extends TableTestBase {
     fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
   }
 
-  @Test
-  def testGetFieldInfoCRowNames(): Unit = {
-    val fieldInfo = tEnv.getFieldInfo(
-      cRowType,
-      Array(
-        UnresolvedFieldReference("name1"),
-        UnresolvedFieldReference("name2"),
-        UnresolvedFieldReference("name3")
-      ))
-
-    fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
-    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
-  }
-
   @Test
   def testGetFieldInfoCClassNames(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(
@@ -224,45 +203,6 @@ class TableEnvironmentTest extends TableTestBase {
       ))
   }
 
-  @Test
-  def testGetFieldInfoCRowAlias1(): Unit = {
-    val fieldInfo = tEnv.getFieldInfo(
-      cRowType,
-      Array(
-        Alias(UnresolvedFieldReference("f0"), "name1"),
-        Alias(UnresolvedFieldReference("f1"), "name2"),
-        Alias(UnresolvedFieldReference("f2"), "name3")
-      ))
-
-    fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
-    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
-  }
-
-  @Test
-  def testGetFieldInfoCRowAlias2(): Unit = {
-    val fieldInfo = tEnv.getFieldInfo(
-      cRowType,
-      Array(
-        Alias(UnresolvedFieldReference("f2"), "name1"),
-        Alias(UnresolvedFieldReference("f0"), "name2"),
-        Alias(UnresolvedFieldReference("f1"), "name3")
-      ))
-
-    fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
-    fieldInfo._2.zip(Array(2, 0, 1)).foreach(x => assertEquals(x._2, x._1))
-  }
-
-  @Test(expected = classOf[TableException])
-  def testGetFieldInfoCRowAlias3(): Unit = {
-    tEnv.getFieldInfo(
-      cRowType,
-      Array(
-        Alias(UnresolvedFieldReference("xxx"), "name1"),
-        Alias(UnresolvedFieldReference("yyy"), "name2"),
-        Alias(UnresolvedFieldReference("zzz"), "name3")
-      ))
-  }
-
   @Test
   def testGetFieldInfoCClassAlias1(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
index ebfac0a2caece..0c2505a87ca5d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
@@ -164,25 +164,6 @@ class TableEnvironmentITCase(
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
-  @Test
-  def testToDataSetWithTypeOfCRow(): Unit = {
-    val env = ExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
-    val t = CollectionDataSets.get3TupleDataSet(env)
-      .toTable(tEnv, 'a, 'b, 'c)
-      .select('a, 'b, 'c)
-
-    val expected = "+1,1,Hi\n" + "+2,2,Hello\n" + "+3,2,Hello world\n" +
-      "+4,3,Hello world, how are you?\n" + "+5,3,I am fine.\n" + "+6,3,Luke Skywalker\n" +
-      "+7,4,Comment#1\n" + "+8,4,Comment#2\n" + "+9,4,Comment#3\n" + "+10,4,Comment#4\n" +
-      "+11,5,Comment#5\n" + "+12,5,Comment#6\n" + "+13,5,Comment#7\n" + "+14,5,Comment#8\n" +
-      "+15,5,Comment#9\n" + "+16,6,Comment#10\n" + "+17,6,Comment#11\n" + "+18,6,Comment#12\n" +
-      "+19,6,Comment#13\n" + "+20,6,Comment#14\n" + "+21,6,Comment#15\n"
-    val results = t.toDataSet[CRow].collect()
-    TestBaseUtils.compareResultAsText(results.asJava, expected)
-  }
-
   @Test
   def testToTableFromCaseClass(): Unit = {
     val env = ExecutionEnvironment.getExecutionEnvironment
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
index dde7f890bfbb3..d49076379e2c5 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
@@ -67,13 +67,12 @@ class RetractionITCase extends StreamingWithStateTestBase {
       .groupBy('count)
       .select('count, 'word.count as 'frequency)
 
-    // to DataStream with CRow
-    val results = resultTable.toDataStream[CRow]
-    results.addSink(new StreamITCase.StringSinkWithCRow)
+    val results = resultTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = Seq("+1,1", "+1,2", "+1,1", "+2,1", "+1,2", "+1,1", "+2,2", "+2,1", "+3,1",
-      "+3,0", "+4,1", "+4,0", "+5,1", "+5,0", "+6,1", "+1,2")
+    val expected = Seq("1,1", "1,2", "1,1", "2,1", "1,2", "1,1", "2,2", "2,1", "3,1", "3,0",
+      "4,1", "4,0", "5,1", "5,0", "6,1", "1,2")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
index c446d64f780ed..ceae6c610703b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
@@ -23,7 +23,7 @@ import java.io.File
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api.scala.stream.utils.StreamTestData
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.sinks.CsvTableSink
+import org.apache.flink.table.sinks.{CsvTableSink, CsvRetractTableSink}
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
 import org.apache.flink.table.api.TableEnvironment
@@ -59,5 +59,34 @@ class TableSinkITCase extends StreamingMultipleProgramsTestBase {
 
     TestBaseUtils.compareResultsByLinesInMemory(expected, path)
   }
-  
+
+  @Test
+  def testStreamTableSinkNeedRetraction(): Unit = {
+
+    val tmpFile = File.createTempFile("flink-table-sink-test", ".tmp")
+    tmpFile.deleteOnExit()
+    val path = tmpFile.toURI.toString
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setParallelism(4)
+
+    val input = StreamTestData.get3TupleDataStream(env)
+      .map(x => x).setParallelism(1) // increase DOP to 4
+
+    val results = input.toTable(tEnv, 'a, 'b, 'c)
+      .where('a < 5 || 'a > 17)
+      .select('c, 'b)
+      .groupBy('b)
+      .select('b, 'c.count)
+      .writeToSink(new CsvRetractTableSink(path))
+
+    env.execute()
+
+    val expected = Seq(
+      "true,1,1", "true,2,1", "false,2,1", "true,2,2", "true,3,1", "true,6,1", "false,6,1",
+      "true,6,2", "false,6,2", "true,6,3", "false,6,3", "true,6,4").mkString("\n")
+
+    TestBaseUtils.compareResultsByLinesInMemory(expected, path)
+  }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
index 6c75d53ca221c..497869d25fc15 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
@@ -49,12 +49,4 @@ object StreamITCase {
       }
     }
   }
-
-  final class StringSinkWithCRow extends RichSinkFunction[CRow]() {
-    def invoke(value: CRow) {
-      testResults.synchronized {
-        testResults += value.toString
-      }
-    }
-  }
 }

From 09183bb7aac5e9c4f9982d03aa429b59edb90c1a Mon Sep 17 00:00:00 2001
From: Fabian Hueske <fhueske@apache.org>
Date: Fri, 28 Apr 2017 01:59:57 +0200
Subject: [PATCH 161/253] [FLINK-6093] [table] Add stream TableSinks and
 DataStream conversion with support for retraction.

---
 .../connectors/kafka/KafkaTableSink.java      |   6 +-
 .../table/api/BatchTableEnvironment.scala     |  16 +-
 .../table/api/StreamTableEnvironment.scala    | 428 ++++++++++-----
 .../apache/flink/table/api/TableConfig.scala  |  17 -
 .../flink/table/api/TableEnvironment.scala    |  48 +-
 .../api/java/StreamTableEnvironment.scala     |  81 ++-
 .../api/scala/StreamTableEnvironment.scala    |  33 +-
 .../table/api/scala/TableConversions.scala    |  19 +
 .../org/apache/flink/table/api/table.scala    |   4 +-
 .../nodes/datastream/DataStreamCalc.scala     |   8 +-
 .../datastream/DataStreamGroupAggregate.scala |   2 +
 .../DataStreamGroupWindowAggregate.scala      |   4 +
 .../nodes/datastream/retractionTraits.scala   |  37 +-
 .../DataStreamRetractionRules.scala           |  16 +-
 .../runtime/CRowCorrelateFlatMapRunner.scala  |   2 +-
 .../table/runtime/CRowFlatMapRunner.scala     |   2 +-
 .../table/runtime/CRowInputMapRunner.scala    |   2 +-
 .../CRowInputTupleOutputMapRunner.scala       |  53 +-
 .../table/runtime/CRowOutputMapRunner.scala   |   2 +-
 .../runtime/CorrelateFlatMapRunner.scala      |   2 +-
 .../flink/table/runtime/FlatJoinRunner.scala  |   2 +-
 .../flink/table/runtime/FlatMapRunner.scala   |   2 +-
 .../table/runtime/MapJoinLeftRunner.scala     |   2 +-
 .../table/runtime/MapJoinRightRunner.scala    |   2 +-
 .../flink/table/runtime/MapRunner.scala       |   2 +-
 .../table/runtime/MapSideJoinRunner.scala     |   2 +-
 ...etSessionWindowAggregatePreProcessor.scala |   2 +-
 .../aggregate/GroupAggProcessFunction.scala   |  55 +-
 .../runtime/io/CRowValuesInputFormat.scala    |   2 +-
 .../table/runtime/io/ValuesInputFormat.scala  |   2 +-
 ...Sink.scala => AppendStreamTableSink.scala} |   8 +-
 .../flink/table/sinks/CsvTableSink.scala      | 100 +---
 .../table/sinks/RetractStreamTableSink.scala  |  55 ++
 .../flink/table/sinks/StreamRetractSink.scala |  35 --
 .../table/sinks/UpsertStreamTableSink.scala   |  79 +++
 .../flink/table/TableEnvironmentTest.scala    |   1 -
 .../api/scala/stream/RetractionITCase.scala   |  80 ++-
 .../api/scala/stream/TableSinkITCase.scala    |  33 +-
 .../api/scala/stream/sql/SqlITCase.scala      | 102 ++--
 .../table/GroupAggregationsITCase.scala       |  61 +--
 .../scala/stream/table/OverWindowTest.scala   |  18 +-
 .../api/scala/stream/utils/StreamITCase.scala |  35 +-
 .../plan/rules/RetractionRulesTest.scala      |   2 +-
 .../table/sinks/StreamTableSinksITCase.scala  | 511 ++++++++++++++++++
 .../table/utils/MockTableEnvironment.scala    |   5 -
 .../flink/table/utils/TableTestBase.scala     |   4 +-
 46 files changed, 1403 insertions(+), 581 deletions(-)
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/{StreamTableSink.scala => AppendStreamTableSink.scala} (79%)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/RetractStreamTableSink.scala
 delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala

diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSink.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSink.java
index 97f5fbae7be44..a8a2fd033a816 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSink.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSink.java
@@ -20,7 +20,7 @@
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.types.Row;
 import org.apache.flink.api.java.typeutils.RowTypeInfo;
-import org.apache.flink.table.sinks.StreamTableSink;
+import org.apache.flink.table.sinks.AppendStreamTableSink;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.connectors.kafka.partitioner.KafkaPartitioner;
 import org.apache.flink.streaming.util.serialization.SerializationSchema;
@@ -29,12 +29,12 @@
 import java.util.Properties;
 
 /**
- * A version-agnostic Kafka {@link StreamTableSink}.
+ * A version-agnostic Kafka {@link AppendStreamTableSink}.
  *
  * <p>The version-specific Kafka consumers need to extend this class and
  * override {@link #createKafkaProducer(String, Properties, SerializationSchema, KafkaPartitioner)}}.
  */
-public abstract class KafkaTableSink implements StreamTableSink<Row> {
+public abstract class KafkaTableSink implements AppendStreamTableSink<Row> {
 
 	protected final String topic;
 	protected final Properties properties;
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index c7bacfe19140c..2a3cedf0979a6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -36,7 +36,7 @@ import org.apache.flink.table.expressions.{Expression, RowtimeAttribute, TimeAtt
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.DataSetRel
 import org.apache.flink.table.plan.rules.FlinkRuleSets
-import org.apache.flink.table.plan.schema.{DataSetTable, TableSourceTable}
+import org.apache.flink.table.plan.schema.{DataSetTable, RowSchema, TableSourceTable}
 import org.apache.flink.table.runtime.MapRunner
 import org.apache.flink.table.sinks.{BatchTableSink, TableSink}
 import org.apache.flink.table.sources.{BatchTableSource, TableSource}
@@ -133,14 +133,14 @@ abstract class BatchTableEnvironment(
     * Creates a final converter that maps the internal row type to external type.
     *
     * @param physicalTypeInfo the input of the sink
-    * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
+    * @param schema the input schema with correct field names (esp. for POJO field mapping)
     * @param requestedTypeInfo the output type of the sink
     * @param functionName name of the map function. Must not be unique but has to be a
     *                     valid Java class identifier.
     */
-  override protected def getConversionMapper[IN, OUT](
+  protected def getConversionMapper[IN, OUT](
       physicalTypeInfo: TypeInformation[IN],
-      logicalRowType: RelDataType,
+      schema: RowSchema,
       requestedTypeInfo: TypeInformation[OUT],
       functionName: String):
     Option[MapFunction[IN, OUT]] = {
@@ -153,7 +153,7 @@ abstract class BatchTableEnvironment(
 
       val converterFunction = generateRowConverterFunction[OUT](
         physicalTypeInfo.asInstanceOf[TypeInformation[Row]],
-        logicalRowType,
+        schema,
         requestedTypeInfo,
         functionName
       )
@@ -334,7 +334,11 @@ abstract class BatchTableEnvironment(
       case node: DataSetRel =>
         val plan = node.translateToPlan(this)
         val conversion =
-          getConversionMapper(plan.getType, logicalType, tpe, "DataSetSinkConversion")
+          getConversionMapper(
+            plan.getType,
+            new RowSchema(logicalType),
+            tpe,
+            "DataSetSinkConversion")
         conversion match {
           case None => plan.asInstanceOf[DataSet[A]] // no conversion necessary
           case Some(mapFunction: MapFunction[Row, A]) =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index bd063051bf8b8..aef2b1b5a946c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -19,19 +19,23 @@
 package org.apache.flink.table.api
 
 import _root_.java.util.concurrent.atomic.AtomicInteger
+import _root_.java.lang.{Boolean => JBool}
 
 import org.apache.calcite.plan.RelOptUtil
 import org.apache.calcite.plan.hep.HepMatchOrder
-import org.apache.calcite.rel.RelNode
+import org.apache.calcite.rel.{RelNode, RelVisitor}
 import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode}
+import org.apache.calcite.sql.SqlKind
 import org.apache.calcite.sql2rel.RelDecorrelator
 import org.apache.calcite.tools.{RuleSet, RuleSets}
 import org.apache.flink.api.common.typeinfo.AtomicType
 import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.api.common.functions.MapFunction
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
-import org.apache.flink.api.java.typeutils.{GenericTypeInfo, TupleTypeInfo}
+import org.apache.flink.api.java.typeutils.TupleTypeInfo
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.apache.flink.table.calcite.RelTimeIndicatorConverter
@@ -39,12 +43,12 @@ import org.apache.flink.table.explain.PlanJsonParser
 import org.apache.flink.table.expressions.{Expression, ProctimeAttribute, RowtimeAttribute, UnresolvedFieldReference}
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.datastream.{DataStreamRel, UpdateAsRetractionTrait}
+import org.apache.flink.table.plan.nodes.datastream._
 import org.apache.flink.table.plan.rules.FlinkRuleSets
-import org.apache.flink.table.plan.schema.StreamTableSourceTable
-import org.apache.flink.table.plan.schema.DataStreamTable
-import org.apache.flink.table.runtime.{CRowInputMapRunner, CRowInputTupleOutputMapRunner}
+import org.apache.flink.table.plan.schema.{DataStreamTable, RowSchema, StreamTableSourceTable}
+import org.apache.flink.table.runtime.{CRowInputJavaTupleOutputMapRunner, CRowInputMapRunner, CRowInputScalaTupleOutputMapRunner}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.apache.flink.table.sinks.{StreamRetractSink, StreamTableSink, TableSink}
+import org.apache.flink.table.sinks.{AppendStreamTableSink, RetractStreamTableSink, TableSink, UpsertStreamTableSink}
 import org.apache.flink.table.sources.{StreamTableSource, TableSource}
 import org.apache.flink.table.typeutils.TypeCheckUtils
 import org.apache.flink.types.Row
@@ -127,97 +131,192 @@ abstract class StreamTableEnvironment(
   override private[flink] def writeToSink[T](table: Table, sink: TableSink[T]): Unit = {
 
     sink match {
-      case streamSink: StreamTableSink[T] =>
+
+      case retractSink: RetractStreamTableSink[_] =>
+        // retraction sink can always be used
         val outputType = sink.getOutputType
         // translate the Table into a DataStream and provide the type that the TableSink expects.
-        val result: DataStream[T] = translate(table)(outputType)
-        // Give the DataSet to the TableSink to emit it.
-        streamSink.emitDataStream(result)
-
-      case streamRetractSink: StreamRetractSink[T] =>
+        val result: DataStream[T] =
+          translate(table, updatesAsRetraction = true, withChangeFlag = true)(outputType)
+        // Give the DataStream to the TableSink to emit it.
+        retractSink.asInstanceOf[RetractStreamTableSink[Any]]
+          .emitDataStream(result.asInstanceOf[DataStream[JTuple2[JBool, Any]]])
+
+      case upsertSink: UpsertStreamTableSink[_] =>
+        // optimize plan
+        val optimizedPlan = optimize(table.getRelNode, updatesAsRetraction = false)
+        // check for append only table
+        val isAppendOnlyTable = isAppendOnly(optimizedPlan)
+        upsertSink.setIsAppendOnly(isAppendOnlyTable)
+        // extract unique key fields
+        val tableKeys: Option[Array[String]] = getUniqueKeyFields(optimizedPlan)
+        // check that we have keys if the table has changes (is not append-only)
+        tableKeys match {
+          case Some(keys) => upsertSink.setKeyFields(keys)
+          case None if isAppendOnlyTable => upsertSink.setKeyFields(null)
+          case None if !isAppendOnlyTable => throw new TableException(
+            "UpsertStreamTableSink requires that Table has a full primary keys if it is updated.")
+        }
+        val outputType = sink.getOutputType
+        // translate the Table into a DataStream and provide the type that the TableSink expects.
+        val result: DataStream[T] =
+          translate(optimizedPlan, table.getRelNode.getRowType, withChangeFlag = true)(outputType)
+        // Give the DataStream to the TableSink to emit it.
+        upsertSink.asInstanceOf[UpsertStreamTableSink[Any]]
+          .emitDataStream(result.asInstanceOf[DataStream[JTuple2[JBool, Any]]])
+
+      case appendSink: AppendStreamTableSink[_] =>
+        // optimize plan
+        val optimizedPlan = optimize(table.getRelNode, updatesAsRetraction = false)
+        // verify table is an insert-only (append-only) table
+        if (!isAppendOnly(optimizedPlan)) {
+          throw new TableException(
+            "AppendStreamTableSink requires that Table has only insert changes.")
+        }
         val outputType = sink.getOutputType
-        this.config.setNeedsUpdatesAsRetractionForSink(streamRetractSink.needsUpdatesAsRetraction)
         // translate the Table into a DataStream and provide the type that the TableSink expects.
-        val result: DataStream[JTuple2[Boolean, T]] = translate(table, true)(outputType)
-        // Give the DataSet to the TableSink to emit it.
-        streamRetractSink.emitDataStreamWithChange(result)
+        val result: DataStream[T] =
+          translate(optimizedPlan, table.getRelNode.getRowType, withChangeFlag = false)(outputType)
+        // Give the DataStream to the TableSink to emit it.
+        appendSink.asInstanceOf[AppendStreamTableSink[T]].emitDataStream(result)
+
       case _ =>
-        throw new TableException("StreamTableSink required to emit streaming Table")
+        throw new TableException("Stream Tables can only be emitted by AppendStreamTableSink, " +
+          "RetractStreamTableSink, or UpsertStreamTableSink.")
     }
   }
 
-
   /**
     * Creates a final converter that maps the internal row type to external type.
     *
     * @param physicalTypeInfo the input of the sink
-    * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
+    * @param schema the input schema with correct field names (esp. for POJO field mapping)
     * @param requestedTypeInfo the output type of the sink
     * @param functionName name of the map function. Must not be unique but has to be a
     *                     valid Java class identifier.
     */
-  override protected def getConversionMapper[IN, OUT](
+  protected def getConversionMapper[IN, OUT](
       physicalTypeInfo: TypeInformation[IN],
-      logicalRowType: RelDataType,
+      schema: RowSchema,
       requestedTypeInfo: TypeInformation[OUT],
       functionName: String):
-  Option[MapFunction[IN, OUT]] = {
+    MapFunction[IN, OUT] = {
 
-    if (requestedTypeInfo.getTypeClass == classOf[CRow]) {
-      // only used to explain table
-      None
-    } else if (requestedTypeInfo.getTypeClass == classOf[Row]) {
+    if (requestedTypeInfo.getTypeClass == classOf[Row]) {
       // CRow to Row, only needs to be unwrapped
-      Some(
-        new MapFunction[CRow, Row] {
-          override def map(value: CRow): Row = value.row
-        }.asInstanceOf[MapFunction[IN, OUT]]
-      )
+      new MapFunction[CRow, Row] {
+        override def map(value: CRow): Row = value.row
+      }.asInstanceOf[MapFunction[IN, OUT]]
     } else {
       // Some type that is neither CRow nor Row
       val converterFunction = generateRowConverterFunction[OUT](
         physicalTypeInfo.asInstanceOf[CRowTypeInfo].rowType,
-        logicalRowType,
+        schema,
         requestedTypeInfo,
         functionName
       )
 
-      Some(new CRowInputMapRunner[OUT](
+      new CRowInputMapRunner[OUT](
         converterFunction.name,
         converterFunction.code,
         converterFunction.returnType)
-        .asInstanceOf[MapFunction[IN, OUT]])
+        .asInstanceOf[MapFunction[IN, OUT]]
     }
   }
 
+  /** Validates that the plan produces only append changes. */
+  protected def isAppendOnly(plan: RelNode): Boolean = {
+    val appendOnlyValidator = new AppendOnlyValidator
+    appendOnlyValidator.go(plan)
+
+    appendOnlyValidator.isAppendOnly
+  }
+
+  /** Extracts the unique keys of the table produced by the plan. */
+  protected def getUniqueKeyFields(plan: RelNode): Option[Array[String]] = {
+    val keyExtractor = new UniqueKeyExtractor
+    keyExtractor.go(plan)
+    keyExtractor.keys
+  }
+
   /**
-    * Creates a final converter that maps the internal CRow type to external Tuple2 type.
+    * Creates a converter that maps the internal CRow type to Scala or Java Tuple2 with change flag.
     *
     * @param physicalTypeInfo the input of the sink
-    * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
-    * @param requestedTypeInfo the output type of the sink
+    * @param schema the input schema with correct field names (esp. for POJO field mapping)
+    * @param requestedTypeInfo the output type of the sink.
     * @param functionName name of the map function. Must not be unique but has to be a
     *                     valid Java class identifier.
     */
-  protected def getTupleConversionMapper[IN, OUT](
-      physicalTypeInfo: TypeInformation[IN],
-      logicalRowType: RelDataType,
-      requestedTypeInfo: TypeInformation[OUT],
-      functionName: String):
-  Option[MapFunction[IN, JTuple2[Boolean, OUT]]] = {
+  private def getConversionMapperWithChanges[OUT](
+    physicalTypeInfo: TypeInformation[CRow],
+    schema: RowSchema,
+    requestedTypeInfo: TypeInformation[OUT],
+    functionName: String):
+  MapFunction[CRow, OUT] = {
+
+    requestedTypeInfo match {
+
+      // Scala tuple
+      case t: CaseClassTypeInfo[_]
+        if t.getTypeClass == classOf[(_, _)] && t.getTypeAt(0) == Types.BOOLEAN =>
+
+        val reqType = t.getTypeAt(1).asInstanceOf[TypeInformation[Any]]
+        if (reqType.getTypeClass == classOf[Row]) {
+          // Requested type is Row. Just rewrap CRow in Tuple2
+          new MapFunction[CRow, (Boolean, Row)] {
+            override def map(cRow: CRow): (Boolean, Row) = {
+              (cRow.change, cRow.row)
+            }
+          }.asInstanceOf[MapFunction[CRow, OUT]]
+        } else {
+          // Use a map function to convert Row into requested type and wrap result in Tuple2
+          val converterFunction = generateRowConverterFunction(
+            physicalTypeInfo.asInstanceOf[CRowTypeInfo].rowType,
+            schema,
+            reqType,
+            functionName
+          )
+
+          new CRowInputScalaTupleOutputMapRunner(
+            converterFunction.name,
+            converterFunction.code,
+            requestedTypeInfo.asInstanceOf[TypeInformation[(Boolean, Any)]])
+            .asInstanceOf[MapFunction[CRow, OUT]]
 
-    val converterFunction = generateRowConverterFunction(
-      physicalTypeInfo.asInstanceOf[CRowTypeInfo].rowType,
-      logicalRowType,
-      requestedTypeInfo,
-      functionName
-    )
+        }
 
-    Some(new CRowInputTupleOutputMapRunner[OUT](
-      converterFunction.name,
-      converterFunction.code,
-      new TupleTypeInfo(BasicTypeInfo.BOOLEAN_TYPE_INFO, requestedTypeInfo))
-           .asInstanceOf[MapFunction[IN, JTuple2[Boolean, OUT]]])
+      // Java tuple
+      case t: TupleTypeInfo[_]
+        if t.getTypeClass == classOf[JTuple2[_, _]] && t.getTypeAt(0) == Types.BOOLEAN =>
+
+        val reqType = t.getTypeAt(1).asInstanceOf[TypeInformation[Any]]
+        if (reqType.getTypeClass == classOf[Row]) {
+          // Requested type is Row. Just rewrap CRow in Tuple2
+          new MapFunction[CRow, JTuple2[JBool, Row]] {
+            val outT = new JTuple2(true.asInstanceOf[JBool], null.asInstanceOf[Row])
+            override def map(cRow: CRow): JTuple2[JBool, Row] = {
+              outT.f0 = cRow.change
+              outT.f1 = cRow.row
+              outT
+            }
+          }.asInstanceOf[MapFunction[CRow, OUT]]
+        } else {
+          // Use a map function to convert Row into requested type and wrap result in Tuple2
+          val converterFunction = generateRowConverterFunction(
+            physicalTypeInfo.asInstanceOf[CRowTypeInfo].rowType,
+            schema,
+            reqType,
+            functionName
+          )
+
+          new CRowInputJavaTupleOutputMapRunner(
+            converterFunction.name,
+            converterFunction.code,
+            requestedTypeInfo.asInstanceOf[TypeInformation[JTuple2[JBool, Any]]])
+            .asInstanceOf[MapFunction[CRow, OUT]]
+        }
+    }
   }
 
   /**
@@ -380,9 +479,10 @@ abstract class StreamTableEnvironment(
     * Generates the optimized [[RelNode]] tree from the original relational node tree.
     *
     * @param relNode The root node of the relational expression tree.
+    * @param updatesAsRetraction True if the sink requests updates as retraction messages.
     * @return The optimized [[RelNode]] tree
     */
-  private[flink] def optimize(relNode: RelNode): RelNode = {
+  private[flink] def optimize(relNode: RelNode, updatesAsRetraction: Boolean): RelNode = {
 
     // 1. decorrelate
     val decorPlan = RelDecorrelator.decorrelateQuery(relNode)
@@ -410,7 +510,7 @@ abstract class StreamTableEnvironment(
     // 5. optimize the physical Flink plan
     val physicalOptRuleSet = getPhysicalOptRuleSet
     val physicalOutputProps = relNode.getTraitSet.replace(FlinkConventions.DATASTREAM).simplify()
-    var physicalPlan = if (physicalOptRuleSet.iterator().hasNext) {
+    val physicalPlan = if (physicalOptRuleSet.iterator().hasNext) {
       runVolcanoPlanner(physicalOptRuleSet, logicalPlan, physicalOutputProps)
     } else {
       logicalPlan
@@ -419,13 +519,18 @@ abstract class StreamTableEnvironment(
     // 6. decorate the optimized plan
     val decoRuleSet = getDecoRuleSet
     val decoratedPlan = if (decoRuleSet.iterator().hasNext) {
-
-      if (this.config.getNeedsUpdatesAsRetractionForSink) {
-        physicalPlan = physicalPlan.copy(
+      val planToDecorate = if (updatesAsRetraction) {
+        physicalPlan.copy(
           physicalPlan.getTraitSet.plus(new UpdateAsRetractionTrait(true)),
           physicalPlan.getInputs)
+      } else {
+        physicalPlan
       }
-      runHepPlanner(HepMatchOrder.BOTTOM_UP, decoRuleSet, physicalPlan, physicalPlan.getTraitSet)
+      runHepPlanner(
+        HepMatchOrder.BOTTOM_UP,
+        decoRuleSet,
+        planToDecorate,
+        planToDecorate.getTraitSet)
     } else {
       physicalPlan
     }
@@ -440,14 +545,17 @@ abstract class StreamTableEnvironment(
     * Table API calls and / or SQL queries and generating corresponding [[DataStream]] operators.
     *
     * @param table The root node of the relational expression tree.
+    * @param updatesAsRetraction Set to true to encode updates as retraction messages.
+    * @param withChangeFlag Set to true to emit records with change flags.
     * @param tpe The [[TypeInformation]] of the resulting [[DataStream]].
     * @tparam A The type of the resulting [[DataStream]].
     * @return The [[DataStream]] that corresponds to the translated [[Table]].
     */
-  protected def translate[A](table: Table)(implicit tpe: TypeInformation[A]): DataStream[A] = {
+  protected def translate[A](table: Table, updatesAsRetraction: Boolean, withChangeFlag: Boolean)
+      (implicit tpe: TypeInformation[A]): DataStream[A] = {
     val relNode = table.getRelNode
-    val dataStreamPlan = optimize(relNode)
-    translate(dataStreamPlan, relNode.getRowType)
+    val dataStreamPlan = optimize(relNode, updatesAsRetraction)
+    translate(dataStreamPlan, relNode.getRowType, withChangeFlag)
   }
 
   /**
@@ -456,87 +564,65 @@ abstract class StreamTableEnvironment(
     * @param logicalPlan The root node of the relational expression tree.
     * @param logicalType The row type of the result. Since the logicalPlan can lose the
     *                    field naming during optimization we pass the row type separately.
+    * @param withChangeFlag Set to true to emit records with change flags.
     * @param tpe         The [[TypeInformation]] of the resulting [[DataStream]].
     * @tparam A The type of the resulting [[DataStream]].
     * @return The [[DataStream]] that corresponds to the translated [[Table]].
     */
   protected def translate[A](
       logicalPlan: RelNode,
-      logicalType: RelDataType)
+      logicalType: RelDataType,
+      withChangeFlag: Boolean)
       (implicit tpe: TypeInformation[A]): DataStream[A] = {
 
-    TableEnvironment.validateType(tpe)
+    // if no change flags are requested, verify table is an insert-only (append-only) table.
+    if (!withChangeFlag && !isAppendOnly(logicalPlan)) {
+      throw new TableException(
+        "Table is not an append-only table. " +
+          "Output needs to handle update and delete changes.")
+    }
 
-    logicalPlan match {
-      case node: DataStreamRel =>
-        val plan = node.translateToPlan(this)
-        val conversion =
-          getConversionMapper(plan.getType, logicalType, tpe, "DataStreamSinkConversion")
-        conversion match {
-          case None => plan.asInstanceOf[DataStream[A]] // no conversion necessary
-          case Some(mapFunction: MapFunction[CRow, A]) =>
-            plan.map(mapFunction)
-              .returns(tpe)
-              .name(s"to: ${tpe.getTypeClass.getSimpleName}")
-              .asInstanceOf[DataStream[A]]
-        }
+    // get CRow plan
+    val plan: DataStream[CRow] = translateToCRow(logicalPlan)
 
-      case _ =>
-        throw TableException("Cannot generate DataStream due to an invalid logical plan. " +
-          "This is a bug and should not happen. Please file an issue.")
+    // convert CRow to output type
+    val conversion = if (withChangeFlag) {
+      getConversionMapperWithChanges(
+        plan.getType,
+        new RowSchema(logicalType),
+        tpe,
+        "DataStreamSinkConversion")
+    } else {
+      getConversionMapper(
+        plan.getType,
+        new RowSchema(logicalType),
+        tpe,
+        "DataStreamSinkConversion")
     }
-  }
 
-  /**
-    * Translates a [[Table]] into a [[DataStream]] with change information.
-    *
-    * The transformation involves optimizing the relational expression tree as defined by
-    * Table API calls and / or SQL queries and generating corresponding [[DataStream]] operators.
-    *
-    * @param table       The root node of the relational expression tree.
-    * @param wrapToTuple True, if want to output chang information
-    * @param tpe         The [[TypeInformation]] of the resulting [[DataStream]].
-    * @tparam A The type of the resulting [[DataStream]].
-    * @return The [[DataStream]] that corresponds to the translated [[Table]].
-    */
-  protected def translate[A](table: Table, wrapToTuple: Boolean)(implicit tpe: TypeInformation[A])
-  : DataStream[JTuple2[Boolean, A]] = {
-    val relNode = table.getRelNode
-    val dataStreamPlan = optimize(relNode)
-    translate(dataStreamPlan, relNode.getRowType, wrapToTuple)
+    val rootParallelism = plan.getParallelism
+
+    conversion match {
+      case mapFunction: MapFunction[CRow, A] =>
+        plan.map(mapFunction)
+          .returns(tpe)
+          .name(s"to: ${tpe.getTypeClass.getSimpleName}")
+          .setParallelism(rootParallelism)
+    }
   }
 
   /**
-    * Translates a logical [[RelNode]] into a [[DataStream]] with change information.
+    * Translates a logical [[RelNode]] plan into a [[DataStream]] of type [[CRow]].
     *
-    * @param logicalPlan The root node of the relational expression tree.
-    * @param logicalType The row type of the result. Since the logicalPlan can lose the
-    * @param wrapToTuple True, if want to output chang information
-    * @param tpe         The [[TypeInformation]] of the resulting [[DataStream]].
-    * @tparam A The type of the resulting [[DataStream]].
-    * @return The [[DataStream]] that corresponds to the translated [[Table]].
+    * @param logicalPlan The logical plan to translate.
+    * @return The [[DataStream]] of type [[CRow]].
     */
-  protected def translate[A](
-      logicalPlan: RelNode,
-      logicalType: RelDataType,
-      wrapToTuple: Boolean)(implicit tpe: TypeInformation[A]): DataStream[JTuple2[Boolean, A]] = {
-
-    TableEnvironment.validateType(tpe)
+  protected def translateToCRow(
+    logicalPlan: RelNode): DataStream[CRow] = {
 
     logicalPlan match {
       case node: DataStreamRel =>
-        val plan = node.translateToPlan(this)
-        val conversion =
-          getTupleConversionMapper(plan.getType, logicalType, tpe, "DataStreamSinkConversion")
-        conversion match {
-          case None => plan.asInstanceOf[DataStream[JTuple2[Boolean, A]]] // no conversion necessary
-          case Some(mapFunction: MapFunction[CRow, JTuple2[Boolean, A]]) =>
-            plan.map(mapFunction)
-              .returns(new TupleTypeInfo[JTuple2[Boolean, A]](BasicTypeInfo.BOOLEAN_TYPE_INFO, tpe))
-              .name(s"to: ${tpe.getTypeClass.getSimpleName}")
-              .asInstanceOf[DataStream[JTuple2[Boolean, A]]]
-        }
-
+        node.translateToPlan(this)
       case _ =>
         throw TableException("Cannot generate DataStream due to an invalid logical plan. " +
           "This is a bug and should not happen. Please file an issue.")
@@ -551,10 +637,8 @@ abstract class StreamTableEnvironment(
     */
   def explain(table: Table): String = {
     val ast = table.getRelNode
-    val optimizedPlan = optimize(ast)
-    val dataStream = translate[CRow](
-      optimizedPlan,
-      ast.getRowType)(new GenericTypeInfo(classOf[CRow]))
+    val optimizedPlan = optimize(ast, updatesAsRetraction = false)
+    val dataStream = translateToCRow(optimizedPlan)
 
     val env = dataStream.getExecutionEnvironment
     val jsonSqlPlan = env.getExecutionPlan
@@ -574,4 +658,90 @@ abstract class StreamTableEnvironment(
         s"$sqlPlan"
   }
 
+  private class AppendOnlyValidator extends RelVisitor {
+
+    var isAppendOnly = true
+
+    override def visit(node: RelNode, ordinal: Int, parent: RelNode): Unit = {
+      node match {
+        case s: DataStreamRel if s.producesUpdates =>
+          isAppendOnly = false
+        case _ =>
+          super.visit(node, ordinal, parent)
+      }
+    }
+  }
+
+  /** Identifies unique key fields in the output of a RelNode. */
+  private class UniqueKeyExtractor extends RelVisitor {
+
+    var keys: Option[Array[String]] = None
+
+    override def visit(node: RelNode, ordinal: Int, parent: RelNode): Unit = {
+      node match {
+        case c: DataStreamCalc =>
+          super.visit(node, ordinal, parent)
+          // check if input has keys
+          if (keys.isDefined) {
+            // track keys forward
+            val inNames = c.getInput.getRowType.getFieldNames
+            val inOutNames = c.getProgram.getNamedProjects.asScala
+              .map(p => {
+                c.getProgram.expandLocalRef(p.left) match {
+                    // output field is forwarded input field
+                  case i: RexInputRef => (i.getIndex, p.right)
+                    // output field is renamed input field
+                  case a: RexCall if a.getKind.equals(SqlKind.AS) =>
+                    a.getOperands.get(0) match {
+                      case ref: RexInputRef =>
+                        (ref.getIndex, p.right)
+                      case _ =>
+                        (-1, p.right)
+                    }
+                    // output field is not forwarded from input
+                  case _: RexNode => (-1, p.right)
+                }
+              })
+              // filter all non-forwarded fields
+              .filter(_._1 >= 0)
+              // resolve names of input fields
+              .map(io => (inNames.get(io._1), io._2))
+
+            // filter by input keys
+            val outKeys = inOutNames.filter(io => keys.get.contains(io._1)).map(_._2)
+            // check if all keys have been preserved
+            if (outKeys.nonEmpty && outKeys.length == keys.get.length) {
+              // all key have been preserved (but possibly renamed)
+              keys = Some(outKeys.toArray)
+            } else {
+              // some (or all) keys have been removed. Keys are no longer unique and removed
+              keys = None
+            }
+          }
+        case _: DataStreamOverAggregate =>
+          super.visit(node, ordinal, parent)
+          // keys are always forwarded by Over aggregate
+        case a: DataStreamGroupAggregate =>
+          // get grouping keys
+          val groupKeys = a.getRowType.getFieldNames.asScala.take(a.getGroupings.length)
+          keys = Some(groupKeys.toArray)
+        case w: DataStreamGroupWindowAggregate =>
+          // get grouping keys
+          val groupKeys =
+            w.getRowType.getFieldNames.asScala.take(w.getGroupings.length).toArray
+          // get window start and end time
+          val windowStartEnd = w.getWindowProperties.map(_.name)
+          // we have only a unique key if at least one window property is selected
+          if (windowStartEnd.nonEmpty) {
+            keys = Some(groupKeys ++ windowStartEnd)
+          }
+        case _: DataStreamRel =>
+          // anything else does not forward keys or might duplicate key, so we can stop
+          keys = None
+      }
+    }
+
+  }
+
 }
+
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala
index d2969786bd669..6448657c5111f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala
@@ -36,11 +36,6 @@ class TableConfig {
    */
   private var nullCheck: Boolean = true
 
-  /**
-    * Defines whether sink table requires that update and delete changes are sent with retraction
-    */
-  private var needsUpdatesAsRetractionForSink: Boolean = false
-
   /**
     * Defines the configuration of Calcite for Table API and SQL queries.
     */
@@ -71,18 +66,6 @@ class TableConfig {
     this.nullCheck = nullCheck
   }
 
-  /**
-    * Returns the need retraction property for table sink.
-    */
-  def getNeedsUpdatesAsRetractionForSink = needsUpdatesAsRetractionForSink
-
-  /**
-    * Set the need retraction property for table sink.
-    */
-  def setNeedsUpdatesAsRetractionForSink(needsUpdatesAsRetraction: Boolean ): Unit = {
-    this.needsUpdatesAsRetractionForSink = needsUpdatesAsRetraction
-  }
-
   /**
     * Returns the current configuration of Calcite for Table API and SQL queries.
     */
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 5b752ab1612a5..bb0de3e529037 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -51,15 +51,14 @@ import org.apache.flink.table.catalog.{ExternalCatalog, ExternalCatalogSchema}
 import org.apache.flink.table.codegen.{CodeGenerator, ExpressionReducer, GeneratedFunction}
 import org.apache.flink.table.expressions.{Alias, Expression, UnresolvedFieldReference}
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils._
-import org.apache.flink.table.functions.{ScalarFunction, TableFunction, AggregateFunction}
+import org.apache.flink.table.functions.AggregateFunction
 import org.apache.flink.table.expressions._
 import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{checkForInstantiation, checkNotSingleton, createScalarSqlFunction, createTableSqlFunctions}
 import org.apache.flink.table.functions.{ScalarFunction, TableFunction}
 import org.apache.flink.table.plan.cost.DataSetCostFactory
 import org.apache.flink.table.plan.logical.{CatalogNode, LogicalRelNode}
 import org.apache.flink.table.plan.rules.FlinkRuleSets
-import org.apache.flink.table.plan.schema.RelTable
-import org.apache.flink.table.runtime.types.CRowTypeInfo
+import org.apache.flink.table.plan.schema.{RelTable, RowSchema}
 import org.apache.flink.table.sinks.TableSink
 import org.apache.flink.table.sources.{DefinedFieldNames, TableSource}
 import org.apache.flink.table.validate.FunctionCatalog
@@ -620,7 +619,7 @@ abstract class TableEnvironment(val config: TableConfig) {
         throw new TableException(
           "An input of GenericTypeInfo<Row> cannot be converted to Table. " +
             "Please specify the type of the input with a RowTypeInfo.")
-      case a: AtomicType[A] =>
+      case a: AtomicType[_] =>
         exprs.zipWithIndex flatMap {
           case (UnresolvedFieldReference(name), idx) =>
             if (idx > 0) {
@@ -691,53 +690,32 @@ abstract class TableEnvironment(val config: TableConfig) {
     (fieldNames.toArray, fieldIndexes.toArray)
   }
 
-  /**
-    * Creates a final converter that maps the internal row type to external type.
-    *
-    * @param physicalTypeInfo the input of the sink
-    * @param logicalRowType the logical type with correct field names (esp. for POJO field mapping)
-    * @param requestedTypeInfo the output type of the sink
-    * @param functionName name of the map function. Must not be unique but has to be a
-    *                     valid Java class identifier.
-    */
-  protected def getConversionMapper[IN, OUT](
-      physicalTypeInfo: TypeInformation[IN],
-      logicalRowType: RelDataType,
-      requestedTypeInfo: TypeInformation[OUT],
-      functionName: String):
-    Option[MapFunction[IN, OUT]]
-
   protected def generateRowConverterFunction[OUT](
       inputTypeInfo: TypeInformation[Row],
-      logicalRowType: RelDataType,
+      schema: RowSchema,
       requestedTypeInfo: TypeInformation[OUT],
       functionName: String):
     GeneratedFunction[MapFunction[Row, OUT], OUT] = {
 
     // validate that at least the field types of physical and logical type match
     // we do that here to make sure that plan translation was correct
-    val logicalRowTypeInfo = FlinkTypeFactory.toInternalRowTypeInfo(logicalRowType)
-    if (logicalRowTypeInfo != inputTypeInfo) {
+    if (schema.physicalTypeInfo != inputTypeInfo) {
       throw TableException("The field types of physical and logical row types do not match." +
         "This is a bug and should not happen. Please file an issue.")
     }
 
-    // convert to type information
-    val logicalFieldTypes = logicalRowType.getFieldList.asScala
-      .map(t => FlinkTypeFactory.toTypeInfo(t.getType))
-
-    // field names
-    val logicalFieldNames = logicalRowType.getFieldNames.asScala
+    val fieldTypes = schema.physicalFieldTypeInfo
+    val fieldNames = schema.physicalFieldNames
 
     // validate requested type
-    if (requestedTypeInfo.getArity != logicalFieldTypes.length) {
+    if (requestedTypeInfo.getArity != fieldTypes.length) {
       throw new TableException("Arity of result does not match requested type.")
     }
 
     requestedTypeInfo match {
       // POJO type requested
       case pt: PojoTypeInfo[_] =>
-        logicalFieldNames.zip(logicalFieldTypes) foreach {
+        fieldNames.zip(fieldTypes) foreach {
           case (fName, fType) =>
             val pojoIdx = pt.getFieldIndex(fName)
             if (pojoIdx < 0) {
@@ -752,7 +730,7 @@ abstract class TableEnvironment(val config: TableConfig) {
 
       // Tuple/Case class/Row type requested
       case tt: TupleTypeInfoBase[_] =>
-        logicalFieldTypes.zipWithIndex foreach {
+        fieldTypes.zipWithIndex foreach {
           case (fieldTypeInfo, i) =>
             val requestedTypeInfo = tt.getTypeAt(i)
             if (fieldTypeInfo != requestedTypeInfo) {
@@ -763,11 +741,11 @@ abstract class TableEnvironment(val config: TableConfig) {
 
       // Atomic type requested
       case at: AtomicType[_] =>
-        if (logicalFieldTypes.size != 1) {
+        if (fieldTypes.size != 1) {
           throw new TableException(s"Requested result type is an atomic type but " +
             s"result has more or less than a single field.")
         }
-        val fieldTypeInfo = logicalFieldTypes.head
+        val fieldTypeInfo = fieldTypes.head
         if (fieldTypeInfo != at) {
           throw new TableException(s"Result field does not match requested type. " +
             s"Requested: $at; Actual: $fieldTypeInfo")
@@ -787,7 +765,7 @@ abstract class TableEnvironment(val config: TableConfig) {
 
     val conversion = generator.generateConverterResultExpression(
       requestedTypeInfo,
-      logicalFieldNames)
+      fieldNames)
 
     val body =
       s"""
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
index a649584ae9bef..a70bcca79c36f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
@@ -18,12 +18,14 @@
 package org.apache.flink.table.api.java
 
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.java.typeutils.{TupleTypeInfo, TypeExtractor}
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.table.api._
 import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
 import org.apache.flink.table.expressions.ExpressionParser
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+import _root_.java.lang.{Boolean => JBool}
 
 /**
   * The [[TableEnvironment]] for a Java [[StreamExecutionEnvironment]].
@@ -132,7 +134,10 @@ class StreamTableEnvironment(
   }
 
   /**
-    * Converts the given [[Table]] into a [[DataStream]] of a specified type.
+    * Converts the given [[Table]] into an append [[DataStream]] of a specified type.
+    *
+    * The [[Table]] must only have insert (append) changes. If the [[Table]] is also modified
+    * by update or delete changes, the conversion will fail.
     *
     * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
     * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
@@ -145,11 +150,16 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T](table: Table, clazz: Class[T]): DataStream[T] = {
-    translate[T](table)(TypeExtractor.createTypeInfo(clazz))
+    val typeInfo = TypeExtractor.createTypeInfo(clazz)
+    TableEnvironment.validateType(typeInfo)
+    translate[T](table, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
   }
 
   /**
-    * Converts the given [[Table]] into a [[DataStream]] of a specified type.
+    * Converts the given [[Table]] into an append [[DataStream]] of a specified type.
+    *
+    * The [[Table]] must only have insert (append) changes. If the [[Table]] is also modified
+    * by update or delete changes, the conversion will fail.
     *
     * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
     * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
@@ -162,7 +172,68 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T](table: Table, typeInfo: TypeInformation[T]): DataStream[T] = {
-    translate[T](table)(typeInfo)
+    TableEnvironment.validateType(typeInfo)
+    translate[T](table, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
+  }
+
+  /**
+    * Converts the given [[Table]] into a [[DataStream]] of add and retract messages.
+    * The message will be encoded as [[JTuple2]]. The first field is a [[JBool]] flag,
+    * the second field holds the record of the specified type [[T]].
+    *
+    * A true [[JBool]] flag indicates an add message, a false flag indicates a retract message.
+    *
+    * The fields of the [[Table]] are mapped to the requested type as follows:
+    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+    * types: Fields are mapped by position, field types must match.
+    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+    *
+    * @param table The [[Table]] to convert.
+    * @param clazz The class of the requested record type.
+    * @tparam T The type of the requested record type.
+    * @return The converted [[DataStream]].
+    */
+  def toRetractStream[T](table: Table, clazz: Class[T]):
+    DataStream[JTuple2[JBool, T]] = {
+
+    val typeInfo = TypeExtractor.createTypeInfo(clazz)
+    TableEnvironment.validateType(typeInfo)
+    val resultType = new TupleTypeInfo[JTuple2[JBool, T]](Types.BOOLEAN, typeInfo)
+    translate[JTuple2[JBool, T]](
+      table,
+      updatesAsRetraction = true,
+      withChangeFlag = true)(resultType)
+  }
+
+  /**
+    * Converts the given [[Table]] into a [[DataStream]] of add and retract messages.
+    * The message will be encoded as [[JTuple2]]. The first field is a [[JBool]] flag,
+    * the second field holds the record of the specified type [[T]].
+    *
+    * A true [[JBool]] flag indicates an add message, a false flag indicates a retract message.
+    *
+    * The fields of the [[Table]] are mapped to the requested type as follows:
+    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+    * types: Fields are mapped by position, field types must match.
+    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+    *
+    * @param table The [[Table]] to convert.
+    * @param typeInfo The [[TypeInformation]] of the requested record type.
+    * @tparam T The type of the requested record type.
+    * @return The converted [[DataStream]].
+    */
+  def toRetractStream[T](table: Table, typeInfo: TypeInformation[T]):
+    DataStream[JTuple2[JBool, T]] = {
+
+    TableEnvironment.validateType(typeInfo)
+    val resultTypeInfo = new TupleTypeInfo[JTuple2[JBool, T]](
+      Types.BOOLEAN,
+      typeInfo
+    )
+    translate[JTuple2[JBool, T]](
+      table,
+      updatesAsRetraction = true,
+      withChangeFlag = true)(resultTypeInfo)
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
index 0552d7c64fbb7..e5ad6c23e35c5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
@@ -17,10 +17,11 @@
  */
 package org.apache.flink.table.api.scala
 
+import org.apache.flink.api.scala._
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.table.api.{TableEnvironment, Table, TableConfig}
-import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
+import org.apache.flink.table.api.{Table, TableConfig, TableEnvironment}
 import org.apache.flink.table.expressions.Expression
+import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
 import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
 import org.apache.flink.streaming.api.scala.asScalaStream
 
@@ -127,11 +128,14 @@ class StreamTableEnvironment(
   }
 
   /**
-    * Converts the given [[Table]] into a [[DataStream]] of a specified type.
+    * Converts the given [[Table]] into an append [[DataStream]] of a specified type.
+    *
+    * The [[Table]] must only have insert (append) changes. If the [[Table]] is also modified
+    * by update or delete changes, the conversion will fail.
     *
     * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
-    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
-    * types: Fields are mapped by position, field types must match.
+    * - [[org.apache.flink.types.Row]] and Scala Tuple types: Fields are mapped by position, field
+    * types must match.
     * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
     *
     * @param table The [[Table]] to convert.
@@ -139,7 +143,24 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T: TypeInformation](table: Table): DataStream[T] = {
-    asScalaStream(translate(table))
+    val returnType = createTypeInformation[T]
+    asScalaStream(translate(table, updatesAsRetraction = false, withChangeFlag = false)(returnType))
+  }
+
+/**
+  * Converts the given [[Table]] into a [[DataStream]] of add and retract messages.
+  * The message will be encoded as [[Tuple2]]. The first field is a [[Boolean]] flag,
+  * the second field holds the record of the specified type [[T]].
+  *
+  * A true [[Boolean]] flag indicates an add message, a false flag indicates a retract message.
+  *
+  * @param table The [[Table]] to convert.
+  * @tparam T The type of the requested data type.
+  * @return The converted [[DataStream]].
+  */
+  def toRetractStream[T: TypeInformation](table: Table): DataStream[(Boolean, T)] = {
+    val returnType = createTypeInformation[(Boolean, T)]
+    asScalaStream(translate(table, updatesAsRetraction = true, withChangeFlag = true)(returnType))
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
index 2a0d57131acde..5efff62599185 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
@@ -57,5 +57,24 @@ class TableConversions(table: Table) {
     }
   }
 
+  /** Converts the [[Table]] to a [[DataStream]] of add and retract messages.
+    * The message will be encoded as [[Tuple2]]. The first field is a [[Boolean]] flag,
+    * the second field holds the record of the specified type [[T]].
+    *
+    * A true [[Boolean]] flag indicates an add message, a false flag indicates a retract message.
+    *
+    */
+  def toRetractStream[T: TypeInformation]: DataStream[(Boolean, T)] = {
+
+    table.tableEnv match {
+      case tEnv: ScalaStreamTableEnv =>
+        tEnv.toRetractStream(table)
+      case _ =>
+        throw new TableException(
+          "Only tables that originate from Scala DataStreams " +
+            "can be converted to Scala DataStreams.")
+    }
+  }
+
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index dd8265b6f01c5..310a75f3a98a3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -755,7 +755,9 @@ class Table(
     *
     * A batch [[Table]] can only be written to a
     * [[org.apache.flink.table.sinks.BatchTableSink]], a streaming [[Table]] requires a
-    * [[org.apache.flink.table.sinks.StreamTableSink]].
+    * [[org.apache.flink.table.sinks.AppendStreamTableSink]], a
+    * [[org.apache.flink.table.sinks.RetractStreamTableSink]], or an
+    * [[org.apache.flink.table.sinks.UpsertStreamTableSink]].
     *
     * @param sink The [[TableSink]] to which the [[Table]] is written.
     * @tparam T The data type that the [[TableSink]] expects.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
index 59f723acb27fc..ce0f966e9c179 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
@@ -100,11 +100,17 @@ class DataStreamCalc(
       calcProgram,
       config)
 
+    val inputParallelism = inputDataStream.getParallelism
+
     val mapFunc = new CRowFlatMapRunner(
       genFunction.name,
       genFunction.code,
       CRowTypeInfo(schema.physicalTypeInfo))
 
-    inputDataStream.flatMap(mapFunc).name(calcOpName(calcProgram, getExpressionString))
+    inputDataStream
+      .flatMap(mapFunc)
+      .name(calcOpName(calcProgram, getExpressionString))
+      // keep parallelism to ensure order of accumulate and retract messages
+      .setParallelism(inputParallelism)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index 056cda9d24e94..18f1fc89b76ed 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -67,6 +67,8 @@ class DataStreamGroupAggregate(
 
   override def consumesRetractions = true
 
+  def getGroupings: Array[Int] = groupings
+
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamGroupAggregate(
       cluster,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index f61828b628f66..1be1896a96fdf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -59,6 +59,10 @@ class DataStreamGroupWindowAggregate(
 
   override def consumesRetractions = true
 
+  def getGroupings: Array[Int] = grouping
+
+  def getWindowProperties: Seq[NamedWindowProperty] = namedProperties
+
   override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
     new DataStreamGroupWindowAggregate(
       window,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala
index c3b43ba69f1f3..173b7d32112e1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/retractionTraits.scala
@@ -82,19 +82,38 @@ object AccModeTrait {
 }
 
 /**
-  * The AccMode indicates which kinds of messages a [[org.apache.calcite.rel.RelNode]] might
-  * produce.
-  * In [[AccMode.Acc]] the node only emit accumulate messages.
-  * In [[AccMode.AccRetract]], the node produces accumulate messages for insert changes,
-  * retraction messages for delete changes, and accumulate and retraction messages
-  * for update changes.
+  * The [[AccMode]] determines how insert, update, and delete changes of tables are encoded
+  * by the messeages that an operator emits.
   */
 object AccMode extends Enumeration {
   type AccMode = Value
 
-  val Acc        = Value // Operator produces only accumulate (insert) messages
-  val AccRetract = Value // Operator produces accumulate (insert, update) and
-                         //   retraction (delete, update) messages
+  /**
+    * An operator in [[Acc]] mode emits change messages as
+    * [[org.apache.flink.table.runtime.types.CRow]] which encode a pair of (Boolean, Row).
+    *
+    * An operator in [[Acc]] mode may only produce update and delete messages, if the table has
+    * a unique key and all key attributes are contained in the Row.
+    *
+    * Changes are encoded as follows:
+    * - insert: (true, NewRow)
+    * - update: (true, NewRow) // the Row includes the full unique key to identify the row to update
+    * - delete: (false, OldRow) // the Row includes the full unique key to idenify the row to delete
+    *
+    */
+  val Acc = Value
+
+  /**
+    * * An operator in [[AccRetract]] mode emits change messages as
+    * [[org.apache.flink.table.runtime.types.CRow]] which encode a pair of (Boolean, Row).
+    *
+    * Changes are encoded as follows:
+    * - insert: (true, NewRow)
+    * - update: (false, OldRow), (true, NewRow) // updates are encoded in two messages!
+    * - delete: (false, OldRow)
+    *
+    */
+  val AccRetract = Value
 }
 
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
index 97c0dbbc23e0d..f0b725db748b0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamRetractionRules.scala
@@ -102,17 +102,17 @@ object DataStreamRetractionRules {
       val rel = call.rel(0).asInstanceOf[DataStreamRel]
       val traits = rel.getTraitSet
 
-      val traitsWithUpdateAsRetrac =
+      val traitsWithUpdateAsRetraction =
         if (null == traits.getTrait(UpdateAsRetractionTraitDef.INSTANCE)) {
         traits.plus(UpdateAsRetractionTrait.DEFAULT)
       } else {
         traits
       }
       val traitsWithAccMode =
-        if (null == traitsWithUpdateAsRetrac.getTrait(AccModeTraitDef.INSTANCE)) {
-          traitsWithUpdateAsRetrac.plus(AccModeTrait.DEFAULT)
+        if (null == traitsWithUpdateAsRetraction.getTrait(AccModeTraitDef.INSTANCE)) {
+          traitsWithUpdateAsRetraction.plus(AccModeTrait.DEFAULT)
       } else {
-        traitsWithUpdateAsRetrac
+        traitsWithUpdateAsRetraction
       }
 
       if (traits != traitsWithAccMode) {
@@ -122,8 +122,8 @@ object DataStreamRetractionRules {
   }
 
   /**
-    * Rule that annotates all [[DataStreamRel]] nodes that need to sent out update and delete
-    * changes as retraction messages.
+    * Rule that annotates all [[DataStreamRel]] nodes that need to sent out update changes with
+    * retraction messages.
     */
   class SetUpdatesAsRetractionRule extends RelOptRule(
     operand(
@@ -131,7 +131,7 @@ object DataStreamRetractionRules {
     "SetUpdatesAsRetractionRule") {
 
     /**
-      * Checks if a [[RelNode]] requires that update and delete changes are sent with retraction
+      * Checks if a [[RelNode]] requires that update changes are sent with retraction
       * messages.
       */
     def needsUpdatesAsRetraction(node: RelNode): Boolean = {
@@ -142,7 +142,7 @@ object DataStreamRetractionRules {
     }
 
     /**
-      * Annotates a [[RelNode]] to send out update and delete changes as retraction messages.
+      * Annotates a [[RelNode]] to send out update changes with retraction messages.
       */
     def setUpdatesAsRetraction(relNode: RelNode): RelNode = {
       val traitSet = relNode.getTraitSet
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
index 66e51b1d4b06e..ff3821afde318 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
@@ -37,7 +37,7 @@ class CRowCorrelateFlatMapRunner(
     flatMapCode: String,
     collectorName: String,
     collectorCode: String,
-    @transient returnType: TypeInformation[CRow])
+    @transient var returnType: TypeInformation[CRow])
   extends RichFlatMapFunction[CRow, CRow]
   with ResultTypeQueryable[CRow]
   with Compiler[Any] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
index 9a4650b518ad9..9701cb9b37813 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
@@ -35,7 +35,7 @@ import org.slf4j.LoggerFactory
 class CRowFlatMapRunner(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[CRow])
+    @transient var returnType: TypeInformation[CRow])
   extends RichFlatMapFunction[CRow, CRow]
   with ResultTypeQueryable[CRow]
   with Compiler[FlatMapFunction[Row, Row]] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala
index 8e95c935afe43..109c6e1ebe35c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputMapRunner.scala
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory
 class CRowInputMapRunner[OUT](
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT])
+    @transient var returnType: TypeInformation[OUT])
   extends RichMapFunction[CRow, OUT]
   with ResultTypeQueryable[OUT]
   with Compiler[MapFunction[Row, OUT]] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala
index 54bbf7ee3127f..7c964371827f9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowInputTupleOutputMapRunner.scala
@@ -18,6 +18,8 @@
 
 package org.apache.flink.table.runtime
 
+import java.lang.{Boolean => JBool}
+
 import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
@@ -28,36 +30,63 @@ import org.apache.flink.types.Row
 import org.slf4j.LoggerFactory
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 
-
 /**
-  * Convert [[CRow]] to a [[Tuple2]]
+  * Convert [[CRow]] to a [[JTuple2]]
   */
-class CRowInputTupleOutputMapRunner[OUT](
+class CRowInputJavaTupleOutputMapRunner(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[JTuple2[Boolean, OUT]])
-  extends RichMapFunction[CRow, JTuple2[Boolean, OUT]]
-          with ResultTypeQueryable[JTuple2[Boolean, OUT]]
-          with Compiler[MapFunction[Row, OUT]] {
+    @transient var returnType: TypeInformation[JTuple2[JBool, Any]])
+  extends RichMapFunction[CRow, Any]
+          with ResultTypeQueryable[JTuple2[JBool, Any]]
+          with Compiler[MapFunction[Row, Any]] {
 
   val LOG = LoggerFactory.getLogger(this.getClass)
 
-  private var function: MapFunction[Row, OUT] = _
-  private var tupleWrapper: JTuple2[Boolean, OUT] = _
+  private var function: MapFunction[Row, Any] = _
+  private var tupleWrapper: JTuple2[JBool, Any] = _
 
   override def open(parameters: Configuration): Unit = {
     LOG.debug(s"Compiling MapFunction: $name \n\n Code:\n$code")
     val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
     LOG.debug("Instantiating MapFunction.")
     function = clazz.newInstance()
-    tupleWrapper = new JTuple2[Boolean, OUT]()
+    tupleWrapper = new JTuple2[JBool, Any]()
   }
 
-  override def map(in: CRow): JTuple2[Boolean, OUT] = {
+  override def map(in: CRow): JTuple2[JBool, Any] = {
     tupleWrapper.f0 = in.change
     tupleWrapper.f1 = function.map(in.row)
     tupleWrapper
   }
 
-  override def getProducedType: TypeInformation[JTuple2[Boolean, OUT]] = returnType
+  override def getProducedType: TypeInformation[JTuple2[JBool, Any]] = returnType
+}
+
+/**
+  * Convert [[CRow]] to a [[Tuple2]]
+  */
+class CRowInputScalaTupleOutputMapRunner(
+  name: String,
+  code: String,
+  @transient var returnType: TypeInformation[(Boolean, Any)])
+  extends RichMapFunction[CRow, (Boolean, Any)]
+    with ResultTypeQueryable[(Boolean, Any)]
+    with Compiler[MapFunction[Row, Any]] {
+
+  val LOG = LoggerFactory.getLogger(this.getClass)
+
+  private var function: MapFunction[Row, Any] = _
+
+  override def open(parameters: Configuration): Unit = {
+    LOG.debug(s"Compiling MapFunction: $name \n\n Code:\n$code")
+    val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
+    LOG.debug("Instantiating MapFunction.")
+    function = clazz.newInstance()
+  }
+
+  override def map(in: CRow): (Boolean, Any) =
+    (in.change, function.map(in.row))
+
+  override def getProducedType: TypeInformation[(Boolean, Any)] = returnType
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala
index 966dea904e112..cb8f69556e351 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputMapRunner.scala
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory
 class CRowOutputMapRunner(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[CRow])
+    @transient var returnType: TypeInformation[CRow])
   extends RichMapFunction[Any, CRow]
   with ResultTypeQueryable[CRow]
   with Compiler[MapFunction[Any, Row]] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala
index a0415e1ef7d00..478b6b64cfcad 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala
@@ -32,7 +32,7 @@ class CorrelateFlatMapRunner[IN, OUT](
     flatMapCode: String,
     collectorName: String,
     collectorCode: String,
-    @transient returnType: TypeInformation[OUT])
+    @transient var returnType: TypeInformation[OUT])
   extends RichFlatMapFunction[IN, OUT]
   with ResultTypeQueryable[OUT]
   with Compiler[Any] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala
index 715848d026893..67acc0b10ee1d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory
 class FlatJoinRunner[IN1, IN2, OUT](
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT])
+    @transient var returnType: TypeInformation[OUT])
   extends RichFlatJoinFunction[IN1, IN2, OUT]
   with ResultTypeQueryable[OUT]
   with Compiler[FlatJoinFunction[IN1, IN2, OUT]] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
index 2e37baf0a6f3e..938da59ea4aa4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
@@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory
 class FlatMapRunner(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[Row])
+    @transient var returnType: TypeInformation[Row])
   extends RichFlatMapFunction[Row, Row]
   with ResultTypeQueryable[Row]
   with Compiler[FlatMapFunction[Row, Row]] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala
index 644e8553c8e2a..5f3dbb4cbaf98 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala
@@ -24,7 +24,7 @@ import org.apache.flink.util.Collector
 class MapJoinLeftRunner[IN1, IN2, OUT](
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT],
+    returnType: TypeInformation[OUT],
     broadcastSetName: String)
   extends MapSideJoinRunner[IN1, IN2, IN2, IN1, OUT](name, code, returnType, broadcastSetName) {
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala
index eee38d19b9ec2..e2d9331187623 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala
@@ -24,7 +24,7 @@ import org.apache.flink.util.Collector
 class MapJoinRightRunner[IN1, IN2, OUT](
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT],
+    returnType: TypeInformation[OUT],
     broadcastSetName: String)
   extends MapSideJoinRunner[IN1, IN2, IN1, IN2, OUT](name, code, returnType, broadcastSetName) {
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
index 32562c7e3ec42..14eeecfb451b5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
@@ -28,7 +28,7 @@ import org.slf4j.LoggerFactory
 class MapRunner[IN, OUT](
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT])
+    @transient var returnType: TypeInformation[OUT])
   extends RichMapFunction[IN, OUT]
   with ResultTypeQueryable[OUT]
   with Compiler[MapFunction[IN, OUT]] {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala
index 090e1849f22e7..00b7b8eee0ce1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala
@@ -28,7 +28,7 @@ import org.slf4j.LoggerFactory
 abstract class MapSideJoinRunner[IN1, IN2, SINGLE_IN, MULTI_IN, OUT](
     name: String,
     code: String,
-    @transient returnType: TypeInformation[OUT],
+    @transient var returnType: TypeInformation[OUT],
     broadcastSetName: String)
   extends RichFlatMapFunction[MULTI_IN, OUT]
     with ResultTypeQueryable[OUT]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
index 22a2682ba3e8d..9bcac3031d52c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
@@ -41,7 +41,7 @@ class DataSetSessionWindowAggregatePreProcessor(
     genAggregations: GeneratedAggregationsFunction,
     keysAndAggregatesArity: Int,
     gap: Long,
-    @transient intermediateRowType: TypeInformation[Row])
+    @transient var intermediateRowType: TypeInformation[Row])
   extends AbstractRichFunction
   with MapPartitionFunction[Row,Row]
   with GroupCombineFunction[Row,Row]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index 745f24d8ca858..6ee37e62f1655 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -17,6 +17,8 @@
  */
 package org.apache.flink.table.runtime.aggregate
 
+import java.lang.{Long => JLong}
+
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.types.Row
@@ -24,6 +26,7 @@ import org.apache.flink.util.Collector
 import org.apache.flink.api.common.state.ValueStateDescriptor
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.api.common.state.ValueState
+import org.apache.flink.table.api.Types
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.slf4j.LoggerFactory
 import org.apache.flink.table.runtime.types.CRow
@@ -47,7 +50,10 @@ class GroupAggProcessFunction(
   private var newRow: CRow = _
   private var prevRow: CRow = _
   private var firstRow: Boolean = _
+  // stores the accumulators
   private var state: ValueState[Row] = _
+  // counts the number of added and retracted input records
+  private var cntState: ValueState[JLong] = _
 
   override def open(config: Configuration) {
     LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
@@ -65,6 +71,9 @@ class GroupAggProcessFunction(
     val stateDescriptor: ValueStateDescriptor[Row] =
       new ValueStateDescriptor[Row]("GroupAggregateState", aggregationStateType)
     state = getRuntimeContext.getState(stateDescriptor)
+    val inputCntDescriptor: ValueStateDescriptor[JLong] =
+      new ValueStateDescriptor[JLong]("GroupAggregateInputCounter", Types.LONG)
+    cntState = getRuntimeContext.getState(inputCntDescriptor)
   }
 
   override def processElement(
@@ -74,11 +83,14 @@ class GroupAggProcessFunction(
 
     val input = inputC.row
 
-    // get accumulators
+    // get accumulators and input counter
     var accumulators = state.value()
+    var inputCnt = cntState.value()
+
     if (null == accumulators) {
       firstRow = true
       accumulators = function.createAccumulators()
+      inputCnt = 0L
     } else {
       firstRow = false
     }
@@ -92,29 +104,44 @@ class GroupAggProcessFunction(
 
     // update aggregate result and set to the newRow
     if (inputC.change) {
+      inputCnt += 1
       // accumulate input
       function.accumulate(accumulators, input)
       function.setAggregationResults(accumulators, newRow.row)
     } else {
+      inputCnt -= 1
       // retract input
       function.retract(accumulators, input)
       function.setAggregationResults(accumulators, newRow.row)
     }
 
-    // update accumulators
-    state.update(accumulators)
-
-    // if previousRow is not null, do retraction process
-    if (generateRetraction && !firstRow) {
-      if (prevRow.row.equals(newRow.row)) {
-        // ignore same newRow
-        return
-      } else {
-        // retract previous row
-        out.collect(prevRow)
+    if (inputCnt != 0) {
+      // we aggregated at least one record for this key
+
+      // update the state
+      state.update(accumulators)
+      cntState.update(inputCnt)
+
+      // if this was not the first row and we have to emit retractions
+      if (generateRetraction && !firstRow) {
+        if (prevRow.row.equals(newRow.row)) {
+          // newRow is the same as before. Do not emit retraction and acc messages
+          return
+        } else {
+          // retract previous result
+          out.collect(prevRow)
+        }
       }
-    }
+      // emit the new result
+      out.collect(newRow)
 
-    out.collect(newRow)
+    } else {
+      // we retracted the last record for this key
+      // sent out a delete message
+      out.collect(prevRow)
+      // and clear all state
+      state.clear()
+      cntState.clear()
+    }
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
index ec73fa6058b53..1cb3a6e0283ce 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
@@ -30,7 +30,7 @@ import org.slf4j.LoggerFactory
 class CRowValuesInputFormat(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[CRow])
+    @transient var returnType: TypeInformation[CRow])
   extends GenericInputFormat[CRow]
   with NonParallelInput
   with ResultTypeQueryable[CRow]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
index d536b39f68798..43ce6056610cd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory
 class ValuesInputFormat(
     name: String,
     code: String,
-    @transient returnType: TypeInformation[Row])
+    @transient var returnType: TypeInformation[Row])
   extends GenericInputFormat[Row]
   with NonParallelInput
   with ResultTypeQueryable[Row]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamTableSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/AppendStreamTableSink.scala
similarity index 79%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamTableSink.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/AppendStreamTableSink.scala
index 360252e619a55..abdca172ad23f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamTableSink.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/AppendStreamTableSink.scala
@@ -21,11 +21,15 @@ package org.apache.flink.table.sinks
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.table.api.Table
 
-/** Defines an external [[TableSink]] to emit a batch [[Table]].
+/**
+  * Defines an external [[TableSink]] to emit streaming [[Table]] with only insert changes.
+  *
+  * If the [[Table]] is also modified by update or delete changes, a
+  * [[org.apache.flink.table.api.TableException]] will be thrown.
   *
   * @tparam T Type of [[DataStream]] that this [[TableSink]] expects and supports.
   */
-trait StreamTableSink[T] extends TableSink[T] {
+trait AppendStreamTableSink[T] extends TableSink[T] {
 
   /** Emits the DataStream. */
   def emitDataStream(dataStream: DataStream[T]): Unit
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
index 809afd2fb4ec4..7214394f2e750 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSink.scala
@@ -25,7 +25,6 @@ import org.apache.flink.types.Row
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.core.fs.FileSystem.WriteMode
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 
 /**
   * A simple [[TableSink]] to emit data as CSV files.
@@ -40,7 +39,7 @@ class CsvTableSink(
     fieldDelim: Option[String],
     numFiles: Option[Int],
     writeMode: Option[WriteMode])
-  extends TableSinkBase[Row] with BatchTableSink[Row] with StreamTableSink[Row] {
+  extends TableSinkBase[Row] with BatchTableSink[Row] with AppendStreamTableSink[Row] {
 
   /**
     * A simple [[TableSink]] to emit data as CSV files.
@@ -134,100 +133,3 @@ class CsvFormatter(fieldDelim: String) extends MapFunction[Row, String] {
     builder.mkString
   }
 }
-
-/**
-  * A simple [[TableSink]] to emit data as CSV files.
-  *
-  * @param path The output path to write the Table to.
-  * @param fieldDelim The field delimiter
-  * @param numFiles The number of files to write to
-  * @param writeMode The write mode to specify whether existing files are overwritten or not.
-  */
-class CsvRetractTableSink(
-    path: String,
-    fieldDelim: Option[String],
-    numFiles: Option[Int],
-    writeMode: Option[WriteMode])
-  extends TableSinkBase[Row] with StreamRetractSink[Row] {
-
-  override def needsUpdatesAsRetraction: Boolean = true
-
-  /**
-    * A simple [[TableSink]] to emit data as CSV files.
-    *
-    * @param path The output path to write the Table to.
-    * @param fieldDelim The field delimiter, ',' by default.
-    */
-  def this(path: String, fieldDelim: String = ",") {
-    this(path, Some(fieldDelim), None, None)
-  }
-
-  /**
-    * A simple [[TableSink]] to emit data as CSV files.
-    *
-    * @param path The output path to write the Table to.
-    * @param fieldDelim The field delimiter.
-    * @param numFiles The number of files to write to.
-    * @param writeMode The write mode to specify whether existing files are overwritten or not.
-    */
-  def this(path: String, fieldDelim: String, numFiles: Int, writeMode: WriteMode) {
-    this(path, Some(fieldDelim), Some(numFiles), Some(writeMode))
-  }
-
-
-  override def emitDataStreamWithChange(dataStream: DataStream[JTuple2[Boolean,Row]]): Unit = {
-    val csvRows = dataStream
-      .map(new CsvRetractFormatter(fieldDelim.getOrElse(",")))
-      .returns(TypeInformation.of(classOf[String]))
-
-
-    if (numFiles.isDefined) {
-      csvRows.setParallelism(numFiles.get)
-    }
-
-    val sink = writeMode match {
-      case None => csvRows.writeAsText(path)
-      case Some(wm) => csvRows.writeAsText(path, wm)
-    }
-
-    if (numFiles.isDefined) {
-      sink.setParallelism(numFiles.get)
-    }
-  }
-
-  override protected def copy: TableSinkBase[Row] = {
-    new CsvRetractTableSink(path, fieldDelim, numFiles, writeMode)
-  }
-
-  override def getOutputType: TypeInformation[Row] = {
-    new RowTypeInfo(getFieldTypes: _*)
-  }
-}
-
-/**
-  * Formats a [[Tuple2]] with change information into a [[String]] with fields separated by the
-  * field delimiter.
-  *
-  * @param fieldDelim The field delimiter.
-  */
-class CsvRetractFormatter(fieldDelim: String) extends MapFunction[JTuple2[Boolean,Row], String] {
-  override def map(rowT: JTuple2[Boolean,Row]): String = {
-
-    val row: Row = rowT.f1
-
-    val builder = new StringBuilder
-
-    builder.append(rowT.f0.toString)
-
-    // write following values
-    for (i <- 0 until row.getArity) {
-      builder.append(fieldDelim)
-      val v = row.getField(i)
-      if (v != null) {
-        builder.append(v.toString)
-      }
-    }
-    builder.mkString
-  }
-}
-
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/RetractStreamTableSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/RetractStreamTableSink.scala
new file mode 100644
index 0000000000000..3ab997ed4a2d5
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/RetractStreamTableSink.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.sinks
+
+import java.lang.{Boolean => JBool}
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TupleTypeInfo
+import org.apache.flink.table.api.Types
+
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.table.api.Table
+
+/**
+  * Defines an external [[TableSink]] to emit a streaming [[Table]] with insert, update, and delete
+  * changes.
+  *
+  * The table will be converted into a stream of accumulate and retraction messages which are
+  * encoded as [[JTuple2]].
+  * The first field is a [[JBool]] flag to indicate the message type.
+  * The second field holds the record of the requested type [[T]].
+  *
+  * A message with true [[JBool]] flag is an accumulate (or add) message.
+  * A message with false flag is a retract message.
+  *
+  * @tparam T Type of records that this [[TableSink]] expects and supports.
+  */
+trait RetractStreamTableSink[T] extends TableSink[JTuple2[JBool, T]] {
+
+  /** Returns the requested record type */
+  def getRecordType: TypeInformation[T]
+
+  /** Emits the DataStream. */
+  def emitDataStream(dataStream: DataStream[JTuple2[JBool, T]]): Unit
+
+  override def getOutputType = new TupleTypeInfo(Types.BOOLEAN, getRecordType)
+
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala
deleted file mode 100644
index 7f7c9449441e2..0000000000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/StreamRetractSink.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.sinks
-
-import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
-import org.apache.flink.streaming.api.datastream.DataStream
-
-trait StreamRetractSink[T] extends TableSink[T]{
-
-  /**
-    * Whether the [[StreamTableSink]] requires that update and delete changes are sent with
-    * retraction messages.
-    */
-  def needsUpdatesAsRetraction: Boolean = false
-
-  /** Emits the DataStream with change infomation. */
-  def emitDataStreamWithChange(dataStream: DataStream[JTuple2[Boolean,T]]): Unit
-
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala
new file mode 100644
index 0000000000000..2ae3406a72d47
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.sinks
+
+import java.lang.{Boolean => JBool}
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.api.java.typeutils.TupleTypeInfo
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.table.api.{Table, Types}
+
+/**
+  * Defines an external [[TableSink]] to emit a streaming [[Table]] with insert, update, and delete
+  * changes. The [[Table]] must be have unique key fields (atomic or composite) or be append-only.
+  *
+  * If the [[Table]] does not have a unique key and is not append-only, a
+  * [[org.apache.flink.table.api.TableException]] will be thrown.
+  *
+  * The unique key of the table is configured by the [[UpsertStreamTableSink#setKeyFields()]]
+  * method.
+  *
+  * The [[Table]] will be converted into a stream of upsert and delete messages which are encoded as
+  * [[JTuple2]]. The first field is a [[JBool]] flag to indicate the message type. The second field
+  * holds the record of the requested type [[T]].
+  *
+  * A message with true [[JBool]] field is an upsert message for the configured key.
+  * A message with false flag is a delete message for the configured key.
+  *
+  * If the table is append-only, all messages will have a true flag and must be interpreted
+  * as insertions.
+  *
+  * @tparam T Type of records that this [[TableSink]] expects and supports.
+  */
+trait UpsertStreamTableSink[T] extends TableSink[JTuple2[JBool, T]] {
+
+  /**
+    * Configures the unique key fields of the [[Table]] to write.
+    * The method is called after [[TableSink.configure()]].
+    *
+    * The keys array might be empty, if the table consists of a single (updated) record.
+    * If the table does not have a key and is append-only, the keys attribute is null.
+    *
+    * @param keys the field names of the table's keys, an empty array if the table has a single
+    *             row, and null if the table is append-only and has no key.
+    */
+  def setKeyFields(keys: Array[String]): Unit
+
+  /**
+    * Specifies whether the [[Table]] to write is append-only or not.
+    *
+    * @param isAppendOnly true if the table is append-only, false otherwise.
+    */
+  def setIsAppendOnly(isAppendOnly: Boolean): Unit
+
+  /** Returns the requested record type */
+  def getRecordType: TypeInformation[T]
+
+  /** Emits the DataStream. */
+  def emitDataStream(dataStream: DataStream[JTuple2[JBool, T]]): Unit
+
+  override def getOutputType = new TupleTypeInfo(Types.BOOLEAN, getRecordType)
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index 675e5d9287574..ba3b5918f5958 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -21,7 +21,6 @@ package org.apache.flink.table
 import org.apache.flink.api.scala._
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.{TupleTypeInfo, TypeExtractor}
 import org.apache.flink.table.api.scala._
 import org.apache.flink.api.java.typeutils.{GenericTypeInfo, RowTypeInfo, TupleTypeInfo, TypeExtractor}
 import org.apache.flink.table.api.TableException
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
index d49076379e2c5..40f4c7da3c1b7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
@@ -27,10 +27,8 @@ import org.junit.Test
 import org.apache.flink.table.api.scala._
 import org.apache.flink.api.scala._
 import org.apache.flink.streaming.api.TimeCharacteristic
-import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.table.utils.TableFunc0
 
-import scala.collection.mutable
 
 /**
   * tests for retraction
@@ -55,51 +53,47 @@ class RetractionITCase extends StreamingWithStateTestBase {
   def testWordCount(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
-    env.setParallelism(1)
+    StreamITCase.clear
     env.setStateBackend(getStateBackend)
 
     val stream = env.fromCollection(data)
     val table = stream.toTable(tEnv, 'word, 'num)
     val resultTable = table
       .groupBy('word)
-      .select('word as 'word, 'num.sum as 'count)
+      .select('num.sum as 'count)
       .groupBy('count)
-      .select('count, 'word.count as 'frequency)
+      .select('count, 'count.count as 'frequency)
 
-    val results = resultTable.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    val results = resultTable.toRetractStream[Row]
+    results.addSink(new StreamITCase.RetractingSink)
     env.execute()
 
-    val expected = Seq("1,1", "1,2", "1,1", "2,1", "1,2", "1,1", "2,2", "2,1", "3,1", "3,0",
-      "4,1", "4,0", "5,1", "5,0", "6,1", "1,2")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    val expected = Seq("1,2", "2,1", "6,1")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
   // keyed groupby + non-keyed groupby
   @Test
   def testGroupByAndNonKeyedGroupBy(): Unit = {
-
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
-    env.setParallelism(1)
+    StreamITCase.clear
     env.setStateBackend(getStateBackend)
 
     val stream = env.fromCollection(data)
     val table = stream.toTable(tEnv, 'word, 'num)
     val resultTable = table
       .groupBy('word)
-      .select('word as 'word, 'num.sum as 'count)
-      .select('count.sum)
+      .select('word as 'word, 'num.sum as 'cnt)
+      .select('cnt.sum)
+
+    val results = resultTable.toRetractStream[Row]
 
-    val results = resultTable.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    results.addSink(new StreamITCase.RetractingSink).setParallelism(1)
     env.execute()
 
-    val expected = Seq("1", "2", "1", "3", "4", "3", "5", "3", "6", "3", "7", "3", "8", "3", "9",
-      "10")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    val expected = Seq("10")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
   // non-keyed groupby + keyed groupby
@@ -108,8 +102,7 @@ class RetractionITCase extends StreamingWithStateTestBase {
 
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
-    env.setParallelism(1)
+    StreamITCase.clear
     env.setStateBackend(getStateBackend)
 
     val stream = env.fromCollection(data)
@@ -119,13 +112,12 @@ class RetractionITCase extends StreamingWithStateTestBase {
       .groupBy('count)
       .select('count, 'count.count)
 
-    val results = resultTable.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    val results = resultTable.toRetractStream[Row]
+    results.addSink(new StreamITCase.RetractingSink)
     env.execute()
 
-    val expected = Seq("1,1", "1,0", "2,1", "2,0", "3,1", "3,0", "4,1", "4,0", "5,1", "5,0", "6," +
-      "1", "6,0", "7,1", "7,0", "8,1", "8,0", "9,1", "9,0", "10,1")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    val expected = Seq("10,1")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
   // test unique process, if the current output message of unbounded groupby equals the
@@ -150,9 +142,9 @@ class RetractionITCase extends StreamingWithStateTestBase {
     )
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
-    env.setParallelism(1)
+    StreamITCase.clear
     env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
 
     val stream = env.fromCollection(data)
     val table = stream.toTable(tEnv, 'pk, 'value)
@@ -162,12 +154,13 @@ class RetractionITCase extends StreamingWithStateTestBase {
       .groupBy('sum)
       .select('sum, 'pk.count as 'count)
 
-    val results = resultTable.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    val results = resultTable.toRetractStream[Row]
+    results.addSink(new StreamITCase.RetractMessagesSink)
     env.execute()
 
-    val expected = Seq("1,1", "2,1", "3,1", "3,0", "6,1", "1,2", "1,3", "6,2", "6,1", "12,1","12," +
-      "0", "18,1", "8,1")
+    val expected = Seq(
+      "+1,1", "+2,1", "+3,1", "-3,1", "+6,1", "-1,1", "+1,2", "-1,2", "+1,3", "-6,1", "+6,2",
+      "-6,2", "+6,1", "+12,1", "-12,1", "+18,1", "+8,1")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -176,8 +169,7 @@ class RetractionITCase extends StreamingWithStateTestBase {
   def testCorrelate(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
-    env.setParallelism(1)
+    StreamITCase.clear
     env.setStateBackend(getStateBackend)
 
     val func0 = new TableFunc0
@@ -186,19 +178,17 @@ class RetractionITCase extends StreamingWithStateTestBase {
     val table = stream.toTable(tEnv, 'word, 'num)
     val resultTable = table
       .groupBy('word)
-      .select('word as 'word, 'num.sum as 'count)
+      .select('word as 'word, 'num.sum as 'cnt)
       .leftOuterJoin(func0('word))
-      .groupBy('count)
-      .select('count, 'word.count as 'frequency)
+      .groupBy('cnt)
+      .select('cnt, 'word.count as 'frequency)
 
-    val results = resultTable.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    val results = resultTable.toRetractStream[Row]
+    results.addSink(new StreamITCase.RetractingSink)
     env.execute()
 
-    val expected = Seq(
-      "1,1", "1,2", "1,1", "2,1", "1,2", "1,1", "2,2", "2,1", "3,1", "3,0", "4,1", "4,0", "5,1",
-      "5,0", "6,1", "1,2")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    val expected = Seq("1,2", "2,1", "6,1")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
index ceae6c610703b..c446d64f780ed 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSinkITCase.scala
@@ -23,7 +23,7 @@ import java.io.File
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api.scala.stream.utils.StreamTestData
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.sinks.{CsvTableSink, CsvRetractTableSink}
+import org.apache.flink.table.sinks.CsvTableSink
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
 import org.apache.flink.table.api.TableEnvironment
@@ -59,34 +59,5 @@ class TableSinkITCase extends StreamingMultipleProgramsTestBase {
 
     TestBaseUtils.compareResultsByLinesInMemory(expected, path)
   }
-
-  @Test
-  def testStreamTableSinkNeedRetraction(): Unit = {
-
-    val tmpFile = File.createTempFile("flink-table-sink-test", ".tmp")
-    tmpFile.deleteOnExit()
-    val path = tmpFile.toURI.toString
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setParallelism(4)
-
-    val input = StreamTestData.get3TupleDataStream(env)
-      .map(x => x).setParallelism(1) // increase DOP to 4
-
-    val results = input.toTable(tEnv, 'a, 'b, 'c)
-      .where('a < 5 || 'a > 17)
-      .select('c, 'b)
-      .groupBy('b)
-      .select('b, 'c.count)
-      .writeToSink(new CsvRetractTableSink(path))
-
-    env.execute()
-
-    val expected = Seq(
-      "true,1,1", "true,2,1", "false,2,1", "true,2,2", "true,3,1", "true,6,1", "false,6,1",
-      "true,6,2", "false,6,2", "true,6,3", "false,6,3", "true,6,4").mkString("\n")
-
-    TestBaseUtils.compareResultsByLinesInMemory(expected, path)
-  }
+  
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
index abbcbdd083cb6..249d5052cf828 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
@@ -53,19 +53,19 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT b, COUNT(a) FROM MyTable GROUP BY b"
 
-    val t = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c)
+    val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c)
     tEnv.registerTable("MyTable", t)
 
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
+    val result = tEnv.sql(sqlQuery).toRetractStream[Row]
+    result.addSink(new StreamITCase.RetractingSink).setParallelism(1)
     env.execute()
 
-    val expected = mutable.MutableList("1,1", "2,1", "2,2")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    val expected = List("1,1", "2,2", "3,3", "4,4", "5,5", "6,6")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
   /** test selection **/
@@ -74,7 +74,7 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT a * 2, b - 1 FROM MyTable"
 
@@ -85,7 +85,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList("2,0", "4,1", "6,1")
+    val expected = List("2,0", "4,1", "6,1")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -95,7 +95,7 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT * FROM MyTable WHERE a = 3"
 
@@ -106,7 +106,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList("3,2,Hello world")
+    val expected = List("3,2,Hello world")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -116,7 +116,7 @@ class SqlITCase extends StreamingWithStateTestBase {
 
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT * FROM MyTable WHERE _1 = 3"
 
@@ -127,7 +127,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList("3,2,Hello world")
+    val expected = List("3,2,Hello world")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -136,7 +136,7 @@ class SqlITCase extends StreamingWithStateTestBase {
   def testUnion(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT * FROM T1 " +
       "UNION ALL " +
@@ -151,7 +151,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,1,Hi", "1,1,Hi",
       "2,2,Hello", "2,2,Hello",
       "3,2,Hello world", "3,2,Hello world")
@@ -163,7 +163,7 @@ class SqlITCase extends StreamingWithStateTestBase {
   def testUnionWithFilter(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT * FROM T1 WHERE a = 3 " +
       "UNION ALL " +
@@ -178,7 +178,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "2,2,Hello",
       "3,2,Hello world")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
@@ -189,7 +189,7 @@ class SqlITCase extends StreamingWithStateTestBase {
   def testUnionTableWithDataSet(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT c FROM T1 WHERE a = 3 " +
       "UNION ALL " +
@@ -204,7 +204,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList("Hello", "Hello world")
+    val expected = List("Hello", "Hello world")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -213,7 +213,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     // for sum aggregation ensure that every time the order of each element is consistent
     env.setParallelism(1)
@@ -232,7 +232,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "Hello World,1,7", "Hello World,2,15", "Hello World,3,35",
       "Hello,1,1", "Hello,2,3", "Hello,3,6", "Hello,4,10", "Hello,5,15", "Hello,6,21")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
@@ -243,7 +243,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
@@ -259,7 +259,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "Hello World,1", "Hello World,2", "Hello World,3",
       "Hello,1", "Hello,2", "Hello,3", "Hello,4", "Hello,5", "Hello,6")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
@@ -270,7 +270,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     // for sum aggregation ensure that every time the order of each element is consistent
     env.setParallelism(1)
@@ -289,7 +289,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "Hello World,7,28", "Hello World,8,36", "Hello World,9,56",
       "Hello,1,1", "Hello,2,3", "Hello,3,6", "Hello,4,10", "Hello,5,15", "Hello,6,21")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
@@ -300,7 +300,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
@@ -314,7 +314,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList("1", "2", "3", "4", "5", "6", "7", "8", "9")
+    val expected = List("1", "2", "3", "4", "5", "6", "7", "8", "9")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
@@ -363,7 +363,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
       "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
       "Hello,3,3,7", "Hello,4,3,9", "Hello,5,3,12",
@@ -420,7 +420,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
       "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
       "Hello,3,3,7",
@@ -490,7 +490,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
       "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
       "Hello,3,4,9",
@@ -562,7 +562,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
       "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
       "Hello,3,4,9",
@@ -584,7 +584,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
 
@@ -608,7 +608,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
     env.setStateBackend(getStateBackend)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
@@ -661,7 +661,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,2,Hello,2,1,2,2,2",
       "1,3,Hello world,5,2,2,3,2",
       "1,1,Hi,6,3,2,3,1",
@@ -687,7 +687,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
     env.setStateBackend(getStateBackend)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val sqlQuery = "SELECT a, b, c, " +
       "SUM(b) over (" +
@@ -731,7 +731,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,2,Hello,2,1,2,2,2",
       "1,3,Hello world,5,2,2,3,2",
       "1,1,Hi,6,3,2,3,1",
@@ -757,7 +757,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
     env.setStateBackend(getStateBackend)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
@@ -793,7 +793,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "2,2,Hello,2,1,2,2,2",
       "3,5,Hello,7,2,3,5,2",
       "1,3,Hello,10,3,3,5,2",
@@ -812,7 +812,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
     env.setStateBackend(getStateBackend)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
@@ -849,7 +849,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "2,2,Hello,2,1,2,2,2",
       "3,5,Hello,7,2,3,5,2",
       "1,3,Hello,10,3,3,5,2",
@@ -869,7 +869,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
     env.setStateBackend(getStateBackend)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
@@ -907,7 +907,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "2,1,Hello,1,1,1,1,1",
       "1,1,Hello,7,4,1,3,1",
       "1,2,Hello,7,4,1,3,1",
@@ -932,7 +932,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
     env.setStateBackend(getStateBackend)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
     env.setParallelism(1)
 
     val sqlQuery = "SELECT a, b, c, " +
@@ -975,7 +975,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,1,Hello,6,3,2,3,1",
       "1,2,Hello,6,3,2,3,1",
       "1,3,Hello world,6,3,2,3,1",
@@ -1000,7 +1000,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setParallelism(1)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val t = StreamTestData.get5TupleDataStream(env)
       .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
@@ -1017,7 +1017,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,0,0",
       "2,1,1",
       "2,3,1",
@@ -1043,7 +1043,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setParallelism(1)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val t = StreamTestData.get5TupleDataStream(env)
       .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
@@ -1060,7 +1060,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,0,0",
       "2,1,1",
       "2,3,1",
@@ -1087,7 +1087,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setParallelism(1)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val t = StreamTestData.get5TupleDataStream(env)
       .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
@@ -1104,7 +1104,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,0,0",
       "2,1,0",
       "2,3,0",
@@ -1130,7 +1130,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     env.setStateBackend(getStateBackend)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     env.setParallelism(1)
-    StreamITCase.testResults = mutable.MutableList()
+    StreamITCase.clear
 
     val t = StreamTestData.get5TupleDataStream(env)
       .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
@@ -1146,7 +1146,7 @@ class SqlITCase extends StreamingWithStateTestBase {
     result.addSink(new StreamITCase.StringSink)
     env.execute()
 
-    val expected = mutable.MutableList(
+    val expected = List(
       "1,0,0",
       "2,1,0",
       "2,3,0",
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
index 271e90ba23bab..910cbf267dbc5 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
@@ -23,6 +23,7 @@ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
 import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala.stream.utils.StreamITCase.RetractingSink
 import org.apache.flink.types.Row
 import org.junit.Assert.assertEquals
 import org.junit.Test
@@ -38,29 +39,24 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
   def testNonKeyedGroupAggregate(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
-    env.setParallelism(1)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.clear
 
     val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
             .select('a.sum, 'b.sum)
 
-    val results = t.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    val results = t.toRetractStream[Row]
+    results.addSink(new StreamITCase.RetractingSink).setParallelism(1)
     env.execute()
 
-    val expected = mutable.MutableList(
-      "1,1", "3,3", "6,5", "10,8", "15,11", "21,14", "28,18", "36,22", "45,26", "55,30", "66,35",
-      "78,40", "91,45", "105,50", "120,55", "136,61", "153,67", "171,73", "190,79", "210,85",
-      "231,91")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    val expected = List("231,91")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
   @Test
   def testGroupAggregate(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
-    env.setParallelism(1)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.clear
 
@@ -68,15 +64,12 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
       .groupBy('b)
       .select('b, 'a.sum)
 
-    val results = t.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    val results = t.toRetractStream[Row]
+    results.addSink(new StreamITCase.RetractingSink)
     env.execute()
 
-    val expected = mutable.MutableList(
-      "1,1", "2,2", "2,5", "3,4", "3,9", "3,15", "4,7", "4,15",
-      "4,24", "4,34", "5,11", "5,23", "5,36", "5,50", "5,65", "6,16", "6,33", "6,51", "6,70",
-      "6,90", "6,111")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    val expected = List("1,1", "2,5", "3,15", "4,34", "5,65", "6,111")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
   @Test
@@ -88,30 +81,22 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
 
     val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
       .groupBy('b)
-      .select('a.sum as 'd, 'b)
-      .groupBy('b, 'd)
-      .select('b)
+      .select('a.count as 'cnt, 'b)
+      .groupBy('cnt)
+      .select('cnt, 'b.count as 'freq)
 
-    val results = t.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
-    env.execute()
+    val results = t.toRetractStream[Row]
 
-    val expected = mutable.MutableList(
-      "1",
-      "2", "2",
-      "3", "3", "3",
-      "4", "4", "4", "4",
-      "5", "5", "5", "5", "5",
-      "6", "6", "6", "6", "6", "6")
-
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+    results.addSink(new RetractingSink)
+    env.execute()
+    val expected = List("1,1", "2,1", "3,1", "4,1", "5,1", "6,1")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 
   @Test
   def testGroupAggregateWithExpression(): Unit = {
     val env = StreamExecutionEnvironment.getExecutionEnvironment
     env.setStateBackend(getStateBackend)
-    env.setParallelism(1)
     val tEnv = TableEnvironment.getTableEnvironment(env)
     StreamITCase.clear
 
@@ -119,14 +104,14 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
       .groupBy('e, 'b % 3)
       .select('c.min, 'e, 'a.avg, 'd.count)
 
-    val results = t.toDataStream[Row]
-    results.addSink(new StreamITCase.StringSink)
+    val results = t.toRetractStream[Row]
+    results.addSink(new RetractingSink)
     env.execute()
 
     val expected = mutable.MutableList(
-      "0,1,1,1", "1,2,2,1", "2,1,2,1", "3,2,3,1", "1,2,2,2",
-      "5,3,3,1", "3,2,3,2", "7,1,4,1", "2,1,3,2", "3,2,3,3", "7,1,4,2", "5,3,4,2", "12,3,5,1",
-      "1,2,3,3", "14,2,5,1")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+      "0,1,1,1", "7,1,4,2", "2,1,3,2",
+      "3,2,3,3", "1,2,3,3", "14,2,5,1",
+      "12,3,5,1", "5,3,4,2")
+    assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
   }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
index ea3ab221a60e2..96e5eb59baba9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/OverWindowTest.scala
@@ -35,7 +35,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows as 'w)
       .select('c, 'b.count over 'x)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -43,7 +43,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over partitionBy 'c orderBy 'abc preceding 2.rows as 'w)
       .select('c, 'b.count over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -51,7 +51,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over partitionBy 'c orderBy 'rowtime preceding 2 following "xx" as 'w)
       .select('c, 'b.count over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -59,7 +59,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_RANGE as 'w)
       .select('c, 'b.count over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -67,7 +67,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over partitionBy 'a + 'b orderBy 'rowtime preceding 2.rows as 'w)
       .select('c, 'b.count over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -77,7 +77,7 @@ class OverWindowTest extends TableTestBase {
     val result = table2
       .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows as 'w)
       .select('c, 'b.count over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -85,7 +85,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over orderBy 'rowtime preceding -1.rows as 'w)
       .select('c, 'b.count over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -93,7 +93,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over orderBy 'rowtime preceding 1.rows following -2.rows as 'w)
       .select('c, 'b.count over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test(expected = classOf[ValidationException])
@@ -103,7 +103,7 @@ class OverWindowTest extends TableTestBase {
     val result = table
       .window(Over orderBy 'rowtime preceding 1.minutes as 'w)
       .select('c, weightedAvg('b, 'a) over 'w)
-    streamUtil.tEnv.optimize(result.getRelNode)
+    streamUtil.tEnv.optimize(result.getRelNode, updatesAsRetraction = true)
   }
 
   @Test
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
index 497869d25fc15..effde8e7c713c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/utils/StreamITCase.scala
@@ -25,16 +25,18 @@ import org.junit.Assert._
 
 import scala.collection.mutable
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
-import org.apache.flink.table.runtime.types.CRow
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 
 object StreamITCase {
 
-  var testResults = mutable.MutableList.empty[String]
+  var testResults: mutable.MutableList[String] = mutable.MutableList.empty[String]
+  var retractedResults: ArrayBuffer[String] = mutable.ArrayBuffer.empty[String]
 
   def clear = {
     StreamITCase.testResults.clear()
+    StreamITCase.retractedResults.clear()
   }
 
   def compareWithList(expected: java.util.List[String]): Unit = {
@@ -49,4 +51,33 @@ object StreamITCase {
       }
     }
   }
+
+  final class RetractMessagesSink extends RichSinkFunction[(Boolean, Row)]() {
+    def invoke(v: (Boolean, Row)) {
+      testResults.synchronized {
+        testResults += (if (v._1) "+" else "-") + v._2
+      }
+    }
+  }
+
+  final class RetractingSink() extends RichSinkFunction[(Boolean, Row)] {
+    def invoke(v: (Boolean, Row)) {
+      retractedResults.synchronized {
+        val value = v._2.toString
+        if (v._1) {
+          retractedResults += value
+        } else {
+          val idx = retractedResults.indexOf(value)
+          if (idx >= 0) {
+            retractedResults.remove(idx)
+          } else {
+            throw new RuntimeException("Tried to retract a value that wasn't added first. " +
+              "This is probably an incorrectly implemented test. " +
+              "Try to set the parallelism of the sink to 1.")
+          }
+        }
+      }
+    }
+  }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala
index 580029f5c2101..861f70e3f3d20 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/plan/rules/RetractionRulesTest.scala
@@ -292,7 +292,7 @@ class StreamTableTestForRetractionUtil extends StreamTableTestUtil {
 
   def verifyTableTrait(resultTable: Table, expected: String): Unit = {
     val relNode = resultTable.getRelNode
-    val optimized = tEnv.optimize(relNode)
+    val optimized = tEnv.optimize(relNode, updatesAsRetraction = false)
     val actual = TraitUtil.toString(optimized)
     assertEquals(
       expected.split("\n").map(_.trim).mkString("\n"),
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala
new file mode 100644
index 0000000000000..2dfb658320555
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.sinks
+
+import java.lang.{Boolean => JBool}
+
+import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.streaming.api.functions.sink.SinkFunction
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.apache.flink.table.api.{TableEnvironment, TableException}
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.scala.stream.utils.StreamTestData
+import org.apache.flink.types.Row
+
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+class StreamTableSinksITCase extends StreamingMultipleProgramsTestBase {
+
+  @Test(expected = classOf[TableException])
+  def testAppendSinkOnUpdatingTable(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'id, 'num, 'text)
+
+    t.groupBy('text)
+      .select('text, 'id.count, 'num.sum)
+      .writeToSink(new TestAppendSink)
+
+    // must fail because table is not append-only
+    env.execute()
+  }
+
+  @Test
+  def testAppendSinkOnAppendTable(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+        .assignAscendingTimestamps(_._1.toLong)
+        .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+    t.window(Tumble over 5.millis on 'rowtime as 'w)
+      .groupBy('w)
+      .select('w.end, 'id.count, 'num.sum)
+      .writeToSink(new TestAppendSink)
+
+    env.execute()
+
+    val result = RowCollector.getAndClearValues.map(_.f1.toString).sorted
+    val expected = List(
+      "1970-01-01 00:00:00.005,4,8",
+      "1970-01-01 00:00:00.01,5,18",
+      "1970-01-01 00:00:00.015,5,24",
+      "1970-01-01 00:00:00.02,5,29",
+      "1970-01-01 00:00:00.025,2,12")
+      .sorted
+    assertEquals(expected, result)
+  }
+
+  @Test
+  def testRetractSinkOnUpdatingTable(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text)
+
+    t.select('id, 'num, 'text.charLength() as 'len)
+      .groupBy('len)
+      .select('len, 'id.count, 'num.sum)
+      .writeToSink(new TestRetractSink)
+
+    env.execute()
+    val results = RowCollector.getAndClearValues
+
+    val retracted = restractResults(results).sorted
+    val expected = List(
+      "2,1,1",
+      "5,1,2",
+      "11,1,2",
+      "25,1,3",
+      "10,7,39",
+      "14,1,3",
+      "9,9,41").sorted
+    assertEquals(expected, retracted)
+
+  }
+
+  @Test
+  def testRetractSinkOnAppendTable(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+    t.window(Tumble over 5.millis on 'rowtime as 'w)
+      .groupBy('w)
+      .select('w.end, 'id.count, 'num.sum)
+      .writeToSink(new TestRetractSink)
+
+    env.execute()
+    val results = RowCollector.getAndClearValues
+
+    assertFalse(
+      "Received retraction messages for append only table",
+      results.exists(!_.f0))
+
+    val retracted = restractResults(results).sorted
+    val expected = List(
+      "1970-01-01 00:00:00.005,4,8",
+      "1970-01-01 00:00:00.01,5,18",
+      "1970-01-01 00:00:00.015,5,24",
+      "1970-01-01 00:00:00.02,5,29",
+      "1970-01-01 00:00:00.025,2,12")
+      .sorted
+    assertEquals(expected, retracted)
+
+  }
+
+  @Test
+  def testUpsertSinkOnUpdatingTableWithFullKey(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text)
+
+    t.select('id, 'num, 'text.charLength() as 'len, ('id > 0) as 'cTrue)
+      .groupBy('len, 'cTrue)
+      .select('len, 'id.count as 'cnt, 'cTrue)
+      .groupBy('cnt, 'cTrue)
+      .select('cnt, 'len.count, 'cTrue)
+      .writeToSink(new TestUpsertSink(Array("cnt", "cTrue"), false))
+
+    env.execute()
+    val results = RowCollector.getAndClearValues
+
+    assertTrue(
+      "Results must include delete messages",
+      results.exists(_.f0 == false)
+    )
+
+    val retracted = upsertResults(results, Array(0, 2)).sorted
+    val expected = List(
+      "1,5,true",
+      "7,1,true",
+      "9,1,true").sorted
+    assertEquals(expected, retracted)
+
+  }
+
+  @Test(expected = classOf[TableException])
+  def testUpsertSinkOnUpdatingTableWithoutFullKey(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text)
+
+    t.select('id, 'num, 'text.charLength() as 'len, ('id > 0) as 'cTrue)
+      .groupBy('len, 'cTrue)
+      .select('len, 'id.count, 'num.sum)
+      .writeToSink(new TestUpsertSink(Array("len", "cTrue"), false))
+
+    // must fail because table is updating table without full key
+    env.execute()
+  }
+
+  @Test
+  def testUpsertSinkOnAppendingTableWithFullKey1(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+    t.window(Tumble over 5.millis on 'rowtime as 'w)
+      .groupBy('w, 'num)
+      .select('num, 'w.end as 'wend, 'id.count)
+      .writeToSink(new TestUpsertSink(Array("wend", "num"), true))
+
+    env.execute()
+    val results = RowCollector.getAndClearValues
+
+    assertFalse(
+      "Received retraction messages for append only table",
+      results.exists(!_.f0))
+
+    val retracted = upsertResults(results, Array(0, 1, 2)).sorted
+    val expected = List(
+      "1,1970-01-01 00:00:00.005,1",
+      "2,1970-01-01 00:00:00.005,2",
+      "3,1970-01-01 00:00:00.005,1",
+      "3,1970-01-01 00:00:00.01,2",
+      "4,1970-01-01 00:00:00.01,3",
+      "4,1970-01-01 00:00:00.015,1",
+      "5,1970-01-01 00:00:00.015,4",
+      "5,1970-01-01 00:00:00.02,1",
+      "6,1970-01-01 00:00:00.02,4",
+      "6,1970-01-01 00:00:00.025,2").sorted
+    assertEquals(expected, retracted)
+  }
+
+  @Test
+  def testUpsertSinkOnAppendingTableWithFullKey2(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+    t.window(Tumble over 5.millis on 'rowtime as 'w)
+      .groupBy('w, 'num)
+      .select('w.start as 'wstart, 'w.end as 'wend, 'num, 'id.count)
+      .writeToSink(new TestUpsertSink(Array("wstart", "wend", "num"), true))
+
+    env.execute()
+    val results = RowCollector.getAndClearValues
+
+    assertFalse(
+      "Received retraction messages for append only table",
+      results.exists(!_.f0))
+
+    val retracted = upsertResults(results, Array(0, 1, 2)).sorted
+    val expected = List(
+      "1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,1,1",
+      "1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,2,2",
+      "1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,3,1",
+      "1970-01-01 00:00:00.005,1970-01-01 00:00:00.01,3,2",
+      "1970-01-01 00:00:00.005,1970-01-01 00:00:00.01,4,3",
+      "1970-01-01 00:00:00.01,1970-01-01 00:00:00.015,4,1",
+      "1970-01-01 00:00:00.01,1970-01-01 00:00:00.015,5,4",
+      "1970-01-01 00:00:00.015,1970-01-01 00:00:00.02,5,1",
+      "1970-01-01 00:00:00.015,1970-01-01 00:00:00.02,6,4",
+      "1970-01-01 00:00:00.02,1970-01-01 00:00:00.025,6,2").sorted
+    assertEquals(expected, retracted)
+  }
+
+  @Test
+  def testUpsertSinkOnAppendingTableWithoutFullKey1(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+    t.window(Tumble over 5.millis on 'rowtime as 'w)
+      .groupBy('w, 'num)
+      .select('w.end as 'wend, 'id.count as 'cnt)
+      .writeToSink(new TestUpsertSink(null, true))
+
+    env.execute()
+    val results = RowCollector.getAndClearValues
+
+    assertFalse(
+      "Received retraction messages for append only table",
+      results.exists(!_.f0))
+
+    val retracted = results.map(_.f1.toString).sorted
+    val expected = List(
+      "1970-01-01 00:00:00.005,1",
+      "1970-01-01 00:00:00.005,2",
+      "1970-01-01 00:00:00.005,1",
+      "1970-01-01 00:00:00.01,2",
+      "1970-01-01 00:00:00.01,3",
+      "1970-01-01 00:00:00.015,1",
+      "1970-01-01 00:00:00.015,4",
+      "1970-01-01 00:00:00.02,1",
+      "1970-01-01 00:00:00.02,4",
+      "1970-01-01 00:00:00.025,2").sorted
+    assertEquals(expected, retracted)
+  }
+
+  @Test
+  def testUpsertSinkOnAppendingTableWithoutFullKey2(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+
+    val t = StreamTestData.get3TupleDataStream(env)
+      .assignAscendingTimestamps(_._1.toLong)
+      .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+    t.window(Tumble over 5.millis on 'rowtime as 'w)
+      .groupBy('w, 'num)
+      .select('num, 'id.count as 'cnt)
+      .writeToSink(new TestUpsertSink(null, true))
+
+    env.execute()
+    val results = RowCollector.getAndClearValues
+
+    assertFalse(
+      "Received retraction messages for append only table",
+      results.exists(!_.f0))
+
+    val retracted = results.map(_.f1.toString).sorted
+    val expected = List(
+      "1,1",
+      "2,2",
+      "3,1",
+      "3,2",
+      "4,3",
+      "4,1",
+      "5,4",
+      "5,1",
+      "6,4",
+      "6,2").sorted
+    assertEquals(expected, retracted)
+  }
+
+  /** Converts a list of retraction messages into a list of final results. */
+  private def restractResults(results: List[JTuple2[JBool, Row]]): List[String] = {
+
+    val retracted = results
+      .foldLeft(Map[String, Int]()){ (m: Map[String, Int], v: JTuple2[JBool, Row]) =>
+        val cnt = m.getOrElse(v.f1.toString, 0)
+        if (v.f0) {
+          m + (v.f1.toString -> (cnt + 1))
+        } else {
+          m + (v.f1.toString -> (cnt - 1))
+        }
+      }.filter{ case (_, c: Int) => c != 0 }
+
+    assertFalse(
+      "Received retracted rows which have not been accumulated.",
+      retracted.exists{ case (_, c: Int) => c < 0})
+
+    retracted.flatMap { case (r: String, c: Int) => (0 until c).map(_ => r) }.toList
+  }
+
+  /** Converts a list of upsert messages into a list of final results. */
+  private def upsertResults(results: List[JTuple2[JBool, Row]], keys: Array[Int]): List[String] = {
+
+    def getKeys(r: Row): List[String] =
+      keys.foldLeft(List[String]())((k, i) => r.getField(i).toString :: k)
+
+    val upserted = results.foldLeft(Map[String, String]()){ (o: Map[String, String], r) =>
+      val key = getKeys(r.f1).mkString("")
+      if (r.f0) {
+        o + (key -> r.f1.toString)
+      } else {
+        o - key
+      }
+    }
+
+    upserted.values.toList
+  }
+
+}
+
+private class TestAppendSink extends AppendStreamTableSink[Row] {
+
+  var fNames: Array[String] = _
+  var fTypes: Array[TypeInformation[_]] = _
+
+  override def emitDataStream(s: DataStream[Row]): Unit = {
+    s.map(
+      new MapFunction[Row, JTuple2[JBool, Row]] {
+        override def map(value: Row): JTuple2[JBool, Row] = new JTuple2(true, value)
+      })
+      .addSink(new RowSink)
+  }
+
+  override def getOutputType: TypeInformation[Row] = new RowTypeInfo(fTypes, fNames)
+
+  override def getFieldNames: Array[String] = fNames
+
+  override def getFieldTypes: Array[TypeInformation[_]] = fTypes
+
+  override def configure(
+    fieldNames: Array[String],
+    fieldTypes: Array[TypeInformation[_]]): TableSink[Row] = {
+    val copy = new TestAppendSink
+    copy.fNames = fieldNames
+    copy.fTypes = fieldTypes
+    copy
+  }
+}
+
+private class TestRetractSink extends RetractStreamTableSink[Row] {
+
+  var fNames: Array[String] = _
+  var fTypes: Array[TypeInformation[_]] = _
+
+  override def emitDataStream(s: DataStream[JTuple2[JBool, Row]]): Unit = {
+    s.addSink(new RowSink)
+  }
+
+  override def getRecordType: TypeInformation[Row] = new RowTypeInfo(fTypes, fNames)
+
+  override def getFieldNames: Array[String] = fNames
+
+  override def getFieldTypes: Array[TypeInformation[_]] = fTypes
+
+  override def configure(
+      fieldNames: Array[String],
+      fieldTypes: Array[TypeInformation[_]]): TableSink[JTuple2[JBool, Row]] = {
+    val copy = new TestRetractSink
+    copy.fNames = fieldNames
+    copy.fTypes = fieldTypes
+    copy
+  }
+
+}
+
+private class TestUpsertSink(
+    expectedKeys: Array[String],
+    expectedIsAppendOnly: Boolean)
+  extends UpsertStreamTableSink[Row] {
+
+  var fNames: Array[String] = _
+  var fTypes: Array[TypeInformation[_]] = _
+
+  override def setKeyFields(keys: Array[String]): Unit =
+    if (keys != null) {
+      assertEquals("Provided key fields do not match expected keys",
+        expectedKeys.sorted.mkString(","),
+        keys.sorted.mkString(","))
+    } else {
+      assertNull("Provided key fields should not be null.", expectedKeys)
+    }
+
+  override def setIsAppendOnly(isAppendOnly: Boolean): Unit =
+    assertEquals(
+      "Provided isAppendOnly does not match expected isAppendOnly",
+      expectedIsAppendOnly,
+      isAppendOnly)
+
+  override def getRecordType: TypeInformation[Row] = new RowTypeInfo(fTypes, fNames)
+
+  override def emitDataStream(s: DataStream[JTuple2[JBool, Row]]): Unit = {
+    s.addSink(new RowSink)
+  }
+
+  override def getFieldNames: Array[String] = fNames
+
+  override def getFieldTypes: Array[TypeInformation[_]] = fTypes
+
+  override def configure(
+      fieldNames: Array[String],
+      fieldTypes: Array[TypeInformation[_]]): TableSink[JTuple2[JBool, Row]] = {
+    val copy = new TestUpsertSink(expectedKeys, expectedIsAppendOnly)
+    copy.fNames = fieldNames
+    copy.fTypes = fieldTypes
+    copy
+  }
+}
+
+class RowSink extends SinkFunction[JTuple2[JBool, Row]] {
+  override def invoke(value: JTuple2[JBool, Row]): Unit = RowCollector.addValue(value)
+}
+
+object RowCollector {
+  private val sink: mutable.ArrayBuffer[JTuple2[JBool, Row]] =
+    new mutable.ArrayBuffer[JTuple2[JBool, Row]]()
+
+  def addValue(value: JTuple2[JBool, Row]): Unit = {
+    sink.synchronized {
+      sink += value
+    }
+  }
+
+  def getAndClearValues: List[JTuple2[JBool, Row]] = {
+    val out = sink.toList
+    sink.clear()
+    out
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
index 79e957a677a00..8626b07410a1d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
@@ -39,9 +39,4 @@ class MockTableEnvironment extends TableEnvironment(new TableConfig) {
 
   override protected def getBuiltInPhysicalOptRuleSet: RuleSet = ???
 
-  override protected def getConversionMapper[IN, OUT](
-      physicalTypeInfo: TypeInformation[IN],
-      logicalRowType: RelDataType,
-      requestedTypeInfo: TypeInformation[OUT],
-      functionName: String) = ???
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
index 73bc2f8ab07ee..0e6d4617b874a 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
@@ -211,7 +211,7 @@ case class StreamTableTestUtil() extends TableTestUtil {
 
   def verifyTable(resultTable: Table, expected: String): Unit = {
     val relNode = resultTable.getRelNode
-    val optimized = tEnv.optimize(relNode)
+    val optimized = tEnv.optimize(relNode, updatesAsRetraction = false)
     val actual = RelOptUtil.toString(optimized)
     assertEquals(
       expected.split("\n").map(_.trim).mkString("\n"),
@@ -221,7 +221,7 @@ case class StreamTableTestUtil() extends TableTestUtil {
   // the print methods are for debugging purposes only
   def printTable(resultTable: Table): Unit = {
     val relNode = resultTable.getRelNode
-    val optimized = tEnv.optimize(relNode)
+    val optimized = tEnv.optimize(relNode, updatesAsRetraction = false)
     println(RelOptUtil.toString(optimized))
   }
 

From 2ec90d2b95a041fd7a0a6f72aa3e0b82efdbc325 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 4 May 2017 11:25:24 +0200
Subject: [PATCH 162/253] [hotfix] [build] Updated outdated config keys in
 default flink-conf.yaml

---
 flink-dist/src/main/resources/flink-conf.yaml | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/flink-dist/src/main/resources/flink-conf.yaml b/flink-dist/src/main/resources/flink-conf.yaml
index 446d99280ff46..12c90e176943a 100644
--- a/flink-dist/src/main/resources/flink-conf.yaml
+++ b/flink-dist/src/main/resources/flink-conf.yaml
@@ -165,15 +165,25 @@ jobmanager.web.port: 8081
 # High Availability
 #==============================================================================
 
+# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
+#
+# high-availability: zookeeper
+
+# The path where metadata for master recovery is persisted. While ZooKeeper stored
+# the small ground truth for checkpoint and leader election, this location stores
+# the larger objects, like persisted dataflow graphs.
+# 
+# Must be a durable file system that is accessible from all nodes
+# (like HDFS, S3, Ceph, nfs, ...) 
+#
+# high-availability.storageDir: hdfs:///flink/ha/
+
 # The list of ZooKeeper quorum peers that coordinate the high-availability
 # setup. This must be a list of the form:
-#
 # "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
 #
-
-# high-availability: zookeeper
 # high-availability.zookeeper.quorum: localhost:2181
-# high-availability.zookeeper.storageDir: hdfs:///flink/ha/
+
 
 # ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
 # It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)

From 1c9ad361efb6c0f782137a2136cd2f6bccb09aa7 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 4 May 2017 12:47:47 +0200
Subject: [PATCH 163/253] [FLINK-6401] [rocksdb] Harden RocksDB performance
 test

---
 .../benchmark/RocksDBPerformanceTest.java      | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
index 011703ef01a6a..71475830d9baf 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
@@ -19,6 +19,8 @@
 package org.apache.flink.contrib.streaming.state.benchmark;
 
 import org.apache.flink.core.memory.MemoryUtils;
+import org.apache.flink.testutils.junit.RetryOnFailure;
+import org.apache.flink.testutils.junit.RetryRule;
 import org.apache.flink.util.TestLogger;
 
 import org.junit.Rule;
@@ -26,6 +28,7 @@
 import org.junit.rules.TemporaryFolder;
 
 import org.rocksdb.CompactionStyle;
+import org.rocksdb.NativeLibraryLoader;
 import org.rocksdb.Options;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksIterator;
@@ -46,9 +49,16 @@ public class RocksDBPerformanceTest extends TestLogger {
 	@Rule
 	public final TemporaryFolder TMP = new TemporaryFolder();
 
+	@Rule
+	public final RetryRule retry = new RetryRule();
+
 	@Test(timeout = 2000)
+	@RetryOnFailure(times = 3)
 	public void testRocksDbMergePerformance() throws Exception {
-		final File rocksDir = TMP.newFolder("rdb");
+		final File rocksDir = TMP.newFolder();
+
+		// ensure the RocksDB library is loaded to a distinct location each retry
+		NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());
 
 		final String key = "key";
 		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
@@ -121,8 +131,12 @@ public void testRocksDbMergePerformance() throws Exception {
 	}
 
 	@Test(timeout = 2000)
+	@RetryOnFailure(times = 3)
 	public void testRocksDbRangeGetPerformance() throws Exception {
-		final File rocksDir = TMP.newFolder("rdb");
+		final File rocksDir = TMP.newFolder();
+
+		// ensure the RocksDB library is loaded to a distinct location each retry
+		NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());
 
 		final String key = "key";
 		final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

From 4aadba06a4ce52ebcd93764a4f5d368dd3f42087 Mon Sep 17 00:00:00 2001
From: Nico Kruber <nico@data-artisans.com>
Date: Tue, 11 Apr 2017 14:20:40 +0200
Subject: [PATCH 164/253] [hotfix] fix typo in error message

---
 .../org/apache/flink/api/common/io/DelimitedInputFormat.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java b/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
index 3dc567ea24d43..e20f646b701a4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
@@ -450,7 +450,7 @@ public FileBaseStatistics getStatistics(BaseStatistics cachedStats) throws IOExc
 		}
 		catch (Throwable t) {
 			if (LOG.isErrorEnabled()) {
-				LOG.error("Unexpected problen while getting the file statistics for file '" + this.filePath + "': "
+				LOG.error("Unexpected problem while getting the file statistics for file '" + this.filePath + "': "
 						+ t.getMessage(), t);
 			}
 		} finally {

From 20003334c459e8317ed2ed6bfeb68b1f00c06b6f Mon Sep 17 00:00:00 2001
From: Nico Kruber <nico@data-artisans.com>
Date: Tue, 11 Apr 2017 15:29:41 +0200
Subject: [PATCH 165/253] [hotfix] fix typo in taskmanager.sh usage string

---
 flink-dist/src/main/flink-bin/bin/taskmanager.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-dist/src/main/flink-bin/bin/taskmanager.sh b/flink-dist/src/main/flink-bin/bin/taskmanager.sh
index 3bac51f54e076..8431408ef7941 100755
--- a/flink-dist/src/main/flink-bin/bin/taskmanager.sh
+++ b/flink-dist/src/main/flink-bin/bin/taskmanager.sh
@@ -18,7 +18,7 @@
 ################################################################################
 
 # Start/stop a Flink TaskManager.
-USAGE="Usage: taskmanager.sh start|start-foreground|stop|stop-all)"
+USAGE="Usage: taskmanager.sh (start|start-foreground|stop|stop-all)"
 
 STARTSTOP=$1
 

From b81d79baaffd80c87d8102d0549d9fbf28e65fb9 Mon Sep 17 00:00:00 2001
From: Nico Kruber <nico@data-artisans.com>
Date: Thu, 6 Apr 2017 14:41:52 +0200
Subject: [PATCH 166/253] [FLINK-4545] [network] replace the network buffers
 parameter

Instead, allow the configuration with the following three new (more flexible)
parameters:
 * "taskmanager.network.memory.fraction": fraction of JVM memory to use for network buffers (default: 0.1)
 * "taskmanager.network.memory.min": minimum memory size for network buffers (default: 64 MB)
 * "taskmanager.network.memory.max": maximum memory size for network buffers (default: 1 GB)

This closes #3721
---
 docs/setup/config.md                          |  80 ++++-
 docs/setup/yarn_setup.md                      |   2 +-
 .../addons/hbase/HBaseConnectorITCase.java    |  37 +++
 .../configuration/TaskManagerOptions.java     |  20 ++
 flink-dist/pom.xml                            |  24 ++
 flink-dist/src/main/flink-bin/bin/config.sh   | 133 ++++++++
 .../src/main/flink-bin/bin/taskmanager.sh     |  31 +-
 flink-dist/src/test/bin/calcTMHeapSizeMB.sh   |  42 +++
 flink-dist/src/test/bin/calcTMNetBufMem.sh    |  39 +++
 ...anagerHeapSizeCalculationJavaBashTest.java | 306 ++++++++++++++++++
 .../src/test/resources/log4j-test.properties  |  35 ++
 .../clusterframework/BootstrapTools.java      |   2 +-
 .../ContaineredTaskManagerParameters.java     |  26 +-
 .../io/network/buffer/NetworkBufferPool.java  |   9 +-
 .../partition/SpillableSubpartition.java      |   6 +-
 .../minicluster/MiniClusterConfiguration.java |   9 +-
 .../taskexecutor/TaskManagerServices.java     | 235 +++++++++++++-
 .../TaskManagerServicesConfiguration.java     | 114 ++++++-
 .../minicluster/LocalFlinkMiniCluster.scala   |  11 +-
 .../NetworkEnvironmentConfiguration.scala     |   4 +-
 .../TaskManagerServicesConfigurationTest.java | 111 +++++++
 .../taskexecutor/TaskManagerServicesTest.java | 289 +++++++++++++++++
 ...kManagerComponentsStartupShutdownTest.java |   9 +-
 ...actEventTimeWindowCheckpointingITCase.java |   2 +
 .../apache/flink/yarn/cli/FlinkYarnCLI.java   |   2 +-
 .../flink/yarn/cli/FlinkYarnSessionCli.java   |   2 +-
 26 files changed, 1477 insertions(+), 103 deletions(-)
 create mode 100755 flink-dist/src/test/bin/calcTMHeapSizeMB.sh
 create mode 100755 flink-dist/src/test/bin/calcTMNetBufMem.sh
 create mode 100644 flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java
 create mode 100644 flink-dist/src/test/resources/log4j-test.properties
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfigurationTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java

diff --git a/docs/setup/config.md b/docs/setup/config.md
index a3daac000b7c7..c4a7354ea7e77 100644
--- a/docs/setup/config.md
+++ b/docs/setup/config.md
@@ -82,13 +82,13 @@ without explicit scheme definition, such as `/user/USERNAME/in.txt`, is going to
 
 ### Managed Memory
 
-By default, Flink allocates a fraction of `0.7` of the total memory configured via `taskmanager.heap.mb` for its managed memory. Managed memory helps Flink to run the batch operators efficiently. It prevents `OutOfMemoryException`s because Flink knows how much memory it can use to execute operations. If Flink runs out of managed memory, it utilizes disk space. Using managed memory, some operations can be performed directly on the raw data without having to deserialize the data to convert it into Java objects. All in all, managed memory improves the robustness and speed of the system.
+By default, Flink allocates a fraction of `0.7` of the free memory (total memory configured via `taskmanager.heap.mb` minus memory used for network buffers) for its managed memory. Managed memory helps Flink to run the batch operators efficiently. It prevents `OutOfMemoryException`s because Flink knows how much memory it can use to execute operations. If Flink runs out of managed memory, it utilizes disk space. Using managed memory, some operations can be performed directly on the raw data without having to deserialize the data to convert it into Java objects. All in all, managed memory improves the robustness and speed of the system.
 
 The default fraction for managed memory can be adjusted using the `taskmanager.memory.fraction` parameter. An absolute value may be set using `taskmanager.memory.size` (overrides the fraction parameter). If desired, the managed memory may be allocated outside the JVM heap. This may improve performance in setups with large memory sizes.
 
 - `taskmanager.memory.size`: The amount of memory (in megabytes) that the task manager reserves on-heap or off-heap (depending on `taskmanager.memory.off-heap`) for sorting, hash tables, and caching of intermediate results. If unspecified (-1), the memory manager will take a fixed ratio with respect to the size of the task manager JVM as specified by `taskmanager.memory.fraction`. (DEFAULT: -1)
 
-- `taskmanager.memory.fraction`: The relative amount of memory (with respect to `taskmanager.heap.mb`) that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of `0.8` means that a task manager reserves 80% of its memory (on-heap or off-heap depending on `taskmanager.memory.off-heap`) for internal data buffers, leaving 20% of free memory for the task manager's heap for objects created by user-defined functions. (DEFAULT: 0.7) This parameter is only evaluated, if `taskmanager.memory.size` is not set.
+- `taskmanager.memory.fraction`: The relative amount of memory (with respect to `taskmanager.heap.mb`, after subtracting the amount of memory used by network buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of `0.8` means that a task manager reserves 80% of its memory (on-heap or off-heap depending on `taskmanager.memory.off-heap`) for internal data buffers, leaving 20% of free memory for the task manager's heap for objects created by user-defined functions. (DEFAULT: 0.7) This parameter is only evaluated, if `taskmanager.memory.size` is not set.
 
 - `taskmanager.memory.off-heap`: If set to `true`, the task manager allocates memory which is used for sorting, hash tables, and caching of intermediate results outside of the JVM heap. For setups with larger quantities of memory, this can improve the efficiency of the operations performed on the memory (DEFAULT: false).
 
@@ -174,7 +174,11 @@ will be used under the directory specified by jobmanager.web.tmpdir.
 
 - `fs.output.always-create-directory`: File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to *true*, writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to *false*, the writer will directly create the file directly at the output path, without creating a containing directory. (DEFAULT: false)
 
-- `taskmanager.network.numberOfBuffers`: The number of buffers available to the network stack. This number determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value (DEFAULT: 2048).
+- `taskmanager.network.memory.fraction`: Fraction of JVM memory to use for network buffers. This determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value or the min/max values below. (DEFAULT: 0.1)
+
+- `taskmanager.network.memory.min`: Minimum memory size for network buffers in bytes (DEFAULT: 64 MB)
+
+- `taskmanager.network.memory.max`: Maximum memory size for network buffers in bytes (DEFAULT: 1 GB)
 
 - `state.backend`: The backend that will be used to store operator state checkpoints if checkpointing is enabled. Supported backends:
    -  `jobmanager`: In-memory state, backup to JobManager's/ZooKeeper's memory. Should be used only for minimal state (Kafka offsets) or testing and local debugging.
@@ -257,11 +261,17 @@ The following parameters configure Flink's JobManager and TaskManagers.
 
 - `taskmanager.tmp.dirs`: The directory for temporary files, or a list of directories separated by the system's directory delimiter (for example ':' (colon) on Linux/Unix). If multiple directories are specified, then the temporary files will be distributed across the directories in a round robin fashion. The I/O manager component will spawn one reading and one writing thread per directory. A directory may be listed multiple times to have the I/O manager use multiple threads for it (for example if it is physically stored on a very fast disc or RAID) (DEFAULT: **The system's tmp dir**).
 
-- `taskmanager.network.numberOfBuffers`: The number of buffers available to the network stack. This number determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value (DEFAULT: **2048**).
+- `taskmanager.network.memory.fraction`: Fraction of JVM memory to use for network buffers. This determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value or the min/max values below. Also note, that `taskmanager.network.memory.min` and `taskmanager.network.memory.max` may override this fraction. (DEFAULT: **0.1**)
+
+- `taskmanager.network.memory.min`: Minimum memory size for network buffers in bytes (DEFAULT: **64 MB**). Previously, this was determined from `taskmanager.network.numberOfBuffers` and `taskmanager.memory.segment-size`.
+
+- `taskmanager.network.memory.max`: Maximum memory size for network buffers in bytes (DEFAULT: **1 GB**). Previously, this was determined from `taskmanager.network.numberOfBuffers` and `taskmanager.memory.segment-size`.
+
+- `taskmanager.network.numberOfBuffers` (deprecated, replaced by the three parameters above): The number of buffers available to the network stack. This number determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value (DEFAULT: **2048**). If set, it will be mapped to `taskmanager.network.memory.min` and `taskmanager.network.memory.max` based on `taskmanager.memory.segment-size`.
 
 - `taskmanager.memory.size`: The amount of memory (in megabytes) that the task manager reserves on the JVM's heap space for sorting, hash tables, and caching of intermediate results. If unspecified (-1), the memory manager will take a fixed ratio of the heap memory available to the JVM, as specified by `taskmanager.memory.fraction`. (DEFAULT: **-1**)
 
-- `taskmanager.memory.fraction`: The relative amount of memory that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of 0.8 means that TaskManagers reserve 80% of the JVM's heap space for internal data buffers, leaving 20% of the JVM's heap space free for objects created by user-defined functions. (DEFAULT: **0.7**) This parameter is only evaluated, if `taskmanager.memory.size` is not set.
+- `taskmanager.memory.fraction`: The relative amount of memory (with respect to `taskmanager.heap.mb`, after subtracting the amount of memory used by network buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of `0.8` means that a task manager reserves 80% of its memory (on-heap or off-heap depending on `taskmanager.memory.off-heap`) for internal data buffers, leaving 20% of free memory for the task manager's heap for objects created by user-defined functions. (DEFAULT: 0.7) This parameter is only evaluated, if `taskmanager.memory.size` is not set.
 
 - `taskmanager.debug.memory.startLogThread`: Causes the TaskManagers to periodically log memory and Garbage collection statistics. The statistics include current heap-, off-heap, and other memory pool utilization, as well as the time spent on garbage collection, by heap memory pool.
 
@@ -614,9 +624,54 @@ You have to configure `jobmanager.archive.fs.dir` in order to archive terminated
 
 ## Background
 
+
 ### Configuring the Network Buffers
 
-If you ever see the Exception `java.io.IOException: Insufficient number of network buffers`, please use the following formula to adjust the number of network buffers:
+If you ever see the Exception `java.io.IOException: Insufficient number of network buffers`, you
+need to adapt the amount of memory used for network buffers in order for your program to run on your
+task managers.
+
+Network buffers are a critical resource for the communication layers. They are used to buffer
+records before transmission over a network, and to buffer incoming data before dissecting it into
+records and handing them to the application. A sufficient number of network buffers is critical to
+achieve a good throughput.
+
+<div class="alert alert-info">
+Since Flink 1.3, you may follow the idiom "more is better" without any penalty on the latency (we
+prevent excessive buffering in each outgoing and incoming channel, i.e. *buffer bloat*, by limiting
+the actual number of buffers used by each channel).
+</div>
+
+In general, configure the task manager to have enough buffers that each logical network connection
+you expect to be open at the same time has a dedicated buffer. A logical network connection exists
+for each point-to-point exchange of data over the network, which typically happens at
+repartitioning or broadcasting steps (shuffle phase). In those, each parallel task inside the
+TaskManager has to be able to talk to all other parallel tasks.
+
+#### Setting Memory Fractions
+
+Previously, the number of network buffers was set manually which became a quite error-prone task
+(see below). Since Flink 1.3, it is possible to define a fraction of memory that is being used for
+network buffers with the following configuration parameters:
+
+- `taskmanager.network.memory.fraction`: Fraction of JVM memory to use for network buffers (DEFAULT: 0.1),
+- `taskmanager.network.memory.min`: Minimum memory size for network buffers in bytes (DEFAULT: 64 MB),
+- `taskmanager.network.memory.max`: Maximum memory size for network buffers in bytes (DEFAULT: 1 GB), and
+- `taskmanager.memory.segment-size`: Size of memory buffers used by the memory manager and the
+network stack in bytes (DEFAULT: 32768 (= 32 KiBytes)).
+
+#### Setting the Number of Network Buffers directly
+
+<div class="alert alert-warning">
+  <strong>Note:</strong> This way of configuring the amount of memory used for network buffers is deprecated. Please consider using the method above by defining a fraction of memory to use.
+</div>
+
+The required number of buffers on a task manager is
+*total-degree-of-parallelism* (number of targets) \* *intra-node-parallelism* (number of sources in one task manager) \* *n*
+with *n* being a constant that defines how many repartitioning-/broadcasting steps you expect to be
+active at the same time. Since the *intra-node-parallelism* is typically the number of cores, and
+more than 4 repartitioning or broadcasting channels are rarely active in parallel, it frequently
+boils down to
 
 ```
 #slots-per-TM^2 * #TMs * 4
@@ -624,16 +679,11 @@ If you ever see the Exception `java.io.IOException: Insufficient number of netwo
 
 Where `#slots per TM` are the [number of slots per TaskManager](#configuring-taskmanager-processing-slots) and `#TMs` are the total number of task managers.
 
-Network buffers are a critical resource for the communication layers. They are used to buffer records before transmission over a network, and to buffer incoming data before dissecting it into records and handing them to the
-application. A sufficient number of network buffers is critical to achieve a good throughput.
-
-In general, configure the task manager to have enough buffers that each logical network connection you expect to be open at the same time has a dedicated buffer. A logical network connection exists for each point-to-point exchange of data over the network, which typically happens at repartitioning- or broadcasting steps (shuffle phase). In those, each parallel task inside the TaskManager has to be able to talk to all other parallel tasks. Hence, the required number of buffers on a task manager is *total-degree-of-parallelism* (number of targets) \* *intra-node-parallelism* (number of sources in one task manager) \* *n*. Here, *n* is a constant that defines how many repartitioning-/broadcasting steps you expect to be active at the same time.
-
-Since the *intra-node-parallelism* is typically the number of cores, and more than 4 repartitioning or broadcasting channels are rarely active in parallel, it frequently boils down to `#slots-per-TM^2 * #TMs * 4`.
-
-To support for example a cluster of 20 8-slot machines, you should use roughly 5000 network buffers for optimal throughput.
+To support, for example, a cluster of 20 8-slot machines, you should use roughly 5000 network
+buffers for optimal throughput.
 
-Each network buffer has by default a size of 32 KiBytes. In the above example, the system would allocate roughly 300 MiBytes for network buffers.
+Each network buffer has by default a size of 32 KiBytes. In the example above, the system would thus
+allocate roughly 300 MiBytes for network buffers.
 
 The number and size of network buffers can be configured with the following parameters:
 
diff --git a/docs/setup/yarn_setup.md b/docs/setup/yarn_setup.md
index 3149ec25ef9b4..1ce45ad9c73a3 100644
--- a/docs/setup/yarn_setup.md
+++ b/docs/setup/yarn_setup.md
@@ -122,7 +122,7 @@ The system will use the configuration in `conf/flink-conf.yaml`. Please follow o
 
 Flink on YARN will overwrite the following configuration parameters `jobmanager.rpc.address` (because the JobManager is always allocated at different machines), `taskmanager.tmp.dirs` (we are using the tmp directories given by YARN) and `parallelism.default` if the number of slots has been specified.
 
-If you don't want to change the configuration file to set configuration parameters, there is the option to pass dynamic properties via the `-D` flag. So you can pass parameters this way: `-Dfs.overwrite-files=true -Dtaskmanager.network.numberOfBuffers=16368`.
+If you don't want to change the configuration file to set configuration parameters, there is the option to pass dynamic properties via the `-D` flag. So you can pass parameters this way: `-Dfs.overwrite-files=true -Dtaskmanager.network.memory.min=536346624`.
 
 The example invocation starts 11 containers (even though only 10 containers were requested), since there is one additional container for the ApplicationMaster and Job Manager.
 
diff --git a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseConnectorITCase.java b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseConnectorITCase.java
index c1aa9a0eebe03..33bbe12fc03b5 100644
--- a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseConnectorITCase.java
+++ b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseConnectorITCase.java
@@ -22,7 +22,11 @@
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.ExecutionEnvironmentFactory;
+import org.apache.flink.api.java.LocalEnvironment;
 import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.TableConfig;
 import org.apache.flink.table.api.TableEnvironment;
@@ -36,6 +40,7 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -74,6 +79,12 @@ public class HBaseConnectorITCase extends HBaseTestingClusterAutostarter {
 	public static void activateHBaseCluster() throws IOException {
 		registerHBaseMiniClusterInClasspath();
 		prepareTable();
+		LimitNetworkBuffersTestEnvironment.setAsContext();
+	}
+
+	@AfterClass
+	public static void resetExecutionEnvironmentFactory() {
+		LimitNetworkBuffersTestEnvironment.unsetAsContext();
 	}
 
 	private static void prepareTable() throws IOException {
@@ -335,4 +346,30 @@ public Tuple1<Integer> reduce(Tuple1<Integer> v1, Tuple1<Integer> v2) throws Exc
 	}
 
 
+	/**
+	 * Allows the tests to use {@link ExecutionEnvironment#getExecutionEnvironment()} but with a
+	 * configuration that limits the maximum memory used for network buffers since the current
+	 * defaults are too high for Travis-CI.
+	 */
+	private static abstract class LimitNetworkBuffersTestEnvironment extends ExecutionEnvironment {
+
+		public static void setAsContext() {
+			Configuration config = new Configuration();
+			// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
+			config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 80L << 20); // 80 MB
+			final LocalEnvironment le = new LocalEnvironment(config);
+
+			initializeContextEnvironment(new ExecutionEnvironmentFactory() {
+				@Override
+				public ExecutionEnvironment createExecutionEnvironment() {
+					return le;
+				}
+			});
+		}
+
+		public static void unsetAsContext() {
+			resetContextEnvironment();
+		}
+	}
+
 }
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
index e915c0bde6117..c5063d145a1fe 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
@@ -85,11 +85,30 @@ public class TaskManagerOptions {
 	/**
 	 * Number of buffers used in the network stack. This defines the number of possible tasks and
 	 * shuffles.
+	 *
+	 * @deprecated use {@link #NETWORK_BUFFERS_MEMORY_FRACTION}, {@link #NETWORK_BUFFERS_MEMORY_MIN},
+	 * and {@link #NETWORK_BUFFERS_MEMORY_MAX} instead
 	 */
+	@Deprecated
 	public static final ConfigOption<Integer> NETWORK_NUM_BUFFERS =
 			key("taskmanager.network.numberOfBuffers")
 			.defaultValue(2048);
 
+	/** Fraction of JVM memory to use for network buffers. */
+	public static final ConfigOption<Float> NETWORK_BUFFERS_MEMORY_FRACTION =
+			key("taskmanager.network.memory.fraction")
+			.defaultValue(0.1f);
+
+	/** Minimum memory size for network buffers (in bytes) */
+	public static final ConfigOption<Long> NETWORK_BUFFERS_MEMORY_MIN =
+			key("taskmanager.network.memory.min")
+			.defaultValue(64L << 20); // 64 MB
+
+	/** Maximum memory size for network buffers (in bytes) */
+	public static final ConfigOption<Long> NETWORK_BUFFERS_MEMORY_MAX =
+			key("taskmanager.network.memory.max")
+			.defaultValue(1024L << 20); // 1 GB
+
 
 	/** Minimum backoff for partition requests of input channels. */
 	public static final ConfigOption<Integer> NETWORK_REQUEST_BACKOFF_INITIAL =
@@ -101,6 +120,7 @@ public class TaskManagerOptions {
 			key("taskmanager.net.request-backoff.max")
 			.defaultValue(10000);
 
+
 	/**
 	 * Number of network buffers to use for each outgoing/ingoing channel (subpartition/input channel).
 	 *
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index ff87d537dda67..5f6a39bcc47ca 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -254,6 +254,16 @@ under the License.
 			<scope>provided</scope>
 		</dependency>
 		<!-- end optional Flink libraries -->
+
+		<!-- test dependencies -->
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-test-utils-junit</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<!-- end test dependencies -->
 	</dependencies>
 
 	<profiles>
@@ -312,6 +322,20 @@ under the License.
 	<build>
 		<plugins>
 
+			<!--unit tests-->
+			<!--plugin must appear BEFORE the shade-plugin to not mess up package order and include the non-uber JAR into the assembly-->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<configuration>
+					<reuseForks>false</reuseForks>
+					<!-- <workingDirectory>${project.build.testOutputDirectory}</workingDirectory> -->
+					<systemPropertyVariables>
+						<log.level>WARN</log.level>
+					</systemPropertyVariables>
+				</configuration>
+			</plugin>
+
 			<!-- binary compatibility checks -->
 			<plugin>
 				<groupId>com.github.siom79.japicmp</groupId>
diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index 08944818fcbc5..7e3c1d452fa54 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -108,6 +108,11 @@ KEY_TASKM_MEM_MANAGED_FRACTION="taskmanager.memory.fraction"
 KEY_TASKM_OFFHEAP="taskmanager.memory.off-heap"
 KEY_TASKM_MEM_PRE_ALLOCATE="taskmanager.memory.preallocate"
 
+KEY_TASKM_NET_BUF_FRACTION="taskmanager.network.memory.fraction"
+KEY_TASKM_NET_BUF_MIN="taskmanager.network.memory.min"
+KEY_TASKM_NET_BUF_MAX="taskmanager.network.memory.max"
+KEY_TASKM_NET_BUF_NR="taskmanager.network.numberOfBuffers" # fallback
+
 KEY_TASKM_COMPUTE_NUMA="taskmanager.compute.numa"
 
 KEY_ENV_PID_DIR="env.pid.dir"
@@ -231,6 +236,31 @@ if [ -z "${FLINK_TM_MEM_PRE_ALLOCATE}" ]; then
     FLINK_TM_MEM_PRE_ALLOCATE=$(readFromConfig ${KEY_TASKM_MEM_PRE_ALLOCATE} "false" "${YAML_CONF}")
 fi
 
+
+# Define FLINK_TM_NET_BUF_FRACTION if it is not already set
+if [ -z "${FLINK_TM_NET_BUF_FRACTION}" ]; then
+    FLINK_TM_NET_BUF_FRACTION=$(readFromConfig ${KEY_TASKM_NET_BUF_FRACTION} 0.1 "${YAML_CONF}")
+fi
+
+# Define FLINK_TM_NET_BUF_MIN and FLINK_TM_NET_BUF_MAX if not already set (as a fallback)
+if [ -z "${FLINK_TM_NET_BUF_MIN}" -a -z "${FLINK_TM_NET_BUF_MAX}" ]; then
+    FLINK_TM_NET_BUF_MIN=$(readFromConfig ${KEY_TASKM_NET_BUF_NR} -1 "${YAML_CONF}")
+    FLINK_TM_NET_BUF_MAX=${FLINK_TM_NET_BUF_MIN}
+fi
+
+# Define FLINK_TM_NET_BUF_MIN if it is not already set
+if [ -z "${FLINK_TM_NET_BUF_MIN}" -o "${FLINK_TM_NET_BUF_MIN}" = "-1" ]; then
+    # default: 64MB = 67108864 bytes (same as the previous default with 2048 buffers of 32k each)
+    FLINK_TM_NET_BUF_MIN=$(readFromConfig ${KEY_TASKM_NET_BUF_MIN} 67108864 "${YAML_CONF}")
+fi
+
+# Define FLINK_TM_NET_BUF_MAX if it is not already set
+if [ -z "${FLINK_TM_NET_BUF_MAX}" -o "${FLINK_TM_NET_BUF_MAX}" = "-1" ]; then
+    # default: 1GB = 1073741824 bytes
+    FLINK_TM_NET_BUF_MAX=$(readFromConfig ${KEY_TASKM_NET_BUF_MAX} 1073741824 "${YAML_CONF}")
+fi
+
+
 # Verify that NUMA tooling is available
 command -v numactl >/dev/null 2>&1
 if [[ $? -ne 0 ]]; then
@@ -463,3 +493,106 @@ TMSlaves() {
 useOffHeapMemory() {
     [[ "`echo ${FLINK_TM_OFFHEAP} | tr '[:upper:]' '[:lower:]'`" == "true" ]]
 }
+
+HAVE_AWK=
+# same as org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateNetworkBufferMemory(long totalJavaMemorySize, Configuration config)
+calculateNetworkBufferMemory() {
+    local network_buffers_bytes
+    if [ "${FLINK_TM_HEAP}" -le "0" ]; then
+        echo "Variable 'FLINK_TM_HEAP' not set (usually read from '${KEY_TASKM_MEM_SIZE}' in ${FLINK_CONF_FILE})."
+        exit 1
+    fi
+
+    if [[ "${FLINK_TM_NET_BUF_MIN}" = "${FLINK_TM_NET_BUF_MAX}" ]]; then
+        # fix memory size for network buffers
+        network_buffers_bytes=${FLINK_TM_NET_BUF_MIN}
+    else
+        if [[ "${FLINK_TM_NET_BUF_MIN}" -gt "${FLINK_TM_NET_BUF_MAX}" ]]; then
+            echo "[ERROR] Configured TaskManager network buffer memory min/max '${FLINK_TM_NET_BUF_MIN}'/'${FLINK_TM_NET_BUF_MAX}' are not valid."
+            echo "Min must be less than or equal to max."
+            echo "Please set '${KEY_TASKM_NET_BUF_MIN}' and '${KEY_TASKM_NET_BUF_MAX}' in ${FLINK_CONF_FILE}."
+            exit 1
+        fi
+
+        # Bash only performs integer arithmetic so floating point computation is performed using awk
+        if [[ -z "${HAVE_AWK}" ]] ; then
+            command -v awk >/dev/null 2>&1
+            if [[ $? -ne 0 ]]; then
+                echo "[ERROR] Program 'awk' not found."
+                echo "Please install 'awk' or define '${KEY_TASKM_NET_BUF_MIN}' and '${KEY_TASKM_NET_BUF_MAX}' instead of '${KEY_TASKM_NET_BUF_FRACTION}' in ${FLINK_CONF_FILE}."
+                exit 1
+            fi
+            HAVE_AWK=true
+        fi
+
+        # We calculate the memory using a fraction of the total memory
+        if [[ `awk '{ if ($1 > 0.0 && $1 < 1.0) print "1"; }' <<< "${FLINK_TM_NET_BUF_FRACTION}"` != "1" ]]; then
+            echo "[ERROR] Configured TaskManager network buffer memory fraction '${FLINK_TM_NET_BUF_FRACTION}' is not a valid value."
+            echo "It must be between 0.0 and 1.0."
+            echo "Please set '${KEY_TASKM_NET_BUF_FRACTION}' in ${FLINK_CONF_FILE}."
+            exit 1
+        fi
+
+        network_buffers_bytes=`awk "BEGIN { x = lshift(${FLINK_TM_HEAP},20) * ${FLINK_TM_NET_BUF_FRACTION}; netbuf = x > ${FLINK_TM_NET_BUF_MAX} ? ${FLINK_TM_NET_BUF_MAX} : x < ${FLINK_TM_NET_BUF_MIN} ? ${FLINK_TM_NET_BUF_MIN} : x; printf \"%.0f\n\", netbuf }"`
+    fi
+
+    # recalculate the JVM heap memory by taking the network buffers into account
+    local tm_heap_size_bytes=$((${FLINK_TM_HEAP} << 20)) # megabytes to bytes
+    if [[ "${tm_heap_size_bytes}" -le "${network_buffers_bytes}" ]]; then
+        echo "[ERROR] Configured TaskManager memory size (${FLINK_TM_HEAP} MB, from '${KEY_TASKM_MEM_SIZE}') must be larger than the network buffer memory size (${network_buffers_bytes} bytes, from: '${KEY_TASKM_NET_BUF_FRACTION}', '${KEY_TASKM_NET_BUF_MIN}', '${KEY_TASKM_NET_BUF_MAX}', and '${KEY_TASKM_NET_BUF_NR}')."
+        exit 1
+    fi
+
+    echo ${network_buffers_bytes}
+}
+
+# same as org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config)
+calculateTaskManagerHeapSizeMB() {
+    if [ "${FLINK_TM_HEAP}" -le "0" ]; then
+        echo "Variable 'FLINK_TM_HEAP' not set (usually read from '${KEY_TASKM_MEM_SIZE}' in ${FLINK_CONF_FILE})."
+        exit 1
+    fi
+
+    local tm_heap_size_mb=${FLINK_TM_HEAP}
+
+    if useOffHeapMemory; then
+
+        local network_buffers_mb=$(($(calculateNetworkBufferMemory) >> 20)) # bytes to megabytes
+        tm_heap_size_mb=$((tm_heap_size_mb - network_buffers_mb))
+
+        if [[ "${FLINK_TM_MEM_MANAGED_SIZE}" -gt "0" ]]; then
+            # We split up the total memory in heap and off-heap memory
+            if [[ "${tm_heap_size_mb}" -le "${FLINK_TM_MEM_MANAGED_SIZE}" ]]; then
+                echo "[ERROR] Remaining TaskManager memory size (${tm_heap_size_mb} MB, from: '${KEY_TASKM_MEM_SIZE}' (${FLINK_TM_HEAP} MB) minus network buffer memory size (${network_buffers_mb} MB, from: '${KEY_TASKM_NET_BUF_FRACTION}', '${KEY_TASKM_NET_BUF_MIN}', '${KEY_TASKM_NET_BUF_MAX}', and '${KEY_TASKM_NET_BUF_NR}')) must be larger than the managed memory size (${FLINK_TM_MEM_MANAGED_SIZE} MB, from: '${KEY_TASKM_MEM_MANAGED_SIZE}')."
+                exit 1
+            fi
+
+            tm_heap_size_mb=$((tm_heap_size_mb - FLINK_TM_MEM_MANAGED_SIZE))
+        else
+            # Bash only performs integer arithmetic so floating point computation is performed using awk
+            if [[ -z "${HAVE_AWK}" ]] ; then
+                command -v awk >/dev/null 2>&1
+                if [[ $? -ne 0 ]]; then
+                    echo "[ERROR] Program 'awk' not found."
+                    echo "Please install 'awk' or define '${KEY_TASKM_MEM_MANAGED_SIZE}' instead of '${KEY_TASKM_MEM_MANAGED_FRACTION}' in ${FLINK_CONF_FILE}."
+                    exit 1
+                fi
+                HAVE_AWK=true
+            fi
+
+            # We calculate the memory using a fraction of the total memory
+            if [[ `awk '{ if ($1 > 0.0 && $1 < 1.0) print "1"; }' <<< "${FLINK_TM_MEM_MANAGED_FRACTION}"` != "1" ]]; then
+                echo "[ERROR] Configured TaskManager managed memory fraction '${FLINK_TM_MEM_MANAGED_FRACTION}' is not a valid value."
+                echo "It must be between 0.0 and 1.0."
+                echo "Please set '${KEY_TASKM_MEM_MANAGED_FRACTION}' in ${FLINK_CONF_FILE}."
+                exit 1
+            fi
+
+            # recalculate the JVM heap memory by taking the off-heap ratio into account
+            local offheap_managed_memory_size=`awk "BEGIN { printf \"%.0f\n\", ${tm_heap_size_mb} * ${FLINK_TM_MEM_MANAGED_FRACTION} }"`
+            tm_heap_size_mb=$((tm_heap_size_mb - offheap_managed_memory_size))
+        fi
+    fi
+
+    echo ${tm_heap_size_mb}
+}
diff --git a/flink-dist/src/main/flink-bin/bin/taskmanager.sh b/flink-dist/src/main/flink-bin/bin/taskmanager.sh
index 8431408ef7941..b16abc9340f1d 100755
--- a/flink-dist/src/main/flink-bin/bin/taskmanager.sh
+++ b/flink-dist/src/main/flink-bin/bin/taskmanager.sh
@@ -57,39 +57,10 @@ if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then
 
     if [ "${FLINK_TM_HEAP}" -gt "0" ]; then
 
-        TM_HEAP_SIZE=${FLINK_TM_HEAP}
+        TM_HEAP_SIZE=$(calculateTaskManagerHeapSizeMB)
         # Long.MAX_VALUE in TB: This is an upper bound, much less direct memory will be used
         TM_MAX_OFFHEAP_SIZE="8388607T"
 
-        if useOffHeapMemory; then
-            if [[ "${FLINK_TM_MEM_MANAGED_SIZE}" -gt "0" ]]; then
-                # We split up the total memory in heap and off-heap memory
-                if [[ "${FLINK_TM_HEAP}" -le "${FLINK_TM_MEM_MANAGED_SIZE}" ]]; then
-                    echo "[ERROR] Configured TaskManager memory size ('${KEY_TASKM_MEM_SIZE}') must be larger than the managed memory size ('${KEY_TASKM_MEM_MANAGED_SIZE}')."
-                    exit 1
-                fi
-                TM_HEAP_SIZE=$((FLINK_TM_HEAP - FLINK_TM_MEM_MANAGED_SIZE))
-            else
-                # Bash only performs integer arithmetic so floating point computation is performed using awk
-                command -v awk >/dev/null 2>&1
-                if [[ $? -ne 0 ]]; then
-                    echo "[ERROR] Program 'awk' not found."
-                    echo "Please install 'awk' or define '${KEY_TASKM_MEM_MANAGED_SIZE}' instead of '${KEY_TASKM_MEM_MANAGED_FRACTION}' in ${FLINK_CONF_FILE}."
-                    exit 1
-                fi
-                # We calculate the memory using a fraction of the total memory
-                if [[ `awk '{ if ($1 > 0.0 && $1 < 1.0) print "1"; }' <<< "${FLINK_TM_MEM_MANAGED_FRACTION}"` != "1" ]]; then
-                    echo "[ERROR] Configured TaskManager managed memory fraction '${FLINK_TM_MEM_MANAGED_FRACTION}' is not a valid value."
-                    echo "It must be between 0.0 and 1.0."
-                    echo "Please set '${KEY_TASKM_MEM_MANAGED_FRACTION}' in ${FLINK_CONF_FILE}."
-                    exit 1
-                fi
-                # recalculate the JVM heap memory by taking the off-heap ratio into account
-                OFFHEAP_MANAGED_MEMORY_SIZE=`awk "BEGIN { printf \"%.0f\n\", ${FLINK_TM_HEAP} * ${FLINK_TM_MEM_MANAGED_FRACTION} }"`
-                TM_HEAP_SIZE=$((FLINK_TM_HEAP - OFFHEAP_MANAGED_MEMORY_SIZE))
-            fi
-        fi
-
         export JVM_ARGS="${JVM_ARGS} -Xms${TM_HEAP_SIZE}M -Xmx${TM_HEAP_SIZE}M -XX:MaxDirectMemorySize=${TM_MAX_OFFHEAP_SIZE}"
 
     fi
diff --git a/flink-dist/src/test/bin/calcTMHeapSizeMB.sh b/flink-dist/src/test/bin/calcTMHeapSizeMB.sh
new file mode 100755
index 0000000000000..3956643a0fd33
--- /dev/null
+++ b/flink-dist/src/test/bin/calcTMHeapSizeMB.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Wrapper script to compare the TM heap size calculation of config.sh with Java
+USAGE="Usage: calcTMHeapSizeMB.sh <memTotal> <offHeap> <netBufFrac> <netBufMin> <netBufMax> <managedMemMB> <managedMemFrac>"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+FLINK_TM_HEAP=$1
+FLINK_TM_OFFHEAP=$2
+FLINK_TM_NET_BUF_FRACTION=$3
+FLINK_TM_NET_BUF_MIN=$4
+FLINK_TM_NET_BUF_MAX=$5
+FLINK_TM_MEM_MANAGED_SIZE=$6
+FLINK_TM_MEM_MANAGED_FRACTION=$7
+
+if [[ -z "${FLINK_TM_MEM_MANAGED_FRACTION}" ]]; then
+  echo "$USAGE"
+  exit 1
+fi
+
+FLINK_CONF_DIR=${bin}/../../main/resources
+. ${bin}/../../main/flink-bin/bin/config.sh
+
+calculateTaskManagerHeapSizeMB
diff --git a/flink-dist/src/test/bin/calcTMNetBufMem.sh b/flink-dist/src/test/bin/calcTMNetBufMem.sh
new file mode 100755
index 0000000000000..9948d9c0aa892
--- /dev/null
+++ b/flink-dist/src/test/bin/calcTMNetBufMem.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Wrapper script to compare the TM heap size calculation of config.sh with Java
+USAGE="Usage: calcTMNetBufMem.sh <memTotal> <netBufFrac> <netBufMin> <netBufMax>"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+FLINK_TM_HEAP=$1
+FLINK_TM_NET_BUF_FRACTION=$2
+FLINK_TM_NET_BUF_MIN=$3
+FLINK_TM_NET_BUF_MAX=$4
+
+if [[ -z "${FLINK_TM_NET_BUF_MAX}" ]]; then
+  echo "$USAGE"
+  exit 1
+fi
+
+FLINK_CONF_DIR=${bin}/../../main/resources
+. ${bin}/../../main/flink-bin/bin/config.sh
+
+calculateNetworkBufferMemory
diff --git a/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java b/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java
new file mode 100644
index 0000000000000..11d8ec7319bc6
--- /dev/null
+++ b/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.dist;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.taskexecutor.TaskManagerServices;
+import org.apache.flink.util.OperatingSystem;
+import org.apache.flink.util.TestLogger;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Random;
+
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Unit test that verifies that the task manager heap size calculation used by the bash script
+ * <tt>taskmanager.sh</tt> returns the same values as the heap size calculation of
+ * {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)}.
+ *
+ * NOTE: the shell script uses <tt>awk</tt> to perform floating-point arithmetic which uses
+ * <tt>double</tt> precision but our Java code restrains to <tt>float</tt> because we actually do
+ * not need high precision.
+ */
+public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger {
+
+	/** Key that is used by <tt>config.sh</tt>. */
+	private static final String KEY_TASKM_MEM_SIZE = "taskmanager.heap.mb";
+
+	/**
+	 * Number of tests with random values.
+	 *
+	 * NOTE: calling the external test script is slow and thus low numbers are preferred for general
+	 * testing.
+	 */
+	private static final int NUM_RANDOM_TESTS = 20;
+
+	@Before
+	public void checkOperatingSystem() {
+		Assume.assumeTrue("This test checks shell scripts which are not available on Windows.",
+			!OperatingSystem.isWindows());
+	}
+
+	/**
+	 * Tests that {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} has the same
+	 * result as the shell script.
+	 */
+	@Test
+	public void compareNetworkBufShellScriptWithJava() throws Exception {
+		int managedMemSize = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue().intValue();
+		float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue();
+
+		// manual tests from org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateHeapSizeMB()
+
+		compareNetworkBufJavaVsScript(
+			getConfig(1000, false, 0.1f, 64L << 20, 1L << 30, managedMemSize, managedMemFrac), 0.0f);
+
+		compareNetworkBufJavaVsScript(
+			getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, 10 /*MB*/, managedMemFrac), 0.0f);
+
+		compareNetworkBufJavaVsScript(
+			getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, managedMemSize, 0.1f), 0.0f);
+
+		// some automated tests with random (but valid) values
+
+		Random ran = new Random();
+		for (int i = 0; i < NUM_RANDOM_TESTS; ++i) {
+			// tolerate that values differ by 1% (due to different floating point precisions)
+			compareNetworkBufJavaVsScript(getRandomConfig(ran), 0.01f);
+		}
+	}
+
+	/**
+	 * Tests that {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} has the same
+	 * result as the shell script.
+	 */
+	@Test
+	public void compareHeapSizeShellScriptWithJava() throws Exception {
+		int managedMemSize = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue().intValue();
+		float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue();
+
+		// manual tests from org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateHeapSizeMB()
+
+		compareHeapSizeJavaVsScript(
+			getConfig(1000, false, 0.1f, 64L << 20, 1L << 30, managedMemSize, managedMemFrac), 0.0f);
+
+		compareHeapSizeJavaVsScript(
+			getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, 10 /*MB*/, managedMemFrac), 0.0f);
+
+		compareHeapSizeJavaVsScript(
+			getConfig(1000, true, 0.1f, 64L << 20, 1L << 30, managedMemSize, 0.1f), 0.0f);
+
+		// some automated tests with random (but valid) values
+
+		Random ran = new Random();
+		for (int i = 0; i < NUM_RANDOM_TESTS; ++i) {
+			// tolerate that values differ by 1% (due to different floating point precisions)
+			compareHeapSizeJavaVsScript(getRandomConfig(ran), 0.01f);
+		}
+	}
+
+	/**
+	 * Returns a flink configuration object with the given values.
+	 *
+	 * @param javaMemMB
+	 * 		total JVM memory to use (in megabytes)
+	 * @param useOffHeap
+	 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
+	 * @param netBufMemFrac
+	 * 		fraction of JVM memory to use for network buffers
+	 * @param netBufMemMin
+	 * 		minimum memory size for network buffers (in bytes)
+	 * @param netBufMemMax
+	 * 		maximum memory size for network buffers (in bytes)
+	 * @param managedMemSizeMB
+	 * 		amount of managed memory (in megabytes)
+	 * @param managedMemFrac
+	 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
+	 * 		<tt>-1</tt>)
+	 *
+	 * @return flink configuration
+	 */
+	private static Configuration getConfig(
+			final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
+			final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
+			final float managedMemFrac) {
+
+		Configuration config = new Configuration();
+
+		config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
+		config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);
+
+		config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, netBufMemMin);
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, netBufMemMax);
+
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB);
+		config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);
+
+		return config;
+	}
+
+	/**
+	 * Returns a flink configuration object with random values (only those relevant to the tests in
+	 * this class.
+	 *
+	 * @param ran  random number generator
+	 *
+	 * @return flink configuration
+	 */
+	private static Configuration getRandomConfig(final Random ran) {
+
+		float frac = Math.max(ran.nextFloat(), Float.MIN_VALUE);
+
+		// note: we are testing with integers only here to avoid overly complicated checks for
+		// overflowing or negative Long values - this should be enough for any practical scenario
+		// though
+		long min = TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue() + ran.nextInt(Integer.MAX_VALUE);
+		long max = ran.nextInt(Integer.MAX_VALUE) + min;
+
+		int javaMemMB = Math.max((int) (max >> 20), ran.nextInt(Integer.MAX_VALUE)) + 1;
+		boolean useOffHeap = ran.nextBoolean();
+
+		int managedMemSize = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue().intValue();
+		float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue();
+
+		if (ran.nextBoolean()) {
+			// use fixed-size managed memory
+			Configuration config = getConfig(javaMemMB, useOffHeap, frac, min, max, managedMemSize, managedMemFrac);
+			long totalJavaMemorySize = ((long) javaMemMB) << 20; // megabytes to bytes
+			final int networkBufMB =
+				(int) (TaskManagerServices.calculateNetworkBufferMemory(totalJavaMemorySize, config) >> 20);
+			// max (exclusive): total - netbuf
+			managedMemSize = Math.min(javaMemMB - networkBufMB - 1, ran.nextInt(Integer.MAX_VALUE));
+		} else {
+			// use fraction of given memory
+			managedMemFrac = Math.max(ran.nextFloat(), Float.MIN_VALUE);
+		}
+
+		return getConfig(javaMemMB, useOffHeap, frac, min, max, managedMemSize, managedMemFrac);
+	}
+
+	// Helper functions
+
+	/**
+	 * Calculates the heap size via
+	 * {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} and the shell script
+	 * and verifies that these are equal.
+	 *
+	 * @param config     flink configuration
+	 * @param tolerance  tolerate values that are off by this factor (0.01 = 1%)
+	 */
+	private void compareNetworkBufJavaVsScript(final Configuration config, final float tolerance)
+			throws IOException {
+
+		final long totalJavaMemorySizeMB = config.getLong(KEY_TASKM_MEM_SIZE, 0L);
+
+		long javaNetworkBufMem = TaskManagerServices.calculateNetworkBufferMemory(totalJavaMemorySizeMB << 20, config);
+
+		String[] command = {"src/test/bin/calcTMNetBufMem.sh",
+			String.valueOf(totalJavaMemorySizeMB),
+			String.valueOf(config.getFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION)),
+			String.valueOf(config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN)),
+			String.valueOf(config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX))};
+		String scriptOutput = executeScript(command);
+
+		long absoluteTolerance = (long) (javaNetworkBufMem * tolerance);
+		if (absoluteTolerance < 1) {
+			assertEquals(
+				"Different network buffer memory sizes with configuration: " + config.toString(),
+				String.valueOf(javaNetworkBufMem), scriptOutput);
+		} else {
+			Long scriptNetworkBufMem = Long.valueOf(scriptOutput);
+			assertThat(
+				"Different network buffer memory sizes (Java: " + javaNetworkBufMem + ", Script: " + scriptNetworkBufMem +
+					") with configuration: " + config.toString(), scriptNetworkBufMem,
+				allOf(greaterThanOrEqualTo(javaNetworkBufMem - absoluteTolerance),
+					lessThanOrEqualTo(javaNetworkBufMem + absoluteTolerance)));
+		}
+	}
+
+	/**
+	 * Calculates the heap size via
+	 * {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} and the shell script
+	 * and verifies that these are equal.
+	 *
+	 * @param config     flink configuration
+	 * @param tolerance  tolerate values that are off by this factor (0.01 = 1%)
+	 */
+	private void compareHeapSizeJavaVsScript(final Configuration config, float tolerance)
+			throws IOException {
+
+		final long totalJavaMemorySizeMB = config.getLong(KEY_TASKM_MEM_SIZE, 0L);
+
+		long javaHeapSizeMB = TaskManagerServices.calculateHeapSizeMB(totalJavaMemorySizeMB, config);
+
+		String[] command = {"src/test/bin/calcTMHeapSizeMB.sh",
+			String.valueOf(totalJavaMemorySizeMB),
+			String.valueOf(config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)),
+			String.valueOf(config.getFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION)),
+			String.valueOf(config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN)),
+			String.valueOf(config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX)),
+			String.valueOf(config.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE)),
+			String.valueOf(config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION))};
+		String scriptOutput = executeScript(command);
+
+		long absoluteTolerance = (long) (javaHeapSizeMB * tolerance);
+		if (absoluteTolerance < 1) {
+			assertEquals("Different heap sizes with configuration: " + config.toString(),
+				String.valueOf(javaHeapSizeMB), scriptOutput);
+		} else {
+			Long scriptHeapSizeMB = Long.valueOf(scriptOutput);
+			assertThat(
+				"Different heap sizes (Java: " + javaHeapSizeMB + ", Script: " + scriptHeapSizeMB +
+					") with configuration: " + config.toString(), scriptHeapSizeMB,
+				allOf(greaterThanOrEqualTo(javaHeapSizeMB - absoluteTolerance),
+					lessThanOrEqualTo(javaHeapSizeMB + absoluteTolerance)));
+		}
+	}
+
+	/**
+	 * Executes the given shell script wrapper and returns its output.
+	 *
+	 * @param command  command to run
+	 *
+	 * @return raw script output
+	 */
+	private String executeScript(final String[] command) throws IOException {
+		ProcessBuilder pb = new ProcessBuilder(command);
+		pb.redirectErrorStream(true);
+		Process process = pb.start();
+		BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+		StringBuilder sb = new StringBuilder();
+		String s;
+		while ((s = reader.readLine()) != null) {
+			sb.append(s);
+		}
+		return sb.toString();
+	}
+
+}
diff --git a/flink-dist/src/test/resources/log4j-test.properties b/flink-dist/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000000000..9f248378dbba8
--- /dev/null
+++ b/flink-dist/src/test/resources/log4j-test.properties
@@ -0,0 +1,35 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=OFF, console
+
+# -----------------------------------------------------------------------------
+# Console (use 'console')
+# -----------------------------------------------------------------------------
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+# -----------------------------------------------------------------------------
+# File (use 'file')
+# -----------------------------------------------------------------------------
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.file=${log.dir}/${mvn.forkNumber}.log
+log4j.appender.file.append=false
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
index e9d3cbd36007a..ea508d191e7e5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
@@ -309,7 +309,7 @@ public static void substituteDeprecatedConfigPrefix(
 	 * Get an instance of the dynamic properties option.
 	 *
 	 * Dynamic properties allow the user to specify additional configuration values with -D, such as
-	 *  -Dfs.overwrite-files=true  -Dtaskmanager.network.numberOfBuffers=16368
+	 * <tt> -Dfs.overwrite-files=true  -Dtaskmanager.network.memory.min=536346624</tt>
      */
 	public static Option newDynamicPropertiesOption() {
 		return new Option(DYNAMIC_PROPERTIES_OPT, true, "Dynamic properties");
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java
index 8ff3c257aa270..9d679cf2f09a2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java
@@ -20,7 +20,7 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.taskexecutor.TaskManagerServices;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -141,26 +141,10 @@ public static ContaineredTaskManagerParameters create(
 
 		final long javaMemorySizeMB = containerMemoryMB - cutoff;
 
-		// (2) split the Java memory between heap and off-heap
+		// (2) split the remaining Java memory between heap and off-heap
+		final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(javaMemorySizeMB, config);
+		final long offHeapSize = javaMemorySizeMB == heapSizeMB ? -1L : javaMemorySizeMB - heapSizeMB; 
 
-		final boolean useOffHeap = config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP);
-
-		final long heapSizeMB;
-		long offHeapSize = -1;
-		if (useOffHeap) {
-			offHeapSize = config.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE);
-
-			if (offHeapSize <= 0) {
-				double fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
-
-				offHeapSize = (long) (fraction * javaMemorySizeMB);
-			}
-
-			heapSizeMB = javaMemorySizeMB - offHeapSize;
-		} else {
-			heapSizeMB = javaMemorySizeMB;
-		}
-		
 		// (3) obtain the additional environment variables from the configuration
 		final HashMap<String, String> envVars = new HashMap<>();
 		final String prefix = ConfigConstants.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;
@@ -172,7 +156,7 @@ public static ContaineredTaskManagerParameters create(
 				envVars.put(envVarKey, config.getString(key, null));
 			}
 		}
-		
+
 		// done
 		return new ContaineredTaskManagerParameters(
 			containerMemoryMB, heapSizeMB, offHeapSize, numSlots, envVars);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java
index a36bdf47592dd..1eb44c387e94d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java
@@ -194,12 +194,15 @@ public BufferPool createBufferPool(int numRequiredBuffers, int maxUsedBuffers) t
 			if (numTotalRequiredBuffers + numRequiredBuffers > totalNumberOfMemorySegments) {
 				throw new IOException(String.format("Insufficient number of network buffers: " +
 								"required %d, but only %d available. The total number of network " +
-								"buffers is currently set to %d. You can increase this " +
-								"number by setting the configuration key '%s'.",
+								"buffers is currently set to %d of %d bytes each. You can increase this " +
+								"number by setting the configuration keys '%s', '%s', and '%s'.",
 						numRequiredBuffers,
 						totalNumberOfMemorySegments - numTotalRequiredBuffers,
 						totalNumberOfMemorySegments,
-						TaskManagerOptions.NETWORK_NUM_BUFFERS.key()));
+						memorySegmentSize,
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(),
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key(),
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key()));
 			}
 
 			this.numTotalRequiredBuffers += numRequiredBuffers;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
index 11c6d1647ddc6..654d5289bba1b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java
@@ -52,8 +52,10 @@
  * <p>Since the network buffer pool size for outgoing partitions is usually
  * quite small, e.g. via the {@link TaskManagerOptions#NETWORK_BUFFERS_PER_CHANNEL}
  * and {@link TaskManagerOptions#NETWORK_EXTRA_BUFFERS_PER_GATE} parameters
- * for bounded channels or from the default value of
- * {@link TaskManagerOptions#NETWORK_NUM_BUFFERS}, most spillable partitions
+ * for bounded channels or from the default values of
+ * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
+ * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN}, and
+ * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}, most spillable partitions
  * will be spilled for real-world data sets.
  */
 class SpillableSubpartition extends ResultSubpartition {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
index 823b3f2aceca8..b8d6bbb7aa907 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
@@ -23,6 +23,7 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.taskexecutor.TaskManagerServices;
 import org.apache.flink.runtime.util.EnvironmentInformation;
 import scala.concurrent.duration.FiniteDuration;
 
@@ -190,7 +191,7 @@ public String toString() {
 	 * 3. Distribute the available free memory equally among all components (JMs, RMs and TMs) and
 	 * calculate the managed memory from the share of memory for a single task manager.
 	 *
-	 * @return
+	 * @return size of managed memory per task manager (in megabytes)
 	 */
 	private long getOrCalculateManagedMemoryPerTaskManager() {
 		if (managedMemoryPerTaskManager == -1) {
@@ -206,9 +207,6 @@ private long getOrCalculateManagedMemoryPerTaskManager() {
 				// share the available memory among all running components
 
 				float memoryFraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
-				long networkBuffersMemory =
-					(long) config.getInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS) *
-						(long) config.getInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE);
 
 				long freeMemory = EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag();
 
@@ -217,12 +215,13 @@ private long getOrCalculateManagedMemoryPerTaskManager() {
 				long memoryPerComponent = freeMemory / (numTaskManagers + numResourceManagers + numJobManagers);
 
 				// subtract the network buffer memory
+				long networkBuffersMemory = TaskManagerServices.calculateNetworkBufferMemory(memoryPerComponent, config);
 				long memoryMinusNetworkBuffers = memoryPerComponent - networkBuffersMemory;
 
 				// calculate the managed memory size
 				long managedMemoryBytes = (long) (memoryMinusNetworkBuffers * memoryFraction);
 
-				return managedMemoryBytes >>> 20;
+				return managedMemoryBytes >> 20; // bytes to megabytes
 			} else {
 				return memorySize;
 			}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index e3c834501d109..ecf81d97c4b7c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -18,6 +18,8 @@
 
 package org.apache.flink.runtime.taskexecutor;
 
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.memory.MemoryType;
 import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
@@ -46,7 +48,6 @@
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.util.EnvironmentInformation;
 import org.apache.flink.util.Preconditions;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -256,9 +257,11 @@ private static MemoryManager createMemoryManager(TaskManagerServicesConfiguratio
 			}
 			memorySize = configuredMemory << 20; // megabytes to bytes
 		} else {
+			// similar to #calculateNetworkBufferMemory(TaskManagerServicesConfiguration tmConfig)
 			float memoryFraction = taskManagerServicesConfiguration.getMemoryFraction();
 
 			if (memType == MemoryType.HEAP) {
+				// network buffers already allocated -> use memoryFraction of the remaining heap:
 				long relativeMemSize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * memoryFraction);
 				if (preAllocateMemory) {
 					LOG.info("Using {} of the currently free heap space for managed heap memory ({} MB)." ,
@@ -269,7 +272,10 @@ private static MemoryManager createMemoryManager(TaskManagerServicesConfiguratio
 				}
 				memorySize = relativeMemSize;
 			} else if (memType == MemoryType.OFF_HEAP) {
-				// The maximum heap memory has been adjusted according to the fraction
+				// The maximum heap memory has been adjusted according to the fraction (see
+				// calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config)), i.e.
+				// maxJvmHeap = jvmHeapNoNet - jvmHeapNoNet * memoryFraction = jvmHeapNoNet * (1 - memoryFraction)
+				// directMemorySize = jvmHeapNoNet * memoryFraction
 				long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
 				long directMemorySize = (long) (maxMemory / (1.0 - memoryFraction) * memoryFraction);
 				if (preAllocateMemory) {
@@ -321,9 +327,19 @@ private static NetworkEnvironment createNetworkEnvironment(
 
 		NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskManagerServicesConfiguration.getNetworkConfig();
 
+		final long networkBuf = calculateNetworkBufferMemory(taskManagerServicesConfiguration);
+		int segmentSize = networkEnvironmentConfiguration.networkBufferSize();
+
+		// tolerate offcuts between intended and allocated memory due to segmentation (will be available to the user-space memory)
+		final long numNetBuffersLong = networkBuf / segmentSize;
+		if (numNetBuffersLong > Integer.MAX_VALUE) {
+			throw new IllegalArgumentException("The given number of memory bytes (" + networkBuf
+				+ ") corresponds to more than MAX_INT pages.");
+		}
+
 		NetworkBufferPool networkBufferPool = new NetworkBufferPool(
-			networkEnvironmentConfiguration.numNetworkBuffers(),
-			networkEnvironmentConfiguration.networkBufferSize(),
+			(int) numNetBuffersLong,
+			segmentSize,
 			networkEnvironmentConfiguration.memoryType());
 
 		ConnectionManager connectionManager;
@@ -375,6 +391,217 @@ private static NetworkEnvironment createNetworkEnvironment(
 			networkEnvironmentConfiguration.extraNetworkBuffersPerGate());
 	}
 
+	/**
+	 * Calculates the amount of memory used for network buffers based on the total memory to use and
+	 * the according configuration parameters.
+	 *
+	 * The following configuration parameters are involved:
+	 * <ul>
+	 *  <li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
+	 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
+	 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
+	 *  <li>{@link TaskManagerOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
+	 * </ul>.
+	 *
+	 * @param totalJavaMemorySize
+	 * 		overall available memory to use (heap and off-heap, in bytes)
+	 * @param config
+	 * 		configuration object
+	 *
+	 * @return memory to use for network buffers (in bytes)
+	 */
+	@SuppressWarnings("deprecation")
+	public static long calculateNetworkBufferMemory(long totalJavaMemorySize, Configuration config) {
+		Preconditions.checkArgument(totalJavaMemorySize > 0);
+
+		int segmentSize = config.getInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE);
+
+		final long networkBufBytes;
+		if (TaskManagerServicesConfiguration.hasNewNetworkBufConf(config)) {
+			// new configuration based on fractions of available memory with selectable min and max
+			float networkBufFraction = config.getFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
+			long networkBufMin = config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN);
+			long networkBufMax = config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX);
+
+			TaskManagerServicesConfiguration
+				.checkNetworkBufferConfig(segmentSize, networkBufFraction, networkBufMin, networkBufMax);
+
+			networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin,
+				(long) (networkBufFraction * totalJavaMemorySize)));
+
+			TaskManagerServicesConfiguration
+				.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
+					"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
+					"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
+					"Network buffer memory size too large: " + networkBufBytes + " >= " +
+						totalJavaMemorySize + " (total JVM memory size)");
+		} else {
+			// use old (deprecated) network buffers parameter
+			int numNetworkBuffers = config.getInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS);
+			networkBufBytes = (long) numNetworkBuffers * (long) segmentSize;
+
+			TaskManagerServicesConfiguration.checkNetworkConfigOld(numNetworkBuffers);
+
+			TaskManagerServicesConfiguration
+				.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
+					networkBufBytes, TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
+					"Network buffer memory size too large: " + networkBufBytes + " >= " +
+						totalJavaMemorySize + " (total JVM memory size)");
+		}
+
+		return networkBufBytes;
+	}
+
+	/**
+	 * Calculates the amount of memory used for network buffers inside the current JVM instance
+	 * based on the available heap or the max heap size and the according configuration parameters.
+	 *
+	 * For containers or when started via scripts, if started with a memory limit and set to use
+	 * off-heap memory, the maximum heap size for the JVM is adjusted accordingly and we are able
+	 * to extract the intended values from this.
+	 *
+	 * The following configuration parameters are involved:
+	 * <ul>
+	 *  <li>{@link TaskManagerOptions#MANAGED_MEMORY_FRACTION},</li>
+	 *  <li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
+	 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
+	 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
+	 *  <li>{@link TaskManagerOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
+	 * </ul>.
+	 *
+	 * @param tmConfig task manager services configuration object
+	 *
+	 * @return memory to use for network buffers (in bytes)
+	 */
+	public static long calculateNetworkBufferMemory(TaskManagerServicesConfiguration tmConfig) {
+		final NetworkEnvironmentConfiguration networkConfig = tmConfig.getNetworkConfig();
+
+		final float networkBufFraction = networkConfig.networkBufFraction();
+		final long networkBufMin = networkConfig.networkBufMin();
+		final long networkBufMax = networkConfig.networkBufMax();
+
+		if (networkBufMin == networkBufMax) {
+			// fixed network buffer pool size
+			return networkBufMin;
+		}
+
+		// relative network buffer pool size using the fraction
+
+		final MemoryType memType = networkConfig.memoryType();
+
+		final long networkBufBytes;
+		if (memType == MemoryType.HEAP) {
+			// use fraction parts of the available heap memory
+
+			final long relativeMemSize = EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag();
+			networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin,
+				(long) (networkBufFraction * relativeMemSize)));
+
+			TaskManagerServicesConfiguration
+				.checkConfigParameter(networkBufBytes < relativeMemSize,
+					"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
+					"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
+					"Network buffer memory size too large: " + networkBufBytes + " >= " +
+						relativeMemSize + "(free JVM heap size)");
+		} else if (memType == MemoryType.OFF_HEAP) {
+			// The maximum heap memory has been adjusted accordingly as in
+			// calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config))
+			// and we need to invert these calculations.
+
+			final long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
+
+			// check if a value has been configured
+			long configuredMemory = tmConfig.getConfiguredMemory() << 20; // megabytes to bytes
+
+			final long jvmHeapNoNet;
+			if (configuredMemory > 0) {
+				// The maximum heap memory has been adjusted according to configuredMemory, i.e.
+				// maxJvmHeap = jvmHeapNoNet - configuredMemory
+
+				jvmHeapNoNet = maxMemory + configuredMemory;
+			} else {
+				// The maximum heap memory has been adjusted according to the fraction, i.e.
+				// maxJvmHeap = jvmHeapNoNet - jvmHeapNoNet * managedFraction = jvmHeapNoNet * (1 - managedFraction)
+
+				final float managedFraction = tmConfig.getMemoryFraction();
+				jvmHeapNoNet = (long) (maxMemory / (1.0 - managedFraction));
+			}
+
+			// finally extract the network buffer memory size again from:
+			// jvmHeapNoNet = jvmHeap - networkBufBytes
+			//              = jvmHeap - Math.min(networkBufMax, Math.max(networkBufMin, jvmHeap * netFraction)
+			networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin,
+				(long) (jvmHeapNoNet / (1.0 - networkBufFraction) * networkBufFraction)));
+
+			TaskManagerServicesConfiguration
+				.checkConfigParameter(networkBufBytes < maxMemory,
+					"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
+					"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
+						TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
+					"Network buffer memory size too large: " + networkBufBytes + " >= " +
+						maxMemory + "(maximum JVM heap size)");
+		} else {
+			throw new RuntimeException("No supported memory type detected.");
+		}
+
+		return networkBufBytes;
+	}
+
+	/**
+	 * Calculates the amount of heap memory to use (to set via <tt>-Xmx</tt> and <tt>-Xms</tt>)
+	 * based on the total memory to use and the given configuration parameters.
+	 *
+	 * @param totalJavaMemorySizeMB
+	 * 		overall available memory to use (heap and off-heap)
+	 * @param config
+	 * 		configuration object
+	 *
+	 * @return heap memory to use (in megabytes)
+	 */
+	public static long calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config) {
+		Preconditions.checkArgument(totalJavaMemorySizeMB > 0);
+
+		final long totalJavaMemorySize = totalJavaMemorySizeMB << 20; // megabytes to bytes
+
+		// split the available Java memory between heap and off-heap
+
+		final boolean useOffHeap = config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP);
+
+		final long heapSizeMB;
+		if (useOffHeap) {
+
+			// subtract the Java memory used for network buffers
+			final long networkBufMB = calculateNetworkBufferMemory(totalJavaMemorySize, config) >> 20; // bytes to megabytes
+			final long remainingJavaMemorySizeMB = totalJavaMemorySizeMB - networkBufMB;
+
+			long offHeapSize = config.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE);
+
+			if (offHeapSize <= 0) {
+				// calculate off-heap section via fraction
+				double fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
+				offHeapSize = (long) (fraction * remainingJavaMemorySizeMB);
+			}
+
+			TaskManagerServicesConfiguration
+				.checkConfigParameter(offHeapSize < remainingJavaMemorySizeMB, offHeapSize,
+					TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
+					"Managed memory size too large for " + networkBufMB +
+						" MB network buffer memory and a total of " + totalJavaMemorySizeMB +
+						" MB JVM memory");
+
+			heapSizeMB = remainingJavaMemorySizeMB - offHeapSize;
+		} else {
+			heapSizeMB = totalJavaMemorySizeMB;
+		}
+
+		return heapSizeMB;
+	}
+
 	/**
 	 * Validates that all the directories denoted by the strings do actually exist, are proper
 	 * directories (not files), and are writable.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
index 366be34d8d824..3fee689205dbb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
@@ -34,6 +34,8 @@
 import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
 import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
 import org.apache.flink.util.MathUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.net.InetAddress;
@@ -47,6 +49,7 @@
  * the io manager and the metric registry
  */
 public class TaskManagerServicesConfiguration {
+	private static final Logger LOG = LoggerFactory.getLogger(TaskManagerServicesConfiguration.class);
 
 	private final InetAddress taskManagerAddress;
 
@@ -58,6 +61,11 @@ public class TaskManagerServicesConfiguration {
 
 	private final QueryableStateConfiguration queryableStateConfig;
 
+	/**
+	 * Managed memory (in megabytes).
+	 *
+	 * @see TaskManagerOptions#MANAGED_MEMORY_SIZE
+	 */
 	private final long configuredMemory;
 
 	private final boolean preAllocateMemory;
@@ -126,6 +134,13 @@ public float getMemoryFraction() {
 		return memoryFraction;
 	}
 
+	/**
+	 * Returns the size of the managed memory (in megabytes), if configured.
+	 *
+	 * @return managed memory or a default value (currently <tt>-1</tt>) if not configured
+	 *
+	 * @see TaskManagerOptions#MANAGED_MEMORY_SIZE
+	 */
 	public long getConfiguredMemory() {
 		return configuredMemory;
 	}
@@ -228,6 +243,7 @@ public static TaskManagerServicesConfiguration fromConfiguration(
 	 * @param slots to start the task manager with
 	 * @return Network environment configuration
 	 */
+	@SuppressWarnings("deprecation")
 	private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfiguration(
 		Configuration configuration,
 		boolean localTaskManagerCommunication,
@@ -245,11 +261,6 @@ private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfigurat
 		checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
 			"Number of task slots must be at least one.");
 
-		final int numNetworkBuffers = configuration.getInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS);
-
-		checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers,
-			TaskManagerOptions.NETWORK_NUM_BUFFERS.key(), "");
-
 		final int pageSize = configuration.getInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE);
 
 		// check page size of for minimum size
@@ -284,6 +295,27 @@ private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfigurat
 			}
 		}
 
+		// network buffer memory fraction
+
+		float networkBufFraction = configuration.getFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
+		long networkBufMin = configuration.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN);
+		long networkBufMax = configuration.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX);
+		checkNetworkBufferConfig(pageSize, networkBufFraction, networkBufMin, networkBufMax);
+
+		// fallback: number of network buffers
+		final int numNetworkBuffers = configuration.getInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS);
+		checkNetworkConfigOld(numNetworkBuffers);
+
+		if (!hasNewNetworkBufConf(configuration)) {
+			// map old config to new one:
+			networkBufMin = networkBufMax = numNetworkBuffers * pageSize;
+		} else {
+			if (configuration.contains(TaskManagerOptions.NETWORK_NUM_BUFFERS)) {
+				LOG.info("Ignoring old (but still present) network buffer configuration via {}.",
+					TaskManagerOptions.NETWORK_NUM_BUFFERS.key());
+			}
+		}
+
 		final NettyConfig nettyConfig;
 		if (!localTaskManagerCommunication) {
 			final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
@@ -317,7 +349,9 @@ private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfigurat
 			TaskManagerOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);
 
 		return new NetworkEnvironmentConfiguration(
-			numNetworkBuffers,
+			networkBufFraction,
+			networkBufMin,
+			networkBufMax,
 			pageSize,
 			memType,
 			ioMode,
@@ -328,6 +362,69 @@ private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfigurat
 			nettyConfig);
 	}
 
+	/**
+	 * Validates the (old) network buffer configuration.
+	 *
+	 * @param numNetworkBuffers		number of buffers used in the network stack
+	 *
+	 * @throws IllegalConfigurationException if the condition does not hold
+	 */
+	@SuppressWarnings("deprecation")
+	protected static void checkNetworkConfigOld(final int numNetworkBuffers) {
+		checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers,
+			TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
+			"Must have at least one network buffer");
+	}
+
+	/**
+	 * Validates the (new) network buffer configuration.
+	 *
+	 * @param pageSize 				size of memory buffers
+	 * @param networkBufFraction	fraction of JVM memory to use for network buffers
+	 * @param networkBufMin 		minimum memory size for network buffers (in bytes)
+	 * @param networkBufMax 		maximum memory size for network buffers (in bytes)
+	 *
+	 * @throws IllegalConfigurationException if the condition does not hold
+	 */
+	protected static void checkNetworkBufferConfig(
+			final int pageSize, final float networkBufFraction, final long networkBufMin,
+			final long networkBufMax) throws IllegalConfigurationException {
+
+		checkConfigParameter(networkBufFraction > 0.0f && networkBufFraction < 1.0f, networkBufFraction,
+			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(),
+			"Network buffer memory fraction of the free memory must be between 0.0 and 1.0");
+
+		checkConfigParameter(networkBufMin >= pageSize, networkBufMin,
+			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key(),
+			"Minimum memory for network buffers must allow at least one network " +
+				"buffer with respect to the memory segment size");
+
+		checkConfigParameter(networkBufMax >= pageSize, networkBufMax,
+			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
+			"Maximum memory for network buffers must allow at least one network " +
+				"buffer with respect to the memory segment size");
+
+		checkConfigParameter(networkBufMax >= networkBufMin, networkBufMax,
+			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
+			"Maximum memory for network buffers must not be smaller than minimum memory (" +
+				TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ": " + networkBufMin + ")");
+	}
+
+	/**
+	 * Returns whether the new network buffer memory configuration is present in the configuration
+	 * object, i.e. at least one new parameter is given or the old one is not present.
+	 *
+	 * @param config configuration object
+	 * @return <tt>true</tt> if the new configuration method is used, <tt>false</tt> otherwise
+	 */
+	@SuppressWarnings("deprecation")
+	public static boolean hasNewNetworkBufConf(final Configuration config) {
+		return config.contains(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION) ||
+			config.contains(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN) ||
+			config.contains(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX) ||
+			!config.contains(TaskManagerOptions.NETWORK_NUM_BUFFERS);
+	}
+
 	/**
 	 * Creates the {@link QueryableStateConfiguration} from the given Configuration.
 	 */
@@ -353,8 +450,11 @@ private static QueryableStateConfiguration parseQueryableStateConfiguration(Conf
 	 * @param parameter         The parameter value. Will be shown in the exception message.
 	 * @param name              The name of the config parameter. Will be shown in the exception message.
 	 * @param errorMessage  The optional custom error message to append to the exception message.
+	 *
+	 * @throws IllegalConfigurationException if the condition does not hold
 	 */
-	private static void checkConfigParameter(boolean condition, Object parameter, String name, String errorMessage) {
+	static void checkConfigParameter(boolean condition, Object parameter, String name, String errorMessage)
+			throws IllegalConfigurationException {
 		if (!condition) {
 			throw new IllegalConfigurationException("Invalid configuration value for " + 
 					name + " : " + parameter + " - " + errorMessage);
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index 3db91b7d5dd5c..8677307461258 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -363,11 +363,6 @@ class LocalFlinkMiniCluster(
     if (config.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE) ==
         TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) {
 
-      val bufferSize: Int = config.getInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE)
-
-      val bufferMem: Long = config.getInteger(
-        TaskManagerOptions.NETWORK_NUM_BUFFERS).toLong * bufferSize.toLong
-
       val numTaskManager = config.getInteger(
         ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
         ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER)
@@ -381,10 +376,10 @@ class LocalFlinkMiniCluster(
       // each TaskManagers and each JobManager
       memorySize /= numTaskManager + 1 // the +1 is the job manager
 
-      // for each TaskManager, subtract the memory needed for memory buffers
-      memorySize -= bufferMem
+      // for each TaskManager, subtract the memory needed for network memory buffers
+      memorySize -= TaskManagerServices.calculateNetworkBufferMemory(memorySize, config)
       memorySize = (memorySize * memoryFraction).toLong
-      memorySize >>>= 20 // bytes to megabytes
+      memorySize >>= 20 // bytes to megabytes
       config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, memorySize)
     }
   }
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
index 4ecfe5977f48f..d74bb3be35156 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
@@ -23,7 +23,9 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager.IOMode
 import org.apache.flink.runtime.io.network.netty.NettyConfig
 
 case class NetworkEnvironmentConfiguration(
-    numNetworkBuffers: Int,
+    networkBufFraction: Float,
+    networkBufMin: Long,
+    networkBufMax: Long,
     networkBufferSize: Int,
     memoryType: MemoryType,
     ioMode: IOMode,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfigurationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfigurationTest.java
new file mode 100644
index 0000000000000..ddd6462bfebde
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfigurationTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unit test for {@link TaskManagerServicesConfiguration}.
+ */
+public class TaskManagerServicesConfigurationTest {
+	/**
+	 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
+	 * returns the correct result for old configurations via
+	 * {@link TaskManagerOptions#NETWORK_NUM_BUFFERS}.
+	 */
+	@SuppressWarnings("deprecation")
+	@Test
+	public void hasNewNetworkBufConfOld() throws Exception {
+		Configuration config = new Configuration();
+		config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
+
+		assertFalse(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+	}
+
+	/**
+	 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
+	 * returns the correct result for new configurations via
+	 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
+	 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN} and {@link
+	 * TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}.
+	 */
+	@Test
+	public void hasNewNetworkBufConfNew() throws Exception {
+		Configuration config = new Configuration();
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+
+		// fully defined:
+		config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, 1024);
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 2048);
+
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+
+		// partly defined:
+		config = new Configuration();
+		config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 1024);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+
+		config = new Configuration();
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, 1024);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+		config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+
+		config = new Configuration();
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 1024);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, 1024);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+	}
+
+	/**
+	 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
+	 * returns the correct result for mixed old/new configurations.
+	 */
+	@SuppressWarnings("deprecation")
+	@Test
+	public void hasNewNetworkBufConfMixed() throws Exception {
+		Configuration config = new Configuration();
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+
+		config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
+		assertFalse(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));
+
+		// old + 1 new parameter = new:
+		Configuration config1 = config.clone();
+		config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
+
+		config1 = config.clone();
+		config1.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, 1024);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
+
+		config1 = config.clone();
+		config1.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 1024);
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
+	}
+
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java
new file mode 100644
index 0000000000000..bf90634ea2a81
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.core.memory.MemoryType;
+import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
+import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
+import org.apache.flink.runtime.util.EnvironmentInformation;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.net.InetAddress;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+/**
+ * Unit test for {@link TaskManagerServices}.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(EnvironmentInformation.class)
+public class TaskManagerServicesTest {
+
+	/**
+	 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using old
+	 * configurations via {@link TaskManagerOptions#NETWORK_NUM_BUFFERS}.
+	 */
+	@SuppressWarnings("deprecation")
+	@Test
+	public void calculateNetworkBufOld() throws Exception {
+		Configuration config = new Configuration();
+		config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
+
+		// note: actual network buffer memory size is independent of the totalJavaMemorySize
+		assertEquals(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue().longValue(),
+			TaskManagerServices.calculateNetworkBufferMemory(10L << 20, config));
+		assertEquals(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue().longValue(),
+			TaskManagerServices.calculateNetworkBufferMemory(64L << 20, config));
+
+		// test integer overflow in the memory size
+		int numBuffers = (int) ((2L << 32) / TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()); // 2^33
+		config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, numBuffers);
+		assertEquals(2L << 32, TaskManagerServices.calculateNetworkBufferMemory(2L << 33, config));
+	}
+
+	/**
+	 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using new
+	 * configurations via {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
+	 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN} and
+	 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}.
+	 */
+	@Test
+	public void calculateNetworkBufNew() throws Exception {
+		Configuration config = new Configuration();
+
+		// (1) defaults
+		final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
+		final Long defaultMin = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue();
+		final Long defaultMax = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue();
+		assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, defaultMax),
+			TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config));
+		assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, defaultMax),
+			TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config));
+
+		calculateNetworkBufNew(config);
+	}
+
+	/**
+	 * Helper to test {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} with the
+	 * new configuration parameters.
+	 *
+	 * @param config configuration object
+	 */
+	private static void calculateNetworkBufNew(final Configuration config) {
+		// (2) fixed size memory
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, 1L << 20); // 1MB
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 1L << 20); // 1MB
+
+		// note: actual network buffer memory size is independent of the totalJavaMemorySize
+		assertEquals(1 << 20, TaskManagerServices.calculateNetworkBufferMemory(10L << 20, config));
+		assertEquals(1 << 20, TaskManagerServices.calculateNetworkBufferMemory(64L << 20, config));
+		assertEquals(1 << 20, TaskManagerServices.calculateNetworkBufferMemory(1L << 30, config));
+
+		// (3) random fraction, min, and max values
+		Random ran = new Random();
+		for (int i = 0; i < 1_000; ++i){
+			float frac = Math.max(ran.nextFloat(), Float.MIN_VALUE);
+			config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, frac);
+
+			long min = Math.max(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue(), ran.nextLong());
+			config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, min);
+
+			long max = Math.max(min, ran.nextLong());
+			config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, max);
+
+			long javaMem = Math.max(max + 1, ran.nextLong());
+
+			final long networkBufMem = TaskManagerServices.calculateNetworkBufferMemory(javaMem, config);
+			assertTrue("Lower bound not met with configuration: " + config.toString(),
+				networkBufMem >= min);
+			assertTrue("Upper bound not met with configuration: " + config.toString(),
+				networkBufMem <= max);
+			if (networkBufMem > min && networkBufMem < max) {
+				assertEquals(
+					"Wrong network buffer memory size with configuration: " + config.toString(),
+					(long) (javaMem * frac), networkBufMem);
+			}
+		}
+	}
+
+	/**
+	 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using mixed
+	 * old/new configurations.
+	 */
+	@SuppressWarnings("deprecation")
+	@Test
+	public void calculateNetworkBufMixed() throws Exception {
+		Configuration config = new Configuration();
+		config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
+
+		final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
+		final Long defaultMin = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue();
+		final Long defaultMax = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue();
+
+		// old + 1 new parameter = new:
+		Configuration config1 = config.clone();
+		config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
+		assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
+			TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
+		assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
+			TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));
+
+		config1 = config.clone();
+		long newMin = TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue(); // smallest value possible
+		config1.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, newMin);
+		assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
+			TaskManagerServices.calculateNetworkBufferMemory((10L << 20), config1));
+		assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
+			TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));
+
+		config1 = config.clone();
+		long newMax = Math.max(64L << 20 + 1, TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue());
+		config1.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, newMax);
+		assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
+			TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
+		assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
+			TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));
+		assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
+
+		// old + any new parameter = new:
+		calculateNetworkBufNew(config);
+	}
+
+	/**
+	 * Returns the value or the lower/upper bound in case the value is less/greater than the lower/upper bound, respectively.
+	 *
+	 * @param value value to inspec
+	 * @param lower lower bound
+	 * @param upper upper bound
+	 *
+	 * @return <tt>min(upper, max(lower, value))</tt>
+	 */
+	private static long enforceBounds(final long value, final long lower, final long upper) {
+		return Math.min(upper, Math.max(lower, value));
+	}
+
+	/**
+	 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(TaskManagerServicesConfiguration)}
+	 * using the same (manual) test cases as in {@link #calculateHeapSizeMB()}.
+	 */
+	@Test
+	public void calculateNetworkBufFromHeapSize() throws Exception {
+		PowerMockito.mockStatic(EnvironmentInformation.class);
+		// some defaults:
+		when(EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag()).thenReturn(1000L << 20); // 1000MB
+		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(1000L << 20); // 1000MB
+
+		TaskManagerServicesConfiguration tmConfig;
+
+		tmConfig = getTmConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue(),
+			TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
+			0.1f, 60L << 20, 1L << 30, MemoryType.HEAP);
+		when(EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag()).thenReturn(1000L << 20); // 1000MB
+		assertEquals(100L << 20, TaskManagerServices.calculateNetworkBufferMemory(tmConfig));
+
+		tmConfig = getTmConfig(10, TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
+			0.1f, 60L << 20, 1L << 30, MemoryType.OFF_HEAP);
+		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(890L << 20); // 890MB
+		assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
+			TaskManagerServices.calculateNetworkBufferMemory(tmConfig));
+
+		tmConfig = getTmConfig(-1, 0.1f,
+			0.1f, 60L << 20, 1L << 30, MemoryType.OFF_HEAP);
+		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(810L << 20); // 810MB
+		assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
+			TaskManagerServices.calculateNetworkBufferMemory(tmConfig));
+	}
+
+	/**
+	 * Returns a task manager services configuration for the tests
+	 *
+	 * @param managedMemory         see {@link TaskManagerOptions#MANAGED_MEMORY_SIZE}
+	 * @param managedMemoryFraction see {@link TaskManagerOptions#MANAGED_MEMORY_FRACTION}
+	 * @param networkBufFraction	see {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION}
+	 * @param networkBufMin			see {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN}
+	 * @param networkBufMax			see {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}
+	 * @param memType				on-heap or off-heap
+	 *
+	 * @return configuration object
+	 */
+	private static TaskManagerServicesConfiguration getTmConfig(
+		final long managedMemory, final float managedMemoryFraction, float networkBufFraction,
+		long networkBufMin, long networkBufMax,
+		final MemoryType memType) {
+
+		final NetworkEnvironmentConfiguration networkConfig = new NetworkEnvironmentConfiguration(
+			networkBufFraction,
+			networkBufMin,
+			networkBufMax,
+			TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue(),
+			memType,
+			null,
+			TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL.defaultValue(),
+			TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX.defaultValue(),
+			TaskManagerOptions.NETWORK_BUFFERS_PER_CHANNEL.defaultValue(),
+			TaskManagerOptions.NETWORK_EXTRA_BUFFERS_PER_GATE.defaultValue(),
+			null);
+
+		return new TaskManagerServicesConfiguration(
+			mock(InetAddress.class),
+			new String[] {},
+			networkConfig,
+			QueryableStateConfiguration.disabled(),
+			1,
+			managedMemory,
+			false,
+			managedMemoryFraction,
+			mock(MetricRegistryConfiguration.class),
+			0);
+	}
+
+	/**
+	 * Test for {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} with some
+	 * manually calculated scenarios.
+	 */
+	@Test
+	public void calculateHeapSizeMB() throws Exception {
+		Configuration config = new Configuration();
+		config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, 64L << 20); // 64MB
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 1L << 30); // 1GB
+
+		config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, false);
+		assertEquals(1000, TaskManagerServices.calculateHeapSizeMB(1000, config));
+
+		config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, true);
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 10); // 10MB
+		assertEquals(890, TaskManagerServices.calculateHeapSizeMB(1000, config));
+
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, -1); // use fraction of given memory
+		config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 0.1f); // 10%
+		assertEquals(810, TaskManagerServices.calculateHeapSizeMB(1000, config));
+	}
+
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
index 973fddfecce4e..7837b27c9e75b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
@@ -121,18 +121,21 @@ public void testComponentsStartupShutdown() throws Exception {
 				config,
 				false); // exit-jvm-on-fatal-error
 
+			final int networkBufNum = 32;
+			// note: the network buffer memory configured here is not actually used below but set
+			// accordingly to be consistent
 			final NetworkEnvironmentConfiguration netConf = new NetworkEnvironmentConfiguration(
-					32, BUFFER_SIZE, MemoryType.HEAP, IOManager.IOMode.SYNC,
+					0.1f, networkBufNum * BUFFER_SIZE, networkBufNum * BUFFER_SIZE, BUFFER_SIZE, MemoryType.HEAP, IOManager.IOMode.SYNC,
 					0, 0, 2, 8, null);
 
 			ResourceID taskManagerId = ResourceID.generate();
 			
 			final TaskManagerLocation connectionInfo = new TaskManagerLocation(taskManagerId, InetAddress.getLocalHost(), 10000);
 
-			final MemoryManager memManager = new MemoryManager(32 * BUFFER_SIZE, 1, BUFFER_SIZE, MemoryType.HEAP, false);
+			final MemoryManager memManager = new MemoryManager(networkBufNum * BUFFER_SIZE, 1, BUFFER_SIZE, MemoryType.HEAP, false);
 			final IOManager ioManager = new IOManagerAsync(TMP_DIR);
 			final NetworkEnvironment network = new NetworkEnvironment(
-				new NetworkBufferPool(netConf.numNetworkBuffers(), netConf.networkBufferSize(), netConf.memoryType()),
+				new NetworkBufferPool(32, netConf.networkBufferSize(), netConf.memoryType()),
 				new LocalConnectionManager(),
 				new ResultPartitionManager(),
 				new TaskEventDispatcher(),
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
index dbef01f879aec..fea2b794fd518 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
@@ -103,6 +103,8 @@ public static void startTestCluster() {
 		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
 		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM / 2);
 		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 48L);
+		// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
+		config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 80L << 20); // 80 MB
 
 		cluster = new LocalFlinkMiniCluster(config, false);
 		cluster.start();
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java
index 557fa389f8bf9..6ce8d176b4177 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnCLI.java
@@ -66,7 +66,7 @@ public class FlinkYarnCLI implements CustomCommandLine<YarnClusterClientV2> {
 
 	/**
 	 * Dynamic properties allow the user to specify additional configuration values with -D, such as
-	 *  -D fs.overwrite-files=true  -D taskmanager.network.numberOfBuffers=16368
+	 * <tt> -Dfs.overwrite-files=true  -Dtaskmanager.network.memory.min=536346624</tt>
 	 */
 	private final Option DYNAMIC_PROPERTIES;
 
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
index 9277d21d35202..3d82132f5f958 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
@@ -119,7 +119,7 @@ public class FlinkYarnSessionCli implements CustomCommandLine<YarnClusterClient>
 
 	/**
 	 * Dynamic properties allow the user to specify additional configuration values with -D, such as
-	 *  -D fs.overwrite-files=true  -D taskmanager.network.numberOfBuffers=16368
+	 * <tt> -Dfs.overwrite-files=true  -Dtaskmanager.network.memory.min=536346624</tt>
 	 */
 	private final Option DYNAMIC_PROPERTIES;
 

From f8c72db49fbdb72c883a1a71fa33d7d468610ecb Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Sat, 6 May 2017 19:39:49 +0200
Subject: [PATCH 167/253] [FLINK-4545] [network] (followup) Replace awk lshift
 by multiplication

'lshift(...)' is not defined by default in some commonly used awk versions.
---
 flink-dist/src/main/flink-bin/bin/config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index 7e3c1d452fa54..d73b22037b997 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -533,7 +533,7 @@ calculateNetworkBufferMemory() {
             exit 1
         fi
 
-        network_buffers_bytes=`awk "BEGIN { x = lshift(${FLINK_TM_HEAP},20) * ${FLINK_TM_NET_BUF_FRACTION}; netbuf = x > ${FLINK_TM_NET_BUF_MAX} ? ${FLINK_TM_NET_BUF_MAX} : x < ${FLINK_TM_NET_BUF_MIN} ? ${FLINK_TM_NET_BUF_MIN} : x; printf \"%.0f\n\", netbuf }"`
+        network_buffers_bytes=`awk "BEGIN { x = ${FLINK_TM_HEAP} * 1048576 * ${FLINK_TM_NET_BUF_FRACTION}; netbuf = x > ${FLINK_TM_NET_BUF_MAX} ? ${FLINK_TM_NET_BUF_MAX} : x < ${FLINK_TM_NET_BUF_MIN} ? ${FLINK_TM_NET_BUF_MIN} : x; printf \"%.0f\n\", netbuf }"`
     fi
 
     # recalculate the JVM heap memory by taking the network buffers into account

From b30802ecae345550409dec8c32ddba346bbd429d Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 5 May 2017 16:40:08 +0200
Subject: [PATCH 168/253] [hotfix] [runtime] Migrate
 NetworkEnvironmentConfiguration to Java

---
 .../taskexecutor/TaskManagerServices.java     |   2 +-
 .../NetworkEnvironmentConfiguration.java      | 206 ++++++++++++++++++
 .../NetworkEnvironmentConfiguration.scala     |  36 ---
 ...kManagerComponentsStartupShutdownTest.java |   2 +-
 .../runtime/taskmanager/TaskManagerTest.java  |   2 +-
 5 files changed, 209 insertions(+), 39 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.java
 delete mode 100644 flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index ecf81d97c4b7c..86a2fdfb3d9a8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -388,7 +388,7 @@ private static NetworkEnvironment createNetworkEnvironment(
 			networkEnvironmentConfiguration.partitionRequestInitialBackoff(),
 			networkEnvironmentConfiguration.partitionRequestMaxBackoff(),
 			networkEnvironmentConfiguration.networkBuffersPerChannel(),
-			networkEnvironmentConfiguration.extraNetworkBuffersPerGate());
+			networkEnvironmentConfiguration.floatingNetworkBuffersPerGate());
 	}
 
 	/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.java
new file mode 100644
index 0000000000000..193fd90f0bfb5
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+import org.apache.flink.core.memory.MemoryType;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager.IOMode;
+import org.apache.flink.runtime.io.network.netty.NettyConfig;
+
+import javax.annotation.Nullable;
+
+/**
+ * Configuration object for the network stack.
+ */
+public class NetworkEnvironmentConfiguration {
+
+	private final float networkBufFraction;
+
+	private final long networkBufMin;
+
+	private final long networkBufMax;
+
+	private final int networkBufferSize;
+
+	private final MemoryType memoryType;
+
+	private final IOMode ioMode;
+
+	private final int partitionRequestInitialBackoff;
+
+	private final int partitionRequestMaxBackoff;
+
+	private final int networkBuffersPerChannel;
+
+	private final int floatingNetworkBuffersPerGate;
+
+	private final NettyConfig nettyConfig;
+
+	/**
+	 * Constructor for a setup with purely local communication (no netty).
+	 */
+	public NetworkEnvironmentConfiguration(
+			float networkBufFraction,
+			long networkBufMin,
+			long networkBufMax,
+			int networkBufferSize,
+			MemoryType memoryType,
+			IOMode ioMode,
+			int partitionRequestInitialBackoff,
+			int partitionRequestMaxBackoff,
+			int networkBuffersPerChannel,
+			int floatingNetworkBuffersPerGate) {
+
+		this(networkBufFraction, networkBufMin, networkBufMax, networkBufferSize,
+				memoryType, ioMode,
+				partitionRequestInitialBackoff, partitionRequestMaxBackoff,
+				networkBuffersPerChannel, floatingNetworkBuffersPerGate,
+				null);
+		
+	}
+
+	public NetworkEnvironmentConfiguration(
+			float networkBufFraction,
+			long networkBufMin,
+			long networkBufMax,
+			int networkBufferSize,
+			MemoryType memoryType,
+			IOMode ioMode,
+			int partitionRequestInitialBackoff,
+			int partitionRequestMaxBackoff,
+			int networkBuffersPerChannel,
+			int floatingNetworkBuffersPerGate,
+			@Nullable NettyConfig nettyConfig) {
+
+		this.networkBufFraction = networkBufFraction;
+		this.networkBufMin = networkBufMin;
+		this.networkBufMax = networkBufMax;
+		this.networkBufferSize = networkBufferSize;
+		this.memoryType = memoryType;
+		this.ioMode = ioMode;
+		this.partitionRequestInitialBackoff = partitionRequestInitialBackoff;
+		this.partitionRequestMaxBackoff = partitionRequestMaxBackoff;
+		this.networkBuffersPerChannel = networkBuffersPerChannel;
+		this.floatingNetworkBuffersPerGate = floatingNetworkBuffersPerGate;
+		this.nettyConfig = nettyConfig;
+	}
+
+	// ------------------------------------------------------------------------
+
+	public float networkBufFraction() {
+		return networkBufFraction;
+	}
+
+	public long networkBufMin() {
+		return networkBufMin;
+	}
+
+	public long networkBufMax() {
+		return networkBufMax;
+	}
+
+	public int networkBufferSize() {
+		return networkBufferSize;
+	}
+
+	public MemoryType memoryType() {
+		return memoryType;
+	}
+
+	public IOMode ioMode() {
+		return ioMode;
+	}
+
+	public int partitionRequestInitialBackoff() {
+		return partitionRequestInitialBackoff;
+	}
+
+	public int partitionRequestMaxBackoff() {
+		return partitionRequestMaxBackoff;
+	}
+
+	public int networkBuffersPerChannel() {
+		return networkBuffersPerChannel;
+	}
+
+	public int floatingNetworkBuffersPerGate() {
+		return floatingNetworkBuffersPerGate;
+	}
+
+	public NettyConfig nettyConfig() {
+		return nettyConfig;
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public int hashCode() {
+		int result = 1;
+		result = 31 * result + networkBufferSize;
+		result = 31 * result + memoryType.hashCode();
+		result = 31 * result + ioMode.hashCode();
+		result = 31 * result + partitionRequestInitialBackoff;
+		result = 31 * result + partitionRequestMaxBackoff;
+		result = 31 * result + networkBuffersPerChannel;
+		result = 31 * result + floatingNetworkBuffersPerGate;
+		result = 31 * result + (nettyConfig != null ? nettyConfig.hashCode() : 0);
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj) {
+			return true;
+		}
+		else if (obj == null || getClass() != obj.getClass()) {
+			return false;
+		}
+		else {
+			final NetworkEnvironmentConfiguration that = (NetworkEnvironmentConfiguration) obj;
+
+			return this.networkBufFraction == that.networkBufFraction &&
+					this.networkBufMin == that.networkBufMin &&
+					this.networkBufMax == that.networkBufMax &&
+					this.networkBufferSize == that.networkBufferSize &&
+					this.partitionRequestInitialBackoff == that.partitionRequestInitialBackoff &&
+					this.partitionRequestMaxBackoff == that.partitionRequestMaxBackoff &&
+					this.networkBuffersPerChannel == that.networkBuffersPerChannel &&
+					this.floatingNetworkBuffersPerGate == that.floatingNetworkBuffersPerGate &&
+					this.memoryType == that.memoryType &&
+					this.ioMode == that.ioMode && 
+					(nettyConfig != null ? nettyConfig.equals(that.nettyConfig) : that.nettyConfig == null);
+		}
+	}
+
+	@Override
+	public String toString() {
+		return "NetworkEnvironmentConfiguration{" +
+				"networkBufFraction=" + networkBufFraction +
+				", networkBufMin=" + networkBufMin +
+				", networkBufMax=" + networkBufMax +
+				", networkBufferSize=" + networkBufferSize +
+				", memoryType=" + memoryType +
+				", ioMode=" + ioMode +
+				", partitionRequestInitialBackoff=" + partitionRequestInitialBackoff +
+				", partitionRequestMaxBackoff=" + partitionRequestMaxBackoff +
+				", networkBuffersPerChannel=" + networkBuffersPerChannel +
+				", floatingNetworkBuffersPerGate=" + floatingNetworkBuffersPerGate +
+				", nettyConfig=" + nettyConfig +
+				'}';
+	}
+}
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
deleted file mode 100644
index d74bb3be35156..0000000000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager
-
-import org.apache.flink.core.memory.MemoryType
-import org.apache.flink.runtime.io.disk.iomanager.IOManager.IOMode
-import org.apache.flink.runtime.io.network.netty.NettyConfig
-
-case class NetworkEnvironmentConfiguration(
-    networkBufFraction: Float,
-    networkBufMin: Long,
-    networkBufMax: Long,
-    networkBufferSize: Int,
-    memoryType: MemoryType,
-    ioMode: IOMode,
-    partitionRequestInitialBackoff : Int,
-    partitionRequestMaxBackoff : Int,
-    networkBuffersPerChannel: Int,
-    extraNetworkBuffersPerGate: Int,
-    nettyConfig: NettyConfig = null)
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
index 7837b27c9e75b..2a4c0365d034c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
@@ -145,7 +145,7 @@ public void testComponentsStartupShutdown() throws Exception {
 				netConf.partitionRequestInitialBackoff(),
 				netConf.partitionRequestMaxBackoff(),
 				netConf.networkBuffersPerChannel(),
-				netConf.extraNetworkBuffersPerGate());
+				netConf.floatingNetworkBuffersPerGate());
 
 			network.start();
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index 0f5afc095d7c7..e790ea814e63a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -1047,7 +1047,7 @@ public void testTaskManagerServicesConfiguration() throws Exception {
 		assertEquals(tmConfig.getNetworkConfig().partitionRequestInitialBackoff(), 100);
 		assertEquals(tmConfig.getNetworkConfig().partitionRequestMaxBackoff(), 200);
 		assertEquals(tmConfig.getNetworkConfig().networkBuffersPerChannel(), 10);
-		assertEquals(tmConfig.getNetworkConfig().extraNetworkBuffersPerGate(), 100);
+		assertEquals(tmConfig.getNetworkConfig().floatingNetworkBuffersPerGate(), 100);
 	}
 
 	/**

From 577a7047ac7c4d25a446f621f90aa2301026a8bf Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 4 May 2017 21:40:26 +0200
Subject: [PATCH 169/253] [hotfix] [config] Harmonize configuration keys for
 TaskManager network settings.

This preserves old config keys as deprecated keys where the key was already present
in an earlier release.

This also re-arranges config options to form logical sections in the file
and harmonized JavaDoc formatting style.
---
 .../configuration/TaskManagerOptions.java     | 100 ++++++++-----
 .../runtime/io/network/netty/NettyConfig.java | 136 +++++++-----------
 2 files changed, 116 insertions(+), 120 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
index c5063d145a1fe..8480045bbbcea 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
@@ -29,22 +29,41 @@
 public class TaskManagerOptions {
 
 	// ------------------------------------------------------------------------
-	//  TaskManager Options
+	//  General TaskManager Options
 	// ------------------------------------------------------------------------
 
 	// @TODO Migrate 'taskmanager.*' config options from ConfigConstants
-	
-	/** Whether to kill the TaskManager when the task thread throws an OutOfMemoryError */
-	public static final ConfigOption<Boolean> KILL_ON_OUT_OF_MEMORY =
-			key("taskmanager.jvm-exit-on-oom")
-			.defaultValue(false);
 
-	/** JVM heap size (in megabytes) for the TaskManagers */
+	/**
+	 * JVM heap size (in megabytes) for the TaskManagers
+	 */
 	public static final ConfigOption<Integer> TASK_MANAGER_HEAP_MEMORY =
 			key("taskmanager.heap.mb")
 			.defaultValue(1024);
-		   
-	/** Size of memory buffers used by the network stack and the memory manager (in bytes). */
+
+	/**
+	 * Whether to kill the TaskManager when the task thread throws an OutOfMemoryError
+	 */
+	public static final ConfigOption<Boolean> KILL_ON_OUT_OF_MEMORY =
+			key("taskmanager.jvm-exit-on-oom")
+			.defaultValue(false);
+
+	/**
+	 * Whether the quarantine monitor for task managers shall be started. The quarantine monitor
+	 * shuts down the actor system if it detects that it has quarantined another actor system
+	 * or if it has been quarantined by another actor system.
+	 */
+	public static final ConfigOption<Boolean> EXIT_ON_FATAL_AKKA_ERROR =
+			key("taskmanager.exit-on-fatal-akka-error")
+			.defaultValue(false);
+
+	// ------------------------------------------------------------------------
+	//  Managed Memory Options
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Size of memory buffers used by the network stack and the memory manager (in bytes).
+	 */
 	public static final ConfigOption<Integer> MEMORY_SEGMENT_SIZE =
 			key("taskmanager.memory.segment-size")
 			.defaultValue(32768);
@@ -73,7 +92,9 @@ public class TaskManagerOptions {
 			key("taskmanager.memory.off-heap")
 			.defaultValue(false);
 
-	/** Whether TaskManager managed memory should be pre-allocated when the TaskManager is starting. */
+	/**
+	 * Whether TaskManager managed memory should be pre-allocated when the TaskManager is starting.
+	 */
 	public static final ConfigOption<Boolean> MANAGED_MEMORY_PRE_ALLOCATE =
 			key("taskmanager.memory.preallocate")
 			.defaultValue(false);
@@ -94,53 +115,65 @@ public class TaskManagerOptions {
 			key("taskmanager.network.numberOfBuffers")
 			.defaultValue(2048);
 
-	/** Fraction of JVM memory to use for network buffers. */
+	/**
+	 * Fraction of JVM memory to use for network buffers.
+	 */
 	public static final ConfigOption<Float> NETWORK_BUFFERS_MEMORY_FRACTION =
 			key("taskmanager.network.memory.fraction")
 			.defaultValue(0.1f);
 
-	/** Minimum memory size for network buffers (in bytes) */
+	/**
+	 * Minimum memory size for network buffers (in bytes)
+	 */
 	public static final ConfigOption<Long> NETWORK_BUFFERS_MEMORY_MIN =
 			key("taskmanager.network.memory.min")
 			.defaultValue(64L << 20); // 64 MB
 
-	/** Maximum memory size for network buffers (in bytes) */
+	/**
+	 * Maximum memory size for network buffers (in bytes)
+	 */
 	public static final ConfigOption<Long> NETWORK_BUFFERS_MEMORY_MAX =
 			key("taskmanager.network.memory.max")
 			.defaultValue(1024L << 20); // 1 GB
 
-
-	/** Minimum backoff for partition requests of input channels. */
-	public static final ConfigOption<Integer> NETWORK_REQUEST_BACKOFF_INITIAL =
-			key("taskmanager.net.request-backoff.initial")
-			.defaultValue(100);
-
-	/** Maximum backoff for partition requests of input channels. */
-	public static final ConfigOption<Integer> NETWORK_REQUEST_BACKOFF_MAX =
-			key("taskmanager.net.request-backoff.max")
-			.defaultValue(10000);
-
-
 	/**
 	 * Number of network buffers to use for each outgoing/ingoing channel (subpartition/input channel).
 	 *
 	 * Reasoning: 1 buffer for in-flight data in the subpartition + 1 buffer for parallel serialization
 	 */
 	public static final ConfigOption<Integer> NETWORK_BUFFERS_PER_CHANNEL =
-		key("taskmanager.net.memory.buffers-per-channel")
+			key("taskmanager.network.memory.buffers-per-channel")
 			.defaultValue(2);
 
-	/** Number of extra network buffers to use for each outgoing/ingoing gate (result partition/input gate). */
+	/**
+	 * Number of extra network buffers to use for each outgoing/ingoing gate (result partition/input gate).
+	 */
 	public static final ConfigOption<Integer> NETWORK_EXTRA_BUFFERS_PER_GATE =
-		key("taskmanager.net.memory.extra-buffers-per-gate")
+			key("taskmanager.network.memory.floating-buffers-per-gate")
 			.defaultValue(8);
 
+	/**
+	 * Minimum backoff for partition requests of input channels.
+	 */
+	public static final ConfigOption<Integer> NETWORK_REQUEST_BACKOFF_INITIAL =
+			key("taskmanager.network.request-backoff.initial")
+			.defaultValue(100)
+			.withDeprecatedKeys("taskmanager.net.request-backoff.initial");
+
+	/**
+	 * Maximum backoff for partition requests of input channels.
+	 */
+	public static final ConfigOption<Integer> NETWORK_REQUEST_BACKOFF_MAX =
+			key("taskmanager.network.request-backoff.max")
+			.defaultValue(10000)
+			.withDeprecatedKeys("taskmanager.net.request-backoff.max");
+
 	/**
 	 * Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue
 	 * lengths.
 	 */
 	public static final ConfigOption<Boolean> NETWORK_DETAILED_METRICS =
-			key("taskmanager.net.detailed-metrics")
+			key("taskmanager.network.detailed-metrics")
 			.defaultValue(false);
 
 	// ------------------------------------------------------------------------
@@ -176,15 +209,6 @@ public class TaskManagerOptions {
 			key("task.checkpoint.alignment.max-size")
 			.defaultValue(-1L);
 
-	/**
-	 * Whether the quarantine monitor for task managers shall be started. The quarantine monitor
-	 * shuts down the actor system if it detects that it has quarantined another actor system
-	 * or if it has been quarantined by another actor system.
-	 */
-	public static final ConfigOption<Boolean> EXIT_ON_FATAL_AKKA_ERROR =
-		key("taskmanager.exit-on-fatal-akka-error")
-		.defaultValue(false);
-
 	// ------------------------------------------------------------------------
 
 	/** Not intended to be instantiated */
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java
index b9a1b9013fc1f..e716a82886e84 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java
@@ -19,6 +19,8 @@
 package org.apache.flink.runtime.io.network.netty;
 
 import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.net.SSLUtils;
 import org.slf4j.Logger;
@@ -38,19 +40,40 @@ public class NettyConfig {
 
 	// - Config keys ----------------------------------------------------------
 
-	public static final String NUM_ARENAS = "taskmanager.net.num-arenas";
-
-	public static final String NUM_THREADS_SERVER = "taskmanager.net.server.numThreads";
-
-	public static final String NUM_THREADS_CLIENT = "taskmanager.net.client.numThreads";
-
-	public static final String CONNECT_BACKLOG = "taskmanager.net.server.backlog";
-
-	public static final String CLIENT_CONNECT_TIMEOUT_SECONDS = "taskmanager.net.client.connectTimeoutSec";
-
-	public static final String SEND_RECEIVE_BUFFER_SIZE = "taskmanager.net.sendReceiveBufferSize";
-
-	public static final String TRANSPORT_TYPE = "taskmanager.net.transport";
+	public static final ConfigOption<Integer> NUM_ARENAS = ConfigOptions
+			.key("taskmanager.network.netty.num-arenas")
+			.defaultValue(-1)
+			.withDeprecatedKeys("taskmanager.net.num-arenas");
+
+	public static final ConfigOption<Integer> NUM_THREADS_SERVER = ConfigOptions
+			.key("taskmanager.network.netty.server.numThreads")
+			.defaultValue(-1)
+			.withDeprecatedKeys("taskmanager.net.server.numThreads");
+
+	public static final ConfigOption<Integer> NUM_THREADS_CLIENT = ConfigOptions
+			.key("taskmanager.network.netty.client.numThreads")
+			.defaultValue(-1)
+			.withDeprecatedKeys("taskmanager.net.client.numThreads");
+
+	public static final ConfigOption<Integer> CONNECT_BACKLOG = ConfigOptions
+			.key("taskmanager.network.netty.server.backlog")
+			.defaultValue(0) // default: 0 => Netty's default
+			.withDeprecatedKeys("taskmanager.net.server.backlog");
+
+	public static final ConfigOption<Integer> CLIENT_CONNECT_TIMEOUT_SECONDS = ConfigOptions
+			.key("taskmanager.network.netty.client.connectTimeoutSec")
+			.defaultValue(120) // default: 120s = 2min
+			.withDeprecatedKeys("taskmanager.net.client.connectTimeoutSec");
+
+	public static final ConfigOption<Integer> SEND_RECEIVE_BUFFER_SIZE = ConfigOptions
+			.key("taskmanager.network.netty.sendReceiveBufferSize")
+			.defaultValue(0) // default: 0 => Netty's default
+			.withDeprecatedKeys("taskmanager.net.sendReceiveBufferSize");
+
+	public static final ConfigOption<String> TRANSPORT_TYPE = ConfigOptions
+			.key("taskmanager.network.netty.transport")
+			.defaultValue("nio")
+			.withDeprecatedKeys("taskmanager.net.transport");
 
 	// ------------------------------------------------------------------------
 
@@ -111,101 +134,50 @@ public int getNumberOfSlots() {
 		return numberOfSlots;
 	}
 
-	// ------------------------------------------------------------------------
-	// Setters
-	// ------------------------------------------------------------------------
-
-	public NettyConfig setServerConnectBacklog(int connectBacklog) {
-		checkArgument(connectBacklog >= 0);
-		config.setInteger(CONNECT_BACKLOG, connectBacklog);
-
-		return this;
-	}
-
-	public NettyConfig setServerNumThreads(int numThreads) {
-		checkArgument(numThreads >= 0);
-		config.setInteger(NUM_THREADS_SERVER, numThreads);
-
-		return this;
-	}
-
-	public NettyConfig setClientNumThreads(int numThreads) {
-		checkArgument(numThreads >= 0);
-		config.setInteger(NUM_THREADS_CLIENT, numThreads);
-
-		return this;
-	}
-
-	public NettyConfig setClientConnectTimeoutSeconds(int connectTimeoutSeconds) {
-		checkArgument(connectTimeoutSeconds >= 0);
-		config.setInteger(CLIENT_CONNECT_TIMEOUT_SECONDS, connectTimeoutSeconds);
-
-		return this;
-	}
-
-	public NettyConfig setSendAndReceiveBufferSize(int bufferSize) {
-		checkArgument(bufferSize >= 0);
-		config.setInteger(SEND_RECEIVE_BUFFER_SIZE, bufferSize);
-
-		return this;
-	}
-
-	public NettyConfig setTransportType(String transport) {
-		if (transport.equals("nio") || transport.equals("epoll") || transport.equals("auto")) {
-			config.setString(TRANSPORT_TYPE, transport);
-		}
-		else {
-			throw new IllegalArgumentException("Unknown transport type.");
-		}
-
-		return this;
-	}
-
 	// ------------------------------------------------------------------------
 	// Getters
 	// ------------------------------------------------------------------------
 
 	public int getServerConnectBacklog() {
-		// default: 0 => Netty's default
-		return config.getInteger(CONNECT_BACKLOG, 0);
+		return config.getInteger(CONNECT_BACKLOG);
 	}
 
 	public int getNumberOfArenas() {
 		// default: number of slots
-		return config.getInteger(NUM_ARENAS, numberOfSlots);
+		final int configValue = config.getInteger(NUM_ARENAS);
+		return configValue == -1 ? numberOfSlots : configValue;
 	}
 
 	public int getServerNumThreads() {
 		// default: number of task slots
-		return config.getInteger(NUM_THREADS_SERVER, numberOfSlots);
+		final int configValue = config.getInteger(NUM_THREADS_SERVER);
+		return configValue == -1 ? numberOfSlots : configValue;
 	}
 
 	public int getClientNumThreads() {
 		// default: number of task slots
-		return config.getInteger(NUM_THREADS_CLIENT, numberOfSlots);
+		final int configValue = config.getInteger(NUM_THREADS_CLIENT);
+		return configValue == -1 ? numberOfSlots : configValue;
 	}
 
 	public int getClientConnectTimeoutSeconds() {
-		// default: 120s = 2min
-		return config.getInteger(CLIENT_CONNECT_TIMEOUT_SECONDS, 120);
+		return config.getInteger(CLIENT_CONNECT_TIMEOUT_SECONDS);
 	}
 
 	public int getSendAndReceiveBufferSize() {
-		// default: 0 => Netty's default
-		return config.getInteger(SEND_RECEIVE_BUFFER_SIZE, 0);
+		return config.getInteger(SEND_RECEIVE_BUFFER_SIZE);
 	}
 
 	public TransportType getTransportType() {
-		String transport = config.getString(TRANSPORT_TYPE, "nio");
-
-		if (transport.equals("nio")) {
-			return TransportType.NIO;
-		}
-		else if (transport.equals("epoll")) {
-			return TransportType.EPOLL;
-		}
-		else {
-			return TransportType.AUTO;
+		String transport = config.getString(TRANSPORT_TYPE);
+
+		switch (transport) {
+			case "nio":
+				return TransportType.NIO;
+			case "epoll":
+				return TransportType.EPOLL;
+			default:
+				return TransportType.AUTO;
 		}
 	}
 

From 501fa9ed4a11a6aee95ae5fd729daa22cbd0579f Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 5 May 2017 13:41:29 +0200
Subject: [PATCH 170/253] [FLINK-6470] [core] Add a utility to parse memory
 sizes

---
 .../flink/configuration/MemorySize.java       | 251 ++++++++++++++++++
 .../flink/configuration/MemorySizeTest.java   | 204 ++++++++++++++
 2 files changed, 455 insertions(+)
 create mode 100644 flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
new file mode 100644
index 0000000000000..c34c721e9553a
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.configuration;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.util.Locale;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * MemorySize is a representation of a number of bytes, viewable in different units.
+ * 
+ * <h2>Parsing</h2>
+ * 
+ * The size can be parsed from a text expression. If the expression is a pure number,
+ * the value will be interpreted as bytes.
+ * 
+ * <p>To make larger values more compact, the common size suffixes are supported:
+ * 
+ * <ul>
+ *     <li>q or 1b or 1bytes (bytes)
+ *     <li>1k or 1kb or 1kibibytes (interpreted as kibibytes = 1024 bytes)
+ *     <li>1m or 1mb or 1mebibytes (interpreted as mebibytes = 1024 kibibytes)
+ *     <li>1g or 1gb or 1gibibytes (interpreted as gibibytes = 1024 mebibytes)
+ *     <li>1t or 1tb or 1tebibytes (interpreted as tebibytes = 1024 gibibytes)
+ * </ul>
+ */
+@PublicEvolving
+public class MemorySize implements java.io.Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private static final String[] BYTES_UNITS = { "b", "bytes" };
+
+	private static final String[] KILO_BYTES_UNITS = { "k", "kb", "kibibytes" };
+
+	private static final String[] MEGA_BYTES_UNITS = { "m", "mb", "mebibytes" };
+
+	private static final String[] GIGA_BYTES_UNITS = { "g", "gb", "gibibytes" };
+
+	private static final String[] TERA_BYTES_UNITS = { "t", "tb", "tebibytes" };
+
+	private static final String ALL_UNITS = concatenateUnits(
+			BYTES_UNITS, KILO_BYTES_UNITS, MEGA_BYTES_UNITS, GIGA_BYTES_UNITS, TERA_BYTES_UNITS);
+
+	// ------------------------------------------------------------------------
+
+	/** The memory size, in bytes */
+	private final long bytes;
+
+	/**
+	 * Constructs a new MemorySize.
+	 * 
+	 * @param bytes The size, in bytes. Must be zero or larger.
+	 */
+	public MemorySize(long bytes) {
+		checkArgument(bytes >= 0, "bytes must be >= 0");
+		this.bytes = bytes;
+	}
+
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Gets the memory size in bytes.
+	 */
+	public long getBytes() {
+		return bytes;
+	}
+
+	/**
+	 * Gets the memory size in Kibibytes (= 1024 bytes).
+	 */
+	public long getKibiBytes() {
+		return bytes >> 10;
+	}
+
+	/**
+	 * Gets the memory size in Mebibytes (= 1024 Kibibytes).
+	 */
+	public long getMebiBytes() {
+		return bytes >> 20;
+	}
+
+	/**
+	 * Gets the memory size in Gibibytes (= 1024 Mebibytes).
+	 */
+	public long getGibiBytes() {
+		return bytes >> 30;
+	}
+
+	/**
+	 * Gets the memory size in Tebibytes (= 1024 Gibibytes).
+	 */
+	public long getTebiBytes() {
+		return bytes >> 40;
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public int hashCode() {
+		return (int) (bytes ^ (bytes >>> 32));
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		return obj == this || obj.getClass() == this.getClass() && ((MemorySize) obj).bytes == this.bytes;
+	}
+
+	@Override
+	public String toString() {
+		return bytes + " bytes";
+	}
+
+	// ------------------------------------------------------------------------
+	//  Parsing
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Parses the given string as as MemorySize.
+	 * The supported expressions are listed under {@link MemorySize}.
+	 * 
+	 * @param text The string to parse
+	 * @return The parsed MemorySize
+	 * 
+	 * @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
+	 */
+	public static MemorySize parse(String text) throws IllegalArgumentException {
+		return new MemorySize(parseBytes(text));
+	}
+
+	/**
+	 * Parses the given string as bytes.
+	 * The supported expressions are listed under {@link MemorySize}.
+	 * 
+	 * @param text The string to parse
+	 * @return The parsed size, in bytes.
+	 * 
+	 * @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
+	 */
+	public static long parseBytes(String text) throws IllegalArgumentException {
+		checkNotNull(text, "text");
+
+		final String trimmed = text.trim();
+		checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
+
+		final int len = trimmed.length();
+		int pos = 0;
+
+		char current;
+		while (pos < len && (current = trimmed.charAt(pos)) >= '0' && current <= '9') {
+			pos++;
+		}
+
+		final String number = trimmed.substring(0, pos);
+		final String unit = trimmed.substring(pos).trim().toLowerCase(Locale.US);
+
+		if (number.isEmpty()) {
+			throw new NumberFormatException("text does not start with a number");
+		}
+
+		final long value;
+		try {
+			value = Long.parseLong(number); // this throws a NumberFormatException on overflow
+		}
+		catch (NumberFormatException e) {
+			throw new IllegalArgumentException("The value '" + number +
+					"' cannot be re represented as 64bit number (numeric overflow).");
+		}
+
+		final long multiplier;
+		if (unit.isEmpty()) {
+			multiplier = 1L;
+		}
+		else {
+			if (matchesAny(unit, BYTES_UNITS)) {
+				multiplier = 1L;
+			}
+			else if (matchesAny(unit, KILO_BYTES_UNITS)) {
+				multiplier = 1024L;
+			}
+			else if (matchesAny(unit, MEGA_BYTES_UNITS)) {
+				multiplier = 1024L * 1024L;
+			}
+			else if (matchesAny(unit, GIGA_BYTES_UNITS)) {
+				multiplier = 1024L * 1024L * 1024L;
+			}
+			else if (matchesAny(unit, TERA_BYTES_UNITS)) {
+				multiplier = 1024L * 1024L * 1024L * 1024L;
+			}
+			else {
+				throw new IllegalArgumentException("Memory size unit '" + unit + 
+						"' does not match any of the recognized units: " + ALL_UNITS);
+			}
+		}
+
+		try {
+			return Math.multiplyExact(value, multiplier);
+		}
+		catch (ArithmeticException e) {
+			throw new IllegalArgumentException("The value '" + text + 
+					"' cannot be re represented as 64bit number of bytes (numeric overflow).");
+		}
+	}
+
+	private static boolean matchesAny(String str, String[] variants) {
+		for (String s : variants) {
+			if (s.equals(str)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	private static String concatenateUnits(final String[]... allUnits) {
+		final StringBuilder builder = new StringBuilder(128);
+
+		for (String[] units : allUnits) {
+			builder.append('(');
+
+			for (String unit : units) {
+				builder.append(unit);
+				builder.append(" | ");
+			}
+
+			builder.setLength(builder.length() - 3);
+			builder.append(") / ");
+		}
+
+		builder.setLength(builder.length() - 3);
+		return builder.toString();
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
new file mode 100644
index 0000000000000..dbdd96b91594f
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.configuration;
+
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+/**
+ * Tests for the {@link MemorySize} class.
+ */
+public class MemorySizeTest {
+
+	@Test
+	public void testUnitConversion() {
+		final MemorySize zero = new MemorySize(0);
+		assertEquals(0, zero.getBytes());
+		assertEquals(0, zero.getKibiBytes());
+		assertEquals(0, zero.getMebiBytes());
+		assertEquals(0, zero.getGibiBytes());
+		assertEquals(0, zero.getTebiBytes());
+
+		final MemorySize bytes = new MemorySize(955);
+		assertEquals(955, bytes.getBytes());
+		assertEquals(0, bytes.getKibiBytes());
+		assertEquals(0, bytes.getMebiBytes());
+		assertEquals(0, bytes.getGibiBytes());
+		assertEquals(0, bytes.getTebiBytes());
+
+		final MemorySize kilos = new MemorySize(18500);
+		assertEquals(18500, kilos.getBytes());
+		assertEquals(18, kilos.getKibiBytes());
+		assertEquals(0, kilos.getMebiBytes());
+		assertEquals(0, kilos.getGibiBytes());
+		assertEquals(0, kilos.getTebiBytes());
+
+		final MemorySize megas = new MemorySize(15 * 1024 * 1024);
+		assertEquals(15_728_640, megas.getBytes());
+		assertEquals(15_360, megas.getKibiBytes());
+		assertEquals(15, megas.getMebiBytes());
+		assertEquals(0, megas.getGibiBytes());
+		assertEquals(0, megas.getTebiBytes());
+
+		final MemorySize teras = new MemorySize(2L * 1024 * 1024 * 1024 * 1024 + 10);
+		assertEquals(2199023255562L, teras.getBytes());
+		assertEquals(2147483648L, teras.getKibiBytes());
+		assertEquals(2097152, teras.getMebiBytes());
+		assertEquals(2048, teras.getGibiBytes());
+		assertEquals(2, teras.getTebiBytes());
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testInvalid() {
+		new MemorySize(-1);
+	}
+
+	@Test
+	public void testStandardUtils() throws IOException {
+		final MemorySize size = new MemorySize(1234567890L);
+		final MemorySize cloned = CommonTestUtils.createCopySerializable(size);
+
+		assertEquals(size, cloned);
+		assertEquals(size.hashCode(), cloned.hashCode());
+		assertEquals(size.toString(), cloned.toString());
+	}
+
+	@Test
+	public void testParseBytes() {
+		assertEquals(1234, MemorySize.parseBytes("1234"));
+		assertEquals(1234, MemorySize.parseBytes("1234b"));
+		assertEquals(1234, MemorySize.parseBytes("1234 b"));
+		assertEquals(1234, MemorySize.parseBytes("1234bytes"));
+		assertEquals(1234, MemorySize.parseBytes("1234 bytes"));
+	}
+
+	@Test
+	public void testParseKibiBytes() {
+		assertEquals(667766, MemorySize.parse("667766k").getKibiBytes());
+		assertEquals(667766, MemorySize.parse("667766 k").getKibiBytes());
+		assertEquals(667766, MemorySize.parse("667766kb").getKibiBytes());
+		assertEquals(667766, MemorySize.parse("667766 kb").getKibiBytes());
+		assertEquals(667766, MemorySize.parse("667766kibibytes").getKibiBytes());
+		assertEquals(667766, MemorySize.parse("667766 kibibytes").getKibiBytes());
+	}
+
+	@Test
+	public void testParseMebiBytes() {
+		assertEquals(7657623, MemorySize.parse("7657623m").getMebiBytes());
+		assertEquals(7657623, MemorySize.parse("7657623 m").getMebiBytes());
+		assertEquals(7657623, MemorySize.parse("7657623mb").getMebiBytes());
+		assertEquals(7657623, MemorySize.parse("7657623 mb").getMebiBytes());
+		assertEquals(7657623, MemorySize.parse("7657623mebibytes").getMebiBytes());
+		assertEquals(7657623, MemorySize.parse("7657623 mebibytes").getMebiBytes());
+	}
+
+	@Test
+	public void testParseGibiBytes() {
+		assertEquals(987654, MemorySize.parse("987654g").getGibiBytes());
+		assertEquals(987654, MemorySize.parse("987654 g").getGibiBytes());
+		assertEquals(987654, MemorySize.parse("987654gb").getGibiBytes());
+		assertEquals(987654, MemorySize.parse("987654 gb").getGibiBytes());
+		assertEquals(987654, MemorySize.parse("987654gibibytes").getGibiBytes());
+		assertEquals(987654, MemorySize.parse("987654 gibibytes").getGibiBytes());
+	}
+
+	@Test
+	public void testParseTebiBytes() {
+		assertEquals(1234567, MemorySize.parse("1234567t").getTebiBytes());
+		assertEquals(1234567, MemorySize.parse("1234567 t").getTebiBytes());
+		assertEquals(1234567, MemorySize.parse("1234567tb").getTebiBytes());
+		assertEquals(1234567, MemorySize.parse("1234567 tb").getTebiBytes());
+		assertEquals(1234567, MemorySize.parse("1234567tebibytes").getTebiBytes());
+		assertEquals(1234567, MemorySize.parse("1234567 tebibytes").getTebiBytes());
+	}
+
+	@Test
+	public void testUpperCase() {
+		assertEquals(1L, MemorySize.parse("1 B").getBytes());
+		assertEquals(1L, MemorySize.parse("1 K").getKibiBytes());
+		assertEquals(1L, MemorySize.parse("1 M").getMebiBytes());
+		assertEquals(1L, MemorySize.parse("1 G").getGibiBytes());
+		assertEquals(1L, MemorySize.parse("1 T").getTebiBytes());
+	}
+
+	@Test
+	public void testTrimBeforeParse() {
+		assertEquals(155L, MemorySize.parseBytes("      155      "));
+		assertEquals(155L, MemorySize.parseBytes("      155      bytes   "));
+	}
+
+	@Test
+	public void testParseInvalid() {
+		// null
+		try {
+			MemorySize.parseBytes(null);
+			fail("exception expected");
+		} catch (NullPointerException ignored) {}
+
+		// empty
+		try {
+			MemorySize.parseBytes("");
+			fail("exception expected");
+		} catch (IllegalArgumentException ignored) {}
+
+		// brank
+		try {
+			MemorySize.parseBytes("     ");
+			fail("exception expected");
+		} catch (IllegalArgumentException ignored) {}
+
+		// no number
+		try {
+			MemorySize.parseBytes("foobar or fubar or foo bazz");
+			fail("exception expected");
+		} catch (IllegalArgumentException ignored) {}
+
+		// wrong unit
+		try {
+			MemorySize.parseBytes("16 gjah");
+			fail("exception expected");
+		} catch (IllegalArgumentException ignored) {}
+
+		// multiple numbers
+		try {
+			MemorySize.parseBytes("16 16 17 18 bytes");
+			fail("exception expected");
+		} catch (IllegalArgumentException ignored) {}
+
+		// negavive number
+		try {
+			MemorySize.parseBytes("-100 bytes");
+			fail("exception expected");
+		} catch (IllegalArgumentException ignored) {}
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testParseNumberOverflow() {
+		MemorySize.parseBytes("100000000000000000000000000000000 bytes");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testParseNumberTimeUnitOverflow() {
+		MemorySize.parseBytes("100000000000000 tb");
+	}
+}

From a7d3ba84b6f6f1c018061dfc4a133838f6a882d4 Mon Sep 17 00:00:00 2001
From: David Anderson <david@alpinegizmo.com>
Date: Thu, 4 May 2017 17:02:48 +0200
Subject: [PATCH 171/253] [FLINK-6447] [docs] Update aws/emr docs

This closes #3828
---
 docs/fig/flink-on-emr.png | Bin 103880 -> 0 bytes
 docs/setup/aws.md         |  21 +++++++++++++++------
 2 files changed, 15 insertions(+), 6 deletions(-)
 delete mode 100644 docs/fig/flink-on-emr.png

diff --git a/docs/fig/flink-on-emr.png b/docs/fig/flink-on-emr.png
deleted file mode 100644
index f71c004e43144ec236c7bfd2207a0419fdf43946..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 103880
zcmeEtRajk1(<bgtaEBnl-8FcCKyY_=clY2BEVu-B*|=+PcXxO9+4;`-zmuGKp1GKt
zxtV>n_Uf+guIlcRw`zTr`z($KhYJS+0)i+hAtDa~0^SJ%0_Fh&1$@(O{jC551g^?V
zSXfR{SeRJO-p1I>(g*}Z;%h=8w4#C<ZvUCbL1dy_Uf?Rkfzqe_Y?w(Q#gAYl@!04e
zYe<+m{SAafAf{0X!l*bg1^fI}QH<mid-#Q*AdT>8L&FAnbzym$O<cxa9jx2d9#*~9
zURJ%%GOnO~Ju}HsestP{{J3HX%x)vR-cMHL{tQMe2qppo&KypM#`y6g7-&x8y^B{v
z$gr4Wzh%>GyUtrh+}``Jc@RMo7)o1_W5+;QlH7DU6k`zIk5J<ygDBDtTwSarlf+%(
z1>bpwPNvcrhg?i{14G<Nc@aP|hV@|^L1I?P+}tTr!>386Q0mqlTs;F@QJjq^YdB%B
zh+7!>yk=O5#OJ9tG7oLvlv#w-zx8f@a=TSC4!qm^3Sm~7_U6FzT0c3}<6yB*9FRtP
z4KJDlS8d`{R08&6SXLE{x1V=(_!;Lj=9B0!lg2g<Pd|R5N(X+p0vWc$Fxu$a(2#EM
zSe30(VJ*Lam1I_}H>twlSJJaeuFsnfjC-6MT)gH{EN-z<yQ=ZHn2ZAb3Mp7&!g+j@
zuwFmLlIc^<B1ckmXvnyhZQ4Ri%>>d!7;MZkh)B{IexAq7zWWh0e3!XPGZN0+{o~G<
zm}nlqpz>LG_b5=}4%!ve1|{@)pBPLP)I%4x)Q<>eP+~aaHQ#TKS|}=}UuHCmB}U<!
zgvsT?F8qNqW}-8QbY|y*%Jb>#@PdE*`BmdE1PCVC8MJ-MSTD&@#sa8s+MMTS<gX&|
zAoT22y<Ni#`0kI$0cn<Mg5br1==)$~y1~~b&}n`=lT`vRP#eThr686u2$Oy&hJFBD
zGAf8(UFdm4>JCs_m}x(QbqG^XlTJ}vjAf8|K_XX(_qs4GU<T{p$$`W$&|iqDf54du
zJ;r=6gya$|kKuR+6ZDZuj<+Z1J_cJI-pW5L8?6wG8)imWK-eTJTn=j*p3fI53w59F
z8b_?tFd53C1L~Y~9@H2(8D|UrZdb{MW&_A7R8sfM`IiQqHkdWvCtcWQI6`sIcd&VZ
zlVabq@#QiKbror;VaY|x#4)l5<z@=S6!D7DouJ8uB6Ik&t4yg=qS%Fba$m+#P1)Vq
z-1*#Tn!vO|ti^Q1)`XC=`p4H8-$iiuz1s@Z)6Zejl*Y_QQ3x*TOzYF^yw(e5B&t*`
zMVm#jh`#CW(^aUnsMM%>s+5^`wq$Td;fd<rP_)bE#L{Gp3L5Ib*u340+N58%-U_xQ
zx_IA$xe9j{5Z!xq@o>-ljQZ^MZ1oKP{9Y(3OV*YY0-ZAuzstEx?<3v@g<dG5WIQD<
znpWsW&$~@AeXgol*P#34`zR68gBW~CDM?;=!lE?CG{;C*!AB8g;?x+v0Xst`H@wEg
z<<H)N-s0JE2vn!>s-K{Y@bWV3zh1^)ehH>Fpzn^~_-Pqa6mOQ;NOz^4NL<1&6r)9w
zN~1yHA%B~HT!=3;DmbbTMP9F6E`J)K*}mMstF$W5ugb6GUGj|jK>wfznda~3Uk@gp
zEw~ZWMIIzX9ab!L^P{>bwJ@YerBIlab|j|nN8WgtS-&Y?Et4puC|<8buVwG2UWYI=
z^7uGEG=&c`X3dkZc^;oK6WHQ$2EPt!#N#E{(8w0j%Gs$%Dt?eRlRr>+DE*{zqb#Ge
zQnZ-!D1umKoKvSrCuP;Xhyj>@tc4tfpN9{_OjA9VBT(k}f~=&I>n+i$;iW08(5)hz
zRi&~O*H@Sym)~q0W?Z!EHiAl5oLGV>TcUX6KP%Nlnl2=pn>58R)n7XUc&Y7Oga&xj
zQaT|xkvd*Fr8s8nqd5^cesh#^;&JT1uD$lZo;_sS=bNV7=h>$`I-59~&SYd{4P~CU
z9M@;GEY$zWI?QIt=xHg!WY3~*rD)|ihi!4Jn`YiJcAp4qsc#WyDQ{*!^^P6Dvh)3v
z<wPq~ZNDL=z0zH;9cLjzFVSpPmtD8qkkB;4W>x>yJ$5x`UX%yRiT=@8&mz{?ZhkX1
z&p)r-sLd#KM}r2PrcNnuM#~}$5Pbu6q<qA|p3nZ3{mJTdN!}{Jl4Hqv-WyO;kG{yZ
z7&(8m(7&XzaK|%_{}I2AThpP3)1A}WmiOp>U|{q^_d(!P=BV*#(WrP@2o@gwXH8Gn
zi>&Y{_@SMw@ScQI+_T5rk{+3*$Q|XA+hMJ_^eyeJSAV@>2|awHmfOm66!)~-<a5VM
z&Ao-wpv%h}>dV6m<@2cX@^kOQ+^y~#vvZs)z>U_{(rL&A{Q1(s+}N~k_4k`$CLfPC
zr`II#pP+i67@%WdGGIY|RzkKhKI7$C{ZiLS=Sj5?0lLQL<?ECr!6YTZ-fCf5xYf1x
z2Ul`K<ACgRz3IvXJ@GF7K#xEhWS@6>??|HdL*pYO!b_vmLNCLpzs89$glS`Wk%`d-
zV@=_Phet+~eI?M@ZMD3s&gNWd=xXp7LQFDYI8)EABem*nOsJ+&qmk`1*v#FG?(xxs
ztSMI8f*lSl4rHpM^gw>X=ucM|#F2EAq?PoH-Hua?V;y$dINgkT-%OrF9(~E{hS9?D
z!TSTWJcWFKf`vlCWX@#D<i)|ofyq7Gv*PQHQc6j+$_<<}WDwYZdBIGxN;E8ZARDPf
zt>$!!knr&5`gw`xbXVdjwi@Ou17*Y60m}zVZ)cY%$<f3#tVaGufij^Ay@~_Xl2+%f
zML(wqC+IohBlKAp%kdStN<FGEjtXlsJA=bMoh!OasmnE+u*GTRtS>PtjQ8nhE{H+Q
zL0w(1BcrLvp3$!vcV1(dM|2+bo-~{9v>58B=xJW76IyQ`krM^t-xvXOW*TG8%_mPT
zx-RD~%S#7!HRU30@9Pb1RmRJA<=xG`96YVixVR)N#1qU5FO#j3eL$wQSt+S^>0f2Q
z7{KsKAWV=^I#kjqrnt4cU0FSAYrpXA^V<Z^^x1xKeakxQ+%*q4tQMXZ{vMJ+6veyz
z@U2_+hRvL<mrcjw0T9<T-ZXS`dvhZ`9c^jji`RolCcYK5%DwAU3YaLp(ETQ_aA9Yw
z>G`}fb|J9FgKC?z^`m>BPvNnk__Km67Mt)0*N6e#0A1W{^6V<hs|ojF+S2DY{a2|I
zwUe5Jy?0|)0<ye1w%&&yPlGO;g`W0BPkTF~5(uK{MRl4y+zxg!Xh2oUwQd^Si@r^`
zZKMnVYP3FUnUwQr+BE99<TftVJzZDSwP&1G9htS6`PAGbZk5%o_O|ZXG_SO|_8*PT
zG!Hj(JSJY>tcEwZv<sB$*k?NU+zTxRa3He~edZtZh;Y-q7CDt%P`D|WX~%h+eP8u0
z4euOx-y8Gxc!JpDeC8}?tsMb_w}6+yBl{-Dd*1y1ICSi1S<;X;QY%_Z3O?Pl@N0DO
zoLGhI%fYM3o$B@cINnBTQN6}mQCo?Z%~hKLui^7S$TjjebWtKDfmtv6`{@U+!>jOt
zQicYG9i7PbIxqP9TUTl?^G_XU)yAbDa5%Q|XdpHgpjka2f~`UKkl)p!dC5#?=f+U(
zWqFTsO#(TS)LlSGQ4xNEugmd(JW9MLWr7$8X$GHPROfyr`4R`1v80oS*OwX0&tD~3
zMKM9>n0qZM8b~ZEs$i3-K?XH_V>}uuAP>3sx#JaMB|A1&#JX2`fgI<Al)8U1d;|dj
zO*d0ibySs+<~FdgqSrID(Kn)ZwXy{|w;&+AuH3*!D<eleVpl6mYX@#uKGOg6;08Yb
zdd)ye{GTq47JQ_tGIGSiHugruZ1k-3jHLW<#KgqB_J+pX@*-k?lLP<pk(xR>+Hx~6
zxVX5`yRgvP*qbmgadB}mFfubRGt&Wk&^fqSJL<X8Sv!#ZHOT*rBVy!WU~gvYXl7$g
z{A*l2eH$l7K2p+O6aDx1*E)?{&HkCm+Tm}ufDAJHYGGiaXJq*A*g#U=UvIhP%v_Bu
zRYlCKjI14ibMSMqGx7eX|NqwV&y2rOs{KRB&cgOP<!>#2Q}QzWV&FH1{*u;z-U7wN
z568>!U(xf!aaItef`ABuNQww4x`H0BAZjWpwF$gN!-jTA%GZ9&M#4_`dQHscjN=jG
z&UG<~mLII570tqqKp}&M)!fO7n>zwYNzl~@e!$2yWp_AM)-jp6>tvgcEoPIA@kKQv
z=99DHM}(MM{E#=7ldGRs-G19YO~&9COwIta+DEO<QEN4`+8&Cjmg{1maB{w2@3KH3
zsRaLe#0CXRiu7f#{`&jNzuIMdg&1uh{x<?9Fa&>bX_cDne^Yc2C*S=u@ZWPpV1q(+
zlI52y$o+dDU6f&uKi2{z5sLr`SVx_jv84L%3jZF65oVX~?=}DZ2FMASn6J=z=z%$V
z{2xpKJ77ax5&qNfuci8-l64Tf_BgWPefqZ?5d7Vc|0S8Mj!;xx6kh$N6vVuLvsKYa
zZYn18Amn+zVZz9kJ{NYIStx$aR>SnvCkKIWIRjOH8laQXSdQ8MxnB1eLjmR8{#*q-
zYihA=<*|XM^K_utK+)Aj2)`kwb<>U*gqEU2&Gkh{VG$3!&DjM^s}mAfiXx8&(+AG`
zxMKsC$10ul(VjoV-4O#HVgAjEv0T3S&FDuNd)Ks;9(PaaBL*7Q@G*ODo^uZGm~T87
zx+BwfD(SST`dv0f(!Hdytiq2$ZCW;ZOlGd^cKl#X{Ctk$BT7F#-=p@@D}^uBBD%D_
zVWUM->?5Y?Y9404SeOa!6o^kO!bvCF2{$v~LQO{t&%x7v|3lig!n$(#!YY{g2H!0}
zO5ueJ9a(d7Nl|lZ=E_k;@svp)cI==0q+aXmWwU1cB>7jpEG$;MX?;wMnn5^eN#Q1#
zCPHmL=HyU;Ba>rgHQMAMsTVgPyKsW(rt?cbDkG;?P<yA{;L4alpQx|73+*9owad%L
z#HIMxBXKynJ-kNlxTrxO%l~ndu+%zEU<VO*+*H47m|FxBUVO(1O+}wBSIfWr$eR#X
z5{agQtsrOn#NHFc{KGWlJ+~Tc$rmdgY_uA<c6UcmFaWUvfwWl70Z1KhYFIk;7g55D
zB|AK3OsFM$gqMmfjF3{SiT(|8gBFALi;6_>0UHn$!%gn{G)#&r*e8fL2|+0X;}<;P
z0~`!~pp+<%dlA`9+(Un!qBe3+j(~NjmRC=|vSRM{JI{ph!USJZb73FPRiF8xRpXOO
z2x#W*Q0xY-@b?IKFj)?9adoR)J5~0gLE1-+xF#2!hY=~J2RhP|DzF@1{q~uLKn6nw
z73M(hp@<nf)QB7$z*E$R<>47zBMIPr1OyiF?u{3p#E)xbSZ4c6yLWpU)8VeB#)Rx|
z#qibRb0CL|AbU+!_YuShaDI#;BFZpo&~D~@bHCexhh~umI8EZ4$=bN}J#;u(Y6zHs
zCf+hGE$Le)KC*U^8XbZCh+g)yXRQuTw|@L|lReYzo*8j-ll1hX;8Q~#(terO=w-;y
z=J5PWls~RDBcopish_qKtIA0*?`5$SiQRz~RLAW{PvKltqPTEgoL);_N6pm1BbX6d
zr}-))vg+``;{m{0QziBffp5O&f*x!m^XU%?mQ>b>z8>g(XA7YKQH+zzzyX*;hp+JK
zR2u0g4i6(#650_haD2);?V&Nq_3H=6>9avamSZ<5gD%8R50>!@izh7&gmxCQq~5E7
zV#awd5509cq?Hue4j{)3Q8t`t2&fWlVF?Fb_|{SV_#Osy6n4X>+c4iV2=8i?LNkdD
zsruxb@rDwu1RV^~?7dl$j?kIMKg(kUD;;HagW*)!P%(6%e{dV!Yb}Xn?6DR$*mAU7
zOn_XgpNu800qsJ-iv_SQ+aa>5Z0ziDbf}sWt<`0Oh3WVW-Pz3Kd*QKLcTd_1s7jN&
z4pu53q69}SRTGD3vd=I;9VCe*SzNEG$eYd)%2)Gu`2xo1MJp>{87L{c_Ae2fg~C7~
zCyyq~9wah|0tfwBONVJ38tcl;R}#+h>!rFIrT$QZ2-UuPP55$EnP*EDLj}UeU3l&v
zy0Prqm!g99=yk8L1vvW@>DYcg>iHYIn+C-6g^~W?eMX&aZNhPw(Nxmx_=;<B{XM;5
zn61d5J;xOMwfl|QvJTqdg_?-7fN$2kE0{F^;yXQ!-~a~hpiVGJ?|y^w69GEhEQRjJ
zQRg0KCfrqis}R|d{);xc!r6Y_&~a^dqtQu(V>NTX2p(?n%$XdeaKnjWBvA_@ypk_5
zv7vX!W!-0nfod2&<*$Kkg>)mqo&2dQY&Hgv?C)Qp%NOf#%6u4Xp*reVBO?cQVGIdA
zxU!{BL}dWzVXS2)G^1*JKbXD&f^QV<GMvPGA04%6gbp1pSu{|*XPtXeW?%OjYi8SQ
zTg7C($XkPwh&Vdt`Csd>pcTHO!kWF#zb-pkI>7pt!IsWZKkNr$LTshsUO;PBM3P)v
zzd`x}#Gb$Xp(e5M{e;x8;U7&sRo0}o^AC@9;03<3A9JHSz1hrhf~^-~e0Q~aqCZ}|
z!DTNGs{&NwmiQ>aSYU%i*jc$qAh6@=beHpaRMH8ce{nw=IJD}G%ChR%Spr*+`N{dC
zY^djPzR2~+%>#rs_eW~Pe49n|v=g5FY)@y~tI9wp?i}u|Rr?L?vDBxPht{K~#C|rA
zM0|1v-V+$#n^>C49l(ivy%yu{50WlreWwZY7I|xp&m}JrIwF)-)5mub9?U8t3l6$_
zZ{Hb@LQ-RFhWm^6QI6~m&SW0MEElR_=RVbISQe$X)6T!Nuz*A{mUfm*R(LpD&6U<V
zIx~7u*bqHwHy6#`2An*uIFH+vD(GlM?+(`EgI1;x=q}e0c-$U$)u}k>hE{5|<Up|H
zn&?d>dT!f9)a$@al(2Utw?n2P)Zq7XpCaVB^jlo8{HaD<!K@)VM+%=i*oS8~aK0$<
zIVfdIe`nKuxZEM|-d1%7-9AvjDP4+wPM;+i55?e;h)i%9{S<h-gW-D)$t=o7r=Axl
zniY2Lt<^6_1IE=ZCDOnzh_`SJpAyhdSPwZG8FHQ%cC$=P#7y7}&tA}P@Vd+eGxG*7
zQ+Ll~KEk5Ps89iUrLFA^lYVE=v#{A+6KQT)>!Wcs0Vh5lMc*&v@j+HOVO8J9=F2?X
zGd~|@=jC&C5^q2Wa(Z;c+-#>ZAGXd*$re1MN6{d^n_h!4ve{VLd_Ri8)5nc(hu_W-
z5kAK7!%R^uD*F*~picR3zl?PXjiP#a=pDKU<G@+v4jn%1M&w-?`Z0V>+>weIcf0th
zLoVd}B;-7|48CZyrs8eNbp<F(qvEBT+PYcn-qb#aN9dgl<Xqx<xZLn(@c%&|G#5N1
zVtK@Fw}`MG;J>pn+EzzK;Eh=25>j$O_NTN%!Mw2BPEO4Y7|9hsmV}k>rD~fNB`x%z
ztvh2JE1q@4?s7&XtdC`Q*!w~%B_z=2hRL(R)|IBhW)j$RwFvH==kb?c)#{yHyhaTc
z$5S`pn?ArW5$m-bUA4NuZ>dHUXkt_st5LEw6mk$I`0RZ{&i`ShobTwk!L{8bEjMf5
z9=~=EEH!#5z*H@bcIS1}&#Xg}Hhk6q@Yt1I3g(m;)X??C5Qb0_<?!Bc<sz7D^~6@}
z$&{$>^e_$Iq3Njd{VYGyi8Q{W3F}ApcRY`MZpOYI>5p(QX-^P!t%<%9ud_vW=62;x
zP2cXTchN1^%youGl$IWLdfjfS*9q%vR+>Ya&fonZlW_r14u4$+*g4CI@gj=apH5nH
z%(yJ%XeG)rLG|@~wh7ZSy+7%x65_W?vonU?&8OqgRMmtuwZCnA=?NLL{ZW~)e!sjE
zg|`#3?5@y!TX@;(DZVweVv}=z(n@tgaer1#*ouFh$|QwXqy+l(P(wh@V^YVem~(#?
z5@d(l#-W3mV_!Rk?DwkWa&Y6a4128Dse|}AEkFw87#GI5ZPqu6^N@u=<)EoB`ixa%
z@Q!lk)`kko5!EEW{PB}JeC|l*=7<Vh)g41#;pyQJwh8LXl0&Ucvg=*Nt+dQ<M#2x3
zZd^2Q(k1%hweFxQQq;A6_e&2HPeHUh<Y<pSB+C+p@pp9tc>g7WS^oVS5^NKi@~MTI
z0^1pE55|=P%S@!}`sb2s0wQ&*7>qEYIE=9Lq^XcVJuzFVuI}e5wx#)<ta=tFS&z{=
zjMhdXSO+X~g_h=_FD{iJc-1pWu-p;Nc_{Uq8S!-^bEouMy`20_;o<6^x9*7{JBqJI
zy~tUrjYN+^3`H&W^2Jqq5CTULLf$UQ7#9nodC;#7#$H)38Lv;pX_Eg47s$SJlo4~f
zu99{nmo@IZ9uO>Nwq=E^k3?|TN1Q|w)O;2`-#+6oh++wL{$Y`L2UYL>w7811JeJiR
zc`0lx?K~}X!kW~63gg?YKY2c!>!O<JFe5*3HP|b*-30OTg!%5a(*|mHNnC6QuCl^9
zTl(dc{u|EQd^wmdAc#z21jNP($svQO#H4`7<!Iy0L)<8a4jcWLDeb0X{cXWqhoTAW
z?RhZ5Hi<#JHk@r{^P9J}5>!R=3ioZobt0|4F|w}_Y7M?yFwZok;w(K2CXY(tY2Eti
zX9SM3)5C$SKf(q<O1~j)f`fk0wRMlmuC^4p11Me<)fubPVgd$Ee+duLCsy6>*x_Y1
z%8ST6KB>+I7sALJ*}dU%ss=hli*8Nzd@h4$*P4X}aF*C;=zGl&9rEG3(Q?e=AgUID
zM3}TS6~R_ri`)3q&QqKU15#1C8dw-Xx?f#h@-?B4xRoR4&-JT@9A@f1rXcG|#quJT
z)>SfU&Txs<W#Ih~c%ca4R32JYKac*Lcj{wQGW1cmZYqj_HjTz;iACM~^-%R-!F%||
zrJybDn}KBw+((43C*ZnVD%$d!DXmj&(FSln1qzFP8@nq`bZ&p_@Vos_z%9nEhEO=3
zvakfjmP<(u-BAlFWM-QOV}i$(R-zhKkLo$5U)4QwA=@M^n~I{jR3@!C2f?k4mHtX2
zv-8v4IiPIxOGDX~slH0tfvY7ju?As=aqrLx%31=dVrHZz13QUWROhl>>@sj$ZLZkQ
zy~qze>;(aZgE-5sfSTBwZ{Xq<dBR=2=v)I!4tc;>N>@!jNyg=!;UsYGOk2bCu`R-t
z*8*Bv9kRFGxLsM`>3FMweC?>kHnFTn=qaxqe2A<pj}6m=n9GDi8gZmLCK=Y_YO=EF
zr#9XmM!$5!<ZWl*GaZjyf!7{Mo07+cqQ6s}AB)qO$E*v?&>YRuYXmsO5dxT$1Ho;$
zUf9G#WY8^49rdaIb$Z$S;iWL+cj-cl{oU{+p!9g475)r`*+K96A#SeDHxrdyY4rlw
zn^am^u)bbS)sMhmWq#_<kU!!v3U^SLn4z8dsbmLpD3`^jYvwYKTDYjgI{kLpoa)SD
zY#+nkR763Tw3|05+f2j5pS-Qra;IF@QCF(kU)6R3p=;)h)aLzu;&FAmPu@}@Hq)B9
zzszIRXt36NucF6}eDJ;3&8Q{zcEMJDv~~c|80@}l!%z~l)86<8ha~8h%Jrhi`_wWW
znlSbbVgW6y+b=2=S!Z3u-3wwkmC83h4dIV2!)smy{lu<`o{c%HM6^*wP*2wvQRF{)
z3W?gVF1VIZIA$~wEo!W9UuSv^a~SPD_e)>8HMqF|gTI_V;{T?1whr&-)$?`E!LaMC
zI>Yuu=^4OqXri=ZziN5Zk!j<d)P~^2Ei>TbEZOVI4MDmrJL;)c(GwlOYw9Q&%CK4-
z21XTSi=mfo4XSL|KRHyq(5|)3CLKwIPY1PJUsV}x!Sz2mr(?~LnD=s9DxS3gY@B3f
z`7{V9_WKscuAi4U-VFp*=tYtYGNm^dKzWG(eZyY6awo9bt(G-uL(4I7PsQY|7zgp_
z<R6hgWi#d;%=ENW#$cjjUG~(6&deu1h}Z77_&SYY$TXfYo8JWjWG%k}_(uXshTW&R
z64crt?oY%-T<q{f23o&0c5&Apt7o1*eu6BzlHc);IMht0e-C%WZbZ76Cm%^C&$0Gh
z^*bi?+|4+}!xAR#Q87`w%vsdvv-EFf0~8KHRTqdqfsd{ziAmK+lk-JQMVhx&VmYcv
z`)d=8bKM_OSGXrx2<JLc1Co#)1?Q6Jof5J$`=RnY-2`kM&~fpMic6Dd=P|I?leL#M
zO}fm?ZZSaJO)Jn&O>Pz`IdVs@F+OLqcaxr#S=|tZ52(ll@n?n4bT693t1TX;Hz<oU
z*dd^Z--SH$X#WTqvwuk`&SeZAs~y*Mn~4|AcNl6o*`GMSNq9^h(o>Dk8Nm-7@<hBK
z5GxqDd^I|%b#3$k1K-#a+nIUtOHIMeHePr-1ha%Mm0STlUDx8bbiF(s>wDadWAKXp
z!F4Sb_A_yV*TXX_DWrLfvyDGHn+8@wu!{U1Pa9PX40bb%PZ(mac{V-YWW+;qg$vmr
zL2R2P>mRNKqa3Ih5qN?@8zDsg<|-AzAn@Xt>y?UT9t&wozt4YUHP8_v{G{|ulS)tR
z2U?ln@AD@VvK9v-?%z(nAM1jHuKigmuxj`L$v_3%YZ}flua9*6hWPHopRgK26%c|W
zfg(KU<(O999N!bSdsYNb!6hfoV8p0jz112<IBfZJBT=va^ar6|HWm;qyY6cN`(ydQ
zu@TTg>oZ`z_tIU0C0SGv|BVA;OM<zqy6yR<48^X4RxzRfo7^?0!ze@5Z1zvk=@+)g
zhyuhG!M_*~{(H!C319=^f>r6CqWrZM5g=eFIGr5wM`#8dj91=Qs8ZY+kLGuG=RcbX
z{tE>}VcX6ALkR%!)981)D95oYeV-Kmz@FY^bqGRWLm-Ppy8REx`aeKq2N@8T^f0Wi
zn*75m|9@8gZ+7JzSAyc#rtnIwcld#4DCx~60yotc8g}?QOMig*Q1M^)AV7B&T$o<B
zyrfIumPiC}V9{Yh`q6mB-zOCvPG>0r&z#zBzXvCx`O-ZQ8XCdPOY9KxKDst`@ye0|
zc*5erp5yGOoYdHXr_wL`*GXV4r=6jY(@u_xU!EK#Y@)Jc&ezW;vaP*uuWago(2}Nr
zLtlAeUx|oI)6pMlbMGmAK-3rB0bBmpS(t~omu9pHB0*<NFysQ))xy?CfD#c8RIQj(
zjYw#x60GsOyoH^PlDWJfjBmn!D@34BU`wIRh0f!}{P>5C(c}ihhi!9ww08yc$OmA}
zoz~7u5{0NlUQ`A*NmaRu<Dyj$`G$dcE~A88(6iM~C$2X-Y1mP?qGm{G!LUVX)bPN6
z4+^n;$x)Kf#tsWEZgpiU0LjGX%7dKvMiFE%u_e&mg-1)}dir5oNdtcQ_9Gf~MRYGi
z99Bm~BNSx&ia)`bm+^?h3ZI0;rw;s9tYNn1*hIBZ8nP|hvxFH}Jz-EV#MBsoV}6Zs
zqWX5(YN)I&`L5_^A<0g5{8*S(n@20x=L4l=njH{h{P`*jjr#TgvI0fdvk=KC6TN{!
zN=AnHvBjwR=*?oY8pem=r+cY`4}v}fgkFbVBHqps4Xr$6hk&z7ojY!VsZs`)Fza+B
z`nL;W9PDurz@saZT5DI<STE=Pad{tUJBS;`6s6}G{2XrX2NOwl2cWG$D{lnPyG-<2
zijXiFboR+7OYA?~YrdFJl_gT$TJot|^^Z^@WvfS}vjIR7sh^pdz%lGUW1$5fn9#_5
z(F!|`mv^;!WxGvG!Ovs+l);=9!^`$lGtF1OA1AA-6s9Pc+HyHD2xk{vUJUST$T`W%
zu~1<>^!~)Hy%a1C4g;-yslG2vztsCy-Es{-+#sl3#Jv3G2R|(WC+Pu=v}!8I_SvVP
z*{Cpcu6%QwC21phmn5)>(CQpSRe=+ALt2Es*R)K1r&y^}T$&h4NblBDrT3!%?=oEm
zbIuoQZg1?HB-wse9|nASH@t#;A7qDu2Nw-dsVO`BqIP^1cZsc(#)E6KMuC>><}RY!
z_MS(LU<<e9f`7z@%>|Z*#ApBbs9|MxwSoLz_t^T|ItLYu(4(L@$d<&x$=0&dapctP
zbmp!Bkp*aFV25hjIcs&qQYAG78meAwN8$JRmOVCXOP2C2_O@WY)TGVrC5_U~GJaf*
zy0gv<KCzyCrC5xgYbUt=L)kIdrArO4sf3L}<hIt#HH+%-#9ft0OLxaYpW`$YtY$KE
zNA<;=-1r>b`Er!EJCYNE)?SC53E4R9K9kob0P8RM=A1hl=*0-8>q74%)mt|$ohmrk
zy7U*rb%1L_qG6ZR*iOOQBCEX?ZD;So!Cito*s6%u@dU{gd%^AJ`N2WtE*cfxMI>N-
zXq(?0Epwl%7oRA;Q}enU`4ZQ<*=}&Y2^|?%1^adu^<?{0H`Yue!ed>24_@we5kNR)
zN|}nZF^3%CZ`}nUotzbLR|tJRt7qZ$q$!r9!nC!nVY9LRK`P0_E#@CHv_6SiiJYU!
zO{Xsv9Mq8v>kg}6)T8A6-0$Q$I~S2sjGIip+Mus)+J1P%<HW9rtx7?Tr+~;fsMJMz
z)|q2el}uDRo7(2fMDU5%u6EY_#{)6*H-trR0G&VY<pYx{eocmO%tNZk@PQFs+4uU<
zS>?J6Zeb(EaPK1GIV|`|pAZY=mhiZ@CBNEjVXALmbe?vpQK~tSHb5^kS7ltt5Jb_!
zLe@KjHq<A1_Cm6=nm#TGxKt}&DG-HeyQ<3%=8Fx?%(|lJ3|^ahF*pGzPVe`2?AOg&
zoETA~SM-U}0qZslCg!m1W>zQ3*-K-)(Y};xwEbf?nW2f82XYlv$)uu*&=3Ztv!*_!
z@t0_=?hhq>CJ3k1#{0MG9cC>JruoyJeAGuOn<-V1_e1$an{5`#SuMSx>--55UKc(-
z>!4{|<DnW93o=czLoC0U*9@ADSlzLcW<4#3iVm}6!xBBSgAHdVfMV;{z@7xvMtkzt
zyFzx6#nYDCPX1$Rq880u^$Izl7u{ccj~#eWrsphbhRQ6D@N?M$-q|Ye;isb-Af2ly
zV=SR;BG7IN%!m>NH8wUhBBH~I6J8wgBdMKNg^&@9v6hjHW9N5;azi1>UF6u96i|bS
z&z{cCU^zKmwL{L2TP=J)Yu^%kJ1xJ`Mh*T1eG-BV>W{=Lkoka&5FX<$PD;Cp`P5)v
zu<60K{uG&L7-Uy5y;zq{-~AX7eUfQwI|c5lJ0-Mdg2U&%F1ly(^!2J59UnhT@>{;{
zVy#&+yODgs#7p!Qq4;R$6mx_R%zz7CrS*z^f8hnuoeO?+qTJn<3JU+Bb&J=uWqZW!
zlIOlm>HU&ri^hFs!z-MQ>%-@+ySFQ!m{y4Ud-x;MO1|!F0s4>)Q4fz{SFeW^V5Jk=
zNOryV33h9n#{-ew@?@Y&;#%<gffnPa(b%)|rPQ>8IGxT9N)5BY(C1f~B@@?Q=6@{q
zM*ef)J92?ul82&;o1BqXg{Wm9plt^7!_Nf`@*y!vgfA*z*Q|fmu)PPXckv90<)bfs
z1qlJn13;f{RXB02%U8l{e?-pT-Q2jYs)q6{IiQ&FJPnYzz0MguJx1V$K;@65S@Yq>
z2JRW9T<(sjNvOZ`8`vr!Wp*#GyH8A9FJ)7+@p0$zS4fKA<L{l-Kf1};aa-vc!BM^G
z7@>!KZ0A`-eF-7nXrwi?T8$_*Ng_48bRBNGL2~?YbcOLADkD86xLNxia&?B3XMH>s
z=U(F@`|KKn*Lz4Uk2^0{nt{+<_3VB<%?r%a@RqtM@@>GF>syDw`9){Ie7lbT65%2K
zbe*?B7Y+uU#%Ru?jYgG=G{T{VhApuG%#igO^;Yxj$>jyVY21ee^sYC8uh4K8FJ|V3
zYmI87#p`^896B4kP!GjMc@v@H<u(HD9=EAu7ZV(}sRRC7Pp`o?(+a!00jd`rQ}<T_
z8soFK<oeR0_iLZlV~+`#+Yo-7JW_fS37{wDR&1p_T~;1t-k4fEpxMp5MHR}RDOy;$
z!aw!BLWqB!B_nCkGSZ~!^ZLiZ_S2&tAHwg|Q@2na0*5ekw;kku$KRsv!QMbchAN74
zQbO>dge7E<fQl*9053OHcd0@!OpmMU+3*AP(zMZuhz0oIw4>gG+p{6tZpd`@a-u<B
z1dUMzMr56?^PW2RC(>`{g9hPQLnSf2mDP4@m+~Sqh2nPKoz$x~o}%aNo9{}TYLQeV
zQH!VJz{v86=JZ^w{mSH$^iI4r`#2o^c-PXPbEW-0Ygq~!P9wx)C7CPyaGI!>;^eF>
zQ|jvc%CQzbW8tzspdujlF!*)8<(O_;Ww*;j6m8nI>FxL_Hn^nf>io(T;Vm3%dn&1B
z2|8tQ#>n_)R=@~csG4kNb1`%Mu7o`9)cjl9D^yL@W5HTKMzv2b_>k?K<r5ZulGJ!Q
z>9?h~;F~90t$Z^VDpTi>v6q*9HbM@`44=1X0H;j@bokpnRDRLmPv=HzkIRJfgrcR8
z*VlZ{22`E+mahVXs~ndI5B+X=rk@&QEV>uBpNu9LxH^BoI9m{$XvcP1yFwU9Y2j<F
zO{h$IXhOA1*P&_&0gh@f^G<ksWD1!9XPqg@C`TJ&7$70Uv?4Mdi87;4Q2qK+>S?zv
zc1~uXml3_{a@d&Pf(tMqBy&JK1v=fejm&s8k!!s9wHK4$%dQS8YZ=4)I)#2-49sIG
zV<43;cw}#HDDia$qbEX-=%2+Y9TQSH7Ac45FcCgJDeqP(AV81KUR>~Y_}~y>b*&(U
z?wi*>Ae6Ri^09~x1nFyp_$a!=64u0+tWM@#_jCX3`qqymxoVdZh{mIf^&9l_k-1y{
zd|k8Cm!ED1M4w*QGHq2(<)y1K?H|0_pnk3@g5*Z@;YQkYQGwb%U1%irES`#R5~33E
zASGFBjoN?2(`v_ubs3l%e6Sg^S5N1tutPKT(Tu>YIpTsx8iKjj!_qk^Rm0u1AgaET
zua9!NO7SbL6T_Eif}}*=HF$)uv0StvKvD^qxgA*&e>A*Rl#czd>aKQ7t3S{@f7kC}
zWtF<0R%AWavG%)xAsg?D1Y7eLw3s@mQ_%66aXy?rsGH^2tN`RmN-a%VZO+zFe7L$k
z$gew{tgWl71=I~OxP6x5`+-E~RoLwt_mfR_h%Ke5WZD<b%3|vIb2_eVqmhwO6b;eO
zwi2t`NASzWJi0gU+JQA?E3@XdX?~M2v86}wo7q~*@TJo@hYl^PUP^)mEoa=N#XJ`-
z2i=z!D~mg|A-m&x{FuvA^QAdcsriPAlh=i#CB@=C@S~+ubsc(-H%qA)i`9hIu{_qN
z(*gGH)yu(w&vgs{BuldrPZpVy_To(CK6Q&WG8_Jd;DrS%olkG|o)6Zk5i<2gCC=%~
zbGO@Z0IMB!0^BA0vp9<#l|jhEB|}~f{3uf_i=yEq`!gF46%1mR<F@D8y4tz-TK(+v
zC4E52b%bKrzC1bgkXCN|5+kp42RHJ#gTC|Zv@m4LGZ>1gmTa*s-TCE>ux_+)nD+TM
zW{<^|ZN8opjsbL(bx){<mx0cQ_j3p^`>tAd;inZkvMP#Ox#$4u$4x(0*q7DfPTA8h
zTP-rPs0vTQW8Y2%t3`hJ@H{Czr+OvSAVJE;F@1Tvdkyo6%#0i|4&2lUDH*2_;72sJ
zO0d3BbNK=KxW)YCnSy8--+^%J?zE@fIB1jkd#m@&y>Z0rTDosb2+m##c{_vm)7uLi
z^2<I_Df|;y$zF4mfNYn~Y*H=>JHAS^mT}2|<#w9~#8sZROuyl$nv?xnHz?k(Tn&pF
zbS?l0GpSRsX}Pl>M<_VS!L}>77K6>RInSL(fS2aC=l5M_sZm}F(C2drQrq8mq8azh
zmrO5@AzGepUEXUYY~jv=-BbPpllT_EM#u{_7bvCN0830BDQ(v~K!GuEfzS8hkc+FT
zIo;!C$|`s5xi7Xy#gya9E^xVE<oK)S8VI+0r_{7$kLKTL_RAY`rp%|y-G*hDs}|SV
zr`}nGFVLb_wXUBn4c+-(zM9-nGw7^!zUg-ZGJZt`yehhaKJD(-4r&g+Y(r8YNG5FC
z`=;2)tPW!MpVWiM5+G?+?}-GCC_t1|@SV0sOnJ0RUI@d+Rse%K{%bVuu)a##YxH2c
zPRmmMi`wA279XKH2L15<%T+@lP|111I#Hy4^gD&%l%GgPi2jAM*vk6)v^VYbFaJ6^
z^ua)Eiy@f1Lh_H07`PxT(X1rsP5Mviu3zBfbYUQ-O(8k{F=C)_5ssjrLkJICK$c|x
zB-Mq0wdKBsIkaR?_{UZGh3|Ao{KB^;&p_%j|J4A425j)>PL=tyz=w))5{ULhU-S5$
z{1!(2f`S<Rfu2uv)~K<<9|+oiRs}4UdqL9u6Iuf{5dW&8E38>i{GACPpJZA<JWHY-
zeBS%t4HCeH|95Ou#3FUvD(L=zNfYoPyt%nql$DZ_iXI{4ai*mJMno_7_xIPyiIh(i
z<Fm8D-za4_=c^1BqX*fX4i!&KF1H7;*~}H!;Pk0~<EkD)Y(QXmahclh`v*ZkJqFv;
zWiHRhg8~>hxJj_3iCpnJ)jfDDdecsOD=Rwdh7G6)9G3k4Yu<$#K?4IaZJ!q>d_MPg
zgRhTA6?4q^+>S7V?m$o3^}G{0d+NZQKGfv5%~-}<cQmbsRRe(xZtqCa@h*bw_(mZ!
z>0h~OUJAyRpbR-eUD2!|EW>r1G}p0dEd)MjBP8?S9Y4p4Kvm!TeKd_LK;U@;SskQw
zP~NW`&Q&Ww8w0<Dm74CiL+F={0X#RgZM5lN-5=LeZQ3Tpe4jLM*vQp)oLqR+)1qcR
zG>>op8hb5J>qAMbysfEV<I2c&AD&Guy!DLYHL}37+>>2;A~2DdkoeEd$?6z~JO93(
zePeX+4jU89e9+29A*;E#a4`3>soCx6egLIVpk;;5RPX^DrTGK;+EqGJ*7B+0<7PMm
z{@7cweM@U=_T16MggB89-`AXsDd&~xt1jv>m$oZ{x$Zv@0TJM#3<w>_0YYP-aIA_O
zJ6QIWWwl_D<0S7l33)T(fCXAdI$si->@64WdEO-`4MBcHsn-WRbUy0?lz0cs%fVJo
z__n$+thd8|A{RhGB?!0>zXQFR1y;hs@w+?F@WV)nxK?sF|9ArvC&TP)rzPr>{|2$=
z-oyLo@SZ-cNU814!ffF2I>dc<&MB4I2+sR?3+M~9o_E2Yh<%-bO$ZUH3<%}<udUV{
z2VO{I?ho)kfNGLlw@7GL_$yVSG%6F+un1d(sHlY7J+^(F3Gs9Z+!$Y)zdUSI?$$TP
zog?|}on7-J7)!H+HOzQJzOab&NSLq8>k}4Z1|HU+gWJ+SX^*JqDfx>!Jp5)~b}Ojm
z;<ZR9b1U_g5cnB}6UJzm$n>!=A!e`b=}xM>Ubl`a+O3o78XIxMqwqVg52gTHT4gXR
zOtQ^j_kREAmJTbRc&*W^LEz*%2>cc|1<XKghjGc!_6`6!r*Ef2u2eIjNzTe7AJ!3J
zeL=hXk}6qw1#dC+^L?1*BhY>pk7x#!p@AIb%;`E@@}7*}+01UW1zK9Iy@ud2C@*o{
zgz!W4jeH$1XE#Ax^>C$&O1Jbsn9?rKyKJhOnVHdBs<V>#^vOQ=o$+sK%~sqO-k>qJ
zzXb@iE`i-pw#6~$*tXMwC__RGHxe;j)@3a|Vy2b8ibRR%uOd=|)LC;3hgY9<d6vre
zMfvR)YT);`__sp`YzrK5ZdQ{)-?*$=A^Ohfy<H?5@~kWYG(R0oo%Vc1e~KSvn-IKg
z*xlp*uSLib90OJ!d)#iQ0X2FkWD7v<#{RypIf~=iO1OTNe``B}p+#%AOm$sT?(m~j
zYvNQvcd^FkyA!xNFS)70^i)38(-ZYke6OXNcI5et#kDmeR8-X4=*2ef-6p5uE%&IW
z{~i=yuv2(O80WPZa-fR!dWXFt%3Z<aXI~3*Hk+JoBIDD~u2y57CQ_$upopRwai=*t
z7az|#`6$!)IRg{wlN2&fl$!Y)(svTXgc<=2o4G?-HR>|;b@RjNq9{v0KR+N?;OMFR
zbeQa=VAe>Ls1?xuU)9qw4$}}(hA;xDO8tv*I((~V5S!BmG;iCet_%SIq_^7_a+m=r
zQEPu*%H~}=%`LBY4<$852_}4Txh8{Ms!~~Vqi<;ZM}&R%TfM3lK7^_hP^)3&dV<zM
zaMcv0Leudqu1Z75(w7IBUi<#q<umP@wbKIkoyaknnH6PVWtgpxOKY!CPyVWhY?8M@
zm3`4f$p2O@j4*`iI*FhC^hk7l)@*#%FnGuB;9ab1ale%bW~e7Lt^l<zq}Vfs35NH@
zHPm*wOjSCf@MOLTqb%P0bTXlBT;8NQ%!QJflre6RUXknui89JEcAXyMRyafcfaPVt
z{0DJhIWVen%hZK*C3|vd-pL`djaARQ`054?GTEJdS^4Fir!-`}z{7y-IZ&H#?N8*o
z0<%lpM#=0}xP+ePLi>*9(y6~|L^-&KxY*FT>=uxre4yLVCg|h@z|@sVG1LJowsGQi
zI(&zVw=*M5LgF>FZcJU%@^Rlko~dsP`POEC81PhebFvlc&WYJsw-+em!J_xm71=_Y
zu7~$c{hhTJo_9T6pTF(W^yV(OY8$6gV*G00h*a6x+lkbSduyV@uDwo}MUvQylE>w?
zBtDO`^wSs0nQN097Pm{o{(akK>F3|lZ;bn3w>~CF?wzziWDuJlOhw0wjsN9Zq3k0%
zg|wgzS(iCPLt7gm003y%2*zsz1mAx;U$VQp<PPxn*L-*ENkrp~iP)}?Hy=Zp+d%>~
zJnJv-!`iAqJe3W9ZuU&Q%%6t?Y2jehYf3Qt)A8`tvQ?)<AP*p0k}s&^wv(_JeYB~1
zqHytB3a&#A-uAs~VUQX>WboIHhpRn6hcaY69UUF8p!0aXO0Ki|@#5lOF95)o$Xodh
zQOP6rO&;{qVDXNc9s#L7q3C2Te0Dv<hZW6$E`b+VQJ)V%l9)Y{A);h@Mqs-OAyH3u
z3)&Sk(bd0K<mleR0*{!tR~PP=e<#*ZV9zlmg&~IY|9fHn2B`m!G{DhFRWbi8b_HJS
zZdss<bPoI$Ec73x``3iQznmwO&T7@)+>ie}VtWHsP>4=urPyz};y;g!BEMQ-5_|v5
zz5@Nmb9A7Ye|`*P`?HDwXf#QFIgTQgw7>FJ{{Zd4$KXt03U*_TvJ~|n?e|v)gxFun
zS%0Ide^F2|KL5%&w)5*n{wvW0)Rn(lP-(+|)8qe{&ToTqo$a}}=a-n*f_)b(#k{=u
z+8%cC3SWSJl<xJfjQCCOPLfU(EINhh)0&u;Yvd{9yzI)-Il_O}g&o>a$i4s-7yB#O
zIfhYKRwk;YMF3PA(NzZh=AGsB^&ILN8UWvhygW#g(KI1lU1EQKf8P3RLT!iJZy_$k
zCI91qcT@uNsl2q!z29hzgsdLK3;CA9>{PU)&)0qR=CN6d2L*cOO!_w-Lv0}tavx9-
z*%CM%HxVeb9ne35dqJ;!hHMN$CJKggy>aKGZ!UeJBz?W9@aZ&UHuJjxdV;mQ0oo*(
z^LZdoMN*iAKnLJ&F$ItUKM9e;Rtr_g!f~q#|1u7<uh0A?nX})`yy2LQ$XCyfT6&zs
zL4#!X9x*8lbjJ<q$2*CU;Gd`C*}~^{cXRpNCCcqCJrVUZsdC=S5iE3XUH4T<B%Wam
zZ30t7^)mHP8|H(K3JU1u?XQm;;5T$LA~Nw@ZkO+)2zc^e3Wq+Yb1OZ4=XGNii^OGn
z&vq~`>(%&E;<TXc-t}WjxKq2EKjM+ZUm*&%w4WX;6npZBz8aQ-Actf*z0mo4qO)+d
z$kPhGMLa+I*+9v2!WFOTa69ADQRNzvN(BXr8J6}#H64J_%Z<fxVN8_$!e@>*_%lT|
zGXeN+M~OC%oJ4JzXypfwRNNKI;;yx~D&&4?8Q*roxfMq|Yh+~q8wOl>^TDC<LcDSo
z2u)&O?z#%KqWzUSijc3kaiJZ;_35M@uvbtZlNt<7<cY$aUhXBvA>ede(1uy?2&i~b
zRB-@-=D#nvRv`1FIAl7iYfMHjJ>|1T0+j+E9E#Ep8^(lv#(d4Z)L*fQy0!m1FAt26
z+JfFJh=fus&50m~IV%ll>a2aB#Xqx25ei7B@WV?f^ZU7VdKWIF2Mu=#H+WA_zh^*5
zxH694Hq<{7H>ot8acJm2I=eU8UBY(Q(hPi3%rQ0LQv5JC;_*(Fk|EFutPV1v`7M-A
z!(}rx?fNP;Q0<Ps(?Du(VN|2!am?r4=FD>$)i>71gZhn&0u{aTwksb@_4HN#u#)_d
zu5FgHErz^;_uWpe*G1w?uiYFmfD^tlg1MC%2w6({eK}||lbYh5_sh;E_=Mq^XNAHu
ztGt%?XX%BB%Ov%pTgndU0R1R|gc+%M`Ex8PYJv6x)#OSvyDNW!50BNkm@2<4(sZpc
z+hRPR9%)J7HNU^q#%tP_{X1GZuK<dQwp$*1q&gp2Ddy;*ISqq^_$Te6n|L?`^3Xsg
zEEtxKWWYQ3G-10z^LIa~?g%+dV3IAI$vzknWw7QO?4m?$m+WO<U0nh8a+CrTvr53U
z0Y)Y!x!Ku5@KJB)Ap*J!)kcdoGmDZOZ~ODIFLdR1-qq`Zs|pfyh2vp$F6fuls8#yt
z%=#vS7skgDJ_tq|7BPe+uTOX+sd<=O(G>ySGd3uA)EqO7-4bf)Vf({#F}x<GL`My(
zcS|q_YV3cP#g9v7UA5oxkZ%C1GRg&&HW*@4H1aWuk>SCk6tkMT0s*jsujPu#viKjJ
zoY+QF)Ge{vd{eyg3fTwtt)yt^T{8vM0MBIF-j*auiRG3i?6V=GAN#s=6){Wu26>Oz
zrX|@1^_k0iJ^H841!Nb+TZ&^-NrY0YSd`uw+|<nJKkceIB3#vSD=7|<mvsgt_9_M+
zzuRM*K6g=@fGW*&E-OQll_b!dlp+X&XO?@G1g|sd%`x&pLSqR#P7-&r^TR~oLHarR
zSJstvxsHupnOHWfP5X`-ZEtZ&PhjcAhD}L1@?CN4`U~(YmD`PjQ}pJvF9+7hgyD^s
zJAs20gWs8r!ldAithFQ)F+q;VREW2Vo=~?7Ud_$K9-`<Z2G}=?(k%3((-4VTJzW@K
za+kFe(F?74BW9V>i<zmQS6#Od#X(Bp6v;}7yzjG>>lW*6s9nF95qVvIc_OLzZoZuo
zgsYCecA1mKRH(uP2@$F2dZgSQ!YxZRzqoW5+I|Xq>+*XL8-~S>-0h}_hgO4<heRHo
z3iZ0Y8%@Vtx{hIDskGCgaAbK7X_3J?^2WOizWCcz{YnG86mxdhc9kX;z$bsv(ne0F
z9ef8M6&Hi4IbX^YUBp!iK;N4_<vgjmkv%G6Dkmxp*_I{P<Y)`yYmNwouOci>I!Lcn
z+%VDr!AGk{YQ8-d@>Z`+%&Ouf%FJ+j=L^%#ZJBh%C0WO<=o8gM|8`<i;!BT)r?UjP
z)r2ihf_i~hWg@%BZCM4`p%*M&hScX|H_MCBIRC{7s`sG|L^U%06M-|%T*oW|U3zaV
zR{JcNE`hS8yG(f%^sl_H12FG9=y=*@bI2xP&Z-azc19%t4%k=H@lkY7klj()aa=Vg
zXIv{H1QS<wF9k+8Mb9X0wvch8H>Uh*1(CDOyKD*D9Wl!L_RIb+ZUYhwrPFb2iYKCv
z_TZ<jciPK%7J}-tNM$zDh0n<7k#T*2*F^N<dU*)3L&_>`;AX+98jpr8n)0~>DxI!+
z>dbIMC6zpsYNM37%PGCiZ7&rJZTDRl&lTJTIwHrp(fpHO8&`W{^+Q=1Sjuit1`a2b
zM*K0Er%#el9I}K{#co?CYfkhqdsu-)6z@qy1i;Hih{zlsWJ#j)a_#nfI&~#N66PXa
zAk`RqKIusKV8_+ib{IY(_iB>6DZUwUJsThnP|fQs@k!P^bEK3%eoVj$uT5Jql`EPt
zwX6mmnu0IQ+=*#4=ZG^fh;7UrRM&QK1JsOAxp7UYjqRA(x6hfG3@h<e@+zMuQJd_g
zTDS~VIeIFR?44>9y>KX<50lw@H8Ri@Y`0I$eVH<RgU@bE7b=a{+BeO`<&zZ~jcT2w
z-{VWI>2_z(pjo}i#Jci6Y7*o9`zy_cYhc)}^XSDptiB%(dC=b7N13D-Ch4EJz2i#N
z&6RPqlDT+zDx}yD5$t|oxA)8_{&}csOqU=NQK+li0kHQaU{}CTB~lDq#|w3X9`t5d
z!^9%DO~ItlEX%wGdUIBm;0-|yKg1S?e`?K<qf`Z<GXTDuTy2kAwcqz9^r;iS4C%2E
zxto|Q+q~o<2ae4gsn*hN$eXxPiDj3Bq%TpBKa)qTgs^P3E(a?kSnt7}*M?}tnm3do
z{VlOR@9;G##kj<u3qZiTSjab3yb8aZM3kWYzu0@rwz{%rYZwR+T!Ita-QC^Y-QC?i
zxVyW%yE_Dj;1FDbyW6{SclSBneePfIUe7o7wKjXrIoF(3HAYp9WlhmTQSxI^i`Iew
z;f?!D{p9}8yN>Q>X#Z`{moF`wGCs}l#nZ>yaM=TgoRo4pn-p=fMN*X0d`Bx7p|TPJ
znNI1LN3uS`0Z<oVtuzzQ&iuQDdZ3Td$$Vu9@Fhl)Js|@KwtR)UV37<4*{an+DrV(X
zd^QpUEXc}yxTxWE++5Jrs6`Gx=SLr6d|wS;e~I$W`Bb=)xvp~@9JzJZ<Y>Q?)wpvM
zpETzX<!wf0G|_5=tMC%>pNx~S%RwrB!|3jVu2U~gp3eFeL*B<&>Ak=P*I)xbg2J%s
zgGUQlapPjJZfzgRDpb&BK&k*+Az$<pj@p4XqtMqS-$jHeJ|3pETHS=IQYRQ>4GsD1
zL@lCnD2y%{hrbaY$6h|65d%5KB&l}?CD60LB)6Rj{sekKTR8&l3u|uv70aMEuhPr9
zwGMj+?+JY+GwP>pd$lCwm|R7R=HY>5ZOc9|$*u{UQjx9+k~&*7%ouY<sHx{jQ9Gkl
zuPTy<t6RHuOncbjT(Hj-<94MR9@7`o?oy0q**HOuk8DyYj%{<pou3vxE1xQ$kc~}y
z9~Y=9)tW%$69DDF#4c++RJFQ|0~gT`Q!Va2&ZSVie+K-3&gBxpnAGZS2-q`+)e11;
zG~U5g153Rk=rBH|c{XG;O~GN8rrSDSi@~L92%8e?c;4YvIE_X`qmvNFsQX?Et&X6Y
z&d$n~4sR{d?Eaz3ipsndFn67W?mP6*oR*JItM8fy^)$zN{@#tX-h~#pvpkBfAom*W
z4DK(jLci_<h>>wvZ8(OrlJHM!Atumy!A+;4TB1_cEPvkEqMWX(GF00-EOs4gEjHXq
zf6Tck`6$bjOrpwdd|7l)cPeS5`P|v`?(9^57wHtTNQkD92YyOuu!GPd*PW}@ih!?f
z{%96}7H~*A)Lz!>9oAdCS!7V0g25T@XiUU`DPmZE9|W?KA7|=@4%V|4L7aisdyEsn
zJiMu|gEsBep#-h#`_^<K$~1nn{y(Ibi|CRN_DyT{$#l0R?;UB5<*Ln#IPrDHw>x%o
zH~RtK0=E)*5Y+<a3LX$j9AB(D$8U8^cWCH0-@TXZFA~JGDeey(xHRRW5C%l7bTF%n
zVlsR`oohD(`@V}(&wWg41kzBz4$nWWh6K582JS~F#y(u@Uhxh0yTqW)pHF?8_{ukz
z{L$C1ceiPTbq*sVXt1H;l};yX0$*q#d_=H!P>ko!n)_}dwt7vZG>3EZZpL-(IyQDp
zbEkIMMNY@nh>Gi_6LO(lq+B(CyZADFaqKQ8y=3^*6&5NbHZ+%Z%@^3HX{`}{;IKDd
z38TG7(HY;(XDO@Z4^;Y>7Qp(aI5FMEFJPK3<J`?N&#a;Uq2v}l?WTx2A1*bpI9G_`
zocTgdICxz;UAre=Ce@+O|HQ;1$p9c2x3IZsVJWMun!i{mSYfy~^i@@=dPD`_QK_e!
zNX8wjlE+dyU;<>Zc@3*<wNdU@LWHz6bMp&nSb8v6#oa6u4=pL>Ub`W;Z*CbB9>H-W
zCKBfHmmi;ZqqUm*<VOzsR<E+WqF8oRsDV%WVx&_!RcRYs;SEBpZ(R?hTguBv8>V}+
zSwG-oCNgOYhMJb!ekAizFsw=jZ#7DbH0F%gcaN3jeWC5SW|=y<@YCohsmbtAQIhsm
z0iJuLFR**Ays7Zp7;81iOfl4`rYPw+Gs-$H?3F2@h9yaz=!Tf%%zu_K=fFlU7@)#P
zrkv6a^K^6$4Q&@Kp5&o{Mg}t%pEh6MjZnG!8MFj8Z~4%VUzw5ycH9wQzdOv2!<f?B
zs*|joAn(jx93k_I8E6se;!+d<PMfXQNC-2<v8Q0(MVEXq&5kUfoMqpT5jb$mG7fjY
z2XoA0RSP|RpcLemDt%qxsvi#BbLX(+GkxhVX*&=+Q+M_@=ZTk`9!o*m-SZV~JjUj6
z0(oS$>=6<5T~ashu><s?={tvs-=xIO)h*;aO03b2dSNk`L^RB)np#h<_PDkJZ*K#K
zdb=P=5cMXGTq2UhNK?TyC(kHu^rW2rIL0{Ci1Bk@EdW0bnADz-vT<e2&}^jDnQAHL
z9Bvwpe=sXjahb@!;Fx}#63WB}EgF{aGFB&pjZ=G=>Zk&)ZO{F1Q}^xuSOeInyB?Pb
z{y)%TD-(bQ(8YXXtJKiPRkWs>9AbE~|9J~vq!)}Om!~7{kN6|zq?WU#H+j$)St*(c
zwPvX6Vnqd`x#-sq+8#^9uaE#u2A68-!SACdm+EEC0Zar2_TRXhPm62cqY_*cLpNcg
zIgqPZaw!<TI2RQ)AG#IL@j5YRpb3o{2pQ9%nCh*Bxgx5mQvBe_+J6eB@xv#A7YR#L
zL>oueO(W6J(JGgYTNK`mb~ByRd6*BMs;a8Gyt?9dn^8NhYLwYNWiB9sfLBNm%G<xN
zF*bAVt~_;AGm@{r_bpSqyr+d7tqv7>&n{7_$6*y+aswl5vg5=ddxCzau<<3Q5lDMA
z5+c&ih@^5^Z}c)?rM~qDbE`5HG@T5HIp-J=>&mIN<$nb5&R5gE?+v@e9<P<xFvAC{
zvY)7B?;Ks<j3~YjGz@W6%Ek!t@!^?FrIox5zSV+`8_!>Fnd*Yv49++=ynWW^*pFB{
zly0~RHdu`Ma_jwIA?~gKje|YY+_~#?tcjTu9fF<nZLjM`!r4+B*qKxOeKYy{BB8YN
zBE$ZVxND14<0K1|<W<kwiq3|bN3?mx`)Wo?C4JPie%m8E)$&~n%W}nn_p^_USsABC
z!^X2>o~0|=&u7D&pBi>Ki|!$bXwIEGQXe#Hk_<uqAg+3eDSAx<XlJ9lQYY(WpU%qR
zc4;yK0CgLUrWcs|ZT(u1e$&CDWM$0@+Z8`hCtmqM@=pQXbH-Uxz|Fo^s>S0y+FirT
zA~ua}5yx%2s&xbs+^iIoERsN#29n-|!0bJ0Ti4V2DIi#2vc3Cf!68_rjal~CCKt^w
zpPf25-1PfsPn~1QvjEXJ+Lx%PnreaA*8&c<do-~K$RA_O%3?rPbCxu#z-Ju5YR?&1
zgF$7B8FCiTeu4zMaW{in)atq)a?^Dz%*^J5?GZMe`O^^wWwVA!I#D#`uiD)3(7Gon
zaAZIITC`BH=Z;lWgU>8@EQN$h=yz0@4Lh$a-LL1mvxJJix7X<SCnn#2@v_wo9h`?j
zK3@r9p52jC@e`zBoan-qZ`gy@^dtR}5SMB7)@FO`e)M2}ZDsnw^77($*7o}LR`mS$
zG(k1*OZ6L5tZ<#jDQVg0QTjvEq4k3X?t!xkH6Q%RTEUrqab~@6f)AsK`Nx|ujjTZa
zBn$f!qLf@jp}S^*;GOQ=%LT4WGU<b5orZQrM4p_kuQM@rehT{w1^Ye6d=@B)*|_o&
zKCOF|mpm%4pcW|Rez%9dYn6(Ntb0v1z4Xh);kWC8DUVcmy_I}nF^>lD0`DsCgI(*F
z4gm$_<YcxVtM^*dYqHNe^um-czYb}P$LF+*EG$CaO{5%A&8O!MH?c3WYylCrBXQcF
z^`~}Wkjveh-GdrK2U<eFyo1JPaV4)FtNrq=Cn08L@k8l*xWzu**tP>{V$A1#NY^xr
zs2DGhFu&L?T+1pGI~#w33G6b|mi!d{gqYEe@xZ%6wPfzUv<w7lR&yr5(jP}mN!?-N
z!^oL(<l%ELN7dL-XnV+D;!^odb%f(l5?kw|=``diOucni6xJ+OY=##PD2wb>05QjS
zFeXD86MFdhv(xsz#*gZUg_9@>O3EfYmh1HKy2k@Zw}Mlpr7-R2cRYhHN*;HW<sN5K
z6KU6xW+jto@>k}|TIv4lLZ6~5#fJMczRf7S0ADX1SQNaALq3d(>AW0<kxD5$g18XW
z(1Gx#a>HHh73M2Nm)6j98VMiPGhn=Vk9|&qbltgLG$i9D<3}N|FABG4f>;gg>25;P
zab5T4OcDAHwZL$L*MPzs(*bMByBRbOD_pO4=Nd=#ec|8>rzX79&EfCNsZ<unLY+XP
z2dhJ2<lY)<Q;kUYLJEk5qPcQ*-?G+ASqb^%Pwk3B9}ts`9vF{Zi=z_J!W^IAi|OQ$
ziXQf0))phPi0{I>hv*qW=CZ&ppw#{AoF@F<stFzZ-9i?VgKw-+1D=|?d^YyXJHD5N
zDkvzR0ELrSW_@hbO(sNKtDm{t4_`4jdwv9%^+DqZiP=yr@}vhh-ijhd8a^tvQw1!d
z2N7SZ@*DTnG%l3wGUN=;v42VI=|LZcH4p8i$G!0w%kbev9iKFu@C0`UK6-p_>Rr?H
zA>L+-bup#$`lQk5X{p)wu~m#aIgs}7Zlsx^_oeYl(;ZIRy2!RCY>TmF>pr6~W|0@f
zcIb;Sp0$)H=8g?^%!6+8A|rvhP6<o<-l_A0!G>*@N;Kr3)ND#=FBt~AS%=Wtun8dQ
zp#cG;Jyu_ANTr*>IeI)!pgZ5YVS1b!xZO^SxEw!R8yiYwvIhZm1c=B+-Xtv*RFlla
zZa$M5iRGmO-ADbgw-u(fb|$%4<|2L9L(BFjBIvuu{>1LsG8>BZ0d_&u_G(N!!zo)?
zxlU(Z`{!6(g#ML=!L2Hb4GPIr3`#Ay<^YKp5>i73UBMkJUo0{=xH5N&$YvW44EkxY
zD8X1;MjaNa;m`E!qw(fV-YfGjzP~9pSFj3IxStN=)X_mh^SWuD;c@0SxVe2^-KaF1
zVU8d4tg&Cpc=zwia`qM!r)!dOe>CM7RkG4p!|ohvdRyEd;PHEgG~uMJ?fvX4u#9~e
zXC@<aGzphk4J9=)cA{11vv|r!Ryr|uHdYKO=$Stm$=2prdV;Bv6vU)(-8xEISw$mS
z)z=1{V|fR0b^E~PO$2LsHL|bpQa6%(@sQF%F`+Z+MQ#RJ+kaa<u(_T}aa9-0PMh|4
z#yG8P-el%5U-HoGpdw3o2q{@sRZC$J75zNs_}IPN$o+}Y`>PSk4QXtJ-tE=${Y%;-
zy!leqS6#tQt`eArPT8lF`p1+Aexp~8!r?L&jf*;N*VS|D1v2$*uQskFtTyuus>X$@
z2re|3l`eqKdP%?hGwY;h1Odp`V7yJJDDWZ4)fisg+<cQM7OgnD$YIX?)hmwmnvZxl
zS(UnV7^T@=AUKiv{c*$j8s2vW(|EN)qpo?kQTogX%{SG!uL%4uwlIYj<kKgu;vCmF
zHMq1Qs|J{UFel6_)gx*L>$}(sv$B>Nj+q-1ddq23oz^YlBhD$hU+k764N(jzA9qW-
zBeMn7ih5aP#edM-a-P2RH@~({f79EEs~0};D`3)5*Nv4nRImHlzao-{)+0%dsC3if
zwo;q$_-pwChCCgc4b4j<w^^C=8&95WC~Yq^)f+XcH+!lMI~P661-Z}AYSr}Qc^!N?
zR`qz7LN>KAZ}D|)#c2-S7BzpmW&c69*_8n@V%_r|iD!9y@E{hmg##h*f?YHc(p_Dr
z_2cc*RoS0a6myk(=?sTqMm&Q%ChZ`z<qEn<jgemg=OOxN*5bM0IgN_&Si6hHoDQ9<
zJZJBYKX{0|Rr_mPi;q16IyhlnBp}%kC1lzBN1L5U^KW?Z^TX;50!2>IW}!lDUO);^
zsyhp!g@Rt;rwXQbIF&WP$^cTC9dc$4bqlfovY38`_-0fspt71{|H9bQyWZO!N$P9q
z)%m}gKfeG?c6KRLTw?x`y1i62(ei{{6wAsl7V^YR=m{QO(NNI~V)m{zt8KT=!LraR
zvI8ZwYDj<O;Amh0^$;l{y!9ym&V9{%_CLpkI2AM@`9D2+P;T)+-}GdUsTcoM&IJUv
z4#>WpqT(|CyL-=%6tLL$g&PLi|4uUYQva^VxT0tLD|z(SK>puN{@-%_wHE-1<==n)
zzuoo!lXF7b;?QBl!aBb*@C|q8=D5I&gO=3mH7^G<GQ+kLtj~itCGq?${lwx@-xzkP
z*~9R4UTy*sAXAIy2%qnV!Mu-~Uwls0*)>9VyT}e<Rz6hsqbd~{E7Fg@V-37+%m&rC
zTJg-XZ<(?fkIe4!+>y0n==2|3iH{YeVAbg4KnoY1RnK;cHcnn?*7`a2m$F(;ClK(@
z;$LfF=|9uNmgP>RDkBCggRU;CtNQbvQ6W1?wu006TR{z}O*2AHvfH52Vu<ovXVv>!
zrPu5ZUt;iQ%>p)RA%}|?vq-rCxWI2rHYUFbC6#U$9JEd7=0BP4)ZB>gLi#&cvKA;O
zf7eX(M<m}e=`;hhHWiH8g?S0|6CiR~b4`fC=Qs8j=BZq)`-x6R3F;qnvBpX-H?80R
zogkpHKswL#X-aUt-fI*};DKn$o!Zo%i4cogE&iqU2a}Lq<m!bo!d4*;$SX9lyP|(2
z#X~u>ju>?Z53*-n&_x4#+C0=B{B3;*B>}=7%L*?ni>sfuV$rRhqzMAF^ZgH2-`s3g
zEE2=a4X-p#LjekAeHupdzWWCWkE1MO=m}2>9ejgpq;g{k2*&X-sK^gJb_aH<TW&-6
zmWa=6oko$ak<S^&hWd1wy@Beg9iCwkDZY)YR<7t?^DQ(qYkL_fy?#g#?T*3Y^5_37
z_!v+I4CqnIyQ@l@gQ!Zz3=OCP$%o&uj>Z)1TykPz4!g=0q#Yt)auU|ef@xr9A1uok
z@6{tL!Ml@Fh@e>m=7}}}<3HkiK(xBFzcuvQ?*+)*kB>2dHi1CFTbZP=ZO#0KcEEJA
zctJcYom`_BSqrMm>ZbQ^msUO0|0C3AMr+<8v$}mPEjr>6W%(f-=raXEb*7(OD}1~;
zgLkLujf>~K32izue)r_a`j+e7m|BExzP4O<`EYY$%;EjwS<`agXFsjx(c;KdjJp94
z8vdtSeJ0piWbB(vqBXpdC~(O_(QGDcjsQaY-Q6oQFf}dJN@Ih>u#-6*ln!^wcMrWn
zmcXyIL`^j3rz}e&T*`9N2n=Q}i&vvfkR=spGdOr@c`5t^38+gXE0-ZHnDUu@e7j*K
z7>M<^VE>w9D5O=&XjhSB8j)fS&|a>i_;^r5*n^R=d>CtkVLsyk$!Ip!E9Nk!*B?o?
zGD0h(KfBNPLkF*E=>AXPKjrfXh$lPf?5PcM=A^XMds7i=XNARK8RMgVgV=W|kp53E
z=wK{Mxmzt6v8wCa>Q>t5J0FZ-_8+{S7oX6S)9ycfrR0!BX5~mzocfB0!X(N7J$s`$
z!?KcO86)0Uu4h9#0<sA3&k&mJfoGqr+Y2>gse0CHb`NZQF!i4MwE`Fj$Y){fXVvtT
z>*d8~^_>!DeU`kZ(t2P%%7kjxmgo-G58q$9FBr2=w3fVo{&>s6T;*PLK$ZMKL#5M+
zS$t`<_wK=_6EPj2NUbz7x1=*fWw5GHFali$7GGA|B!B`NFw2H`;&RY?@&+?_s7xT#
z0nQ#FP$dqe$j#E-joCg1?umshJVdM~v_OKqfSISQ7%7~$5!eGV-k3HYj8I8F1uV=C
z+NUKs5d{Yj<b?~yOb6_N98W*Y^rW=V2#8z`nB~-X2<#+lc&0uxp@JW*c*JwK&d*Ln
zC`z33RfVT!Q>7qzPc=oM5is>AD@Ys7QeM5FsX$12AX0vibigEl4}3Kcw`-{4?pNQ*
zD$7H{8VhCm25{}wIgnA4IUsxQIP3GI9Z3U$V3RnArgQ_KKB1U!Ky@VmQAw~$1niTK
z8?V(h$rQw@ikUI26Tqkm)8%_j%Rk9ITlVE~>yjOUdwFeXhFF?lktr`?hs>^;h_l%A
z4Ieh01*%w*2sXi89R(q@3n9`-O6tO@WmOC369A2?{;DZLWS^$7(@d^whA4y17DXY~
z>VPTcD4dS4r3H$J9M<qSszqPvNVght%4mw;y<Sm_m$R(Yg089ns+_OmxayHN%)eXC
zInDyrJE~`b7~%6N)HCO%P4bX;gJ}U*gl)<Rltcqo)1sjqO#s!i_Y$V?PtE4ZhuB6-
zLbqSQq0vnY0tf|hd<{uXkD4iN&+!zvTreFjsLd9czl9A4{HlD3mg<md$sV{?vgk&g
z4FsvatP`DSB<zeSv4OY^N@a_LtlOIBr0D;w@}En78_bszsCThtX7;ISfd$W2nBah$
z9!XMxK`wXG-`iRGEXW{kT}}p5p(25jc}Hd6@MS7%S-n!QokWVf7C3EuBj~#))Z)Hv
zZ?LnZeiens5#}$JJwjJuIHz_mqChhB`B}9V{*c#K=T^^C?NH@#3o>pX1~HX>$kVb$
z%lmRl_!Uq!HJMxm`FoK`m~XNE(#{uxC1jL>2FU`do!CW1obsLQXsdL2D1~yG{vyIq
zI20ptxceL+1I%K&Nl{A%qy@*cl=AsX*h#Tb)N5taoGp5jbLhmXpHyCFY2-8`9NwFA
zUukK@Pb!7ILP>ru7EhkciK|kC)GyfP>4>Rw>sReLYpY>rb&6P69!(h-kqVKQVai>%
z0V`%}hrqVOs$o_|Mh~E`^GjG)<eE}z44!&$FD$;(ok>`cz!h<}5V{tL$Q0`}?O$OT
z@HS$vz*|k6$=41weGOT=dOxYZ1DPF|bLL3*EMkHiQg~Y|r@Uv4=k8&6SR7oDXrAhF
zUL|j4TlyBxnmoYLF@O}?W6YV<gVet6R8Jc#C^!q%Nx4A~x@DywHRU#r@ksmkWAgg}
zb`j5z;-l};n9XW(5D94t)3#y2HYLrVRSIj&8Hkg~d|wW2H|bzgQf}p+EdhpX^CkZA
z6UZ&>u6V;jXxLa^Ig&sQ!)aN#KGfHb*PBG`4Q6){G9mXKKG$1aRG<p3_veqK=a9{C
z$af1+Igh$}M-8XObv!|GI7)@uq9e<Bwz=}^VaOu$gW<PuJJirL#RY|O$-^E+1yOzX
z9jelzrVgCTd+7drH(|2@VXe+%wNfvvtn8RO4ExYb9$0>z&vki&c-jz>qrw_gXL*t*
zVcc9^h#V1(`9~6(;)I<`{>mK+n&ox;_9DgM5~4C%g;ml%eQ|EgYZP*-yIYWq9>W@O
zd)O2x*3xN66irFs5I+ojL(nl`;g~anQ*fiAR&kMRlsox2BmE1-T*)Lk3h&WW#^R1s
zl}9y@fqCnyR|lZ^5$%_269cVYIH8u-yEt4iKi@GoQq-HykEPI2s!2P_Qn}t2Kl^@)
zVz{Hh48mEH%=UB*T-BjjKFnU*00symw{S#2i}&g>8X)pn_Hmo}RD2_dYcftcGmf^o
z5xqyFlbCMb`AG?s8TO}j`yW&H)t?8dPboSCvyPm-aOh{G^UBf8un@kS^+L(_yZLx`
z$GHI0x&j?BT<S-U7%G_Z{*;~Xx%d5px4P7YNofv`&nG7PVi3d*tpqGq&Jgj+jcl$f
zK_j-ugyFsVz1iO!*n<UiB^Xrn62avah?C-L$?93p%f9Fq;$;Sr&v_Vqxhd)K+o2vR
zq?SrpCBkCINXM+JG5E|-f9K%we3C!$38X1B2Khu{Uynma@+4uOZgDB#xt(Kvh3Ae<
zw7&oghRvZ~nce1tcr5M+7@E1qDy7$R)p2J3kkEI)gbPFOoY>N`&fm=>+54;<>jW~R
z(^C52@r(gHI%Tjf>l+kwRk=rq&^u$kwLre0(*?ZtFn{G!ly3|AP~og>#ASg4HC+?>
zj9LpkCzOt`zjJJ+MdLK<Qrlr-q>L14)vn;wjzUgiC1mE@qs+*TnX!xNQvZ{;D6JbO
z$`t9pZJd*^pOfAj5?ALN0vENyGnE9<Z2x+|uwc$^*XsgV0w^@$BFO<^B@>PyN)f)u
zbKAq<$=65>BGS+*U-&EpoYFVXB{2rmEMbYAPL16BdN^`KL?UD#{3Pe;iIr3i1J<!{
znB|xKmc4mPs{3RG0jO>H2X+Wr9CWL9(&jGB&}(?g*%@hS<E^)cc$QUsw$$5XBjUWl
z?f8@JjZY(pXyszt)o9lHfm>)TO}_qkjVZ@K6_?h>j3+MrU6!669yXRALhHp2jBlMJ
zc$tlLur@)n*(xFz1X?5h;g^?~k-q1v!Ts=K=Cmh1<)f=injf9phr7kTi8j}SjDBCP
zK#CAc%+z1VEOnmGT{qy2Z}9(ZMTp>UaRM;*+g8Z9?QvRo(*p<9Rr(}pX1_IqLgk;?
z&n@ys#=<y!A*veSj<iKqRANuk9ynN!=3yEy!=rt5utA3GGj5c%XNHmdc9}E}hjO01
z7xNvh1Vos)bmN72Vpc>+f<lEkf0SztGw@;eY42?rXC4AFBQSSM?lE&6G;85@A)Ak*
zeNy=Axtn?w+@Re_1kecd&f^ZoI!j|le(YBytEHXOLtZ9JPwAgLn5QH~`&csf1zM{<
zSfguhV_w%J)-{^g=mFxu{1C8LeZ@`fJQ{|Ys)U@<PM*eIgZ~OIdSxYY;pd7mq&+E$
z&Z43tt*l5v*F5*5LQHvO1nlCF_r>46ji<YuBt9Fk+jNLDVh!629k|Y&O%*TPJ9M}f
zW<Qf*^d`qaUio-6n-ai9Sd~MVzYuS3(XH}@HPdli6Z5>BmReO;W`}hIB~>Cz?FoIS
z07^hA($WybH0mHjb|(TI_r%k5RO|Uf8r6g|w#kCQgh2c5%w60jzmsmZxy2$|GxY$|
zoa6|ziAmk49ShBUEtUB)#+kd{3nA;ZEoywWz703ET4|GSc~xUO3E})xrJ@NkgkyA7
zzVQ-qL*5Siv*M&m2B20hAvz`7Q%EO)3WcwsPKs0=&Phop<;vPc(f`Ik@aVqbt0?bg
z6*7|FtV&wjcmO-_zc<Dke=fXbfcF-!II3KfvHHxCbLG?O#}$sI=Gs;S(9lt0-Kn(K
z3>hfcp<Kg4_nFKu>=t(2(^IR~uVMWUHWP7~*ch=XX{3<t<)L)u-n64_%7MQE>R{B>
z>tk_HZxEVha4L(}{8xocfm7q>m6Vo|+OgWIvG1~(gw;=0{iN59wG@7o>7yM?=EOYR
z9OTa&TeN#qz+iK0gfSZ^>N|HJQ-?^%3r%&GTUOW)8Q#K=v};fPjJU-4Dn^U5S1bOZ
z-XW!YVN+k_-mKjS`M&2jfl7|6hRIv+e&Nzal+_;n$?9MXUrZ{}3BixN?TAu|%`Eh1
z66;`tBX!rqBj<jb@blh+ZA#NawDoF;7eil)fEa|0qkG79wM?|(|HiBTvFN!U5V!mg
z_r2dRD_o6o@yu}?peppt=*QvTQ61Rk*F&+}$VVv@ah^X7(5In}@igk?&3)1!#roiK
znP(DrALQ7f@CH}73n$87B$!iHTvp$lp5N9h00tLq__m~N%SFe!POd7X7VA(gq87MQ
zC>AciJ*&GaohO{AZ242})%BEdG@yHEo~lLBG$oZrM<M+5*Z2#LS-<gBO*0<kO3=#;
zfjN};J;OZSA$&zrro3Bj0*~h-ug%(*F}}4^BRTf#jR{nA3l}~2bgBtKlqg*Vc3$+8
z=;syX&Syj_`CtH7FaG_fu;VZhdkF=t*hqN(ee;Z<XpEu+v!38P98*HFCR?LLHSI|K
zW0Us6FRz~@a!WfG9-JKq!%#+MKN8<Uzf-)0e`}RCIb3WAP9I?EfiWv~KWBO54jdX|
zD{x@Wo?Zd|#>9u($fP3ftmav&{#ZM(>ey0bPxOB_rBtr02ib)l?BWnMpMrlSAb+Om
zAKw|RfgW(Wg!+9df4-rXX4{ZxKg<hOEu$U%Oc;ZCuEA%&FzKw+2)+VjSuG6ZDk)Ja
z%4*foEqy1ALeB$P$&xauDVykVu9<BS;T53@nua`1);wm-+(MK{5RonjRJCr2Dk-ZN
z#kd2yvF9QnEp+^ilY?dQj&H9XCQ)U~b1!+ns4t7y`B6a{&8D~Wa_`y+*}Q=$%A7b!
z@hG^B#jvh2?v&6nW8&A$S@g@V*wO((_z8_>a9m0(iwG4#NnF>ya_75CUjz+I2J9|L
zb29Y^u-hXFr}^PL!#85-MmmjZDm)heFYtvH-DHt8myHy4;~Tg2nxpznsX;xU31L#&
zgnr7m&|W6KdF{*DLc4hlhX=6_<K*lCA5Hm(dxtrJDE)uX)nD<4e?}@FE>j;vlb$px
zOO2)kj(3$v^_h9_5Ru`)Jr*0!&F2vl{eg0}z}D}@a1nj!lV1^zF34h6$FU7~6QcvN
zB?3qWX_mv^BC;Y?-p+@)z7wU#SKtcN4%F-Pe1~6Y-h)g-l3~aG_L*SrIu5~j(yFHz
zDzd-ac_q}vtS7Fqk0*S9a*0KPy-P4oITO0d&i52k313XD2^fP0ZdfZJ%n{z1h{dY7
zJ?pu$P$`6v+Z7s>-7V#u*)pkvxkAwVLMzSLvqW(|I2y}`9Rv}}CH$>uDBUUqkA5^*
zB@Nj{(L|T(c^p%yqYo8;3{FASn@*Rrqlfux$8{6*zB+oIQ3w%N_G8w`)-f}+jEU;>
ziAgN=&MvK>;~EE()9Q&-1gkp&Z}g3>b_3hX<KMHb#bMHax2(k{&VRGm-neW|8vp!%
z!i<pK+zWCiQhnFC)bCQQPq*yoC?R}01tQxlyu5C0DaO{q=^E*0?q1&8Ys%o7Q@9s=
zr||2m7J<LgQ^Hq&Rk=0S&vf?%J0i0~S@4!f$8p$VB`vKRBubBPD{+sX$ko%~r${+|
zf9>S7e*CVER5Sd%OXumPFCYh`4z8xh61U-Uspsa7sOSjW=%`3c-9q9Cy@R!xDc3W}
zq-w<I0AFfm)AkJJ$oFFiKkXz#L!T-As0xIg5RiIIBgsgIWfNr+eJa<-KA?d*d1Sb2
zX;a5Bg~v42O()b|hr>=8QvD}N2SWPA=<kCP@Uo6cvqge86MCC=1>hJcD6lVzg;xV*
zzu4DNzO<-OSK*U7x?!t-VoNL81a)hT`1h*4=7Oq`-+kNJ?_Wb|#bzu}DlL9X3Noy0
z+ndxtLW*7sNv##t%{^U^>QECNY5-ct!5R63CirV31Cz{%f>c7D6Y@=Md<6lwl9COq
zUhOWaHY7hT1>GwgYOkDjk?I@y_kHR7HxTvkTV2=LRCy81%*zM1Yhrfy(oh>Q#qn^w
zInRB!_(5M<9@5w07wi%L&p!j#B1|OwMOggrNP0nSFhRUEkUBjd&NIM~T@(Ku^aJ>f
zYFrRTlZju-hH6Oblve*27XZEjPT;QzZcxxR`G4;VxauM37qqn@x6AckuF~N_)*^7>
zg+zUw{8!lXdyrlDy#jW&rS>?#X_mi_8t?%<7E}!aCw5>s%H&_J4KPjr4FteivCgx>
z|Jxb^#sDJ*LInyxs=q73_zw&72bp6FQH!wt)gm+Gzbpkf5in}N=e=6{g0z2)20#bp
zvVwAXB!t}no;KYjWa^|zCbuUl8jY48Ters@1voqo3@t5fBsx7vfoM!}^k*O0wtt<-
zUKzca2jtU|-w|N%Bwub>8RX>TBs#kTQ84UR63CA1>})ej%LLFJ$rPIH*C&9O&&{`<
zo{vetz?ad#FOOXUVDgV77-r|ioFJ%7i(WyBXlrz@i@JDBqtn@=bCK!CzS{&`S2rCc
z{AboZ^SGkaQO;~$$6PjVLtPNpH^(sq9m>H74ly{~Q23YS3l{4Rk|UG^)!H54-}>+%
zhXHZ{@z+ijUu6Cjsa6{T)>U`p+ko^-9%QO~Dg&B(Q+~FY6^*pNGK@|2=y1RULm9Ar
z+gcvvu`Yj8F<*pI7jSv16*7s;OCF~M<;Ugr0H7`|PAW@y4{c=PF}E6pnRJrP_Xtx2
zt^+J@w~LN*`7^v!;tQ2jv8JXbJ)p}}ll|Xp95{u4^a9_fCed{pk{N3d9XBDQf<hX8
z<pg2&W<A~SXxDWV;PP)&KoJ2UWk&X%69}?JQ@HPvseBR4JqA@Z5bU_#A-4~(%i5da
zjwH~52dZ8Xj)=6?5qNVEUIe~NeSz)e6iN$BeG&~PT$us?+#|1Upv&}&F(c`g?QoEf
zKi}v;<O9JiG}Vo{56l99Z+Nfxo8&F5w>n3}IIV`eu`lp+Kk%ngfow0rQju~%rG<d>
z%mw<wnb`vELm_ZupuFj$!<dk8dvC^54B9oihmOagYo8~-lx6pR&bRMrTL}sVbMAN#
zewFSZ?;+EJ!AZ^QyVxPvmL#8y>WIzO4z=ILoWNT0V^ulXF3}Fj6rSI(0o{H)2I!|C
zm`wJ=^2xX}Hsf~Am`^?QSoA#*p_d3f>#w)k$73|95mI59IwMOtU*MYuMblKoX;%(3
z`mhjQl<GvA{`0i`BnBK&S8@CHuL*t+*r{K^E33dQT$S>LA^JY^1R3E=b7xBi*eIXd
z*5!}to%GDAMFdQ^d9_$De!1X@=vrtZT3^gh9;*d6w8l_*-G;N$#f?}j*CH}OQK!wz
zlcI$jXolb@{Xr|J;NPSYPKPO9JT0)2hl+eS7?11))dZ^~`Dut8L%Pa3TG+{hN-Dhy
z^JlpTQaF6(2~*cyzSx6YFP#+pW%6}|1E6oyw(IoTy07H+3m+V~)ki7fkAs;A5V(I@
zr7aGE-h-K?Sv@;NA1DwVh=}N9N|wYey$FmD-gh}3QpH~{A>P5f;vlt9kSEQ#jg82b
zJ7N~aa<`<|FyH*Q9V`OPRrNWwr={Ui3^BV!8ipe4VZ(v2@`IwQFXpR=&Kd-3=LvNY
z{RA6Vmc`ATDwrGsR=G|~SviS{2YS&fH5%Yfe4JApy+}<Bz7i!#5S#nE&S$1b<pe!B
zy7CZo&EQ15=4ZlvJEF8IXCm-A!h`@#x;PEg-n8zIjzr!=KJOofp*ESy3|n4CMn10)
z9U6u?@;aDME8II3h-d;)B)nZY3N30>@SrZLiIZefLguT`4j`5sBn2ZsI#Lrj400|M
zE}7A4_;e(>=V3Bz5!Y1*a*dK}uL_rJWF_RTAKj%hSO0a{XH6oz=WjO9P7n!Ta)NPm
z&b3@2N=pDla8aSRx9?Y;$9@Z_H##pFR(>AByoad$nfRGI5C8mY=JOF&mG9vcP~9Ik
z5Q5ulr0UX%U+u@{BSrKrrTH5k%^ALcbIl`$#vqD1)Mm2fn`JM|aB;)-a|v>L12;;<
z4<otA#}nxpeXt|`5d5}%vF0rZrw>QXB@j{j(JH@+q;?9uo)sfT8!$>cc0|VH(})6o
zyuFT*=jO)JrACLkjr19K<JOv5;#x)U;Y^8$$c!$S>Db*oY;@imT3Ud?pubaIvmVn{
z6{4yGgHtD3lz#=0D_5MaUd%T4w#+s^E^tAz5HyF0n9pI+d77J099lOD{FDDZCX8Yh
z4z$8je!D&!lf8Xt_m4mZZIZ^7{W%Wfo3a73(s_K!8W4IVTfo~M=acuhn;woik1<F7
z;fL1wK-)i%y+1jpcl3j1Lb%EHRU(M#<Wo9>HA^}Yp5v*X`Z?GPUT!1NzwQX*5y2#g
zzzsc(6IyHN;b+RUN$UW4qNG?GyXo*mi0n_yeoj6_sCZ9JM%fLI@8{1iPR)~uzoAa?
zTT3MJ-%eeRaBdb@#%3~$RT^EvNtKU)G1aUQe-iSMkA9hn%(|Ev_rG+KLb3%Zo)$!x
zs5Kwa87vkOQzxSE$F$PHC3WBKh8IQ5aRrrR?F>?Xzg4{5Bbw)p3ltRX20rHvk*J4*
zm(>YrWPew9_YA=r+G!HCPlifI#R!QMYa^ys1|uUd_H=WgG5G%6PA43x53z`Rh^JIL
z#}!qZIW$V;j^t?;)s(|v0oiszpyJ?hk4~dHAy^m3=J>bd3lLK4He@ZyV=%74x!w|J
z=Vk9H*`ZOnSs<93e(i`HF%QC3%>kb2o5$X(J|f6S^kCu(E`RJdzH%~+tnsVtK)a#)
zHm9}2c#<z>0^(u$IK}fihQ9hggf4hXE4f)tN8pIX?P>=wwc`6ckpf*!^`EIG(naC0
z11*UrTpkF2JB1#y5pB99-vEL<I-X}p7MpdkoUD0C1@IK){Hb!T;^vBh0qZrnpQuz!
z2OF)EvhHYy^uQ0|tG07$o?wI(jp!flYfH<i8BvMEbwohWu&;KjZ7VnvQJi$XK{(~5
zPMut&p)^{;l;l&&^w7q@Y7)o(FNYI(goM#pNW+UD1~54Gx6jhMzvvPbXx<0@5eFE4
z(UZQ6Vy|AW!1;3NdnwPbM@{Ic!wxzX5iga3$OOIG3-W7}TzQp?y1#%VueefyAx=e@
zH;|1quF|{6nn*)<0_q!04Q6mX6&24N<ZC*myk%1zs(R;z0993<wzfFQ5@lm4DSusC
zy@I(UGC@$`Pf<zj?b)`fDp<i_82-`^y2Pawm5pZGMoFpC9*J<oC(dh_F4vIGb9Li#
ze;HBiJU+=#@$<@|=4lv+=tRI9vFi<00;Ef<IBRtfH^KKTnh?h87WCy)5!*l;2p{yE
z`&#aWia9#J^dx^7#ABsaU?6PKJ>3$E&lQJIdD2t+EY6nxcaOX^Owyaa_0`j>&Gjeo
zmM7~$pc+yI9c#Q)UeQ*Bu<uk5iBKsSY^f6HR_`U-UN@b#IAQnc!Q}nuEINPq55MJ<
zI2=MWn^I!n<=kL`FCc<K7Vr}u?u=jZjgsuE2!JX;3hGR56d)w^HSJWR&J0FA6N*Ig
zSz1;U&SrT|m)fuo)^kjLISL|C+e6!Lgg)RvPDalS*9q(I7x?a;YKNzymI@z#_MLv2
z_jIay)z}4Bzx*kL2xz2o6+~MFbBC$HF67Xiy{<6QMx@V|G%r1}dg+Z>-bX(VnRsF}
z3fITRGY>ys7AtXbb%8>^V$%N~Q#T>mdUW$<blmFRyanyd`%WITldzH&?254C;Pe<(
z21cDig?_LLZA)7LX-noHw&e)3(H9|xptXmvA2m9N#dU2NhkJQ-7sT8_9T$RY@MCfJ
zJDRkac^7DmKF{S{Gqly&c;o)*qW>~ZS#`C{7lQ#`dXSs*uTc->(@tp*N44|_M?)HV
zb=(TIM*XLmlb0(j-|JM@Gs~vX5d7NG`IL^G9~+OzQ!MWlJ8aaVkthp3!zWD6^gOo$
z6xGfL0Hp$T1hQ86MZ;Ia7j{<#DY<9|JKGOCul`Vpx`~n)KAL1EX8Ce_nhM0lFnu3`
zAq7<?Um1cJ8+zHeL`cp@DEB=6nSAP<6bFEi49E4hr7pa!?K9{e9xALe86S9B+sE0G
zo^N!9IE*GR=@}XHCZ}qzK2xAxvX)*g=iEQ_m8aKfOFBd}^vH4dZEfMK`_Ipe4G?Ke
z*l1!^B)UxRwfK5a*T!QcG`8=P;xZ^wjE9j#;lHWQD2m0<h^mO=#Fx6^zlsQl<W&}8
zS|&KTk;tjjX{-C8)ZaBJIpw-wF9O;RnT^h}(o06diMo($pk}w4RGG)xbrnz^cn!*M
z{&l9`kkExpqBnIvCihSo08kPa6b_dM#@5!h6uIF*Hl{JxY}5SoqXp4a+8X`rEUb)I
z72jl4jmxW~MyV+79-JjjYR*OIZ8kVNQJ+3zMk2b&%d4x1xd(MM|GR*w&Y9AYjYJW5
zx->yUN{qJS1qsGKS(tS#0O;y&8_w|bOh7`IssczQK*7VqLm(Y#+8ixY`>7#ON##<b
zLX!aWXVy#~Nkm6~{v%#Q-}(*OwmywcngYPx%=|o28bDcCRWJlD2~-Oo9ok<NVl0JL
zcNlCA=;i6DZ&P=pq=og)?qB%Nt_y&KA3eSz+Q1_{#zCZhZ&Wk$2&$!w4!WVNgRfP4
zpbRw3Be&@Re-yRAQvHWb>;?6~1E7=-KD{=z{{kvo-yxhj@h+b&DO*DSew_Y7&U`t)
z(IKf<X4-$b3jXCcI-E1J*Zf;#^cQf=l?BZF9EoRY;_oK<&#RxpfS~(6nK!mp{WBQ-
z&uwQ&0Rw)3#5MhIIM-hYu&w=>G}cxA%WQrj0S5FE7{mCF_~H*XqL+`LSKyKlliG&z
zzd+>M-vjdQA4&T!Gtt8XtiBsw6lzP-e_4cQq22=7Hg-n9kA4d1ec1Gkp;WETZE8Xx
zNq_`A+2GLKKj+Nge{13U8Ts3<<tx^?r*+#$ymeL|I-G1abo^a#&b^bTyhmyt?`I~N
zOg4OPA;^|ZpN?A$fW{s;>=)-h0|5VbBDzo?FKplamj)}E2uLknP%7%uGM-2l*=ZMR
zOnRFy0k7U`Lmg3cUu!RVd;GU|?PSe>z{fO+@NY3;z{L>&U8Ptek?7mao><?1KbBT?
zNTQTfeI!=#OXL{Qxi$x?R<)WIZ7>>5zc|6q+pLP*b}j9$0kyMcgNs#~_OFljFgRT3
z0FA%oOI#i=bk!Q2p+HEa%L&G5z9NO6l>_~6|9NJwcz|dB&R^@0w}cJ+gU9T!sBeaU
zV1D-K(Ncn_lcUupie&*=Pp2G5&b+pW4(=y%R@I7c{l!K13!Le6wy>PsjO-$Kb?Wcc
zw&O-}^9i8><+K0~oS~dL+a$vJ(Ze6kX2qmC@Wy-7>&rd=4rZz9&F%Zn#bUJ#bVVId
zQ$rh{kWjSSWVw`IR3yxS10WPoX?0+({_+^lAwV^d=eW^YV|OxowWP0=)-W)lQc^J9
z`H1<QTC9mi`kGze)AM(2r}3|w=mlqPM6;tK-tuZ77|rxmM*eg=>qxC0)(BL$h?{ft
z=AKoD0ymaX2|vOt{SLP@Yah_HPleqsx*n`RYDayJ=7_P_?D(ET4J{D_>cz(Y{DtS3
zUcS16(;gg0aaL~NvpFUzC<UkPX<_~E>q<-rD7!^0X;Ec}&pmsFBWR*{gFE!DJkgJP
zZc>Vz>ojO*=$gl4S1#}1UiMOj_abJu2A$|=WoBWM%PodTEY_P*hNgQbh9ok5{e@@<
zh+D9!8FHx)sNwgD+J(jUDlvWY>=ALfY1L*$!tbuJH6I8B9W4(+6JfFeBm})1el}Rz
zoo<Loq%x->WBy5nck38h0Z@h_&P4;bYHvG>06ToSi-ZtR%`CR6Ck*CwVu$Ay@R^6(
z2gS6i=S}5~&iL^@fLZmS=i}|dcWv<_Tr0a#Nwi93x1y#yexf!A{__alB6I{7@5j?L
zcEQts9yEY96Zd28>jE8F0jKe&OqpmNX7$w$VQglcJJZ07<EF&{GhU#Z(V(u=smN=G
zFkdw;MP#Uyq+2+m=Ou?WZUQH-nl<Aq;(@+lbm5;@(0zoOTRfIu)AM76%-b-E{VAsz
zJn`6*``utT`gUlPGOu0AkdEF6rfSWaPrMJR5N%PCS$Y3G`5G;+53glNuT+@dgv+5K
zU0Y-BsZ34N<t=bm4Mk!&P_`>dg*A%fk+o_MLWJ#G9IvdtZ%1I2+s8u;KeGqba5PTv
z$h#8Fm5U<QAD*OLcq=>z3xXnt<=KW%`l3|-+vRnE;MF4$d}X1nJ&kU`2?4A()$Nfv
zf~61kz~j~KRmSEkcn?7EsL0mU=d0EY+rgpXAGYon2Ee>|4o4^g$_H=DZs;8kq>dW!
znt()ROKOd%AnrtVKR=Nn9UhzQ+0qJ|m%-6nAj(I;b{O?3Wq$7q#I{8i3-PC~sBLIO
zUb{&Urxnk0;C5kc%FjEXQo42FrNBpP&%}D&@9E#KG#e}%FI1K@TIn4@BcL?pW+}8p
zKt&CJG;+44Pu!IiF2pJRg0yIz*PLh|t(BVC^GmiJZi$r=h0R-A_Wk0%#uJzlej}=*
zEGYc^XIWLN9Wbp;S)clK^7AC?=1d7t9?z7xj883ytnriK^a@L3);lO+vgr05)XNFX
z;k?)IZ`1TEy%&YyTQYBKw|6j1za(RD?omH8SI--i%uEa01BM55wHAuGFvoLM@oqNw
zc1a&SM9tesG<Jh<+YQ87`bcsr{2v5r?#ORHM9(l~C-?>Z7;<~<X8ww#&9nGzRA$lO
zd%SV%o?JNh{T*a1qnsxlu5epzLGg^-Eo!!IJ?MAJ&A1?oXV2S2um*;W{&X}n!%qFr
zbBf7&J}O0-{_o+K-|3r>F~b??Q$TNS&!i#XbVDE6`CGl(#mS+`JPg?zmEKRl797BL
zeY^nNE4A+FknZs~hjnSbyEn|mc?#HAfu`c+eEg;$B~wdbn)aO!*oN+1VkQj}n6hbT
z=o6g+jwe*lM;BI)%|{#DS3UM_lppc5TM52hL7p^h886?{sMzXT5P|*Q_h1)35sLlb
z^m#=`qg^+7CA@&8di4Y^j=A<5NrmWazn8D1T0i9C!EO$q{f|3bMQRlQXPg@m0HJwn
z99rKGvyEJeRo@7PPU%VM9&(@1#*yHVkTqY+d^a@HwQ)571d8%nTI*SbnZbYfz-NEI
z0fG@9vZx4v-}Cb$@nqa7hGbPYnMXDuWR3w<oLl!X2_>k?Cs2=9zqt&<&n;?%fX)fb
zYnA$-nH#}xRYQlz<-tM=krjF@jBE9JXCR404-7vwjZ-H%O3#CW5mcJDOH5jz2y%;y
z?ZieJldP?BLK`dps#jVSDC0$uaP>h`zi2#YN8#FPCMAYW5``6hlOh_BTz04F#`}au
z)@n!^xj8)?RNivcU_NbdT@<smvhMeA<6mzm^7{1pg8uax0@Y?&5Kaq9?s2UybTe02
zn&9^f61=rS*Mkj9K7#8CM4V@;9Y?bma{`wPLsOR{7dBTW=gu%D%xpM9_sk3MU&O$0
zCf9?{D6<2#F<7eK6#mizu<8pE**gEeYqj!<>jolL?N4vY!cqS7uRUo45lS5+gpi#R
z^!Znl=cvlQr@fT_d+3Qdt{fJ*&;BHF5DtH;;DZ*D*GW~&j_${E3R&A4|K9eWfyCSI
z^uqi0Xb}t%NcN)ng|JMe0iu|I-KQh1uQDwJTy6xAgG7b#?zboEjzqA9`kQdbJ`Pw&
z`NcodH@<kvc8wS?tnjeLMt)6t=a?f1`q>D%|CM#OVi7-;X$O5n4s`36az-4(Px$2a
zC>Q+Yxxj#107Hv6_@U1|*GaY9Vh=XwRCs310Ua?SS5(b6*pvRvd`}db+Iji${Bvu*
z4=DdmXHGxr7pIHwudL1QXSPcDm)F<yJa49yI2{5bY3iYwd<3OWHe3=P3h&=CUE*-d
zJ1%^cIz|2boctp*7!|Vv#lYt%KFqItcem_z>}UPfay(!^UM7sNe0gENw;XKG)sV!#
z=>bZK(Sqe24!8Vsc&`W+X><b$R}%Eu6@!KhaRtsltOq}&P=Y$(vo}uB$GI`0jR6UZ
z4$#2+CHi%NMOuO7q(D9Vr@1(USaQszLb0dtA>tfE3~cO(H$UEwSI0GDTwGiYC_-9S
z#JHW-@^YyDh8||M2ZZMA8`eL<$J=Z_Wz1jRkeLaPFm@o(m<wANj(HTNL3JL~*=h=5
zrL(5QuvNeEUG8hNJj#bV5Bufnn=y^mHJFlv4V@H;%ZC_r0)I3?tc!^D8?+153=Aev
zoQ@T62N&@*dAMzl#c9lATE^Y3A<axdiU=Md?qAD2f=&jo)B?2!gBMQzAmFn2VlhTR
zjAX8SD;TOMaU4U-D?fU+OV7^@s!9i(<5mB8_fx0cyW$s@5Q;Sca!R)<`0%L{ciSdg
zjD2QI*+JSNL4|?UE$(cl=7Vo;+vM?$bqHxayfW!~NNyjyHJ59|CY_D<zH}@6!LV;J
zo4Z$#hK1`mhGL%YWajx_Wq{!XYR%+#&G9S81MC%z5mDYczvgLB1b_U;ytEVHEk0=o
z>*Sqlqtc`Eer1Okch#_a!lZM$#Oq{P`?Q()k?n<`l@w@*j|_dN#tr*B60irz&#iF@
z0dNpL7uUK<BpYhFK?MTUTsRljpQhXw;xhe@osS1Xu^Qfw+DW4Ck?Ym`&};7yiW26!
zNF*3;fw!G3l^A4L{%g-!4f)h^+@whuF7@Vf<=hxN4eUZVP$it8$2$Y48`zd>MPqyc
zm7z7{wHQ!0I$7fo0n_f9V|VDZc90#SacOWhUx>stNEKA%a5^uKw1z1lVeLr6gs%Xi
zL;+%_=Z58=n@W>3r|Wy}j?8AKkBpPX{WV7RYjoCAQ7c<z%f+b3_IW6TkXVkGRF)9<
z*x=5B?3fsIH+l^3JKF(|3-5XN3qDtoJJv226<h0_wbf7G_;yFs8d<D}p$ou^!c2JD
zJ`3aipbX+1>2t8*-Oy=4A*JRC)xn5i&2~c)olLB}!&@ZvibVUkNf~QaP*C{;IR8Qf
zz6y|aa3jvtnRQ6H)72U}BedZcN&GxG5dVmJNT5?_jbs@@uC1v#9i%r~Tax30Nhlhf
zxJs6(<NbOyvfSm_a*H^kBC19Z_4h~>?;3NhL-*p{@>9wnXtf{+YO02YeHf0D*Q-|t
zYPDHM0jdOijQ-`FbDbjUlE&oa?X`bPif7-P6-%F$%?Vx9(0l-@#RMJhCfxWhn4P|F
zpT&P6zo&9S5a|_1W(Tt-j$gUwY*lRv=boRHT;`y?`=x)RNvy8Fxml=D%thhqbCB!7
zcfX-)Yko^7k`0%wA99uZ=GYiCmi|P3i%X2SrqYRBE?~3HLt#lVg}ok*k#~b1(O0;o
z{Q~)(jto5B*OCXdV2HcGyKsoLA)V&`;Uy^J?E1{&b>9<x5}Z}B%lJFQ;!_1J?sgYu
zaBb#6hU~hW7WIw!Aaw~f0-?@M-U+YH-R2aHqW}Mhd#CWqwytY5HY=*QVpMF~c2aT0
zwr$%+#kOtRc2cpevuf}Cp6A`)|2@z7@6N@#$Xc0mjxpw(ee}_KYi-zktX<mPMK2zF
z*IL;1(6Z=jay(VR;gzBNxdB)Uk#46NXAy|{);|P+{u3y$|13)3NiYE8rt?Sl3<V8+
zT<dGZSfVl_)du>bW9;ocxjVxCh$a@+5>4c}+1vKpi5a(ZR75>DJz)3Q)4<m%Y0WQy
zld&@RPB+{TA`$>hQ!71~lJ=*Hir@y_gDZC<<XKW0wJ{E-kGeX7^jA7i{J>>%!`Y-Y
zdNbK<dyVj@$Fm2F0Uob$tRL8^oR5)z=I|@UKULGBoJmIpU;I=bH4Rd(@Tik%``hUC
zZ<SGht-Uglp(Y6k)Qa6J!nALE44ISa&etTD0!^s=#GE3>b46zxfpyxcxX^6=#;1LK
z1#<M!#`unbo8kc_X&Pv#sXVf@G(h?lhe`(cmu(}GZ?3MSJ*wydT3u+A^@ef8@03NB
z_4XTm&guiZ>4`Xx>3^<EcLfEdgZSsAZ|;G9dWln{oz?YtK1gN*>?{wzsd0h#XJB)V
zz$)V)f0yO2bhuf@yjmY)onMh==xGtKLK|Rl+ZKb%%IbRxOqtpv{JFE;vXVX%x!&p@
znw%(4@=#FmJT(i0fF<^KSDX%)Ye1<tkBsM^!q_+DX$yBHQUY3VC!Rt!3iV|^MY-ta
zFi}6Mr)bFF?U}Y^RQ1AM@4tWo(BMBfhH-gwfmF^ESOA>`0do+NPlfr$izYe7U(<DS
zC{)OOPt&>`EEgBovgv{})5W?nphZ?x0FYx#ZEvLG=fCTVe*h6*nLmL}X8Vfd@5KM#
z@P7>elJU@wBzycP75h6l{)5<X{9zD~**yJuUj8SxvaSY5g*$5o<K6!UMD*7gGXF3L
zP}n^F4<7XYMQZ^V5j0Nc|0s|Dbyik%z|(JCe!ai>*TnFzwx;lr{mIS25OnnZkG9hJ
zVuu69mUYp^&ida6{Y6B`K>@+b)ZCK<baA~KY}|-(GK)JL;B?D}qtoGr<`0P&l$?wK
zKstN@dkIJmvC@HDVEy-ibW4l<p&nTEdf6wZc1uhD-%<~xC&D<V{u|G-(f@<Y4Pwb&
zz@z+5@cIAZ5U8{182r1dpg!H(a4?`bnd|-7u|5Tu%sTUYCM42$Ud;<d*%h}F5NBnT
z15L(jbWZE!nkMCazC@z7jT4Z2BChwm@{+LkM?hHVGPx7RR3fg9RJsq|2OHmO0It$J
zJUn0j!j7=!O-XIF3KYbRW{YGp<LY}ejbdoh<Ww8*MW;YcY>9NqbouosFxYJ6jkadh
z{6WcR6w>4oOjnaPHM<a*?<cP0qXj>A9W0`Q71vgqej(D&W;|i}m%R6#)4j;);@5>e
zD0ziw72xYxTY}1XEx}C5pWs_oazwp?(?h~-^?66gR&f|w(dS(EH#OUo!1swf<>cug
z+DpQ!kF4vYym5L~x^<h24f)UNne>NRr_VfeV0Z)}9&2MKZu!7kc}-DKM?UTHF^W8I
zg;^yXDivjH@jVaSQFbZ*?BK3RnRYn)D$}6ZyG_u?d1y?ZcOX#4XdU{gT+4CX{Mex<
z<7u0rDnnEnF+_39`U%-S<vtdF`!~z3TUG>sHt=G)RW8eA!CR`N!VwZ<a@rp{Jbhck
zRxeBg8JAoDrG#%p9o%uTqEZNt;B#P+P%OMJ6f5Go=d_Dyd8c~=Iow^SXTM@O7d&fc
zl7m+oK+l)Db(Hv2%nnJ0?;U?O=6(GUoqhvY^NJbGX{Cy!pRnj@$3=~*61AxXn{F<a
zN|C3qBffmq%VvXT24_Nl8LBB@LdI37Wg>hJbx~48mOtjm7cQ*`yzgt$cPAxqKJ-8p
zt`IO?z2ix=v|10`pXYXwPxjTFaZW>FG?mdryzN<6irOFeVPZ2UI6;ZKTlh$(!<s0U
zu-A+~l^Qx9XOad=fjb5KIFKWf9W@U^&sa_X_T90HNZ_0j=KG!nfd60dMpK{d4d7_a
z2&kUMumw!&=ma<oa1zHNByIssbJ6CzUVN2eZn<+;5_%La>GRx4`E|S3d3f1=S6L-t
z&*{}L_XaJi<{z;<?UXA0T3lr+e7-9jU?feyD`3eBCy~ZOSM&wvC~?k(BU1Ep<47X!
zIsEVSBm^Ds5?K)NI-1o>H=%B?RR<wZc+k>`{00N$TxOQ;%QBIy@E)BJ6jw=c4I+*2
z;t}fL#+p6Bw58-EUT|(Ma8NiY96EGRRG+=AdPc{Ye=zC2Oxr<YmRATWRyCvXBwq^i
z;~;!@v+W)+qgjoJ8r!5DKGl<wPPrz7?yG2I3JY0xk^|I;+6jRkG0+sj9a8yC?BGCB
z_d!m9TM;o(_-}@Ht9GOk-_yK0$t1kY)wR9wWycklWpjn-n>NX_pX`AUd-C=0%9RM2
z0nBB!xi2M0{w)YqVjwn9NtN!<b$;*p5RFXAJCS#2(^YCgF%uk8$$1-&6*{{(L+_OX
z&%;q5Zf-?tX-<G6bng_}zkJf!+}O7#r38-iwg;b|GTUTjZmQ*s4S*I=8%)Y-UFDRK
zg~cW)`*r_*U5D9HZW$v6W-~hGczfBQXKOD5GIix>f7lcpo&j^5{8>#l7=k*=<t%YV
z;Z}O#Fv$tnS_+e+-1=E7>~zNz<pyvl-1l2WdSyq1%ZeWeTH2H_mupeWK*lG6WJ7uz
zSXkK59NPyo5ssm$#lomYQfA&5kD=`M{kGcdj^n0WFPA*drT(&ihiP5DF95Wh4`B^{
z+NAGjnzJMyZdqkB<hZ7Gnz$aJ%D~$vH@>JFUsPVfHh|+ia{Bjx8af6Av|wbLU5u6s
z*5gzw{sfxv)2~ILsT7o4S01#qV}!{Jn$dQeOkUsC*1Tp08jzR-4D|W)7wj9>#O<$!
z_YZjf6pR?V&f$dn!TO3sFt?wGPUlrUn5{C<Hb?Vcxw7eCgKxA43g3tJ{oui$lt-Oi
zs84@rvb%nPFPixJI86oiJbP|vC9uN@PX1C2p@6J^(Vr1`JuNi+UjIPA@F4H;XiLDu
zWfydU>HmAcG$cHqiO_pfgM8H+j1*K$;Xh#N#Z5IWYe8ODFKqqQ164ckBWcK=V!83#
z)AdC-ooAqQqDTO(mVY&OiQ4O=4AYdj(B(cQ<LG_9IE;9A>$1MsC{=fnw5)zo*IT@D
z7ZG-9A`udb2XMNBVcY{ZUBn<MGhVKjpF4MLJfp4Gjgqc6cqzOHfM9Wc6f}>5(z)6Q
ztrJ8{$i&wkNS~{>rxT;22{8)csGE{tEmOQ6f7q(!!sv}cX>Q7wBcQvy=VI$h*v>P>
ze{%Arq(8zqS$7BjQ>wQIGr@8>%w0u_-zNo;`hNLIgeL5}dC7#Gwt4!*J!eeDkMO{$
zt>9j?GI8CUS<t>~9n&!+Og=yV7UUP!MSdO9BjsQ&zIRXLx%ZCZwgl%EJYZN=Lh8nj
z92K2fdAD4o<n+KF{7$|DOWqs|wMm3d^b5|lfdRTFyc6#4IyWU*W}#)<jW#QiAq5|j
zw~kX4(*a(=<y<qWXac{`^`2+dge6z~Zv0rU3Of}>vC(n;V}~Arkk8i~cjxr>mo@`=
zgCSzTLoi=qw)nkff=!U7mCXXC-0@7qSEuazmKGE`c6pS-E5W$^j(vX=bL;0Th;9Fh
z*1uALB$zMUw-VteZQ!H=_Kvb&oF8>nm(vj1!)ingt@A4=Hm#v0tSp8Dhx`p$_MKo^
zy#_mc%gI@rEJK)H8qKYIWGPQ&e50FuyzP@~0HJAd`dBmWizCC5o0GSkX`v!g9&Upf
zJ7K<iKVo=NQI3lj=_J;g;^UPH(J}(A;|prql&@L?2|WL~!M2y471^E*ahF3I`M~n!
z?Tqv#A;l*X!{(%qGKFO+ZU1G`Wp6LDIeoa7TiKnfi=+<WCq#hI9QOxEp`$6xwj47)
z0G<Dn@j6O(n?8>_ZYWr-!vmI8_-h5h1~I8W-856?o6Ov%EmYiG7d&7V;2VmmW(0Kv
z5cfogJmHGxjdz?|locrdkatXqgLr5W4T)pqV7|gPkgL^vec>Rm<AAiE(^gVd)qjL`
z8!nV9((%n-xDhNQ-_kYVOQk`XGED9hpr&}z;<}NbJkG^QVVg-GWyxtJMm$dM3_3a>
zO3<beO)xBfhe@W%=&y`pP*PFd1*=d5ON@P80|4`-`H&PdUZwu48ZnsBXIj#P*yWz&
zY8_a@3-ETlCAcr;iN$90`l1{MMIYFILHH5#gK>Un0Mq8=8|TO9WeD-|OEmTEO)mha
zM@BUFI<JB*rqKK%LXL}zGc5;oC*JMaXFYtn9V1-~1}^(u5{wsBIC`qP5V5ls&47Ik
zO3)D*00(&SrTc(cqGJ8#a_li<QWvU~G|BfZqoLrR%Qqn=Gq)`&*=w;qk)!UU(~JYS
ze?Q|glxPN+RWq?8d>EJ3us8xcG0XQ%D2s`!M~)hK_mhwj(HzQ@Y<_O23!W5f*cykJ
zP&j%@0eN%;EZalBnvNr8lOPyuW^(f)bA0i+jF=I?Mk%d#1qUlEOR8g98jTSLq5HKL
zlYs<!86;s);St?$#|SgGCoks&hpgaDq+3ee7dGJ9uer}01F_2IX~W7xdbTC(mj|GJ
zB7jtz6rsKu=T_GzNGT1#0;Sp0hPy>=151N6@cYv{$Q~4QQrIRQnioWYZNu~v%YEjN
z{c5C&uFu!wQJ0Io#i<no!b2%Yyvd@bh`X2=tYgq<5qPe8KJ#Mouod7i{gsSm>UuXT
zh*btAB+Zno&;*ihwz@U+5lfrb{G=7&7HhF6b^77FpXakb7sb>T<;a3w>6_(KKsMy0
ztGw-BeCAN@cERt?GDnc0h<BgA2G|S}N9(N;bAoyzPM>{8b8>oJ;51xTfu;;;jr}Oe
z6I@M-*30bJGBd?^rU}<ihBm;P#DPA!HO-v6gO|>MP^F+~?^x$aO#`UU>sc65+Cw8_
z4OEal4#w=*b8!}d%Jap)W%~tJCT_?*v`N=0XAfnFzY5KAzT&Y3pB<bDi&DN;XPw)Q
ztyv%CT?SGY#<bxPjaXlpA3BIPhd+ZQ?V{mJzUjg1eA<8bfV#RUF$<KSZHVhHz@j<O
zYYE`aJH%ANJint^O+B+`a)!tvVb@X@2g=ud`k|OhVUm6ZcA^Ina-U?S%XBpI$ORF&
z+oKmgGZ7v&U6bb?rTInT?_3L4>h9mo>cE=rCol*4rYXC8x8i_1U{{keiW{tje>$=s
zvzgB)L>zS{<RODa&Ic<WUC;wJ4TW>>8uUMftr@h2t}rhsvTi`va0uTvudHbhn_t(=
z^_QcBw*1lC>WqYa7I>3kV2~J7-O&mpD{45+T!09u8+B&-t;t??SM4svv03w-UtiMY
z|JW~reEU-cq6e8c0Cy?7LmF(IVH%;>_(VIL?j>^2`NKH0#ZRlb#dQ-y1nf&rD?#sx
z*DK~+{}*{K=CC6;x3WCuYs?{wgfwZn8C8S|fnpbKWLIhs(6OrIA$lEv8jFU#DSGkM
z7yNrz&ZhyvtQUo9ahOCf4YdMC>g{re8Uw9z8n&z<_hO}RoL|_W4XE_DA&)vdJVa{K
z_RhCfOvJ4R=14fx`j5B*u4!I~z2MvGbP3oH%>d;lP+4M-(@hCMz)RpK38$*K@(CQe
z81kbe{*V!OcLb7E&npe#GGTtt@W-{-W#HmB<Jk5|JD3yX>_?KQ^lYKB6xfp?dT)8E
zAEH39ii3NJ70EvdR@OOP2pTQ2K*#6^hMkbwJ?X~XMrMU*!&QD(xi&L^(&}l>E97%q
z5x}#;>)&wXParn{JtDA&4VaJQt{c*87+(BE@cUAX-**t^O3o4&Bf0u0O_JXbp&fcZ
znljZX8%buu<BYQ(YQYqn#7Ku=&S+qN$Q~UzgmJ4P$8yau!t4?w57ARlGfN{@;7S;L
zbv056ojE1VJUm&Jw7PVCV}}SP6%IxLaxkqBe1<Wb?^rEqy!A8k{7wVOjC`aQB$vYy
zV?UW#plplYn6YzRs1xotg8HMXx(3Zf5qq7j`(T}#76r!gf~MQ%>t*I-L%B%{&XDVc
zhxfScVsU(4)p6{NC$SyY!*MM68+$Qxim`IRsBr5Wv;NX5RT!-3Y&<{BQhAZ5G0l4v
zmSoItyjF`}l}cn6_O!KlA@T>gA=-4EUkP4T0}xFvN||S_id-AQyp}+Z0sE}32d53U
z(Z!dOe0=;sVIVws?Ok>I_?rmg>-&e&<jZaa+PYBKYIh7Q?Vy)`gvsQ&Mrvx<-)Av1
zDH!7DKrV>*qbn>1pQDggErhSb<r@jLWZ8g`5q9wpA5%bC<g4Gn)KE?U7y(me!zli)
zK$|YRt<Vo^tdJ-0Ig(&aK-|*3)q>T%{<gCeEmgHOA-r%gC09m7#`@Y7GybK(k0y?Z
zuP44ePu-vM`o>@nU=hD;@4}#lE<=bxOHA-R9*p9~>^n4?U0io+ogrh7&vA%KRbss|
z@%ztp;#L_y2SIt#LxrY!f}Irc!y?(K<KMostAt0FWvW$9$MCD{qOXiQ)anU)G%2UN
zzE|u)dD*&qgQOfL1w#_EAxWseXb1$nVw{s_EO9!BbCYp*4|*Ukt@^b&I9ha2wL>!(
zCc>t_!{PK@3#Bk!Ilw(_ynYRx5N4a_t1LkyVB=Wj&pW~m8z7ddDyDN_W0i`8B=!~E
zOPOK?;h0Sr{fX>^F8b?bPGH}o1><$ZASFn2Rw?{6S^OI~%`a&w<Yb+tx}wg4VR9~U
zT8Hu;=hgT-y2?I0Na<tG$8i4SKFglR8GZ^H?fZcQOk%#+(5Ky0;Zd%X$Yd4i;OY-_
zk?u~f2fKsyhpuEpcRBZfgD(MSr#9+a)CT25`@$WrFwnKyxx<vyBJZ73HT20+{awFY
zx4tbyQoiuW%K=!Q^W^bL1YoeYp&Y4NsHr*Mdx*Awnkbu>KxPK(N+YoF%K0PSgBSBu
zCzn(FK2T8gU422Q(C2Zk#X4135=g#dxY){?+tDoBYT!pVI^vyI9+|x#3<JqGIw2w8
zq;Y3V-Gx=qg5(#&O^irPqsYfeCIy=SN7UrV7d=@OX#uL*mSNLL)OIQUm2LYw91@Qd
z*wV4UWdp>4IeD_t$-2_f$-DWyq5!k)<K~OXTsYSW@nQ$A@hff;=i$xL6{gBuTTKjG
z<ju|KLF<^Qb!xb~OgocWB6hUK2q8a$gT5w5fK(dxxj()UO}<We`}xx<7o>A$alQ`5
zH`fQPdaP|N)2!m}tltW52otHa-*Mu@*xa5_J3Qk)dChc4|5024I8NI)DJ(H*la{zC
zU{p^Y8Z{NiBtOJw)fvqLazpApujl9%vG_IOm%+w+RJap>$)>a#c|F+Ka<6fY^(R)2
z9}S71B^16Q$o_071w(XGL+hDU@Hf0wEO1QS<s#<7@ipK%dYD<Kfep`Z+%7ShP|_D&
zJs_xxl_;l9^{^Qzy~ct_tZ2_oNv!#QOSWQkbn_~4b9gUGr26hj;f;d@Vnd2Gg{o(-
zYYUBHQLdWNftZe>=3$Q)-=H)7!s;zKGF>wA_0itx7xmVRA;3h#;xE7z;X8<t?Jx`}
zW{&rz?0%BZdixy}4pD+f_R|gN&|z^7K{{WnS!8qBCGAiWRI-m#u&;ioVnPH{kZ9G#
zK&%VoxinmR-gsq_4Fn1g@YUUETaZ<BDkXQIHi>2_lp@PmqQc)HOWQ%In$Q}XS9jVW
zJ@T2XyIF*;25T8+=VXkxJDQu~m6KH59X6RzR}n#(Tb(}2JsV0KBpYm0i)n)%qXi^n
z+yfMs`BR!y9rZ}onsyE9BYyn2A~F$#$+}Mac;z%m0*O%bigqSObL3;Ovs-1!`|hKO
z8)(5yxVVcJQv9XW2=am7I5@sY_FWF0Gzeg?^3+dD*5B-Tx!9=`6F<GaO!SnkK-`Fw
zi%RD~_rz&fEqP2VYY~qwZSxyywV1=avR8;sJyZX%h!N&XfoK`?CaSt@BKv(~^7JGQ
z1yg+-UvOtsrzYtwHnN6hEWx5AUg`nwwEVTCtYv<|QQjc>67fZPJZDSD)>(9*q8D~Q
zx!drBlDffr;c}BEa~uJ&R<xD-<h%VzBnQqT1UTE4j>A@1wxaW+@ifsLXIyfqEF!c1
zaxQKHK=~EkB2UCRZ8HdqC@kgUk}V87MSp--y4#aIg7N&1MGBk?NE6%>o(&`fI`ACt
z$WBy@&2LvGIyiY^&7g4E6g^V0Xf@6*vzo_}+!P$b5$lI^M2~!^0$(URYwtHR@&e{2
z1%w+J3$*Iv5m6`vT<}ZrN6OVU^@Q_8`}EVTzBh2#YTX`h{jDxBP*OvLp0x9V;gMlJ
zkBEjynu1xBBrS^?om(t!1rFp@xzpLfK}1+fKL_LUcm{3mP!RRtWF50^MQ^p}+1bqD
z#h?*1huq`FCIOx=c$sTB0pFKSH`p<rH{60qr}_c~biXk!0~pR`#wB+YRO4h?)UHmO
zSBPv8CK$>yU(sQDSaF^|vpdQaD4mtzOgYS1e^$}KCgE91UZu7JsZq|^?dIffJSvV5
zrnx+Av^^F(NPp;mqn%?~)W`XyiO@0h0=t5(vxw8ALx0(~+hQNvUV$w7J+NBVsFaDU
zcJ>Rs@gmMTEkm=IR~8LO1o^mrG5&jj<7{NUUjsy65qTjgyz_{4hsA5BX7qZ3LjpD#
zKT5LSePF?IyM$mecJ-%P8FlI_=q%x&*|7q(_EZ&MJj+c_VZ~o1$o!H^4)NoFaO%up
z9e(U*NHfAZyTp~$LLtzG??WARZSGwrEGw(5rfQ_|fd5Es^}g00+w=n+aA=x&qo5w=
zhcSgW=B7$!?vI52ypG-!+B^|R{{)gBF9^S}hT-BcF|g09mm)OSfosCcwUo~Ff9X{P
z;81R+JTP^;ZUDVnEzPR7*@|Ri=QI+%mMxxy|IfE8lmlmJn!QsA0<SRn6E_-1<I4N9
z4-@_#SOFkX%Q7pn*;WD|McSIKjKM^0E~(Sq0YPs%j4Vi8Ov#+}4~8+-ZB{hfyLj#T
z5}?az>928gifP!J@r5zQpfBL+aS{CqsQF^qIzQ{Lg11*z#z+e^KmIOE_yY59Cjrk$
zSnb0L<{Zy`5r$IQE8sOIhzuBmyB=|i92YpSLbrUM@6lWcSqRS-mU#JM+EJ81NcTmi
z&0<<zuYv{qSkh!7gHShkXcRA>6YR(nQZFz-B$t>G`55!~Nm<Vq2N^wlWOGrtsbTqT
zQslS&T&&7|Ipjk%3<m`6;gqHy7KWo>_Z}9*6k0BN$tubG6iEo#c_!4!*Bd>H<h_Ds
zCvOP%zgPoteMR|Oh5C6PS{Ah2gk;z}^@;>r4duv0A^=iq!;(x*0Qw(#eltX$@mf9x
zL>jxI77e&=Wl$DFX!N~skfh&35p{20J;3ET^Lf96QJ=gc7VLsUqqF@9ZCP0{T4WZl
zD~*z=-ps@ZTzqLj(953&jh(Y}dRZyD_l#Fe1u}CI{$6Q3k8}yWJ-|<k*~F}zzvsJm
zLICmvT$xc77XJBWut;koy&x?2hz`UTxwHU_HPX8*2#-52_LKAqqf`ZZB&Oi;=VLlr
z9?U>A=Hil7U(|E6lUTJ|oU;$<u#Ch;O)P(cq0$ll1v?6BeSO^0hqJ`W7r4iO<!sl(
z@l;$6sapikr1~<-ufwR`wj#i#Cm}yP;bH6*Vp7Pa#B)t>xTLb?bv!krS_egxV&$;1
zMg8E8XQNo1t||L2y<!3>u<rWXs|8NB!Ups_Il)iQi5r&hql+wLwd8}sT16NJNNMUJ
zH1^HESzk<|`d%yI7ky7uOR9-w>WW+z)%ZGKoFu1-+8dbKXoH2uiNcs{fod0vDn*7}
za8vjuV4|Pt&)7!VE_xaiSaQitB%AvyqkeL<sI>COsG6wyY3i<56Z(Z(24UdHuhKp6
zqrFM?rk_Ei66GX0JaM0(Goq{Unm4>TsB;UNCnv4sg=CMxEndm>LV{>sv|L>)`uYl;
z;fJ9qM^-6)9yrRqW*jfTZ3|JeA6|x)2`jg>)~Sr_DP=6BNUtGpZNQflA-S%aE$ZRx
zxUbXmLbT7d$wXD<aHOW`rCCe;?0@6A+b=}jRm@dg^ew&`LU=m7<ngbfMnQ@h0em{+
z^8vYHY-Oc1eO-s)4Kakp$mn{6Rg+;rSYIO@y`45_lNPVi%uKOy*hR(QEQc7z@Kq~n
z<Hk@ZeyAy45i0IY&>40KA+o|3SLVA>6!$f#wOF8wxTVM7NoTp5o@+S9Mll)Iqrtg;
zR8$l=$+-&LVl$E9$#cfZ8O0Phi&|d&{&}ZMyZCOY7J7Ckr4%=0EFLTP+^aWxUAxz6
z9EM%_>yG9pl*k@yOPP(^3e2bueCF=3q%a)A$CE>HVMx?K=y~MH$q6w*BT>B3&|T$_
z9(DCH9tB%U0CN%FE4OtKJ;_ub<LafSpS#gifn&LOO~)HKA?{$0d3~26#bSKnlq88S
z<8vs*BKcaA>NKsPF-!h4FRNm^?YFs=;ySg=wBmedq24s!QG_zFZ;6}Y(+@mo=Bmq5
z7k%|*`7H$XN&;&}@DKVrRBo2Iof;os?#dJiu=h)I#HWPwm+=l#4WVeog=bEfW{(we
zD$MnI5p#<bqN#(k_(%JZ0!r@c7wN(^!(@e#9Q#xvADQ0Rohh`nJJZVP6cbAYW29am
zV1~?%_Dsy}H9G-{#A;gf@*kmviKz&#;u^w6B~viBTW<IiQ$K<m1||bTo-6yVBCFz$
z%(+$M+$#t;TuJ=Q>~qMTH@^R-NP9eBy1b1c-Y7K_0_Zv-aG)xr-7vi86|6HZG|VY>
z*I;Yd(}8VeV^d3zBti3jAuFipF^W3kIKS4gepDSbkZb1(8YvVg>54HOl#%lrYD1J7
zqv^E@W_rzyw7IQ==cE0OQKu+}AfIL{cRLUs`T3Owdxo6&H~nIhl0#QxgF*n-qqGgV
z*nS?}a-VNzXnR<=*v)KagTQmemm}19dUHChvnndI?-Ml7etTOzeZMOUnA>n+-ft{^
zhutR477X*~7xJKgE2-rxB_I|O?&6g&)EXzrxosicWN`BGW^<Bo!B39#i=0nlat*P7
z)~hWdbWw9Z+*MVW-<cfE)k&44KDN*X`7nECUG>Tyn^5HyyN%#dnh&DtSHe<fpXQnz
z9a>RUnUzhp%bpATz0d5jYD?YD%<lcS458Zxz}jcx+Cm&ovd^<N0TEE7qYcHg8PPv`
zxX02gI2WOrkr3k5c@-?59wwgQ=#6RC+&2N47Fdhv9!hR3JYH$T4POaJT9mx{Hr3kM
zds;47og!sIa=aLP;@G(xQ;Z#KqzGFK19y8LD2cW)EDiFdVaCGBzbF~8-F=zY99&vj
zy4m6Q%U}eqdF08%W9nQHy~Q%!(`y@scAJ(QFG=}Yv9Q9ilm%JUI$nQaBHl^npqL9n
zR(e=!%UTWjFlb?t3S42!42pvhmU|r?uyg#B075j?KKR&JI#YCh8B>Gi;56PiviyVc
zD0{*~*2QuA7kmCEvQueo=&k{&wa0q7GVblu)1#!&W-^>q^(7&9M+mty(~e_M7hXEV
zx&-dTpR1yHEptHU95?l+FCYVk3`mM~`(j~>PT_7;4m#R+Qjht%XQXnxQomw7o5|J_
zRt*yhT0o{~hI=Fj0#l4W*-sd<+rYH!3V9mk@HZy-&0V!7NSc;jcRs2YKlMEx$!@di
zRq^m2yMVbVg6ICh$L~dRil%gybP$_`SxJFM$`e|h2FX)J!h;=`jwTwvPbh3fdy1Mj
z^n-R}pDTMsr?|Md_C@nt_(4KbbY9Th(M^t}gV8-IJCqUAj!KQWy`lyHxj$X~{5p`;
zV>O<}NvWBSk+yv*43xA=Q643&y5GWY;i0tx1syMN3jQU|1gzuuKv32nBeKK1W|1KP
zl(`Irnf|fe)BQn_a`eIy$v#)Mrw`UE(<KhwFx<U2wgghTlg3@F*W}cS`{vH=jnRq%
z56jw*hcP;dxA?>RVG^=(<aF~BAJ&8^Um;xUq*}YWo&nP`QJwRoT>>)+<2;~YV(dNQ
z@#H&i`Ef-;_&apdg1jc=JKCTPwp6m}d!);n>LEn=&94K6GfRrHy5&{4U~2z$iaJw(
zGh$xN;KL*LRD1w2F&HZK_G7#Dh<(=WXZw_E+|jE?*Adq;&3K0(yIyHd^iZH8cDoGs
zjuqm);PHl-^Aj46(+t+9>*}vDbRS_xLBLHC3H+^J+=5Ey1y>pUJz}Nksm9<!(Iy1t
z^BOuf;~%F9T@d827wGyRj{fN!D^ODDbzm*&!_{;ri#dM786>B<iYqqe#gG0nENsa$
z*?_&Q^V-_5kEXJN>0&!CTDR9gLN@XPq3fsJ!aVjyC=O`^c59n+?ETT3^pHDb%x1V&
z<{h~jJZ~BUrAE9Puw%QP;m>!qNG(GOWX8K6CfMUz9u1D2ds|_Yr0yoAs!#P5=(2y?
z2d(|Z{YWtYTgGvVgC}<mhK>mEp^LmYJ&j+4l1&#$x0s=67pS8E)2@N7M~eI8$J#cT
z1@saAN{nO3^xrg=ZaM(>1rGv2l-_qls+<3E8Z}Lu;0&gV&J{h~D0Jyts8&v3A=~!%
zSW-sR3+#W9SMtdKN~U~({`cE|fBla^M>8~-4$^O#9?}1WI;0B{9{^+>!7L6D*VY3z
z2<yuuiIKrNAb`88B`4#KFO%_>p}oL}Tc@udv<02QVO*zfb7JpS6MX_P9vO<zp-}@H
zGNCtE1B@0IujhHoW=YqC587D+BwmxZ#m&KFP4?m@yKV2roOBYDo)d-8SXPI^;ufgL
zhk?w#-=Nibj8pB9)h&H}P2_c-G%*{hpPexn`RPEmyCC@3)p)PqacMgK6&Zt!>d=YJ
zsucwqc=X2dG?5egTr*Bxuj$t<amOX6_Bki*`UD)>rOXjWl*49Q9s7C5)Am$0&}J1>
z(Nc{TJ|q4$edROx#LQkwd`7@;$+xCXLW|#}=jUevPt@0a+DFMsP1w^ZcmrJTK+OjT
z{RnLbPbn&+f&Z{V*hm3p2zxFz&8AYiy~x$+#~szrYI3cJuHpW$NgWl+YC4)n3*5|&
zXCKV7`oS=UcQnLIi9A8I=%>0bZ153n4swV=VE3u;E5ocC(Vi1-TT5CqoECKhfRBWH
znGO!qCy6kdyZrr=+x$J;p$f@qb(@es0qxB%r~FVSFq={&%eqDtK8^K|*_T7)uG%4s
z=`V<SsVU*A-x=P)x_OigtN1s7pf3nclG4qC2`rhV5*l60F~eYpqd0RmZxjuNAiYB9
zT3hym&iDj({EIE7usCS*iF0>14w|EgG9EWu&Wu1+j?&xxxjxt?5Z)A;P<JX@Vv%><
zZVc{jH8ENEtO1!#a>H~%9pcNkrRRc$^Ce$;PRFS^yR;&4eHBUoiSe(;7$v!zghwN*
z8TT*=Fv-z9fW3@EPCq>mFfBFkg_JtGn>W3c5&8ZMM)KLKt-fvX?wJ6++*!%cOE|b3
z*j3*^BB;6Cwh{-DS-j|86zz(P-dc~GFeK%&mf#O0y?Zq#DvZ0z54)3Z7MF$z@3z=U
z<<QiB$L6iIzm?=923D)<fHA+wio95Q{MI!Kq<?C+PkoEBQAEwB?8=@|FW6+W6K3P_
zgw0L4=F{*Uq$6z<*j|xn)2j&D(NbVE3VW5cB)?k+SuyadOx}DA{&}Mm)xDd_PCZTn
zm~O1S)BbR?RKZ0tkug~1S*;-Cm5eN65@#PsGSg_wetp{Ys5h=(%LEu^=4Cq++T~V=
z{Dk}GII?0v%zQ8R@=c~mMo}GXBZ|v`>Hu9@<4)rI`*0Q~{sE^%ZkjiA)eDwBOV;6I
z!|eSMVJ1W-VP5%~;se@k06)oLNyJ;&LNE9~IYU1|x49qLq?)Z{YIidUoy{9)`_@7C
zTS9+~WxX@LAJRr6SD7eCDwOB|O<%N=0T5{9@3Y&fi*AiVk=&&IMMp{G0y#p0cj$cu
zW?cBoxTq()466Oi#4wv}z5x-`1t)m)-@gS+8_^rID%H=liYZ0~)zz^)Yvt7zz}9rl
zXtQxe(@KIV0;j2o7!8F{;}}$9i(%RycZ#%+H8*WYcH`q>Vq(5|?h?5HZU>a;3bgi@
zQq{@14I_sgqkQ5Q=y`2Z0eAPS;3jIG5gr%E&+TUnt18f7^RjCBbn3^<6x0QrfIeJl
zQS&2gx&*7cepO}%guzt<s}&k6Y}hj03ww#{s7eTn@lCej6am~0X;hT}L=N%`--%g4
zw7EIn{=r|X1bf`AEkcN2dug}x9-~2ON@Au*{4Y(-i(O;|l8u>eH+idGJ!J0S@3sZ`
zZ-sox9l5-ox)=(IKY$6(xB`>7yZjF##>ZBSMU->H?N>#<nwiVD&Ln5It=AY4B}*oO
z+;zi@P{_&q7>6}~cf=LdXwq@wfaf{NXV0S+KJ6jh7q_Q`U%?Ky>1KE`->%0K2@H-$
zSBAIVt;<ccf%-czYxaSup?JU}@;GNgN1*vGk8{NsQg75dOZbi19N_WZ&<GFh7|}k{
z=p9C)viKm%u2xg6eh0y@6@eO*ME^FcXu@{5B}`aQ1DPFXfrNddDrG{!+b8ZuTN}3N
zPsKUPWP?-m+n^EE2Dzhxo36|RxN0b~ZwC1Luc;XU>APQg4S10br`fp+{Ejz3QjUhp
zHeL{3=}qHP`D&q#U6n(K-^QIyJ7_Bmd9FUa2%$gHleP{ixT}YP(s~&bewbb@kEV2s
z=X5i8A}58;wwYb|bh3-J>h^#ojE}HVyPz!4c7}CIb(DKLJMB0dbY-R#0G~)9EeD=j
zk_+=YV(%(24KIufh>mI3e{K4CnR!1QYk17$+th(T>PX3ADLy3HuuxpK9`QU<MacaE
zCV@XA_!~aHu3qdL%wQ|WeigfK={rfv-kInWAL0wi;4hZ#-EL@@)4`RJl8}O6%A!?{
zoQ&jvws`9SCic)HcL)2>T47od{y>YDO~YEgOY2A+%1y<(Gt7zh$1x5Utevuhi<jHj
z9=41=ckc6)#mjC9fWw1}G|y^3&isMiq_HKPlB%TX0sBc}WV`-k&?wW=ery3B!~<6y
z%t6_R5<v*I<A&D(9P81d&~!#kV~&dk>=#Cd;mk)&sRc*!6}({gh|`?>VAZvHa1&?S
zH0Msxg+vGGSQNxx%m4B9Ljn`n<Y6HFi=7by%|_cT+rmO@?y^q>V?Jb>E1@(zGrG)a
z>RDwgrta*+!7(kOhM$*hAtp);7N7K^t8hjuV@e*~k0QiCg`g3_u0tB)i;wU6Mwf7*
zQZB@pK5Q;4KwoB;L@>fjWLhp9v(b(nz2COPL#8j@Ue@PJnOoht$W0elP}}(SbL)PL
ze>U5B?2doeA0j(5@Y292x3vb#`UlImbO7wWnr^Bzzx>gyWkO7_d=7C|B<?nyTPsEc
zuX-59&Q|Y+EgT*UiX4YYJO6_Y>;4Aoe4x(OM4Dd4!wsSX|4rnJPhBH+u5+p{uY)W+
z9tGX*z#t75%5<7m;}2yodrX+H`Q?;CQ3)B~MG`neXA1j~4cepU9b>a%7iacaTd><r
zvW_M3+Y4h`yF(^6F%*wYJE(K+`fu1JYWLJ`U-!9a3Q&d0t>3f2bm>{O`&)^xbI4Jk
zV_k<?oy1Jvx(3rCJLZHOf8>_AfzZ#jav_SpouVBjD>e=DzGa|Ln25JE;-H=G-*>={
z&=3~A08IbO{dey_^IhRgBy<8~)T=T=FE^dI7cJ~{f)$+pg8-Um0NBV&Phi^@jJ{>k
z-L8e%rAqoiOL2-rp<NP2Re9!PR|4a@KM1uLoRki>MP|oGqr6@wafNNY4s>gOdc0?Q
zPz|I>;3o&P)-f7wTLe>i;G8rPh<}j?LeFVG4<^Rp3HW^8<FP2S>VN5lIHMhL82j+T
zv?$(^{?#bqAMBTKl)N)5dv1(`mhB6tq<>i^8e7o?YMtKCaW38LesN!XPsz@&=$o9J
za@{)O<21Y5)Zl*pKr)N0eSd>i3q&<ZWZYVR>N8G_hUi-1AEo|=RI4P?K6hE+h=FN%
zl!xJ%5sz2uVcT;W8p=t4J7h)UisqKs@zANQHVNa{YEu`Rk=ql51FzTLRp;Is_c({g
zuqOhO1iiC-(AhwhNJR(&+oL}kRA0usX~CD_M8<SIPiK}qzZ;T^NGdc(({vGIA9d1v
zdhtvH(>($09NUFyOgo#G>SbF&s{f4ZpYbbxE+(Y)C%Cih&$GV?@xJDMe@48Y&Ymo_
z`-vbY#gUQuP=9M8Bf~f!LgpU6fG*dPn0mP_vpW8~LSx76Ap*(ba)8U(CxLPquv@Ew
zjj>00)zz_$80*ci{B9b7&Y2>yrgpqKU0^FavM3!fh-AeY1DSlze#iPffSVHymDG@e
z+kKq6boLq7AFr%|;PP4ZRlZ;Tb1!6K=`5!TBKhDCWp`g>@?uLO1@me!lPtQ0YL(1(
zoI__Q@GU#(d=u*<ON9&!%KMWN+T>N(iz$-6Ck>5kK(Wq2mE@m#wxCQ5{~SD*%2w+*
zMu*uTf#8X<q5%h>*$7(i_+1Fa2M^Y&{YpJTzED-(md4Y{EeUo?OR3(LoktJRn+f%0
z#h;igEZGI}4U&izZY;bZ+8Z*c&z5q|EsoI!lb*IeZ6;&SzO&nAjxL=@Nj>foM{F7?
zWUTe;$V_bgEd;QkMrpoKzu<bz1Sky3WVa930^uzXH+hV1mArc2c%B#307V^+Fo83|
zRllMmC>ghUJx`ZY$6Cy&w%AQNE&);Z!)EIQNPuyQP+Q*_o^7&JDzBTW`q1}X)HrC?
ziUG~+;C8MhvR9`&Pnhu=@5|ymY~Hf+!NwQfw(x>H(4vu27dGeJ=uc|lDs?<w&06|!
z6t3N_A_D$SaQ6D3S&7zKxYdHM2CJOEE!Y%fXKe?an3hz&$rkaycs42G)s!?+S-}<J
zp1@{{9K=FgP=_zEs4q6j%l5JQI~^b>4kIEJx|yPKuBiv3?&DtKvIDZ<O#8PdLu%*W
z4*`<lRT?(K;ROYb<lmje^+!l87SD%Ahy4yeIBkYH9(Ol|YccI`EMpSsjDHhlj+MPK
z9A!9Ed?8yco;uGU+f0@k5q&Tj$PADEdnoBr|F}4&iYC7ff(?QK7X>=~D8x-<T@-73
zrPND~ktYik0iXI+GUqqBh-JYXlarVd7)T7d#%Z3f6$Gor$xN%>kfzY2e;RGBdvPM9
zxL|LRZx~OpLJ?=w9M6<!TCZ4{3brw3D=_$#?i~(3xNX|88T~WM`VJK%{e#Z3l+kAw
zK9EiK!Y@!8S~;IqL~?B;m-9P$XW;!W|Hw5t`8AljXF9X*Fvpqo2FLAZPNx1<e3l>m
ztCFb-HFT?@fx-eMw7jx;elBm%;Nql^WtYB#4H_|&_x!~0vBqN4Xj3HW8q*T`3y#ge
zuF<F{&~7v{E9aIK*wtxI;|;S$LJRCI#g@!{9x*rOW8_^}=8Ru*nx9oDIydh$QmE1E
z)57G|2_>9YfHV`pZyJ|A_}YM-;Y(=eHDM+$V3RBOyAKs}tUYpUVUrjgHu&u!B0^+!
zzNcoW9&v>S`5V>G3WiSbOXUg!=Pn0ik@<V7_EY)u$RcKHBX&G-<;D(D8)^YFjp+rC
z&y1CE_GjFBiR_6=2n|$us2%1PZDF*kw2ezIClOhHJ50`y{oBuD8;f+?xycbee!KF|
za)lHu$V6ijv72x|ZKblqvrqq8NtuRcQT3OFeUv8IIhE!|p2dj8A~KCR<g1psNFp?^
zP)Q{uI0Vtnnn4c}jXrFaM0D(}#6T;&4XC59`hJz}g9LgWx|=i!L(~2*-&Lg_Imvn`
zigEm~x`vTOcJx_Wso@pr-{s?EXUlC$4Nxx%rTz99t1fNIsg@(NjGN{v8wO^B)Cn2r
zi7;dlSHxE{C!)F>q0gO2t*$<mAC8VDciCUQIGd}w`<J&QN=0&WC8%|>fClY~m~?ta
z7YXfzsz6Iuec5u?41ci{LAADu-$h|$h*@3@X*?mUFGA!Yk0C6eU5U_~MMY`KR<Z11
z)EJY~%D9w>3t7*Uouk>_z0+{fOpfrFS}q;g=Rl5(gc*m*Ga2wO01#h6X+gub=5oDE
z+b^x62oApl8H%nOP2J4Cx1}Ai2ZTKmCvbJ<@$oA?lId-toc`QrDOMKD;(9_oqhK|o
ze>hu-!pA_*vrA9$r_47h_!YQ0fjb?jyXT$xoP_-AlExf(O5M_XGWW#k2M}0RUf)zp
zVj{Lj`U!h@(h=N5OLkl30>e^6sZ61&p6Bwe{^8}w;6LU#&7{y$&tdD#EA|&mfOK1j
z5t!_(k+GStuv&g}2OPL0R|XPiGTdO1D{`}hgPelnRsH&R8Vr|~PWS-c&V@o7X;bJ~
zw&%%Ls>t2*dJ<;#0tN>|nP6PwVM@k0^qwF@7T4<L@4I<RRHpd+vQfyw+Sm$OD#~|e
z6v20HB{t+>?nStbw?B!nSycxR+O!zNvqjc}a&l@jcTp;RmZqbAjpSuW=8-?!#i?V#
z#NO(GoR}Xka3?Dw9XQ_M^=~f85IRb;=xGK1Kui$Ou!CCDsdVI4f>Ih;8X;BNAWF$g
z=04hQ<?o^$jeY9`6BzpWPv9|yhHJq{t>-?70%VnRlTZPIn9R-o=EH;F$T6*0=Jc`}
zOb`IZCx0=Ee7oH~Z~D$%X?`*C@FT%1PZ*fft>j&83DsGP26vbyL0lYI)m74NUx^OU
z&^NTouNS$%iMTX81Zl`z9P`cKQ?3Vk7AExfgiUXv$<#pMEJ4B9K_g1=ihFI#ch;D4
zS!RN%zV+R<>K+=irI4q88g8>el=i3LAWOIrUxNQx$u<ZBgf>fyF80=czl*wlU#cB(
zyR!i9S_Ml=@G<NTc8d`ER>J`=l)NUXIZw?ie|NPTj0=E}J>=;AZ<e-FP9W`^cLw-B
zFYx~t49o?fkUQh=`5!Jyf9dg3&HyLPIwKl?|DOi&e*|=hzLHxJbBbW@0Fa}35t_}i
z!{ta;RG0nd07gn!OX#B>d&k^;*!j|4eb4%4Wcr0B*Qm7be_$=BhLO5CtH6V&qu_dO
z+5dO}+^yoFS%pSp&LDJoCeg52hXryd?JT_n?0oz^K7YY>+vddG*!rH4S1o3wbeK@f
z9Ol=<hi0m(N)9+N@3vU&?UKPiw>IrcfX;$r0ff!R<O_eV>tUTb1Ir4Z#%AP#`-#3z
zg_5#d)E5c2_lO}9z2IwbDrLmm^=YfY!G&ivKFd*T;dCzL>VKL1ZkHl`J&d|2ou5kl
zqq@w6UcMfc6$4T!_*H)*PL;H|u<O(EU_4yKlG{#zMv_{iztvX_HD0JtBodKtVkY|^
zNQFCr0HM#&-eExk#71)wo$Hl$4B6{Vdf$?!ukrH{Aq_SOzNC7}11QoOU%*}qwj9;{
z(IyaaWCHj<_xUeGNi`ji?y07;Ee(N@q5vNi{YH+{8P#g;yRB6KkE>lGEL#8>NBWYB
z3t#XV)oALhe{t-r<yVYFk#_GQ(KKslbk@DIic!;c;Yb<HG(s7s^UemdNKF!N#%Ttd
z4ZQd}*P{)_(2=jjlzY<6-oYXFWQC!9vkhqKlLPVE%W1i&ZQ^(DgqS#2cq77Pgy8!n
z%UMA;99NZWaK65b*w9qjFWd3wiQG^MgY73sm##72XNu73AWAM>Xq<mr5i<i40}p0X
zdo3y5T4bm$%#jg34Gm1ZjVGQ(vky#X&r_9g<sm+unz029&KGKKlgr#_O3c=ENcw3j
zzY!(B3tFS@Vx2H;240$CD?!aLh?cY;zfD%c(f3%+I_mIRiMOS!AtRpMO1K>L&<lfG
z!lJ&JF0BO4;-Abg<Oo<T$Oc1of1`Ix?Sv8tK0Pb#0evx2J>-Mrq`Hhn%CaLuP(u6F
zTB-GAn2||#7dnzd4x-=aFqcpfbu%;)(E=eQ2=rh6SA&QE@#fH!nvG;iHyCja-(mE_
zcKC^%5^G~lYkx<SN~Dq&0^S@|e0|AWein)TktB&2lKSenvK$7t&nO292P+kc;k9Y4
z!s4IBqh(=Yiyi_(#ij@}Ex_Vy&L{+Aw|e7Ak_zS24%bMGAHOR+EE_c*C?lkon?wzB
zuLuNvrG4GCXh|HYN=Y>xc|l1hNhRF7T_;*Z5I71xK|eWW*q{9fT<6@O+5u<szg3x6
z4FC<&llj0@BBQT3A0M*+Sv!mY?@!UH0&LMw52&hh=G+0k-jV{OD)e>#TZ6XpQ8zVo
zp8}d-{Q>_U#TI4$&jGkt(%{<KdV{B=psUgC;f+o;PR`Tf&y02kz{_Y<0lJFfH6BT|
zl=(nhYO7bBw+c+q>$I1>3^ip-t@z(qteN2olV|R$#6ymBCU$`BsQxQR4YC7!$zHMH
zf3w0v`8oxbvFJ~}ni5PJ4hp48NXi3!yuzV0zo?IrDxPP^YtmA_pYQ=NR+mcbg`Cv#
zX(2^4x4V%y<p@i?Rn?QM6q($=uTXlvU?Zu3j*{e8k-%BePtHBj%~m}by9)E$Q`xm5
z0Fsmm)v!ix(|+8=uMIeZIr>MJ{<j{tTZkA?Oc26hwqiv2N(R>3XPQ@%h0wnE5~~xQ
zkiw{1o~z79E)y79lQ>>bp<U7{z6V=uyX3Jg<_UQ&+<-gLfHpj58soKfj-+=q52e$y
z^u29=flC|6bA}~<`o~72okJ^_)u#gyZ+6gEGekFz>l)R<y0|M&{5pd+tE#ULe;4Ea
zsa;4h0YuTV^~ie<48GSw-`tQLLivK}@v$Y%ZK~^FwX}@pRSte?;D)wSd`74}GHm#Y
zd{L98)1YEoR|@nXO68g(RfSC;{!Lk;SV3)aUY~Pn1A2o;k+jLOKDPq@$e`V(({-?s
z8#s^cUo_g_U@<hu%egInnauUOq@xbP)YO!`{QMWwj#>hB{j}vUEt8m~B{N!Xb0Eu@
zgtRb~_+96*Kl<3et>*U<fU2ypi$MD(^gvMj`B0fi+<mgoeR2mvEbnaF`Xe;C-i0_q
z6KSrQX)v3dv179JW8#Vd@{*6CmAJRpj-iI)EhC}pLlz1edLq!!ib#_(8UgoK&BOK|
za~r~vqNe&(sv!)=!OK}Q5xaw5+Hd?XzpZ`Ziu2op{M?CEu<-t5KmMAS-#01kxh!?b
zK3Swy9;jIK`>Gecrt5C{P9f)XtWA1z=j`8}guGyd9>P{{Ou-tx0G&_($#!+#C7h|$
z2p9(gB@E0I5euhTec4P#Bg&v7v<(|hvrBA3mqb(8@`8R53c?GNWx0TX)9O!7{2=mA
z2UXaZIp8%+NJ7+DIRvLevxUj~e+j)c2|%*e=D#cPV2lB-)gZu^E5zgp!l3}M)`e8X
zNr?PohwQDDF(Cd*5<faGvhmlS(xK`YN+q9?QcR<E1&a;$zh()28J{JvXm$mf7{F}p
z4YH*c7eZpDV{i1y<ngZv0C4e#0A>m9#7b!SOaK4R1NNV$eZ>F(vs^SmzSuqg9XI~U
z5OhIQ{r^lw{tQe1+mCyHV2M4wug(8{5dLnZ4f7vuJ~O#J?!Q|7_aAgLfFJ*VxM)Ch
zxQY$YpN!@Ntg}}A>LB~WhnDj*q)O%k%_ajFH8zFeNJTUG3Rly4>XBX;*L`7a0PSDz
zC7thgaothX{YnFHO%^~X+l%fNp0jUWesGuikq}j7^&_O}&<j>)GAsC0<{e&9<_%mi
z3V%TLl9_<&01|4Bi~ED4oBDrf{FB81WPVA2+gl1C1YISbE3$dtTl3AcUL7F9Tp8|{
z=WH}zNby#_dQ#}Vg6%}VgDC#8f=NsI(~tjhHs}@t_#IpcVNzI;p?n1e!H~cQNB4ll
z=u9R`boy-#h-0o0hw%s*JdY$sGF=Gb_NZ!!NTDRRs0wKzpR4dVuSA3}Uf8wZPUaY!
zAA6i^KS}0J#F(wR?4d8D!+I9foqyJ)*jn%f?&qs*ab&WjtVIKFsrlqD7stGRP3wjZ
ze+J!7;zYuBY;a?&N4tm_*_6djJ;}zh5pe9oDb7p0uzJPKtX?t60=Gks4N-u=BP*JK
z87lz}kX79=YQPHX9sx<jB9<DDbb>1enW7GEKoR|@<h#-9V#my9a<0y+g7x+|<-Svc
zq`Mbk*0aux77j;2iU{J|TE^*kMP;}66j8|bPsz4nj3zkbXJEHrVFMX}XUwu$opIkK
zp8UySw3Uy*ap4<wK=*esxY_AXc5)uY{rHAO^i3+Ki^~xQgGEV<yUJyck;wrLtRN<}
zSYR^@yWlAV<JM;5Oe2Unk|jMX#@Zx)OtHiVSal^-{p<Hde%$)Pns7So+F0vW{MWSm
zD`dy>zY^17O5qBxYZ4`sziN{|4onuwv;;nCCKy~_O@*cLCXx|owe>#Q`EG}F)C%aO
zQS`C8OZ+12jV+N`RND@V6KD}MoIX*HYm$l7SLG5PpiUH=b6RvMaFR&(54-%yB8A0r
z^!%fgeVp3-Td^c^t1Mz=j}`om7^L~1I|+%OrL3GQYq(mE+$sJa!rlTXkEYue#tH83
z?hxGF-8~RAxH|-QcXxMp2n2@^++Bma>jVFj<bA*Mo&VmdTQx-$HOx$R)4g@Ay~iJ(
z1ORw83|~}90E|surm_RByKq_@&O&=uV|2yCl`{sPt0x{ZN~FSF6mUHGVEmKh_ozqT
zo@<qBDeF`JQqqS=K;Ks*9G1jve;3n8l@+oMaJf43wif`v4sd{AKtI>LPrhTfENKeV
zZC5gV7uy&$|K$E?5hVD4T%-He@0V$M*Q_B0R>xzCCX?g&fjI$l0@118_LnB^mD;?`
z$KzB)-Ayeq2|0>#QH<~p?|prwA9aJ|2~c-)Sw=jt)yM}cOQFKMM|%^SbD=A95+Js!
z7<jLk9>$02DZybYqTSyf9tuRzE@VAcz|@jc`*BEOEx5{^1z)2HlWlrdFmAkOXVPb9
zh+%q;)iMfWi^*PhTd!XCS(5hNp(3sB!Uz+Li$Ik}HnbyN&>ge4Clsexr=sCzf4!d9
z!<GPUYX@DIfX`N#FpRdoO__RZ32C0_TQhKw_*TN)b+T!GBI8rfxCp4Wdw*T!U2AV_
z6-M>jWog-3&U*1n^zFNOMn!)=d{|QSzBDZuz%|B5`Fb40R<w=oRX~_2i;I%#u!szb
zeJI{c4nk%8a{iTG#XDv*)h@W6hp5@@OwzT=?x?>xarf_O-&!F9p7LhS{k2`IIKL%G
zT$<P|y7_1>^ZLTKkpr><;+b_kUIJNTX)?Jdac;v+-x(%5CiuZ<@~>a=3#eCR)09YQ
zoX{LsH~S-V?|T*6lWx7In()4}u?JM~<wwX4t=z?ud0D^@t|;q1!q-V4?X9X)SAvh8
z(8UEY&V5sYG|p5)N+@?)kg%RFg>Iv{w@R5Tr>D+#30-FR2>8k=$6zu<E2B<jbf&6E
z#w(2(oVC#Z=&@(Z)&fV6h^yUhwa%?Ky!}j)0FPxSDSGt1&}L;l&FfijuK9;ck@><d
z&AN<qn}ahuacWTZQuC^E9KnU=gMBGlWo|creSYBtEV-S(0KagAP8NXN*@fXFK8eJe
zh<zLCm9#$Y_YKO!RX<?rc|-xbg7(akSvj4S@5zM2DCjGCGGf;``ys-`mYpkVKv#0_
zy+V?}J(`%HKIpm%Gu1C<R_>WAx{O`Y@^R}vH1X_5q+M06x6m$kL1m<QNw2ui5{qE%
z1(KXbXHJ6RU|*(|rP(rQ>lUY^Zsdap=9-r}D7?yC@ie~OW|^DOx~5bGw$YlD3ca!r
z<jfKde%TdvUS*dmkhpc>*v9I7D5wr>Q*L+o@v!@&+B582H-lE`c_Mm>+&d(}a+>np
z(}v675y!{SGO@=0R}}pi-y33`+*mtzjptx*4ALh+Vk#<^ecQeGX^od2%RHuHsEUSQ
zOn{3lAV{a@IerS*kMXg@xeW5=@xsXB6AaG?+lp{0>BuV0{FM~xwyvx9^@Yqe<*Qoj
z5&@TclP}nG<s5--y`0Vta<q%?jncam^D{J9vOrq0_rH4kk-wJ@I<1FYNa!2X(GJ0b
ztj-qHjm1}K+>Ll2G0%K{OyEWn&FUh;gjF+JJD>pRRHcd2HW9N?E+OZ_ukwUnl|rtY
zlnquW?fsVHs%EE))Nh!Xv<-CLDsJw`LEMD0v15-6yUMhxsX!m?My8JQq_d;%J6NpC
z%!6n3f+v+f;OB0ed?u>Ws5}vULmPI8FcPE3Q7+g{$H&$Y7(hjUIm%>9v%$D~1H)7<
zD})1P9mzp*MM4;{GMnI(HMOBga(_JeD*--WY}y?OdlkaC>}yZ$?TaLWHKog3YRwCm
z2V-Sx%uwn$jNr=Y3UWWwxGdaYgwM^0oG#hNQbF7oatW=?-`Ekt!yPu}m()d}S_1RI
z!qd~d7S{W|dG;?(T3hko@E<S3-0M&SPxN$j6+J1IiDzG#yS}6xqyw<;TW@<-7J`g-
zED+lT6P18`q;NbbY``yb=;(QF;vI>#9BisVq_G(BB+{r?HVtrwGLEL5H~ID{(=ky=
zS$2!o&LU@l&-aQ~p!~_oRxIBjwtCTFPH3mr<d8yw&^O$WJ&JzVo<zp)$1$F`-Zm|6
zQ+;QZ#lg?T7gfxRhuG3EK`37gUx+}UCINMKt<dOzdd54M66tFy&VAlK-~$w1>a)x<
zmX%Pc-LS@GcEXh3oSL)R_UkvoMQLuNB~Rvczfq8*IC<q_g#KU|5SnBKvvTcXNmx|T
zMMR7CHmrG_>g7zCC>igM=@WU~T98sqy5rZ*EA_t&s{c2{B18;qdW9$!+@ID%PDZ=|
z{#yL3QLcGGK-4oBi6yxv8c*1!J<8OjxFlHGHc8b=J+13S9{Tikb$m%Qp@`NLPM=mL
zwMi)BqBQTYcl_g-ZQqJsdK#s4x<k{K%hZ{S-i;Dc>D9!}m)4Oe3_(p?`d9|?vvvJf
z3bS;2_(wHTh!}~(#gP@RTxy<pxdH;rnXGstUN<UO7E}MDiSl)=&&$zoH%W)l??F?a
zEr5wn+xjW$FFT#Bda>-QheMy+lmL$wB8NUQkpKAgBHH`3Ws!tJrr9vrcqybLh79*z
zSuh8;`MB~lmE>CT0Y07(D$|wr1C07GlbPi<MbX{`LOK`irt>zAt$|-3n$P{?At-I0
z`h^)F(2pUiZq0EB%&UP--VmG%Y{7RzU(tq$8DyhVI$EVTJ5i@^M?dD*LVJJTv>Dzh
z0`&}I7V5LCFsts2lQq_?>5pme0rxaGV1kPi+H-A@!Kp7lrZ5?fM6B(3T9_{*!lt$E
zb75eJ!dq+B(}G0Dynrq!js)kDU72JM9D+TNoNM7aee{4yk(o7tgwp>U&AE4csH;F!
zr|&t{n7UFsx)h`5@v{iO@?2%tKTAf@);IPr*B<<SmVEG7Nt-;A)vP8hP_(jDc?^<f
z<ADxerk<_OFvL~9yT}3aiiQU6YvNPxG`oh*I%Bh){n#sNu%z+PXmXCwi6Po81;b2W
z-299xm?HY`AmblX7M|qeuUcOzSVZ7&20Lh6cEg$|4IP#5a(S8aTY9I0H^t~Qd!>#6
z?TRl!b@Z8NUZp6VS2_f+YnL9S<WeU5?vr^qMWmp(Hux3^KnI%@?cfONb8`IdGF$0;
zVr+4NTybomt(}AG5Pa4d@(S;kQY(VAsOyIUu(&7!b?&z5Lb{UNb|*>Ytc#A`QXbii
zlWCVn#j<iz)&(TQ{6fhleuWt#gxg4-#QN75VNf<6ZiMhMQKG@6SftHZ+k}*$9`uG>
zZKR)IAB!=-wj!V#&H1ee#;8uE!Zb-~SIFw6X}*4^R21}np~%(NK^Q5$t2I({E@mRr
zYxs<qq!P5O^u05lG6o6>AstEdhHGRFV5W$X7aX+TR&_>w#o6d;s@+D4R>rzjHXpt^
zW@099<!lkjI7h^TxX$1`)1NN?GuYl|x-{V?A-qrf6Ec!ok*^lBhW29(IY?(hg4huz
ztv5@j0{&Yn^|)8C2&m+H=*%ZQ_j9l$>61%KfK1bOy~v1%%|S1d6-B)Q?Nis@6PK*G
zgE~r=cwXi{<BLEI_3$r>QU;0vC16ZU%)2g$pl6aB;+ah*#P&LhWu!1n-PQvKRIBxg
zHKM!wX7&$nu(eMY(lj0&8TV3)Qz???chMi5_fJJ<CvzpV$@E+-S;*d#8GCA#@}ltn
zR5~^QTVw&o*=qNu54u0|LX=E1JYXS8Bj<42zzPV`t(V7^y>pIzNFB>B4eRt&RzP0d
zctgobf2;L(*l9LvE<2U#t*!Xwl5+sCyQEltWy7zbq)`tniL(7X7694yL(lG2M8}j1
zLv~-RL-bpW^KPv|Z}wGvU_eWl3m17;8Zz9-1ctg!waG@6g2Lc+GG4vq9xpZP2w3Z<
z17IHc$pwB`Zj>fy>!YT7oex)O7>}R9RgTZI-3$iFMF+bOn1VT{_RCR^f5dxqf<yzh
z)Iy%+$#+tL0lM<61wrXAjo}?hc{&E1iYLP0O{)VLt6aZQ1S7|KA<NFma|%g)VEAE~
zx3ce%3<Q?~2h<^p(9VPE7vP<;v_RqlC=v|tUVGy{VMk-wEO4v?xs!^kzC`}4dOpo*
zqylIdCKM!Dg^f^1wj44(US<p$b~P!lKgzYItlQCEe!`0=pE~xQbh+1jmf01A{BCbX
zuy0=TL$rITTQ@l@Dk^y?!I5*67IHR!V!;ChI!1Cg|CanHUqQ{8(&v-q;4v85SQDpk
zz><Jh*hzn86TS%FTpQKU0V#e$0_K;(!}zaLWUxiYu3IqUGloQr@C<&U|JbTJ6hP!&
z?R@-!&u3MLlCgFS!-elM-U<=vjOn6qiNFWNJhy?hNhXj>+Ypv%&{s4<G&^1)0=ivK
zu_9J~nS4#$c%t37cYH0})fZoyS|DcQa@)o<q&Yr7kME;b3y!+;82Rbs-C!rVS=v(t
z1gWz3YI0J+5}?zc?Z5wa5(r)l=xO{E5@S|LM+Q{6SR@frkF9(28SC3)PJgGeZJUDq
z%SpuqZPi21OFzD}aI{Ee@VXJK7papWw^_aAL9GyAU2?&^EUUj%k%O37FGsJS>PdlW
z*>hhFzhgpI=@2pz1bTB<Lj#e2;=E}WRvYY9AoR&7gppBd7%Q~7qU___rmcgd`z@W)
z<90$Q>FX6N8jzn_Pii(z-M0k@mgCt<tjZOdG<PT($C(s~d|p_!K`>F)P1uR)F<sB?
zB&7Wb(~r{IbUzuqB~(kD9wa&0rq&F$wAd;}H9pwH1vmW+TR}TLsNa_2?<HnE-d}<J
zHQn)c`|_)yq{H>-e!~7W`g2}Km4>JLFGN8%jl+i@!JXFM_ZoXIx)r8ak;!ZH2zT;B
z5JISeK;%-0hBTFXz%GeJ@BgB-gnsu*3>6!IN$VM)ve)Z#L;hkL&(?vo;m{w(If$BR
z7Jh4FOdtO_iFJ-f3Qm=JT}|_(xvb1aJarJv$%^56!8&ma*U9NTl=RuQN*;nI>WR97
z+izl`zbSnhN;)#NN_&5L%RO?s{s<{?!WLu@57wb)4KnfxB;t0^+=BOrg`Hp}Ly5$2
zmSUb&;L*+!y!nQBeeB=^O{lZ^FayT`Fw3oTpk3{YDX(Uh+77eoXcC>N=33?x6ye6H
zRHes=ipva#$f+Lh8chgA{>2cKX85$|N>)}@)X*lC_pU4!EkCzA`*`ref}rf9MVIkD
zl{vqVj@aUXLFrWQ?6b~{NJ+6x&;ub5rQ$_Hi2+v#XbMq)lWCldYpD;p=Y)E@{@l=V
z{pNKvFI!!m(}0A3dW}61$@6W!b(ZB&?Ssva$NNt_V*rk)Uvcy0o0OkH+=LHRtKD~#
z!4Hq|+6RL>i%b>K<B#|a#(W`)DQN@uB@ed2sn^?}j3OR8T)$oI8*0%5YlvGI7L@3$
zu1xVhz7BD=zl^a<ddWS_+*t}N`PrP%?Xh<@2nX;MZ^oaJ2VJ)`Lsn;F>ZnZl@vTH_
zBOf1~681Sv6ui)sAShzYx9zusavM>3;MStg6hLPj+$Um|e+zC!TsQ9VdL_`4pRo(2
z-BS>I1!VVLSr?q>*gxR;RwNPp8)HHO{oV<s$n{%Qr-?z-bRN94J$}M0p?@J!vF$>s
zyvPXNoIM}TUqebTQ++;RHr-<FNL>#-$#Jzo+)HKV`$7@bNp{(HZ^WiCJ4~QRnb<zb
zFQ2LN!*b&^fuHNfnj~rnRR@}$N`{1W2#lqWa#|5KjZ=1kPnK$Q(A~$3vBkDVOj-a&
zs;@jjx$PyKScglcJ|gp$m{FoSfb{K!dW=g5ci7U}76oQrYFtV})|i);^Lr35BJ<c@
zIl-ct-Dk~n)sLB4UJlm(08X-_XC%-?czkcG^O9j4r&CD^`uIfjlXsh^kF2=v4TSej
z21VpEC(JX=!{FW7iYfjAW|DOx)DT+r1nuxlkon|O-Y^iB5Wn(~5smZlJu{ogPh85Z
zaZnTq`fY_qHqOO#u&HD9XZ-S6!OEkb&Tv^U4f$=xR8CevyULSOyoidumly0gU||bn
z1h3<VDeVC{skS(;uVyO94dyh5R)1`=rtBw3@_eIB1vUP6G@XP`0hk`1Qa3)b7PxPn
zc@T|iu;l!&=qk83%9<L)4^)g8+BA|!$-0!Myw4vs5OkTww<A*hXG)dNI`nr{SmTyv
z4w*~u&u8;KhhOm5hyystA~@m5OgwE9g`afrrZx0}?r(MI2?hcVi@B-i0pP#!9qU+p
z%}&FOiZY9Kc^*F7eK@p0Y@6*BtyIO2hm!(T`i>H4RGV8yi*}kE2<_s?xR2f-xnWnO
zMo?!$e9}w*lvQ2i@~0`<emcfA>j8EHT7>YAEIF{hn%(8<Jb#w~c%LV~_g-uO*vvPH
zx?d>y08@g}SFDzANPV^)kcJthfhBH|T-M%o;lZ`ruws&uKfJ)($@C;k`(s#}USdgE
zU}1f?2brpY;gog8&0E^!{!M)7F+yS`TX1BY;-r<sBA0%JYZ^X;0v;P@QNsK@0$cDU
zOV#xJXB4iD7~C^Uw$ig82R$$bP1iVWp7l!Tax$S=Net)eTPHOoN-3k#b*N1DgKCjr
z^Ndpetl6WWMAuyvsT;|$<(stqj9LE~S;H9pxpQxXQHsj=b5Bl=)6jYc7f_q}F0@W_
zeZ-S81!(Ue?hxlLuCuyqBvG|G?o;O;ABkHsRGG;;c54J<RA8P(Y$6h-tX7Y6jkS=u
zJQD67#9z3$fmmxUTBYoo4lrH`hJp-snpzxd^z4u2V7mt;-J|E@g?^0a8}aaF1_HQ;
zODm4*DezSxvAU9Pi`@xAJ1Sigqylx<%rgoNqA4h#aM~e3(FfgzT>)^_eL6}WT|M!f
zk8qnZeFm%|D|B8wzY(isEBDt<-ZAxmmlocY#<6}ESx^ku^Q|qqvFkTEJ4iNpbV(+t
zi4t)(SYDhhy1cXxi#;94a(UsG4)K)2=D#U{nWsd1D_9NiI#@EKttlynqqUy=Kmrmu
zQzj5xs^Flocw;|51J1m6$oQCVL+BEQz)+&U0&*rv4-%U}VD`uOm-7MZ2iVBJZ8#yN
z_pnP4zt(^5d8+j7hqvOyN7~!%UbW4e9P>l3;hjYqq}xk#d{zzndO8%-!<j;E6u$EF
z)ax{YP?JKB4uG#YB{`QyUG*|=EB3a}S;m?yjb~Jj##-mi+Lc4^DF!L$C#F1w1I>w=
z34*Xs`40<Dog7Gb(FBHH2Z4Mb&>w6aXAeDkoBe}n5L^fJJ%h_mD6Yp4eTsylT6R>I
zZ@#|+`eZErgiL@P*ME~Kf2Q%#UkM1TKok$#P46M6xJ;f8V0yoW{<{1m`PJpRw#f~-
z+LwY=`U;mE$c$Q&vx?BM)*Ad%Wcr_+3oz~?NBBGYms9^M=YPNXZ=2!|V6Qn_$bSF3
z;oUz)k13HrH?Xa)qbMGNV7}TwSYxurAe6;rp`Ud2H_Spgy)bX#!NsBD!9=zbfYWXh
z`7G|qUW9?COrW*V2oR1Uf4Ajeffl^u?n0M#0pA}Iduj2K0>?jW&guc_!q|t>;jQjI
z=3PhRpr*ExAszH~1Vup!JUSv!eB2KpFDupy$u|B7mD*<nvAa`OFzbleIxvjL>spA%
zX@~gVi8ntvU<GOqO5oh=mm$VNtbb*m4|jjz|2m-S`C#7S{$Q>_V|jO@{GX`zk?A5%
z8Hkk?WYb_Yb+z5lPF`(FePQ`6b5gSwbY$3UPn#^(<GeykKlRrlkh8$@_-__eG7XZK
zv^ceV#(M~(8MGhqn%EHa0oQ630X47U<_Y#5#`fX#BC(MK%^sD?>-h!sl?W`Zh4`F`
zo6G-Q5&AQjHR9RNz)7IeYXcrbRp}iy@q~%#$zX8<4#@EwTBCxvu155|3`n1F#)ml|
zw}zo2f<p%EBQj?bfxh}J!5RZh&yN*?rPl9+ysai6{6Jn$0YQBz7c_1T7FZo_=4kZN
z=kL&&Ds$$w8&M_tDsa9Cw$Ey$xMA$g6h_DJ&l5d4fLayZ6J^C*{yp8eFi>p$M*{0q
zcBn#A=4*#~wr@gUb$;BuhfC46Yhv1`f9>u+TLC!@n4I%(x2Fc`Z(|W5%*aU@)8(sP
zqWglYLz;$F{-^?L21~>TgoT}^JWFGa2+Pi*Isqqe*yd#^poyg8N@%IW5NHAT`Y#+r
z?Pc&Wm*+C2JU(cI-de8}T^W`FbtYHHmUL?RlJQhD!8Zs{1Qs*<2HlW2r$lWW2bfDt
z7;3xsTT^(ae>?EbmZu(_9R*1J92)AQbzTvp<;9J-R+_g5VHdA)H&U73Vu)IuGNBe}
zF%ZiA28*`AEBzW^*&g0>EU}jNp|piIzerI|g=fKP2jnF4u*ECdzcRlIT7JaIVPZoo
zBBTWO7sh-HOk{jiWMeA~ce2gMh|HDbfs4SX!_DXq{r@5WJ_5VdF6Dm61wxMOVIYE_
zu358APXgyHUA#wpcetcX0>d!Q6AKBrO!y^bw(W$N4T2pmgt>lEFLmlDFS7at1##Ou
z&l}16P*@lw=FMBg7o7Ulb3sEjlJj>swjsgN&tl~llzo^yoTXO6Y+tYM9**Q9`*ltM
zhkO+NVfeK(C~hfG5uDmh+`?_sBa(_e>8AdZc=J~W!;kc7wHL(K9I5%$Oi&$bpAH*r
zB^A90e6D#LlltErppR)~4PM<zIBb@K=9V>hs2Cd6*#^QFW)P@Zn+;tnpxWDY^5#KA
zfA$bh5#NP81?f5*juXOa2iYZDp5lPR)@NGP`DB6;40z0@e4@{s4j5%2<LNkZ(FR>R
z)QU3sRQ34u%u>d-H{v0bmI{s6IUTF(x9^1fuki<5KyomWmKZ~s-%(-zcw}N`w>*?*
z_GnNvxJ2{T#N;&Q$8HWph9SV?NKQC8EZ@BZE5(nV>!6R{ax~>}$KkP~J+685<e}hK
z*gpcUK^D$OL;Cd6Hm|;dkSs8TJ6FUYHlZvG9||H3JOgN9L4CFYT6^$74FUY9W`cJp
zRVF!Iy9X3nv<xZw{{D-ayYiY}DiD>-raeu%)+Ya@T*!!&Rp5C^Z>bk!8fo^*&`9n_
z@W!|D5E1m6h+nlC=HX0YW)HA=e#;|PlGI%(Lnz3&7<GyJY}*Yk?-k{o5>UY}9)<-_
zP#j{Lkywo9jO$dx>>b)ONG6Py8YI+1?c<4ssF<O$X4~CQjQ^*^@1LW#CEj!4I!PDO
z^=EBR(2!NM|Diq@0I3i9K<Wdtg)?~j<Ao{s5KGgy#n2*C1-3=){DqQ)JXaJzLYy1?
z<blnA_Vr{MGdO`rg6f@Y#;$>geV(TTA*NOXdOq#uS&DQIe!41URe$9j-SnF)1MP+{
z)<tuZEEvsEwy9)rWoqr^htjYtn(BPa`1|uh<mc6@CUHldt7^Hv6?zO<eIoelI~e9>
z=q{qkOY=>$0SeNS-sYva_OzXE2;HV<6*7qoa6Kx*mQ`BS5_(;B<>!q*4YwKly_?bz
zmNYp1*4A~AMg0*UzQ@<%u4aEtGv0E*qXhr229b+gpbs{~>@ohGKw@yu9t!fzC)o3%
z#ns0TWr(5VQIM#ree(}Ova;^c3}1c?<cPzu8~hx#n||=n9wD#X)ZM`fQ~fc>^eL>o
zVl~h<i@X%}j={>g`ogtOVMVf4<lz)&y3dPwl3<H*zrfmKZBa7Xq0}{J0EHwjwr82w
zs@{{;qWK3!=@;R~J%v*y)<+zGr&W3=E=2A_^iY)4yM!lakad^3&a3Al&!)qrf@AsH
zPm^ukYJ%Et1_=TB^xpWCS4DvfbcwilPe_*et50<1sNY_=fx@FY>bLCwtJgs(w}AbI
z42bZ*{sxOspb_JQ{2~`Tpz3`ZkTBd^L05swtkVq(rbRdpM-h}}Eazk<P&{npzqn{-
zQ_tVvFdop+%B|p|QV!D-gY(3T-6~Suo$<Thu3nU`i0(w0<v;qAV=>Dd`HjI200~K_
zmuWk_Vz@#FN8?nUg<R7qkBv!XeDH=!E~N;4T6fN}C2!Gsy3I;w*DgPUOmxD9n99Iz
zTDQYa);vF-WIz$-vY_J9i`v##3Ugwi3RiV2B*fT&WH+b!v~gY983mj^ql7@ncFP#O
z34$DC_oE0b&-)Ryxiy}U<7ih};iV>YtmE2Oy1(5({#k)X<hF@}sJ~lZGhS$nA896H
znGn^dN@_Xo*mJlJ_2**AjO+Nk)@MtR(!qd!a*%_h`TRRp_n@74j@b=uva_mBhbu0g
z(-rec%>=r>pTM0osWK{ZM9FA8RU1^vmunj-V`uDww5cikcjM!Q`)qY#Xf2TnDjPA1
zgH6%%hbp{`EG}q)PPoOvel)v4$WXV9)i9nNVaT9SimN||vfe@q$1SY13m29PRRika
zYUXuu2SbWNT0~9(d($r|cY^b)zBo;F$f{5-t1#B&M>S;x#7rcWmun%+QGd=2*uj8?
zr#d*TQeGwhVqE@P9)ep>o|%L2_hgZH%PKF4SkrpgDA%<$iktYqdGSA6BmYAkB^NqL
z3=mml00ALI3s=PNycebXQ%5!5<JLhLE<U?JM0?e`sv|Z?CRLSbH9#dB=wN|!v|<-2
zxkMmR;9+(q!aLnolhgC5$gFNEcB&+*QohEGp6542kJz!w7&_qSUaECyrSr@&co|W#
z4RiLZb;)B2<p;P9!!<ZA2>2-mO-Xqpktb39k56dFfsq<fl~h<}#cH@Y4X-13c0o#h
zb5%ZrYXaFI>st8zqi>z9Iw|VB#>%*d4szq=_OS5WK0nlPGGtkYJ%7tg%oL;zdbxrG
zOY$aTRX=!}p*Don1>P@S#%_nAcejW*?4U?80IS}4+f(Y~!M~|O;Ck4jrCnoi1ACVK
zfluy6_|1Ys!-!)sh0OBW4L_!5tKEyUZ0A5xbZK%XIZb7#LrZr=P#4FR*B9p_b2Cxq
zb-U_K9e^m5qA4U?qstf;od&H-oaB^eO*M~JJI<_FN^=yQ<w#~wsrib1z3A2X=ITiJ
z#_m;L`3V{qmPpLB7V~as3ji}@MDC*RnxP!CYPvWK`9DT<VoZ<LhN3(V*BS{>rFDM+
z+lGx266k7@h-t76p3!_vrfbe0oQqt$CW6H96Yl`}IY(p%B>n=H&VpUr#`LeFXL;kD
ziY*B#?YMS6MT5`2G2$0iorXn_lRw&jhMXxHV4kbgUOFW0x(4g^5B?9V87&S3$YI{f
zb4dKNEXJZepup16r^~BSwRxnSw(9t7c)0`LOQ@pChdiuTMvMPv#Bv7Kzp+vr`0eQb
zWySn^KMe%J-R%(Oqtu$%|K%<=9?*@5MUZ3+{;%!&@BfMt1Q=<j>LGSO`(HrUztZmi
z^-&;j93*f+{{Pp(<B$HU{|!T0vj3&>Uvl$b&&B`@*jdFL3?7y9zYoyABN_?;nykkC
z@v{H7J}@vs#9zUh;xmC5;~os+Eqku1&COWf*Igpyw@54|QhIuL*kG{5&%XR$#xp3M
zYK7)Mk^e_*|M~pm%7@=1iA4=wO<*71o<)(o)>bTUZ|^j2C(07VTt=8q6cm&;`|SX7
z<Xm1i#8Smv31??EBO{~k6b{GYf5-TDQd=^=8wSZJ?Xjfya8VEkt(~9Se(=`>w)N~d
z38)Qwy#C2#*sMd*>;Croq>|&X!eEQ}zXyZO3|xH4s0cI`|8T{vrFvsv7(PiZTXF@@
zki~m{42NE;j@TSCb1=#{_Y<-3GOsj5cRMkRWwG1R;|=fs0mXlsai^|PQqAUoTZ!eq
zJS&D#;`v>WBG6Kp#i+36?Kml%$CjfKvQ)s0^wS@=&Y1kyGrgYu2D<DGvy^2hMs|u*
z8-)PohIH1wKo-{%cUD@=ay@g~sbRv<E7v|b6$y`!D*l23{!=H|k4R(5IVKBuXf-K?
zYuSqo-Nzv?@<9o`K5N%Lqc>n7iI`eb+YUVX=sQ2O3cP(G0)o@w0hr9QDBj2mW>Qq;
z4e(=(lO}CLZg#)RBU)e%8xjL_sGpA8>+dc!K>gie7!`riEf@w~QYcLbj`hIK=O(kC
z0uK8D*hN*3B@^)rt__EIyj=s-NabH|j&YBYPspK$IJ6k1{pn&$+86~jU>ys3@;toW
zWnm|Ne7?pBg8<sZHw%jCVuoZT(eaee)d@r6U@^a`8z<5Md$|(M-uFo05G$6UFuuus
zCpwcrzBo3&YS5}?S_{XbYOdd9=^yq$)(x0rOCb=|T*0fx#@MWNnxXnnEe!PrTp(f#
z7l_!p3LEgjwqV=GscCtfG;JiwtSNq}uhou?9}CMnDko%ajZ=@%0T|T^M9uxJT=54T
z7R@$`UD8-UsR(Brsu>^d{MN@)1_)NiQmsuQtfiHD=e3HgYws9OL<}pe2hc>HnZD+^
zClCGO+F)-&1gwh0T(~-&0Cy-hBxwEBe7S1rWwhN_a3egwX&*5GV4xmtL|iUgxEE)s
z@m>A2wpNng`##A|m=q>C1j`3nH%~4V1@rtBf**Xn?c*5>tUp2RYY>EiI;~9f%k<hO
z*<1kiW)kden$m_rdsbiIel`bx!p>Y=(!L=Ia_d?=E2C%`td8noSU88RH?*59n0<pQ
z^!-`w@B{<Sj26G$G5pf9zzyh}TCxc69#XN=lNK%fm%lD7S1v<r<YmIvH_;D04%$mk
zj{Ns$8wxke5e=_hhiF%+$0g&EPcbT0IZXSYTY{-%B{NY<;xS98z1coei{`fDCD&$(
z1gK>5c+Yh3w4M!lIk-G~$x;q=$28I)6DtjgK8pGh<!y|-Q9z`ao@ixe3yAvO-cq|$
z>23LnUho<^_c9P97Fy({gA2^$WOu-L-Ou;8slIC5O)b6cUk|xT$>aYc!giB?V_scM
z+JLMEc$g1~U+PyImjF*HStC@}k&a-Ujn2D1WVXa!BEc#!6y8P&Gxhpt&{~^ft=Vig
z!5(GoZu#Z3NEQ|zA6NbVdfU!9Ax)=zb4`e_IW0Bd)GK8!gBHqK(|!U}1{M`Pg`Q;W
z{SiRqQ9(;0EGbp;UUGL+N%Fv&Vb+c6?T28a8ND{qGfWW-sZssNssXOC>}^!SQ_M6O
zxOj#Ysa)&o!%%vt@D^+OieCQ}*!FCT<q^8-;N+Df*RE>HU+vFnUG?ZpT{I9JS5rc+
zeT&u=CEcU=VoxQ3UFYUJD-Kg?pf)X)Z_&ZPLGgCr)-mfwMuZ6W_E^)3bJSZOyW~9x
zVKld*XZz>sDJ`Q}tl_)=oRhkthtpkT5Kj^b54xp_%am$R?(YO0_WTc7B#AoA>7Ik6
zv;hG}o3Z>y^2CO>6lzxMpBf(95`SuV#7*%}g(-MELd!O}6{uAPZ|Cv$#e~d6Phrk`
zu9&iBn!`s<J;kb;FW>NLO~kc3^t97|8IWe1N&D$!dzgh=<<6DM=W7vU$~ne|2?Io+
zl~J!9W(@17v40Dn{G_c136UL|ziUbui~*&Ffv})#!nL)%tBS$TaZ*+o$?5@jd(>nt
zgcCiutSBNW;xUYJ#nnF<Oox)26ciK$afC)R4UVf)gn2IP+Nb30sTmm#tF--tsu2aP
zIEaUYz>{D+-rw!A)zFvi^>|P9uU1a0`%Sf~6CFk{10$m%)(<RFDVL}Qy<fnNd@ab@
z&Z-Tw8O-M#L^vV+nwX?(=E<v+T`Q@r2R#>l^`UX@ceG(89->+y65SL!YLNS5nzY^N
z;eduW%*w<>C?jX;Sh-nVS3Rl+Q#=Tc1DP!qiF%v=ay>bGaPNji!q4M$&}fCwsMRxb
zW=v6CPrn66D2HuMGVb)9mC;LTn9^ow#6F|5!dvDk&C>v%;w6k#{}8h>c?Ke*@GoeN
zoE9HfX`ip)w8{*IsfMR!zlce@&^2)EiKH?k5iraX+9aRV`Hxv39Oel`_?WDHMVC<R
zR16FnF}AT1dl<(V8K2*A^@4_9UFiRXdXA*QA7T?Cv2;)@QaWp(ugF_LvthpkCH|^X
zG}cm3??{Ij_qun=Lb5ppw#R}nV~l<%c+Ue>6>5K9pqMPGAi-+82$)=Cw*}S(8A<zH
z7i2cje(bv}pZQU5DJT>V2P+Z=hB$QrbCOvURpC$L!O6u`;bI!e32+$neH0vr*O5+V
z!qChvld`aMPL(AP0lp)%)jV=l&lRzW+YId-yL;oM%^A_%l^|or{NqV-cWvk|J0Um+
zmxR&V^*){~C<ctONK#NbN;c+$0MPxU%58mU=0Jgd>6J4*YZEHp-w`AQ<)|<O_So&i
zJ`NjD+*L&q(bP!!mEf{U`oK8Rz1JT58>Km9+HT-y7VCaP(XQF$C-GP0DBP=#^G41O
zNaJOYa$K$3%s#I9phK7OA3t#-g+SgeE2(Npz_7c^b%cacMvT8Aq1*Z*!l9pYK@0Ga
z%7Gf$15CA3+kWnn{WUY9+`!)GtcJY2+JB??K7oN64$->?b>mZAL$ija*sWz=Sx-yL
zk@d#wvqu#&*c4S3drene$egg3O1sTEEZM$Nq?Biyk9fr;G2?v1z4>*Ql>v|4WM%NY
z2U-O<s!rfa$SU@yObkjKlN(S+-wD*-4%2H#<lP5DI`q%}PA??V7P#PqTUH{Lw5UlO
z4q<EV<M%7#j@tGWa$~xBpy5!6Uz7Mo((6*(g}VYz`jFbffkK5$JvDHc?I-eHU0(Eg
zw5p?90kP_b!Jcf@&id{KJivi?ttpD3>b&eo43^s*oqe|p`!V!H{H8y+SMU%G4zI7g
zRjRo2KJ6HTC%S1-u_G^uYq<)XA-xt2q-jl?s?K-IG7s=g-+jze5^1sI21q81Vp+<D
z|Ai^Tt@7$dw~0m1BadYpA<u)8KZ5g+EpF((V`A!iq&7DucC&etw0!Rwz~yA`YOy*b
zS{$BnA3%(b<30VhMX`a%!lhL#Nv2XSnHp3#K^BPe3obTGeDjQpq3O`*bcDwEqNts=
z3i(wJex_m0yxc2t|5ZkeNRZYd{)YzMh1+U9mxUih4`{+qmA!U_<>2J2sdjkhmQhpa
z9M!kG2q)%{=&4!@cyyX{F|q-^A-s2K{^Fk{x8jFI?6c{Q=QfMR;~a~g<}3(Au7q!F
zhL4YW>9?K4_<C?m$?i3fF4N#;Rjm0={VLpYGGyI{JfFXz)}K4!S*+wz$Fbj3+B$_o
z@!kJMrun!x?IA(ysa7jd8Bkaxn;r;Rsguyc2>php1&))rISTxfFDWAK5qwBA!fr>C
zGjIpPV$vT-qEQV|L?_h%7S)At6~R6gz8JQxkFy=$MUB7P(y<;v&%9o8$SLAimq3K+
z*)coddk(%0HFg)UBb!L@9g4y^%*)g)d)`hf=@U_@X;!n7&|yFOn&~N+ZXuwMiTfkq
z&_nvf4HHWWS-h=^M9s818PY(ReqC%+K7bV$;qlUr%HjIR*T45?1WOE?Z>Q9yKPl9*
zhl%TR5J6XpEk59e<eNl*Rb)OkLyE{ZnE(rEC;kq^_wa?3^T8J)d=bgmB|f@|W>+<5
zQn<}z2m@s%Zd5MQ^x65EsL(Fk%YFwUm={wUyHN|hO&lD3TqaZ01+%ic>iLMtnAKvu
z_bTWMho6RX`w-?$4(6<`JH|W6c+J?8@#*IoHql8M(ITj)YYz|Gqh-jQPO%3Ljnn;b
zOzc%i%<pIxEKksgUbd`wb=9^Y?VYW0Z?S~hZSM$rmJSCvhR?MjtF-#)`&?I=N4+tZ
zFP*`3=H2r+_DR;;cC53iF(wS=cvSyjz{st^GgII|&&hJDx}CJe5I9S}HgqV6J)4}C
zjrK+pc0wPIwHH!T%EJAa&cd-JM7s5*PyUT$+nD}gajR9XphIRQxWRqOcUod4`$@m|
zz@p?U!0bsMj*f7Qv~iCoO>rq6ndnhEx_I92)hMyaG`r^3^hclYxmunb-mx2+MKTmp
zv;nICbh_XBX*pP!_rgv?OQ9GVQJE6rUyiZAv{VnERS>3gNDoMY>5&(zI>NXBbm0qY
zrAa`4R$Y_i%=sOp>ku{^*K1!?K5Ax-FTx@9xK4S56roJ{^x7AkR?$I#vRzMIwjO)Y
z+8x8WvU2GuOR#dJnvfOwjEc%3u}j)#Z`FaB!P;<t7Rl5tw05m)j_^zP^-+>?O;?HZ
z>;oQJF2t4eq`v4$szvEm3gFQ5TGxHo*NUQ1$;FX2rFJb$oo-V&UHAh+Up$CNCboR4
zBAD+_+K-B#(u%-uvqo`c1StUU$;cl{j5<!Ro?nA|Zxa-0Z`km63_*%<4tdXH3<m2N
zXT)bt^d<I(ZrYw6ghC9wThUxS{kAdFCvs3kwz|>|5HBY&FS9try@bt={IO(-LZ$hF
z^IcB*=D6R58Ju~2gDDz=md}O*@l3-96f(qql{|;Y7)Cr&kVoKND!f3~5uFO}RDjLE
zxb<mSg>WlWxhVFcUe$@2r-ZtYrKt)$(RA@Zy3JH(m)#0WE)eaYgyxYgOlnSNm>O>2
zn-l(=wJR~}mew{m8Yf2@M-B>33k7Cqn+*?Wk_&G)i|9l)T&%X%-pwv-dJ4Xse|xv1
zqlQh4gGii)PK1;pL&y(DHI@m(qp|7}Z|R$LuDOM=y)$A?pu)ujz$r$METLbcry6uB
z>D9FTZ2Q4357*=v-v+f3)Ibv9k*_6JHl|wqrC>lIq7Zxd?A`6$YXb-Q^YG!rFQfBR
zr-_TK8}5g+r;YD7sE>Y?xFb>#8NOnsnNdMf?D@8UhdB2XKzUY+n+aRc^<hc$38Y4f
zdYua=ajhIb_?1@U5w}UH+&yoh=>arM90*KY4^|e4_$AgiaBXkYG<g3HIJsY+Z!P4q
zE8Y*Y<Z^o0XumhGDl2`zpi)4I<Mn{ap&e7_QGi>>b}_I2spz&0R_=TPif&CWX6VTQ
zE1PvV7s9D)5@X7--!T)~2~mP;$$ve>TlYib<9TjFs81=99XoDKs<=za4_7fz!$*h?
zsFzFa14+^V?^eI?fm3dyV#%~SKq)1@BgZJF1e^;~dh?OXdfdC%3drtru|5+H>4|D>
ztk|5DsFogglG1YH0myaYnoC{29v~rAx@y+MPR$`~cS{l)RP!lLJg`G&@Z!r@e>K>u
z#cH^bHsnZJeCUe+b78v?!G4$JJ*8VR>-6^MX=@J^4X(pYki_Z>*C|_jNHyYaH79yy
zCZ!JwbIFzJ<@_oBuc(m+Vf~0Q`{NO4^AlWQ%N6f5hMO_7^&E1y&dXJRYf)Q^I?voo
z&Km1ZOb~H|uKt>dDYlaG%eP;swHL=+S06XY7*(`yt5i0UO>E})CQvch+H&{u;>|i~
zp1n`85o+t4Z%dI|g#WDtpzw-+BKm!`-fPRk1Et8xjnGrkDl|iuz#-+}l5VI`u(azs
zZ`EYQUtZs!?Rz0~lgFaOTK3kRP-iC3)NK}}U*m$U_%!NdkGY%k)NcO!bF}MwzIeFs
z?mB073!*#--i*d}@%36G>=!oyLTNt;DL;rBLa@)>o*8{9z-<vFxz%w}(|@%)%w%pz
zpZs|_ojOPnoyMVtJ>VBTY9!%gT(SO1{WqC?2%YLBI<U694xH@Vh{wbve*ST4twp~s
z066xcH=N!@#F8+Wt#aQ(6DlLe>$4^SHg-;kILjg`**JAld+#~`aF~RV0C(aghQGJl
zdv)&|oetehY@OSVO;?Fak(<Oi6-lYx)gjkDHuHuFvv@-L61eGX?sA%<o(I>|<*cxk
zH_zyjMlFTc?AdIn*IpT2&E5cAHQ8qUqekp9Z!W(?9djQSS4effab;?`X>BkpsvSDE
z^H^(x+U0PQF7mBv>YH9Uiy5~HwlzZ)=DR5^N0nYkpP7s}+p1DaxAJOA#dqcEfi4x@
zaIb<4ALVD84N=)*9Dc3$K#5z#+n+MKlcFUnqzs($H}HxrUB~L<B8lG&;>G7+;r6I~
z6#=zu^)>?y7g(fvIq6|-kUO2pC=(QuhGyT@_ksOgy#vSE6()KKgxTtLyv)9&e685N
zE^kxauQr~@b}ZS{&TuHrf`svzY~Yv>lq!JhWH)^tm~`0Lv9s!H>Q-8zrE5dFJr~m!
z?rNQC<PSBAbkkLFO5N+6I(hEK%G~QD<~xYl<_`K?c2Ua={`vbGnzO5OUjN<VnN9|;
z)@NTMyo3Z6d&#Jm=8Q^%QVexVm%HTXmG8pe-xa;ldpIvL!vWdZgsTMW;5nds=d?Fc
zVB|tz<n}J8A07n<=a;2gA|Ps9z%0EAeNaMW24Vz@Jif|UGS;mSO4|sd@`WYg-H5tc
zkFMHgowr-aeU7YnRZe1KGjQ<=blpr#i5<nbLBVAHeDBx1oo)#yqj4grWhgb2F}_o-
z@@_e<AW<Q#kx`9>wQ6%?LUGW%M4r*qwh_Rh=_%R(!}Qw2Cfr<LSg{86txZ|Om!nYb
zZH<K63L(eC5{A@tx_YJx@k(l5mO#CVJ|j6f;=YA&uEqyv@<4vgws5ZUxU4c-&l4{(
z5#bPxVGl2=LC(d@CY{n}-n$ve_D6yigLKW@)uU5`eTufmPgiwjhR-{Vy9PG2!E#o~
zkkJt?ljdVa-|CJTJU|VGB6+d;f9Xn1-GrZ9OgI+b>YB{B?WN-+nGHo5VMRSwHESIh
z?bX=T6*f8d<xP%9CwqHQq<anxG<Me5A_y>6iQgTA@z;hm$WQLa@UD<N$FQMvVcr&%
z7{=WDE^<quVXVz2hP-s|0|FiwC|I_%eU!Q|uPC$YCOm%)92rMLX^X$=q6rK?T+j~0
za@O`f&(ACvq2!{@)gAppTGi_>C;K?<R5qiCf8hzh$c<P<8o}QL19Tc7gTD45ISz9y
zI>4`dv1oBsOR@OOryhhE#hqFAk!HyI$jNytmIVXkYPpT%GIhKhMQgq@JpI_Thx%=*
z_IsFRO--Ekq=4`~>ZFxSk=YV7jN)$zvSmqeNs)f=uvppbV2uS6=a<?uHHEB@!xx^;
zxp~^f{iCr^GRENf_9!fRK6shDQJyi~NiB2Lb6+m)6dL#OWS&NqeLiTL2DKDvHfySE
z$q_%bw+lr%EQvpoia>$@TcE~=LXw!42iIRa8?|;!;s?vm(F)alU7kI9(vREjkTPAT
zM#Q9+RW^#4n(L)ei4wJe@X|z78^#BY*+C7PO%0PH$XgZ%3zG+f5;?*z4q1q?aq2Bp
zSMWKSIH^>#%elr~)p0E`58k}1PS|`K{zKo2my+9Y_D!W%Xm2c|yRJst_}H{y|7y)<
zIX}`zopbLNKn7Wz@4L^_HV#a`b_n}R08V_DKS6`})LOffx7|(FZ^=q2EhJ`NV@;6K
z3?px;?h1uU4Ep=}zzjM8gXjQV=Z@Nz1%Zl+$|lVF^_Kcgx7}z?D9+(c?VC&~MBV5x
z@Yo4~9msB_K>pA&xK#+{`pi2f`R6nWfnf!L@b_W`lG*?jd6XY6E<Yb%)i_EHhYfCD
ze?JN)CRFKcc{Qo`@f^*oIA0GlGjr;f54TuRSe3H>yx`BFzC+IY(Czu0z(C^fMEa5T
zMrYngQ|7e7srfNkhfUTdun2QXCDJ0>Mz+X_HZJ_Hx&Y4W_q7E}EQInE(iyDs&oF<#
z?mw5k#|L|b%f0O}m4E*D?<0SH)_{N=)P;^<DCuhW1K$!sZqRyf^d1IUPUEnGDUu_@
zb9Q*PjI_prMdL6%HRJ85vrjAwd*|@X_GQf-4|Uh*_4=v!23z$<KQkvg<n%RD>a0Le
z>!t_Mf+)w=nDVs5uSA#(a`<jYnnbKvdvg}81!nKm=bzHpzT~Qw8h_xwK-^B@RYfVK
zJ%Lexzq=PQa@N2F@}n7SBCh23rG2-l`WHaC>|z2P#_m<$TC!sd@p)d&mY*$nz6=do
zMgdeGKQG&8-~$@1(M{Vxos0l4lbprtE3v@ZjUkzD`;6qc!?w&OL<x`ilOOx<q8|76
zxc}@Mpuq(K^9dFVAJLnpY=#o%lQD!Ro};q$6Q%yTdYJiU3-3Ltn*~gqYH&&4)1l2s
zqBF-P`g1!80v<=TsOP65SAJU_Y^C{`prAl`cW0NbpE1HQma(5<t|-4XP!KUc&r<-L
zZx;+L92)1$u=Y>p?~@J=ZQ{5h*<RGqCmtwvgaWNGjB%J@D!+=s<;ra*QXECR&Unb_
zErl0i(WPP<pKG5YxhxoZh&L)~OH{2t*J`11YBYf%o>qB+&7TPXsMp5()(?$)y^oK6
z0(%!!Q?6c^LVt|vSR0<AUpKSoCp-Vz8;jVKBIuu!gg`wh51#{+h)gBb(paNG|JfIs
zApWwyyxPEYhG>7U1_#)-syHNWK_31$A43_gFBN#IcWIa9tiMR%R$cjm(^yOG_iv@~
zH#Va?8g$gq@q30e$F!(nUFCRMr)g%9aY(#k-jpEfNWj%y6pg$0_3*{*W+SO<S`AdK
zplBO+*SHy%9ae6s>uvB?zo;8Ms#Sl|Y;1OV2o(km&#E}cO!1S)t2B^OYGKsJK0gy$
zwR1DpbmC1?>oSuYeoUee@hC*?x#C`WeHAoV80mDc$pf1oTxI@laS9i_eO{dbqA<Q1
zcDLD%M)bWMqq360y*D8QS2ZEl5Wx1e!jpoe`Eb>r(2E1}`5T(OPA5BPmC8g^Mp;4!
zg8?i)wDn=MONZU3o-3o34nNC}bpP&=9#g6w?%q@aZa8v(t8hO?YX$cVdb5yBdr>iL
zQc^M$9D=}s;N`OA773N5y6Wo9WESFVnaVeEqAbaeWl4IY>yI5~?lurAjzp3Qj(q!L
z*phPGRds5~q@1j1>DQ^5nn5KShShv06S!6Mc4CsM(U_^}>5&B$luKWm7L-aM#XZ_R
zH<H!epXAjwDUOGc7|VHmzz&p~>|VI5hLhb~C^QzF4d$p^?2KuyWikgIXnUFchOS8!
z0Wv49edmb_56-`~pxx~Z&|xfobhifP@FJcL<qcGM*C0zX$A`tp4maX84zzEC_z=V@
zb}N4~wNSOyt@_-8JdF}BrE`tGAjRxm?RD$<Ty!r;Fu>$%b=P^<YsLF*G1p5m)Js?S
z*eE+)H)u%V2j@y`Me_Rx_Bey0v(S;BbdukpzMy+$9@otz3Lx!8uQSFdcyN<Cw4V!n
z4iAa(hq30KvMCqN;RTGUu(=?4&~i-a1SemAIz2NN%i$A6=$ZjPm|`Ln<s<ncpMQ&J
z8qkQXxE-@?a{FdvC8JAK{FJUx8;;Bq#R(*W0p;*?En1m?r5D45`li5(yUqdGk6s+`
ztKy^~<06(7zm6(~+{B_dT%7wY@lx8W>Zz-i1NsK*QE{*(rk?>|Y)!slkR|F`jiW*7
zEdp~`?+_X&9fi54!v?81$#>Z!+Rac;+O7GuU4%@pA`poXFB?!ejB9=ZMZ~`2xF+K(
z(2d1Zb#cs#xrjs|-b>@6$4ra4o|_+}o)g{O>Y!g@0T|4S**ylCkS}b8RLTl;8DsqR
zW9bY|F`-J0P00H-f~0gzL?uz_hZZ$M5g7^=4@=0YdO;Z(j!CkayTcMM>)elTgzYiI
z^h8KDk9V9ux=_Blb18kv?RB7t!Wt7#8kWIEJx@`w4&*|uVqy8EjiCBD*;H&&wziy(
zp9}i(W?z{6AT5sGZB0u8I^$Byfx$D#Mc;>ThV;dtCa(L0H+tCt=K1v`9;a=$8-48~
z0ye6ljtW!GX%Kytb|WlKEBBNsWB<KAv~{*%$}}iyYY(}@Q=8te?&1bydEZAzCEaxw
z#k{I~gqT&9xLmtFn;FhK)1T9Ing?5a_b^a<?bPLL{a>Vp@cT;-=9tb){;^<z+6h?)
zD1Zrc-srO-YEZ#Qbq1ZZ9@E@m0wLJJ+;@->182JWhr39*(z>$bm(<e&PR&}kZ=U7L
z?5rfcO*?-!q2N40y7wiW3o>aB#^_AXLl;#qM<rP6k#Dl!fut-F_SCD&6rHemtFQkY
zk&lXlN7_+4P-%+FQcIt%Und<_vwmB2BriWa4gFa9%3F4xa04Tf`JpCH3@oV}mk)6F
zofCAt+5JQhZ@(0;c(gNrtk(095&o=d=Co@0nbl;t+_yzu3tcm!FXZgeTbM`1gw1nm
zi|-62Rd{#ri`HE9^>W~35}Z!6yokw~9?R5f)n|w&wjl`J4=2Xa#Ts2bd{njroDEJz
z-;b|STp%4yYwEkqQj;tPLP9SjbQ^an`=~d{DzOxuFvs>uy<_TW;1E1wd#`JKdtw+i
zhnu{S3fXTglIUkc^&fZz(BiXWR~!>H0H@Gq#t$<r^)9%Q0&7f?;DpTljBBJttNZz1
z)@8$Dx)ItbKwC_-9vjCwuxA!__O2;17-m~%l-@=xwOY;xc6Ez<Kam*AXY)yh&oJ$k
zrFb2zG1-dpCHzsCKrImz1R`!PRoAG5+P3n6T2P;sV*z7}GS*Jz&4arg31eqI(>aZK
zoPF4}ioety$83n=xP;UwURNz}trQ=-fv~50^K9sZRBaWHcFLAv(261wfMpONC9+q8
z34iJh9qqDuwE^8Z%~^&y8}UHH1dU`4wxq6$lG4#K*OePmWp@Im6^YmnAf2e_-1z_4
zd&{6Wv#x6xcS3N700Dx#dvFLI+%32VcXt{HuEE{i-QC^Y-R<j4X6BxG?)u*U?^{n*
z|2WlGbsxRX*?X<E_xVox&Lb$^xytZ>Z6kX~5C`KHv)cTVK(eV8Ka9evwIB_(1<xmm
zvEEkAl4xRhq#!4*<%f#6MfsbLG246$`cTr>!G-bVPhR*G6ug!iC#TzdDSZHOJrXz3
zC;-8uFKfKhrP9~I58<}B)_9`d_2Q~?36`{%0+&0>w(2lgkety^6YcPmmJh6<`k0xX
zE<R64q(oj}PtNjUgCTY?MD^!w^TDR4-)@UB&aB5mwm5Z7dJ$ZhI^N`Wthn5p_L6wz
zVAMbYe$FO@6$>9Y8NXDCO$JPGJ)3j9yEW_Tv?FMkd!gmmg8SGJgGraR<Kb3(lToMv
z)k+Sg;tv#CG<r<Ht^bCOexXOCL_q6LynON#tcDqc5FXZ)wX$A{GXE!ttgIw};uH%i
zQN`Q4l1$C8%JOJh?!F3g)|6Vy=XF*1Q>ucPnNSks<b=!@nkdD9WQq99R(EKqG_7M2
z6~zwL>75+`Ja_usZ#om3u};PpOZk@L!eM0V2D)Vjs55c_tB(N`l{8ZepBH%?iDzpz
zYtkIhs$!=&K1CWE6pNvc%tnBUQm#e~_U`dzmwb1hR+K$nZm3_-G~UGD+)4?7tjbz;
z96iFa*`gk}t2i?B8uGiUcVKjouA=3&aDCk9m$A8tW!|ed9e1ynI^&LyY8OPP6qc<`
zbnHl?TKvlFCUO9N*^GD7m7yDVhgwHA>R&4F-OY8{{<Z7m)&6_pnFaOd3e~WI^X=l|
z9+hMVq(U?MF<T)GR)?Bp|It)@<FjRdR>_vzc*RCU;ZUpdqII;nH$)ldn;L>L#eJr*
zM&WYZTI)H%`XeWb#~#BZ=s$gi58jgeK+gk(KSvw|nANmjW0pxDhamscHH5<R<if;t
z=i3Z7z}W<LHR0#2!M)`??03Z5jjB<A5JaBSROrWK9#JxkJE415Z&LrPJ3bcBlq$Je
zYFio^a|;*Eu_JA4Lf6$P6VvKDT&oVUC2nz8wflTYM5Wn4xhBhS4maAT4^M#=er{V^
zr*9YtHHJ<mM{NVLy5JI+2LydE7X6oU1~hb^8<!5uj)Z1pO6n)92=%nhesHDRH{mEP
z`>&m10xHdta|K(@27sob7|hq2iJ)Y1l=2gIJcp#n{1-w4EUOkGE6h!+q;ue<V$Pdu
zi8}PknIB{xs(w_Q;xb$+ZI!1$`)!jt71Ga50scTC$GrA<{EQYXxK^!@K~ri#n?fe-
zNp7XhqdK4!3<*tRt&Y68<W%bs4jP84g^LRfEN_~8(?Kkek1Nq_d}%F_JifnQB`rSL
zpIjZ?-k62q*L&{FImT@BaT55;ld+bAb$iHE%VPL72MLj0@;}Zt;!C(D78gTuMQUu%
zvhvTcCwBEb$SGvMLo#46Aw5(^%cnP}C>>e|Ilh5FLAgwPRrX`tIh1-*hp@EbGHj$x
z=e?JLek@V?K14YVc|)Am4edORQ-80ZlNhC9FgU);T}D-B{BT!0+wal0s$1vHLDo1M
za3$fx?_9hS+zJnnTyilety24PsFn4a#%MWEXSjX~L#Y&#ndJzg1O#Q9Wdl4MgT1%`
z%84(VJR`#zu&P{e$TskiWd3aN+<~btO4p9R0H|_tApTUyW#C;(S)+D2C}MH5=Zr)X
zoBS8TrEj!yWsEvzp=O!X$5dQHIv;qUm7sBzfynrkL)b6J^ZhIU{;1W3y3Do$gGgO=
zC2UltMXKhc@@I`cT|nw)sG|L`xoF<ByN(rA2tmm>Fw*px&uc5)?E`#}V!!DGJ{B+?
z;L{RYg*w&vNmncbn5Fu}o$>LEUC!j)-I=id0h|TP;nzeXg5!ba`IpTxALYNDi1?6p
z=V!OZuZkI(C?CA(XhD(dI==c82~>#OfynQ0oCpe*p!{^9@6IE(+R?usjYdn)m_}U)
z4Jfe1J>L{jJy{52e6S1LE@pAsEg;qZt}C=zCz~|mEuorVQe*o$B%X<c3jKuft=8dY
z2*|%>UVgmUtjXu(42ceaDn2zG50R8+ZD*mG(|k_Ins}donagi?7$P%8(}jj?71Rz+
zrpjMfhK=2t$!N=~HQx+%q&$~dSuZMMDJFa{n`a<(9PHcn^1-BQv-}@GkT(N(a@xqu
z)7CUhvFYRWmlxsBK<`N|mhThw@TWNR8pWV7wNQJ>-lAYSby8cyjG_?MgjV~qxVaZ(
z{qbO|DS!LcQ2VCQ#ozE+P4>Z%oldE_h{X=*aN}H9n=2y{jp4Vi#ix^-KdLGtmlpy`
zA8om>uQ-c2pQ#Ufo{kF(5dztCyYwGneE8KHE;aByOTS|5T;|FQt$hmBX}C+hwFpIX
z1bx}GWCQ$X9@5+6Unw^(F^5q80D*|#W7+rnOiE-3|AJ~vaW{f}6ebA?{j<ZT{VEl9
zskwHjH_fqEL{hWLL%oCYtKxS?ZEXP3?0AiIswc^AHoo(<gnb9+FhZhGA$K20VMFEi
zu(84Tq}JLv#G>IFa~9%>ZjCw(0Q*(;#R`X;F`F9h2f&~~(~vrRy&;=JgxoKQtOc#J
z=|86S#>e6-wP}NhnST6bML+;gD?A7|tEj)e09qEmKN_hJ?Bw@{$I3uh^M8UBA7Fq^
zzz12U(Eoa}e)ti1{Mp}9sSE$LS^N!ty#Hmfy#Lhv-%o%TDiChre2Oh0{>#t!4G8^m
zI6iw3v%{dR{`Zr5PFPt8?bMe+rT=dI|FK`6e9A^W42%9}ue>09fT+(qF#o#$eo`t8
zrmJjJHBSkAwW^S@@bP;p^!v`^+nym0pNA8f5*Q8Ph>3}<SM29DdU36$N5{SeUi`fx
zeihDziO+M|B<_h76YNPrQR=P11O)}L2nc#_=ClC^Q+Z~3Y~0*<KndwA#Kpi5@{0wd
zbRse`sEUdTOsx+zf6vOVLOVKeNbcbdV3d3zXrVz#ZISTUK}em#MMzqocP@;Cp6@*0
z*vwgz$-+=kP^!7(vm&T{|20vxq`>Ku6pybJUx)hxs}m9yHW5rES7%`{c)Y*ZO5k!l
z^;1o`C2D>deJUrY+_3x7ld}C(Zmdg8Og3x}8YRu;;`(lVw`sS_Da{{2{Z8-9)w7*w
z2_|>dpc@|^L(fBF81k!WX0f!2t1G=;LTF6=elSWzxZV7?CzjY3=1p@um19;a-q)*7
z#0XXIF}V_LMRZU2{E|^ok)2t$^vxW!9FJ$UjtLn)u#k?n@u;PrLR)emiKt1TZ6SH7
zw?)+N4^1b0fcQ#?hi}tq-DQrYPwbm5TbEM41b_y>13q7~r*2d)>F{v+lpjD1M`NlY
z3nU#>t4~}t$ymE@riw8_#hvkc-kn)*KDKo{f6P*PJ#W`~-k7o;mXh+OV+F!tghBXz
zPs&Vgy)^fx^2K{iC65ZEO?DaEcTz4LF9(dd3micU@@)wWCo^rzS!`>h7Cd8~Fm6Y~
zc7IqUzpmd9K4+O2G5-h)VhU4@v-`~V+V{q8gy;D<m0qnowS^d7d&jmhF>^!P1jP}t
zzxNxiiWgGi`~&W4RlT8v+k1MsP7``cnO`b@M;gaVQzB5vy;*2`SU8eDES2OJZYQ<L
zFSskOVp~{VWc|uvKp<>(K$3ql&{5O`_aYpS9;>S&0Y5&zWg66tEw8+hGq?<U2~s<X
zv#nU2QD}&{3w=n+_k?+tSx1D~6K54+0Vt9o3QHOdj%u>B>`nHw33=Kz<X#+CiGi+q
zRXNxp8@d6+bzps3%`Qp2e1?J#uvuJN`x-HGqsgnfS>bTRr!TI---qZ(a#z9lfRU?v
zWVRD@MX|J~7U8o<+r4Me8E4qpZz6oq1-Xx0gq`y(hI-b&g0g#J$t$M^BmL4%8WDTm
zoK`?S8$8@uQnwa#;2~K~|MKBf!`icP(5_-I`pjkG+4?_7+SeNHv9nkibtj2Gb{gvu
zWGqkWZI(bEL^AK9crUg4!~Kb@`kgrSpK+k<b9}KfJIW5dP|zMXg2?TcS@na>;7txJ
z18eWh)nY*r)XEEHI9Z2!>X}IKn0iL1LFi@-zHnKbB!NT#j~Hd`CI8%9&H0&1MCSs2
znHQvrIbLk4y|izI`|w*IsgA2}G7Bm|cc@}P;wm?@!0>H7(DyBzL{QtEaSz8Uw-mpY
zNOXzG@OZ2qolE9vhh;h%p&kw@n}Vr?$ry75dBZoN^C}7@6N$NMxO3V?hxIDy!6D#m
zRWQVFNjt7^N`dSZuY|?LDQ6>XbB!DI!na{zsXtdY<S$jjCbL=1z8==j>PjW09bZ7B
z_5u~D>MWq<!z%t3Wwq_i&NR^sL5G{3ebzdM3I6$1BhgKo^W`;5xMpX)P{{4{N$1Td
z?I_1QWhP)MsdehBXyON`oq491*f~q*ixK#fxfsXB&gjW=yZ!1}4$tSYS~MPu2s|?B
z+`e3aDO%u|%jvWVQz+#oz!ZgKLIdFx)_CO^cL{ZPC$8u_g?@G4)BfC=+v_1d`Sb~Q
z!JyQ;z~mE+m;#gUJYGFO{GJtvJ!FIsOc)Bst)sM!AY`!tcRY@Piv%XYg0IkOMM{~~
zxdNzKiHV8*!^48kf`MazQ@LY{!mhz)jo=^m8o^e_0klo{#*{Rl70y5$X3@pnkYj4O
zHq;X*Kp!^RP#T<Bu&~g}@?n}|cS?9~TY_cqIkGj~URqsSy7)-+FH(J2g4oE`s}($S
zLR$)Bo#pp%Vj2%)l9eGO?;__SyWk`hDQs(Q4I~x{BN{(q8<%tr7Lxsekae?NLs6QM
zH{uROk)ISFT~{1b5@0ys5Vq7I1r706!f?Iyx)l9P<$yj9F?`?KJfadu+m%F;$-q|!
zk<4Z()LSwIhH@02j=$dn@{LG}iGVqHZDeCrDFzP7aqTOMgUh^{<1EaP$am(M{foqg
zPtmnY&73`Bv-g;fxQH@s((TF%){D29B9F^2+%q+;!7?vlT;mqVGLEG#tJ$7PahU^k
zZ`qJreKNa=H#EZ$t=}GqvrC0HqCX0W^okL3hr1m#wcbIyqaN<*Mu~k97m$~azKzG?
zg~^eLT_2Xh;i}aAsp)Kh==6A?(Jdm{nNSVk|1dx-verxZ5>sTkTJMCeG$4tuZih_`
zeyF}JbOdi$9vog1yJi<YiX|MFqpDF~RPO8>wA%soJgeM&m`l&K3*|RZco7v96%-UE
zGMVYD_T}d>bI@5s<Fz3K??Kw@_XP!Lbw^(}?ZJt&(k7>}nY-d|c~3cssHvBlV+w1!
ze<H5wr$5fN`8tUcL>E1RxYM6&U$wIYUc+tp((OxLcylgd{ygrUfd<0U)jB$?Ml8fM
zs~8mam1`-14lwTnP{KjxH}bjgd0s@BrL{^vpu*t~NHTr-FGx}?PJdS5x>|L7*_GsS
zuGm|=SxYlcf?F=nwJ<=4Ic|Cg>GEHwoI^bM)x9j?VJN)Z$}+UREpcvBpi_Wo|CuXt
z@fwYg-I!eQysTF@lc!&f*?ZrX%JR7cIPfS~$IvrWV_Mg*LY5?)ceh>N;+?B)7UuP6
z`#xK2ZiuT0uiRDF=!z975Lh~1XJnXjP%qLsV&^`)V|y&W5@94L>5fpMep~#C?+I2E
z#J5)=LqD$th?Zb74=@;J5aS_b*WjHSZ6ufbcKUf3nA;<vlxY&Bo=<N#O9;ZrSsWIw
zDyP=6Vm;!Hhg0TrOInaHov8tPDeZk)SI5y_1QQ!8*n$5}J_TLh9Xu1M5q;t&P5XpZ
zhkj4ZK6VPIf(LJ<rQ^>P4e#c{cLzq!+UElv2o8yIs%yb1ie{1N_PCGPCjvY_lT=C5
zd3$tKGc;&gbKeFYGJM%785--863Se}<>HimSP!m>pIKC59ifxViZZ_EtEbq{@pxAT
zu(DeY@q4tw)*;^h)OsxI*g51=e_lvaEulyQd@|^Vj|!(cy;=6wC*;iL?e$Y#brl#p
zHl{9_<*t^gZ>94d*sOd*l3?QoG(75moXEOAcV{`}#F&G0A4B$DzENDXTa~>)Y2<p9
z!j+3mdE<`jZfxc5axK^VlFf6C_;DpZBWLBcakwCjo}J-*w@EDoH{A5lmntZ7wVFa^
zy2Pn<C5vygo}`Z;%L-+{{HU|+?uahj{KYCuba%I|<u&bfbI2LS;!;!exg@AtyqU_q
z?rluF^SB<OA&#CA+;KVH2RHj9iM%lnJI>ukrAF!XB*El|kuHlv+LL-h1^N#DZ$qLe
z+Uv<^TXBJ%6=;R?lm850C_>Mm`<VaS5hy77;c3C*zS#ryaA{Fad*PI_Hz@2Gj9XV?
zZa4d|ZAziyVMpR)0paBWQisYby=gMdzIKJI(5lg;FB@Ua4w*$q>Yo^;c)o#o{i~p|
zN3Iq98~o%9#L$G6=CNBdGz(ub|C1P_M4h6ZO)=xEee-n)_GtP3I`HO}y)FMn`RD3K
za$_r|&~X+?0@&WK#V(iP+qBmq@`551!=#}#`^V5{or5_&#>USqtvBXijWJ#-RoJ8p
zE{{#Q1nrlL;Oq{ARNaiT)8cNK+wl!QF0nc6u~Nf@^Nk6F0^;V3-sLC_&Liy=1PdST
zXYlp6tJrKO;&vCs6A@HCZ9{%$Dpcl!6V-&bMFrmtdqlI5D42)s`s84V-o$wCpdg7T
zA44(qCS*Ed`?dN7!fKBWqb!q_OL?Y++K}S)F9aH|uQHp^L~ujjLSW0NuqkU6_F;4G
zEdz0jMX|-t%|hCVmEiu+-l_PDR(AgSeh;f$cR+QGQ-fRZ?U%&X*nmq>rLIbQa)P(L
z?BK0BgDvy1;#<6<oB<=O&0RW?l&8g2Kw+LlwTJ4{xbljlFF_zzor#_46{|bf{S{>c
zahUkI^c+4qok@Y@bSeL)(;GP;4Ny6WKwV}oqMC!nu&dKDma_7COLnnB))hl^dPq+~
zFvj?^vl4vvY~(Ik)SNbsqs^OzZCBU5*0kj4Wcb`>SVA?R#^l@AIZOV%&rFk9U`)<s
zBq&pn-cZdKvxElHo(Qkn0NUb-I_SWZh9Iu%wAr?zN%ST_b(lilYKS4}KLBloHGJsp
zquiPib6US@hy*^=Cc>ij43fT`SCz6xiO%1CowWrj)|qGO94mTl8bm20n9*oTM||sb
z4!mjXVA(MQB63APO`*W4!aD?2B%QT0hpn3vjB2(Z_L{iOnrPRI?<qukON}72VSXDO
zqP7Mw#!N0q96JBOY18iReirq4!W6{u74c)Xmr8s(Ff1q@FPkK=h1Idg4%o*!*()c&
z+-BmmVUVb$?wz{TRYAJWx=VFXNor!2*&t*~koyiuqo5k<tHMn5*&#+7<g~yF?-%)R
zoIA&MtW2?!1$=D^qTEu_x!8PDV8#p&P;8Z^339|i{>p79?{QodnF5Hz`LRo(9%oN}
zDFnmZ=FIPB5<=jqm2PCMkzyZ^(q|7^ya3b1v#`Lg4$k|Wr72WyYW9v`9HPAFJ@U|&
zq*W|0I_?pi^^aXiCFgBVQS!Tt&)lPGL1IafXZ;lbQdLR}_Z2EzJ$WYKNft9Z1AOQw
z7SN=t{+2YwFd|kd=Zj09pM=oc7xFUmP8Wn+Rb!9CWZ_q~9g7QIn{c7E+!`J;RGZE>
z^;S2v5AKT-l{Hcz$>(<ztF`x}7Y$gkMC){=R02&>Y1Ju+`OF2F)LVh`Ju<p8w9Xte
z;Kc<h8KFWn5AQt+Xb_CrkLcL+0Lxs@r7j*#=W1j0aG1SLVZC{V8)?aTCPl4V)9+V}
zk5K4Nb@A{aGu@MNU+q;YrA2(d$Nt=WAUnRFNL@Tao>|ftq-L~3Ft+l*yv=NIsv8Ml
z#x^{+5d`-jCs;ms;v|)XYDFf86ysR6mcVRDe+s*Z1Pco#xUQu+bg^gwu_E}(g!JO`
zrhC%91nz_DrXPRqUKhH1x(BkQA2kdTnoCzq&C-Ojq3h-;dK3(D_o*OjNZMY>O-*Nx
zhzmWvOUx=#2uIL3@2@!@?H`!hM?c%whwRC(IqtfF4XtnpH~FG0g!A08un0K1XWq{4
zG3%;0Y7{P!BAyq@E~c(Cw!<XJ)Q@E5!zaTgSiT;FiYjk!B77CE%3QdkLt#RYSd>Cv
zt46f1q_or(Cv_M=+v36~>t1|jPM;*L)2$tDM?erU*fHpT5r|PfxfW}0|GH<3pJ*MZ
zbT%CZskMP^hh&@0P|NdTslC#Wkm9Bm4lp%C%a!4b&&C|v)eN?&JW<F5@~dr0>m*5J
zyG4#eH><iAY74sctBQdXrOM5DZq?BkISr0{zEd~S#?AL~@quocMh@n7h7lH01aLwi
zBr|@E%G$o-%_*pmc_;W55oVRcNbJcFN>|2&QAFz}wNAwg2EB2e-^<zw>mtRLos}!Q
zd$(p)wR#TsPa-E!ZPf|JS*x(WJ(xM}%a~Q|(D1D!l6AdY!Nbd`Bmg=yZOs+00w*Ha
z_$mQ3iXy6?&X^t#FE;5kR#z$ts=4e4H7NV)E-cTQE-9kfNWepL;9OVVK+cBf;RS(L
zlg&ryceyULW!0ROvPB#H37V{ndJ)efLDPu=0j2jup^GWJCvPMd8tR3YR9xOf)W?75
zWL@%j4JL32gBR?=Y_T&=Ad8#@4C|{$l0l%mx2Q~E9LYh>>aIamamyt!lqsVhk|j^u
zG*KfJ#u&qvuCsx>FWrw3ooT!^sr|bvoCmaByGNrVg2b*xuXlBvj(x(oHdzj(lGHmU
zt5thmTuKqFsshStAGYlATiyyr_(m4)DTiZ8``KtZl)DVY>fjWv1=sUV5w!y|J5ioJ
zIYh#~D{igP_R3Eyd^6s>=<+7Aiek*O1=Mqf-}Mtqtv2o1=RR6)hdVw$o{pcb&Y5Ft
z+NF*yl!s;FbCtF~g{6gCsl*%>I2_*|2<9PVoJ2Pjt`Qpmv`7!$d3=}UUFnD@Rf62H
zbn9Wac`mukmw6tJh%E+}8q1L9PEs|i%Pe3d3~o(6Z7h0k&3&!iHgU>b{fXg2@7tuw
z5x-WVaDZc3bV*L_2QFdvIx<Y|t^=O-)O(8`s*FPz5_}3p4MT_fhh+t@xH}C(YYxX3
z&gs_=CU10#!_5d%DIOl2_bM(s?n-73HKEGNtutz#5Je^io9>%Kbs3e-^1M}%Xe4ym
zm75a^PtB~;t_Qw$zJ^n0kgc+~e5Cq+Vrx1GKn#jO%*hv`loxDH(-A*{kk3OPO$Vr~
z#YWZJHQ{r58t$pYu_bAIf~Kecu=LbFSZc^fj%wlw&ypzzZ%UBwq~ltmbiAS(s8<CO
zy^S{<jy6#cqh`d#`BcK#U5_pdP%`ckhkinGYR<__e)c&%k>eZJuaKr0B&mW)cb3C2
zi*AuAnht5F!u53#`0?t|a7)tKdfv2IamxSJ4)S>C@bhrsvjxFs^ODu<pq!qm?4sfn
zWvDwM;?IIZ)#`y3pm?_4vT;|w8@es(c{0rs`3&;rko*l6uNvwi(gb5BMd!spw*yM5
zUR^@#+-Xo#yqCl6w60&d&1sa0N!hf>ph&m1f$@uLb_tkboU}?wi?K?bmf28dlU1x@
zBiSy4w0G$eko!OxTNSHv0>#bte3~#R_?&^GO=VY4c>VpfTWOTVVNms+jP|v*2r0cw
zT9w7^rz80w++j;k5c#tWvfg0>wN(u7AyC#UQ!?IMTEHm7b7_8IDY?a4bqaC1_1+^R
z0gLJU=7>+gNEDh=!*k`OOO3dV*Us>%J6B7?VEvs118W@(2h{b#*_%T15K61_+Lz<A
z<Lmj;eS&j0zG?_mHzzHIdA@cBXLX%n=WeUhvOyQ+ZxL=0KMP7c!6X~Y-Je>zwz7l$
z4iXX7-ok$Hx8Y+179%(-5)qgPiWJuB-ITzeMzQZ?N((-+ca121TcU+m<GR&juYad3
zj`i0{(Euw2)!*u6@&WGrlplW9vK?YxWD1#=6O9;|WhTKF*NVz7HH0Bf&?{PkYb}4G
zB2AVf^E{1Aseu~ft`XwypnbskL+a@^09G>|{d|u8huD+l<;hWIHzQ{_EM(^LNaM4u
z5=J!F(zv2_L>g)1$dctXp>>f^A(v3D8zye0!+@St?7gBAY4M@s^%u?j#wn+W3HS>I
zRh4~F^jPDfcs?0F<lITbei=-w`F&*f5d(cjPO~9y6ZtFFOat-a`?@y&Un1P+8GZXE
z#t-n`hB<?u)p_zYUl`NIek?-P>%<yOpmw5MRKwhr>ZywD8@0@DF4oPUo;ea@8?ElE
zDKFA>+C{y7#lMW`=dCd%sExo>tvrvbHICU@2Znat%pB~7htBE!*niw~fge;l(i@I6
zk8YjXBQ%Tt^s^)_1d<1PsFzK(xR?_!ik)HY$q{|CFL_Sq<xp*bO1V1^Lch?Q{6@EF
z)oM}@$a4^QN@K7xDGze%wD?uK9q?HQkf)Ho9*-4UPTU-@Ls{4nF8DT9esTT4J<*#Y
zrILVVu2K!|i5H0}{D(mHtv64^?a>^~b3Yza1SP4+Fs-hyGNoT*(^)DQ^<B<8J07PE
zh3hcEv*XF;M>B%fuDNIjf~b4MZLGtI<|6p?)vUG(?5z_(GqRm<Q|*Z};q5cTO2AoF
z$!T}CzVET3rpg-WQqhB_)eXWS6N2NnZv}`UQE9oZomn-pOEm0Bu^Z&$ExRO9t=ys_
z+RRzCmmuF(o)pSB##HNXh0-n1g2_2#elM<44lwO#XK&AzU9}OYAMGq+I>yE@<T7d9
z!^!q^dkr7hUAAF>nI0nM=jVf~!a3>PGz!LGgoH&&I<LC4ltkhls+t(tx~Qod3l(!`
zaMYusEHNvFijT**3UBvg3Hy*KtENDo3Mhi!ePOeq8KrQT*6q7MChtU;LMDCs=~75t
zB(ujOA2!^GuR=C-#(CVwKss0_<-#Csa%E#9B08{ILRqG;NH%-@n64m2E4?WfD2V%w
z+c4?);Lv2ar-CNjSCEC3SmKtrb-vek?&8=jRc(#Gb`a-^yWz_rs{y-b<g0?XPoK<X
zBBGYXDv);g&8pP<z-pvvGbq!8D1}3mqJv4apRTnLl9-OI>vSdcfo4on$(*Y~mJbpl
z2V*_J<8)W-T9@qOm|IrFk$>sVdUp1qu4V}~iA~$yvqM3q=wwxi70xcQex@S^s_`)>
zGI>>Cehjx#Z3hfAo1|kv4B3jG2?s^C84!K(%eB&{Qx(waGTOsswOp$&fmpO&b0q8w
zPakGFV(ldPTkVG#0I~<+soNNWPU;O$9vK-466gdM2q5V3q*5-%fngj?4^ck&>JoeQ
zxr+uqx}u$ZwY(p`#sl_JCNaW!z$C#@eVUYB`iC|In0CF<Z74{r{m0*(_ong!2^7=@
zO>L))Y~LaK86bj$gk)ytmnW4RV6E!S@rAow8at#|$(z^y5bc$iU>!@f#9p3YeP{(F
z?k|P$Zxdxt97w`&cQD|>x=q05J2#kCP?0*^a5NfK3s+6c8rFElseLh9)*aHm?yY+L
zClxeD4LteGYL6DI|D2A$sjm9lU+UnRxjl1+zeeJ3Hm(Zdb3@Pp3w$afxpz@)`~Eq|
z>rsg#gE4N$baGhGlLXlAhz#5^?Hvls8myTWk_dub6bj~!9f6zTUX`NWv}IRD770wU
zWqYwomiZ$Zci7^I?2#w{c0;bRW{Ul-dvI1pF!0^h$K0Vf#omjlaE3wLf3_J@!mBHT
zF6P4(kT^7kfKn@xRaq*XqVE^5z%2G>Hg~DPqu9$6UJ)U)`Tmp0j)Iy)5hxS$Y8)1w
zNU1<>lE9EAF7u4mlB5X1b#b#K)AxN46=Q=6QBzLjrQpefLe;Vuj30@8?#j0(pGEts
zKfdR%SSSS)%U(2eDg0xQyd{M{oMW!!PzE{xnblG~u-g{IBoNl@UEQh2Gh*qbWk)pd
z)TN5nPM^&ijB0ZRO<TbER}!5R9NO%xsHO(EJJFP<S?SRXB`J_Tqmmvw$UE@BwGMdy
z*nFp3%y}Y<^E(*PO$IxB3_P1bxFz}G<E&6Y8WqijfcxU6J{TPxm>ZzDwI`sYC&B2R
z$)m!AXZWVaxX~5bcI&xBy9a=fEb<@DlZXUJtTJ9xCqR^9fU!0|Hw)wZ@E(aBdVgf1
z1c5Sjb>X(kTg>XX<<;dEBc@aDN$yc;&(-*79zE%GKW(*naN{o);_$QP`H4N@5hl~6
zwqu#}0*b^|>6sAi8+k+V;>5%`v4`HlB;H7YDo6e6`V#*mRLu71Z43hYc?VtHR?GHh
zY@8CeimdsiYm{2nwhY7>C5Ndv-r#6McyJ*aohLD8FGMHX85FEoGATC@_${gla>1;b
z@;&}U9Fs3XAwgC=2w54Kh+>-lo}=k!tp_}uyx!f!`dcp9W;Y=nl{do1&edRjB0j2j
z-*&M7v0U4GehHv#*eBs&HJTvsM9;ku+*y+<-5|;_MJFq9j@5P>2~I^O4o&C73i`G6
z!QMG!WHey&JwLqA?~%7(*bpX}J$c<D6ssls0+J4QTv$-#wYyM|lTulzWEWV8C!AF-
zX2vSAR>4SBx6arPXvXbt2U>ZLBPPy}24qj7A8EVu63ctPlRPfgWvyrgoH*8e(5A3U
zf8TWfWZW<pEGE|!p^JO#$d)0{OWT+1<LuzhI9O0)f=ZpaBIy}}yL`lR!fH-U(<7xF
zd}&bKQUve9-w=)cPez!Q<rjS_m2fM*Zc0usY_*Pm!RDgI$vGY}8G_G0jPSAwd*M;~
z0!r}R0|Ixm;-W8BC0{De{XK~lbx(3>O5XHD0{B(jXXDYC2d0Xr3OGDYTMZF*&#)6>
zFGQfTlJiM(=wf_Dgj+-QXprZ2J+6)<Voa!KH>*~Ia5Uv*-TYWfjSO2FFycoqR0vrj
z;u>|QQek9jr?$BK!NlE?!$nSUEo;WAqL1kqwM@E8uiW}VZ%k@Q3m%Tjkf3J5F9%d_
z`9~1>zh3WHAYCckBd-6^`UC|Fary_-w-3a`U|D^<LTl?Vr|ix+Q4~YxYD0S0&$X~D
zXD1y^Cp^R<!s(3_!78k@F0z>Ihg#D&2?rs#Jkw<~WQKzYd(z?M^-obo`8~8_sq5t8
zxj4<0KQXOTaK^7*3BKRl(@LUqBN5A=Pf~q*ey~Pj#hmMGetfDo5=D&e7Q~}QI9|@3
zu{z1RgABcG+r#S}CFJuxWGx7k3y6ASCS-V}aQdf+z&aSlIvsc&UfehFbv2Ulo8-HV
z?<}0e47e=nrn0K4!Gn46F0$;8$>-;jJD-Y5`;dpdO)Of;yo8?~Av{@5BDm}>6XHbF
z?*_?l_IUTUQg2x?HY$AdrqymY9{qY$iURZisK+n(#)}I@HKd#(wayX)y-0;$A9=6U
z@VB;cdXq;O!_<}hTG?)?&oE9}K6&0VNP!q_TM30WK!{t$->}a*C;x+hU*ALqsvVrD
zO@bjxrNA~B+_?pBg$@gPhFWexpt|lCLcfi*fNPG}riF}fIb>93zh987v;iziQ^+p3
z*gDA&8?Dg}DS~QhLUXgF3by%&MFyCS*X*^v1>>L91@Xt~PB$f)%EnlAMnBoBE7Xvt
zz9<kr-^j>nulVVvFOkcL5mEXz^m-gAua0eaR0m6PJzNM2DEZxzfgc+?6=G?2>SR5z
z#3W!qrE?u8`|th#tP=|qG0#H0fywW77g_n0-*wteh2K`+eJycA*kvFAxSk)e-F2Ft
z2<tl&P-oIvXli1~hmy#k>790L$@l-v!D5k0-9NDuUbs|qA3dE8W*qcU7@S&Mh#2*t
zg1W8E6jRbys&puS^XhXwsik*aW!tHSKN`gCxiafnO-1}9OT{~MIA;=!Y<<l}%*2G6
zBL#j+ZH-rJdV6{wat1f!^1aO|1L$OBDyQ^kfpgXPS0R`<5<SaUWo2avzAY_yzv|Wz
zuIR`8>m`WyYFF(&(eyzvi3U0vha~S4eUg;fvQGU8GaE8cP7q7UaO5j$DN)-f3cAAs
zV6xp@@h2BUCNwm`c2Jdy`1i6;1nuG>%T8Uf*usfWOP!tV0TvLpW?G<yX#(!<{<Sl*
zZXy`bOH1nnj65TKq{lpf?lqpoz&J)GN+m}V(vL>st)r~pXX3Mr#8ELG5kw<YRlEwH
z+kR$ugl*EsW2zrETr}j>%#?|CUB{*Z@UKe6+#jo{mr0ot&st~t#uI8mj8YaSwXqIN
z`Gd94WeQUYoy9^Xl;E$Hw_yLZKa>9wqSW7iGl>HBQ;l@}`@F8Q>#7LWdsKW4G|p6M
zb{P)jj9VfU?cy#ExT%jPepWjof#cOL_CmBOA*2+ghS_+o6}G^boH}gvxQ06vg23h&
z4oXkPTvd$Y=;)Mp^D<Hb)k=XMoKv!_@sv-3qEDj9id7?r|0Vpf&iN(zk#`Na0SbFy
zKIn+*uV&E?EiV;6*VJ3pg2#opsFn_TA(k-{tpINyV!Qy($v<MHo?(E*-o`rQFENZ%
zyRPt5+Z$KX+P;qz*$vK*D%tWzkOP|zo>ArR>M9)nG$0~>34CB3Fn&u0{#GUd3-95;
zH1Ijmrd7%QF;>5=FMcs#ZGjM6@P8Z7e_L!YkierrR@Nx-pKr<UGSM$(70NO7UtHvW
zn*RX}$b}0Eq#FLM<^EgV^B+<7-&Oos@Rt46bV&O+@&DKue^&Y)1q`i&sAKf=FS*&j
zKc5AF;;H}b6mSgxcc%VY8~?X1zZ{gaoxgGc09XG1zF&X&hX0Rl8vOJAv4`!EH!#v&
z!9a~)cdt-{ndNsUm_QL}V(Mq8;o$HNNDOXEjxAy1owo;O%LE>1hN$?a>kCtRixoor
z+RYaaD(RGWeK&R}415eYg5ZJHi}a{Az57M)10Fr_dd0u=9oMUby`9v<C6AK*OG4U?
zF0n4@8Ed_2Z+9%bI*Bsh?M0LaC*^<D$1<}N8K@VU&75#<s=!NIw|x;qm=1`eA%lyy
zaXp+AQgKwu_}RE(wj3E47k}1)3Mh`-BPU39Fc*;UV|xt!zKey^{;E%oOyEJ%8i#h|
z-m&%I@kjXT!`BY&Y)eDUf9~O>@?e;|D5)qBSfE)#=fedXsKjvThY}oilw{Hz8HraB
z*D;=(F;1n}v~-}jT1%T4?NA@d$GWYBVwC)`7x!ROc<r3czB5R2ox4Cv__VIu@Ev?@
z!~DT!@$b(1zptl9cH<>YXHWlv$p%nQ($FZXW~Bw~Lw2Lj?A!g?;@A05Y6BvTOE56j
zRiVJP8Aq|T+qy}@O{W5;eg35h8?d*t@gtx8qb#)c#Y;cjKaVvHTnO4W7=pl~{H};C
za#2!N1l_4T5=zENuHx@0+}06#PV?`Cgh<F1x#DIyOaoVFYk?Q}@&IMhaOb2}`O`g?
zd=YYal&lC-JyX~eeO3N)PxN0Ev%(q9YeB}(xRA*#f<l6CL%EWr)|>+838N%T9Q=Cx
z6fdg<CPlIQ4h&+JKA8)4O&TB6iEk(3oL<2Z4+oh~F|Hj#rEqOTzT(@YNxFl5`Lb)9
zvM0u)y*HJYMx5g=87DQ)3wW_JTgzVw*w}is`xX=uwKuFl{3mGm_Zd6-dHtC3gc^#L
zd2x0rWRY+tQ#8Eb7G5NQ_3H;do>BWbP{|2-=4<Q<hb0;0w^ya&G7Dc@dY4_2rKgGW
z&d~B}ghy40A@$HFA?Evl(WKaXV_ZKXd-UPhhE<MEX{y3j1uq8%b20VM^jNwJ<Z;3L
z0nG^3V9C+7FdQH(vN}9lQj>3SdcjhbimPxXdvVOz-5?Z_wdRvkgsU_BsYfO~$-m_7
z8etJi5k8F3Xu5a=8y|xpx%Zr&*ldK8J2rJwA)cY<VWr$bt-K)@)P1m7Mr^cS^kRyv
z`w=ehVo@h3@W8W7_;phV0J%3VO|7kkD7uI<{nCZdEGrw#D^r|;6<u-bPpBiFi8Ah>
zB9Z>tS7uKw<m#u(No_OdbPzA~7_|^)bVWf96a{aR>(4CY89oNVuB}?abZ<q;TwT`|
z6|uO66&6#=I@A^WBiY&I!0l!!J*J*iHLk`)NkGG3Lbn|=0aC^OQr=6Pe6i*Id#DqN
zn0^7KOWP5w;NebIJ7hi4Cpk*HG*@oJrb5=Vmk-T5_Am}gFC>La_$mr@*mGx#esCs5
zR$`;7p@e9(7rb4qQQYKz;h?`wt8)_T3!!t>U5G%bD=Out@;g@IvUzUS&rSNiIhi1A
zvnTwPnWgj6CV~YP`LrWHg8M-aM_Vz40de>6jPX0Pb~P|GZ`LnTkLJ@+Y0`Ms+b$3Z
zQ?)Xv?UXlSbHEGxG<?`GN%L;pr9g`6uOTH;mL~2E2!>yi)P>DDX7xn`Owb&rw;Q<|
z@~)pwV*7tkRIXyMS3e2jv#4k-<Qm;RRcDmcVm(<X&PVgD(UHvyd;SQjdaTIQ$T=~y
zYQJ=#s%=%utTX!#q6k3i<k6oSTkZcq9qT_|<w^3IW1T_PE`$FhlnfqGss>cdC?<+N
z%4*)rB+L-QYi@WoN5Z%@F=W;}0s49jX*cvw6m}uZL|+6k;{^`+<|7_mls_jfl>{o>
z?^|f1rQI*v51p+`@6qUY?(O3}GE-}WYm6^#(b@{xABvF&;1T=erSvk;bgHwCMzL8#
zrK{Kn)c6mr^xWe=B^6!B$W@h-u0wih6Knl#?fxzpL(nF{@IJ1VO}Gq!-WOoA8DN9D
zp1<%lmT8Cr)w(2gAQ)#GyEiH^#%LKkUN<Mo<*l8W!8xOjCB!~GTa+DHS(;$%ydw}(
zf{yy5Yi^JwbZ$!|UJ6n<pFEyHyZtzHah*`<ic4?18{qopFpNOHk)}PCySCv|lsXsI
zy0>l?!Jpyd2&W9inz@gaJXW3~Htg-cC5nPjSuH<$sz79gwK()E@wd%a6YLVm3Av}1
zN9+<0%w19D9D=EEmlS=ef1<LZzcV0Dvxe8vhbp8@Z=ci8vL3v3eSWIyF~-Q?F{bb~
zl!O106hE^P?Uhy1B?>~G^|hWi(f}<m6Vb}T6EqfMX4k>|b><DN%z>CwTjLD5y|-F1
z--#36F0+%~+#SBgKX$Vx#x{BA{PKlHC0C<nGQO3_Y=n&ZZFlZHz5|@A=Z${MG!+L=
zP_3i6n5r8ztJMmb;aL|jB=4s5gV_lgw?iQ4Xb^|(+ugXfu+s>o%+(>3G9lJvx$e|k
zue%L{^*f|>qcM1eI<qAs%LnE<<h2k;4o7A8=RM=n>jOHnwY-K{de1HW=Q()Dd}Wnz
z)p?BvEMjRco2pCGP!R{^5zSkUD-5Qz$3oVgJ)J4LwVpWx$pQsI*Nd%5l<H6s0&|l<
zDWURfX7GP|SM3nLCZFKIOr+#AGWjR_T0Sz~9RAP#`E!K0k3IFoBvA>kJPDgL#_wmo
z>vhkG?vJD^HlhoMY6|8E-<LQZkzt!^B(}}R(j)X{ljbu7lF*gO-{~~!>nTJTdhIT_
zU!{Wgr<ZXTOCP&7&8Y{aoH(E9Z}%41;yKkR&~+zVd6BUO@I6h(vVtEOvKOj*3H5%|
zQ}q@Xvi9vxZx3cFHE4xK{-R!o?%X`=dQ_-+BPqC2i}JD2g={#%r<3Me%3!Q+cV-Gr
z$yw*um{YTXv*spop*#!(6|+}l3|!3s;xLtzH|pCuo{(=+$(}(I(7ZiLET`)J+lu#d
zwsn1K*BYC8Ut2lBsR(JNx{DrSwYRt^7F^Wkj1CJq(G1BOOFQZL?qIi>>#|gZ7Fs(<
zB0G_(p0@P9cauO_HL`&As?60JDZf48ysRrrl}eJZn$w6j9#z07d@*C)vfi}?7~hRJ
zw(fb1MPXsTdhSi;A=Be*S?_fkI4=+(y@e%Pn-$wU>kf3S><uryt%y?|zUGWSnRk!&
zp~~$3Y$;Y<Ej;c^_sg3w5_@=H)aCt$BMl5-NrKg2Mn*^+OK?kGI@<wIfA$!-yM~}N
zAc51e?BGjD^bxZ#BvgeE$FSkNyhtxEGYcg1dMUZr=-JC?r}5gU-Rh4Vs)8$~ckd$4
zIGV1<8e;PBk*Hgrvc!#wLI<!8eK5Cb_SIe3Viz&kch!_Y3kjTMhR2Y^bh$n5GPcAc
z#)BR|G5zeYt1<F<xHE73(iAjTNZ7P1H8ty<YW@+-tHDYj{t``TXN#8f7$V1jy7D@5
zyQ#`eYh7&mSO4S<Ze}l!n$9??{zaAg1J;JfOVMs7OVLO6C+a0b<8rC7*hVmHOVDH1
zj}1!A2EARLys%v(cGkP?Z4G{Ppk^U&9><&N6Yv$>LqV0S6%;RQ$HTpd1nL*}w<vWW
z4r>U3W-$$-*M|(NnApI`<23!@ka4Yd0bx4?pvSfEx`~I-01gLLp(tQ!B!*XRkJAVE
zHaxmDc!5u&iH^l%Y}3Y%wW1(R@NjLsgYG!{*>}r(|AcUWhQC+5*O-j?*~k?~s5TG<
zlE_9<e;4Q3F7MX~<w=Bx!2sE|_8`gJvg%#hIsl)1?|OQ9SKH8+6hh}PJCcdz>vvS9
zK+70EH--K-BP`inL;2&2=qmr2H4z5aoo-zS)3R@1rBz=tv5y6hCcB^cFh}_kOExDW
z{O0m$E&Z1$qLwzd23UNn?+<jF+ms~tL<EixCyA)<&LUU871O>STXpJ~_4kc1glV;|
znv-^u_hAiW8~-9yjcikjAA+}`pg9p06h$=N(YXwb5!|E!>q5ngT(O7V(b3~hkNcx-
zujFlbfkB{ZrE#6$plX1RN+7)6GQv5c`e|<|4>fi>gbQi4mDmZ&@bu#crhgC2moFdc
zFB92i&gaM6x!nZhoth2P*ntGez8o;X(GI-#-rytfA{$Z;xFs{gSWT;#wFT;|F#UHp
zc<c;XEYjiluDDV4AlUiw8#N^JD}2=O94}f_Q{^A9D|*Xx(pWVbp$XrGcQ&{Jlro8^
zy0$uM?$pkWD{wB4--&$8Bm2NW0Vx+)*LA$-?2sqd$By+(=JME$9Tt?1y_{7lkI%zk
z7A<eg{hrs*`CD>?;Z`yw^IjSou162yo74Q+<7ZkQ9?`wEydMR@O;cq;t3L|(rv7zS
zwqvHZ_n>y{^><D@*#SVLM=5%ks0@gx#>i?J(dt)<=~&UaE!{dtUL9pz;~`(alyKtl
z)T)v*ui5-8ju8G9fqcRkY_C11zModBU6q`6%VrxUVA_am0sOJioo7d_=pFAj!{q}?
zb(-THjP}ZM<<wy_>FWzEHN%w>hdcS+QtDH|XY*w|S>d}NHT_vmRnSGxWI~!`Ajf?%
zRY`i;?F!X&g2#iWK@E&s(7m#HN3^-yCM*{a-)FRVe&i(~-T}SpJf(QVzPvwced?fb
zy=Agb#WwU=<^|dyz+n>0>UR9>E>*n%wHY4k3U8qK?CNxCVEXVnw%3(0_8ZQ>_n$@>
zHx7f53j{<d5#0?vLQrZJTQ@IhLL?g<{FmKW34`t9CCyOz@b#R4Iw5XIh?);kNpm<K
zhu!HFn#@o;)To}Ih9*CYDHc&kib<sA;Wr&pX_8ngRP39TR&=9m5p?3EEAP{Jk(Sm~
zDGcFWcNrP#^YEfLuAr8sE>f5EyRBC=IK<KPWC3_XL_^eOGP}0?DC!O4PDu$(>V7T}
zBn(H))iP(5<w5g^N^fxzTF&rU5Dtx0V5-khDpP(W*Pm4hFYeQ<9{bciGPRJkM_P>p
zd7a&YyX147rGB4|#e~xltlc)rXn6vbP7-_G{Pfi@jp0HaY;Es)aJ__b<e$(bKiC|G
z_js@lAg~naO?Lr4FxN4ql0}Am@T&RgCGFlVOb>5Kkl}9Qg8AiEOqq0c#$IQ>raWe4
zG5pepan46bze<EGw<}f=z`P7yC&)%dF;lf!6~dft=o0X`$P6%7xx-gJmy2X$&lxab
zK_OaE`w}fddL)=W4M27Gpy8TBU-6xJQ}J&RbkK|mfv&HsftulQ`MxOV7_QkqsLu`-
z`WioA&k|IL-iUACbx@vB^R1<pvAS_|P(s+xT0Jc{^czqrH;|+>Umaond5i32J<Z+B
z<@ptMP!WSp1g6U!z<bKb>~pwtR9@u?lU2E%P|^mTLv<ZnxI%gW7-t5Y*Rfa$7a^D_
zcW`iqMFiOAD=vLa6|Dcs%d-er8%S$qU9vvML|4W;vjgeM{u49(Q@k7tbbZ&-%wL#b
zgE^sMQRw8@`aTcPk;*mVNNihuVLS<ty3HTKQ}K^WBDHpX?#)H$indQacTujWbyC!}
zBPICJChlL(u1dNXL>nJC5$Tbqd_P#reJ4&j6Ze_Y?J5H;eWPLO{H@NB)ml;UVf8IB
zzkg$UNGEmpIwUGuEJuY?C~i9|9-8V?p!(K@2q6^2nHcvX#0Y<`eMa)T18#5IMDy$c
zq&Zp@7Yf2ot+1lG@sJF;D>&tV!-Q-!I(&%Bhq9Kj4ZPKk_)RB|B6g$nmRYT%wpKIW
zy}H*oV91j5D$O)g=#gZUIqeL*y)!z)<2hKj!AZ<Xjf*rvuK)b|D*~kgqir>bNIcC+
zsSTZH{e3^nQ7&ocN1mYS@)eb$M};*(o>qpSI9aRMUA_yv5J~03F>u;Nn5yUR(z)f#
z+@`J(m3C>gpHs*ByZQS7;IG>Zx<st7&@ZYT=qhcDI7dxI!p)_Rg}O+~$k=3TQH#w*
zD7ebJVWm9dROL%cRpH=-3OB>wkea|%PJO3U><JShE0~a~QXJZ5mu~Ant**S5E^v{S
zaN-~0@BW<)>Q7uA6AW_?mVg0c$~>a`loi|7azx+HV(uHEX0ubjeyA;L_fS_5CppRI
ztvTJULs?M~3txw0izT|j*1U==FFb6ps0Jo@IRQ;GW=LQby?KmpOCxCtOoE2c2I`HD
zrC0Cf$d+x&T3SCW+Z9bEX-9Pt4R8b@)$A>qy$4iSeVVlx98VK8=ToO!UC$s`eeAO|
zm&Nv$JLnf@>YD4F_9$^df(aEJ9c97io)~?I(668}p<?|g!rt9Xg4!WRF<(JCco9VE
z%Qxw0A^gSIOM&7k0~R?Lw_Gtza@nx9dwvCH1Nl-zC)G~oPOwgY?!a5(wUdaBgg_vb
zl^-5PQYhej;)md1#JjMLTp{Gn)E^XR*s9p9GH>?vi)2c!tpF(A+(PlTCuL&Fb_Y)r
z*8A^^&F0k2$3FL;oh4hQtgwn^PM;a|j%yYTP}cX;?RWwIM4o^U2PX=at(JVB0^cpU
zGq1Hi_~B8B7x#O|YJmP$UauPbs6@gwmUA)S_S1$((yVETphfC5IZ#?n0zf?^-gs11
zcnc}m17~BN78KTT#SxWsW$mEL7{(Y<<=}uKBYe`2eN@hK`DWWflfA^={>r5mS-z_V
zJA&S{?HP7~9MO{T-qp<PsHT#DKm;59!j_ZrNiG71Mlqk)3_Svs!P!TkGpWQlF32~7
zF*x&9ifSypi}EJuqXL@4b`NlbD@$i6Z;y1h3DQLFzrW!Xc-(B+hMB|%K!cvqvR**^
z%LmcefPnIbY3+oqas?-rt>S{5z$N{t@tj;#*BaoDYK{tL+pAUs-4>!*R&vHF5#-?Q
zl%M!F0`zA==L3X?x#$|dxxOmfM;$n|50`acW<!d?D=@ZN%0EG@xq;)z9E)0JxUbY8
zzg?eeuw{4A{vCq<`-s<|26geFi~CU4whciYSW862A+y9>Vz$#7w04e144yh?<Ve%Q
z^sn6uxQC771No=Z{PSfQ8U9&&!iVEeg>8>0_C^LH_Z;g11yweI<{GR?7w~*9_v2vy
zIzO%DfTR&Er#16Gnd#r;k<K~<?O@*R>(d!hv#{3RNk9H^<Y0>HtaA5dSwhRB^K^h`
z^i=-OcJYJJO084I*QWLUk8b|zY&!^$PeTI12>mBe_<I!USwTZ|<_P%IWd4igq6Yys
z7~HET^S|pVl>&;zYx)4Q|6Lh^7f{&%qEqqdUvvJeGA}dKpD+$57g;y|{fFSd4369&
zD%Af?6Rs};xjRkQXJelK`osTow~*#o(|@(^I<WoWYR~3$nS+^``DaB1^2o@D>|Vm}
zh4dYa!Von@79mQ1uPq9n@5u?V!(uCNb5vXq+4@8_@5t$Q%Gq_eUyaa>(HQMjnvU@d
zNy^1=(&R$){vq0=fmF)u8w*maLY?%!^>)$B=4M7&+eFjA^H&AynJ{1Ot*BzAXC@<w
z7KJiy7aR}<YWC`?K@GK^y|!xhF806Z1O$JckBB(?H7LM(u-=`(Wo_HCvTmKQ42Vfd
z^(56vYoHtf&PM$>Xy1urH%&N+n>HiFRx1%vQAi&Tw?_$l%f1AmB0ApN69sj{4oFi3
z&T>7k_begE%~Ynx;;32Qq7jE>)lSVTz0eRLap%0<{QGIv7q7dbk3WL{;HcVjV8&68
z<-2w`dvW@LOK#fhH@axUCyWYp3yS@63XpvrTUSD4sJcaFwvgj<m*NXCNPWs0c9SF(
z@sEDxsq?0a;7^9p*vpO!`&3aR|5OnYS(~b`7l_ybN0^FMcfKrnZIxOW^SiS3O5ieR
zJq}v%B9^ToN5zKJ*7w+6x!-L8YqmRP*1-xa-#6$UP6xX5vXL(+a7cuLS@~kVCV1Zh
zL`ir}p;pJ-Fv<vc3w*1$H>0wx&fQ$Vo)-&cBd^GhIW!`s2Cq!%bq<GNih}^%5CEL*
zT+8<Vuf4Afifdik4em0yyM>^^-QC@FfZ!g2ySr<E-~<L|fZ*;L+&u($w@dauXV>|@
zQ+0peA9sGNnyOXv*1Oi~>DB!_Pj`Q6!XfY4u3`40Rg}f=DO|4E8k$W9{&aZmFY~$U
zDb{j2E36&xmr?sg{*X%|@hvXL=!+RM=bmkfcLfA=sKoYp`u%UB4ZDh^pyYbS(isq_
zH%f`B`!_3=T}4b2=Z+JV)PoJra6wypA%P(#F8Qxt9i~Yv8EuZ7*0<PO#v%14nZc#l
z<>(v(?VMwgVl76Jif`VR6eUy!KF%Pb6y1c=e`aPTSSE=EWh2rsQkT&2<`E~FJvLzP
z-AiCSr*}lsr^^21h_yELafUw*<<1Je6#bLePfaMBK-{|wTy*{^C{oSFP5GpXU`8dM
zhXcMx4e!4ZS4V6}s5`_%dV~DzpalQx85Q+N+D2!ejGop9&z+JIa^qhe`8OGsW?Isz
zQ@9Wi!XfVJQO;^A0!L`tE8g;Rb&-0DsRthf`)=*Pg9ao~-ebqs$u`K&F)-#5i_)xX
z;jLi-al`a)<j1$}A48ECg@QhqZgvsDQe!0P!1)hh;GEtZL^(mtl$_4u?EQ4@>^SNV
zF2_GE4C8)2xhBum8_#EkoVgAMgs&pf5`Osbf;c@BQB72i)3VlIKoO`=Dses{QJM@J
zB#PlU`yDb{rXeXH6%lYcg${+j>(ChhG9m{n9~RE(`n2s-#q@C<Hc*<x#?`^Jku-vN
zd|D2{W~Uia>-uo7kJGjx7G#e>jWzsc-lw>W&K1~xy)h1JF_*7IKdUYn#Q(THA({Y}
zip;Ce>^#s-{a*0CUv3?dQ?~EhN!)*}y4*DzcIWQa`K*KH#^-sgHvoV6qE0gEPL8f1
zSK~_lIx~6$YBjuCHyof=th+226GvU$FNh7i1b4UAg0-KaSIt3<C?W!p&h1V2ix$^d
zEV;_C)2u&sixIvzjeE*6cnB-F{ut)1+`gw7KFM%fN@|jhb-DouU}hL*(API8i#&H&
z`a}e-&`fSM!|XIw8?Q9QIvw<1lf_>Xe4uvgC`B?}5C7~gSLx9B!xd?*%4+tow$1ZF
zBHw)vcKNz}FkbLi#eGploCIBYLLfNoEVj5wp~<g3_p4<iI{tFevjGqMbc_Fr-D!ST
zE$(YRf-&V5*RgQTLNz&!v`x*K(ZyA@jozbAn?Yjsw~DZHBtYkc-&sg7Y{SOnoF2O0
zd^z{B*7nWQCZ*%8bSUzY(^)pOw6P<An|5>Z0j>Va>89A(vPX}~<tmzX%^}eX&FMki
zZFsq3r9z6vt{HI+5!>GkSrg8m*AW48TI1Z8E{Z$+cN0S{OE9Dx0X8MS4aynxqxz6=
z$jr4+o`TXSIQ1hWbiQz`FyZZLXR|<4JcAiP+Ps7V7a!xzVqM+kvobXzZgzycLVtnV
zOd?4V8M+p^XIDbiK3!3m)L6)C+OJO4ojnpSC!{qJpR3u_G^$QiF?b_<FjK4StxvNv
z-1hv+Jv(XW_ZX0LHfMYF1Iq2#(D{tJ4E1=v3N5_5Lh*d2T?Gx8>U`j-D0vkL-`1ST
zc6W1_>BjtdGfK&tFsLow=8|BZv>%sV^bw<IKIueirMU#1&So_f`3Gqg=|b4&+tO`v
z_-HQ}3<jniS13bI8zBt#i7GgI)K(cq-&y;D)nv@70h7V!tj|ui0+CeTx4{L(9vnnI
zn7@SEUdY8p-5-N<{I;&CRxod|Mu%fN6)Y_UV&-$3jq1cwmh1Q5Q`rfExXiKR+IS{Y
zzn~fNfpo06g8q8bu#iFSQ5j1<T44u#xcmy>LhHfZx;AKOgjwgvonM>X3(pd;-7RcQ
zw%g~X<^9H~lm6F>Vxxq-)A4t1Xv<@RXJy~qZ30Wf;SxZ*$pV+=gt42j&r4aACQD8u
z`K=9WGL5O0XH!txb7r$M6{8My*SIGlJZflF(M$u)yb94)R<>_;slAMyV>s}WU!l{?
z;0@ADu^!1mW4_u|by^y3W-AT3`!F55-*R+>7T^4&uSW?K)Gvt;3a&P2>=2y=vpP-&
zYsCR&>yoja(3lIinW;5#f!0DhW}W#ce=>P|4#;)RIO|(VKBlI#f+eLjaa2(!Rncyn
zb@@Kfk1yn2uef)<kuVYSqup~&kLOt#+vv5HMD6@%G=>81SoaZD;j2#2n0y?3yLlxw
zZbR47zRzyr#2F#Dq<%<@)2-yOW_q=zrwu||q4myR{VZW)+i~Dt!?je5+KCjvDelaB
zd`EOb5ek@<UElqpx)WWEh#vAulJtZmFA7`E?)$?Eb2M8s+$o(MJdv*Va5r0^vYehl
zM#f9Gr=}KR+@HO&{&L{~YY6Piy&(B6nZipC+ZE<l!)&dA#2txU+78CO-t%%H3xj)r
z8LkzeTmJ8us#4-lVAg*)T;Fgkw^RI7Zfm+5wouYYwpfQ9)=)&jqtu$S1pxEj>bibQ
zoQj!+vrGK=j?m4VLOFOM55!G{yR_1%IFD?Do1{pajT(h0*jo^Q)JJL%gqtQ_Ab!#P
z(Xg96VqWv`i<p|B@P#*a$qL<;9iN~bUhMmASCeCPg)p&}8Lm|T&Bs>gl-fBh+#?Zf
z*K(z>+LXScY6dl%P`&d_LcVK$z$2x=k)He`#m%DpsH(yoaaY6qT0hUUL8erjm;~<S
zCZVxiEIwvio6}hIkm|G~LpGaX?}PRVC)Dqz(-9uyCR+P8Lih8S+p{CFX_iW{<?V!r
zK0WwWUmRx(P~XsnME4wz$5uPI%D5Zu@oe}**xBk{?#MV>qt$k#%g>)nuKes`X>-&L
zZ4nDxil)EZIe5?m*PlGzQ&<=J9QpyJP#s5>Ub_yfS}8n+yKm1njR;v>il@Z`zncH)
ziWTsJuDoWn=b9p3_!jR}VQGqLxkYKd4sD{jnKOLS<1^UPhx=D{2TL0eyRHCN5>r16
zgTU=w`OM;5N;^)BnLw8%Sn)KwZj5>>Wq!{3ELsK7y>n?Hsoyaf3RLBP^scVhLbUbs
zAdDUK`>_hYk)!JnAmUaWD{N>@ejqnb$!jG<Gm|Q6)<|U(tk{CkKM<%~UxQ4&Z=sG2
z2eggZvGG!e*$!B+x3;%QsHjpe6Tg5$LN{LXA2sKqq+bZXQuTRc1X|)f@p;jkvoY1(
zvq5`gi)W~=CI(XO0YSN3qwPNilTi8+Nm(elT@i=nWWi@Z;DUayB{lO{)SzrMUgiAk
z`J?^^^_)~tiOMq=j%M5geq?VjV;Ak4)gYCvLL>(BIi4Q*^73rKIoq+LI4uN7(Kq6A
zPvt=D!d?OnAh>1@q4UILiMxC#2;WU?M_Zzc3rt8xn=Gw41jMWV9!%H|%(rS<;0BO-
zSCejoo7~#qjs9fJZ>ir5uTjg6#>pD3Xz131irCqXUWMh0$cyCgiS`8ZMJMih&H&Qm
z61x*f+zE-Rp8(z>^^cG+Kt#mfCJ?TVMc+d*x;l(;IXI;)%&@=GesOclE_zHi%y`!p
zoJ`Klg;rEMT|32vOqk=PyhFc%OA@7=;;?Kz3_J*uM!3|VRn|<Wg}h5YLa*@Cbn13f
z0s$}ISCk%`GgPKFeahuQ7}O=sn=S3G;s~%o7Tqe|L#K;TMD-&$9Bp2xWT@L6k<@?B
zqh##-Wm+oKN}B*@a!t8{W{v;svJ`G<LcsBL^aeN(V0$*JJ;me?Me#GfRXI41Kiq2Z
zv<@BnTY$)Sg?Epz17wHuvLS}WJO1-|5Qnc%E&LW%3~QD4ljFGpp30)70m4au610xk
zu_{s=qXyYeHTQciteX?08xt<Mz1N2~)}NM*2+C^21o<DAUxUJMp5J<#&kmL-az$30
zU3tizA0+No-}YL0zdsCrRl3T$%P2{{ZX#)1LGiv=#ajVe0#34bLmHhUr`T&-0%|n+
zx&B0^zeD^lplt{ZzbjOk$&983*)?q*lvmTtKpl)EdTQm)dT2CAR1l{+Gcq<;HKmnh
z{H-Ww8H7hpFaX}qd&Q$f@?`<RoZZ%TT`Wotcn|03OLP5_?BSM|BzC|1w-<~OHD_iy
z!u<&&rdDSu`DDty-GARdS729w7jAFhxHUDypHz26QN;|M5`zT}_XiuOK%?ct!tyvD
z3up=|8x%i77qskuu>levwBsUCcL?e8F5ms>D92_Yt?z=ipW+p|aftGXJ+AVaIg9lo
zZO<&`MAp3kBI}fOi;r<49I6V`xnfD}bA5F;_w2qo#lu~DPXcWrVLAUbF{Z{|pAW_;
z?G<)=oTO6#xJOaeN+PFMdp7dggyc{NyV+^+oz}2w@8ppx{=UU_rlW;OBkOd4xx<TK
zx1&Z0r(QP8>Zb*-;}=A?r|sLpM_xpSCrKKJaT3X&+Bz<_b<~`XP_6a+I}|5SUR8IB
zZ>eG|w8hK1i*8fhvs}Uj((CHN<sERxkr7(gM=$a{f7r0YSAqV{Mx*S3h`pl`t(63#
zNBKW-Zsngk@9X|(oEF4@ez9<GvL8OTI~%=3AcWi%dmoQV{PG=<f&Or0EUHm!AwM2e
zdU|o&ayS;FNHe?6{fAT5pWLWx*<vklsvQi4b+R5xcC@u8%=|ug9JYd%qCYS0DFy@D
zlu)|13($BX%ydX>Q*`<Hi*im5H+23o5ww8L7*D)tjIQR<*|<tpNHj&#NWWK*!pqG*
z#Pp3fHtO@Sme7V7tc0T<ULEqEFdG@?Ey3f~mJD>$_kfm^7gaUoSIi}hJ%15N-*&i#
zI5Z&k^I|uR{@ldszGQydQc^-Yo+A*&3goR=ZHrIslM_<(W#WyvJC+lR%Q}jh_%s&6
zYUok(N+Beta?v+NoT=#p{L80DQBP7F`hv5ok75n&fS}7e(LmRiv)l$L*l@muMYSif
zWe@$s3MS*f+afwrLta6*{Is0X#Ap(R#EF72$`y?1G!)HeuC_M(t13wg8lnTY7ZZeF
zobbaM$rAhZ=&h~8cfPRY$7PBMo4Bu@Bpt6|C$}7y%!5#W0Yw2K68?lhl(NlltW<Af
zE22pRbRR`;&j(1i-})V<l>p}c0!NtSzbWu{*=?QKJ(@`0p6cejNEhNP-c8cyO(jXz
zZ*}P)m$_`3l<dDfYI|A#9AhqGcemv>_|oT70s?}wH*8`v%b{IFO%2ocPNeUJ_3yBo
z4pPNor6S1j=4L5w*MP5HLNKe+zQ@3~;x3FYbo(T0A7-xWJmpUas2}!DqWUlzn~^#o
zisoCdW4Wz#5i|=|Xouxj(3^AaL_t7KK;JK~Y(^xhtqS0^_}ASMc!QEExh(4c4kQ0Y
zz5YIKKq5FQoWTTcoBtib{O9o>Tm5gCNuTTuheLuM82J|<_#aE^$Nkx23_c^x|CAlQ
zd8oHXh1G=%ADi|+^mu=P+f5{YG7myF-M{`N|G5h)$ACBVhZ&uSnex9Dbo{?w*juwq
zGVZ^v;eRZ+;rRznau~Jzud2iNPMAL`ePPGe=zpt+|GBcx>o*jyWi5Tm^e=Py&%Fd7
zQEfN{%%6XJJA+H%_<J%>#P0a5gFP(RhduT`7TOqo)5!C~57btc5lJb5!_5uTIR{7!
zjyW6V@}8y(EUgnRMFQ}A@FlCjm9V)a6$!<(8>yWLO~?uhUeg`7l1RgGFKC%8h|~4}
z97S00oJK@NiNA_pn9fV_%j{l%5p4M+(Glcoj03*gTmJX&omWajDkg449_7lpl{N3c
z6#mj<Uk=);iznGHxh=SdHk4Kjfsr&jV(0{ouy~HmI#BX(kg`acNQp#?`M*L_Rd9Hn
zNnd%+EH4>0*5SE)eOS?dPF>KY`vF)``RX{;K(d}40d9%y{&Zo=pGAb)qcr)6^M7gf
z9X<XS+3v2F2pEm<WZKt2=SKIa9PC5Hi|jehjcmj06oD3sGXH_-Q}7;jrRTTV=ZY&k
zgLjXi6s+dN_C!-Pp@8|70K7VU)`&0q1bP$dJ*qkpkrM>0^+2_!D>Z;6f9%SrU1=>Z
zp|Qkl!nURDXTgE(9EAG(vM2Pi15emF-A`R_gUnPkfC1n=Qvt5cS{|Re{c;<Q#|HB9
zZ>?3PqyI?c@Wi3NK3NA1(r0docLstdu#L4WsQ8z^Dvp@gj0y6$oM?C-?!1*FCPhRC
z7Nj@0b0l`h3>sbXqp~ZB|6bizxv&<bCxBT0j<#{Wpr?{~2^ks7(fx+1c}xw3X9z7$
z2~Pu_^z30HqfkX<$ygW&lzK-UZe!<&7o)~_^xmK_b;}Vy{R@rX4fZ|r+O4Me%85ZS
z64>F3^_VIz+1}-I1X!(Kqq@Hg7omJNUr`Tcj*(Cw<XA|$XP6uLtZU61Qj12<a=quW
zF})dUC#tav_+|o$Mc~zPNqO4Ob&9=_4J^?pg10%<`!M{aBe0@nZD44+c{K?Vmuxcv
zP9@`#!BD_k)nOtrN-D9v>vA9A%%)saWIVTe&ynX!4mq8d4f4YlC2Lm!?2qq+XZV#8
z(r4~RBkKquJuLffAa`euU+Jzu!t$F<v@Anx&0!KK8ikCo-+rE00S)6<{TPudH*$3R
z-pCmEsQ(~}<3)fM&p5b!n-4eb@%~Sk7W_x-l)%Nbm<6x|28xQePChv-YyqHUo<T~-
zm|$*!k`q?5D^9r?pSnCTfwTsq4<W!{iBGi}_CuXq_V}ae=ke1TN9U4ttr85aag$Tq
zhv^A~7fj_QH*A`<2BgiTZ2OrP{n3yAI0PYt0Mx`S+8Y0u8-;Hago|*$DBoMAgJM1x
z{$d+Yrq7XM&koQP0VV!{?}mTSjNA%u-!ACsLY&*>=gUuMRi&7FXulV_Q8zM*d%DtS
z+Wh6$lYNtvs=6biUY~QIl9?NeS<A%5Y=J`%w*wp+*7quIJ-jdCU|axOn6#h}YVi<M
zG!icOYB1LNV#kOA-2SW-x&k&Y)qTdWFLTpJl7o?DR0y|?s==LI%j&2UmHD2`b`SIj
zEI3U$%&we$NDcMR@Mt=>MAfuY-<<vkKu7m4M%+}*3=75-GTyxcR%;&x%>OKiR66(3
zV{}d%5RH@Pq9E(^k3>Woj;u{EZ3TFsL6N3>-qkZw37jc$=&oc;IE#<BcL7lr5}GQ7
zP?MI+Dj83YyUK%3Vs`mklQ3dzIdF}yQf|nPwesn>uxEQ}<2@4c%Qug7>X8W}Sl5Cs
zqrA$Cp!f5~OlwJ!t#CdJ-(qd{JbNi?)8F!YGaIo7H;X)EhvuILgY<G)(pO8wOp>WD
zY4qa5T;_Gl-E1Hiun6L=c9AyEXT#imcRfKy*R-Sb($x|<+(u4C#J8ix>jKvqarKj~
zpfS4cwhKNXZojODRYIJ9)x1MAlD)wsUHqeO(0qCjJ5rRYQSUjgF{rNXX40Hz4?gbZ
zr>tca#&o$~B6Func}<-=ttK3y9Y=caavf%iv*rjQFy-L9v5L^v<3Ue+SQ88PR8%*(
z%B?%`m43Kl>)ulZx3e8^c&6!8Gm}s{uhI;2I#0)H?0^j)sT-r9!{g3fU7+>d%_!$>
z$ReP_Dr`<sHhoQT=wY5^JH*tMCF&q)hme{Cw8i*aJ;bHN?zs_cgD95jj<9^3s4;^*
z`}s~axFW|J@aRf&;bun0KnE|PR@rS=j#N!9>p)6V2VzaVIipr&CouU4LvH5<@saJa
znn`rsf8=5cz|MP)^0BIi$s{2K55%Q%&tt7uLFX|<<KI{O)u4|qmGfG2`0Qqi`N4QG
znhneub!rfKHOjb<i4pSc+dKIq$;k!Y`CGjn6;R73_-NpODaeTma+`B|M5JU>_q^-Z
zHN^`h`a#J$-+B(+fRldMhJQPkh?)w7)B7I_YVi>;NH~K^RCBh=VcTJd=(yhnnf-Z~
zpu((=i5s_+MR2x86Fl#w>ym}Ye(F(2kSTjK98!!q4X%3WTNuQWQJ!cmQOWpB3H8_-
z22s*?3ai@P+uyH(=Tn019?R-4D^lB}BS(7kw&CLOY2AA5rt`__M{m+{>}p8X896PC
z*8>Eke?EZUB{<nsHxAEL&rDhK1dk?j>vfC`E<FpjTUE8Z&6mXBm^Sa-j*~O%u?!le
zWfT{Uep;J7ozExWO`Q|c3C?SCNJwxR9Bgz^y^rCJ&qb)q*3d^pM4eeEvl17P#Xua6
z11f1IE~EuJv$oJ`HbgD0KTLm_As%SpkO{Ak@z9sW8D1V}60!vmA92wRY3tbDhJz9B
z@nf6EWbnC_3)ac&NTbsh`)rrjJKT~y{-GFIq!IKvq>uA;B0DnxclWi}lja)l=j<0?
zALoYxAa$#ME1nnQv)D|vJruPgIhF+Tc`CMJqAk=Lh4D&wKbgKj^Xx{%OXcT{#M0EW
zXLlOzh9{}(++k?;_K~w5n=k*;xaM{mmXU^|HZ>R(y?azhP30?QE{{E+4J7IDd7%$q
z!n`a<TUxqDT<7>Q*+k5IvYm7qD52au<Z?k2ydP?#6^+)^=Vnfwm=^xdg8^=gYqTF2
z$DA~2Vdc;y)@qZ47%E)Yks5+6_i0<(5H?YKrf|t%UcvO6+C{E_*yX!bmcDhj2E>xg
zWygc)7gmNjWJXozD~2cE0k@r9JE^C7hX-S;KCs%=;RmF7(;Dv@r9};>*NviZlv@{T
zALB4Tk9%C{M+?e!OP;t?wlEehS`t47K7ndkCa-;1_2G|lY75z%4uvaW2sZIQEKq#Y
zs)cpG#76wKj>Z9~q{dw@hP_b}Ql%@~MIJT@@n2(VZu_OVJ*q*kOd=N!uLhWT94&0|
zUl~;()w<);>2_&3Wbf5m>FLL6PtmlCF6`qHTk+&m79zb7Qkc0}p|@$3rcRGNndNf{
z!&z(XS3O(bp;aw~f6S0%9Y_b*mI|*eY44=Y=iMI?sH@|U^N?dpq)0-qe#{$rxLmWf
zvJnW9J&&(aNI?j!#-r+&>QvT=ykc)7{mHF5Hbj?rjvOT;-s-`N$W=K{^OfjJW6=E0
zSxJ224N)<;g}H+JCIV3)-M^f#qDxDQgJco5j)lSIE4|G*CIDw47$ZVn`hC0%gbZA5
zDkVoFd`yxk024DrCdGVEF-G!)JfkaVMp-T3NCQFJNE<@UpEOkSdQ`ThbH8Y!y-|}K
zC<A#x^<ZutjQ=ZKIKuONZ&{d{lfKogZ#7SHR_Wp?HINvHDh3RRPSG)Nz=mK3YCB_)
z|8n0Xpzy#r2V^s<*&aBI*(pz_ocdgF(MEavP2av*Ig2*Jpq5{;h@5b+W|lX;kGBD|
ztU0ckoQ}?IF|SX{_3vL%<;}nK<)Yk<2UXTQi4_U*m*tSx@V=*Eeg1v+rjOwpxAibI
zG_<_P4BmKOF!^N1Mq_wPcX%jNG$6%*Qs)sBGav1od``5xBi6v>3vs%4)h_C3>}tBr
zMhn4LYq2%9>M*Fg_fh-cgNkZ8qtIiul~!>9`6I{YU1{A~*}$P|EOa7dBLvLGF$nJA
zD2Fc(pPNS=g2agUq{FN1)cF(?uG~x4TLpDIft#5I=}nFB4p^rmGCD=#p>pZX5APLv
zE3eLw5Zl;SFEMUjPZ#m3Z1G&!lE(#AXg4o|-g9VfQ${G>9`tXou0-ffSP<5D9%?eE
zvk_emY}ZkLKl|t){P5Ao;ONdXV}^SL#7JH0@T4h<YFh22klfGylylB~CtRk!8dw$g
z)}o9OM~ZQF&O4MjGh&q1u>bd40x-C<Zqob7FK!iyMC#%r2ZU+g3CQ+62@NK(%tN7~
zI-c_w8s%$ITe$m*g`F^R!zanlg!U_0j1A=gvgg72Yhd%;0I*I-68ssTfBn8L+<}aK
z+q`#FHA!x>>9Dx`yt*N(`Pq7)rG$YxeO*w+tgjt}Wt)r$X-X1gl;%eaa_6E|VnXdM
zp8vU773^{+z|t?^zvAO3vfX`Jqd^l9-Bo^`$q=QysB#Mj9NS76oLCGh((7gweO$ee
z&|Q;*X9b&n`ME`+!+1<y-&E30xc-VMTR0ZZuCs-n|K*^aufasdnZ%F@L$c@57|1B4
z=u7P#a*RW`Tzbz13^-DL-Fr>G1grkoH)PPEF?8YiD&)hT<I#p+ao~H^1luT^bgw8!
zzr%p}Zw12}aX7haGfoDcwi{D*w!iv15Ra!`>ceyVuDkM@UMRJC)f|IKKkOOqD6v9p
zD8R4!sJBHUZZORkP2EQu<iV#-%@}IKqOeu+LE>9*ajU=CF`+Bvjs0@05@9HCn431&
zp#|@FsdaBbWohUkA2N9$)G}{e;}okNN&XYW;`IR%Uja!#vW9BReu5GO^H8uP0wV{l
zEf#YFi07qs>X%bmvw5I!o*hWg60;LxfsBGjxqX&&I8w~nL<|3jU1_qyNdks+Yf4zh
z<$e3p8#7>IG}rUd@F06Py>3uJh0x(%=I#cE%Bx{(vO%3{FGBV^PeI55eIDy1yS*fQ
zzf9@v8?YWRtaN!!MKX}kLW^3TIilxVYWRGl$2`M?FTM@cq=sOe!rxYpkh<7NKU7ts
zB1Bk{{{^h<`C2M(+tEgJy0e#mloY2-kBNHYs#XdQeu`PZYr!K_fPl}!4U|w4MDa_?
zUoTbis$V~}<1SEOA?~}Bak5MVa;Q+r4fT<5STvW#MWUW+zANYSs*~6)cU`uX5~{rH
zP#DK5kZ}vM;kis9f-_lTlzn$NL~98FsSIDV=_#v;kmq|IVU`@8-43vlj=MN4F1=Eq
zXO-kHL^7@jhEUOq1W*rHSG|*9G_9GoKhAjx|M(cG^$g!`jUrV?kXo(%CWzOkTItZz
zq!%o-IjA`;QC}Bx%6%(T`XDvYj0WiEKN7kWI0(BPFRnSj9P?G%*Q83}9Lk0GJn=hg
zH=FxG0B0h}G4M%LoS6=b*^LD!n5o-go`>_$1}DI%DpjTz+AYF+e+en)pW%Ub@8247
zzb0RK+3G!7{7ycMIMc8i^mDgs!PCf~jK0C{#@41j#DYSug-Ly=U7awSpIpVQN@ISW
zqC>e6$q2@^es9TIOel}{sEr=kYoMNJFR9_g(m$7+FK@$zht^APq`hu+(WU~wpk3B*
z^wSm%XHWlo!)5}v9)&4~ig)3uHLQC6^{9#+@Oxw=^h{<S((SIH9$q7}eaFx{G_Rmk
zeg+-o$WOUw4YNi8$IK2rfP3afLWq?l0ON%QHv=!E$5u!=vq9YN!sm4551@J^ZvgKx
z4W5@deyGbpu(A(~=F5QtOW0#U?OpoX*LnDo&l>_dlOrCxZHWOFttT`*?ke5fx!v_V
z6o_y-*7K=polMV-!}pb6o^nzg<hslFe@w(sIQRS-S-M};?qD^tnRRjIj{LF~>Em^i
zZ)U_Ac~J+>Wzk{FW%rZE9jPBg%%IBr&84u<&v>tp8@t*6{iB7oT{S|Zf@>zjfx4JO
z!$Q;&CEv^^Fk7>%^_FxlKS4+W@k_1OV5(c~jVAU1jSlKgHz)^%34jtom?Pr;?L^|Z
zxYTWUquy4SkKAUxqJjj^+NN$1zlQ!B#C^l_lpJgUFtYrMrtYzUq31~p!9+P9!k)=l
zL*7YJ%$WU%mC`6mh9x1dkbH-P<WcM)tIZtF5xhXK05z$xg9_kc=TvJ=(g?Ktg7NEa
zVa!!LaAD#PG5<F(V|)hK$lU1SqB$e-%C=95OPV}K5YhFHAJY{uoCjXC9cO8Zh<bfs
zs12&FYO>Nez>?&3E8LDMNj=?Ol@Lcc-5F=kFrzhVd3O^dJU8umcl;J5)Qp3WdWSz|
zy-OV{P<77n=Z;-YN}Enwy-B)(I!7;ZP!OwrXlHC#hH+FDN93d(txfy6&Uak|EcUJq
zV_yE)Y@?;I*>S`}6$&$`A|QL=oM4pyq}k3=B{$pJ>bqG3KimF#h%8Vn2>)ndU>@Q@
z;U71II@SO)^Rsxn#g`U}ov(k(`rM)n{RlJ*Sn4;SPu9s|V1Xrfns&*YJk2gbGp(E3
zk7h6*92rY8s68|<_|0PLu6Ug8ZL;tBnPc3PpG3Ev=!0J!j8Q8f>h8z#?h2VKr+MS+
z_5o>EDX({}8nmM}L8Lil1(nW7^V#_KrS$LX5oH28()ViEq7-Q+F}AUqt!#2nF`b*k
zYN*ShUml(jwZn~QsmR~Uyxu<&jt?2^6&;HFD4VXzEFihEk5cQhM-ynB?bSv=wT?2r
zrl|0H#4B=MST!i)JWz#Pz$2=%gPl)%r^%rkx%L@j2IJ>buPKA?4t9MEeidu^%ZkV7
zfpW8+?}U7#iRl-60}T^Wkb1%kRUAMz0%r?qFHBZXuvanT+hyYUepHOfuq&gt05=l=
zp}|Q`))N}o@rcT86ay;6I;zMGEKR4feNWea^LhByAfktsko{6J9F}XW{W{Q(`EeRY
zY@g`eJNQi*aS?T?)PMX-6Fc-*s*$4a@nC)afT2wrWui4ONynfomK1NURu+t1mtGu4
zeq|oaT7gtg-E_FcaBScB&_;*6o37z6*I~eTX|C(Ig9OlkxIGOF7OZ_s`8`a>Kh~TB
zJ{%nEAblH<j&>WqHA;;@Rp4eia@^q|hqk$hG_eDT_1xA5Y8`QLFRp6|WIz4{d+)Um
zVMs0>b3Dy!vU6(ho(m7lQIQz8Xu$d*a4<>MKG%6FEd{oi5|X=zRH^CcWwhTz`XZTQ
z7KOxFfpkNPOlQ}M?<zcj`zDzMY>5UxD&}yIFRw!LF3I?om*ciy;d6t3pEWAKPDE`m
z{=5}hAA=zALo)Emd@en5#%7hD>Gd0Expn5Cfz(`Gdz7tqn1W8v6cp2}MNGntaM^*j
z;Pj5H?G^={D|Ru9(QLO&tPh9`v2iO<>O@}Fcy*hO{;<LM#fXyfHr<nww*W>-Vl}pS
zVqNs&K@@?q9jeC1E$@q-uEWs}NRIcI+^^*6wdQPP!Jmv5&eG-2)wgyv76lBi=3h&^
zT28b1P^E5<#ggn!-?^{)xBR-}SuH6;gc8jU?0wV;U$+=OxsbCDD!Niypu2>FIKMPL
z`vD<m9{Zlp0eg-NMx{<_v%qe`s^t?Lb!dH*ScGsRwOAD>@u=_Uv@A$~s^WB0ZZj(@
z`e;F%%nrt$NHeD5yET!5G~#6%$GA^5tGrZjtzX+w>1txYqy2SvmzI%ugM4ZqKVHlx
zZOCC6Ex6esiv>CN_*J^De_5^{bZtWfs2Tf9fb|Cu+aQ1@TJJOHG1Of|fk0D&^7g!_
zbyaJBE#WuDJDz^o1&@3t$-(nMAh=-<cmFcCU_j8^OWSMT^ZNzr5<Txo+r%TU0ls`~
z=_shE<lV(l$50!=k9sqq8zVN{epkI3{Pzn+&JI=cPj?vwtb8k4t?Sg)g9Ia$5Y4Oi
z9N87gPl~td^B<1Pk?)K?j4otkhJin%OsI<+XyZZNVCS)Wq2RRTgxoRb;y9%-+2~z!
z%8RH-dugv5e-IC8Eg8vtzQSxeiRB+f)QO44`;h43Qd3K%)zXPY6m&H})tz_1cE2)<
z(WZI5)WUelCHRYOX>$<H5X0kL55tp*8byx&50#v2z2X!Wm=KB$Z5ggdE4}k|DRuVX
zJEi$5Y|=8dkCG-90Y+($h7wGLIyqgs2nb4aKSv%zska($@2yBFrW^A&Gq4u1M!V>_
zEr=6)Xyw|O{Ago{7uVvyb7>wIT?OSGj|lOa4<1-ooaWSCdd^utoeW(M8nz5;(h27?
z`9`7dQb`)w@&ht*g4iYoe<sBY9rJf%aBYZm*i5yOqi|T+WFNq)ulsH?E7er~Y&$-`
zv_G&V68eXpK^BD+8<!&pcAKJl<J;w=HzxDMYKv9G)77lGy>u@x;t02oU2g`2j>g}l
zkV`(=WQQBTS~v3HQ;e~9S91c3zvjyvu0LR+eiT}u2jGl}u`U_g=*_ir4g;$~QHk^R
zZZ*?}r{{);Ggmn?<egESejoB4FA3W2*P&wfk*g5&m~{`>`{m9ffAgPf<kX4d&tD!4
z){M}%j30ziI<1^~zuMr;3Tpm7xwKGPkwUUbv_EB7>A=~x=#bJ5mT78tSq-bDNO&4=
zTTqFgG~fExrbF_#O8O5CN$^Jap=IWMBd0ps#yBHkQ|!)zb5cFrFh4(D0#U5w;xAhD
z6pR=*@;45vdAX%j*rP?fb9_PiP67W_=+-<ixnpDSy4`la+z!EvbMPW?B=zBUR;1!S
z<5^ymfzQl*kvAzVzr@qc*9VK~tY>=q0x|34Y%tOxpy31;tCP6%?B9jF$cB(wD2CIP
zZ!G}*TQK48uVJCNd{Fn=Vpp;e9)+6VXP)N$x@cWFOo{cx(U_pDQ2w2D2;)K`hn)ll
z(@Jj)Q%(X0&&hrt7&q9VZ{#8T14h4nNlWh(4t!*Cng+cwM;ilF-+F2ONnHMB%J|-d
z^Q+IO#;^aYfI+%52HsJjnh=Gv<-c^?CyH;h^Xlolo7{hgAOB{DR>I%ty8r*1|9=nu
eAFzXsSE#`ECTUJqXwdK8J~9%D;+3Mtf&T~I#(>)Z

diff --git a/docs/setup/aws.md b/docs/setup/aws.md
index cee568053b5da..bc5b221201afc 100644
--- a/docs/setup/aws.md
+++ b/docs/setup/aws.md
@@ -32,17 +32,26 @@ Amazon Web Services offers cloud computing services on which you can run Flink.
 
 [Amazon Elastic MapReduce](https://aws.amazon.com/elasticmapreduce/) (Amazon EMR) is a web service that makes it easy to  quickly setup a Hadoop cluster. This is the **recommended way** to run Flink on AWS as it takes care of setting up everything.
 
-### Create EMR Cluster
+### Standard EMR Installation
 
-The EMR documentation contains [examples showing how to start an EMR cluster](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-gs-launch-sample-cluster.html). You can follow that guide and install any EMR release. You don't need to install *All Applications* part of the EMR release, but can stick to *Core Hadoop*:
+Flink is a supported application on Amazon EMR. [Amazon's documentation](http://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-flink.html)
+describes configuring Flink, creating and monitoring a cluster, and working with jobs.
 
-<img src="{{ site.baseurl }}/fig/flink-on-emr.png" class="img-responsive">
+### Custom EMR Installation
 
-When creating your cluster, make sure to setup [IAM roles](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-iam-roles.html) allowing you to access your S3 buckets if required.
+Amazon EMR services are regularly updated to new releases but a version of Flink which is not available
+can be manually installed in a stock EMR cluster.
 
-{% top %}
+**Create EMR Cluster**
+
+The EMR documentation contains [examples showing how to start an EMR cluster](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-gs-launch-sample-cluster.html). You can follow that guide and install any EMR release. You don't need to install the *All Applications* part of the EMR release, but can stick to *Core Hadoop*.
+
+{% warn Note %}
+Access to S3 buckets requires
+[configuration of IAM roles](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-iam-roles.html)
+when creating an EMR cluster.
 
-### Install Flink on EMR Cluster
+**Install Flink on EMR Cluster**
 
 After creating your cluster, you can [connect to the master node](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-connect-master-node.html) and install Flink:
 

From e5e420021be16ba81f5a20ecd9cd4fda6e326a99 Mon Sep 17 00:00:00 2001
From: David Anderson <david@alpinegizmo.com>
Date: Thu, 4 May 2017 11:27:33 +0200
Subject: [PATCH 172/253] [FLINK-6443] [docs] Add more links to concepts docs

This closes #3822
---
 docs/concepts/programming-model.md | 22 +++++++++++++++++-----
 docs/concepts/runtime.md           | 16 ++++++++--------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/docs/concepts/programming-model.md b/docs/concepts/programming-model.md
index 3d2aebbc43a12..d83cf0096da20 100644
--- a/docs/concepts/programming-model.md
+++ b/docs/concepts/programming-model.md
@@ -48,7 +48,7 @@ Flink offers different levels of abstraction to develop streaming/batch applicat
     for certain operations only. The *DataSet API* offers additional primitives on bounded data sets, like loops/iterations.
 
   - The **Table API** is a declarative DSL centered around *tables*, which may be dynamically changing tables (when representing streams).
-    The Table API follows the (extended) relational model: Tables have a schema attached (similar to tables in relational databases)
+    The [Table API](../dev/table_api.html) follows the (extended) relational model: Tables have a schema attached (similar to tables in relational databases)
     and the API offers comparable operations, such as select, project, join, group-by, aggregate, etc.
     Table API programs declaratively define *what logical operation should be done* rather than specifying exactly
    *how the code for the operation looks*. Though the Table API is extensible by various types of user-defined
@@ -60,7 +60,7 @@ Flink offers different levels of abstraction to develop streaming/batch applicat
 
   - The highest level abstraction offered by Flink is **SQL**. This abstraction is similar to the *Table API* both in semantics and
     expressiveness, but represents programs as SQL query expressions.
-    The SQL abstraction closely interacts with the Table API, and SQL queries can be executed over tables defined in the *Table API*.
+    The [SQL](../dev/table_api.html#sql) abstraction closely interacts with the Table API, and SQL queries can be executed over tables defined in the *Table API*.
 
 
 ## Programs and Dataflows
@@ -81,6 +81,9 @@ arbitrary **directed acyclic graphs** *(DAGs)*. Although special forms of cycles
 Often there is a one-to-one correspondence between the transformations in the programs and the operators
 in the dataflow. Sometimes, however, one transformation may consist of multiple transformation operators.
 
+Sources and sinks are documented in the [streaming connectors](../dev/connectors/index.html) and [batch connectors](../dev/batch/connectors.html) docs.
+Transformations are documented in [DataStream transformations](../dev/datastream_api.html#datastream-transformations) and [DataSet transformations](../dev/batch/dataset_transformations.html).
+
 {% top %}
 
 ## Parallel Dataflows
@@ -112,6 +115,8 @@ Streams can transport data between two operators in a *one-to-one* (or *forwardi
     is preserved, but the parallelism does introduce non-determinism regarding the order in
     which the aggregated results for different keys arrive at the sink.
 
+Details about configuring and controlling parallelism can be found in the docs on [parallel execution](../dev/parallel.html).
+
 {% top %}
 
 ## Windows
@@ -128,6 +133,7 @@ One typically distinguishes different types of windows, such as *tumbling window
 <img src="../fig/windows.svg" alt="Time- and Count Windows" class="offset" width="80%" />
 
 More window examples can be found in this [blog post](https://flink.apache.org/news/2015/12/04/Introducing-windows.html).
+More details are in the [window docs](../dev/windows.html).
 
 {% top %}
 
@@ -165,6 +171,8 @@ This alignment also allows Flink to redistribute the state and adjust the stream
 
 <img src="../fig/state_partitioning.svg" alt="State and Partitioning" class="offset" width="50%" />
 
+For more information, see the documentation on [working with state](../dev/stream/state.html).
+
 {% top %}
 
 ## Checkpoints for Fault Tolerance
@@ -178,17 +186,21 @@ point of the checkpoint.
 The checkpoint interval is a means of trading off the overhead of fault tolerance during execution with the recovery time (the number
 of events that need to be replayed).
 
-More details on checkpoints and fault tolerance are in the [fault tolerance docs]({{ site.baseurl }}/internals/stream_checkpointing.html).
+The description of the [fault tolerance internals]({{ site.baseurl }}/internals/stream_checkpointing.html) provides
+more information about how Flink manages checkpoints and related topics.
+Details about enabling and configuring checkpointing are in the [checkpointing API docs](../dev/stream/checkpointing.html).
+
 
 {% top %}
 
 ## Batch on Streaming
 
-Flink executes batch programs as a special case of streaming programs, where the streams are bounded (finite number of elements).
+Flink executes [batch programs](../dev/batch/index.html) as a special case of streaming programs, where the streams are bounded (finite number of elements).
 A *DataSet* is treated internally as a stream of data. The concepts above thus apply to batch programs in the
 same way as well as they apply to streaming programs, with minor exceptions:
 
-  - Programs in the DataSet API do not use checkpoints. Recovery happens by fully replaying the streams.
+  - [Fault tolerance for batch programs](../dev/batch/fault_tolerance.html) does not use checkpointing.
+    Recovery happens by fully replaying the streams.
     That is possible, because inputs are bounded. This pushes the cost more towards the recovery,
     but makes the regular processing cheaper, because it avoids checkpoints.
 
diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md
index 0d4e01787b535..c598b12f3c682 100644
--- a/docs/concepts/runtime.md
+++ b/docs/concepts/runtime.md
@@ -31,7 +31,7 @@ under the License.
 For distributed execution, Flink *chains* operator subtasks together into *tasks*. Each task is executed by one thread.
 Chaining operators together into tasks is a useful optimization: it reduces the overhead of thread-to-thread
 handover and buffering, and increases overall throughput while decreasing latency.
-The chaining behavior can be configured in the APIs.
+The chaining behavior can be configured; see the [chaining docs](../dev/datastream_api.html#task-chaining-and-resource-groups) for details.
 
 The sample dataflow in the figure below is executed with five subtasks, and hence with five parallel threads.
 
@@ -54,9 +54,9 @@ The Flink runtime consists of two types of processes:
 
     There must always be at least one TaskManager.
 
-The JobManagers and TaskManagers can be started in various ways: directly on the machines, in
-containers, or managed by resource frameworks like YARN. TaskManagers connect to JobManagers, announcing
-themselves as available, and are assigned work.
+The JobManagers and TaskManagers can be started in various ways: directly on the machines as a [standalone cluster](../setup/cluster_setup.html), in
+containers, or managed by resource frameworks like [YARN](../setup/yarn_setup.html) or [Mesos](../setup/mesos.html).
+TaskManagers connect to JobManagers, announcing themselves as available, and are assigned work.
 
 The **client** is not part of the runtime and program execution, but is used to prepare and send a dataflow to the JobManager.
 After that, the client can disconnect, or stay connected to receive progress reports. The client runs either as part of the
@@ -98,7 +98,7 @@ job. Allowing this *slot sharing* has two main benefits:
 
 <img src="../fig/slot_sharing.svg" alt="TaskManagers with shared Task Slots" class="offset" width="80%" />
 
-The APIs also include a *resource group* mechanism which can be used to prevent undesirable slot sharing. 
+The APIs also include a *[resource group](../dev/datastream_api.html#task-chaining-and-resource-groups)* mechanism which can be used to prevent undesirable slot sharing. 
 
 As a rule-of-thumb, a good default number of task slots would be the number of CPU cores.
 With hyper-threading, each slot then takes 2 or more hardware thread contexts.
@@ -107,7 +107,7 @@ With hyper-threading, each slot then takes 2 or more hardware thread contexts.
 
 ## State Backends
 
-The exact data structures in which the key/values indexes are stored depends on the chosen **state backend**. One state backend
+The exact data structures in which the key/values indexes are stored depends on the chosen [state backend](../ops/state_backends.html). One state backend
 stores data in an in-memory hash map, another state backend uses [RocksDB](http://rocksdb.org) as the key/value store.
 In addition to defining the data structure that holds the state, the state backends also implement the logic to
 take a point-in-time snapshot of the key/value state and store that snapshot as part of a checkpoint.
@@ -120,8 +120,8 @@ take a point-in-time snapshot of the key/value state and store that snapshot as
 
 Programs written in the Data Stream API can resume execution from a **savepoint**. Savepoints allow both updating your programs and your Flink cluster without losing any state. 
 
-Savepoints are **manually triggered checkpoints**, which take a snapshot of the program and write it out to a state backend. They rely on the regular checkpointing mechanism for this. During execution programs are periodically snapshotted on the worker nodes and produce checkpoints. For recovery only the last completed checkpoint is needed and older checkpoints can be safely discarded as soon as a new one is completed.
+[Savepoints](..//setup/savepoints.html) are **manually triggered checkpoints**, which take a snapshot of the program and write it out to a state backend. They rely on the regular checkpointing mechanism for this. During execution programs are periodically snapshotted on the worker nodes and produce checkpoints. For recovery only the last completed checkpoint is needed and older checkpoints can be safely discarded as soon as a new one is completed.
 
-Savepoints are similar to these periodic checkpoints except that they are **triggered by the user** and **don't automatically expire** when newer checkpoints are completed.
+Savepoints are similar to these periodic checkpoints except that they are **triggered by the user** and **don't automatically expire** when newer checkpoints are completed. Savepoints can be created from the [command line](../setup/cli.html#savepoints) or when cancelling a job via the [REST API](../monitoring/rest_api.html#cancel-job-with-savepoint).
 
 {% top %}

From 9d0c6e490bea08769d696d911646457efdebf8f6 Mon Sep 17 00:00:00 2001
From: "shuai.xus" <shuai.xus@alibaba-inc.com>
Date: Tue, 18 Apr 2017 14:15:29 +0800
Subject: [PATCH 173/253] [FLINK-5867] [flip-1] Support restarting only
 pipelined sub-regions of the ExecutionGraph on task failure

---
 .../failover/FailoverRegion.java              | 251 +++++++++
 .../failover/FailoverStrategyLoader.java      |   8 +-
 .../RestartPipelinedRegionStrategy.java       | 206 ++++++++
 .../ExecutionGraphTestUtils.java              |  38 +-
 .../executiongraph/FailoverRegionTest.java    | 490 ++++++++++++++++++
 ...ipelinedRegionFailoverConcurrencyTest.java | 353 +++++++++++++
 .../RestartPipelinedRegionStrategyTest.java   | 396 ++++++++++++++
 7 files changed, 1731 insertions(+), 11 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverRegion.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FailoverRegionTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverRegion.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverRegion.java
new file mode 100644
index 0000000000000..b36cfcf1a4f13
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverRegion.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.failover;
+
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.FutureUtils;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.executiongraph.GlobalModVersionMismatch;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
+import org.apache.flink.util.AbstractID;
+import org.apache.flink.util.FlinkException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * FailoverRegion manages the failover of a minimal pipeline connected sub graph.
+ * It will change from CREATED to CANCELING and then to CANCELLED and at last to RUNNING,
+ */
+public class FailoverRegion {
+
+	private static final AtomicReferenceFieldUpdater<FailoverRegion, JobStatus> STATE_UPDATER =
+			AtomicReferenceFieldUpdater.newUpdater(FailoverRegion.class, JobStatus.class, "state");
+
+	/** The log object used for debugging. */
+	private static final Logger LOG = LoggerFactory.getLogger(FailoverRegion.class);
+
+	// ------------------------------------------------------------------------
+
+	/** a unique id for debugging */
+	private final AbstractID id = new AbstractID();
+
+	private final ExecutionGraph executionGraph;
+
+	private final List<ExecutionVertex> connectedExecutionVertexes;
+
+	/** The executor that executes the recovery action after all vertices are in a */
+	private final Executor executor;
+
+	/** Current status of the job execution */
+	private volatile JobStatus state = JobStatus.RUNNING;
+
+
+	public FailoverRegion(ExecutionGraph executionGraph, Executor executor, List<ExecutionVertex> connectedExecutions) {
+		this.executionGraph = checkNotNull(executionGraph);
+		this.executor = checkNotNull(executor);
+		this.connectedExecutionVertexes = checkNotNull(connectedExecutions);
+
+		LOG.debug("Created failover region {} with vertices: {}", id, connectedExecutions);
+	}
+
+	public void onExecutionFail(Execution taskExecution, Throwable cause) {
+		// TODO: check if need to failover the preceding region
+		if (!executionGraph.getRestartStrategy().canRestart()) {
+			// delegate the failure to a global fail that will check the restart strategy and not restart
+			executionGraph.failGlobal(cause);
+		}
+		else {
+			cancel(taskExecution.getGlobalModVersion());
+		}
+	}
+
+	private void allVerticesInTerminalState(long globalModVersionOfFailover) {
+		while (true) {
+			JobStatus curStatus = this.state;
+			if (curStatus.equals(JobStatus.CANCELLING)) {
+				if (transitionState(curStatus, JobStatus.CANCELED)) {
+					reset(globalModVersionOfFailover);
+					break;
+				}
+			}
+			else {
+				LOG.info("FailoverRegion {} is {} when allVerticesInTerminalState.", id, state);
+				break;
+			}
+		}
+	}
+
+	public JobStatus getState() {
+		return state;
+	}
+
+	/**
+	 * get all execution vertexes contained in this region
+	 */
+	public List<ExecutionVertex> getAllExecutionVertexes() {
+		return connectedExecutionVertexes;
+	}
+
+	// Notice the region to failover, 
+	private void failover(long globalModVersionOfFailover) {
+		if (!executionGraph.getRestartStrategy().canRestart()) {
+			executionGraph.failGlobal(new FlinkException("RestartStrategy validate fail"));
+		}
+		else {
+			JobStatus curStatus = this.state;
+			if (curStatus.equals(JobStatus.RUNNING)) {
+				cancel(globalModVersionOfFailover);
+			}
+			else if (curStatus.equals(JobStatus.CANCELED)) {
+				reset(globalModVersionOfFailover);
+			}
+			else {
+				LOG.info("FailoverRegion {} is {} when notified to failover.", id, state);
+			}
+		}
+	}
+
+	// cancel all executions in this sub graph
+	private void cancel(final long globalModVersionOfFailover) {
+		while (true) {
+			JobStatus curStatus = this.state;
+			if (curStatus.equals(JobStatus.RUNNING)) {
+				if (transitionState(curStatus, JobStatus.CANCELLING)) {
+
+					// we build a future that is complete once all vertices have reached a terminal state
+					final ArrayList<Future<?>> futures = new ArrayList<>(connectedExecutionVertexes.size());
+
+					// cancel all tasks (that still need cancelling)
+					for (ExecutionVertex vertex : connectedExecutionVertexes) {
+						futures.add(vertex.cancel());
+					}
+
+					final FutureUtils.ConjunctFuture allTerminal = FutureUtils.combineAll(futures);
+					allTerminal.thenAcceptAsync(new AcceptFunction<Void>() {
+						@Override
+						public void accept(Void value) {
+							allVerticesInTerminalState(globalModVersionOfFailover);
+						}
+					}, executor);
+
+					break;
+				}
+			}
+			else {
+				LOG.info("FailoverRegion {} is {} when cancel.", id, state);
+				break;
+			}
+		}
+	}
+
+	// reset all executions in this sub graph
+	private void reset(long globalModVersionOfFailover) {
+		try {
+			// reset all connected ExecutionVertexes
+			final Collection<CoLocationGroup> colGroups = new HashSet<>();
+			final long restartTimestamp = System.currentTimeMillis();
+
+			for (ExecutionVertex ev : connectedExecutionVertexes) {
+				CoLocationGroup cgroup = ev.getJobVertex().getCoLocationGroup();
+				if (cgroup != null && !colGroups.contains(cgroup)){
+					cgroup.resetConstraints();
+					colGroups.add(cgroup);
+				}
+
+				ev.resetForNewExecution(restartTimestamp, globalModVersionOfFailover);
+			}
+			if (transitionState(JobStatus.CANCELED, JobStatus.CREATED)) {
+				restart(globalModVersionOfFailover);
+			}
+			else {
+				LOG.info("FailoverRegion {} switched from CANCELLING to CREATED fail, will fail this region again.", id);
+				failover(globalModVersionOfFailover);
+			}
+		}
+		catch (GlobalModVersionMismatch e) {
+			// happens when a global recovery happens concurrently to the regional recovery
+			// go back to a clean state
+			state = JobStatus.RUNNING;
+		}
+		catch (Throwable e) {
+			LOG.info("FailoverRegion {} reset fail, will failover again.", id);
+			failover(globalModVersionOfFailover);
+		}
+	}
+
+	// restart all executions in this sub graph
+	private void restart(long globalModVersionOfFailover) {
+		try {
+			if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) {
+				// if we have checkpointed state, reload it into the executions
+				//TODO: checkpoint support restore part ExecutionVertex cp
+				/**
+				if (executionGraph.getCheckpointCoordinator() != null) {
+					executionGraph.getCheckpointCoordinator().restoreLatestCheckpointedState(
+							connectedExecutionVertexes, false, false);
+				}
+				*/
+				//TODO, use restart strategy to schedule them.
+				//restart all connected ExecutionVertexes
+				for (ExecutionVertex ev : connectedExecutionVertexes) {
+					try {
+						ev.scheduleForExecution(
+								executionGraph.getSlotProvider(),
+								executionGraph.isQueuedSchedulingAllowed());
+					}
+					catch (Throwable e) {
+						failover(globalModVersionOfFailover);
+					}
+				}
+			}
+			else {
+				LOG.info("FailoverRegion {} switched from CREATED to RUNNING fail, will fail this region again.", id);
+				failover(globalModVersionOfFailover);
+			}
+		} catch (Exception e) {
+			LOG.info("FailoverRegion {} restart failed, failover again.", id, e);
+			failover(globalModVersionOfFailover);
+		}
+	}
+
+	private boolean transitionState(JobStatus current, JobStatus newState) {
+		if (STATE_UPDATER.compareAndSet(this, current, newState)) {
+			LOG.info("FailoverRegion {} switched from state {} to {}.", id, current, newState);
+			return true;
+		}
+		else {
+			return false;
+		}
+	}
+
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java
index f18a90f79c329..8b6fa6e9c80f8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java
@@ -35,9 +35,12 @@ public class FailoverStrategyLoader {
 	/** Config name for the {@link RestartAllStrategy} */
 	public static final String FULL_RESTART_STRATEGY_NAME = "full";
 
-	/** Config name for the strategy that restarts individual tasks */
+	/** Config name for the {@link RestartIndividualStrategy} */
 	public static final String INDIVIDUAL_RESTART_STRATEGY_NAME = "individual";
 
+	/** Config name for the {@link RestartPipelinedRegionStrategy} */
+	public static final String PIPELINED_REGION_RESTART_STRATEGY_NAME = "region";
+
 	// ------------------------------------------------------------------------
 
 	/**
@@ -59,6 +62,9 @@ public static FailoverStrategy.Factory loadFailoverStrategy(Configuration config
 				case FULL_RESTART_STRATEGY_NAME:
 					return new RestartAllStrategy.Factory();
 
+				case PIPELINED_REGION_RESTART_STRATEGY_NAME:
+					return new RestartPipelinedRegionStrategy.Factory();
+
 				case INDIVIDUAL_RESTART_STRATEGY_NAME:
 					return new RestartIndividualStrategy.Factory();
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java
new file mode 100644
index 0000000000000..0a5baa87c0a53
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.failover;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionEdge;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.IntermediateResultPartition;
+import org.apache.flink.util.FlinkException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A failover strategy that restarts regions of the ExecutionGraph. A region is defined
+ * by this strategy as the weakly connected component of tasks that communicate via pipelined
+ * data exchange.
+ */
+public class RestartPipelinedRegionStrategy extends FailoverStrategy {
+
+	/** The log object used for debugging. */
+	private static final Logger LOG = LoggerFactory.getLogger(RestartPipelinedRegionStrategy.class);
+
+	/** The execution graph on which this FailoverStrategy works */
+	private final ExecutionGraph executionGraph;
+
+	/** The executor used for future actions */
+	private final Executor executor;
+
+	/** Fast lookup from vertex to failover region */
+	private final HashMap<ExecutionVertex, FailoverRegion> vertexToRegion;
+
+
+	/**
+	 * Creates a new failover strategy to restart pipelined regions that works on the given
+	 * execution graph and uses the execution graph's future executor to call restart actions.
+	 * 
+	 * @param executionGraph The execution graph on which this FailoverStrategy will work
+	 */
+	public RestartPipelinedRegionStrategy(ExecutionGraph executionGraph) {
+		this(executionGraph, executionGraph.getFutureExecutor());
+	}
+
+	/**
+	 * Creates a new failover strategy to restart pipelined regions that works on the given
+	 * execution graph and uses the given executor to call restart actions.
+	 * 
+	 * @param executionGraph The execution graph on which this FailoverStrategy will work
+	 * @param executor  The executor used for future actions
+	 */
+	public RestartPipelinedRegionStrategy(ExecutionGraph executionGraph, Executor executor) {
+		this.executionGraph = checkNotNull(executionGraph);
+		this.executor = checkNotNull(executor);
+		this.vertexToRegion = new HashMap<>();
+	}
+
+	// ------------------------------------------------------------------------
+	//  failover implementation
+	// ------------------------------------------------------------------------ 
+
+	@Override
+	public void onTaskFailure(Execution taskExecution, Throwable cause) {
+		final ExecutionVertex ev = taskExecution.getVertex();
+		final FailoverRegion failoverRegion = vertexToRegion.get(ev);
+
+		if (failoverRegion == null) {
+			executionGraph.failGlobal(new FlinkException(
+					"Can not find a failover region for the execution " + ev.getTaskNameWithSubtaskIndex()));
+		}
+		else {
+			failoverRegion.onExecutionFail(taskExecution, cause);
+		}
+	}
+
+	@Override
+	public void notifyNewVertices(List<ExecutionJobVertex> newJobVerticesTopological) {
+		LOG.debug("Generating failover regions for {} new job vertices", newJobVerticesTopological.size());
+		generateAllFailoverRegion(newJobVerticesTopological);
+	}
+
+	@Override
+	public String getStrategyName() {
+		return "Pipelined Region Failover";
+	}
+
+	// Generate all the FailoverRegion from the new added job vertexes
+	private void generateAllFailoverRegion(List<ExecutionJobVertex> newJobVerticesTopological) {
+		for (ExecutionJobVertex ejv : newJobVerticesTopological) {
+			for (ExecutionVertex ev : ejv.getTaskVertices()) {
+				if (getFailoverRegion(ev) != null) {
+					continue;
+				}
+				List<ExecutionVertex> pipelinedExecutions = new ArrayList<>();
+				List<ExecutionVertex> orgExecutions = new ArrayList<>();
+				orgExecutions.add(ev);
+				pipelinedExecutions.add(ev);
+				getAllPipelinedConnectedVertexes(orgExecutions, pipelinedExecutions);
+
+				FailoverRegion region = new FailoverRegion(executionGraph, executor, pipelinedExecutions);
+				for (ExecutionVertex vertex : pipelinedExecutions) {
+					vertexToRegion.put(vertex, region);
+				}
+			}
+		}
+	}
+
+	/**
+	 * Get all connected executions of the original executions
+	 *
+	 * @param orgExecutions  the original execution vertexes
+	 * @param connectedExecutions  the total connected executions
+	 */
+	private static void getAllPipelinedConnectedVertexes(List<ExecutionVertex> orgExecutions, List<ExecutionVertex> connectedExecutions) {
+		List<ExecutionVertex> newAddedExecutions = new ArrayList<>();
+		for (ExecutionVertex ev : orgExecutions) {
+			// Add downstream ExecutionVertex
+			for (IntermediateResultPartition irp : ev.getProducedPartitions().values()) {
+				if (irp.getIntermediateResult().getResultType().isPipelined()) {
+					for (List<ExecutionEdge> consumers : irp.getConsumers()) {
+						for (ExecutionEdge consumer : consumers) {
+							ExecutionVertex cev = consumer.getTarget();
+							if (!connectedExecutions.contains(cev)) {
+								newAddedExecutions.add(cev);
+							}
+						}
+					}
+				}
+			}
+			if (!newAddedExecutions.isEmpty()) {
+				connectedExecutions.addAll(newAddedExecutions);
+				getAllPipelinedConnectedVertexes(newAddedExecutions, connectedExecutions);
+				newAddedExecutions.clear();
+			}
+			// Add upstream ExecutionVertex
+			int inputNum = ev.getNumberOfInputs();
+			for (int i = 0; i < inputNum; i++) {
+				for (ExecutionEdge input : ev.getInputEdges(i)) {
+					if (input.getSource().getIntermediateResult().getResultType().isPipelined()) {
+						ExecutionVertex pev = input.getSource().getProducer();
+						if (!connectedExecutions.contains(pev)) {
+							newAddedExecutions.add(pev);
+						}
+					}
+				}
+			}
+			if (!newAddedExecutions.isEmpty()) {
+				connectedExecutions.addAll(0, newAddedExecutions);
+				getAllPipelinedConnectedVertexes(newAddedExecutions, connectedExecutions);
+				newAddedExecutions.clear();
+			}
+		}
+	}
+
+	// ------------------------------------------------------------------------
+	//  testing
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Finds the failover region that contains the given execution vertex.
+ 	 */
+	@VisibleForTesting
+	public FailoverRegion getFailoverRegion(ExecutionVertex ev) {
+		return vertexToRegion.get(ev);
+	}
+
+	// ------------------------------------------------------------------------
+	//  factory
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Factory that instantiates the RestartPipelinedRegionStrategy.
+	 */
+	public static class Factory implements FailoverStrategy.Factory {
+
+		@Override
+		public FailoverStrategy create(ExecutionGraph executionGraph) {
+			return new RestartPipelinedRegionStrategy(executionGraph);
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
index 0d7e389117e09..140e984675bc9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
@@ -30,6 +30,7 @@
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
 import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.failover.FailoverRegion;
 import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
 import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
 import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider;
@@ -137,6 +138,27 @@ public static void waitUntilExecutionState(Execution execution, ExecutionState s
 		}
 	}
 
+	public static void waitUntilFailoverRegionState(FailoverRegion region, JobStatus status, long maxWaitMillis)
+			throws TimeoutException {
+
+		checkNotNull(region);
+		checkNotNull(status);
+		checkArgument(maxWaitMillis >= 0);
+
+		// this is a poor implementation - we may want to improve it eventually
+		final long deadline = maxWaitMillis == 0 ? Long.MAX_VALUE : System.nanoTime() + (maxWaitMillis * 1_000_000);
+
+		while (region.getState() != status && System.nanoTime() < deadline) {
+			try {
+				Thread.sleep(2);
+			} catch (InterruptedException ignored) {}
+		}
+
+		if (System.nanoTime() >= deadline) {
+			throw new TimeoutException();
+		}
+	}
+
 	/**
 	 * Takes all vertices in the given ExecutionGraph and switches their current
 	 * execution to RUNNING.
@@ -320,21 +342,17 @@ public SimpleActorGateway(ExecutionContext executionContext){
 
 		@Override
 		public Object handleMessage(Object message) {
-			Object result = null;
-			if(message instanceof SubmitTask) {
+			if (message instanceof SubmitTask) {
 				SubmitTask submitTask = (SubmitTask) message;
 				lastTDD = submitTask.tasks();
-
-				result = Acknowledge.get();
+				return Acknowledge.get();
 			} else if(message instanceof CancelTask) {
-				CancelTask cancelTask = (CancelTask) message;
-
-				result = Acknowledge.get();
+				return Acknowledge.get();
 			} else if(message instanceof FailIntermediateResultPartitions) {
-				result = new Object();
+				return new Object();
+			} else {
+				return null;
 			}
-
-			return result;
 		}
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FailoverRegionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FailoverRegionTest.java
new file mode 100644
index 0000000000000..beeefb1b8c4e4
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FailoverRegionTest.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.JobException;
+import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory;
+import org.apache.flink.runtime.executiongraph.failover.RestartPipelinedRegionStrategy;
+import org.apache.flink.runtime.executiongraph.restart.InfiniteDelayRestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
+import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider;
+import org.apache.flink.runtime.instance.Instance;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobgraph.ScheduleMode;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.SimpleActorGateway;
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilExecutionState;
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilFailoverRegionState;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class FailoverRegionTest extends TestLogger {
+
+	/**
+	 * Tests that a job only has one failover region and can recover from task failure successfully
+	 * @throws Exception
+	 */
+	@Test
+	public void testSingleRegionFailover() throws Exception {
+		RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy(10);
+		ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy);
+		RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+
+		ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();
+
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev).getState());
+
+		ev.getCurrentExecutionAttempt().fail(new Exception("Test Exception"));
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev).getState());
+
+		for (ExecutionVertex evs : eg.getAllExecutionVertices()) {
+			evs.getCurrentExecutionAttempt().cancelingComplete();
+		}
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev).getState());
+	}
+
+	/**
+	 * Tests that a job has server failover regions and one region failover does not influence others
+	 * 
+	 * <pre>
+	 *     (a1) ---> (b1) -+-> (c1) ---+-> (d1) 
+	 *                     X          /
+	 *     (a2) ---> (b2) -+-> (c2) -+
+	 *
+	 *           ^         ^         ^
+	 *           |         |         |
+	 *     (pipelined) (blocking) (pipelined)
+	 *
+	 * </pre>
+	 */
+	@Test
+	public void testMultiRegionsFailover() throws Exception {
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+
+		final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20);
+				
+		JobVertex v1 = new JobVertex("vertex1");
+		JobVertex v2 = new JobVertex("vertex2");
+		JobVertex v3 = new JobVertex("vertex3");
+		JobVertex v4 = new JobVertex("vertex4");
+
+		v1.setParallelism(2);
+		v2.setParallelism(2);
+		v3.setParallelism(2);
+		v4.setParallelism(1);
+
+		v1.setInvokableClass(AbstractInvokable.class);
+		v2.setInvokableClass(AbstractInvokable.class);
+		v3.setInvokableClass(AbstractInvokable.class);
+		v4.setInvokableClass(AbstractInvokable.class);
+
+		v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+		v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+		v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		List<JobVertex> ordered = Arrays.asList(v1, v2, v3, v4);
+
+		ExecutionGraph eg = new ExecutionGraph(
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				jobId,
+				jobName,
+				new Configuration(),
+				new SerializedValue<>(new ExecutionConfig()),
+				AkkaUtils.getDefaultTimeout(),
+				new InfiniteDelayRestartStrategy(10),
+				new FailoverPipelinedRegionWithDirectExecutor(),
+				Collections.<BlobKey>emptyList(),
+				Collections.<URL>emptyList(),
+				slotProvider,
+				ExecutionGraph.class.getClassLoader());
+
+		eg.attachJobGraph(ordered);
+
+		RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+
+		// the following two vertices are in the same failover region
+		ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0];
+		ExecutionVertex ev21 = eg.getJobVertex(v2.getID()).getTaskVertices()[0];
+
+		// the following two vertices are in the same failover region
+		ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1];
+		ExecutionVertex ev22 = eg.getJobVertex(v2.getID()).getTaskVertices()[1];
+
+		// the following vertices are in one failover region
+		ExecutionVertex ev31 = eg.getJobVertex(v3.getID()).getTaskVertices()[0];
+		ExecutionVertex ev32 = eg.getJobVertex(v3.getID()).getTaskVertices()[1];
+		ExecutionVertex ev4 = eg.getJobVertex(v3.getID()).getTaskVertices()[0];
+
+		eg.scheduleForExecution();
+
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
+
+		ev21.scheduleForExecution(slotProvider, true);
+		ev21.getCurrentExecutionAttempt().fail(new Exception("New fail"));
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());
+
+		ev11.getCurrentExecutionAttempt().cancelingComplete();
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());
+
+		ev11.getCurrentExecutionAttempt().markFinished();
+		ev21.getCurrentExecutionAttempt().markFinished();
+		ev22.scheduleForExecution(slotProvider, true);
+		ev22.getCurrentExecutionAttempt().markFinished();
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());
+
+		waitUntilExecutionState(ev31.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 2000);
+		waitUntilExecutionState(ev32.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 2000);
+
+		ev31.getCurrentExecutionAttempt().fail(new Exception("New fail"));
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState());
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev31).getState());
+
+		ev32.getCurrentExecutionAttempt().cancelingComplete();
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());
+	}
+
+	/**
+	 * Tests that when a task fail, and restart strategy doesn't support restarting, the job will go to failed
+	 * @throws Exception
+	 */
+	@Test
+	public void testNoManualRestart() throws Exception {
+		NoRestartStrategy restartStrategy = new NoRestartStrategy();
+		ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy);
+
+		ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();
+
+		ev.fail(new Exception("Test Exception"));
+
+		for (ExecutionVertex evs : eg.getAllExecutionVertices()) {
+			evs.getCurrentExecutionAttempt().cancelingComplete();
+		}
+		assertEquals(JobStatus.FAILED, eg.getState());
+	}
+
+	/**
+	 * Tests that two failover regions failover at the same time, they will not influence each orther
+	 * @throws Exception
+	 */
+	@Test
+	public void testMutilRegionFailoverAtSameTime() throws Exception {
+		Instance instance = ExecutionGraphTestUtils.getInstance(
+				new ActorTaskManagerGateway(
+						new SimpleActorGateway(TestingUtils.directExecutionContext())),
+				16);
+
+		Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
+		scheduler.newInstanceAvailable(instance);
+
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+		final Configuration cfg = new Configuration();
+
+		JobVertex v1 = new JobVertex("vertex1");
+		JobVertex v2 = new JobVertex("vertex2");
+		JobVertex v3 = new JobVertex("vertex3");
+		JobVertex v4 = new JobVertex("vertex4");
+
+		v1.setParallelism(2);
+		v2.setParallelism(2);
+		v3.setParallelism(2);
+		v4.setParallelism(2);
+
+		v1.setInvokableClass(AbstractInvokable.class);
+		v2.setInvokableClass(AbstractInvokable.class);
+		v3.setInvokableClass(AbstractInvokable.class);
+		v4.setInvokableClass(AbstractInvokable.class);
+
+		v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+		v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		List<JobVertex> ordered = Arrays.asList(v1, v2, v3, v4);
+
+		ExecutionGraph eg = new ExecutionGraph(
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				jobId,
+				jobName,
+				cfg,
+				new SerializedValue<>(new ExecutionConfig()),
+				AkkaUtils.getDefaultTimeout(),
+				new InfiniteDelayRestartStrategy(10),
+				new RestartPipelinedRegionStrategy.Factory(),
+				Collections.<BlobKey>emptyList(),
+				Collections.<URL>emptyList(),
+				scheduler,
+				ExecutionGraph.class.getClassLoader());
+		try {
+			eg.attachJobGraph(ordered);
+		}
+		catch (JobException e) {
+			e.printStackTrace();
+			fail("Job failed with exception: " + e.getMessage());
+		}
+		eg.scheduleForExecution();
+		RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+
+		ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0];
+		ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1];
+		ExecutionVertex ev31 = eg.getJobVertex(v3.getID()).getTaskVertices()[0];
+		ExecutionVertex ev32 = eg.getJobVertex(v3.getID()).getTaskVertices()[1];
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());
+
+		ev11.getCurrentExecutionAttempt().fail(new Exception("new fail"));
+		ev31.getCurrentExecutionAttempt().fail(new Exception("new fail"));
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState());
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev31).getState());
+
+		ev32.getCurrentExecutionAttempt().cancelingComplete();
+		waitUntilFailoverRegionState(strategy.getFailoverRegion(ev31), JobStatus.RUNNING, 1000);
+
+		ev12.getCurrentExecutionAttempt().cancelingComplete();
+		waitUntilFailoverRegionState(strategy.getFailoverRegion(ev11), JobStatus.RUNNING, 1000);
+	}
+
+	/**
+	 * Tests that if a task reports the result of its preceding task is failed,
+	 * its preceding task will be considered as failed, and start to failover
+	 * TODO: as the report part is not finished yet, this case is ignored temporarily
+	 * @throws Exception
+	 */
+	@Ignore
+	@Test
+	public void testSucceedingNoticePreceding() throws Exception {
+		Instance instance = ExecutionGraphTestUtils.getInstance(
+				new ActorTaskManagerGateway(
+						new SimpleActorGateway(TestingUtils.directExecutionContext())),
+				14);
+
+		Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
+		scheduler.newInstanceAvailable(instance);
+
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+		final Configuration cfg = new Configuration();
+
+		JobVertex v1 = new JobVertex("vertex1");
+		JobVertex v2 = new JobVertex("vertex2");
+
+		v1.setParallelism(1);
+		v2.setParallelism(1);
+
+		v1.setInvokableClass(AbstractInvokable.class);
+		v2.setInvokableClass(AbstractInvokable.class);
+
+		v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+
+		List<JobVertex> ordered = new ArrayList<>(Arrays.asList(v1, v2));
+
+		ExecutionGraph eg = new ExecutionGraph(
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				jobId,
+				jobName,
+				cfg,
+				new SerializedValue<>(new ExecutionConfig()),
+				AkkaUtils.getDefaultTimeout(),
+				new InfiniteDelayRestartStrategy(10),
+				new FailoverPipelinedRegionWithDirectExecutor(),
+				Collections.<BlobKey>emptyList(),
+				Collections.<URL>emptyList(),
+				scheduler,
+				ExecutionGraph.class.getClassLoader());
+		try {
+			eg.attachJobGraph(ordered);
+		}
+		catch (JobException e) {
+			e.printStackTrace();
+			fail("Job failed with exception: " + e.getMessage());
+		}
+		eg.setScheduleMode(ScheduleMode.EAGER);
+		eg.scheduleForExecution();
+		RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+
+		ExecutionVertex ev11 = eg.getJobVertex(v2.getID()).getTaskVertices()[0];
+		ExecutionVertex ev21 = eg.getJobVertex(v2.getID()).getTaskVertices()[0];
+		ev21.getCurrentExecutionAttempt().fail(new Exception("Fail with v1"));
+
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev21).getState());
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState());
+	}
+
+	/**
+	 * Tests that a new failure comes while the failover region is in CANCELLING
+	 * @throws Exception
+	 */
+	@Test
+	public void testFailWhileCancelling() throws Exception {
+		RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy();
+		ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy);
+		RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+
+		Iterator<ExecutionVertex> iter = eg.getAllExecutionVertices().iterator();
+		ExecutionVertex ev1 = iter.next();
+		ev1.getCurrentExecutionAttempt().switchToRunning();
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev1).getState());
+
+		ev1.getCurrentExecutionAttempt().fail(new Exception("new fail"));
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState());
+
+		ExecutionVertex ev2 = iter.next();
+		ev2.getCurrentExecutionAttempt().fail(new Exception("new fail"));
+		assertEquals(JobStatus.RUNNING, eg.getState());
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState());
+	}
+
+	/**
+	 * Tests that a new failure comes while the failover region is restarting
+	 * @throws Exception
+	 */
+	@Test
+	public void testFailWhileRestarting() throws Exception {
+		RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy();
+		ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy);
+		RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+
+		Iterator<ExecutionVertex> iter = eg.getAllExecutionVertices().iterator();
+		ExecutionVertex ev1 = iter.next();
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev1).getState());
+
+		ev1.getCurrentExecutionAttempt().fail(new Exception("new fail"));
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState());
+
+		for (ExecutionVertex evs : eg.getAllExecutionVertices()) {
+			evs.getCurrentExecutionAttempt().cancelingComplete();
+		}
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev1).getState());
+
+		ev1.getCurrentExecutionAttempt().fail(new Exception("new fail"));
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState());
+	}
+
+	private static ExecutionGraph createSingleRegionExecutionGraph(RestartStrategy restartStrategy) throws Exception {
+		Instance instance = ExecutionGraphTestUtils.getInstance(
+				new ActorTaskManagerGateway(
+						new SimpleActorGateway(TestingUtils.directExecutionContext())),
+				14);
+
+		Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
+		scheduler.newInstanceAvailable(instance);
+
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+		final Configuration cfg = new Configuration();
+
+		JobVertex v1 = new JobVertex("vertex1");
+		JobVertex v2 = new JobVertex("vertex2");
+		JobVertex v3 = new JobVertex("vertex3");
+
+		v1.setParallelism(3);
+		v2.setParallelism(2);
+		v3.setParallelism(2);
+
+		v1.setInvokableClass(AbstractInvokable.class);
+		v2.setInvokableClass(AbstractInvokable.class);
+		v3.setInvokableClass(AbstractInvokable.class);
+
+		v3.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		List<JobVertex> ordered = new ArrayList<>(Arrays.asList(v1, v2, v3));
+
+		ExecutionGraph eg = new ExecutionGraph(
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				jobId,
+				jobName,
+				cfg,
+				new SerializedValue<>(new ExecutionConfig()),
+				AkkaUtils.getDefaultTimeout(),
+				restartStrategy,
+				new FailoverPipelinedRegionWithDirectExecutor(),
+				Collections.<BlobKey>emptyList(),
+				Collections.<URL>emptyList(),
+				scheduler,
+				ExecutionGraph.class.getClassLoader());
+		try {
+			eg.attachJobGraph(ordered);
+		}
+		catch (JobException e) {
+			e.printStackTrace();
+			fail("Job failed with exception: " + e.getMessage());
+		}
+
+		eg.scheduleForExecution();
+		return eg;
+	}
+
+	// ------------------------------------------------------------------------
+
+	/**
+	 * A factory to create a RestartPipelinedRegionStrategy that uses a
+	 * direct (synchronous) executor for easier testing.
+	 */
+	private static class FailoverPipelinedRegionWithDirectExecutor implements Factory {
+
+		@Override
+		public FailoverStrategy create(ExecutionGraph executionGraph) {
+			return new RestartPipelinedRegionStrategy(executionGraph, Executors.directExecutor());
+		}
+	}
+
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java
new file mode 100644
index 0000000000000..635ec755a1a9c
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.ManuallyTriggeredDirectExecutor;
+import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.execution.SuppressRestartsException;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy;
+import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory;
+import org.apache.flink.runtime.executiongraph.failover.RestartPipelinedRegionStrategy;
+import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
+import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+import org.apache.flink.util.SerializedValue;
+
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilExecutionState;
+import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilJobStatus;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * These tests make sure that global failover (restart all) always takes precedence over
+ * local recovery strategies for the {@link RestartPipelinedRegionStrategy}
+ * 
+ * <p>This test must be in the package it resides in, because it uses package-private methods
+ * from the ExecutionGraph classes.
+ */
+public class PipelinedRegionFailoverConcurrencyTest {
+
+	/**
+	 * Tests that a cancellation concurrent to a local failover leads to a properly
+	 * cancelled state.
+	 */
+	@Test
+	public void testCancelWhileInLocalFailover() throws Exception {
+
+		// the logic in this test is as follows:
+		//  - start a job
+		//  - cause a task failure and delay the local recovery action via the manual executor
+		//  - cancel the job to go into cancelling
+		//  - resume in local recovery action
+		//  - validate that this does in fact not start a new task, because the graph as a
+		//    whole should now be cancelled already
+
+		final JobID jid = new JobID();
+		final int parallelism = 2;
+
+		final ManuallyTriggeredDirectExecutor executor = new ManuallyTriggeredDirectExecutor();
+
+		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);
+
+		final ExecutionGraph graph = createSampleGraph(
+				jid,
+				new FailoverPipelinedRegionWithCustomExecutor(executor),
+				new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0),
+				slotProvider,
+				2);
+
+		final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next();
+		final ExecutionVertex vertex1 = ejv.getTaskVertices()[0];
+		final ExecutionVertex vertex2 = ejv.getTaskVertices()[1];
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		// let one of the vertices fail - that triggers a local recovery action
+		vertex1.getCurrentExecutionAttempt().fail(new Exception("test failure"));
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+
+		// graph should still be running and the failover recovery action should be queued
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(1, executor.numQueuedRunnables());
+
+		// now cancel the job
+		graph.cancel();
+
+		assertEquals(JobStatus.CANCELLING, graph.getState());
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.CANCELING, vertex2.getCurrentExecutionAttempt().getState());
+
+		// let the recovery action continue
+		executor.trigger();
+
+		// now report that cancelling is complete for the other vertex
+		vertex2.getCurrentExecutionAttempt().cancelingComplete();
+
+		assertEquals(JobStatus.CANCELED, graph.getState());
+		assertTrue(vertex1.getCurrentExecutionAttempt().getState().isTerminal());
+		assertTrue(vertex2.getCurrentExecutionAttempt().getState().isTerminal());
+
+		// make sure all slots are recycled
+		assertEquals(parallelism, slotProvider.getNumberOfAvailableSlots());
+	}
+
+	/**
+	 * Tests that a terminal global failure concurrent to a local failover
+	 * leads to a properly failed state.
+	 */
+	@Test
+	public void testGlobalFailureConcurrentToLocalFailover() throws Exception {
+
+		// the logic in this test is as follows:
+		//  - start a job
+		//  - cause a task failure and delay the local recovery action via the manual executor
+		//  - cause a global failure
+		//  - resume in local recovery action
+		//  - validate that this does in fact not start a new task, because the graph as a
+		//    whole should now be terminally failed already
+
+		final JobID jid = new JobID();
+		final int parallelism = 2;
+
+		final ManuallyTriggeredDirectExecutor executor = new ManuallyTriggeredDirectExecutor();
+
+		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);
+
+		final ExecutionGraph graph = createSampleGraph(
+				jid,
+				new FailoverPipelinedRegionWithCustomExecutor(executor),
+				new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0),
+				slotProvider,
+				2);
+
+		final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next();
+		final ExecutionVertex vertex1 = ejv.getTaskVertices()[0];
+		final ExecutionVertex vertex2 = ejv.getTaskVertices()[1];
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		// let one of the vertices fail - that triggers a local recovery action
+		vertex1.getCurrentExecutionAttempt().fail(new Exception("test failure"));
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+
+		// graph should still be running and the failover recovery action should be queued
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(1, executor.numQueuedRunnables());
+
+		// now cancel the job
+		graph.failGlobal(new SuppressRestartsException(new Exception("test exception")));
+
+		assertEquals(JobStatus.FAILING, graph.getState());
+		assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.CANCELING, vertex2.getCurrentExecutionAttempt().getState());
+
+		// let the recovery action continue
+		executor.trigger();
+
+		// now report that cancelling is complete for the other vertex
+		vertex2.getCurrentExecutionAttempt().cancelingComplete();
+
+		assertEquals(JobStatus.FAILED, graph.getState());
+		assertTrue(vertex1.getCurrentExecutionAttempt().getState().isTerminal());
+		assertTrue(vertex2.getCurrentExecutionAttempt().getState().isTerminal());
+
+		// make sure all slots are recycled
+		assertEquals(parallelism, slotProvider.getNumberOfAvailableSlots());
+	}
+
+	/**
+	 * Tests that a local failover does not try to trump a global failover.
+	 */
+	@Test
+	public void testGlobalRecoveryConcurrentToLocalRecovery() throws Exception {
+
+		// the logic in this test is as follows:
+		//  - start a job
+		//  - cause a task failure and delay the local recovery action via the manual executor
+		//  - cause a global failure that is recovering immediately
+		//  - resume in local recovery action
+		//  - validate that this does in fact not cause another task restart, because the global
+		//    recovery should already have restarted the task graph
+
+		final JobID jid = new JobID();
+		final int parallelism = 2;
+
+		final ManuallyTriggeredDirectExecutor executor = new ManuallyTriggeredDirectExecutor();
+
+		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);
+
+		final ExecutionGraph graph = createSampleGraph(
+				jid,
+				new FailoverPipelinedRegionWithCustomExecutor(executor),
+				new FixedDelayRestartStrategy(2, 0), // twice restart, no delay
+				slotProvider,
+				2);
+		RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)graph.getFailoverStrategy();
+
+		final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next();
+		final ExecutionVertex vertex1 = ejv.getTaskVertices()[0];
+		final ExecutionVertex vertex2 = ejv.getTaskVertices()[1];
+
+		graph.scheduleForExecution();
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex1).getState());
+
+		// let one of the vertices fail - that triggers a local recovery action
+		vertex2.getCurrentExecutionAttempt().fail(new Exception("test failure"));
+		assertEquals(ExecutionState.FAILED, vertex2.getCurrentExecutionAttempt().getState());
+		assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(vertex2).getState());
+
+		// graph should still be running and the failover recovery action should be queued
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(1, executor.numQueuedRunnables());
+
+		// now cancel the job
+		graph.failGlobal(new Exception("test exception"));
+
+		assertEquals(JobStatus.FAILING, graph.getState());
+		assertEquals(ExecutionState.FAILED, vertex2.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.CANCELING, vertex1.getCurrentExecutionAttempt().getState());
+
+		// now report that cancelling is complete for the other vertex
+		vertex1.getCurrentExecutionAttempt().cancelingComplete();
+
+		waitUntilJobStatus(graph, JobStatus.RUNNING, 1000);
+		assertEquals(JobStatus.RUNNING, graph.getState());
+
+		waitUntilExecutionState(vertex1.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000);
+		waitUntilExecutionState(vertex2.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000);
+		vertex1.getCurrentExecutionAttempt().switchToRunning();
+		vertex2.getCurrentExecutionAttempt().switchToRunning();
+		assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState());
+
+		// let the recovery action continue - this should do nothing any more
+		executor.trigger();
+
+		// validate that the graph is still peachy
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex1).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex2).getState());
+		assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState());
+		assertEquals(1, vertex1.getCurrentExecutionAttempt().getAttemptNumber());
+		assertEquals(1, vertex2.getCurrentExecutionAttempt().getAttemptNumber());
+		assertEquals(1, vertex1.getCopyOfPriorExecutionsList().size());
+		assertEquals(1, vertex2.getCopyOfPriorExecutionsList().size());
+
+		// make sure all slots are in use
+		assertEquals(0, slotProvider.getNumberOfAvailableSlots());
+
+		// validate that a task failure then can be handled by the local recovery
+		vertex2.getCurrentExecutionAttempt().fail(new Exception("test failure"));
+		assertEquals(1, executor.numQueuedRunnables());
+
+		// let the local recovery action continue - this should recover the vertex2
+		executor.trigger();
+
+		waitUntilExecutionState(vertex2.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000);
+		vertex2.getCurrentExecutionAttempt().switchToRunning();
+
+		// validate that the local recovery result
+		assertEquals(JobStatus.RUNNING, graph.getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex1).getState());
+		assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex2).getState());
+		assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState());
+		assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState());
+		assertEquals(1, vertex1.getCurrentExecutionAttempt().getAttemptNumber());
+		assertEquals(2, vertex2.getCurrentExecutionAttempt().getAttemptNumber());
+		assertEquals(1, vertex1.getCopyOfPriorExecutionsList().size());
+		assertEquals(2, vertex2.getCopyOfPriorExecutionsList().size());
+
+		// make sure all slots are in use
+		assertEquals(0, slotProvider.getNumberOfAvailableSlots());
+	}
+
+	// ------------------------------------------------------------------------
+	//  utilities
+	// ------------------------------------------------------------------------
+
+	private ExecutionGraph createSampleGraph(
+			JobID jid,
+			Factory failoverStrategy,
+			RestartStrategy restartStrategy,
+			SlotProvider slotProvider,
+			int parallelism) throws Exception {
+
+		// build a simple execution graph with on job vertex, parallelism 2
+		final ExecutionGraph graph = new ExecutionGraph(
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				jid,
+				"test job",
+				new Configuration(),
+				new SerializedValue<>(new ExecutionConfig()),
+				Time.seconds(10),
+				restartStrategy,
+				failoverStrategy,
+				Collections.<BlobKey>emptyList(),
+				Collections.<URL>emptyList(),
+				slotProvider,
+				getClass().getClassLoader());
+
+		JobVertex jv = new JobVertex("test vertex");
+		jv.setInvokableClass(NoOpInvokable.class);
+		jv.setParallelism(parallelism);
+
+		JobGraph jg = new JobGraph(jid, "testjob", jv);
+		graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
+
+		return graph;
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static class FailoverPipelinedRegionWithCustomExecutor implements Factory {
+
+		private final Executor executor;
+
+		FailoverPipelinedRegionWithCustomExecutor(Executor executor) {
+			this.executor = executor;
+		}
+
+		@Override
+		public FailoverStrategy create(ExecutionGraph executionGraph) {
+			return new RestartPipelinedRegionStrategy(executionGraph, executor);
+		}
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java
new file mode 100644
index 0000000000000..45aabe66ff90b
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.JobException;
+import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.executiongraph.failover.FailoverRegion;
+import org.apache.flink.runtime.executiongraph.failover.RestartPipelinedRegionStrategy;
+import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+public class RestartPipelinedRegionStrategyTest {
+
+	/**
+	 * Creates a JobGraph of the following form:
+	 * 
+	 * <pre>
+	 *  v1--->v2-->\
+	 *              \
+	 *               v4 --->\
+	 *        ----->/        \
+	 *  v3-->/                v5
+	 *       \               /
+	 *        ------------->/
+	 * </pre>
+	 */
+	@Test
+	public void testSimpleFailoverRegion() throws Exception {
+		
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+		final Configuration cfg = new Configuration();
+		
+		JobVertex v1 = new JobVertex("vertex1");
+		JobVertex v2 = new JobVertex("vertex2");
+		JobVertex v3 = new JobVertex("vertex3");
+		JobVertex v4 = new JobVertex("vertex4");
+		JobVertex v5 = new JobVertex("vertex5");
+		
+		v1.setParallelism(5);
+		v2.setParallelism(7);
+		v3.setParallelism(2);
+		v4.setParallelism(11);
+		v5.setParallelism(4);
+
+		v1.setInvokableClass(AbstractInvokable.class);
+		v2.setInvokableClass(AbstractInvokable.class);
+		v3.setInvokableClass(AbstractInvokable.class);
+		v4.setInvokableClass(AbstractInvokable.class);
+		v5.setInvokableClass(AbstractInvokable.class);
+
+		v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		
+		List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
+
+        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutor());
+		ExecutionGraph eg = new ExecutionGraph(
+			TestingUtils.defaultExecutor(),
+			TestingUtils.defaultExecutor(),
+			jobId, 
+			jobName, 
+			cfg,
+			new SerializedValue<>(new ExecutionConfig()),
+			AkkaUtils.getDefaultTimeout(),
+			new NoRestartStrategy(),
+            new RestartPipelinedRegionStrategy.Factory(),
+            Collections.<BlobKey>emptyList(),
+            Collections.<URL>emptyList(),
+            scheduler,
+            ExecutionGraph.class.getClassLoader());
+		try {
+			eg.attachJobGraph(ordered);
+		}
+		catch (JobException e) {
+			e.printStackTrace();
+			fail("Job failed with exception: " + e.getMessage());
+		}
+
+        RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+        ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
+        ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
+        ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
+        ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
+        ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
+        FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[2]);
+        FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[3]);
+        FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
+        FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[4]);
+        FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
+
+        assertEquals(region1, region2);
+        assertEquals(region3, region2);
+        assertEquals(region4, region2);
+        assertEquals(region5, region2);
+	}
+
+    /**
+     * Creates a JobGraph of the following form:
+     *
+     * <pre>
+     *  v2 ------->\
+     *              \
+     *  v1---------> v4 --->|\
+     *                        \
+     *                        v5
+     *                       /
+     *  v3--------------->|/
+     * </pre>
+     */
+	@Test
+	public void testMultipleFailoverRegions() throws Exception {
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+		final Configuration cfg = new Configuration();
+
+        JobVertex v1 = new JobVertex("vertex1");
+        JobVertex v2 = new JobVertex("vertex2");
+        JobVertex v3 = new JobVertex("vertex3");
+        JobVertex v4 = new JobVertex("vertex4");
+        JobVertex v5 = new JobVertex("vertex5");
+
+        v1.setParallelism(3);
+        v2.setParallelism(2);
+        v3.setParallelism(2);
+        v4.setParallelism(5);
+        v5.setParallelism(2);
+
+        v1.setInvokableClass(AbstractInvokable.class);
+        v2.setInvokableClass(AbstractInvokable.class);
+        v3.setInvokableClass(AbstractInvokable.class);
+        v4.setInvokableClass(AbstractInvokable.class);
+        v5.setInvokableClass(AbstractInvokable.class);
+
+        v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+        v4.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+        v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+        v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+
+        List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
+
+        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutor());
+		ExecutionGraph eg = new ExecutionGraph(
+			TestingUtils.defaultExecutor(),
+			TestingUtils.defaultExecutor(),
+			jobId, 
+			jobName, 
+			cfg,
+			new SerializedValue<>(new ExecutionConfig()),
+			AkkaUtils.getDefaultTimeout(),
+			new NoRestartStrategy(),
+            new RestartPipelinedRegionStrategy.Factory(),
+            Collections.<BlobKey>emptyList(),
+            Collections.<URL>emptyList(),
+            scheduler,
+            ExecutionGraph.class.getClassLoader());
+		try {
+			eg.attachJobGraph(ordered);
+		}
+		catch (JobException e) {
+			e.printStackTrace();
+			fail("Job failed with exception: " + e.getMessage());
+		}
+
+        // All in one failover region
+        RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+        ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
+        ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
+        ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
+        ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
+        ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
+        FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
+        FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
+        FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
+        FailoverRegion region31 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
+        FailoverRegion region32 = strategy.getFailoverRegion(ejv3.getTaskVertices()[1]);
+        FailoverRegion region51 = strategy.getFailoverRegion(ejv5.getTaskVertices()[0]);
+        FailoverRegion region52 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
+
+        //There should be 5 failover regions. v1 v2 v4 in one, v3 has two, v5 has two
+        assertEquals(region1, region2);
+        assertEquals(region2, region4);
+        assertFalse(region31.equals(region32));
+        assertFalse(region51.equals(region52));
+	}
+
+    /**
+     * Creates a JobGraph of the following form:
+     *
+     * <pre>
+     *  v1--->v2-->\
+     *              \
+     *               v4 --->|\
+     *        ----->/        \
+     *  v3-->/                v5
+     *       \               /
+     *        ------------->/
+     * </pre>
+     */
+	@Test
+	public void testSingleRegionWithMixedInput() throws Exception {
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+		final Configuration cfg = new Configuration();
+
+        JobVertex v1 = new JobVertex("vertex1");
+        JobVertex v2 = new JobVertex("vertex2");
+        JobVertex v3 = new JobVertex("vertex3");
+        JobVertex v4 = new JobVertex("vertex4");
+        JobVertex v5 = new JobVertex("vertex5");
+
+        v1.setParallelism(3);
+        v2.setParallelism(2);
+        v3.setParallelism(2);
+        v4.setParallelism(5);
+        v5.setParallelism(2);
+
+        v1.setInvokableClass(AbstractInvokable.class);
+        v2.setInvokableClass(AbstractInvokable.class);
+        v3.setInvokableClass(AbstractInvokable.class);
+        v4.setInvokableClass(AbstractInvokable.class);
+        v5.setInvokableClass(AbstractInvokable.class);
+
+        v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+        v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+        v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+        v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+        v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+
+        List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
+
+        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutor());
+		ExecutionGraph eg = new ExecutionGraph(
+			TestingUtils.defaultExecutor(),
+			TestingUtils.defaultExecutor(),
+			jobId, 
+			jobName, 
+			cfg,
+			new SerializedValue<>(new ExecutionConfig()),
+			AkkaUtils.getDefaultTimeout(),
+			new NoRestartStrategy(),
+            new RestartPipelinedRegionStrategy.Factory(),
+            Collections.<BlobKey>emptyList(),
+            Collections.<URL>emptyList(),
+            scheduler,
+            ExecutionGraph.class.getClassLoader());
+		try {
+			eg.attachJobGraph(ordered);
+		}
+		catch (JobException e) {
+			e.printStackTrace();
+			fail("Job failed with exception: " + e.getMessage());
+		}
+
+        // All in one failover region
+        RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+        ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
+        ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
+        ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
+        ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
+        ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
+        FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
+        FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
+        FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
+        FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
+        FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);
+
+        assertEquals(region1, region2);
+        assertEquals(region2, region4);
+        assertEquals(region3, region2);
+        assertEquals(region1, region5);
+    }
+
+    /**
+     * Creates a JobGraph of the following form:
+     *
+     * <pre>
+     *  v1-->v2-->|\
+     *              \
+     *               v4
+     *             /
+     *  v3------>/
+     * </pre>
+     */
+	@Test
+	public void testMultiRegionNotAllToAll() throws Exception {
+		final JobID jobId = new JobID();
+		final String jobName = "Test Job Sample Name";
+		final Configuration cfg = new Configuration();
+
+        JobVertex v1 = new JobVertex("vertex1");
+        JobVertex v2 = new JobVertex("vertex2");
+        JobVertex v3 = new JobVertex("vertex3");
+        JobVertex v4 = new JobVertex("vertex4");
+        JobVertex v5 = new JobVertex("vertex5");
+
+        v1.setParallelism(2);
+        v2.setParallelism(2);
+        v3.setParallelism(5);
+        v4.setParallelism(5);
+
+        v1.setInvokableClass(AbstractInvokable.class);
+        v2.setInvokableClass(AbstractInvokable.class);
+        v3.setInvokableClass(AbstractInvokable.class);
+        v4.setInvokableClass(AbstractInvokable.class);
+
+        v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+        v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+        v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+
+        List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4));
+
+        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutor());
+        ExecutionGraph eg = new ExecutionGraph(
+			TestingUtils.defaultExecutor(),
+			TestingUtils.defaultExecutor(),
+			jobId, 
+			jobName, 
+			cfg,
+			new SerializedValue<>(new ExecutionConfig()),
+			AkkaUtils.getDefaultTimeout(),
+			new NoRestartStrategy(),
+            new RestartPipelinedRegionStrategy.Factory(),
+            Collections.<BlobKey>emptyList(),
+            Collections.<URL>emptyList(),
+            scheduler,
+            ExecutionGraph.class.getClassLoader());
+		try {
+			eg.attachJobGraph(ordered);
+		}
+		catch (JobException e) {
+			e.printStackTrace();
+			fail("Job failed with exception: " + e.getMessage());
+		}
+
+        // All in one failover region
+        RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
+        ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
+        ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
+        ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
+        ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
+        FailoverRegion region11 = strategy.getFailoverRegion(ejv1.getTaskVertices()[0]);
+        FailoverRegion region12 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
+        FailoverRegion region21 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
+        FailoverRegion region22 = strategy.getFailoverRegion(ejv2.getTaskVertices()[1]);
+        FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
+        FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
+
+        //There should be 3 failover regions. v1 v2 in two, v3 and v4 in one
+        assertEquals(region11, region21);
+        assertEquals(region12, region22);
+        assertFalse(region11.equals(region12));
+        assertFalse(region3.equals(region4));
+	}
+
+}

From bda3f88d4655c5ccd6789cc605e7cce2cde6d52b Mon Sep 17 00:00:00 2001
From: "shuai.xus" <shuai.xus@alibaba-inc.com>
Date: Thu, 20 Apr 2017 23:56:53 +0200
Subject: [PATCH 174/253] [FLINK-5867] [flip-1] Add tests for pipelined
 failover region construction

---
 .../PipelinedFailoverRegionBuildingTest.java  | 644 ++++++++++++++++++
 1 file changed, 644 insertions(+)
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java
new file mode 100644
index 0000000000000..55bf7114ea34e
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java
@@ -0,0 +1,644 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph.failover;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.JobException;
+import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests that make sure that the building of pipelined connected failover regions works
+ * correctly.
+ */
+public class PipelinedFailoverRegionBuildingTest extends TestLogger {
+
+	/**
+	 * Tests that validates that a graph with single unconnected vertices works correctly.
+	 * 
+	 * <pre>
+	 *     (v1)
+	 *     
+	 *     (v2)
+	 *     
+	 *     (v3)
+	 *     
+	 *     ...
+	 * </pre>
+	 */
+	@Test
+	public void testIndividualVertices() throws Exception {
+		final JobVertex source1 = new JobVertex("source1");
+		source1.setInvokableClass(NoOpInvokable.class);
+		source1.setParallelism(2);
+
+		final JobVertex source2 = new JobVertex("source2");
+		source2.setInvokableClass(NoOpInvokable.class);
+		source2.setParallelism(2);
+
+		final JobGraph jobGraph = new JobGraph("test job", source1, source2);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+		FailoverRegion sourceRegion11 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[0]);
+		FailoverRegion sourceRegion12 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[1]);
+		FailoverRegion targetRegion21 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[0]);
+		FailoverRegion targetRegion22 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[1]);
+
+		assertTrue(sourceRegion11 != sourceRegion12);
+		assertTrue(sourceRegion12 != targetRegion21);
+		assertTrue(targetRegion21 != targetRegion22);
+	}
+
+	/**
+	 * Tests that validates that embarrassingly parallel chains of vertices work correctly.
+	 * 
+	 * <pre>
+	 *     (a1) --> (b1)
+	 *
+	 *     (a2) --> (b2)
+	 *
+	 *     (a3) --> (b3)
+	 *
+	 *     ...
+	 * </pre>
+	 */
+	@Test
+	public void testEmbarrassinglyParallelCase() throws Exception {
+		int parallelism = 10000;
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(parallelism);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(parallelism);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(parallelism);
+
+		vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+		vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+		FailoverRegion preRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[0]);
+		FailoverRegion preRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]);
+		FailoverRegion preRegion3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]);
+
+		assertTrue(preRegion1 == preRegion2);
+		assertTrue(preRegion2 == preRegion3);
+
+		for (int i = 1; i < parallelism; ++i) {
+			FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[i]);
+			FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[i]);
+			FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[i]);
+
+			assertTrue(region1 == region2);
+			assertTrue(region2 == region3);
+
+			assertTrue(preRegion1 != region1);
+		}
+	}
+
+	/**
+	 * Tests that validates that a single pipelined component via a sequence of all-to-all
+	 * connections works correctly.
+	 * 
+	 * <pre>
+	 *     (a1) -+-> (b1) -+-> (c1) 
+	 *           X         X
+	 *     (a2) -+-> (b2) -+-> (c2)
+	 *           X         X
+	 *     (a3) -+-> (b3) -+-> (c3)
+	 *
+	 *     ...
+	 * </pre>
+	 */
+	@Test
+	public void testOneComponentViaTwoExchanges() throws Exception {
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(3);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(5);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(2);
+
+		vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+		FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]);
+		FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[4]);
+		FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]);
+
+		assertTrue(region1 == region2);
+		assertTrue(region2 == region3);
+	}
+
+	/**
+	 * Tests that validates that a single pipelined component via a cascade of joins
+	 * works correctly.
+	 * 
+	 * <p>Non-parallelized view:
+	 * <pre>
+	 *     (1)--+
+	 *          +--(5)-+
+	 *     (2)--+      |
+	 *                 +--(7)
+	 *     (3)--+      |
+	 *          +--(6)-+
+	 *     (4)--+
+	 *     ...
+	 * </pre>
+	 */
+	@Test
+	public void testOneComponentViaCascadeOfJoins() throws Exception {
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(8);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(8);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(8);
+
+		final JobVertex vertex4 = new JobVertex("vertex4");
+		vertex4.setInvokableClass(NoOpInvokable.class);
+		vertex4.setParallelism(8);
+
+		final JobVertex vertex5 = new JobVertex("vertex5");
+		vertex5.setInvokableClass(NoOpInvokable.class);
+		vertex5.setParallelism(4);
+
+		final JobVertex vertex6 = new JobVertex("vertex6");
+		vertex6.setInvokableClass(NoOpInvokable.class);
+		vertex6.setParallelism(4);
+
+		final JobVertex vertex7 = new JobVertex("vertex7");
+		vertex7.setInvokableClass(NoOpInvokable.class);
+		vertex7.setParallelism(2);
+
+		vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+
+		Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator();
+
+		FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next());
+
+		while (evs.hasNext()) {
+			FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next());
+			assertTrue(preRegion == region);
+		}
+	}
+
+	/**
+	 * Tests that validates that a single pipelined component instance from one source
+	 * works correctly.
+	 * 
+	 * <p>Non-parallelized view:
+	 * <pre>
+	 *                 +--(1)
+	 *          +--(5)-+
+	 *          |      +--(2)
+	 *     (7)--+
+	 *          |      +--(3)
+	 *          +--(6)-+
+	 *                 +--(4)
+	 *     ...
+	 * </pre>
+	 */
+	@Test
+	public void testOneComponentInstanceFromOneSource() throws Exception {
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(8);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(8);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(8);
+
+		final JobVertex vertex4 = new JobVertex("vertex4");
+		vertex4.setInvokableClass(NoOpInvokable.class);
+		vertex4.setParallelism(8);
+
+		final JobVertex vertex5 = new JobVertex("vertex5");
+		vertex5.setInvokableClass(NoOpInvokable.class);
+		vertex5.setParallelism(4);
+
+		final JobVertex vertex6 = new JobVertex("vertex6");
+		vertex6.setInvokableClass(NoOpInvokable.class);
+		vertex6.setParallelism(4);
+
+		final JobVertex vertex7 = new JobVertex("vertex7");
+		vertex7.setInvokableClass(NoOpInvokable.class);
+		vertex7.setParallelism(2);
+
+		vertex1.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex2.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		vertex3.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex4.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		vertex5.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex6.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex7, vertex5, vertex6, vertex1, vertex2, vertex3, vertex4);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+
+		Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator();
+
+		FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next());
+
+		while (evs.hasNext()) {
+			FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next());
+			assertTrue(preRegion == region);
+		}
+	}
+
+	/**
+	 * <pre>
+	 *     (a1) -+-> (b1) -+-> (c1) 
+	 *           X
+	 *     (a2) -+-> (b2) -+-> (c2)
+	 *           X
+	 *     (a3) -+-> (b3) -+-> (c3)
+	 *
+	 *           ^         ^
+	 *           |         |
+	 *     (pipelined) (blocking)
+	 *
+	 * </pre>
+	 */
+	@Test
+	public void testTwoComponentsViaBlockingExchange() throws Exception {
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(3);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(2);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(2);
+
+		vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+		FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]);
+		FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]);
+		FailoverRegion region31 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]);
+		FailoverRegion region32 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[1]);
+
+		assertTrue(region1 == region2);
+		assertTrue(region2 != region31);
+		assertTrue(region32 != region31);
+	}
+
+	/**
+	 * <pre>
+	 *     (a1) -+-> (b1) -+-> (c1) 
+	 *           X         X
+	 *     (a2) -+-> (b2) -+-> (c2)
+	 *           X         X
+	 *     (a3) -+-> (b3) -+-> (c3)
+	 *
+	 *           ^         ^
+	 *           |         |
+	 *     (pipelined) (blocking)
+	 * </pre>
+	 */
+	@Test
+	public void testTwoComponentsViaBlockingExchange2() throws Exception {
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(3);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(2);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(2);
+
+		vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+		FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]);
+		FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]);
+		FailoverRegion region31 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]);
+		FailoverRegion region32 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[1]);
+
+		assertTrue(region1 == region2);
+		assertTrue(region2 != region31);
+		assertTrue(region32 != region31);
+	}
+
+	/**
+	 * Cascades of joins with partially blocking, partially pipelined exchanges:
+	 * <pre>
+	 *     (1)--+
+	 *          +--(5)-+
+	 *     (2)--+      |
+	 *              (block)
+	 *                 |
+	 *                 +--(7)
+	 *                 |
+	 *              (block)
+	 *     (3)--+      |
+	 *          +--(6)-+
+	 *     (4)--+
+	 *     ...
+	 * </pre>
+	 *
+	 * Component 1: 1, 2, 5; component 2: 3,4,6; component 3: 7 
+	 */
+	@Test
+	public void testMultipleComponentsViaCascadeOfJoins() throws Exception {
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(8);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(8);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(8);
+
+		final JobVertex vertex4 = new JobVertex("vertex4");
+		vertex4.setInvokableClass(NoOpInvokable.class);
+		vertex4.setParallelism(8);
+
+		final JobVertex vertex5 = new JobVertex("vertex5");
+		vertex5.setInvokableClass(NoOpInvokable.class);
+		vertex5.setParallelism(4);
+
+		final JobVertex vertex6 = new JobVertex("vertex6");
+		vertex6.setInvokableClass(NoOpInvokable.class);
+		vertex6.setParallelism(4);
+
+		final JobVertex vertex7 = new JobVertex("vertex7");
+		vertex7.setInvokableClass(NoOpInvokable.class);
+		vertex7.setParallelism(2);
+
+		vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+		vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+
+		FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[0]);
+		FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[5]);
+		FailoverRegion region5 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex5.getID()).getTaskVertices()[2]);
+
+		assertTrue(region1 == region2);
+		assertTrue(region1 == region5);
+
+		FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]);
+		FailoverRegion region4 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex4.getID()).getTaskVertices()[5]);
+		FailoverRegion region6 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex6.getID()).getTaskVertices()[2]);
+
+		assertTrue(region3 == region4);
+		assertTrue(region3 == region6);
+
+		FailoverRegion region71 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[0]);
+		FailoverRegion region72 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[1]);
+
+		assertTrue(region71 != region72);
+		assertTrue(region1 != region71);
+		assertTrue(region1 != region72);
+		assertTrue(region3 != region71);
+		assertTrue(region3 != region72);
+	}
+
+	@Test
+	public void testDiamondWithMixedPipelinedAndBlockingExchanges() throws Exception {
+		final JobVertex vertex1 = new JobVertex("vertex1");
+		vertex1.setInvokableClass(NoOpInvokable.class);
+		vertex1.setParallelism(8);
+
+		final JobVertex vertex2 = new JobVertex("vertex2");
+		vertex2.setInvokableClass(NoOpInvokable.class);
+		vertex2.setParallelism(8);
+
+		final JobVertex vertex3 = new JobVertex("vertex3");
+		vertex3.setInvokableClass(NoOpInvokable.class);
+		vertex3.setParallelism(8);
+
+		final JobVertex vertex4 = new JobVertex("vertex4");
+		vertex4.setInvokableClass(NoOpInvokable.class);
+		vertex4.setParallelism(8);
+
+		vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+		vertex3.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		vertex4.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+		vertex4.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+
+		final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+
+		Iterator<ExecutionVertex> evs = eg.getAllExecutionVertices().iterator();
+
+		FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next());
+
+		while (evs.hasNext()) {
+			FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next());
+			assertTrue(preRegion == region);
+		}
+	}
+
+	/**
+	 * This test checks that are strictly co-located vertices are in the same failover region,
+	 * even through they are connected via a blocking pattern.
+	 * This is currently an assumption / limitation of the scheduler.
+	 */
+	@Test
+	public void testBlockingAllToAllTopologyWithCoLocation() throws Exception {
+		final JobVertex source = new JobVertex("source");
+		source.setInvokableClass(NoOpInvokable.class);
+		source.setParallelism(10);
+
+		final JobVertex target = new JobVertex("target");
+		target.setInvokableClass(NoOpInvokable.class);
+		target.setParallelism(13);
+
+		target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
+
+		final SlotSharingGroup sharingGroup = new SlotSharingGroup();
+		source.setSlotSharingGroup(sharingGroup);
+		target.setSlotSharingGroup(sharingGroup);
+
+		source.setStrictlyCoLocatedWith(target);
+
+		final JobGraph jobGraph = new JobGraph("test job", source, target);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+		FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
+		FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
+
+		// we use 'assertTrue' here rather than 'assertEquals' because we want to test
+		// for referential equality, to be on the safe side
+		assertTrue(region1 == region2);
+	}
+
+	/**
+	 * This test checks that are strictly co-located vertices are in the same failover region,
+	 * even through they are connected via a blocking pattern.
+	 * This is currently an assumption / limitation of the scheduler.
+	 */
+	@Test
+	public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception {
+		final JobVertex source = new JobVertex("source");
+		source.setInvokableClass(NoOpInvokable.class);
+		source.setParallelism(10);
+
+		final JobVertex target = new JobVertex("target");
+		target.setInvokableClass(NoOpInvokable.class);
+		target.setParallelism(10);
+
+		target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+
+		final SlotSharingGroup sharingGroup = new SlotSharingGroup();
+		source.setSlotSharingGroup(sharingGroup);
+		target.setSlotSharingGroup(sharingGroup);
+
+		source.setStrictlyCoLocatedWith(target);
+
+		final JobGraph jobGraph = new JobGraph("test job", source, target);
+		final ExecutionGraph eg = createExecutionGraph(jobGraph);
+
+		RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
+		FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
+		FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]);
+		FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
+		FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]);
+
+		// we use 'assertTrue' here rather than 'assertEquals' because we want to test
+		// for referential equality, to be on the safe side
+		assertTrue(sourceRegion1 == sourceRegion2);
+		assertTrue(sourceRegion2 == targetRegion1);
+		assertTrue(targetRegion1 == targetRegion2);
+	}
+
+	// ------------------------------------------------------------------------
+	//  utilities
+	// ------------------------------------------------------------------------
+
+	private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
+		// configure the pipelined failover strategy
+		final Configuration jobManagerConfig = new Configuration();
+		jobManagerConfig.setString(
+				JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
+				FailoverStrategyLoader.PIPELINED_REGION_RESTART_STRATEGY_NAME);
+
+		return ExecutionGraphBuilder.buildGraph(
+				null,
+				jobGraph,
+				jobManagerConfig,
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				mock(SlotProvider.class),
+				PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
+				new StandaloneCheckpointRecoveryFactory(),
+				Time.seconds(10),
+				new NoRestartStrategy(),
+				new UnregisteredMetricsGroup(),
+				1000,
+				log);
+	}
+}

From f6275d880ea69fe90556cb635352b7d2c2397247 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Fri, 21 Apr 2017 00:02:19 +0200
Subject: [PATCH 175/253] [FLINK-5867] [flip-1] Improve performance of
 Pipelined Failover Region construction

This method exploits the fact that verties are already in topological order.

This closes #3773
---
 .../RestartPipelinedRegionStrategy.java       | 170 ++++++++++++------
 1 file changed, 115 insertions(+), 55 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java
index 0a5baa87c0a53..1884d1c80e838 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java
@@ -24,14 +24,16 @@
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
-import org.apache.flink.runtime.executiongraph.IntermediateResultPartition;
+import org.apache.flink.runtime.executiongraph.IntermediateResult;
 import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.FlinkRuntimeException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.concurrent.Executor;
 
@@ -91,16 +93,20 @@ public void onTaskFailure(Execution taskExecution, Throwable cause) {
 
 		if (failoverRegion == null) {
 			executionGraph.failGlobal(new FlinkException(
-					"Can not find a failover region for the execution " + ev.getTaskNameWithSubtaskIndex()));
+					"Can not find a failover region for the execution " + ev.getTaskNameWithSubtaskIndex(), cause));
 		}
 		else {
+			LOG.info("Recovering task failure for {} #{} ({}) via restart of failover region",
+					taskExecution.getVertex().getTaskNameWithSubtaskIndex(),
+					taskExecution.getAttemptNumber(),
+					taskExecution.getAttemptId());
+
 			failoverRegion.onExecutionFail(taskExecution, cause);
 		}
 	}
 
 	@Override
 	public void notifyNewVertices(List<ExecutionJobVertex> newJobVerticesTopological) {
-		LOG.debug("Generating failover regions for {} new job vertices", newJobVerticesTopological.size());
 		generateAllFailoverRegion(newJobVerticesTopological);
 	}
 
@@ -109,74 +115,128 @@ public String getStrategyName() {
 		return "Pipelined Region Failover";
 	}
 
-	// Generate all the FailoverRegion from the new added job vertexes
+	/**
+	 * Generate all the FailoverRegion from the new added job vertexes
+ 	 */
 	private void generateAllFailoverRegion(List<ExecutionJobVertex> newJobVerticesTopological) {
+		final IdentityHashMap<ExecutionVertex, ArrayList<ExecutionVertex>> vertexToRegion = new IdentityHashMap<>();
+
+		// we use the map (list -> null) to imitate an IdentityHashSet (which does not exist)
+		final IdentityHashMap<ArrayList<ExecutionVertex>, Object> distinctRegions = new IdentityHashMap<>();
+
+		// this loop will worst case iterate over every edge in the graph (complexity is O(#edges))
+		
 		for (ExecutionJobVertex ejv : newJobVerticesTopological) {
-			for (ExecutionVertex ev : ejv.getTaskVertices()) {
-				if (getFailoverRegion(ev) != null) {
-					continue;
-				}
-				List<ExecutionVertex> pipelinedExecutions = new ArrayList<>();
-				List<ExecutionVertex> orgExecutions = new ArrayList<>();
-				orgExecutions.add(ev);
-				pipelinedExecutions.add(ev);
-				getAllPipelinedConnectedVertexes(orgExecutions, pipelinedExecutions);
-
-				FailoverRegion region = new FailoverRegion(executionGraph, executor, pipelinedExecutions);
-				for (ExecutionVertex vertex : pipelinedExecutions) {
-					vertexToRegion.put(vertex, region);
+
+			// currently, jobs with a co-location constraint fail as one
+			// we want to improve that in the future (or get rid of co-location constraints)
+			if (ejv.getCoLocationGroup() != null) {
+				makeAllOneRegion(newJobVerticesTopological);
+				return;
+			}
+
+			// see if this JobVertex one has pipelined inputs at all
+			final List<IntermediateResult> inputs = ejv.getInputs();
+			final int numInputs = inputs.size();
+			boolean hasPipelinedInputs = false;
+
+			for (IntermediateResult input : inputs) {
+				if (input.getResultType().isPipelined()) {
+					hasPipelinedInputs = true;
+					break;
 				}
 			}
-		}
-	}
 
-	/**
-	 * Get all connected executions of the original executions
-	 *
-	 * @param orgExecutions  the original execution vertexes
-	 * @param connectedExecutions  the total connected executions
-	 */
-	private static void getAllPipelinedConnectedVertexes(List<ExecutionVertex> orgExecutions, List<ExecutionVertex> connectedExecutions) {
-		List<ExecutionVertex> newAddedExecutions = new ArrayList<>();
-		for (ExecutionVertex ev : orgExecutions) {
-			// Add downstream ExecutionVertex
-			for (IntermediateResultPartition irp : ev.getProducedPartitions().values()) {
-				if (irp.getIntermediateResult().getResultType().isPipelined()) {
-					for (List<ExecutionEdge> consumers : irp.getConsumers()) {
-						for (ExecutionEdge consumer : consumers) {
-							ExecutionVertex cev = consumer.getTarget();
-							if (!connectedExecutions.contains(cev)) {
-								newAddedExecutions.add(cev);
+			if (hasPipelinedInputs) {
+				// build upon the predecessors
+				for (ExecutionVertex ev : ejv.getTaskVertices()) {
+
+					// remember the region in which we are
+					ArrayList<ExecutionVertex> thisRegion = null;
+
+					for (int inputNum = 0; inputNum < numInputs; inputNum++) {
+						if (inputs.get(inputNum).getResultType().isPipelined()) {
+
+							for (ExecutionEdge edge : ev.getInputEdges(inputNum)) {
+								final ExecutionVertex predecessor = edge.getSource().getProducer();
+								final ArrayList<ExecutionVertex> predecessorRegion = vertexToRegion.get(predecessor);
+
+								if (thisRegion != null) {
+									// we already have a region. see if it is the same as the predecessor's region
+									if (predecessorRegion != thisRegion) {
+
+										// we need to merge our region and the predecessor's region
+										thisRegion.addAll(predecessorRegion);
+										distinctRegions.remove(predecessorRegion);
+
+										// remap the vertices from that merged region
+										for (ExecutionVertex inPredRegion: predecessorRegion) {
+											vertexToRegion.put(inPredRegion, thisRegion);
+										}
+									}
+								}
+								else if (predecessor != null) {
+									// first case, make this our region
+									thisRegion = predecessorRegion;
+									thisRegion.add(ev);
+									vertexToRegion.put(ev, thisRegion);
+								}
+								else {
+									// throw an uncaught exception here
+									// this is a bug and not a recoverable situation
+									throw new FlinkRuntimeException(
+											"bug in the logic to construct the pipelined failover regions");
+								}
 							}
 						}
 					}
 				}
 			}
-			if (!newAddedExecutions.isEmpty()) {
-				connectedExecutions.addAll(newAddedExecutions);
-				getAllPipelinedConnectedVertexes(newAddedExecutions, connectedExecutions);
-				newAddedExecutions.clear();
-			}
-			// Add upstream ExecutionVertex
-			int inputNum = ev.getNumberOfInputs();
-			for (int i = 0; i < inputNum; i++) {
-				for (ExecutionEdge input : ev.getInputEdges(i)) {
-					if (input.getSource().getIntermediateResult().getResultType().isPipelined()) {
-						ExecutionVertex pev = input.getSource().getProducer();
-						if (!connectedExecutions.contains(pev)) {
-							newAddedExecutions.add(pev);
-						}
-					}
+			else {
+				// no pipelined inputs, start a new region
+				for (ExecutionVertex ev : ejv.getTaskVertices()) {
+					ArrayList<ExecutionVertex> region = new ArrayList<>(1);
+					region.add(ev);
+					vertexToRegion.put(ev, region);
+					distinctRegions.put(region, null);
 				}
 			}
-			if (!newAddedExecutions.isEmpty()) {
-				connectedExecutions.addAll(0, newAddedExecutions);
-				getAllPipelinedConnectedVertexes(newAddedExecutions, connectedExecutions);
-				newAddedExecutions.clear();
+		}
+
+		// now that we have all regions, create the failover region objects 
+		LOG.info("Creating {} individual failover regions for job {} ({})",
+				executionGraph.getJobName(), executionGraph.getJobID());
+
+		for (List<ExecutionVertex> region : distinctRegions.keySet()) {
+			final FailoverRegion failoverRegion = new FailoverRegion(executionGraph, executor, region);
+			for (ExecutionVertex ev : region) {
+				this.vertexToRegion.put(ev, failoverRegion);
 			}
 		}
 	}
 
+	private void makeAllOneRegion(List<ExecutionJobVertex> jobVertices) {
+		LOG.warn("Cannot decompose ExecutionGraph into individual failover regions due to use of " +
+				"Co-Location constraints (iterations). Job will fail over as one holistic unit.");
+
+		final ArrayList<ExecutionVertex> allVertices = new ArrayList<>();
+
+		for (ExecutionJobVertex ejv : jobVertices) {
+
+			// safe some incremental size growing
+			allVertices.ensureCapacity(allVertices.size() + ejv.getParallelism());
+
+			for (ExecutionVertex ev : ejv.getTaskVertices()) {
+				allVertices.add(ev);
+			}
+		}
+
+		final FailoverRegion singleRegion = new FailoverRegion(executionGraph, executor, allVertices);
+		for (ExecutionVertex ev : allVertices) {
+			vertexToRegion.put(ev, singleRegion);
+		}
+	}
+
 	// ------------------------------------------------------------------------
 	//  testing
 	// ------------------------------------------------------------------------

From 5f645e4c1ac555cd585010f92797de7d663ff0f3 Mon Sep 17 00:00:00 2001
From: "shuai.xus" <shuai.xus@alibaba-inc.com>
Date: Tue, 25 Apr 2017 15:42:48 +0800
Subject: [PATCH 176/253] [hotfix] [runtime] Correct some JavaDocs for
 RestartIndividualStrategy

---
 .../executiongraph/failover/RestartIndividualStrategy.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java
index 0a449b83d2e16..0e7bca55cf1b5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartIndividualStrategy.java
@@ -60,7 +60,7 @@ public class RestartIndividualStrategy extends FailoverStrategy {
 	private final SimpleCounter numTaskFailures;
 
 	/**
-	 * Creates a new failover strategy that recovers from failures by restarting all tasks
+	 * Creates a new failover strategy that recovers from failures by restarting only the failed task
 	 * of the execution graph.
 	 * 
 	 * <p>The strategy will use the ExecutionGraph's future executor for callbacks.
@@ -72,7 +72,7 @@ public RestartIndividualStrategy(ExecutionGraph executionGraph) {
 	}
 
 	/**
-	 * Creates a new failover strategy that recovers from failures by restarting all tasks
+	 * Creates a new failover strategy that recovers from failures by restarting only the failed task
 	 * of the execution graph.
 	 *
 	 * @param executionGraph The execution graph to handle.

From 32a013311f40e3130cfc43da6e9c94270d9721e6 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Sun, 7 May 2017 01:51:29 +0200
Subject: [PATCH 177/253] [hotfix] [core] Avoid Java 8 specific functions in
 MemorySize

---
 .../org/apache/flink/configuration/MemorySize.java     | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
index c34c721e9553a..800a2169941fc 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
@@ -212,13 +212,15 @@ else if (matchesAny(unit, TERA_BYTES_UNITS)) {
 			}
 		}
 
-		try {
-			return Math.multiplyExact(value, multiplier);
-		}
-		catch (ArithmeticException e) {
+		final long result = value * multiplier;
+
+		// check for overflow
+		if (result / multiplier != value) {
 			throw new IllegalArgumentException("The value '" + text + 
 					"' cannot be re represented as 64bit number of bytes (numeric overflow).");
 		}
+
+		return result;
 	}
 
 	private static boolean matchesAny(String str, String[] variants) {

From 703be93adcb209d59a3726485c50efa37c22353f Mon Sep 17 00:00:00 2001
From: Tony Wei <tony19920430@gmail.com>
Date: Wed, 14 Dec 2016 10:18:25 +0800
Subject: [PATCH 178/253] [FLINK-4821] Implement rescalable non-partitioned
 state for Kinesis Connector

---
 .../flink-connector-kinesis/pom.xml           |  16 +
 .../kinesis/FlinkKinesisConsumer.java         | 150 ++++--
 .../kinesis/internals/KinesisDataFetcher.java |   2 +-
 .../FlinkKinesisConsumerMigrationTest.java    | 149 ++++++
 .../kinesis/FlinkKinesisConsumerTest.java     | 440 ++++++++++++++++--
 ...-consumer-migration-test-flink1.1-snapshot | Bin 0 -> 1140 bytes
 ...mer-migration-test-flink1.1-snapshot-empty | Bin 0 -> 468 bytes
 7 files changed, 690 insertions(+), 67 deletions(-)
 create mode 100644 flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
 create mode 100644 flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.1-snapshot
 create mode 100644 flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.1-snapshot-empty

diff --git a/flink-connectors/flink-connector-kinesis/pom.xml b/flink-connectors/flink-connector-kinesis/pom.xml
index d45719936c3b2..080626f053ed8 100644
--- a/flink-connectors/flink-connector-kinesis/pom.xml
+++ b/flink-connectors/flink-connector-kinesis/pom.xml
@@ -55,6 +55,14 @@ under the License.
 			<version>${guava.version}</version>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-runtime_2.10</artifactId>
+			<version>${project.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-tests_2.10</artifactId>
@@ -63,6 +71,14 @@ under the License.
 			<type>test-jar</type>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-streaming-java_2.10</artifactId>
+			<version>${project.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-test-utils_2.10</artifactId>
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
index a62dc100c054b..dfcd552bc9450 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
@@ -17,15 +17,26 @@
 
 package org.apache.flink.streaming.connectors.kinesis;
 
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedRestoring;
 import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard;
-import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
+import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumber;
+import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
 import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
 import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
 import org.apache.flink.streaming.connectors.kinesis.util.KinesisConfigUtil;
@@ -55,8 +66,10 @@
  *
  * @param <T> the type of data emitted
  */
-public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T>
-	implements CheckpointedAsynchronously<HashMap<KinesisStreamShard, SequenceNumber>>, ResultTypeQueryable<T> {
+public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> implements
+	ResultTypeQueryable<T>,
+	CheckpointedFunction,
+	CheckpointedRestoring<HashMap<KinesisStreamShard, SequenceNumber>> {
 
 	private static final long serialVersionUID = 4724006128720664870L;
 
@@ -91,6 +104,14 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T>
 
 	private volatile boolean running = true;
 
+	// ------------------------------------------------------------------------
+	//  State for Checkpoint
+	// ------------------------------------------------------------------------
+
+	/** The name is the key for sequence numbers state, and cannot be changed. */
+	private static final String sequenceNumsStateStoreName = "Kinesis-Stream-Shard-State";
+
+	private transient ListState<Tuple2<KinesisStreamShard, SequenceNumber>> sequenceNumsStateForCheckpoint;
 
 	// ------------------------------------------------------------------------
 	//  Constructors
@@ -194,8 +215,7 @@ public void run(SourceContext<T> sourceContext) throws Exception {
 		// all subtasks will run a fetcher, regardless of whether or not the subtask will initially have
 		// shards to subscribe to; fetchers will continuously poll for changes in the shard list, so all subtasks
 		// can potentially have new shards to subscribe to later on
-		fetcher = new KinesisDataFetcher<>(
-			streams, sourceContext, getRuntimeContext(), configProps, deserializer);
+		fetcher = createFetcher(streams, sourceContext, getRuntimeContext(), configProps, deserializer);
 
 		boolean isRestoringFromFailure = (sequenceNumsToRestore != null);
 		fetcher.setIsRestoringFromFailure(isRestoringFromFailure);
@@ -203,17 +223,35 @@ public void run(SourceContext<T> sourceContext) throws Exception {
 		// if we are restoring from a checkpoint, we iterate over the restored
 		// state and accordingly seed the fetcher with subscribed shards states
 		if (isRestoringFromFailure) {
-			for (Map.Entry<KinesisStreamShard, SequenceNumber> restored : lastStateSnapshot.entrySet()) {
-				fetcher.advanceLastDiscoveredShardOfStream(
-					restored.getKey().getStreamName(), restored.getKey().getShard().getShardId());
-
-				if (LOG.isInfoEnabled()) {
-					LOG.info("Subtask {} is seeding the fetcher with restored shard {}," +
-							" starting state set to the restored sequence number {}",
-						getRuntimeContext().getIndexOfThisSubtask(), restored.getKey().toString(), restored.getValue());
+			// Since there may have a situation that some subtasks did not finish discovering before rescale,
+			// and KinesisDataFetcher will always discover the shard from the largest shard id. To prevent from
+			// missing some shards which didn't be discovered and whose id is not the largest one, we force the
+			// consumer to discover once from the smallest id and make sure each shard have its initial sequence
+			// number from restored state or SENTINEL_EARLIEST_SEQUENCE_NUM.
+			List<KinesisStreamShard> newShardsCreatedWhileNotRunning = fetcher.discoverNewShardsToSubscribe();
+			for (KinesisStreamShard shard : newShardsCreatedWhileNotRunning) {
+				SequenceNumber startingStateForNewShard;
+
+				if (lastStateSnapshot.containsKey(shard)) {
+					startingStateForNewShard = lastStateSnapshot.get(shard);
+
+					if (LOG.isInfoEnabled()) {
+						LOG.info("Subtask {} is seeding the fetcher with restored shard {}," +
+								" starting state set to the restored sequence number {}",
+							getRuntimeContext().getIndexOfThisSubtask(), shard.toString(), startingStateForNewShard);
+					}
+				} else {
+					startingStateForNewShard = SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get();
+
+					if (LOG.isInfoEnabled()) {
+						LOG.info("Subtask {} is seeding the fetcher with new discovered shard {}," +
+								" starting state set to the SENTINEL_EARLIEST_SEQUENCE_NUM",
+							getRuntimeContext().getIndexOfThisSubtask(), shard.toString());
+					}
 				}
+
 				fetcher.registerNewSubscribedShardState(
-					new KinesisStreamShardState(restored.getKey(), restored.getValue()));
+					new KinesisStreamShardState(shard, startingStateForNewShard));
 			}
 		}
 
@@ -267,38 +305,78 @@ public TypeInformation<T> getProducedType() {
 	// ------------------------------------------------------------------------
 
 	@Override
-	public HashMap<KinesisStreamShard, SequenceNumber> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
-		if (lastStateSnapshot == null) {
-			LOG.debug("snapshotState() requested on not yet opened source; returning null.");
-			return null;
-		}
+	public void initializeState(FunctionInitializationContext context) throws Exception {
+		TypeInformation<Tuple2<KinesisStreamShard, SequenceNumber>> tuple = new TupleTypeInfo<>(
+			TypeInformation.of(KinesisStreamShard.class),
+			TypeInformation.of(SequenceNumber.class)
+		);
+
+		sequenceNumsStateForCheckpoint = context.getOperatorStateStore().getUnionListState(
+			new ListStateDescriptor<>(sequenceNumsStateStoreName, tuple));
+
+		if (context.isRestored()) {
+			if (sequenceNumsToRestore == null) {
+				sequenceNumsToRestore = new HashMap<>();
+				for (Tuple2<KinesisStreamShard, SequenceNumber> kinesisSequenceNumber : sequenceNumsStateForCheckpoint.get()) {
+					sequenceNumsToRestore.put(kinesisSequenceNumber.f0, kinesisSequenceNumber.f1);
+				}
 
-		if (fetcher == null) {
-			LOG.debug("snapshotState() requested on not yet running source; returning null.");
-			return null;
+				LOG.info("Setting restore state in the FlinkKinesisConsumer. Using the following offsets: {}",
+					sequenceNumsToRestore);
+			} else if (sequenceNumsToRestore.isEmpty()) {
+				sequenceNumsToRestore = null;
+			}
+		} else {
+			LOG.info("No restore state for FlinkKinesisConsumer.");
 		}
+	}
 
-		if (!running) {
+	@Override
+	public void snapshotState(FunctionSnapshotContext context) throws Exception {
+		if (lastStateSnapshot == null) {
+			LOG.debug("snapshotState() requested on not yet opened source; returning null.");
+		} else if (fetcher == null) {
+			LOG.debug("snapshotState() requested on not yet running source; returning null.");
+		} else if (!running) {
 			LOG.debug("snapshotState() called on closed source; returning null.");
-			return null;
-		}
+		} else {
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("Snapshotting state ...");
+			}
 
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Snapshotting state ...");
-		}
+			sequenceNumsStateForCheckpoint.clear();
+			lastStateSnapshot = fetcher.snapshotState();
 
-		lastStateSnapshot = fetcher.snapshotState();
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("Snapshotted state, last processed sequence numbers: {}, checkpoint id: {}, timestamp: {}",
+					lastStateSnapshot.toString(), context.getCheckpointId(), context.getCheckpointTimestamp());
+			}
 
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Snapshotted state, last processed sequence numbers: {}, checkpoint id: {}, timestamp: {}",
-				lastStateSnapshot.toString(), checkpointId, checkpointTimestamp);
+			for (Map.Entry<KinesisStreamShard, SequenceNumber> entry : lastStateSnapshot.entrySet()) {
+				sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey(), entry.getValue()));
+			}
 		}
-
-		return lastStateSnapshot;
 	}
 
 	@Override
 	public void restoreState(HashMap<KinesisStreamShard, SequenceNumber> restoredState) throws Exception {
-		sequenceNumsToRestore = restoredState;
+		LOG.info("Subtask {} restoring offsets from an older Flink version: {}",
+			getRuntimeContext().getIndexOfThisSubtask(), sequenceNumsToRestore);
+
+		sequenceNumsToRestore = restoredState.isEmpty() ? null : restoredState;
+	}
+
+	/** This method is created for tests that can mock the KinesisDataFetcher in the consumer. */
+	protected KinesisDataFetcher<T> createFetcher(List<String> streams,
+													SourceFunction.SourceContext<T> sourceContext,
+													RuntimeContext runtimeContext,
+													Properties configProps,
+													KinesisDeserializationSchema<T> deserializationSchema) {
+		return new KinesisDataFetcher<>(streams, sourceContext, runtimeContext, configProps, deserializationSchema);
+	}
+
+	@VisibleForTesting
+	HashMap<KinesisStreamShard, SequenceNumber> getRestoredState() {
+		return sequenceNumsToRestore;
 	}
 }
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index 8f7ca6c40f1e4..c5b4b0495f1e6 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -461,7 +461,7 @@ public void advanceLastDiscoveredShardOfStream(String stream, String shardId) {
 	 * 3. Update the subscribedStreamsToLastDiscoveredShardIds state so that we won't get shards
 	 *    that we have already seen before the next time this function is called
 	 */
-	private List<KinesisStreamShard> discoverNewShardsToSubscribe() throws InterruptedException {
+	public List<KinesisStreamShard> discoverNewShardsToSubscribe() throws InterruptedException {
 
 		List<KinesisStreamShard> newShardsToSubscribe = new LinkedList<>();
 
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
new file mode 100644
index 0000000000000..2f46e0936c939
--- /dev/null
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kinesis;
+
+import com.amazonaws.services.kinesis.model.Shard;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.operators.StreamSource;
+import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
+import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
+import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard;
+import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
+import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
+import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness;
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for checking whether {@link FlinkKinesisConsumer} can restore from snapshots that were
+ * done using the Flink 1.1 {@link FlinkKinesisConsumer}.
+ *
+ * <p>For regenerating the binary snapshot file you have to run the commented out portion
+ * of each test on a checkout of the Flink 1.1 branch.
+ */
+public class FlinkKinesisConsumerMigrationTest {
+
+	@Test
+	public void testRestoreFromFlink11WithEmptyState() throws Exception {
+		Properties testConfig = new Properties();
+		testConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+
+		final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(testConfig);
+
+		StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
+
+		final AbstractStreamOperatorTestHarness<String> testHarness =
+			new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+		testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+		testHarness.setup();
+		// restore state from binary snapshot file using legacy method
+		testHarness.initializeStateFromLegacyCheckpoint(
+			getResourceFilename("kinesis-consumer-migration-test-flink1.1-snapshot-empty"));
+		testHarness.open();
+
+		// assert that no state was restored
+		assertEquals(null, consumerFunction.getRestoredState());
+
+		consumerOperator.close();
+		consumerOperator.cancel();
+	}
+
+	@Test
+	public void testRestoreFromFlink11() throws Exception {
+		Properties testConfig = new Properties();
+		testConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+
+		final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(testConfig);
+
+		StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator =
+			new StreamSource<>(consumerFunction);
+
+		final AbstractStreamOperatorTestHarness<String> testHarness =
+			new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+		testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+		testHarness.setup();
+		// restore state from binary snapshot file using legacy method
+		testHarness.initializeStateFromLegacyCheckpoint(
+			getResourceFilename("kinesis-consumer-migration-test-flink1.1-snapshot"));
+		testHarness.open();
+
+		// the expected state in "kafka-consumer-migration-test-flink1.1-snapshot"
+		final HashMap<KinesisStreamShard, SequenceNumber> expectedState = new HashMap<>();
+		expectedState.put(new KinesisStreamShard("fakeStream1",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
+			new SequenceNumber("987654321"));
+
+		// assert that state is correctly restored from legacy checkpoint
+		assertNotEquals(null, consumerFunction.getRestoredState());
+		assertEquals(1, consumerFunction.getRestoredState().size());
+		assertEquals(expectedState, consumerFunction.getRestoredState());
+
+		consumerOperator.close();
+		consumerOperator.cancel();
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static String getResourceFilename(String filename) {
+		ClassLoader cl = FlinkKinesisConsumerMigrationTest.class.getClassLoader();
+		URL resource = cl.getResource(filename);
+		if (resource == null) {
+			throw new NullPointerException("Missing snapshot resource.");
+		}
+		return resource.getFile();
+	}
+
+	private static class DummyFlinkKafkaConsumer<T> extends FlinkKinesisConsumer<T> {
+		private static final long serialVersionUID = 1L;
+
+		@SuppressWarnings("unchecked")
+		DummyFlinkKafkaConsumer(Properties properties) {
+			super("test", mock(KinesisDeserializationSchema.class), properties);
+		}
+
+		@Override
+		protected KinesisDataFetcher<T> createFetcher(List<String> streams,
+													  	SourceFunction.SourceContext<T> sourceContext,
+													  	RuntimeContext runtimeContext,
+													  	Properties configProps,
+													  	KinesisDeserializationSchema<T> deserializationSchema) {
+			return mock(KinesisDataFetcher.class);
+		}
+	}
+}
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
index 741f0ca689b64..bf8e44f5c1719 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
@@ -18,13 +18,22 @@
 package org.apache.flink.streaming.connectors.kinesis;
 
 import com.amazonaws.services.kinesis.model.Shard;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.state.OperatorStateStore;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants;
 import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
 import org.apache.flink.streaming.connectors.kinesis.config.ProducerConfigConstants;
 import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard;
+import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumber;
 import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
 import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
@@ -35,19 +44,29 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
+import org.mockito.Matchers;
 import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
 import org.powermock.api.mockito.PowerMockito;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
-import java.text.SimpleDateFormat;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Properties;
 import java.util.UUID;
+import java.io.Serializable;
 
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.never;
 
 /**
  * Suite of FlinkKinesisConsumer tests for the methods called throughout the source life cycle.
@@ -511,28 +530,149 @@ public void testUnparsableLongForAggregationMaxCountInConfig() {
 	// ----------------------------------------------------------------------
 
 	@Test
-	public void testSnapshotStateShouldBeNullIfSourceNotOpened() throws Exception {
+	public void testSnapshotStateShouldNotClearListStateIfSourceNotOpened() throws Exception {
 		Properties config = new Properties();
 		config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
 		config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
 		config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
 
+		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+
+		TestingListState<Serializable> listState = new TestingListState<>();
+
 		FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>("fakeStream", new SimpleStringSchema(), config);
 
-		assertTrue(consumer.snapshotState(123, 123) == null); //arbitrary checkpoint id and timestamp
+		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+
+		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+		when(initializationContext.isRestored()).thenReturn(false);
+
+		consumer.initializeState(initializationContext);
+
+		consumer.snapshotState(new StateSnapshotContextSynchronousImpl(123, 123)); //arbitrary checkpoint id and timestamp
+
+		assertFalse(listState.isClearCalled());
 	}
 
 	@Test
-	public void testSnapshotStateShouldBeNullIfSourceNotRun() throws Exception {
+	public void testSnapshotStateShouldNotClearListStateIfSourceNotRun() throws Exception {
 		Properties config = new Properties();
 		config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
 		config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
 		config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
 
+		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+
+		TestingListState<Serializable> listState = new TestingListState<>();
+
 		FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>("fakeStream", new SimpleStringSchema(), config);
+
+		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+
+		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+		when(initializationContext.isRestored()).thenReturn(false);
+
+		consumer.initializeState(initializationContext);
+
 		consumer.open(new Configuration()); // only opened, not run
 
-		assertTrue(consumer.snapshotState(123, 123) == null); //arbitrary checkpoint id and timestamp
+		consumer.snapshotState(new StateSnapshotContextSynchronousImpl(123, 123)); //arbitrary checkpoint id and timestamp
+
+		assertFalse(listState.isClearCalled());
+	}
+
+	@Test
+	public void testListStateChangedAfterSnapshotState() throws Exception {
+		// ----------------------------------------------------------------------
+		// setting config, initial state and state after snapshot
+		// ----------------------------------------------------------------------
+		Properties config = new Properties();
+		config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
+		config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
+		config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+
+		ArrayList<Tuple2<KinesisStreamShard, SequenceNumber>> initialState = new ArrayList<>(1);
+		initialState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream1",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
+			new SequenceNumber("1")));
+
+		ArrayList<Tuple2<KinesisStreamShard, SequenceNumber>> snapShotState = new ArrayList<>(3);
+		snapShotState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream1",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
+			new SequenceNumber("12")));
+		snapShotState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream1",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))),
+			new SequenceNumber("11")));
+		snapShotState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream1",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))),
+			new SequenceNumber("31")));
+
+		// ----------------------------------------------------------------------
+		// mock operator state backend and initial state for initializeState()
+		// ----------------------------------------------------------------------
+		TestingListState<Serializable> listState = new TestingListState<>();
+		for (Serializable state: initialState) {
+			listState.add(state);
+		}
+
+		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+		when(initializationContext.isRestored()).thenReturn(true);
+
+		// ----------------------------------------------------------------------
+		// mock a running fetcher and its state for snapshot
+		// ----------------------------------------------------------------------
+		HashMap<KinesisStreamShard, SequenceNumber> stateSnapshot = new HashMap<>();
+		for (Tuple2<KinesisStreamShard, SequenceNumber> tuple: snapShotState) {
+			stateSnapshot.put(tuple.f0, tuple.f1);
+		}
+
+		KinesisDataFetcher mockedFetcher = mock(KinesisDataFetcher.class);
+		when(mockedFetcher.snapshotState()).thenReturn(stateSnapshot);
+
+		// ----------------------------------------------------------------------
+		// create a consumer and test the snapshotState()
+		// ----------------------------------------------------------------------
+		FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>("fakeStream", new SimpleStringSchema(), config);
+		FlinkKinesisConsumer<?> mockedConsumer = spy(consumer);
+
+		RuntimeContext context = mock(RuntimeContext.class);
+		when(context.getIndexOfThisSubtask()).thenReturn(1);
+
+		mockedConsumer.setRuntimeContext(context);
+		mockedConsumer.initializeState(initializationContext);
+		mockedConsumer.open(new Configuration());
+		Whitebox.setInternalState(mockedConsumer, "fetcher", mockedFetcher); // mock as consumer is running.
+
+		mockedConsumer.snapshotState(mock(FunctionSnapshotContext.class));
+
+		assertEquals(true, listState.clearCalled);
+		assertEquals(3, listState.getList().size());
+
+		for (Serializable state: initialState) {
+			for (Serializable currentState: listState.getList()) {
+				assertNotEquals(state, currentState);
+			}
+		}
+
+		for (Serializable state: snapShotState) {
+			boolean hasOneIsSame = false;
+			for (Serializable currentState: listState.getList()) {
+				hasOneIsSame = hasOneIsSame || state.equals(currentState);
+			}
+			assertEquals(true, hasOneIsSame);
+		}
 	}
 
 	// ----------------------------------------------------------------------
@@ -557,50 +697,290 @@ public void testFetcherShouldNotBeRestoringFromFailureIfNotRestoringFromCheckpoi
 		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(false);
 	}
 
+	@Test
+	@SuppressWarnings("unchecked")
+	public void testFetcherShouldBeCorrectlySeededIfRestoringFromLegacyCheckpoint() throws Exception {
+		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
+
+		KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
+		List<KinesisStreamShard> shards = new ArrayList<>();
+		shards.addAll(fakeRestoredState.keySet());
+		when(mockedFetcher.discoverNewShardsToSubscribe()).thenReturn(shards);
+		PowerMockito.whenNew(KinesisDataFetcher.class).withAnyArguments().thenReturn(mockedFetcher);
+
+		// assume the given config is correct
+		PowerMockito.mockStatic(KinesisConfigUtil.class);
+		PowerMockito.doNothing().when(KinesisConfigUtil.class);
+
+		TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
+			"fakeStream", new Properties(), 10, 2);
+		consumer.restoreState(fakeRestoredState);
+		consumer.open(new Configuration());
+		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
+
+		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
+			Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
+				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
+		}
+	}
+
 	@Test
 	@SuppressWarnings("unchecked")
 	public void testFetcherShouldBeCorrectlySeededIfRestoringFromCheckpoint() throws Exception {
+		// ----------------------------------------------------------------------
+		// setting initial state
+		// ----------------------------------------------------------------------
+		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
+
+		// ----------------------------------------------------------------------
+		// mock operator state backend and initial state for initializeState()
+		// ----------------------------------------------------------------------
+		TestingListState<Serializable> listState = new TestingListState<>();
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> state: fakeRestoredState.entrySet()) {
+			listState.add(Tuple2.of(state.getKey(), state.getValue()));
+		}
+
+		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+		when(initializationContext.isRestored()).thenReturn(true);
+
+		// ----------------------------------------------------------------------
+		// mock fetcher
+		// ----------------------------------------------------------------------
 		KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
+		List<KinesisStreamShard> shards = new ArrayList<>();
+		shards.addAll(fakeRestoredState.keySet());
+		when(mockedFetcher.discoverNewShardsToSubscribe()).thenReturn(shards);
 		PowerMockito.whenNew(KinesisDataFetcher.class).withAnyArguments().thenReturn(mockedFetcher);
 
 		// assume the given config is correct
 		PowerMockito.mockStatic(KinesisConfigUtil.class);
 		PowerMockito.doNothing().when(KinesisConfigUtil.class);
 
-		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = new HashMap<>();
-		fakeRestoredState.put(
-			new KinesisStreamShard("fakeStream1",
-				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
-			new SequenceNumber(UUID.randomUUID().toString()));
-		fakeRestoredState.put(
-			new KinesisStreamShard("fakeStream1",
-				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))),
-			new SequenceNumber(UUID.randomUUID().toString()));
-		fakeRestoredState.put(
-			new KinesisStreamShard("fakeStream1",
-				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))),
-			new SequenceNumber(UUID.randomUUID().toString()));
-		fakeRestoredState.put(
-			new KinesisStreamShard("fakeStream2",
-				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
-			new SequenceNumber(UUID.randomUUID().toString()));
-		fakeRestoredState.put(
-			new KinesisStreamShard("fakeStream2",
-				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))),
-			new SequenceNumber(UUID.randomUUID().toString()));
+		// ----------------------------------------------------------------------
+		// start to test seed initial state to fetcher
+		// ----------------------------------------------------------------------
+		TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
+			"fakeStream", new Properties(), 10, 2);
+		consumer.initializeState(initializationContext);
+		consumer.open(new Configuration());
+		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
+
+		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
+			Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
+				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
+		}
+	}
+
+	@Test
+	@SuppressWarnings("unchecked")
+	public void testFetcherShouldBeCorrectlySeededOnlyItsOwnStates() throws Exception {
+		// ----------------------------------------------------------------------
+		// setting initial state
+		// ----------------------------------------------------------------------
+		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = getFakeRestoredStore("fakeStream1");
+
+		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredStateForOthers = getFakeRestoredStore("fakeStream2");
+
+		// ----------------------------------------------------------------------
+		// mock operator state backend and initial state for initializeState()
+		// ----------------------------------------------------------------------
+		TestingListState<Serializable> listState = new TestingListState<>();
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> state: fakeRestoredState.entrySet()) {
+			listState.add(Tuple2.of(state.getKey(), state.getValue()));
+		}
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> state: fakeRestoredStateForOthers.entrySet()) {
+			listState.add(Tuple2.of(state.getKey(), state.getValue()));
+		}
 
+		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+		when(initializationContext.isRestored()).thenReturn(true);
+
+		// ----------------------------------------------------------------------
+		// mock fetcher
+		// ----------------------------------------------------------------------
+		KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
+		List<KinesisStreamShard> shards = new ArrayList<>();
+		shards.addAll(fakeRestoredState.keySet());
+		when(mockedFetcher.discoverNewShardsToSubscribe()).thenReturn(shards);
+		PowerMockito.whenNew(KinesisDataFetcher.class).withAnyArguments().thenReturn(mockedFetcher);
+
+		// assume the given config is correct
+		PowerMockito.mockStatic(KinesisConfigUtil.class);
+		PowerMockito.doNothing().when(KinesisConfigUtil.class);
+
+		// ----------------------------------------------------------------------
+		// start to test seed initial state to fetcher
+		// ----------------------------------------------------------------------
 		TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
 			"fakeStream", new Properties(), 10, 2);
-		consumer.restoreState(fakeRestoredState);
+		consumer.initializeState(initializationContext);
 		consumer.open(new Configuration());
 		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
 
 		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredStateForOthers.entrySet()) {
+			// should never get restored state not belonging to itself
+			Mockito.verify(mockedFetcher, never()).registerNewSubscribedShardState(
+				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
+		}
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
-			Mockito.verify(mockedFetcher).advanceLastDiscoveredShardOfStream(
-				restoredShard.getKey().getStreamName(), restoredShard.getKey().getShard().getShardId());
+			// should get restored state belonging to itself
 			Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
 				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
 		}
 	}
+
+	/*
+	 * If the original parallelism is 2 and states is:
+	 *   Consumer subtask 1:
+	 *     stream1, shard1, SequentialNumber(xxx)
+	 *   Consumer subtask 2:
+	 *     stream1, shard2, SequentialNumber(yyy)
+	 * After discoverNewShardsToSubscribe() if there are two shards (shard3, shard4) been created:
+	 *   Consumer subtask 1 (late for discoverNewShardsToSubscribe()):
+	 *     stream1, shard1, SequentialNumber(xxx)
+	 *   Consumer subtask 2:
+	 *     stream1, shard2, SequentialNumber(yyy)
+	 *     stream1, shard4, SequentialNumber(zzz)
+	 *  If snapshotState() occur and parallelism is changed to 1:
+	 *    Union state will be:
+	 *     stream1, shard1, SequentialNumber(xxx)
+	 *     stream1, shard2, SequentialNumber(yyy)
+	 *     stream1, shard4, SequentialNumber(zzz)
+	 *    Fetcher should be seeded with:
+	 *     stream1, shard1, SequentialNumber(xxx)
+	 *     stream1, shard2, SequentialNumber(yyy)
+	 *     stream1, share3, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM
+	 *     stream1, shard4, SequentialNumber(zzz)
+	 *
+	 *  This test is to guarantee the fetcher will be seeded correctly for such situation.
+	 */
+	@Test
+	@SuppressWarnings("unchecked")
+	public void testFetcherShouldBeCorrectlySeededWithNewDiscoveredKinesisStreamShard() throws Exception {
+		// ----------------------------------------------------------------------
+		// setting initial state
+		// ----------------------------------------------------------------------
+		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
+
+		// ----------------------------------------------------------------------
+		// mock operator state backend and initial state for initializeState()
+		// ----------------------------------------------------------------------
+		TestingListState<Serializable> listState = new TestingListState<>();
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> state: fakeRestoredState.entrySet()) {
+			listState.add(Tuple2.of(state.getKey(), state.getValue()));
+		}
+
+		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+		when(initializationContext.isRestored()).thenReturn(true);
+
+		// ----------------------------------------------------------------------
+		// mock fetcher
+		// ----------------------------------------------------------------------
+		KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
+		List<KinesisStreamShard> shards = new ArrayList<>();
+		shards.addAll(fakeRestoredState.keySet());
+		shards.add(new KinesisStreamShard("fakeStream2",
+			new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))));
+		when(mockedFetcher.discoverNewShardsToSubscribe()).thenReturn(shards);
+		PowerMockito.whenNew(KinesisDataFetcher.class).withAnyArguments().thenReturn(mockedFetcher);
+
+		// assume the given config is correct
+		PowerMockito.mockStatic(KinesisConfigUtil.class);
+		PowerMockito.doNothing().when(KinesisConfigUtil.class);
+
+		// ----------------------------------------------------------------------
+		// start to test seed initial state to fetcher
+		// ----------------------------------------------------------------------
+		TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
+			"fakeStream", new Properties(), 10, 2);
+		consumer.initializeState(initializationContext);
+		consumer.open(new Configuration());
+		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
+
+		fakeRestoredState.put(new KinesisStreamShard("fakeStream2",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))),
+			SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get());
+		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
+		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
+			Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
+				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
+		}
+	}
+
+	private static final class TestingListState<T> implements ListState<T> {
+
+		private final List<T> list = new ArrayList<>();
+		private boolean clearCalled = false;
+
+		@Override
+		public void clear() {
+			list.clear();
+			clearCalled = true;
+		}
+
+		@Override
+		public Iterable<T> get() throws Exception {
+			return list;
+		}
+
+		@Override
+		public void add(T value) throws Exception {
+			list.add(value);
+		}
+
+		public List<T> getList() {
+			return list;
+		}
+
+		public boolean isClearCalled() {
+			return clearCalled;
+		}
+	}
+
+	private HashMap<KinesisStreamShard, SequenceNumber> getFakeRestoredStore(String streamName) {
+		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = new HashMap<>();
+
+		if (streamName.equals("fakeStream1") || streamName.equals("all")) {
+			fakeRestoredState.put(
+				new KinesisStreamShard("fakeStream1",
+					new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
+				new SequenceNumber(UUID.randomUUID().toString()));
+			fakeRestoredState.put(
+				new KinesisStreamShard("fakeStream1",
+					new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))),
+				new SequenceNumber(UUID.randomUUID().toString()));
+			fakeRestoredState.put(
+				new KinesisStreamShard("fakeStream1",
+					new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))),
+				new SequenceNumber(UUID.randomUUID().toString()));
+		}
+
+		if (streamName.equals("fakeStream2") || streamName.equals("all")) {
+			fakeRestoredState.put(
+				new KinesisStreamShard("fakeStream2",
+					new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
+				new SequenceNumber(UUID.randomUUID().toString()));
+			fakeRestoredState.put(
+				new KinesisStreamShard("fakeStream2",
+					new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))),
+				new SequenceNumber(UUID.randomUUID().toString()));
+		}
+
+		return fakeRestoredState;
+	}
 }
diff --git a/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.1-snapshot b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.1-snapshot
new file mode 100644
index 0000000000000000000000000000000000000000..b60402e8483276a255befd3629dc71bc53ec5f26
GIT binary patch
literal 1140
zcmbVLJ5Los6h3zmc^eTQjekJYSrJ8$LIraJvJ&E@BQc)4J8+k~yR*#9!a`xRvobN(
z7L+FU2bfqH3u;TPY%OfmLSvz4W*2#xNIc2DX3l)ycg}aV^%>fggi#@v7^dhqI9rUu
zq{)<)9OE!qVlqo~7;~mkH5IGb^?7=)Y1ABws5sE(LEmDQ_&OAc-O_NuBUrb9buFx0
zg1O2Zc5U7iCPBo<J?Lz%+6ARyz+1*ubTb`B?i8xVT}&yVPo!L;7Sg7zz(c6yDXAYn
zpnhAkrC~9Tg<NMfE<=pM4Ib>`9eW~%2j?FBaG<gP$JC#c+eiw*S(wNrjQj)tR-L+q
zzMhPP;#iM(QPNItAWt)Cz*=9=d_USb|Iq<j0&u>xivIVTa+xWYmF2c~e0cNr^n-V8
z;8dYI5*Sp8LC9(7(hbQ~LqtKI-neN=de=JXd7Q2awWE4K?v9zhND^*v6l;bFSD|9D
z2zbP1im$@nS_5U!YQ28-{9Bc@woOB~Z)PJf;yjudvv2(MMeJgX8zR9q<x=-nL!T?R
zgmx{oYthKQ?n1^a#$3aR!r=&$B^m=sgEpS0?<a$ERKOs?vOlnF8+M;!3w!p(#HX#z
zN#B8X4^CmQj6P5F97?*|gz3P{$q^5F8>D@Puiqm^;vYVoc2n^7Av1yLUyuhBU&(ml
z^Vuw}b6FBxKO8~r;9277upKV7=cx$|C(Ff)mv)7Qo<(f(!nYx6-o*d7zgI=?`ORl@
uJzGEbJAViIRaW1d>zSWscloy6dHVSN)0e->(9ktDdTr$DmEp@nRLpOq{gAc*

literal 0
HcmV?d00001

diff --git a/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.1-snapshot-empty b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.1-snapshot-empty
new file mode 100644
index 0000000000000000000000000000000000000000..f4dd96d21134264b96e88b4b622d6468ac4bb6b7
GIT binary patch
literal 468
zcmah_%}N6?5T0(SR-vK@`VJz=K@g<}e<>AK5ZhZXj@`5yo881DTUQU_(TC|vh)>|v
zlP}=Oi{8v;>koJ^ml<Zhd_OaJg__Z@CH0V?!hX!iK=5=#jMWSio(_r5Qp*!YESixa
z9Y627zGKlcC|@OjY6y)%mi8@|sb5)G4;?KnX{o2BQ!;9b&(!1Em_~v%L#U6&eqbyt
zhZi_TnprMr8%=zS$^jZuF^yJg54(h0sN~AIZ~k%r<=M-Sge6j6lMd4y1;1g@FWv2X
zxv_R~_Ypv)2MgvKa){0$1ik%S!{$Hyd*|+2*tZ8l8iwbBm63jK1G&m{K{k0?eV?g!
zUjlF=w?Cbf_g^aFk|Hjd#;wghKRm9SJyjuS!Mu<dwVZ;~M=)n&%}gvsl&hWHQsk=c
K>SEF7N__%m<(jtu

literal 0
HcmV?d00001


From fd3589884620de209bab3aa12f1c18b5c8349c88 Mon Sep 17 00:00:00 2001
From: "Tzu-Li (Gordon) Tai" <tzulitai@apache.org>
Date: Sun, 7 May 2017 16:29:32 +0800
Subject: [PATCH 179/253] [FLINK-4821] [kinesis] General improvements to
 rescalable FlinkKinesisConsumer

This commit adds some general improvements to the rescalable
implementation of FlinkKinesisConsumer, including:
- Refactor setup procedures in KinesisDataFetcher so that duplicate work
  isn't done on a restored run
- Strengthen corner cases where fetcher was not fully seeded with
  initial state when snapshot is taken

This closes #3001.
---
 .../kinesis/FlinkKinesisConsumer.java         | 150 +++++++++--------
 .../kinesis/internals/KinesisDataFetcher.java |  52 +-----
 .../FlinkKinesisConsumerMigrationTest.java    |   5 +-
 .../kinesis/FlinkKinesisConsumerTest.java     | 159 ++++++++++--------
 .../internals/KinesisDataFetcherTest.java     |  65 +++++--
 .../testutils/TestableKinesisDataFetcher.java |  14 ++
 6 files changed, 233 insertions(+), 212 deletions(-)

diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
index dfcd552bc9450..4982f7f39ab14 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
@@ -25,13 +25,14 @@
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.state.FunctionInitializationContext;
 import org.apache.flink.runtime.state.FunctionSnapshotContext;
 import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
 import org.apache.flink.streaming.api.checkpoint.CheckpointedRestoring;
 import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
+import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants.InitialPosition;
 import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
@@ -67,9 +68,9 @@
  * @param <T> the type of data emitted
  */
 public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> implements
-	ResultTypeQueryable<T>,
-	CheckpointedFunction,
-	CheckpointedRestoring<HashMap<KinesisStreamShard, SequenceNumber>> {
+		ResultTypeQueryable<T>,
+		CheckpointedFunction,
+		CheckpointedRestoring<HashMap<KinesisStreamShard, SequenceNumber>> {
 
 	private static final long serialVersionUID = 4724006128720664870L;
 
@@ -86,7 +87,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
 	 * shard list retrieval behaviours, etc */
 	private final Properties configProps;
 
-	/** User supplied deseriliazation schema to convert Kinesis byte messages to Flink objects */
+	/** User supplied deserialization schema to convert Kinesis byte messages to Flink objects */
 	private final KinesisDeserializationSchema<T> deserializer;
 
 	// ------------------------------------------------------------------------
@@ -96,9 +97,6 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
 	/** Per-task fetcher for Kinesis data records, where each fetcher pulls data from one or more Kinesis shards */
 	private transient KinesisDataFetcher<T> fetcher;
 
-	/** The sequence numbers in the last state snapshot of this subtask */
-	private transient HashMap<KinesisStreamShard, SequenceNumber> lastStateSnapshot;
-
 	/** The sequence numbers to restore to upon restore from failure */
 	private transient HashMap<KinesisStreamShard, SequenceNumber> sequenceNumsToRestore;
 
@@ -108,7 +106,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
 	//  State for Checkpoint
 	// ------------------------------------------------------------------------
 
-	/** The name is the key for sequence numbers state, and cannot be changed. */
+	/** State name to access shard sequence number states; cannot be changed */
 	private static final String sequenceNumsStateStoreName = "Kinesis-Stream-Shard-State";
 
 	private transient ListState<Tuple2<KinesisStreamShard, SequenceNumber>> sequenceNumsStateForCheckpoint;
@@ -190,58 +188,34 @@ public FlinkKinesisConsumer(List<String> streams, KinesisDeserializationSchema<T
 	//  Source life cycle
 	// ------------------------------------------------------------------------
 
-	@Override
-	public void open(Configuration parameters) throws Exception {
-		super.open(parameters);
-
-		// restore to the last known sequence numbers from the latest complete snapshot
-		if (sequenceNumsToRestore != null) {
-			if (LOG.isInfoEnabled()) {
-				LOG.info("Subtask {} is restoring sequence numbers {} from previous checkpointed state",
-					getRuntimeContext().getIndexOfThisSubtask(), sequenceNumsToRestore.toString());
-			}
-
-			// initialize sequence numbers with restored state
-			lastStateSnapshot = sequenceNumsToRestore;
-		} else {
-			// start fresh with empty sequence numbers if there are no snapshots to restore from.
-			lastStateSnapshot = new HashMap<>();
-		}
-	}
-
 	@Override
 	public void run(SourceContext<T> sourceContext) throws Exception {
 
 		// all subtasks will run a fetcher, regardless of whether or not the subtask will initially have
 		// shards to subscribe to; fetchers will continuously poll for changes in the shard list, so all subtasks
 		// can potentially have new shards to subscribe to later on
-		fetcher = createFetcher(streams, sourceContext, getRuntimeContext(), configProps, deserializer);
-
-		boolean isRestoringFromFailure = (sequenceNumsToRestore != null);
-		fetcher.setIsRestoringFromFailure(isRestoringFromFailure);
-
-		// if we are restoring from a checkpoint, we iterate over the restored
-		// state and accordingly seed the fetcher with subscribed shards states
-		if (isRestoringFromFailure) {
-			// Since there may have a situation that some subtasks did not finish discovering before rescale,
-			// and KinesisDataFetcher will always discover the shard from the largest shard id. To prevent from
-			// missing some shards which didn't be discovered and whose id is not the largest one, we force the
-			// consumer to discover once from the smallest id and make sure each shard have its initial sequence
-			// number from restored state or SENTINEL_EARLIEST_SEQUENCE_NUM.
-			List<KinesisStreamShard> newShardsCreatedWhileNotRunning = fetcher.discoverNewShardsToSubscribe();
-			for (KinesisStreamShard shard : newShardsCreatedWhileNotRunning) {
-				SequenceNumber startingStateForNewShard;
-
-				if (lastStateSnapshot.containsKey(shard)) {
-					startingStateForNewShard = lastStateSnapshot.get(shard);
+		KinesisDataFetcher<T> fetcher = createFetcher(streams, sourceContext, getRuntimeContext(), configProps, deserializer);
+
+		// initial discovery
+		List<KinesisStreamShard> allShards = fetcher.discoverNewShardsToSubscribe();
+
+		for (KinesisStreamShard shard : allShards) {
+			if (sequenceNumsToRestore != null) {
+				if (sequenceNumsToRestore.containsKey(shard)) {
+					// if the shard was already seen and is contained in the state,
+					// just use the sequence number stored in the state
+					fetcher.registerNewSubscribedShardState(
+						new KinesisStreamShardState(shard, sequenceNumsToRestore.get(shard)));
 
 					if (LOG.isInfoEnabled()) {
 						LOG.info("Subtask {} is seeding the fetcher with restored shard {}," +
 								" starting state set to the restored sequence number {}",
-							getRuntimeContext().getIndexOfThisSubtask(), shard.toString(), startingStateForNewShard);
+							getRuntimeContext().getIndexOfThisSubtask(), shard.toString(), sequenceNumsToRestore.get(shard));
 					}
 				} else {
-					startingStateForNewShard = SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get();
+					// the shard wasn't discovered in the previous run, therefore should be consumed from the beginning
+					fetcher.registerNewSubscribedShardState(
+						new KinesisStreamShardState(shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get()));
 
 					if (LOG.isInfoEnabled()) {
 						LOG.info("Subtask {} is seeding the fetcher with new discovered shard {}," +
@@ -249,9 +223,20 @@ public void run(SourceContext<T> sourceContext) throws Exception {
 							getRuntimeContext().getIndexOfThisSubtask(), shard.toString());
 					}
 				}
+			} else {
+				// we're starting fresh; use the configured start position as initial state
+				SentinelSequenceNumber startingSeqNum =
+					InitialPosition.valueOf(configProps.getProperty(
+						ConsumerConfigConstants.STREAM_INITIAL_POSITION,
+						ConsumerConfigConstants.DEFAULT_STREAM_INITIAL_POSITION)).toSentinelSequenceNumber();
 
 				fetcher.registerNewSubscribedShardState(
-					new KinesisStreamShardState(shard, startingStateForNewShard));
+					new KinesisStreamShardState(shard, startingSeqNum.get()));
+
+				if (LOG.isInfoEnabled()) {
+					LOG.info("Subtask {} will be seeded with initial shard {}, starting state set as sequence number {}",
+						getRuntimeContext().getIndexOfThisSubtask(), shard.toString(), startingSeqNum.get());
+				}
 			}
 		}
 
@@ -260,6 +245,10 @@ public void run(SourceContext<T> sourceContext) throws Exception {
 			return;
 		}
 
+		// expose the fetcher from this point, so that state
+		// snapshots can be taken from the fetcher's state holders
+		this.fetcher = fetcher;
+
 		// start the fetcher loop. The fetcher will stop running only when cancel() or
 		// close() is called, or an error is thrown by threads created by the fetcher
 		fetcher.runFetcher();
@@ -306,13 +295,12 @@ public TypeInformation<T> getProducedType() {
 
 	@Override
 	public void initializeState(FunctionInitializationContext context) throws Exception {
-		TypeInformation<Tuple2<KinesisStreamShard, SequenceNumber>> tuple = new TupleTypeInfo<>(
+		TypeInformation<Tuple2<KinesisStreamShard, SequenceNumber>> shardsStateTypeInfo = new TupleTypeInfo<>(
 			TypeInformation.of(KinesisStreamShard.class),
-			TypeInformation.of(SequenceNumber.class)
-		);
+			TypeInformation.of(SequenceNumber.class));
 
 		sequenceNumsStateForCheckpoint = context.getOperatorStateStore().getUnionListState(
-			new ListStateDescriptor<>(sequenceNumsStateStoreName, tuple));
+			new ListStateDescriptor<>(sequenceNumsStateStoreName, shardsStateTypeInfo));
 
 		if (context.isRestored()) {
 			if (sequenceNumsToRestore == null) {
@@ -323,8 +311,6 @@ public void initializeState(FunctionInitializationContext context) throws Except
 
 				LOG.info("Setting restore state in the FlinkKinesisConsumer. Using the following offsets: {}",
 					sequenceNumsToRestore);
-			} else if (sequenceNumsToRestore.isEmpty()) {
-				sequenceNumsToRestore = null;
 			}
 		} else {
 			LOG.info("No restore state for FlinkKinesisConsumer.");
@@ -333,11 +319,7 @@ public void initializeState(FunctionInitializationContext context) throws Except
 
 	@Override
 	public void snapshotState(FunctionSnapshotContext context) throws Exception {
-		if (lastStateSnapshot == null) {
-			LOG.debug("snapshotState() requested on not yet opened source; returning null.");
-		} else if (fetcher == null) {
-			LOG.debug("snapshotState() requested on not yet running source; returning null.");
-		} else if (!running) {
+		if (!running) {
 			LOG.debug("snapshotState() called on closed source; returning null.");
 		} else {
 			if (LOG.isDebugEnabled()) {
@@ -345,15 +327,33 @@ public void snapshotState(FunctionSnapshotContext context) throws Exception {
 			}
 
 			sequenceNumsStateForCheckpoint.clear();
-			lastStateSnapshot = fetcher.snapshotState();
 
-			if (LOG.isDebugEnabled()) {
-				LOG.debug("Snapshotted state, last processed sequence numbers: {}, checkpoint id: {}, timestamp: {}",
-					lastStateSnapshot.toString(), context.getCheckpointId(), context.getCheckpointTimestamp());
-			}
+			if (fetcher == null) {
+				if (sequenceNumsToRestore != null) {
+					for (Map.Entry<KinesisStreamShard, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
+						// sequenceNumsToRestore is the restored global union state;
+						// should only snapshot shards that actually belong to us
+
+						if (KinesisDataFetcher.isThisSubtaskShouldSubscribeTo(
+								entry.getKey(),
+								getRuntimeContext().getNumberOfParallelSubtasks(),
+								getRuntimeContext().getIndexOfThisSubtask())) {
+
+							sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey(), entry.getValue()));
+						}
+					}
+				}
+			} else {
+				HashMap<KinesisStreamShard, SequenceNumber> lastStateSnapshot = fetcher.snapshotState();
+
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Snapshotted state, last processed sequence numbers: {}, checkpoint id: {}, timestamp: {}",
+						lastStateSnapshot.toString(), context.getCheckpointId(), context.getCheckpointTimestamp());
+				}
 
-			for (Map.Entry<KinesisStreamShard, SequenceNumber> entry : lastStateSnapshot.entrySet()) {
-				sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey(), entry.getValue()));
+				for (Map.Entry<KinesisStreamShard, SequenceNumber> entry : lastStateSnapshot.entrySet()) {
+					sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey(), entry.getValue()));
+				}
 			}
 		}
 	}
@@ -366,12 +366,14 @@ public void restoreState(HashMap<KinesisStreamShard, SequenceNumber> restoredSta
 		sequenceNumsToRestore = restoredState.isEmpty() ? null : restoredState;
 	}
 
-	/** This method is created for tests that can mock the KinesisDataFetcher in the consumer. */
-	protected KinesisDataFetcher<T> createFetcher(List<String> streams,
-													SourceFunction.SourceContext<T> sourceContext,
-													RuntimeContext runtimeContext,
-													Properties configProps,
-													KinesisDeserializationSchema<T> deserializationSchema) {
+	/** This method is exposed for tests that need to mock the KinesisDataFetcher in the consumer. */
+	protected KinesisDataFetcher<T> createFetcher(
+			List<String> streams,
+			SourceFunction.SourceContext<T> sourceContext,
+			RuntimeContext runtimeContext,
+			Properties configProps,
+			KinesisDeserializationSchema<T> deserializationSchema) {
+
 		return new KinesisDataFetcher<>(streams, sourceContext, runtimeContext, configProps, deserializationSchema);
 	}
 
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index c5b4b0495f1e6..99305cb435e3e 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -19,9 +19,7 @@
 
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.connectors.kinesis.FlinkKinesisConsumer;
 import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
-import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants.InitialPosition;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
 import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumber;
@@ -99,12 +97,6 @@ public class KinesisDataFetcher<T> {
 
 	private final int indexOfThisConsumerSubtask;
 
-	/**
-	 * This flag should be set by {@link FlinkKinesisConsumer} using
-	 * {@link KinesisDataFetcher#setIsRestoringFromFailure(boolean)}
-	 */
-	private boolean isRestoredFromFailure;
-
 	// ------------------------------------------------------------------------
 	//  Executor services to run created threads
 	// ------------------------------------------------------------------------
@@ -235,41 +227,7 @@ public void runFetcher() throws Exception {
 		//  Procedures before starting the infinite while loop:
 		// ------------------------------------------------------------------------
 
-		//  1. query for any new shards that may have been created while the Kinesis consumer was not running,
-		//     and register them to the subscribedShardState list.
-		if (LOG.isDebugEnabled()) {
-			String logFormat = (!isRestoredFromFailure)
-				? "Subtask {} is trying to discover initial shards ..."
-				: "Subtask {} is trying to discover any new shards that were created while the consumer wasn't " +
-				"running due to failure ...";
-
-			LOG.debug(logFormat, indexOfThisConsumerSubtask);
-		}
-		List<KinesisStreamShard> newShardsCreatedWhileNotRunning = discoverNewShardsToSubscribe();
-		for (KinesisStreamShard shard : newShardsCreatedWhileNotRunning) {
-			// the starting state for new shards created while the consumer wasn't running depends on whether or not
-			// we are starting fresh (not restoring from a checkpoint); when we are starting fresh, this simply means
-			// all existing shards of streams we are subscribing to are new shards; when we are restoring from checkpoint,
-			// any new shards due to Kinesis resharding from the time of the checkpoint will be considered new shards.
-			InitialPosition initialPosition = InitialPosition.valueOf(configProps.getProperty(
-				ConsumerConfigConstants.STREAM_INITIAL_POSITION, ConsumerConfigConstants.DEFAULT_STREAM_INITIAL_POSITION));
-
-			SentinelSequenceNumber startingStateForNewShard = (isRestoredFromFailure)
-				? SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM
-				: initialPosition.toSentinelSequenceNumber();
-
-			if (LOG.isInfoEnabled()) {
-				String logFormat = (!isRestoredFromFailure)
-					? "Subtask {} will be seeded with initial shard {}, starting state set as sequence number {}"
-					: "Subtask {} will be seeded with new shard {} that was created while the consumer wasn't " +
-					"running due to failure, starting state set as sequence number {}";
-
-				LOG.info(logFormat, indexOfThisConsumerSubtask, shard.toString(), startingStateForNewShard.get());
-			}
-			registerNewSubscribedShardState(new KinesisStreamShardState(shard, startingStateForNewShard.get()));
-		}
-
-		//  2. check that there is at least one shard in the subscribed streams to consume from (can be done by
+		//  1. check that there is at least one shard in the subscribed streams to consume from (can be done by
 		//     checking if at least one value in subscribedStreamsToLastDiscoveredShardIds is not null)
 		boolean hasShards = false;
 		StringBuilder streamsWithNoShardsFound = new StringBuilder();
@@ -290,7 +248,7 @@ public void runFetcher() throws Exception {
 			throw new RuntimeException("No shards can be found for all subscribed streams: " + streams);
 		}
 
-		//  3. start consuming any shard state we already have in the subscribedShardState up to this point; the
+		//  2. start consuming any shard state we already have in the subscribedShardState up to this point; the
 		//     subscribedShardState may already be seeded with values due to step 1., or explicitly added by the
 		//     consumer using a restored state checkpoint
 		for (int seededStateIndex = 0; seededStateIndex < subscribedShardsState.size(); seededStateIndex++) {
@@ -489,10 +447,6 @@ public List<KinesisStreamShard> discoverNewShardsToSubscribe() throws Interrupte
 	//  Functions to get / set information about the consumer
 	// ------------------------------------------------------------------------
 
-	public void setIsRestoringFromFailure(boolean bool) {
-		this.isRestoredFromFailure = bool;
-	}
-
 	protected Properties getConsumerConfiguration() {
 		return configProps;
 	}
@@ -595,7 +549,7 @@ public int registerNewSubscribedShardState(KinesisStreamShardState newSubscribed
 	 * @param totalNumberOfConsumerSubtasks total number of consumer subtasks
 	 * @param indexOfThisConsumerSubtask index of this consumer subtask
 	 */
-	private static boolean isThisSubtaskShouldSubscribeTo(KinesisStreamShard shard,
+	public static boolean isThisSubtaskShouldSubscribeTo(KinesisStreamShard shard,
 														int totalNumberOfConsumerSubtasks,
 														int indexOfThisConsumerSubtask) {
 		return (Math.abs(shard.hashCode() % totalNumberOfConsumerSubtasks)) == indexOfThisConsumerSubtask;
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
index 2f46e0936c939..7629f9d0e2794 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -42,10 +42,7 @@
 
 /**
  * Tests for checking whether {@link FlinkKinesisConsumer} can restore from snapshots that were
- * done using the Flink 1.1 {@link FlinkKinesisConsumer}.
- *
- * <p>For regenerating the binary snapshot file you have to run the commented out portion
- * of each test on a checkout of the Flink 1.1 branch.
+ * done using the Flink 1.1 {@code FlinkKinesisConsumer}.
  */
 public class FlinkKinesisConsumerMigrationTest {
 
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
index bf8e44f5c1719..4b178c74620ca 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
@@ -40,6 +40,7 @@
 import org.apache.flink.streaming.connectors.kinesis.testutils.TestableFlinkKinesisConsumer;
 import org.apache.flink.streaming.connectors.kinesis.util.KinesisConfigUtil;
 import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -57,10 +58,8 @@
 import java.util.List;
 import java.util.Properties;
 import java.util.UUID;
-import java.io.Serializable;
 
 import static org.junit.Assert.fail;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.mockito.Mockito.mock;
@@ -530,7 +529,7 @@ public void testUnparsableLongForAggregationMaxCountInConfig() {
 	// ----------------------------------------------------------------------
 
 	@Test
-	public void testSnapshotStateShouldNotClearListStateIfSourceNotOpened() throws Exception {
+	public void testUseRestoredStateForSnapshotIfFetcherNotInitialized() throws Exception {
 		Properties config = new Properties();
 		config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
 		config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
@@ -538,57 +537,63 @@ public void testSnapshotStateShouldNotClearListStateIfSourceNotOpened() throws E
 
 		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
 
-		TestingListState<Serializable> listState = new TestingListState<>();
-
-		FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>("fakeStream", new SimpleStringSchema(), config);
-
-		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
-
-		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
-
-		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
-		when(initializationContext.isRestored()).thenReturn(false);
-
-		consumer.initializeState(initializationContext);
-
-		consumer.snapshotState(new StateSnapshotContextSynchronousImpl(123, 123)); //arbitrary checkpoint id and timestamp
-
-		assertFalse(listState.isClearCalled());
-	}
-
-	@Test
-	public void testSnapshotStateShouldNotClearListStateIfSourceNotRun() throws Exception {
-		Properties config = new Properties();
-		config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
-		config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
-		config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
-
-		OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+		List<Tuple2<KinesisStreamShard, SequenceNumber>> globalUnionState = new ArrayList<>(4);
+		globalUnionState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
+			new SequenceNumber("1")));
+		globalUnionState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))),
+			new SequenceNumber("1")));
+		globalUnionState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))),
+			new SequenceNumber("1")));
+		globalUnionState.add(Tuple2.of(
+			new KinesisStreamShard("fakeStream",
+				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(3))),
+			new SequenceNumber("1")));
 
-		TestingListState<Serializable> listState = new TestingListState<>();
+		TestingListState<Tuple2<KinesisStreamShard, SequenceNumber>> listState = new TestingListState<>();
+		for (Tuple2<KinesisStreamShard, SequenceNumber> state : globalUnionState) {
+			listState.add(state);
+		}
 
 		FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>("fakeStream", new SimpleStringSchema(), config);
+		RuntimeContext context = mock(RuntimeContext.class);
+		when(context.getIndexOfThisSubtask()).thenReturn(0);
+		when(context.getNumberOfParallelSubtasks()).thenReturn(2);
+		consumer.setRuntimeContext(context);
 
 		when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
 
 		StateInitializationContext initializationContext = mock(StateInitializationContext.class);
 
 		when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
-		when(initializationContext.isRestored()).thenReturn(false);
+		when(initializationContext.isRestored()).thenReturn(true);
 
 		consumer.initializeState(initializationContext);
 
-		consumer.open(new Configuration()); // only opened, not run
+		// only opened, not run
+		consumer.open(new Configuration());
+
+		// arbitrary checkpoint id and timestamp
+		consumer.snapshotState(new StateSnapshotContextSynchronousImpl(123, 123));
 
-		consumer.snapshotState(new StateSnapshotContextSynchronousImpl(123, 123)); //arbitrary checkpoint id and timestamp
+		Assert.assertTrue(listState.isClearCalled());
 
-		assertFalse(listState.isClearCalled());
+		// the checkpointed list state should contain only the shards that it should subscribe to
+		Assert.assertEquals(globalUnionState.size() / 2, listState.getList().size());
+		Assert.assertTrue(listState.getList().contains(globalUnionState.get(0)));
+		Assert.assertTrue(listState.getList().contains(globalUnionState.get(2)));
 	}
 
 	@Test
 	public void testListStateChangedAfterSnapshotState() throws Exception {
+
 		// ----------------------------------------------------------------------
-		// setting config, initial state and state after snapshot
+		// setup config, initial state and expected state snapshot
 		// ----------------------------------------------------------------------
 		Properties config = new Properties();
 		config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
@@ -601,16 +606,16 @@ public void testListStateChangedAfterSnapshotState() throws Exception {
 				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
 			new SequenceNumber("1")));
 
-		ArrayList<Tuple2<KinesisStreamShard, SequenceNumber>> snapShotState = new ArrayList<>(3);
-		snapShotState.add(Tuple2.of(
+		ArrayList<Tuple2<KinesisStreamShard, SequenceNumber>> expectedStateSnapshot = new ArrayList<>(3);
+		expectedStateSnapshot.add(Tuple2.of(
 			new KinesisStreamShard("fakeStream1",
 				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))),
 			new SequenceNumber("12")));
-		snapShotState.add(Tuple2.of(
+		expectedStateSnapshot.add(Tuple2.of(
 			new KinesisStreamShard("fakeStream1",
 				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))),
 			new SequenceNumber("11")));
-		snapShotState.add(Tuple2.of(
+		expectedStateSnapshot.add(Tuple2.of(
 			new KinesisStreamShard("fakeStream1",
 				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))),
 			new SequenceNumber("31")));
@@ -618,8 +623,9 @@ public void testListStateChangedAfterSnapshotState() throws Exception {
 		// ----------------------------------------------------------------------
 		// mock operator state backend and initial state for initializeState()
 		// ----------------------------------------------------------------------
-		TestingListState<Serializable> listState = new TestingListState<>();
-		for (Serializable state: initialState) {
+
+		TestingListState<Tuple2<KinesisStreamShard, SequenceNumber>> listState = new TestingListState<>();
+		for (Tuple2<KinesisStreamShard, SequenceNumber> state: initialState) {
 			listState.add(state);
 		}
 
@@ -633,8 +639,9 @@ public void testListStateChangedAfterSnapshotState() throws Exception {
 		// ----------------------------------------------------------------------
 		// mock a running fetcher and its state for snapshot
 		// ----------------------------------------------------------------------
+
 		HashMap<KinesisStreamShard, SequenceNumber> stateSnapshot = new HashMap<>();
-		for (Tuple2<KinesisStreamShard, SequenceNumber> tuple: snapShotState) {
+		for (Tuple2<KinesisStreamShard, SequenceNumber> tuple: expectedStateSnapshot) {
 			stateSnapshot.put(tuple.f0, tuple.f1);
 		}
 
@@ -644,6 +651,7 @@ public void testListStateChangedAfterSnapshotState() throws Exception {
 		// ----------------------------------------------------------------------
 		// create a consumer and test the snapshotState()
 		// ----------------------------------------------------------------------
+
 		FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>("fakeStream", new SimpleStringSchema(), config);
 		FlinkKinesisConsumer<?> mockedConsumer = spy(consumer);
 
@@ -653,22 +661,22 @@ public void testListStateChangedAfterSnapshotState() throws Exception {
 		mockedConsumer.setRuntimeContext(context);
 		mockedConsumer.initializeState(initializationContext);
 		mockedConsumer.open(new Configuration());
-		Whitebox.setInternalState(mockedConsumer, "fetcher", mockedFetcher); // mock as consumer is running.
+		Whitebox.setInternalState(mockedConsumer, "fetcher", mockedFetcher); // mock consumer as running.
 
 		mockedConsumer.snapshotState(mock(FunctionSnapshotContext.class));
 
 		assertEquals(true, listState.clearCalled);
 		assertEquals(3, listState.getList().size());
 
-		for (Serializable state: initialState) {
-			for (Serializable currentState: listState.getList()) {
+		for (Tuple2<KinesisStreamShard, SequenceNumber> state: initialState) {
+			for (Tuple2<KinesisStreamShard, SequenceNumber> currentState: listState.getList()) {
 				assertNotEquals(state, currentState);
 			}
 		}
 
-		for (Serializable state: snapShotState) {
+		for (Tuple2<KinesisStreamShard, SequenceNumber> state: expectedStateSnapshot) {
 			boolean hasOneIsSame = false;
-			for (Serializable currentState: listState.getList()) {
+			for (Tuple2<KinesisStreamShard, SequenceNumber> currentState: listState.getList()) {
 				hasOneIsSame = hasOneIsSame || state.equals(currentState);
 			}
 			assertEquals(true, hasOneIsSame);
@@ -693,8 +701,6 @@ public void testFetcherShouldNotBeRestoringFromFailureIfNotRestoringFromCheckpoi
 			"fakeStream", new Properties(), 10, 2);
 		consumer.open(new Configuration());
 		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
-
-		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(false);
 	}
 
 	@Test
@@ -718,7 +724,6 @@ public void testFetcherShouldBeCorrectlySeededIfRestoringFromLegacyCheckpoint()
 		consumer.open(new Configuration());
 		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
 
-		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
 			Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
 				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
@@ -728,15 +733,18 @@ public void testFetcherShouldBeCorrectlySeededIfRestoringFromLegacyCheckpoint()
 	@Test
 	@SuppressWarnings("unchecked")
 	public void testFetcherShouldBeCorrectlySeededIfRestoringFromCheckpoint() throws Exception {
+
 		// ----------------------------------------------------------------------
-		// setting initial state
+		// setup initial state
 		// ----------------------------------------------------------------------
+
 		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
 
 		// ----------------------------------------------------------------------
 		// mock operator state backend and initial state for initializeState()
 		// ----------------------------------------------------------------------
-		TestingListState<Serializable> listState = new TestingListState<>();
+
+		TestingListState<Tuple2<KinesisStreamShard, SequenceNumber>> listState = new TestingListState<>();
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> state: fakeRestoredState.entrySet()) {
 			listState.add(Tuple2.of(state.getKey(), state.getValue()));
 		}
@@ -751,6 +759,7 @@ public void testFetcherShouldBeCorrectlySeededIfRestoringFromCheckpoint() throws
 		// ----------------------------------------------------------------------
 		// mock fetcher
 		// ----------------------------------------------------------------------
+
 		KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
 		List<KinesisStreamShard> shards = new ArrayList<>();
 		shards.addAll(fakeRestoredState.keySet());
@@ -762,15 +771,15 @@ public void testFetcherShouldBeCorrectlySeededIfRestoringFromCheckpoint() throws
 		PowerMockito.doNothing().when(KinesisConfigUtil.class);
 
 		// ----------------------------------------------------------------------
-		// start to test seed initial state to fetcher
+		// start to test fetcher's initial state seeding
 		// ----------------------------------------------------------------------
+
 		TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
 			"fakeStream", new Properties(), 10, 2);
 		consumer.initializeState(initializationContext);
 		consumer.open(new Configuration());
 		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
 
-		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
 			Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
 				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
@@ -780,9 +789,11 @@ public void testFetcherShouldBeCorrectlySeededIfRestoringFromCheckpoint() throws
 	@Test
 	@SuppressWarnings("unchecked")
 	public void testFetcherShouldBeCorrectlySeededOnlyItsOwnStates() throws Exception {
+
 		// ----------------------------------------------------------------------
-		// setting initial state
+		// setup initial state
 		// ----------------------------------------------------------------------
+
 		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = getFakeRestoredStore("fakeStream1");
 
 		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredStateForOthers = getFakeRestoredStore("fakeStream2");
@@ -790,7 +801,8 @@ public void testFetcherShouldBeCorrectlySeededOnlyItsOwnStates() throws Exceptio
 		// ----------------------------------------------------------------------
 		// mock operator state backend and initial state for initializeState()
 		// ----------------------------------------------------------------------
-		TestingListState<Serializable> listState = new TestingListState<>();
+
+		TestingListState<Tuple2<KinesisStreamShard, SequenceNumber>> listState = new TestingListState<>();
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> state: fakeRestoredState.entrySet()) {
 			listState.add(Tuple2.of(state.getKey(), state.getValue()));
 		}
@@ -808,6 +820,7 @@ public void testFetcherShouldBeCorrectlySeededOnlyItsOwnStates() throws Exceptio
 		// ----------------------------------------------------------------------
 		// mock fetcher
 		// ----------------------------------------------------------------------
+
 		KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
 		List<KinesisStreamShard> shards = new ArrayList<>();
 		shards.addAll(fakeRestoredState.keySet());
@@ -819,15 +832,15 @@ public void testFetcherShouldBeCorrectlySeededOnlyItsOwnStates() throws Exceptio
 		PowerMockito.doNothing().when(KinesisConfigUtil.class);
 
 		// ----------------------------------------------------------------------
-		// start to test seed initial state to fetcher
+		// start to test fetcher's initial state seeding
 		// ----------------------------------------------------------------------
+
 		TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
 			"fakeStream", new Properties(), 10, 2);
 		consumer.initializeState(initializationContext);
 		consumer.open(new Configuration());
 		consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
 
-		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredStateForOthers.entrySet()) {
 			// should never get restored state not belonging to itself
 			Mockito.verify(mockedFetcher, never()).registerNewSubscribedShardState(
@@ -841,42 +854,49 @@ public void testFetcherShouldBeCorrectlySeededOnlyItsOwnStates() throws Exceptio
 	}
 
 	/*
-	 * If the original parallelism is 2 and states is:
+	 * This tests that the consumer correctly picks up shards that were not discovered on the previous run.
+	 *
+	 * Case under test:
+	 *
+	 * If the original parallelism is 2 and states are:
 	 *   Consumer subtask 1:
 	 *     stream1, shard1, SequentialNumber(xxx)
 	 *   Consumer subtask 2:
 	 *     stream1, shard2, SequentialNumber(yyy)
-	 * After discoverNewShardsToSubscribe() if there are two shards (shard3, shard4) been created:
+	 *
+	 * After discoverNewShardsToSubscribe() if there were two shards (shard3, shard4) created:
 	 *   Consumer subtask 1 (late for discoverNewShardsToSubscribe()):
 	 *     stream1, shard1, SequentialNumber(xxx)
 	 *   Consumer subtask 2:
 	 *     stream1, shard2, SequentialNumber(yyy)
 	 *     stream1, shard4, SequentialNumber(zzz)
-	 *  If snapshotState() occur and parallelism is changed to 1:
-	 *    Union state will be:
+	 *
+	 * If snapshotState() occurs and parallelism is changed to 1:
+	 *   Union state will be:
 	 *     stream1, shard1, SequentialNumber(xxx)
 	 *     stream1, shard2, SequentialNumber(yyy)
 	 *     stream1, shard4, SequentialNumber(zzz)
-	 *    Fetcher should be seeded with:
+	 *   Fetcher should be seeded with:
 	 *     stream1, shard1, SequentialNumber(xxx)
 	 *     stream1, shard2, SequentialNumber(yyy)
 	 *     stream1, share3, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM
 	 *     stream1, shard4, SequentialNumber(zzz)
-	 *
-	 *  This test is to guarantee the fetcher will be seeded correctly for such situation.
 	 */
 	@Test
 	@SuppressWarnings("unchecked")
 	public void testFetcherShouldBeCorrectlySeededWithNewDiscoveredKinesisStreamShard() throws Exception {
+
 		// ----------------------------------------------------------------------
-		// setting initial state
+		// setup initial state
 		// ----------------------------------------------------------------------
+
 		HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
 
 		// ----------------------------------------------------------------------
 		// mock operator state backend and initial state for initializeState()
 		// ----------------------------------------------------------------------
-		TestingListState<Serializable> listState = new TestingListState<>();
+
+		TestingListState<Tuple2<KinesisStreamShard, SequenceNumber>> listState = new TestingListState<>();
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> state: fakeRestoredState.entrySet()) {
 			listState.add(Tuple2.of(state.getKey(), state.getValue()));
 		}
@@ -891,6 +911,7 @@ public void testFetcherShouldBeCorrectlySeededWithNewDiscoveredKinesisStreamShar
 		// ----------------------------------------------------------------------
 		// mock fetcher
 		// ----------------------------------------------------------------------
+
 		KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
 		List<KinesisStreamShard> shards = new ArrayList<>();
 		shards.addAll(fakeRestoredState.keySet());
@@ -904,8 +925,9 @@ public void testFetcherShouldBeCorrectlySeededWithNewDiscoveredKinesisStreamShar
 		PowerMockito.doNothing().when(KinesisConfigUtil.class);
 
 		// ----------------------------------------------------------------------
-		// start to test seed initial state to fetcher
+		// start to test fetcher's initial state seeding
 		// ----------------------------------------------------------------------
+
 		TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
 			"fakeStream", new Properties(), 10, 2);
 		consumer.initializeState(initializationContext);
@@ -915,7 +937,6 @@ public void testFetcherShouldBeCorrectlySeededWithNewDiscoveredKinesisStreamShar
 		fakeRestoredState.put(new KinesisStreamShard("fakeStream2",
 				new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))),
 			SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get());
-		Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
 		for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
 			Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
 				new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
index e79f9b14628b6..800fde566318c 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
@@ -18,9 +18,14 @@
 package org.apache.flink.streaming.connectors.kinesis.internals;
 
 import com.amazonaws.services.kinesis.model.Shard;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.connectors.kinesis.FlinkKinesisConsumer;
+import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard;
 import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
 import org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisBehavioursFactory;
 import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
 import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher;
@@ -42,6 +47,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(TestableKinesisDataFetcher.class)
@@ -67,8 +74,6 @@ public void testIfNoShardsAreFoundShouldThrowException() throws Exception {
 				subscribedStreamsToLastSeenShardIdsUnderTest,
 				FakeKinesisBehavioursFactory.noShardsFoundForRequestedStreamsBehaviour());
 
-		fetcher.setIsRestoringFromFailure(false); // not restoring
-
 		fetcher.runFetcher(); // this should throw RuntimeException
 	}
 
@@ -100,23 +105,30 @@ public void testStreamToLastSeenShardStateIsCorrectlySetWhenNotRestoringFromFail
 				subscribedStreamsToLastSeenShardIdsUnderTest,
 				FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
 
-		fetcher.setIsRestoringFromFailure(false);
+		Properties testConfig = new Properties();
+		testConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
+		testConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+
+		final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(testConfig, fetcher);
 
 		PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
-		Thread runFetcherThread = new Thread(new Runnable() {
+		Thread consumerThread = new Thread(new Runnable() {
 			@Override
 			public void run() {
 				try {
-					fetcher.runFetcher();
+					consumer.run(mock(SourceFunction.SourceContext.class));
 				} catch (Exception e) {
 					//
 				}
 			}
 		});
-		runFetcherThread.start();
-		Thread.sleep(1000); // sleep a while before closing
-		fetcher.shutdownFetcher();
+		consumerThread.start();
 
+		fetcher.waitUntilRun();
+		consumer.cancel();
+		consumerThread.join();
 
 		// assert that the streams tracked in the state are identical to the subscribed streams
 		Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
@@ -192,8 +204,6 @@ public void testStreamToLastSeenShardStateIsCorrectlySetWhenNoNewShardsSinceRest
 				new KinesisStreamShardState(restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
 		}
 
-		fetcher.setIsRestoringFromFailure(true);
-
 		PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
 		Thread runFetcherThread = new Thread(new Runnable() {
 			@Override
@@ -284,8 +294,6 @@ public void testStreamToLastSeenShardStateIsCorrectlySetWhenNewShardsFoundSinceR
 				new KinesisStreamShardState(restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
 		}
 
-		fetcher.setIsRestoringFromFailure(true);
-
 		PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
 		Thread runFetcherThread = new Thread(new Runnable() {
 			@Override
@@ -380,8 +388,6 @@ public void testStreamToLastSeenShardStateIsCorrectlySetWhenNoNewShardsSinceRest
 				new KinesisStreamShardState(restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
 		}
 
-		fetcher.setIsRestoringFromFailure(true);
-
 		PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
 		Thread runFetcherThread = new Thread(new Runnable() {
 			@Override
@@ -477,8 +483,6 @@ public void testStreamToLastSeenShardStateIsCorrectlySetWhenNewShardsFoundSinceR
 				new KinesisStreamShardState(restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
 		}
 
-		fetcher.setIsRestoringFromFailure(true);
-
 		PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
 		Thread runFetcherThread = new Thread(new Runnable() {
 			@Override
@@ -507,4 +511,33 @@ public void run() {
 		assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3") == null);
 		assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4") == null);
 	}
+
+	private static class DummyFlinkKafkaConsumer<T> extends FlinkKinesisConsumer<T> {
+		private static final long serialVersionUID = 1L;
+
+		private KinesisDataFetcher<T> fetcher;
+
+		@SuppressWarnings("unchecked")
+		DummyFlinkKafkaConsumer(Properties properties, KinesisDataFetcher<T> fetcher) {
+			super("test", mock(KinesisDeserializationSchema.class), properties);
+			this.fetcher = fetcher;
+		}
+
+		@Override
+		protected KinesisDataFetcher<T> createFetcher(List<String> streams,
+													  SourceFunction.SourceContext<T> sourceContext,
+													  RuntimeContext runtimeContext,
+													  Properties configProps,
+													  KinesisDeserializationSchema<T> deserializationSchema) {
+			return fetcher;
+		}
+
+		@Override
+		public RuntimeContext getRuntimeContext() {
+			RuntimeContext context = mock(RuntimeContext.class);
+			when(context.getIndexOfThisSubtask()).thenReturn(0);
+			when(context.getNumberOfParallelSubtasks()).thenReturn(1);
+			return context;
+		}
+	}
 }
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
index 57886fe694fef..bb644ba7070e6 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
@@ -18,6 +18,7 @@
 package org.apache.flink.streaming.connectors.kinesis.testutils;
 
 import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.core.testutils.OneShotLatch;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
 import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
@@ -42,6 +43,8 @@ public class TestableKinesisDataFetcher extends KinesisDataFetcher<String> {
 
 	private long numElementsCollected;
 
+	private OneShotLatch runWaiter;
+
 	public TestableKinesisDataFetcher(List<String> fakeStreams,
 									  Properties fakeConfiguration,
 									  int fakeTotalCountOfSubtasks,
@@ -62,6 +65,7 @@ public TestableKinesisDataFetcher(List<String> fakeStreams,
 			fakeKinesis);
 
 		this.numElementsCollected = 0;
+		this.runWaiter = new OneShotLatch();
 	}
 
 	public long getNumOfElementsCollected() {
@@ -81,6 +85,16 @@ protected void emitRecordAndUpdateState(String record, long recordTimestamp, int
 		}
 	}
 
+	@Override
+	public void runFetcher() throws Exception {
+		runWaiter.trigger();
+		super.runFetcher();
+	}
+
+	public void waitUntilRun() throws Exception {
+		runWaiter.await();
+	}
+
 	@SuppressWarnings("unchecked")
 	private static SourceFunction.SourceContext<String> getMockedSourceContext() {
 		return Mockito.mock(SourceFunction.SourceContext.class);

From 0b41889559efa0f89158a58c8c211ccd3b8e73a2 Mon Sep 17 00:00:00 2001
From: sunjincheng121 <sunjincheng121@gmail.com>
Date: Fri, 7 Apr 2017 19:28:19 +0800
Subject: [PATCH 180/253] [FLINK-6257] [table] Refactor OVER window tests.

This closes #3697.
---
 .../scala/stream/sql/OverWindowITCase.scala   | 878 ++++++++++++++++
 .../api/scala/stream/sql/OverWindowTest.scala | 503 +++++++++
 .../api/scala/stream/sql/SqlITCase.scala      | 992 +-----------------
 .../stream/sql/WindowAggregateTest.scala      | 380 +------
 .../runtime/harness/HarnessTestBase.scala     |  87 ++
 .../harness/OverWindowHarnessTest.scala       | 974 +++++++++++++++++
 6 files changed, 2444 insertions(+), 1370 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowITCase.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowTest.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala

diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowITCase.scala
new file mode 100644
index 0000000000000..a7fe1c419b3a7
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowITCase.scala
@@ -0,0 +1,878 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream.sql
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.scala.stream.sql.OverWindowITCase.EventTimeSourceFunction
+import org.apache.flink.streaming.api.functions.source.SourceFunction
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.table.api.{TableEnvironment, TableException}
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
+
+import scala.collection.mutable
+
+class OverWindowITCase extends StreamingWithStateTestBase {
+
+  val data = List(
+    (1L, 1, "Hello"),
+    (2L, 2, "Hello"),
+    (3L, 3, "Hello"),
+    (4L, 4, "Hello"),
+    (5L, 5, "Hello"),
+    (6L, 6, "Hello"),
+    (7L, 7, "Hello World"),
+    (8L, 8, "Hello World"),
+    (20L, 20, "Hello World"))
+
+  /**
+    * All aggregates must be computed on the same window.
+    */
+  @Test(expected = classOf[TableException])
+  def testMultiWindow(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY b ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
+      "from T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+  }
+
+  @Test
+  def testProcTimeBoundedPartitionedRowsOver(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setParallelism(1)
+    StreamITCase.clear
+
+    val t = StreamTestData.get5TupleDataStream(env)
+      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
+    tEnv.registerTable("MyTable", t)
+
+    val sqlQuery = "SELECT a, " +
+      "  SUM(c) OVER (" +
+      "    PARTITION BY a ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS sumC, " +
+      "  MIN(c) OVER (" +
+      "    PARTITION BY a ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS minC " +
+      "FROM MyTable"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "1,0,0",
+      "2,1,1",
+      "2,3,1",
+      "3,3,3",
+      "3,7,3",
+      "3,12,3",
+      "4,6,6",
+      "4,13,6",
+      "4,21,6",
+      "4,30,6",
+      "5,10,10",
+      "5,21,10",
+      "5,33,10",
+      "5,46,10",
+      "5,60,10")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testProcTimeBoundedNonPartitionedRowsOver(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
+    StreamITCase.clear
+
+    val t = StreamTestData.get5TupleDataStream(env)
+      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
+    tEnv.registerTable("MyTable", t)
+
+    val sqlQuery = "SELECT a, " +
+      "  SUM(c) OVER (" +
+      "    ORDER BY proctime ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS sumC , " +
+      "  MIN(c) OVER (" +
+      "    ORDER BY proctime ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS minC " +
+      "FROM MyTable"
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "1,0,0",
+      "2,1,0",
+      "2,3,0",
+      "3,6,0",
+      "3,10,0",
+      "3,15,0",
+      "4,21,0",
+      "4,28,0",
+      "4,36,0",
+      "4,45,0",
+      "5,55,0",
+      "5,66,1",
+      "5,77,2",
+      "5,88,3",
+      "5,99,4")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testProcTimeUnboundedPartitionedRangeOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    // for sum aggregation ensure that every time the order of each element is consistent
+    env.setParallelism(1)
+
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
+      "from T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "Hello World,1,7", "Hello World,2,15", "Hello World,3,35",
+      "Hello,1,1", "Hello,2,3", "Hello,3,6", "Hello,4,10", "Hello,5,15", "Hello,6,21")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testProcTimeUnboundedPartitionedRowsOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "c, " +
+      "count(a) " +
+      " OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND CURRENT ROW)" +
+      "from T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "Hello World,1", "Hello World,2", "Hello World,3",
+      "Hello,1", "Hello,2", "Hello,3", "Hello,4", "Hello,5", "Hello,6")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testProcTimeUnboundedNonPartitionedRangeOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    // for sum aggregation ensure that every time the order of each element is consistent
+    env.setParallelism(1)
+
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY proctime  RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
+      "from T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "Hello World,7,28", "Hello World,8,36", "Hello World,9,56",
+      "Hello,1,1", "Hello,2,3", "Hello,3,6", "Hello,4,10", "Hello,5,15", "Hello,6,21")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testProcTimeUnboundedNonPartitionedRowsOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "count(a) OVER (ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND CURRENT ROW)" +
+      "from T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List("1", "2", "3", "4", "5", "6", "7", "8", "9")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRangeOver(): Unit = {
+    val data = Seq(
+      Left((1500L, (1L, 15, "Hello"))),
+      Left((1600L, (1L, 16, "Hello"))),
+      Left((1000L, (1L, 1, "Hello"))),
+      Left((2000L, (2L, 2, "Hello"))),
+      Right(1000L),
+      Left((2000L, (2L, 2, "Hello"))),
+      Left((2000L, (2L, 3, "Hello"))),
+      Left((3000L, (3L, 3, "Hello"))),
+      Right(2000L),
+      Left((4000L, (4L, 4, "Hello"))),
+      Right(3000L),
+      Left((5000L, (5L, 5, "Hello"))),
+      Right(5000L),
+      Left((6000L, (6L, 6, "Hello"))),
+      Left((6500L, (6L, 65, "Hello"))),
+      Right(7000L),
+      Left((9000L, (6L, 9, "Hello"))),
+      Left((9500L, (6L, 18, "Hello"))),
+      Left((9000L, (6L, 9, "Hello"))),
+      Right(10000L),
+      Left((10000L, (7L, 7, "Hello World"))),
+      Left((11000L, (7L, 17, "Hello World"))),
+      Left((11000L, (7L, 77, "Hello World"))),
+      Right(12000L),
+      Left((14000L, (7L, 18, "Hello World"))),
+      Right(14000L),
+      Left((15000L, (8L, 8, "Hello World"))),
+      Right(17000L),
+      Left((20000L, (20L, 20, "Hello World"))),
+      Right(19000L))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t1 = env
+      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "  c, b, " +
+      "  COUNT(a) OVER (PARTITION BY c ORDER BY rowtime RANGE " +
+      "    BETWEEN INTERVAL '1' SECOND PRECEDING AND CURRENT ROW), " +
+      "  SUM(a) OVER (PARTITION BY c ORDER BY rowtime RANGE " +
+      "    BETWEEN INTERVAL '1' SECOND PRECEDING AND CURRENT ROW)" +
+      " FROM T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
+      "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
+      "Hello,3,4,9",
+      "Hello,4,2,7",
+      "Hello,5,2,9",
+      "Hello,6,2,11", "Hello,65,2,12",
+      "Hello,9,2,12", "Hello,9,2,12", "Hello,18,3,18",
+      "Hello World,7,1,7", "Hello World,17,3,21", "Hello World,77,3,21", "Hello World,18,1,7",
+      "Hello World,8,2,15",
+      "Hello World,20,1,20")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRowsOver(): Unit = {
+    val data = Seq(
+      Left((1L, (1L, 1, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((1L, (1L, 1, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((1L, (1L, 1, "Hello"))),
+      Left((3L, (7L, 7, "Hello World"))),
+      Left((1L, (7L, 7, "Hello World"))),
+      Left((1L, (7L, 7, "Hello World"))),
+      Right(2L),
+      Left((3L, (3L, 3, "Hello"))),
+      Left((4L, (4L, 4, "Hello"))),
+      Left((5L, (5L, 5, "Hello"))),
+      Left((6L, (6L, 6, "Hello"))),
+      Left((20L, (20L, 20, "Hello World"))),
+      Right(6L),
+      Left((8L, (8L, 8, "Hello World"))),
+      Left((7L, (7L, 7, "Hello World"))),
+      Right(20L))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t1 = env
+      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      " c, a, " +
+      "  COUNT(a) " +
+      "    OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW), " +
+      "  SUM(a) " +
+      "    OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) " +
+      "FROM T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
+      "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
+      "Hello,3,3,7", "Hello,4,3,9", "Hello,5,3,12",
+      "Hello,6,3,15",
+      "Hello World,7,1,7", "Hello World,7,2,14", "Hello World,7,3,21",
+      "Hello World,7,3,21", "Hello World,8,3,22", "Hello World,20,3,35")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeBoundedNonPartitionedRangeOver(): Unit = {
+    val data = Seq(
+      Left((1500L, (1L, 15, "Hello"))),
+      Left((1600L, (1L, 16, "Hello"))),
+      Left((1000L, (1L, 1, "Hello"))),
+      Left((2000L, (2L, 2, "Hello"))),
+      Right(1000L),
+      Left((2000L, (2L, 2, "Hello"))),
+      Left((2000L, (2L, 3, "Hello"))),
+      Left((3000L, (3L, 3, "Hello"))),
+      Right(2000L),
+      Left((4000L, (4L, 4, "Hello"))),
+      Right(3000L),
+      Left((5000L, (5L, 5, "Hello"))),
+      Right(5000L),
+      Left((6000L, (6L, 6, "Hello"))),
+      Left((6500L, (6L, 65, "Hello"))),
+      Right(7000L),
+      Left((9000L, (6L, 9, "Hello"))),
+      Left((9500L, (6L, 18, "Hello"))),
+      Left((9000L, (6L, 9, "Hello"))),
+      Right(10000L),
+      Left((10000L, (7L, 7, "Hello World"))),
+      Left((11000L, (7L, 17, "Hello World"))),
+      Left((11000L, (7L, 77, "Hello World"))),
+      Right(12000L),
+      Left((14000L, (7L, 18, "Hello World"))),
+      Right(14000L),
+      Left((15000L, (8L, 8, "Hello World"))),
+      Right(17000L),
+      Left((20000L, (20L, 20, "Hello World"))),
+      Right(19000L))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t1 = env
+      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "  c, b, " +
+      "  COUNT(a) " +
+      "    OVER (ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND PRECEDING AND CURRENT ROW), " +
+      "  SUM(a) " +
+      "    OVER (ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND PRECEDING AND CURRENT ROW) " +
+      " FROM T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
+      "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
+      "Hello,3,4,9",
+      "Hello,4,2,7",
+      "Hello,5,2,9",
+      "Hello,6,2,11", "Hello,65,2,12",
+      "Hello,9,2,12", "Hello,9,2,12", "Hello,18,3,18",
+      "Hello World,7,4,25", "Hello World,17,3,21", "Hello World,77,3,21", "Hello World,18,1,7",
+      "Hello World,8,2,15",
+      "Hello World,20,1,20")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeBoundedNonPartitionedRowsOver(): Unit = {
+
+    val data = Seq(
+      Left((2L, (2L, 2, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((1L, (1L, 1, "Hello"))),
+      Left((1L, (1L, 1, "Hello"))),
+      Left((2L, (2L, 2, "Hello"))),
+      Left((1L, (1L, 1, "Hello"))),
+      Left((20L, (20L, 20, "Hello World"))), // early row
+      Right(3L),
+      Left((2L, (2L, 2, "Hello"))), // late row
+      Left((3L, (3L, 3, "Hello"))),
+      Left((4L, (4L, 4, "Hello"))),
+      Left((5L, (5L, 5, "Hello"))),
+      Left((6L, (6L, 6, "Hello"))),
+      Left((7L, (7L, 7, "Hello World"))),
+      Right(7L),
+      Left((9L, (9L, 9, "Hello World"))),
+      Left((8L, (8L, 8, "Hello World"))),
+      Left((8L, (8L, 8, "Hello World"))),
+      Right(20L))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val t1 = env
+      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val sqlQuery = "SELECT " +
+      "c, a, " +
+      "  COUNT(a) OVER (ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW), " +
+      "  SUM(a) OVER (ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW) " +
+      "FROM T1"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
+      "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
+      "Hello,3,3,7",
+      "Hello,4,3,9", "Hello,5,3,12",
+      "Hello,6,3,15", "Hello World,7,3,18",
+      "Hello World,8,3,21", "Hello World,8,3,23",
+      "Hello World,9,3,25",
+      "Hello World,20,3,37")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeUnBoundedPartitionedRangeOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
+    StreamITCase.clear
+
+    val sqlQuery = "SELECT a, b, c, " +
+      "  SUM(b) OVER (" +
+      "    PARTITION BY a ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  COUNT(b) OVER (" +
+      "    PARTITION BY a ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  AVG(b) OVER (" +
+      "    PARTITION BY a ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  MAX(b) OVER (" +
+      "    PARTITION BY a ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  MIN(b) OVER (" +
+      "    PARTITION BY a ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) " +
+      "FROM T1"
+
+    val data = Seq(
+      Left(14000005L, (1, 1L, "Hi")),
+      Left(14000000L, (2, 1L, "Hello")),
+      Left(14000002L, (1, 1L, "Hello")),
+      Left(14000002L, (1, 2L, "Hello")),
+      Left(14000002L, (1, 3L, "Hello world")),
+      Left(14000003L, (2, 2L, "Hello world")),
+      Left(14000003L, (2, 3L, "Hello world")),
+      Right(14000020L),
+      Left(14000021L, (1, 4L, "Hello world")),
+      Left(14000022L, (1, 5L, "Hello world")),
+      Left(14000022L, (1, 6L, "Hello world")),
+      Left(14000022L, (1, 7L, "Hello world")),
+      Left(14000023L, (2, 4L, "Hello world")),
+      Left(14000023L, (2, 5L, "Hello world")),
+      Right(14000030L))
+
+    val t1 = env
+      .addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "1,1,Hello,6,3,2,3,1",
+      "1,2,Hello,6,3,2,3,1",
+      "1,3,Hello world,6,3,2,3,1",
+      "1,1,Hi,7,4,1,3,1",
+      "2,1,Hello,1,1,1,1,1",
+      "2,2,Hello world,6,3,2,3,1",
+      "2,3,Hello world,6,3,2,3,1",
+      "1,4,Hello world,11,5,2,4,1",
+      "1,5,Hello world,29,8,3,7,1",
+      "1,6,Hello world,29,8,3,7,1",
+      "1,7,Hello world,29,8,3,7,1",
+      "2,4,Hello world,15,5,3,5,1",
+      "2,5,Hello world,15,5,3,5,1")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeUnBoundedPartitionedRowsOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val sqlQuery = "SELECT a, b, c, " +
+      "SUM(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "count(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "avg(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "max(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "min(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row) " +
+      "from T1"
+
+    val data = Seq(
+      Left(14000005L, (1, 1L, "Hi")),
+      Left(14000000L, (2, 1L, "Hello")),
+      Left(14000002L, (3, 1L, "Hello")),
+      Left(14000003L, (1, 2L, "Hello")),
+      Left(14000004L, (1, 3L, "Hello world")),
+      Left(14000007L, (3, 2L, "Hello world")),
+      Left(14000008L, (2, 2L, "Hello world")),
+      Right(14000010L),
+      Left(14000012L, (1, 5L, "Hello world")),
+      Left(14000021L, (1, 6L, "Hello world")),
+      Left(14000023L, (2, 5L, "Hello world")),
+      Right(14000020L),
+      Left(14000024L, (3, 5L, "Hello world")),
+      Left(14000026L, (1, 7L, "Hello world")),
+      Left(14000025L, (1, 8L, "Hello world")),
+      Left(14000022L, (1, 9L, "Hello world")),
+      Right(14000030L))
+
+    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
+             .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "1,2,Hello,2,1,2,2,2",
+      "1,3,Hello world,5,2,2,3,2",
+      "1,1,Hi,6,3,2,3,1",
+      "2,1,Hello,1,1,1,1,1",
+      "2,2,Hello world,3,2,1,2,1",
+      "3,1,Hello,1,1,1,1,1",
+      "3,2,Hello world,3,2,1,2,1",
+      "1,5,Hello world,11,4,2,5,1",
+      "1,6,Hello world,17,5,3,6,1",
+      "1,9,Hello world,26,6,4,9,1",
+      "1,8,Hello world,34,7,4,9,1",
+      "1,7,Hello world,41,8,5,9,1",
+      "2,5,Hello world,8,3,2,5,1",
+      "3,5,Hello world,8,3,2,5,1")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeUnBoundedNonPartitionedRangeOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    env.setParallelism(1)
+    StreamITCase.clear
+
+    val sqlQuery = "SELECT a, b, c, " +
+      "  SUM(b) OVER (ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  COUNT(b) OVER (ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  AVG(b) OVER (ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  MAX(b) OVER (ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  MIN(b) OVER (ORDER BY rowtime RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) " +
+      "FROM T1"
+
+    val data = Seq(
+      Left(14000005L, (1, 1L, "Hi")),
+      Left(14000000L, (2, 1L, "Hello")),
+      Left(14000002L, (1, 1L, "Hello")),
+      Left(14000002L, (1, 2L, "Hello")),
+      Left(14000002L, (1, 3L, "Hello world")),
+      Left(14000003L, (2, 2L, "Hello world")),
+      Left(14000003L, (2, 3L, "Hello world")),
+      Right(14000020L),
+      Left(14000021L, (1, 4L, "Hello world")),
+      Left(14000022L, (1, 5L, "Hello world")),
+      Left(14000022L, (1, 6L, "Hello world")),
+      Left(14000022L, (1, 7L, "Hello world")),
+      Left(14000023L, (2, 4L, "Hello world")),
+      Left(14000023L, (2, 5L, "Hello world")),
+      Right(14000030L))
+
+    val t1 = env
+      .addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "2,1,Hello,1,1,1,1,1",
+      "1,1,Hello,7,4,1,3,1",
+      "1,2,Hello,7,4,1,3,1",
+      "1,3,Hello world,7,4,1,3,1",
+      "2,2,Hello world,12,6,2,3,1",
+      "2,3,Hello world,12,6,2,3,1",
+      "1,1,Hi,13,7,1,3,1",
+      "1,4,Hello world,17,8,2,4,1",
+      "1,5,Hello world,35,11,3,7,1",
+      "1,6,Hello world,35,11,3,7,1",
+      "1,7,Hello world,35,11,3,7,1",
+      "2,4,Hello world,44,13,3,7,1",
+      "2,5,Hello world,44,13,3,7,1")
+
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testRowTimeUnBoundedNonPartitionedRowsOver(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    StreamITCase.clear
+    env.setParallelism(1)
+
+    val sqlQuery = "SELECT a, b, c, " +
+      "  SUM(b) OVER (ORDER BY rowtime ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  COUNT(b) OVER (ORDER BY rowtime ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  AVG(b) OVER (ORDER BY rowtime ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  MAX(b) OVER (ORDER BY rowtime ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " +
+      "  MIN(b) OVER (ORDER BY rowtime ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) " +
+      "FROM T1"
+
+    val data = Seq(
+      Left(14000005L, (1, 1L, "Hi")),
+      Left(14000000L, (2, 2L, "Hello")),
+      Left(14000002L, (3, 5L, "Hello")),
+      Left(14000003L, (1, 3L, "Hello")),
+      Left(14000004L, (3, 7L, "Hello world")),
+      Left(14000007L, (4, 9L, "Hello world")),
+      Left(14000008L, (5, 8L, "Hello world")),
+      Right(14000010L),
+      // this element will be discard because it is late
+      Left(14000008L, (6, 8L, "Hello world")),
+      Right(14000020L),
+      Left(14000021L, (6, 8L, "Hello world")),
+      Right(14000030L)
+    )
+
+    val t1 = env
+      .addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList(
+      "2,2,Hello,2,1,2,2,2",
+      "3,5,Hello,7,2,3,5,2",
+      "1,3,Hello,10,3,3,5,2",
+      "3,7,Hello world,17,4,4,7,2",
+      "1,1,Hi,18,5,3,7,1",
+      "4,9,Hello world,27,6,4,9,1",
+      "5,8,Hello world,35,7,5,9,1",
+      "6,8,Hello world,43,8,5,9,1")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+//  <<<<<<< HEAD
+
+
+  /** test sliding event-time unbounded window with partition by **/
+  @Test
+  def testRowTimeUnBoundedPartitionedRowsOver2(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    env.setStateBackend(getStateBackend)
+    StreamITCase.clear
+    env.setParallelism(1)
+
+    val sqlQuery = "SELECT a, b, c, " +
+      "SUM(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "count(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "avg(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "max(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row), " +
+      "min(b) over (" +
+      "partition by a order by rowtime rows between unbounded preceding and current row) " +
+      "from T1"
+
+    val data = Seq(
+      Left(14000005L, (1, 1L, "Hi")),
+      Left(14000000L, (2, 1L, "Hello")),
+      Left(14000002L, (3, 1L, "Hello")),
+      Left(14000003L, (1, 2L, "Hello")),
+      Left(14000004L, (1, 3L, "Hello world")),
+      Left(14000007L, (3, 2L, "Hello world")),
+      Left(14000008L, (2, 2L, "Hello world")),
+      Right(14000010L),
+      // the next 3 elements are late
+      Left(14000008L, (1, 4L, "Hello world")),
+      Left(14000008L, (2, 3L, "Hello world")),
+      Left(14000008L, (3, 3L, "Hello world")),
+      Left(14000012L, (1, 5L, "Hello world")),
+      Right(14000020L),
+      Left(14000021L, (1, 6L, "Hello world")),
+      // the next 3 elements are late
+      Left(14000019L, (1, 6L, "Hello world")),
+      Left(14000018L, (2, 4L, "Hello world")),
+      Left(14000018L, (3, 4L, "Hello world")),
+      Left(14000022L, (2, 5L, "Hello world")),
+      Left(14000022L, (3, 5L, "Hello world")),
+      Left(14000024L, (1, 7L, "Hello world")),
+      Left(14000023L, (1, 8L, "Hello world")),
+      Left(14000021L, (1, 9L, "Hello world")),
+      Right(14000030L)
+    )
+
+    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
+      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+    tEnv.registerTable("T1", t1)
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "1,2,Hello,2,1,2,2,2",
+      "1,3,Hello world,5,2,2,3,2",
+      "1,1,Hi,6,3,2,3,1",
+      "2,1,Hello,1,1,1,1,1",
+      "2,2,Hello world,3,2,1,2,1",
+      "3,1,Hello,1,1,1,1,1",
+      "3,2,Hello world,3,2,1,2,1",
+      "1,5,Hello world,11,4,2,5,1",
+      "1,6,Hello world,17,5,3,6,1",
+      "1,9,Hello world,26,6,4,9,1",
+      "1,8,Hello world,34,7,4,9,1",
+      "1,7,Hello world,41,8,5,9,1",
+      "2,5,Hello world,8,3,2,5,1",
+      "3,5,Hello world,8,3,2,5,1"
+    )
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+}
+
+object OverWindowITCase {
+
+  class EventTimeSourceFunction[T](
+      dataWithTimestampList: Seq[Either[(Long, T), Long]]) extends SourceFunction[T] {
+    override def run(ctx: SourceContext[T]): Unit = {
+      dataWithTimestampList.foreach {
+        case Left(t) => ctx.collectWithTimestamp(t._2, t._1)
+        case Right(w) => ctx.emitWatermark(new Watermark(w))
+      }
+    }
+
+    override def cancel(): Unit = ???
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowTest.scala
new file mode 100644
index 0000000000000..711b31bd76ef1
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/OverWindowTest.scala
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.api.scala.stream.sql
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.utils.TableTestUtil._
+import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase}
+import org.junit.Test
+
+class OverWindowTest extends TableTestBase {
+  private val streamUtil: StreamTableTestUtil = streamTestUtil()
+  streamUtil.addTable[(Int, String, Long)](
+    "MyTable",
+    'a, 'b, 'c,
+    'proctime.proctime, 'rowtime.rowtime)
+
+  @Test
+  def testProcTimeBoundedPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN 2 preceding AND " +
+      "CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "proctime")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "proctime"),
+          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testProcTimeBoundedPartitionedRangeOver() = {
+
+    val sqlQuery =
+      "SELECT a, " +
+        "  AVG(c) OVER (PARTITION BY a ORDER BY proctime " +
+        "    RANGE BETWEEN INTERVAL '2' HOUR PRECEDING AND CURRENT ROW) AS avgA " +
+        "FROM MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "proctime")
+          ),
+          term("partitionBy", "a"),
+          term("orderBy", "proctime"),
+          term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "proctime",
+            "COUNT(c) AS w0$o0",
+            "$SUM0(c) AS w0$o1"
+          )
+        ),
+        term("select", "a", "/(CASE(>(w0$o0, 0)", "CAST(w0$o1), null), w0$o0) AS avgA")
+      )
+
+    streamUtil.verifySql(sqlQuery, expected)
+  }
+
+  @Test
+  def testProcTimeBoundedNonPartitionedRangeOver() = {
+
+    val sqlQuery =
+      "SELECT a, " +
+        "  COUNT(c) OVER (ORDER BY proctime " +
+        "    RANGE BETWEEN INTERVAL '10' SECOND PRECEDING AND CURRENT ROW) AS countA " +
+      "FROM MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "proctime")
+          ),
+          term("orderBy", "proctime"),
+          term("range", "BETWEEN 10000 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "proctime", "COUNT(c) AS w0$o0")
+        ),
+        term("select", "a", "w0$o0 AS $1")
+      )
+
+    streamUtil.verifySql(sqlQuery, expected)
+  }
+
+  @Test
+  def testProcTimeBoundedNonPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY proctime ROWS BETWEEN 2 preceding AND " +
+      "CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "proctime")
+          ),
+          term("orderBy", "proctime"),
+          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+
+  @Test
+  def testProcTimeUnboundedPartitionedRangeOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "proctime")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "proctime"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+        ),
+        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testProcTimeUnboundedPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
+      "CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          streamTableNode(0),
+          term("partitionBy", "c"),
+          term("orderBy", "proctime"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "b", "c", "proctime", "rowtime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testProcTimeUnboundedNonPartitionedRangeOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "proctime")
+          ),
+          term("orderBy", "proctime"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+        ),
+        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testProcTimeUnboundedNonPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
+      "CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          streamTableNode(0),
+          term("orderBy", "proctime"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "b", "c", "proctime", "rowtime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 5 preceding AND " +
+      "CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "rowtime"),
+          term("rows", "BETWEEN 5 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedPartitionedRangeOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime " +
+      "RANGE BETWEEN INTERVAL '1' SECOND  preceding AND CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "rowtime"),
+          term("range", "BETWEEN 1000 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedNonPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY rowtime ROWS BETWEEN 5 preceding AND " +
+      "CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("orderBy", "rowtime"),
+          term("rows", "BETWEEN 5 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeBoundedNonPartitionedRangeOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY rowtime " +
+      "RANGE BETWEEN INTERVAL '1' SECOND  preceding AND CURRENT ROW) as cnt1 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("orderBy", "rowtime"),
+          term("range", "BETWEEN 1000 PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
+        ),
+        term("select", "c", "w0$o0 AS $1")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedPartitionedRangeOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY c ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt2 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "rowtime"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+        ),
+        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (PARTITION BY c ORDER BY rowtime ROWS UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (PARTITION BY c ORDER BY rowtime ROWS UNBOUNDED preceding) as cnt2 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("partitionBy", "c"),
+          term("orderBy", "rowtime"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "rowtime",
+            "COUNT(a) AS w0$o0",
+            "$SUM0(a) AS w0$o1"
+          )
+        ),
+        term(
+          "select",
+          "c",
+          "w0$o0 AS cnt1",
+          "CASE(>(w0$o0, 0)",
+          "CAST(w0$o1), null) AS cnt2"
+        )
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedNonPartitionedRangeOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt2 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("orderBy", "rowtime"),
+          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
+        ),
+        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+  @Test
+  def testRowTimeUnboundedNonPartitionedRowsOver() = {
+    val sql = "SELECT " +
+      "c, " +
+      "count(a) OVER (ORDER BY rowtime ROWS UNBOUNDED preceding) as cnt1, " +
+      "sum(a) OVER (ORDER BY rowtime ROWS UNBOUNDED preceding) as cnt2 " +
+      "from MyTable"
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamOverAggregate",
+          unaryNode(
+            "DataStreamCalc",
+            streamTableNode(0),
+            term("select", "a", "c", "rowtime")
+          ),
+          term("orderBy", "rowtime"),
+          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
+          term(
+            "select",
+            "a",
+            "c",
+            "rowtime",
+            "COUNT(a) AS w0$o0",
+            "$SUM0(a) AS w0$o1"
+          )
+        ),
+        term(
+          "select",
+          "c",
+          "w0$o0 AS cnt1",
+          "CASE(>(w0$o0, 0)",
+          "CAST(w0$o1), null) AS cnt2"
+        )
+      )
+    streamUtil.verifySql(sql, expected)
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
index 249d5052cf828..95366e1fef6d8 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
@@ -19,34 +19,16 @@
 package org.apache.flink.table.api.scala.stream.sql
 
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.scala.stream.sql.SqlITCase.EventTimeSourceFunction
-import org.apache.flink.streaming.api.functions.source.SourceFunction
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
-import org.apache.flink.streaming.api.watermark.Watermark
-import org.apache.flink.table.api.{TableEnvironment, TableException}
+import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
 import org.apache.flink.types.Row
 import org.junit.Assert._
 import org.junit._
-import org.apache.flink.streaming.api.TimeCharacteristic
-import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
-
-import scala.collection.mutable
 
 class SqlITCase extends StreamingWithStateTestBase {
 
-  val data = List(
-    (1L, 1, "Hello"),
-    (2L, 2, "Hello"),
-    (3L, 3, "Hello"),
-    (4L, 4, "Hello"),
-    (5L, 5, "Hello"),
-    (6L, 6, "Hello"),
-    (7L, 7, "Hello World"),
-    (8L, 8, "Hello World"),
-    (20L, 20, "Hello World"))
-
   /** test unbounded groupby (without window) **/
   @Test
   def testUnboundedGroupby(): Unit = {
@@ -208,976 +190,4 @@ class SqlITCase extends StreamingWithStateTestBase {
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
-  @Test
-  def testUnboundPartitionedProcessingWindowWithRange(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    // for sum aggregation ensure that every time the order of each element is consistent
-    env.setParallelism(1)
-
-    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY proctime  RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
-      "from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "Hello World,1,7", "Hello World,2,15", "Hello World,3,35",
-      "Hello,1,1", "Hello,2,3", "Hello,3,6", "Hello,4,10", "Hello,5,15", "Hello,6,21")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testUnboundPartitionedProcessingWindowWithRow(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
-      "CURRENT ROW)" +
-      "from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "Hello World,1", "Hello World,2", "Hello World,3",
-      "Hello,1", "Hello,2", "Hello,3", "Hello,4", "Hello,5", "Hello,6")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testUnboundNonPartitionedProcessingWindowWithRange(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    // for sum aggregation ensure that every time the order of each element is consistent
-    env.setParallelism(1)
-
-    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, " +
-      "count(a) OVER (ORDER BY proctime  RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
-      "from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "Hello World,7,28", "Hello World,8,36", "Hello World,9,56",
-      "Hello,1,1", "Hello,2,3", "Hello,3,6", "Hello,4,10", "Hello,5,15", "Hello,6,21")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testUnboundNonPartitionedProcessingWindowWithRow(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "count(a) OVER (ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND CURRENT ROW)" +
-      "from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List("1", "2", "3", "4", "5", "6", "7", "8", "9")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testBoundPartitionedEventTimeWindowWithRow(): Unit = {
-    val data = Seq(
-      Left((1L, (1L, 1, "Hello"))),
-      Left((2L, (2L, 2, "Hello"))),
-      Left((1L, (1L, 1, "Hello"))),
-      Left((2L, (2L, 2, "Hello"))),
-      Left((2L, (2L, 2, "Hello"))),
-      Left((1L, (1L, 1, "Hello"))),
-      Left((3L, (7L, 7, "Hello World"))),
-      Left((1L, (7L, 7, "Hello World"))),
-      Left((1L, (7L, 7, "Hello World"))),
-      Right(2L),
-      Left((3L, (3L, 3, "Hello"))),
-      Left((4L, (4L, 4, "Hello"))),
-      Left((5L, (5L, 5, "Hello"))),
-      Left((6L, (6L, 6, "Hello"))),
-      Left((20L, (20L, 20, "Hello World"))),
-      Right(6L),
-      Left((8L, (8L, 8, "Hello World"))),
-      Left((7L, (7L, 7, "Hello World"))),
-      Right(20L))
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    val t1 = env
-      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, a, " +
-      "count(a) OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
-      ", sum(a) OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
-      " from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
-      "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
-      "Hello,3,3,7", "Hello,4,3,9", "Hello,5,3,12",
-      "Hello,6,3,15",
-      "Hello World,7,1,7", "Hello World,7,2,14", "Hello World,7,3,21",
-      "Hello World,7,3,21", "Hello World,8,3,22", "Hello World,20,3,35")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testBoundNonPartitionedEventTimeWindowWithRow(): Unit = {
-
-    val data = Seq(
-      Left((2L, (2L, 2, "Hello"))),
-      Left((2L, (2L, 2, "Hello"))),
-      Left((1L, (1L, 1, "Hello"))),
-      Left((1L, (1L, 1, "Hello"))),
-      Left((2L, (2L, 2, "Hello"))),
-      Left((1L, (1L, 1, "Hello"))),
-      Left((20L, (20L, 20, "Hello World"))), // early row
-      Right(3L),
-      Left((2L, (2L, 2, "Hello"))), // late row
-      Left((3L, (3L, 3, "Hello"))),
-      Left((4L, (4L, 4, "Hello"))),
-      Left((5L, (5L, 5, "Hello"))),
-      Left((6L, (6L, 6, "Hello"))),
-      Left((7L, (7L, 7, "Hello World"))),
-      Right(7L),
-      Left((9L, (9L, 9, "Hello World"))),
-      Left((8L, (8L, 8, "Hello World"))),
-      Left((8L, (8L, 8, "Hello World"))),
-      Right(20L))
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    env.setParallelism(1)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    val t1 = env
-      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, a, " +
-      "count(a) OVER (ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)," +
-      "sum(a) OVER (ORDER BY rowtime ROWS BETWEEN 2 preceding AND CURRENT ROW)" +
-      "from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
-      "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
-      "Hello,3,3,7",
-      "Hello,4,3,9", "Hello,5,3,12",
-      "Hello,6,3,15", "Hello World,7,3,18",
-      "Hello World,8,3,21", "Hello World,8,3,23",
-      "Hello World,9,3,25",
-      "Hello World,20,3,37")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testBoundPartitionedEventTimeWindowWithRange(): Unit = {
-    val data = Seq(
-      Left((1500L, (1L, 15, "Hello"))),
-      Left((1600L, (1L, 16, "Hello"))),
-      Left((1000L, (1L, 1, "Hello"))),
-      Left((2000L, (2L, 2, "Hello"))),
-      Right(1000L),
-      Left((2000L, (2L, 2, "Hello"))),
-      Left((2000L, (2L, 3, "Hello"))),
-      Left((3000L, (3L, 3, "Hello"))),
-      Right(2000L),
-      Left((4000L, (4L, 4, "Hello"))),
-      Right(3000L),
-      Left((5000L, (5L, 5, "Hello"))),
-      Right(5000L),
-      Left((6000L, (6L, 6, "Hello"))),
-      Left((6500L, (6L, 65, "Hello"))),
-      Right(7000L),
-      Left((9000L, (6L, 9, "Hello"))),
-      Left((9500L, (6L, 18, "Hello"))),
-      Left((9000L, (6L, 9, "Hello"))),
-      Right(10000L),
-      Left((10000L, (7L, 7, "Hello World"))),
-      Left((11000L, (7L, 17, "Hello World"))),
-      Left((11000L, (7L, 77, "Hello World"))),
-      Right(12000L),
-      Left((14000L, (7L, 18, "Hello World"))),
-      Right(14000L),
-      Left((15000L, (8L, 8, "Hello World"))),
-      Right(17000L),
-      Left((20000L, (20L, 20, "Hello World"))),
-      Right(19000L))
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    val t1 = env
-      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, b, " +
-      "count(a) OVER (PARTITION BY c ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
-      "preceding AND CURRENT ROW)" +
-      ", sum(a) OVER (PARTITION BY c ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
-      " preceding AND CURRENT ROW)" +
-      " from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
-      "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
-      "Hello,3,4,9",
-      "Hello,4,2,7",
-      "Hello,5,2,9",
-      "Hello,6,2,11", "Hello,65,2,12",
-      "Hello,9,2,12", "Hello,9,2,12", "Hello,18,3,18",
-      "Hello World,7,1,7", "Hello World,17,3,21", "Hello World,77,3,21", "Hello World,18,1,7",
-      "Hello World,8,2,15",
-      "Hello World,20,1,20")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testBoundNonPartitionedEventTimeWindowWithRange(): Unit = {
-    val data = Seq(
-      Left((1500L, (1L, 15, "Hello"))),
-      Left((1600L, (1L, 16, "Hello"))),
-      Left((1000L, (1L, 1, "Hello"))),
-      Left((2000L, (2L, 2, "Hello"))),
-      Right(1000L),
-      Left((2000L, (2L, 2, "Hello"))),
-      Left((2000L, (2L, 3, "Hello"))),
-      Left((3000L, (3L, 3, "Hello"))),
-      Right(2000L),
-      Left((4000L, (4L, 4, "Hello"))),
-      Right(3000L),
-      Left((5000L, (5L, 5, "Hello"))),
-      Right(5000L),
-      Left((6000L, (6L, 6, "Hello"))),
-      Left((6500L, (6L, 65, "Hello"))),
-      Right(7000L),
-      Left((9000L, (6L, 9, "Hello"))),
-      Left((9500L, (6L, 18, "Hello"))),
-      Left((9000L, (6L, 9, "Hello"))),
-      Right(10000L),
-      Left((10000L, (7L, 7, "Hello World"))),
-      Left((11000L, (7L, 17, "Hello World"))),
-      Left((11000L, (7L, 77, "Hello World"))),
-      Right(12000L),
-      Left((14000L, (7L, 18, "Hello World"))),
-      Right(14000L),
-      Left((15000L, (8L, 8, "Hello World"))),
-      Right(17000L),
-      Left((20000L, (20L, 20, "Hello World"))),
-      Right(19000L))
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    val t1 = env
-      .addSource[(Long, Int, String)](new EventTimeSourceFunction[(Long, Int, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, b, " +
-      "count(a) OVER (ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
-      "preceding AND CURRENT ROW)" +
-      ", sum(a) OVER (ORDER BY rowtime RANGE BETWEEN INTERVAL '1' SECOND " +
-      " preceding AND CURRENT ROW)" +
-      " from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
-      "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
-      "Hello,3,4,9",
-      "Hello,4,2,7",
-      "Hello,5,2,9",
-      "Hello,6,2,11", "Hello,65,2,12",
-      "Hello,9,2,12", "Hello,9,2,12", "Hello,18,3,18",
-      "Hello World,7,4,25", "Hello World,17,3,21", "Hello World,77,3,21", "Hello World,18,1,7",
-      "Hello World,8,2,15",
-      "Hello World,20,1,20")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  /**
-    * All aggregates must be computed on the same window.
-    */
-  @Test(expected = classOf[TableException])
-  def testMultiWindow(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    StreamITCase.clear
-
-    val t1 = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
-
-    tEnv.registerTable("T1", t1)
-
-    val sqlQuery = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY b ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
-      "from T1"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-  }
-
-  /** test sliding event-time unbounded window with partition by **/
-  @Test
-  def testUnboundedEventTimeRowWindowWithPartition(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    StreamITCase.clear
-    env.setParallelism(1)
-
-    val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "count(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "avg(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "max(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "min(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row) " +
-      "from T1"
-
-    val data = Seq(
-      Left(14000005L, (1, 1L, "Hi")),
-      Left(14000000L, (2, 1L, "Hello")),
-      Left(14000002L, (3, 1L, "Hello")),
-      Left(14000003L, (1, 2L, "Hello")),
-      Left(14000004L, (1, 3L, "Hello world")),
-      Left(14000007L, (3, 2L, "Hello world")),
-      Left(14000008L, (2, 2L, "Hello world")),
-      Right(14000010L),
-      // the next 3 elements are late
-      Left(14000008L, (1, 4L, "Hello world")),
-      Left(14000008L, (2, 3L, "Hello world")),
-      Left(14000008L, (3, 3L, "Hello world")),
-      Left(14000012L, (1, 5L, "Hello world")),
-      Right(14000020L),
-      Left(14000021L, (1, 6L, "Hello world")),
-      // the next 3 elements are late
-      Left(14000019L, (1, 6L, "Hello world")),
-      Left(14000018L, (2, 4L, "Hello world")),
-      Left(14000018L, (3, 4L, "Hello world")),
-      Left(14000022L, (2, 5L, "Hello world")),
-      Left(14000022L, (3, 5L, "Hello world")),
-      Left(14000024L, (1, 7L, "Hello world")),
-      Left(14000023L, (1, 8L, "Hello world")),
-      Left(14000021L, (1, 9L, "Hello world")),
-      Right(14000030L)
-    )
-
-    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "1,2,Hello,2,1,2,2,2",
-      "1,3,Hello world,5,2,2,3,2",
-      "1,1,Hi,6,3,2,3,1",
-      "2,1,Hello,1,1,1,1,1",
-      "2,2,Hello world,3,2,1,2,1",
-      "3,1,Hello,1,1,1,1,1",
-      "3,2,Hello world,3,2,1,2,1",
-      "1,5,Hello world,11,4,2,5,1",
-      "1,6,Hello world,17,5,3,6,1",
-      "1,9,Hello world,26,6,4,9,1",
-      "1,8,Hello world,34,7,4,9,1",
-      "1,7,Hello world,41,8,5,9,1",
-      "2,5,Hello world,8,3,2,5,1",
-      "3,5,Hello world,8,3,2,5,1"
-    )
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  /** test sliding event-time unbounded window with partition by **/
-  @Test
-  def testUnboundedEventTimeRowWindowWithPartitionMultiThread(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    StreamITCase.clear
-
-    val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "count(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "avg(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "max(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row), " +
-      "min(b) over (" +
-      "partition by a order by rowtime rows between unbounded preceding and current row) " +
-      "from T1"
-
-    val data = Seq(
-      Left(14000005L, (1, 1L, "Hi")),
-      Left(14000000L, (2, 1L, "Hello")),
-      Left(14000002L, (3, 1L, "Hello")),
-      Left(14000003L, (1, 2L, "Hello")),
-      Left(14000004L, (1, 3L, "Hello world")),
-      Left(14000007L, (3, 2L, "Hello world")),
-      Left(14000008L, (2, 2L, "Hello world")),
-      Right(14000010L),
-      Left(14000012L, (1, 5L, "Hello world")),
-      Right(14000020L),
-      Left(14000021L, (1, 6L, "Hello world")),
-      Left(14000023L, (2, 5L, "Hello world")),
-      Left(14000024L, (3, 5L, "Hello world")),
-      Left(14000026L, (1, 7L, "Hello world")),
-      Left(14000025L, (1, 8L, "Hello world")),
-      Left(14000022L, (1, 9L, "Hello world")),
-      Right(14000030L)
-    )
-
-    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "1,2,Hello,2,1,2,2,2",
-      "1,3,Hello world,5,2,2,3,2",
-      "1,1,Hi,6,3,2,3,1",
-      "2,1,Hello,1,1,1,1,1",
-      "2,2,Hello world,3,2,1,2,1",
-      "3,1,Hello,1,1,1,1,1",
-      "3,2,Hello world,3,2,1,2,1",
-      "1,5,Hello world,11,4,2,5,1",
-      "1,6,Hello world,17,5,3,6,1",
-      "1,9,Hello world,26,6,4,9,1",
-      "1,8,Hello world,34,7,4,9,1",
-      "1,7,Hello world,41,8,5,9,1",
-      "2,5,Hello world,8,3,2,5,1",
-      "3,5,Hello world,8,3,2,5,1"
-    )
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  /** test sliding event-time unbounded window without partitiion by **/
-  @Test
-  def testUnboundedEventTimeRowWindowWithoutPartition(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    StreamITCase.clear
-    env.setParallelism(1)
-
-    val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "count(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "avg(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "max(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "min(b) over (order by rowtime rows between unbounded preceding and current row) " +
-      "from T1"
-
-    val data = Seq(
-      Left(14000005L, (1, 1L, "Hi")),
-      Left(14000000L, (2, 2L, "Hello")),
-      Left(14000002L, (3, 5L, "Hello")),
-      Left(14000003L, (1, 3L, "Hello")),
-      Left(14000004L, (3, 7L, "Hello world")),
-      Left(14000007L, (4, 9L, "Hello world")),
-      Left(14000008L, (5, 8L, "Hello world")),
-      Right(14000010L),
-      // this element will be discard because it is late
-      Left(14000008L, (6, 8L, "Hello world")),
-      Right(14000020L),
-      Left(14000021L, (6, 8L, "Hello world")),
-      Right(14000030L)
-    )
-
-    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "2,2,Hello,2,1,2,2,2",
-      "3,5,Hello,7,2,3,5,2",
-      "1,3,Hello,10,3,3,5,2",
-      "3,7,Hello world,17,4,4,7,2",
-      "1,1,Hi,18,5,3,7,1",
-      "4,9,Hello world,27,6,4,9,1",
-      "5,8,Hello world,35,7,5,9,1",
-      "6,8,Hello world,43,8,5,9,1")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  /** test sliding event-time unbounded window without partitiion by and arrive early **/
-  @Test
-  def testUnboundedEventTimeRowWindowArriveEarly(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    StreamITCase.clear
-    env.setParallelism(1)
-
-    val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "count(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "avg(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "max(b) over (order by rowtime rows between unbounded preceding and current row), " +
-      "min(b) over (order by rowtime rows between unbounded preceding and current row) " +
-      "from T1"
-
-    val data = Seq(
-      Left(14000005L, (1, 1L, "Hi")),
-      Left(14000000L, (2, 2L, "Hello")),
-      Left(14000002L, (3, 5L, "Hello")),
-      Left(14000003L, (1, 3L, "Hello")),
-      // next three elements are early
-      Left(14000012L, (3, 7L, "Hello world")),
-      Left(14000013L, (4, 9L, "Hello world")),
-      Left(14000014L, (5, 8L, "Hello world")),
-      Right(14000010L),
-      Left(14000011L, (6, 8L, "Hello world")),
-      // next element is early
-      Left(14000021L, (6, 8L, "Hello world")),
-      Right(14000020L),
-      Right(14000030L)
-    )
-
-    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "2,2,Hello,2,1,2,2,2",
-      "3,5,Hello,7,2,3,5,2",
-      "1,3,Hello,10,3,3,5,2",
-      "1,1,Hi,11,4,2,5,1",
-      "6,8,Hello world,19,5,3,8,1",
-      "3,7,Hello world,26,6,4,8,1",
-      "4,9,Hello world,35,7,5,9,1",
-      "5,8,Hello world,43,8,5,9,1",
-      "6,8,Hello world,51,9,5,9,1")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  /** test sliding event-time non-partitioned unbounded RANGE window **/
-  @Test
-  def testUnboundedNonPartitionedEventTimeRangeWindow(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    StreamITCase.clear
-    env.setParallelism(1)
-
-    val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (order by rowtime range between unbounded preceding and current row), " +
-      "count(b) over (order by rowtime range between unbounded preceding and current row), " +
-      "avg(b) over (order by rowtime range between unbounded preceding and current row), " +
-      "max(b) over (order by rowtime range between unbounded preceding and current row), " +
-      "min(b) over (order by rowtime range between unbounded preceding and current row) " +
-      "from T1"
-
-    val data = Seq(
-      Left(14000005L, (1, 1L, "Hi")),
-      Left(14000000L, (2, 1L, "Hello")),
-      Left(14000002L, (1, 1L, "Hello")),
-      Left(14000002L, (1, 2L, "Hello")),
-      Left(14000002L, (1, 3L, "Hello world")),
-      Left(14000003L, (2, 2L, "Hello world")),
-      Left(14000003L, (2, 3L, "Hello world")),
-      Right(14000020L),
-      Left(14000021L, (1, 4L, "Hello world")),
-      Left(14000022L, (1, 5L, "Hello world")),
-      Left(14000022L, (1, 6L, "Hello world")),
-      Left(14000022L, (1, 7L, "Hello world")),
-      Left(14000023L, (2, 4L, "Hello world")),
-      Left(14000023L, (2, 5L, "Hello world")),
-      Right(14000030L)
-    )
-
-    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "2,1,Hello,1,1,1,1,1",
-      "1,1,Hello,7,4,1,3,1",
-      "1,2,Hello,7,4,1,3,1",
-      "1,3,Hello world,7,4,1,3,1",
-      "2,2,Hello world,12,6,2,3,1",
-      "2,3,Hello world,12,6,2,3,1",
-      "1,1,Hi,13,7,1,3,1",
-      "1,4,Hello world,17,8,2,4,1",
-      "1,5,Hello world,35,11,3,7,1",
-      "1,6,Hello world,35,11,3,7,1",
-      "1,7,Hello world,35,11,3,7,1",
-      "2,4,Hello world,44,13,3,7,1",
-      "2,5,Hello world,44,13,3,7,1"
-    )
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  /** test sliding event-time unbounded RANGE window **/
-  @Test
-  def testUnboundedPartitionedEventTimeRangeWindow(): Unit = {
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
-    env.setStateBackend(getStateBackend)
-    StreamITCase.clear
-    env.setParallelism(1)
-
-    val sqlQuery = "SELECT a, b, c, " +
-      "SUM(b) over (" +
-      "partition by a order by rowtime range between unbounded preceding and current row), " +
-      "count(b) over (" +
-      "partition by a order by rowtime range between unbounded preceding and current row), " +
-      "avg(b) over (" +
-      "partition by a order by rowtime range between unbounded preceding and current row), " +
-      "max(b) over (" +
-      "partition by a order by rowtime range between unbounded preceding and current row), " +
-      "min(b) over (" +
-      "partition by a order by rowtime range between unbounded preceding and current row) " +
-      "from T1"
-
-    val data = Seq(
-      Left(14000005L, (1, 1L, "Hi")),
-      Left(14000000L, (2, 1L, "Hello")),
-      Left(14000002L, (1, 1L, "Hello")),
-      Left(14000002L, (1, 2L, "Hello")),
-      Left(14000002L, (1, 3L, "Hello world")),
-      Left(14000003L, (2, 2L, "Hello world")),
-      Left(14000003L, (2, 3L, "Hello world")),
-      Right(14000020L),
-      Left(14000021L, (1, 4L, "Hello world")),
-      Left(14000022L, (1, 5L, "Hello world")),
-      Left(14000022L, (1, 6L, "Hello world")),
-      Left(14000022L, (1, 7L, "Hello world")),
-      Left(14000023L, (2, 4L, "Hello world")),
-      Left(14000023L, (2, 5L, "Hello world")),
-      Right(14000030L)
-    )
-
-    val t1 = env.addSource(new EventTimeSourceFunction[(Int, Long, String)](data))
-      .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
-
-    tEnv.registerTable("T1", t1)
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "1,1,Hello,6,3,2,3,1",
-      "1,2,Hello,6,3,2,3,1",
-      "1,3,Hello world,6,3,2,3,1",
-      "1,1,Hi,7,4,1,3,1",
-      "2,1,Hello,1,1,1,1,1",
-      "2,2,Hello world,6,3,2,3,1",
-      "2,3,Hello world,6,3,2,3,1",
-      "1,4,Hello world,11,5,2,4,1",
-      "1,5,Hello world,29,8,3,7,1",
-      "1,6,Hello world,29,8,3,7,1",
-      "1,7,Hello world,29,8,3,7,1",
-      "2,4,Hello world,15,5,3,5,1",
-      "2,5,Hello world,15,5,3,5,1"
-    )
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testPartitionedProcTimeOverWindow(): Unit = {
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setParallelism(1)
-    StreamITCase.clear
-
-    val t = StreamTestData.get5TupleDataStream(env)
-      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
-    tEnv.registerTable("MyTable", t)
-
-    val sqlQuery = "SELECT a,  " +
-      " SUM(c) OVER (" +
-      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS sumC , " +
-      " MIN(c) OVER (" +
-      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS minC " +
-      " FROM MyTable"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "1,0,0",
-      "2,1,1",
-      "2,3,1",
-      "3,3,3",
-      "3,7,3",
-      "3,12,3",
-      "4,6,6",
-      "4,13,6",
-      "4,21,6",
-      "4,24,7",
-      "5,10,10",
-      "5,21,10",
-      "5,33,10",
-      "5,36,11",
-      "5,39,12")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testPartitionedProcTimeOverWindow2(): Unit = {
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setParallelism(1)
-    StreamITCase.clear
-
-    val t = StreamTestData.get5TupleDataStream(env)
-      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
-    tEnv.registerTable("MyTable", t)
-
-    val sqlQuery = "SELECT a,  " +
-      " SUM(c) OVER (" +
-      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS sumC , " +
-      " MIN(c) OVER (" +
-      " PARTITION BY a ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS minC " +
-      " FROM MyTable"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "1,0,0",
-      "2,1,1",
-      "2,3,1",
-      "3,3,3",
-      "3,7,3",
-      "3,12,3",
-      "4,6,6",
-      "4,13,6",
-      "4,21,6",
-      "4,30,6",
-      "5,10,10",
-      "5,21,10",
-      "5,33,10",
-      "5,46,10",
-      "5,60,10")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-
-  @Test
-  def testNonPartitionedProcTimeOverWindow(): Unit = {
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setParallelism(1)
-    StreamITCase.clear
-
-    val t = StreamTestData.get5TupleDataStream(env)
-      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
-    tEnv.registerTable("MyTable", t)
-
-    val sqlQuery = "SELECT a,  " +
-      " SUM(c) OVER (" +
-      " ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS sumC , " +
-      " MIN(c) OVER (" +
-      " ORDER BY proctime ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS minC " +
-      " FROM MyTable"
-
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "1,0,0",
-      "2,1,0",
-      "2,3,0",
-      "3,6,1",
-      "3,9,2",
-      "3,12,3",
-      "4,15,4",
-      "4,18,5",
-      "4,21,6",
-      "4,24,7",
-      "5,27,8",
-      "5,30,9",
-      "5,33,10",
-      "5,36,11",
-      "5,39,12")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-  @Test
-  def testNonPartitionedProcTimeOverWindow2(): Unit = {
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setStateBackend(getStateBackend)
-    val tEnv = TableEnvironment.getTableEnvironment(env)
-    env.setParallelism(1)
-    StreamITCase.clear
-
-    val t = StreamTestData.get5TupleDataStream(env)
-      .toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
-    tEnv.registerTable("MyTable", t)
-
-    val sqlQuery = "SELECT a,  " +
-      " SUM(c) OVER (" +
-      " ORDER BY proctime ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS sumC , " +
-      " MIN(c) OVER (" +
-      " ORDER BY proctime ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) AS minC " +
-      " FROM MyTable"
-    val result = tEnv.sql(sqlQuery).toDataStream[Row]
-    result.addSink(new StreamITCase.StringSink)
-    env.execute()
-
-    val expected = List(
-      "1,0,0",
-      "2,1,0",
-      "2,3,0",
-      "3,6,0",
-      "3,10,0",
-      "3,15,0",
-      "4,21,0",
-      "4,28,0",
-      "4,36,0",
-      "4,45,0",
-      "5,55,0",
-      "5,66,1",
-      "5,77,2",
-      "5,88,3",
-      "5,99,4")
-    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
-  }
-
-}
-
-object SqlITCase {
-
-  class EventTimeSourceFunction[T](
-      dataWithTimestampList: Seq[Either[(Long, T), Long]]) extends SourceFunction[T] {
-    override def run(ctx: SourceContext[T]): Unit = {
-      dataWithTimestampList.foreach {
-        case Left(t) => ctx.collectWithTimestamp(t._2, t._1)
-        case Right(w) => ctx.emitWatermark(new Watermark(w))
-      }
-    }
-
-    override def cancel(): Unit = ???
-  }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
index 4c1d6e62ec371..125d071f4767f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/WindowAggregateTest.scala
@@ -31,65 +31,10 @@ class WindowAggregateTest extends TableTestBase {
   streamUtil.addTable[(Int, String, Long)](
     "MyTable", 'a, 'b, 'c, 'proctime.proctime, 'rowtime.rowtime)
 
-  @Test
-  def testNonPartitionedProcessingTimeBoundedWindow() = {
-
-    val sqlQuery = "SELECT a, Count(c) OVER (ORDER BY proctime  " +
-      "RANGE BETWEEN INTERVAL '10' SECOND PRECEDING AND CURRENT ROW) AS countA " +
-      "FROM MyTable"
-      val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "proctime")
-          ),
-          term("orderBy", "proctime"),
-          term("range", "BETWEEN 10000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "proctime", "COUNT(c) AS w0$o0")
-        ),
-        term("select", "a", "w0$o0 AS $1")
-      )
-
-    streamUtil.verifySql(sqlQuery, expected)
-  }
-
-  @Test
-  def testPartitionedProcessingTimeBoundedWindow() = {
-
-    val sqlQuery =
-      "SELECT a, " +
-      "  AVG(c) OVER (PARTITION BY a ORDER BY proctime " +
-      "    RANGE BETWEEN INTERVAL '2' HOUR PRECEDING AND CURRENT ROW) AS avgA " +
-      "FROM MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "proctime")
-          ),
-          term("partitionBy","a"),
-          term("orderBy", "proctime"),
-          term("range", "BETWEEN 7200000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "proctime", "COUNT(c) AS w0$o0", "$SUM0(c) AS w0$o1")
-        ),
-        term("select", "a", "/(CASE(>(w0$o0, 0)", "CAST(w0$o1), null), w0$o0) AS avgA")
-      )
-
-    streamUtil.verifySql(sqlQuery, expected)
-  }
-
   @Test
   def testGroupbyWithoutWindow() = {
     val sql = "SELECT COUNT(a) FROM MyTable GROUP BY b"
+
     val expected =
       unaryNode(
         "DataStreamCalc",
@@ -241,327 +186,4 @@ class WindowAggregateTest extends TableTestBase {
 
     streamUtil.verifySql(sqlQuery, "n/a")
   }
-
-  @Test
-  def testUnboundPartitionedProcessingWindowWithRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY c ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "proctime")
-          ),
-          term("partitionBy", "c"),
-          term("orderBy", "proctime"),
-          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
-        ),
-        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testUnboundPartitionedProcessingWindowWithRow() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
-      "CURRENT ROW) as cnt1 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          streamTableNode(0),
-          term("partitionBy", "c"),
-          term("orderBy", "proctime"),
-          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "b", "c", "proctime", "rowtime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testUnboundNonPartitionedProcessingWindowWithRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "proctime")
-          ),
-          term("orderBy", "proctime"),
-          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
-        ),
-        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testUnboundNonPartitionedProcessingWindowWithRow() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (ORDER BY proctime ROWS BETWEEN UNBOUNDED preceding AND " +
-      "CURRENT ROW) as cnt1 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          streamTableNode(0),
-          term("orderBy", "proctime"),
-          term("rows", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "b", "c", "proctime", "rowtime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testUnboundNonPartitionedEventTimeWindowWithRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt2 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "rowtime")
-          ),
-          term("orderBy", "rowtime"),
-          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
-        ),
-        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testUnboundPartitionedEventTimeWindowWithRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt1, " +
-      "sum(a) OVER (PARTITION BY c ORDER BY rowtime RANGE UNBOUNDED preceding) as cnt2 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "rowtime")
-          ),
-          term("partitionBy", "c"),
-          term("orderBy", "rowtime"),
-          term("range", "BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0", "$SUM0(a) AS w0$o1")
-        ),
-        term("select", "c", "w0$o0 AS cnt1", "CASE(>(w0$o0, 0)", "CAST(w0$o1), null) AS cnt2")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testBoundPartitionedRowTimeWindowWithRow() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY rowtime ROWS BETWEEN 5 preceding AND " +
-      "CURRENT ROW) as cnt1 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "rowtime")
-          ),
-          term("partitionBy", "c"),
-          term("orderBy", "rowtime"),
-          term("rows", "BETWEEN 5 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testBoundNonPartitionedRowTimeWindowWithRow() = {
-    val sql = "SELECT " +
-        "c, " +
-        "count(a) OVER (ORDER BY rowtime ROWS BETWEEN 5 preceding AND " +
-        "CURRENT ROW) as cnt1 " +
-        "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "rowtime")
-          ),
-          term("orderBy", "rowtime"),
-          term("rows", "BETWEEN 5 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testBoundPartitionedRowTimeWindowWithRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY rowtime " +
-      "RANGE BETWEEN INTERVAL '1' SECOND  preceding AND CURRENT ROW) as cnt1 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "rowtime")
-          ),
-          term("partitionBy", "c"),
-          term("orderBy", "rowtime"),
-          term("range", "BETWEEN 1000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testBoundNonPartitionedRowTimeWindowWithRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (ORDER BY rowtime " +
-      "RANGE BETWEEN INTERVAL '1' SECOND  preceding AND CURRENT ROW) as cnt1 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "rowtime")
-          ),
-          term("orderBy", "rowtime"),
-          term("range", "BETWEEN 1000 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "rowtime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
- @Test
-  def testBoundNonPartitionedProcTimeWindowWithRowRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (ORDER BY proctime ROWS BETWEEN 2 preceding AND " +
-      "CURRENT ROW) as cnt1 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "proctime")
-          ),
-          term("orderBy", "proctime"),
-          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
-  @Test
-  def testBoundPartitionedProcTimeWindowWithRowRange() = {
-    val sql = "SELECT " +
-      "c, " +
-      "count(a) OVER (PARTITION BY c ORDER BY proctime ROWS BETWEEN 2 preceding AND " +
-      "CURRENT ROW) as cnt1 " +
-      "from MyTable"
-
-    val expected =
-      unaryNode(
-        "DataStreamCalc",
-        unaryNode(
-          "DataStreamOverAggregate",
-          unaryNode(
-            "DataStreamCalc",
-            streamTableNode(0),
-            term("select", "a", "c", "proctime")
-          ),
-          term("partitionBy", "c"),
-          term("orderBy", "proctime"),
-          term("rows", "BETWEEN 2 PRECEDING AND CURRENT ROW"),
-          term("select", "a", "c", "proctime", "COUNT(a) AS w0$o0")
-        ),
-        term("select", "c", "w0$o0 AS $1")
-      )
-    streamUtil.verifySql(sql, expected)
-  }
-
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
new file mode 100644
index 0000000000000..eb5acd5b9a3c9
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.harness
+
+import java.util.{Comparator, Queue => JQueue}
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
+import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil}
+import org.apache.flink.table.runtime.types.CRow
+
+class HarnessTestBase {
+  def createHarnessTester[IN, OUT, KEY](
+    operator: OneInputStreamOperator[IN, OUT],
+    keySelector: KeySelector[IN, KEY],
+    keyType: TypeInformation[KEY]): KeyedOneInputStreamOperatorTestHarness[KEY, IN, OUT] = {
+    new KeyedOneInputStreamOperatorTestHarness[KEY, IN, OUT](operator, keySelector, keyType)
+  }
+
+  def verify(
+    expected: JQueue[Object],
+    actual: JQueue[Object],
+    comparator: Comparator[Object],
+    checkWaterMark: Boolean = false): Unit = {
+    if (!checkWaterMark) {
+      val it = actual.iterator()
+      while (it.hasNext) {
+        val data = it.next()
+        if (data.isInstanceOf[Watermark]) {
+          actual.remove(data)
+        }
+      }
+    }
+    TestHarnessUtil.assertOutputEqualsSorted("Verify Error...", expected, actual, comparator)
+  }
+}
+
+object HarnessTestBase {
+
+  /**
+    * Return 0 for equal Rows and non zero for different rows
+    */
+  class RowResultSortComparator(indexCounter: Int) extends Comparator[Object] with Serializable {
+
+    override def compare(o1: Object, o2: Object): Int = {
+
+      if (o1.isInstanceOf[Watermark] || o2.isInstanceOf[Watermark]) {
+        // watermark is not expected
+        -1
+      } else {
+        val row1 = o1.asInstanceOf[StreamRecord[CRow]].getValue
+        val row2 = o2.asInstanceOf[StreamRecord[CRow]].getValue
+        row1.toString.compareTo(row2.toString)
+      }
+    }
+  }
+
+  /**
+    * Tuple row key selector that returns a specified field as the selector function
+    */
+  class TupleRowKeySelector[T](
+    private val selectorField: Int) extends KeySelector[CRow, T] {
+
+    override def getKey(value: CRow): T = {
+      value.row.getField(selectorField).asInstanceOf[T]
+    }
+  }
+
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala
new file mode 100644
index 0000000000000..56ca85c18e29d
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala
@@ -0,0 +1,974 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.harness
+
+import java.lang.{Integer => JInt, Long => JLong}
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.streaming.api.operators.KeyedProcessOperator
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
+import org.apache.flink.table.codegen.GeneratedAggregationsFunction
+import org.apache.flink.table.functions.AggregateFunction
+import org.apache.flink.table.functions.aggfunctions.{LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction}
+import org.apache.flink.table.runtime.aggregate._
+import org.apache.flink.table.runtime.harness.HarnessTestBase._
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.types.Row
+import org.junit.Test
+
+class OverWindowHarnessTest extends HarnessTestBase{
+
+  private val rT = new RowTypeInfo(Array[TypeInformation[_]](
+    INT_TYPE_INFO,
+    LONG_TYPE_INFO,
+    INT_TYPE_INFO,
+    STRING_TYPE_INFO,
+    LONG_TYPE_INFO),
+    Array("a", "b", "c", "d", "e"))
+
+  private val cRT = new CRowTypeInfo(rT)
+
+  private val aggregates =
+    Array(new LongMinWithRetractAggFunction,
+      new LongMaxWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
+  private val aggregationStateType: RowTypeInfo = AggregateUtil.createAccumulatorRowType(aggregates)
+
+  val funcCode: String =
+    """
+      |public class BoundedOverAggregateHelper
+      |  extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations {
+      |
+      |  transient org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction
+      |    fmin = null;
+      |
+      |  transient org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction
+      |    fmax = null;
+      |
+      |  public BoundedOverAggregateHelper() throws Exception {
+      |
+      |    fmin = (org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction)
+      |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
+      |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
+      |    "MuTG9uZ01pbldpdGhSZXRyYWN0QWdnRnVuY3Rpb26oIdX_DaMPxQIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
+      |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWluV2l0aFJldHJhY3RBZ2dGdW5jdGlvbq_ZGuzxtA_S" +
+      |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
+      |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
+      |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
+      |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
+      |
+      |    fmax = (org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction)
+      |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
+      |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
+      |    "MuTG9uZ01heFdpdGhSZXRyYWN0QWdnRnVuY3Rpb25RmsI8azNGXwIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
+      |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWF4V2l0aFJldHJhY3RBZ2dGdW5jdGlvbvnwowlX0_Qf" +
+      |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
+      |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
+      |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
+      |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
+      |  }
+      |
+      |  public void setAggregationResults(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row output) {
+      |
+      |    org.apache.flink.table.functions.AggregateFunction baseClass0 =
+      |      (org.apache.flink.table.functions.AggregateFunction) fmin;
+      |    output.setField(5, baseClass0.getValue(
+      |      (org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+      |      accs.getField(0)));
+      |
+      |    org.apache.flink.table.functions.AggregateFunction baseClass1 =
+      |      (org.apache.flink.table.functions.AggregateFunction) fmax;
+      |    output.setField(6, baseClass1.getValue(
+      |      (org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+      |      accs.getField(1)));
+      |  }
+      |
+      |  public void accumulate(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row input) {
+      |
+      |    fmin.accumulate(
+      |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+      |      accs.getField(0)),
+      |      (java.lang.Long) input.getField(4));
+      |
+      |    fmax.accumulate(
+      |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+      |      accs.getField(1)),
+      |      (java.lang.Long) input.getField(4));
+      |  }
+      |
+      |  public void retract(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row input) {
+      |
+      |    fmin.retract(
+      |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+      |      accs.getField(0)),
+      |      (java.lang.Long) input.getField(4));
+      |
+      |    fmax.retract(
+      |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+      |      accs.getField(1)),
+      |      (java.lang.Long) input.getField(4));
+      |  }
+      |
+      |  public org.apache.flink.types.Row createAccumulators() {
+      |
+      |    org.apache.flink.types.Row accs = new org.apache.flink.types.Row(2);
+      |
+      |    accs.setField(
+      |      0,
+      |      fmin.createAccumulator());
+      |
+      |    accs.setField(
+      |      1,
+      |      fmax.createAccumulator());
+      |
+      |      return accs;
+      |  }
+      |
+      |  public void setForwardedFields(
+      |    org.apache.flink.types.Row input,
+      |    org.apache.flink.types.Row output) {
+      |
+      |    output.setField(0, input.getField(0));
+      |    output.setField(1, input.getField(1));
+      |    output.setField(2, input.getField(2));
+      |    output.setField(3, input.getField(3));
+      |    output.setField(4, input.getField(4));
+      |  }
+      |
+      |  public org.apache.flink.types.Row createOutputRow() {
+      |    return new org.apache.flink.types.Row(7);
+      |  }
+      |
+      |/*******  This test does not use the following methods  *******/
+      |  public org.apache.flink.types.Row mergeAccumulatorsPair(
+      |    org.apache.flink.types.Row a,
+      |    org.apache.flink.types.Row b) {
+      |    return null;
+      |  }
+      |
+      |  public void resetAccumulator(org.apache.flink.types.Row accs) {
+      |  }
+      |
+      |  public void setConstantFlags(org.apache.flink.types.Row output) {
+      |  }
+      |}
+    """.stripMargin
+
+
+  private val funcName = "BoundedOverAggregateHelper"
+
+  private val genAggFunction = GeneratedAggregationsFunction(funcName, funcCode)
+
+
+  @Test
+  def testProcTimeBoundedRowsOver(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new ProcTimeBoundedRowsOver(
+        genAggFunction,
+        2,
+        aggregationStateType,
+        cRT))
+
+    val testHarness =
+      createHarnessTester(processFunction,new TupleRowKeySelector[Integer](0),BasicTypeInfo
+        .INT_TYPE_INFO)
+
+    testHarness.open()
+
+    testHarness.setProcessingTime(1)
+
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 1))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 1))
+
+    testHarness.setProcessingTime(2)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 2))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 2))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 2))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 2))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 2))
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 2L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 2L: JLong, 3L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 3L: JLong, 4L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 4L: JLong, 5L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 5L: JLong, 6L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 20L: JLong, 30L: JLong), true), 1))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 6L: JLong, 7L: JLong), true), 2))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 7L: JLong, 8L: JLong), true), 2))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 8L: JLong, 9L: JLong), true), 2))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 9L: JLong, 10L: JLong), true), 2))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 30L: JLong, 40L: JLong), true), 2))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+
+    testHarness.close()
+  }
+
+  /**
+    * NOTE: all elements at the same proc timestamp have the same value per key
+    */
+  @Test
+  def testProcTimeBoundedRangeOver(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new ProcTimeBoundedRangeOver(
+        genAggFunction,
+        1000,
+        aggregationStateType,
+        cRT))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[Integer](0),
+        BasicTypeInfo.INT_TYPE_INFO)
+
+    testHarness.open()
+
+    testHarness.setProcessingTime(3)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 0))
+
+    testHarness.setProcessingTime(4)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 0))
+
+    testHarness.setProcessingTime(5)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 0))
+
+    testHarness.setProcessingTime(6)
+
+    testHarness.setProcessingTime(1002)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 0))
+
+    testHarness.setProcessingTime(1003)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 0))
+
+    testHarness.setProcessingTime(1004)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 0))
+
+    testHarness.setProcessingTime(1005)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 0))
+
+    testHarness.setProcessingTime(1006)
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    // all elements at the same proc timestamp have the same value per key
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 4))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 4))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 3L: JLong), true), 5))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 5))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 5))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), true), 6))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 6L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 1004))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 2L: JLong, 8L: JLong), true), 1005))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 4L: JLong, 10L: JLong), true), 1006))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 4L: JLong, 10L: JLong), true), 1006))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 30L: JLong, 40L: JLong), true), 1006))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+
+    testHarness.close()
+  }
+
+  @Test
+  def testProcTimeUnboundedOver(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new ProcTimeUnboundedPartitionedOver(
+        genAggFunction,
+        aggregationStateType))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[Integer](0),
+        BasicTypeInfo.INT_TYPE_INFO)
+
+    testHarness.open()
+
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 0))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 0))
+
+    testHarness.setProcessingTime(1003)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 1003))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 1003))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 1003))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 1003))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 1003))
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 2L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 5L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 0))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 1L: JLong, 8L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 1L: JLong, 9L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 1L: JLong, 10L: JLong), true), 1003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 10L: JLong, 40L: JLong), true), 1003))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+    testHarness.close()
+  }
+
+  /**
+    * all elements at the same row-time have the same value per key
+    */
+  @Test
+  def testRowTimeBoundedRangeOver(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new RowTimeBoundedRangeOver(
+        genAggFunction,
+        aggregationStateType,
+        cRT,
+        4000))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[String](3),
+        BasicTypeInfo.STRING_TYPE_INFO)
+
+    testHarness.open()
+
+    testHarness.processWatermark(1)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 2))
+
+    testHarness.processWatermark(2)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 3))
+
+    testHarness.processWatermark(4000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 4001))
+
+    testHarness.processWatermark(4001)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 4002))
+
+    testHarness.processWatermark(4002)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 0L: JLong, 0: JInt, "aaa", 4L: JLong), true), 4003))
+
+    testHarness.processWatermark(4800)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 11L: JLong, 1: JInt, "bbb", 25L: JLong), true), 4801))
+
+    testHarness.processWatermark(6500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 6501))
+
+    testHarness.processWatermark(7000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 7001))
+
+    testHarness.processWatermark(8000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 8001))
+
+    testHarness.processWatermark(12000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 12001))
+
+    testHarness.processWatermark(19000)
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    // all elements at the same row-time have the same value per key
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 2))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 3))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 2L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 4002))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 0L: JLong, 0: JInt, "aaa", 4L: JLong, 2L: JLong, 4L: JLong), true), 4003))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 11L: JLong, 1: JInt, "bbb", 25L: JLong, 25L: JLong, 25L: JLong), true), 4801))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 2L: JLong, 6L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 2L: JLong, 6L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 2L: JLong, 7L: JLong), true), 7001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 2L: JLong, 8L: JLong), true), 8001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 25L: JLong, 30L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 8L: JLong, 10L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 8L: JLong, 10L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 40L: JLong, 40L: JLong), true), 12001))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+    testHarness.close()
+  }
+
+  @Test
+  def testRowTimeBoundedRowsOver(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new RowTimeBoundedRowsOver(
+        genAggFunction,
+        aggregationStateType,
+        cRT,
+        3))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[String](3),
+        BasicTypeInfo.STRING_TYPE_INFO)
+
+    testHarness.open()
+
+    testHarness.processWatermark(800)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 801))
+
+    testHarness.processWatermark(2500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 2501))
+
+    testHarness.processWatermark(4000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 4001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 4001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 4001))
+
+    testHarness.processWatermark(4800)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 4801))
+
+    testHarness.processWatermark(6500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 6501))
+
+    testHarness.processWatermark(7000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 7001))
+
+    testHarness.processWatermark(8000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 8001))
+
+    testHarness.processWatermark(12000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 12001))
+
+    testHarness.processWatermark(19000)
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 801))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 2501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 2L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 2L: JLong, 4L: JLong), true), 4801))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 3L: JLong, 5L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 4L: JLong, 6L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 5L: JLong, 7L: JLong), true), 7001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 6L: JLong, 8L: JLong), true), 8001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 7L: JLong, 9L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 8L: JLong, 10L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 20L: JLong, 40L: JLong), true), 12001))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+    testHarness.close()
+  }
+
+  /**
+    * all elements at the same row-time have the same value per key
+    */
+  @Test
+  def testRowTimeUnboundedRangeOver(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new RowTimeUnboundedRangeOver(
+        genAggFunction,
+        aggregationStateType,
+        cRT))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[String](3),
+        BasicTypeInfo.STRING_TYPE_INFO)
+
+    testHarness.open()
+
+    testHarness.processWatermark(800)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 801))
+
+    testHarness.processWatermark(2500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 2501))
+
+    testHarness.processWatermark(4000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 4001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 4001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 4001))
+
+    testHarness.processWatermark(4800)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 4801))
+
+    testHarness.processWatermark(6500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 6501))
+
+    testHarness.processWatermark(7000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 7001))
+
+    testHarness.processWatermark(8000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 8001))
+
+    testHarness.processWatermark(12000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 12001))
+
+    testHarness.processWatermark(19000)
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    // all elements at the same row-time have the same value per key
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 801))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 2501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 3L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), true), 4801))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 6L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 7001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 1L: JLong, 8L: JLong), true), 8001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 1L: JLong, 10L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 1L: JLong, 10L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 10L: JLong, 40L: JLong), true), 12001))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+    testHarness.close()
+  }
+
+  @Test
+  def testRowTimeUnboundedRowsOver(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new RowTimeUnboundedRowsOver(
+        genAggFunction,
+        aggregationStateType,
+        cRT))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[String](3),
+        BasicTypeInfo.STRING_TYPE_INFO)
+
+    testHarness.open()
+
+    testHarness.processWatermark(800)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 801))
+
+    testHarness.processWatermark(2500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 2501))
+
+    testHarness.processWatermark(4000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 4001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 4001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 4001))
+
+    testHarness.processWatermark(4800)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 4801))
+
+    testHarness.processWatermark(6500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 6501))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 6501))
+
+    testHarness.processWatermark(7000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 7001))
+
+    testHarness.processWatermark(8000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 8001))
+
+    testHarness.processWatermark(12000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 12001))
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 12001))
+
+    testHarness.processWatermark(19000)
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 801))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 2501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 2L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 4001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), true), 4801))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 5L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 6501))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 7001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 1L: JLong, 8L: JLong), true), 8001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 1L: JLong, 9L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 1L: JLong, 10L: JLong), true), 12001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 10L: JLong, 40L: JLong), true), 12001))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+    testHarness.close()
+  }
+}

From b1a984c2b95ffb82a905e08e42d7265339f7a35c Mon Sep 17 00:00:00 2001
From: Shuyi Chen <shuyi@uber.com>
Date: Fri, 21 Apr 2017 23:48:28 -0700
Subject: [PATCH 181/253] [FLINK-6033] [table] Add support for SQL UNNEST.

This closes #3793.
---
 docs/dev/table_api.md                         |   2 +
 .../table/calcite/FlinkTypeFactory.scala      |   5 +-
 .../utils/UserDefinedFunctionUtils.scala      |   6 +-
 .../flink/table/plan/nodes/FlinkRelNode.scala |   3 +-
 .../table/plan/rules/FlinkRuleSets.scala      |   3 +
 .../rules/logical/LogicalUnnestRule.scala     | 134 ++++++++++++++++++
 .../table/plan/util/ExplodeFunctionUtil.scala |  91 ++++++++++++
 .../table/typeutils/TypeCheckUtils.scala      |   4 +-
 .../api/scala/batch/sql/JoinITCase.scala      |  54 +++++++
 .../api/scala/stream/sql/SqlITCase.scala      |  90 ++++++++++++
 10 files changed, 388 insertions(+), 4 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/util/ExplodeFunctionUtil.scala

diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index a77d9948916b4..d10518802ef3a 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1482,6 +1482,7 @@ val result2 = tableEnv.sql(
 #### Limitations
 
 Joins, set operations, and non-windowed aggregations are not supported yet.
+`UNNEST` supports only arrays and does not support `WITH ORDINALITY` yet.
 
 {% top %}
 
@@ -1690,6 +1691,7 @@ tableReference:
 tablePrimary:
   [ TABLE ] [ [ catalogName . ] schemaName . ] tableName
   | LATERAL TABLE '(' functionName '(' expression [, expression ]* ')' ')'
+  | UNNEST '(' expression ')'
 
 values:
   VALUES expression [, expression ]*
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
index 9281ad863b045..eba1623b2cab9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
@@ -26,7 +26,7 @@ import org.apache.calcite.sql.`type`.SqlTypeName._
 import org.apache.calcite.sql.`type`.{BasicSqlType, SqlTypeName}
 import org.apache.calcite.sql.parser.SqlParserPos
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
-import org.apache.flink.api.common.typeinfo.{NothingTypeInfo, PrimitiveArrayTypeInfo, SqlTimeTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeinfo._
 import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.api.java.typeutils.ValueTypeInfo._
 import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo, RowTypeInfo}
@@ -180,6 +180,9 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
     case pa: PrimitiveArrayTypeInfo[_] =>
       new ArrayRelDataType(pa, createTypeFromTypeInfo(pa.getComponentType), false)
 
+    case ba: BasicArrayTypeInfo[_, _] =>
+      new ArrayRelDataType(ba, createTypeFromTypeInfo(ba.getComponentInfo), true)
+
     case oa: ObjectArrayTypeInfo[_, _] =>
       new ArrayRelDataType(oa, createTypeFromTypeInfo(oa.getComponentInfo), true)
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
index 689bf0e6da528..11174decc48c0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
@@ -436,7 +436,11 @@ object UserDefinedFunctionUtils {
     expected.isPrimitive && Primitives.wrap(expected) == candidate ||
     candidate == classOf[Date] && (expected == classOf[Int] || expected == classOf[JInt])  ||
     candidate == classOf[Time] && (expected == classOf[Int] || expected == classOf[JInt]) ||
-    candidate == classOf[Timestamp] && (expected == classOf[Long] || expected == classOf[JLong])
+    candidate == classOf[Timestamp] && (expected == classOf[Long] || expected == classOf[JLong]) ||
+    (candidate.isArray &&
+      expected.isArray &&
+      candidate.getComponentType.isInstanceOf[Object] &&
+      expected.getComponentType == classOf[Object])
 
   @throws[Exception]
   def serialize(function: UserDefinedFunction): String = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
index 0b244e9a77363..8509a8ec9de82 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
@@ -61,7 +61,8 @@ trait FlinkRelNode extends RelNode {
         val referenceExpr = getExpressionString(fa.getReferenceExpr, inFields, localExprsTable)
         val field = fa.getField.getName
         s"$referenceExpr.$field"
-
+      case cv: RexCorrelVariable =>
+        cv.toString
       case _ =>
         throw new IllegalArgumentException(s"Unknown expression type '${expr.getClass}': $expr")
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
index fad60fde6683c..980dfd3926447 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
@@ -100,6 +100,9 @@ object FlinkRuleSets {
     PushProjectIntoTableSourceScanRule.INSTANCE,
     PushFilterIntoTableSourceScanRule.INSTANCE,
 
+    // Unnest rule
+    LogicalUnnestRule.INSTANCE,
+
     // translate to flink logical rel nodes
     FlinkLogicalAggregate.CONVERTER,
     FlinkLogicalWindowAggregate.CONVERTER,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala
new file mode 100644
index 0000000000000..f2d9f2a385222
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.rules.logical
+
+import java.util.Collections
+
+import com.google.common.collect.ImmutableList
+import org.apache.calcite.plan.RelOptRule._
+import org.apache.calcite.plan.volcano.RelSubset
+import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelOptRuleOperand}
+import org.apache.calcite.rel.`type`.{RelDataTypeFieldImpl, RelRecordType, StructKind}
+import org.apache.calcite.rel.RelNode
+import org.apache.calcite.rel.core.Uncollect
+import org.apache.calcite.rel.logical._
+import org.apache.calcite.sql.`type`.AbstractSqlType
+import org.apache.flink.table.api.TableException
+import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
+import org.apache.flink.table.plan.schema.ArrayRelDataType
+import org.apache.flink.table.plan.util.ExplodeFunctionUtil
+
+class LogicalUnnestRule(
+    operand: RelOptRuleOperand,
+    description: String)
+  extends RelOptRule(operand, description) {
+
+  override def matches(call: RelOptRuleCall): Boolean = {
+
+    val join: LogicalCorrelate = call.rel(0).asInstanceOf[LogicalCorrelate]
+    val right = join.getRight.asInstanceOf[RelSubset].getOriginal
+
+    right match {
+      // a filter is pushed above the table function
+      case filter: LogicalFilter =>
+        filter.getInput.asInstanceOf[RelSubset].getOriginal match {
+          case u: Uncollect => !u.withOrdinality
+          case _ => false
+        }
+      case u: Uncollect => !u.withOrdinality
+      case _ => false
+    }
+  }
+
+  override def onMatch(call: RelOptRuleCall): Unit = {
+    val correlate = call.rel(0).asInstanceOf[LogicalCorrelate]
+
+    val outer = correlate.getLeft.asInstanceOf[RelSubset].getOriginal
+    val array = correlate.getRight.asInstanceOf[RelSubset].getOriginal
+
+    def convert(relNode: RelNode): RelNode = {
+      relNode match {
+        case rs: RelSubset =>
+          convert(rs.getRelList.get(0))
+
+        case f: LogicalFilter =>
+          f.copy(
+            f.getTraitSet,
+            ImmutableList.of(convert(f.getInput.asInstanceOf[RelSubset].getOriginal)))
+
+        case uc: Uncollect =>
+          // convert Uncollect into TableFunctionScan
+          val cluster = correlate.getCluster
+
+          val arrayType =
+            uc.getInput.getRowType.getFieldList.get(0).getValue.asInstanceOf[ArrayRelDataType]
+          val componentType = arrayType.getComponentType
+
+          // create table function
+          val explodeTableFunc = UserDefinedFunctionUtils.createTableSqlFunctions(
+            "explode",
+            ExplodeFunctionUtil.explodeTableFuncFromType(arrayType.typeInfo),
+            FlinkTypeFactory.toTypeInfo(arrayType.getComponentType),
+            cluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]).head
+
+          // create table function call
+          val rexCall = cluster.getRexBuilder.makeCall(
+            explodeTableFunc,
+            uc.getInput.asInstanceOf[RelSubset]
+              .getOriginal.asInstanceOf[LogicalProject].getChildExps
+          )
+
+          // determine rel data type of unnest
+          val rowType = componentType match {
+            case _: AbstractSqlType =>
+              new RelRecordType(
+                StructKind.FULLY_QUALIFIED,
+                ImmutableList.of(new RelDataTypeFieldImpl("f0", 0, componentType)))
+            case _: RelRecordType => componentType
+            case _ => throw TableException(
+              s"Unsupported array component type in UNNEST: ${componentType.toString}")
+          }
+
+          // create table function scan
+          new LogicalTableFunctionScan(
+            cluster,
+            correlate.getTraitSet,
+            Collections.emptyList(),
+            rexCall,
+            classOf[Array[Object]],
+            rowType,
+            null)
+      }
+    }
+
+    // convert unnest into table function scan
+    val tableFunctionScan = convert(array)
+    // create correlate with table function scan as input
+    val newCorrleate =
+      correlate.copy(correlate.getTraitSet, ImmutableList.of(outer, tableFunctionScan))
+    call.transformTo(newCorrleate)
+  }
+}
+
+object LogicalUnnestRule {
+  val INSTANCE = new LogicalUnnestRule(
+    operand(classOf[LogicalCorrelate], any),
+    "LogicalUnnestRule")
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/util/ExplodeFunctionUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/util/ExplodeFunctionUtil.scala
new file mode 100644
index 0000000000000..1bcc6d9709daa
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/util/ExplodeFunctionUtil.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.plan.util
+
+import org.apache.flink.api.common.typeinfo.{BasicArrayTypeInfo, BasicTypeInfo, PrimitiveArrayTypeInfo, TypeInformation}
+import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo
+import org.apache.flink.table.functions.TableFunction
+
+class ObjectExplodeTableFunc extends TableFunction[Object] {
+  def eval(arr: Array[Object]): Unit = {
+    arr.foreach(collect)
+  }
+}
+
+class FloatExplodeTableFunc extends TableFunction[Float] {
+  def eval(arr: Array[Float]): Unit = {
+    arr.foreach(collect)
+  }
+}
+
+class ShortExplodeTableFunc extends TableFunction[Short] {
+  def eval(arr: Array[Short]): Unit = {
+    arr.foreach(collect)
+  }
+}
+class IntExplodeTableFunc extends TableFunction[Int] {
+  def eval(arr: Array[Int]): Unit = {
+    arr.foreach(collect)
+  }
+}
+
+class LongExplodeTableFunc extends TableFunction[Long] {
+  def eval(arr: Array[Long]): Unit = {
+    arr.foreach(collect)
+  }
+}
+
+class DoubleExplodeTableFunc extends TableFunction[Double] {
+  def eval(arr: Array[Double]): Unit = {
+    arr.foreach(collect)
+  }
+}
+
+class ByteExplodeTableFunc extends TableFunction[Byte] {
+  def eval(arr: Array[Byte]): Unit = {
+    arr.foreach(collect)
+  }
+}
+
+class BooleanExplodeTableFunc extends TableFunction[Boolean] {
+  def eval(arr: Array[Boolean]): Unit = {
+    arr.foreach(collect)
+  }
+}
+
+object ExplodeFunctionUtil {
+  def explodeTableFuncFromType(ti: TypeInformation[_]):TableFunction[_] = {
+    ti match {
+      case pat: PrimitiveArrayTypeInfo[_] => {
+        pat.getComponentType match {
+          case BasicTypeInfo.INT_TYPE_INFO => new IntExplodeTableFunc
+          case BasicTypeInfo.LONG_TYPE_INFO => new LongExplodeTableFunc
+          case BasicTypeInfo.SHORT_TYPE_INFO => new ShortExplodeTableFunc
+          case BasicTypeInfo.FLOAT_TYPE_INFO => new FloatExplodeTableFunc
+          case BasicTypeInfo.DOUBLE_TYPE_INFO => new DoubleExplodeTableFunc
+          case BasicTypeInfo.BYTE_TYPE_INFO => new ByteExplodeTableFunc
+          case BasicTypeInfo.BOOLEAN_TYPE_INFO => new BooleanExplodeTableFunc
+        }
+      }
+      case _: ObjectArrayTypeInfo[_, _] => new ObjectExplodeTableFunc
+      case _: BasicArrayTypeInfo[_, _] => new ObjectExplodeTableFunc
+      case _ => throw new UnsupportedOperationException(ti.toString + "IS NOT supported")
+    }
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala
index 9896a8c0440f8..fea8c2aa0d685 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeCheckUtils.scala
@@ -68,7 +68,9 @@ object TypeCheckUtils {
   def isLong(dataType: TypeInformation[_]): Boolean = dataType == LONG_TYPE_INFO
 
   def isArray(dataType: TypeInformation[_]): Boolean = dataType match {
-    case _: ObjectArrayTypeInfo[_, _] | _: PrimitiveArrayTypeInfo[_] => true
+    case _: ObjectArrayTypeInfo[_, _] |
+         _: PrimitiveArrayTypeInfo[_] |
+         _: BasicArrayTypeInfo[_, _] => true
     case _ => false
   }
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala
index 9df17ada83e66..8a8c0ce82fc56 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala
@@ -26,6 +26,7 @@ import org.apache.flink.api.scala.util.CollectionDataSets
 import org.apache.flink.table.api.{TableEnvironment, TableException, ValidationException}
 import org.apache.flink.test.util.TestBaseUtils
 import org.apache.flink.types.Row
+import org.junit.Assert.assertEquals
 import org.junit._
 import org.junit.runner.RunWith
 import org.junit.runners.Parameterized
@@ -375,6 +376,59 @@ class JoinITCase(
     Assert.assertEquals(0, result)
   }
 
+  @Test
+  def testCrossWithUnnest(): Unit = {
+
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val data = List(
+      (1, 1L, Array("Hi", "w")),
+      (2, 2L, Array("Hello", "k")),
+      (3, 2L, Array("Hello world", "x"))
+    )
+    val stream = env.fromCollection(data)
+    tEnv.registerDataSet("T", stream, 'a, 'b, 'c)
+
+    val sqlQuery = "SELECT a, s FROM T, UNNEST(T.c) as A (s)"
+
+    val result = tEnv.sql(sqlQuery)
+
+    val expected = List("1,Hi", "1,w", "2,Hello", "2,k", "3,Hello world", "3,x")
+    val results = result.toDataSet[Row].collect().toList
+    assertEquals(expected.toString(), results.sortWith(_.toString < _.toString).toString())
+  }
+
+  @Test
+  def testJoinWithUnnestOfTuple(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+    val data = List(
+      (1, Array((12, "45.6"), (2, "45.612"))),
+      (2, Array((13, "41.6"), (1, "45.2136"))),
+      (3, Array((18, "42.6")))
+    )
+    val stream = env.fromCollection(data)
+    tEnv.registerDataSet("T", stream, 'a, 'b)
+
+    val sqlQuery = "" +
+      "SELECT a, b, x, y " +
+      "FROM " +
+      "  (SELECT a, b FROM T WHERE a < 3) as tf, " +
+      "  UNNEST(tf.b) as A (x, y) " +
+      "WHERE x > a"
+
+    val result = tEnv.sql(sqlQuery)
+
+    val expected = List(
+      "1,[(12,45.6), (2,45.612)],12,45.6",
+      "1,[(12,45.6), (2,45.612)],2,45.612",
+      "2,[(13,41.6), (1,45.2136)],13,41.6").mkString(", ")
+    val results = result.toDataSet[Row].collect().map(_.toString)
+    assertEquals(expected, results.sorted.mkString(", "))
+  }
+
   @Test(expected = classOf[TableException])
   def testRightOuterJoinWithNonEquiJoinPredicate(): Unit = {
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
index 95366e1fef6d8..4147358a7c959 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
@@ -190,4 +190,94 @@ class SqlITCase extends StreamingWithStateTestBase {
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
+  @Test
+  def testUnnestPrimitiveArrayFromTable(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val data = List(
+      (1, Array(12, 45), Array(Array(12, 45))),
+      (2, Array(41, 5), Array(Array(18), Array(87))),
+      (3, Array(18, 42), Array(Array(1), Array(45)))
+    )
+    val stream = env.fromCollection(data)
+    tEnv.registerDataStream("T", stream, 'a, 'b, 'c)
+
+    val sqlQuery = "SELECT a, b, s FROM T, UNNEST(T.b) AS A (s)"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "1,[12, 45],12",
+      "1,[12, 45],45",
+      "2,[41, 5],41",
+      "2,[41, 5],5",
+      "3,[18, 42],18",
+      "3,[18, 42],42"
+    )
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testUnnestArrayOfArrayFromTable(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val data = List(
+      (1, Array(12, 45), Array(Array(12, 45))),
+      (2, Array(41, 5), Array(Array(18), Array(87))),
+      (3, Array(18, 42), Array(Array(1), Array(45)))
+    )
+    val stream = env.fromCollection(data)
+    tEnv.registerDataStream("T", stream, 'a, 'b, 'c)
+
+    val sqlQuery = "SELECT a, s FROM T, UNNEST(T.c) AS A (s)"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "1,[12, 45]",
+      "2,[18]",
+      "2,[87]",
+      "3,[1]",
+      "3,[45]")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testUnnestObjectArrayFromTableWithFilter(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val data = List(
+      (1, Array((12, "45.6"), (12, "45.612"))),
+      (2, Array((13, "41.6"), (14, "45.2136"))),
+      (3, Array((18, "42.6")))
+    )
+    val stream = env.fromCollection(data)
+    tEnv.registerDataStream("T", stream, 'a, 'b)
+
+    val sqlQuery = "SELECT a, b, s, t FROM T, UNNEST(T.b) AS A (s, t) WHERE s > 13"
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List(
+      "2,[(13,41.6), (14,45.2136)],14,45.2136",
+      "3,[(18,42.6)],18,42.6")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
 }
+

From 4a08a3e398e6b0a80d03f2b673850c97ed5f318f Mon Sep 17 00:00:00 2001
From: "Tzu-Li (Gordon) Tai" <tzulitai@apache.org>
Date: Tue, 2 May 2017 19:35:18 +0800
Subject: [PATCH 182/253] [hotfix] [core] Catch InvalidClassException in
 TypeSerializerSerializationProxy

Previously, the TypeSerializerSerializationProxy only uses the dummy
ClassNotFoundDummyTypeSerializer as a placeholder in the case where the
user uses a completely new serializer and deletes the old one.

There is also the case where the user changes the original serializer's
implementation and results in an InvalidClassException when trying to
deserialize the serializer. We should also use the
ClassNotFoundDummyTypeSerializer as a temporary placeholder in this
case.
---
 .../TypeSerializerSerializationProxy.java     |  3 +-
 .../TypeSerializerSerializationProxyTest.java | 50 ++++++++++++++++++-
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java
index cb8967b207a60..c94124f0ac606 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java
@@ -29,6 +29,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.io.InvalidClassException;
 import java.util.Arrays;
 
 @Internal
@@ -97,7 +98,7 @@ public void read(DataInputView in) throws IOException {
 		in.readFully(buffer);
 		try {
 			typeSerializer = InstantiationUtil.deserializeObject(buffer, userClassLoader);
-		} catch (ClassNotFoundException e) {
+		} catch (ClassNotFoundException | InvalidClassException e) {
 			if (ignoreClassNotFound) {
 				// we create a dummy so that all the information is not lost when we get a new checkpoint before receiving
 				// a proper typeserializer from the user
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxyTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxyTest.java
index 982e7ff613544..db1b4ef578b0b 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxyTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxyTest.java
@@ -26,13 +26,21 @@
 import org.apache.flink.util.InstantiationUtil;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
 import java.io.IOException;
+import java.io.InvalidClassException;
 import java.net.URL;
 import java.net.URLClassLoader;
 
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
 
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(InstantiationUtil.class)
 public class TypeSerializerSerializationProxyTest {
 
 	@Test
@@ -91,4 +99,44 @@ public void testStateSerializerSerializationProxyClassNotFound() throws Exceptio
 				InstantiationUtil.serializeObject(serializer),
 				((TypeSerializerSerializationProxy.ClassNotFoundDummyTypeSerializer<?>) proxy.getTypeSerializer()).getActualBytes());
 	}
-}
\ No newline at end of file
+
+	@Test
+	public void testStateSerializerSerializationProxyInvalidClass() throws Exception {
+
+		TypeSerializer<?> serializer = IntSerializer.INSTANCE;
+
+		TypeSerializerSerializationProxy<?> proxy = new TypeSerializerSerializationProxy<>(serializer);
+
+		byte[] serialized;
+		try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
+			proxy.write(new DataOutputViewStreamWrapper(out));
+			serialized = out.toByteArray();
+		}
+
+		PowerMockito.spy(InstantiationUtil.class);
+		PowerMockito
+			.doThrow(new InvalidClassException("test invalid class exception"))
+			.when(InstantiationUtil.class, "deserializeObject", any(byte[].class), any(ClassLoader.class));
+
+		proxy = new TypeSerializerSerializationProxy<>(new URLClassLoader(new URL[0], null));
+
+		try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+			proxy.read(new DataInputViewStreamWrapper(in));
+			fail("InvalidClassException expected, leading to IOException");
+		} catch (IOException expected) {
+
+		}
+
+		proxy = new TypeSerializerSerializationProxy<>(new URLClassLoader(new URL[0], null), true);
+
+		try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+			proxy.read(new DataInputViewStreamWrapper(in));
+		}
+
+		Assert.assertTrue(proxy.getTypeSerializer() instanceof TypeSerializerSerializationProxy.ClassNotFoundDummyTypeSerializer);
+
+		Assert.assertArrayEquals(
+			InstantiationUtil.serializeObject(serializer),
+			((TypeSerializerSerializationProxy.ClassNotFoundDummyTypeSerializer<?>) proxy.getTypeSerializer()).getActualBytes());
+	}
+}

From 2e4704c78897b1f3b8785d6f64326416c75fd79b Mon Sep 17 00:00:00 2001
From: "Tzu-Li (Gordon) Tai" <tzulitai@apache.org>
Date: Mon, 8 May 2017 02:04:23 +0800
Subject: [PATCH 183/253] [FLINK-6178] [core] Allow serializer upgrades for
 managed state

This commit adds the functionality of allowing serializer upgrades for
Flink's managed state. It consists of 2 major changes: 1) new
user-facing API in `TypeSerializer`, and 2) activate serializer upgrades
in state backends.

For 1) new user-facing API for `TypeSerializer`, the following is added:
- new class: TypeSerializerConfigSnapshot
- new class: CompatibilityResult
- new method: TypeSerializer#snapshotConfiguration()
- new method:
  TypeSerializer#ensureCompatibility(TypeSerializerConfigSnapshot)

Generally speaking, configuration snapshots contains a point-in-time
view of a serializer's state / configuration, and is persisted along
with checkpoints. On restore, the configuration is confronted with the
new serializer of the state to check for compatibility, which may
introduce reconfiguration of the new serializer to be compatible.

This compatibility check is integrated in the state backends' restore
flow in 2). Currently, if the check results in the need to perform state
migration, the restore simply fails as the state migration feature isn't
yet available.
---
 .../typeutils/runtime/WritableSerializer.java |  45 +-
 .../state/RocksDBKeyedStateBackend.java       | 203 ++++--
 .../streaming/state/RocksDBStateBackend.java  |   2 +-
 .../common/typeutils/CompatibilityResult.java |  80 ++
 ...CompositeTypeSerializerConfigSnapshot.java |  85 +++
 .../GenericTypeSerializerConfigSnapshot.java  |  88 +++
 .../ParameterlessTypeSerializerConfig.java    |  89 +++
 .../api/common/typeutils/TypeSerializer.java  |  53 +-
 .../TypeSerializerConfigSnapshot.java         | 103 +++
 .../TypeSerializerSerializationProxy.java     |  12 +-
 .../common/typeutils/TypeSerializerUtil.java  | 203 ++++++
 .../typeutils/base/BooleanSerializer.java     |   6 +
 .../base/BooleanValueSerializer.java          |   9 +-
 .../common/typeutils/base/ByteSerializer.java |   7 +-
 .../typeutils/base/ByteValueSerializer.java   |   9 +-
 .../common/typeutils/base/CharSerializer.java |   7 +-
 .../typeutils/base/CharValueSerializer.java   |   9 +-
 .../CollectionSerializerConfigSnapshot.java   |  44 ++
 .../common/typeutils/base/DateSerializer.java |   7 +
 .../typeutils/base/DoubleSerializer.java      |   7 +-
 .../typeutils/base/DoubleValueSerializer.java |   9 +-
 .../common/typeutils/base/EnumSerializer.java | 175 ++++-
 .../typeutils/base/FloatSerializer.java       |   7 +-
 .../typeutils/base/FloatValueSerializer.java  |   9 +-
 .../base/GenericArraySerializer.java          |  36 +-
 .../GenericArraySerializerConfigSnapshot.java |  95 +++
 .../common/typeutils/base/IntSerializer.java  |   7 +-
 .../typeutils/base/IntValueSerializer.java    |   9 +-
 .../common/typeutils/base/ListSerializer.java |  30 +-
 .../common/typeutils/base/LongSerializer.java |   7 +-
 .../typeutils/base/LongValueSerializer.java   |   9 +-
 .../common/typeutils/base/MapSerializer.java  |  39 +-
 .../base/MapSerializerConfigSnapshot.java     |  48 ++
 .../typeutils/base/ShortSerializer.java       |   7 +-
 .../typeutils/base/ShortValueSerializer.java  |   9 +-
 .../typeutils/base/SqlDateSerializer.java     |   7 +
 .../typeutils/base/SqlTimeSerializer.java     |  12 +
 .../typeutils/base/StringSerializer.java      |   6 +
 .../typeutils/base/StringValueSerializer.java |   9 +-
 .../base/TypeSerializerSingleton.java         |  32 +
 .../BooleanPrimitiveArraySerializer.java      |   3 +-
 .../array/BytePrimitiveArraySerializer.java   |   2 +-
 .../array/CharPrimitiveArraySerializer.java   |   3 +-
 .../array/DoublePrimitiveArraySerializer.java |   3 +-
 .../array/FloatPrimitiveArraySerializer.java  |   3 +-
 .../array/IntPrimitiveArraySerializer.java    |   3 +-
 .../array/LongPrimitiveArraySerializer.java   |   3 +-
 .../array/ShortPrimitiveArraySerializer.java  |   3 +-
 .../base/array/StringArraySerializer.java     |   3 +-
 .../typeutils/runtime/AvroSerializer.java     | 144 +++-
 .../runtime/CopyableValueSerializer.java      |  42 +-
 .../typeutils/runtime/EitherSerializer.java   |  37 +
 .../EitherSerializerConfigSnapshot.java       |  49 ++
 .../typeutils/runtime/KryoRegistration.java   | 173 +++++
 ...oRegistrationSerializerConfigSnapshot.java | 251 +++++++
 .../api/java/typeutils/runtime/KryoUtils.java |  27 +
 .../typeutils/runtime/PojoSerializer.java     | 681 +++++++++++++++---
 .../java/typeutils/runtime/RowSerializer.java |  81 ++-
 .../runtime/TupleSerializerBase.java          |  42 ++
 .../TupleSerializerConfigSnapshot.java        |  91 +++
 .../typeutils/runtime/ValueSerializer.java    |  86 ++-
 .../runtime/kryo/KryoSerializer.java          | 218 ++++--
 .../common/typeutils/SerializerTestBase.java  |  53 ++
 .../TypeSerializerConfigSnapshotTest.java     | 147 ++++
 .../typeutils/base/EnumSerializerTest.java    | 140 ++++
 .../CharPrimitiveArraySerializerTest.java     |   2 -
 .../typeutils/runtime/PojoSerializerTest.java | 270 ++++++-
 .../kryo/KryoSerializerCompatibilityTest.java | 136 ++++
 .../java/io/CollectionInputFormatTest.java    |  12 +
 .../cep/NonDuplicatingTypeSerializer.java     |  16 +-
 .../java/org/apache/flink/cep/nfa/NFA.java    |  47 +-
 .../AbstractKeyedCEPPatternOperator.java      |  34 +-
 .../valuearray/IntValueArraySerializer.java   |   7 +
 .../valuearray/LongValueArraySerializer.java  |   7 +
 .../StringValueArraySerializer.java           |   7 +
 .../table/runtime/types/CRowSerializer.scala  |  53 +-
 .../MigrationNamespaceSerializerProxy.java    |  17 +-
 .../AbstractMigrationRestoreStrategy.java     |   8 +-
 .../runtime/state/ArrayListSerializer.java    |  31 +-
 .../state/DefaultOperatorStateBackend.java    | 130 ++--
 .../runtime/state/HashMapSerializer.java      |  40 +-
 .../flink/runtime/state/JavaSerializer.java   |  19 +-
 .../state/KeyedBackendSerializationProxy.java | 178 +----
 ...endStateMetaInfoSnapshotReaderWriters.java | 257 +++++++
 .../OperatorBackendSerializationProxy.java    | 168 ++---
 ...endStateMetaInfoSnapshotReaderWriters.java | 233 ++++++
 .../state/RegisteredBackendStateMetaInfo.java | 145 ----
 .../RegisteredKeyedBackendStateMetaInfo.java  | 246 +++++++
 ...egisteredOperatorBackendStateMetaInfo.java | 198 +++++
 .../runtime/state/StateMigrationUtil.java     |  83 +++
 .../state/VoidNamespaceSerializer.java        |   8 +
 .../state/heap/CopyOnWriteStateTable.java     |  12 +-
 .../state/heap/HeapKeyedStateBackend.java     |  68 +-
 .../state/heap/NestedMapsStateTable.java      |   4 +-
 .../flink/runtime/state/heap/StateTable.java  |  12 +-
 .../heap/StateTableByKeyGroupReaders.java     |   9 +-
 .../testutils/types/IntListSerializer.java    |  12 +
 .../testutils/types/IntPairSerializer.java    |  12 +
 .../testutils/types/StringPairSerializer.java |  12 +
 .../query/QueryableStateClientTest.java       |   6 +-
 .../state/OperatorStateBackendTest.java       |   2 +-
 .../state/SerializationProxiesTest.java       | 150 +++-
 .../runtime/state/StateBackendTestBase.java   | 305 +++++++-
 .../state/heap/CopyOnWriteStateTableTest.java |  34 +-
 .../StateTableSnapshotCompatibilityTest.java  |   6 +-
 .../recordutils/RecordSerializer.java         |  12 +
 .../scala/typeutils/EitherSerializer.scala    |  48 +-
 .../scala/typeutils/EnumValueSerializer.scala | 116 ++-
 .../scala/typeutils/NothingSerializer.scala   |  12 +-
 .../scala/typeutils/OptionSerializer.scala    |  52 +-
 .../typeutils/TraversableSerializer.scala     |  13 +-
 .../api/scala/typeutils/TrySerializer.scala   |  58 +-
 .../MultiplexingStreamRecordSerializer.java   |  51 ++
 .../streamrecord/StreamRecordSerializer.java  |  49 ++
 .../api/datastream/CoGroupedStreams.java      |  12 +
 .../api/operators/InternalTimer.java          |  12 +
 .../api/windowing/windows/GlobalWindow.java   |  18 +-
 .../api/windowing/windows/TimeWindow.java     |  19 +-
 .../streamrecord/StreamElementSerializer.java |  53 ++
 .../CheckpointingCustomKvStateProgram.java    |   1 -
 120 files changed, 6429 insertions(+), 1011 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtil.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistration.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistrationSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBackendStateMetaInfo.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java

diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
index 9036d751c406a..1a02e7be6f6f8 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
@@ -20,7 +20,11 @@
 
 
 import com.esotericsoftware.kryo.Kryo;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.util.InstantiationUtil;
@@ -30,7 +34,8 @@
 
 import java.io.IOException;
 
-public class WritableSerializer<T extends Writable> extends TypeSerializer<T> {
+@Internal
+public final class WritableSerializer<T extends Writable> extends TypeSerializer<T> {
 	
 	private static final long serialVersionUID = 1L;
 	
@@ -149,4 +154,42 @@ public boolean equals(Object obj) {
 	public boolean canEqual(Object obj) {
 		return obj instanceof WritableSerializer;
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public WritableSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return new WritableSerializerConfigSnapshot<>(typeClass);
+	}
+
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof WritableSerializerConfigSnapshot
+				&& typeClass.equals(((WritableSerializerConfigSnapshot) configSnapshot).getTypeClass())) {
+
+			return CompatibilityResult.compatible();
+		} else {
+			return CompatibilityResult.requiresMigration(null);
+		}
+	}
+
+	public static final class WritableSerializerConfigSnapshot<T extends Writable>
+			extends GenericTypeSerializerConfigSnapshot<T> {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public WritableSerializerConfigSnapshot() {}
+
+		public WritableSerializerConfigSnapshot(Class<T> writableTypeClass) {
+			super(writableTypeClass);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+	}
 }
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index b8e60cd9961f5..079ea1301e4b7 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -26,7 +26,9 @@
 import org.apache.flink.api.common.state.ReducingStateDescriptor;
 import org.apache.flink.api.common.state.StateDescriptor;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
 import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
@@ -52,6 +54,7 @@
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
+import org.apache.flink.runtime.state.StateMigrationUtil;
 import org.apache.flink.runtime.state.DoneFuture;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
@@ -59,9 +62,9 @@
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
 import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StateObject;
 import org.apache.flink.runtime.state.StateUtil;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.internal.InternalAggregatingState;
 import org.apache.flink.runtime.state.internal.InternalFoldingState;
@@ -151,7 +154,15 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
 	 * Information about the k/v states as we create them. This is used to retrieve the
 	 * column family that is used for a state and also for sanity checks when restoring.
 	 */
-	private Map<String, Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>> kvStateInformation;
+	private Map<String, Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>> kvStateInformation;
+
+	/**
+	 * Map of state names to their corresponding restored state meta info.
+	 *
+	 * TODO this map can be removed when eager-state registration is in place.
+	 * TODO we currently need this cached to check state migration strategies when new serializers are registered.
+	 */
+	private Map<String, RegisteredKeyedBackendStateMetaInfo.Snapshot> restoredKvStateMetaInfos;
 
 	/** Number of bytes required to prefix the key groups. */
 	private final int keyGroupPrefixBytes;
@@ -229,7 +240,7 @@ public void dispose() {
 			// and access it in a synchronized block that locks on #dbDisposeLock.
 			if (db != null) {
 
-				for (Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>> column :
+				for (Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>> column :
 						kvStateInformation.values()) {
 					try {
 						column.f0.close();
@@ -568,23 +579,14 @@ public KeyGroupsStateHandle getSnapshotResultStateHandle() {
 
 		private void writeKVStateMetaData() throws IOException {
 
-			List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> metaInfoList =
+			List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> metaInfoSnapshots =
 					new ArrayList<>(stateBackend.kvStateInformation.size());
 
 			int kvStateId = 0;
-			for (Map.Entry<String, Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>> column :
+			for (Map.Entry<String, Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>> column :
 					stateBackend.kvStateInformation.entrySet()) {
 
-				RegisteredBackendStateMetaInfo<?, ?> metaInfo = column.getValue().f1;
-
-				KeyedBackendSerializationProxy.StateMetaInfo<?, ?> metaInfoProxy =
-						new KeyedBackendSerializationProxy.StateMetaInfo<>(
-								metaInfo.getStateType(),
-								metaInfo.getName(),
-								metaInfo.getNamespaceSerializer(),
-								metaInfo.getStateSerializer());
-
-				metaInfoList.add(metaInfoProxy);
+				metaInfoSnapshots.add(column.getValue().f1.snapshot());
 
 				//retrieve iterator for this k/v states
 				readOptions = new ReadOptions();
@@ -597,7 +599,7 @@ private void writeKVStateMetaData() throws IOException {
 			}
 
 			KeyedBackendSerializationProxy serializationProxy =
-					new KeyedBackendSerializationProxy(stateBackend.getKeySerializer(), metaInfoList);
+					new KeyedBackendSerializationProxy(stateBackend.getKeySerializer(), metaInfoSnapshots);
 
 			serializationProxy.write(outputView);
 		}
@@ -717,7 +719,7 @@ private static final class RocksDBIncrementalSnapshotOperation {
 
 		private Map<String, StreamStateHandle> baseSstFiles;
 
-		private final List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfos = new ArrayList<>();
+		private final List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots = new ArrayList<>();
 
 		private FileSystem backupFileSystem;
 		private Path backupPath;
@@ -800,7 +802,7 @@ private StreamStateHandle materializeMetaData() throws Exception {
 				stateBackend.cancelStreamRegistry.registerClosable(outputStream);
 
 				KeyedBackendSerializationProxy serializationProxy =
-					new KeyedBackendSerializationProxy(stateBackend.keySerializer, stateMetaInfos);
+					new KeyedBackendSerializationProxy(stateBackend.keySerializer, stateMetaInfoSnapshots);
 				DataOutputView out = new DataOutputViewStreamWrapper(outputStream);
 
 				serializationProxy.write(out);
@@ -823,18 +825,9 @@ void takeSnapshot() throws Exception {
 			baseSstFiles = stateBackend.materializedSstFiles.get(stateBackend.lastCompletedCheckpointId);
 
 			// save meta data
-			for (Map.Entry<String, Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>> stateMetaInfoEntry : stateBackend.kvStateInformation.entrySet()) {
-
-				RegisteredBackendStateMetaInfo<?, ?> metaInfo = stateMetaInfoEntry.getValue().f1;
-
-				KeyedBackendSerializationProxy.StateMetaInfo<?, ?> metaInfoProxy =
-					new KeyedBackendSerializationProxy.StateMetaInfo<>(
-						metaInfo.getStateType(),
-						metaInfo.getName(),
-						metaInfo.getNamespaceSerializer(),
-						metaInfo.getStateSerializer());
-
-				stateMetaInfos.add(metaInfoProxy);
+			for (Map.Entry<String, Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>> stateMetaInfoEntry
+					: stateBackend.kvStateInformation.entrySet()) {
+				stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().f1.snapshot());
 			}
 
 			// save state data
@@ -1112,33 +1105,38 @@ private void restoreKVStateMetaData() throws IOException, ClassNotFoundException
 
 			serializationProxy.read(currentStateHandleInView);
 
-			List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> metaInfoProxyList =
-					serializationProxy.getNamedStateSerializationProxies();
+			List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> restoredMetaInfos =
+					serializationProxy.getStateMetaInfoSnapshots();
 
-			currentStateHandleKVStateColumnFamilies = new ArrayList<>(metaInfoProxyList.size());
+			currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());
+			rocksDBKeyedStateBackend.restoredKvStateMetaInfos = new HashMap<>(restoredMetaInfos.size());
 
-			for (KeyedBackendSerializationProxy.StateMetaInfo<?, ?> metaInfoProxy : metaInfoProxyList) {
-				Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>> columnFamily =
-						rocksDBKeyedStateBackend.kvStateInformation.get(metaInfoProxy.getStateName());
+			for (RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> restoredMetaInfo : restoredMetaInfos) {
 
-				if (null == columnFamily) {
+				if (!rocksDBKeyedStateBackend.kvStateInformation.containsKey(restoredMetaInfo.getName())) {
 					ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(
-						metaInfoProxy.getStateName().getBytes(ConfigConstants.DEFAULT_CHARSET),
+						restoredMetaInfo.getName().getBytes(ConfigConstants.DEFAULT_CHARSET),
 						rocksDBKeyedStateBackend.columnOptions);
 
-					RegisteredBackendStateMetaInfo<?, ?> stateMetaInfo =
-							new RegisteredBackendStateMetaInfo<>(metaInfoProxy);
+					RegisteredKeyedBackendStateMetaInfo<?, ?> stateMetaInfo =
+							new RegisteredKeyedBackendStateMetaInfo<>(
+								restoredMetaInfo.getStateType(),
+								restoredMetaInfo.getName(),
+								restoredMetaInfo.getNamespaceSerializer(),
+								restoredMetaInfo.getStateSerializer());
 
-					columnFamily = new Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>(
-							rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor),
-							stateMetaInfo);
+					rocksDBKeyedStateBackend.restoredKvStateMetaInfos.put(restoredMetaInfo.getName(), restoredMetaInfo);
 
-					rocksDBKeyedStateBackend.kvStateInformation.put(stateMetaInfo.getName(), columnFamily);
+					ColumnFamilyHandle columnFamily = rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor);
+
+					rocksDBKeyedStateBackend.kvStateInformation.put(
+						stateMetaInfo.getName(),
+						new Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>(columnFamily, stateMetaInfo));
+
+					currentStateHandleKVStateColumnFamilies.add(columnFamily);
 				} else {
-					//TODO we could check here for incompatible serializer versions between previous tasks
+					// TODO with eager state registration in place, check here for serializer migration strategies
 				}
-
-				currentStateHandleKVStateColumnFamilies.add(columnFamily.f0);
 			}
 		}
 
@@ -1198,7 +1196,7 @@ private RocksDBIncrementalRestoreOperation(RocksDBKeyedStateBackend<?> stateBack
 			this.stateBackend = stateBackend;
 		}
 
-		private List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> readMetaData(
+		private List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> readMetaData(
 				StreamStateHandle metaStateHandle) throws Exception {
 
 			FSDataInputStream inputStream = null;
@@ -1212,7 +1210,7 @@ private RocksDBIncrementalRestoreOperation(RocksDBKeyedStateBackend<?> stateBack
 				DataInputView in = new DataInputViewStreamWrapper(inputStream);
 				serializationProxy.read(in);
 
-				return serializationProxy.getNamedStateSerializationProxies();
+				return serializationProxy.getStateMetaInfoSnapshots();
 			} finally {
 				if (inputStream != null) {
 					stateBackend.cancelStreamRegistry.unregisterClosable(inputStream);
@@ -1294,18 +1292,21 @@ private void restoreInstance(
 				}
 
 				// read meta data
-				List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfoProxies =
+				List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots =
 					readMetaData(restoreStateHandle.getMetaStateHandle());
 
 				List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
 
-				for (KeyedBackendSerializationProxy.StateMetaInfo<?, ?> stateMetaInfoProxy : stateMetaInfoProxies) {
+				stateBackend.restoredKvStateMetaInfos = new HashMap<>(stateMetaInfoSnapshots.size());
+
+				for (RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
 
 					ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(
-						stateMetaInfoProxy.getStateName().getBytes(ConfigConstants.DEFAULT_CHARSET),
+						stateMetaInfoSnapshot.getName().getBytes(ConfigConstants.DEFAULT_CHARSET),
 						stateBackend.columnOptions);
 
 					columnFamilyDescriptors.add(columnFamilyDescriptor);
+					stateBackend.restoredKvStateMetaInfos.put(stateMetaInfoSnapshot.getName(), stateMetaInfoSnapshot);
 				}
 
 				if (hasExtraKeys) {
@@ -1320,23 +1321,27 @@ private void restoreInstance(
 						for (int i = 0; i < columnFamilyHandles.size(); ++i) {
 							ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i);
 							ColumnFamilyDescriptor columnFamilyDescriptor = columnFamilyDescriptors.get(i);
-							KeyedBackendSerializationProxy.StateMetaInfo<?, ?> stateMetaInfoProxy = stateMetaInfoProxies.get(i);
+							RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i);
 
-							Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>> registeredStateMetaInfoEntry =
-								stateBackend.kvStateInformation.get(stateMetaInfoProxy.getStateName());
+							Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>> registeredStateMetaInfoEntry =
+								stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName());
 
 							if (null == registeredStateMetaInfoEntry) {
 
-								RegisteredBackendStateMetaInfo<?, ?> stateMetaInfo =
-									new RegisteredBackendStateMetaInfo<>(stateMetaInfoProxy);
+								RegisteredKeyedBackendStateMetaInfo<?, ?> stateMetaInfo =
+									new RegisteredKeyedBackendStateMetaInfo<>(
+										stateMetaInfoSnapshot.getStateType(),
+										stateMetaInfoSnapshot.getName(),
+										stateMetaInfoSnapshot.getNamespaceSerializer(),
+										stateMetaInfoSnapshot.getStateSerializer());
 
 								registeredStateMetaInfoEntry =
-									new Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>(
+									new Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>(
 										stateBackend.db.createColumnFamily(columnFamilyDescriptor),
 										stateMetaInfo);
 
 								stateBackend.kvStateInformation.put(
-									stateMetaInfoProxy.getStateName(),
+									stateMetaInfoSnapshot.getName(),
 									registeredStateMetaInfoEntry);
 							}
 
@@ -1403,15 +1408,19 @@ private void restoreInstance(
 						columnFamilyDescriptors, columnFamilyHandles);
 
 					for (int i = 0; i < columnFamilyDescriptors.size(); ++i) {
-						KeyedBackendSerializationProxy.StateMetaInfo<?, ?> stateMetaInfoProxy = stateMetaInfoProxies.get(i);
+						RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i);
 
 						ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i);
-						RegisteredBackendStateMetaInfo<?, ?> stateMetaInfo =
-							new RegisteredBackendStateMetaInfo<>(stateMetaInfoProxy);
+						RegisteredKeyedBackendStateMetaInfo<?, ?> stateMetaInfo =
+							new RegisteredKeyedBackendStateMetaInfo<>(
+								stateMetaInfoSnapshot.getStateType(),
+								stateMetaInfoSnapshot.getName(),
+								stateMetaInfoSnapshot.getNamespaceSerializer(),
+								stateMetaInfoSnapshot.getStateSerializer());
 
 						stateBackend.kvStateInformation.put(
-							stateMetaInfoProxy.getStateName(),
-							new Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>>(
+							stateMetaInfoSnapshot.getName(),
+							new Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>(
 								columnFamilyHandle, stateMetaInfo));
 					}
 
@@ -1473,22 +1482,57 @@ void restore(Collection<KeyedStateHandle> restoreStateHandles) throws Exception
 	protected <N, S> ColumnFamilyHandle getColumnFamily(
 			StateDescriptor<?, S> descriptor, TypeSerializer<N> namespaceSerializer) throws IOException {
 
-		Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>> stateInfo =
+		Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>> stateInfo =
 				kvStateInformation.get(descriptor.getName());
 
-		RegisteredBackendStateMetaInfo<N, S> newMetaInfo = new RegisteredBackendStateMetaInfo<>(
-				descriptor.getType(),
-				descriptor.getName(),
-				namespaceSerializer,
-				descriptor.getSerializer());
+		RegisteredKeyedBackendStateMetaInfo<N, S> newMetaInfo = new RegisteredKeyedBackendStateMetaInfo<>(
+			descriptor.getType(),
+			descriptor.getName(),
+			namespaceSerializer,
+			descriptor.getSerializer());
 
 		if (stateInfo != null) {
-			if (newMetaInfo.canRestoreFrom(stateInfo.f1)) {
+			// TODO with eager registration in place, these checks should be moved to restore()
+
+			RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> restoredMetaInfo =
+				restoredKvStateMetaInfos.get(descriptor.getName());
+
+			Preconditions.checkState(
+				newMetaInfo.getName().equals(restoredMetaInfo.getName()),
+				"Incompatible state names. " +
+					"Was [" + restoredMetaInfo.getName() + "], " +
+					"registered with [" + newMetaInfo.getName() + "].");
+
+			if (!newMetaInfo.getStateType().equals(StateDescriptor.Type.UNKNOWN)
+				&& !restoredMetaInfo.getStateType().equals(StateDescriptor.Type.UNKNOWN)) {
+
+				Preconditions.checkState(
+					newMetaInfo.getStateType().equals(restoredMetaInfo.getStateType()),
+					"Incompatible state types. " +
+						"Was [" + restoredMetaInfo.getStateType() + "], " +
+						"registered with [" + newMetaInfo.getStateType() + "].");
+			}
+
+			// check compatibility results to determine if state migration is required
+
+			CompatibilityResult<N> namespaceCompatibility = StateMigrationUtil.resolveCompatibilityResult(
+					restoredMetaInfo.getNamespaceSerializer(),
+					MigrationNamespaceSerializerProxy.class,
+					restoredMetaInfo.getNamespaceSerializerConfigSnapshot(),
+					newMetaInfo.getNamespaceSerializer());
+
+			CompatibilityResult<S> stateCompatibility = StateMigrationUtil.resolveCompatibilityResult(
+					restoredMetaInfo.getStateSerializer(),
+					TypeSerializerSerializationProxy.ClassNotFoundDummyTypeSerializer.class,
+					restoredMetaInfo.getStateSerializerConfigSnapshot(),
+					newMetaInfo.getStateSerializer());
+
+			if (!namespaceCompatibility.requiresMigration() && !stateCompatibility.requiresMigration()) {
 				stateInfo.f1 = newMetaInfo;
 				return stateInfo.f0;
 			} else {
-				throw new IOException("Trying to access state using wrong meta info, was " + stateInfo.f1 +
-						" trying access with " + newMetaInfo);
+				// TODO state migration currently isn't possible.
+				throw new RuntimeException("State migration currently isn't supported.");
 			}
 		}
 
@@ -1497,7 +1541,7 @@ protected <N, S> ColumnFamilyHandle getColumnFamily(
 
 		try {
 			ColumnFamilyHandle columnFamily = db.createColumnFamily(columnDescriptor);
-			Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<N, S>> tuple =
+			Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<N, S>> tuple =
 					new Tuple2<>(columnFamily, newMetaInfo);
 			Map rawAccess = kvStateInformation;
 			rawAccess.put(descriptor.getName(), tuple);
@@ -1832,6 +1876,8 @@ private void restoreOldSavepointKeyedState(Collection<KeyedStateHandle> restoreS
 		// clear k/v state information before filling it
 		kvStateInformation.clear();
 
+		restoredKvStateMetaInfos = new HashMap<>(namedStates.size());
+
 		// first get the column family mapping
 		int numColumns = inputView.readInt();
 		Map<Byte, StateDescriptor<?, ?>> columnFamilyMapping = new HashMap<>(numColumns);
@@ -1846,6 +1892,15 @@ private void restoreOldSavepointKeyedState(Collection<KeyedStateHandle> restoreS
 
 			columnFamilyMapping.put(mappingByte, stateDescriptor);
 
+			// mimic a restored kv state meta info
+			restoredKvStateMetaInfos.put(
+				stateDescriptor.getName(),
+				new RegisteredKeyedBackendStateMetaInfo<>(
+					stateDescriptor.getType(),
+					stateDescriptor.getName(),
+					MigrationNamespaceSerializerProxy.INSTANCE,
+					stateDescriptor.getSerializer()).snapshot());
+
 			// this will fill in the k/v state information
 			getColumnFamily(stateDescriptor, MigrationNamespaceSerializerProxy.INSTANCE);
 		}
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java
index e5a78b69a500d..695aa12b4221b 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java
@@ -75,7 +75,7 @@ public static class FinalSemiAsyncSnapshot {
 		}
 
 		private static void throwExceptionOnLoadingThisClass() {
-			throw new RuntimeException("Attempt to migrate RocksDB state created with semi async snapshot mode failed. "
+			throw new RuntimeException("Attempt to requiresMigration RocksDB state created with semi async snapshot mode failed. "
 					+ "Unfortunately, this is not supported. Please create a new savepoint for the job using fully "
 					+ "async mode in Flink 1.1 and run migration again with the new savepoint.");
 		}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
new file mode 100644
index 0000000000000..cfbb516549230
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * A {@code CompatibilityResult} contains information about whether or not data migration
+ * is required in order to continue using new serializers for previously serialized data.
+ *
+ * @param <T> the type of the data being migrated.
+ */
+@PublicEvolving
+public final class CompatibilityResult<T> {
+
+	/** Whether or not migration is required. */
+	private final boolean requiresMigration;
+
+	/**
+	 * The convert deserializer to use for reading previous data during migration,
+	 * in the case that the preceding serializer cannot be found.
+	 *
+	 * <p>This is only relevant if migration is required.
+	 */
+	private final TypeSerializer<T> convertDeserializer;
+
+	/**
+	 * Returns a strategy that signals that the new serializer is compatible and no migration is required.
+	 *
+	 * @return a result that signals migration is not required for the new serializer
+	 */
+	public static <T> CompatibilityResult<T> compatible() {
+		return new CompatibilityResult<>(false, null);
+	}
+
+	/**
+	 * Returns a strategy that signals migration to be performed.
+	 *
+	 * <p>Furthermore, in the case that the preceding serializer cannot be found or restored to read the
+	 * previous data during migration, a provided convert deserializer can be used (may be {@code null}
+	 * if one cannot be provided).
+	 *
+	 * <p>In the case that the preceding serializer cannot be found and a convert deserializer is not
+	 * provided, the migration will fail due to the incapability of reading previous data.
+	 *
+	 * @return a result that signals migration is necessary, possibly providing a convert deserializer.
+	 */
+	public static <T> CompatibilityResult<T> requiresMigration(TypeSerializer<T> convertDeserializer) {
+		return new CompatibilityResult<>(true, convertDeserializer);
+	}
+
+	private CompatibilityResult(boolean requiresMigration, TypeSerializer<T> convertDeserializer) {
+		this.requiresMigration = requiresMigration;
+		this.convertDeserializer = convertDeserializer;
+	}
+
+	public TypeSerializer<T> getConvertDeserializer() {
+		return convertDeserializer;
+	}
+
+	public boolean requiresMigration() {
+		return requiresMigration;
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..e7e2650c4139c
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * A {@link TypeSerializerConfigSnapshot} for serializers that has multiple nested serializers.
+ * The configuration snapshot consists of the configuration snapshots of all nested serializers.
+ */
+@Internal
+public abstract class CompositeTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot {
+
+	private TypeSerializerConfigSnapshot[] nestedSerializerConfigSnapshots;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public CompositeTypeSerializerConfigSnapshot() {}
+
+	public CompositeTypeSerializerConfigSnapshot(TypeSerializerConfigSnapshot... nestedSerializerConfigSnapshots) {
+		this.nestedSerializerConfigSnapshots = Preconditions.checkNotNull(nestedSerializerConfigSnapshots);
+	}
+
+	@Override
+	public void write(DataOutputView out) throws IOException {
+		super.write(out);
+		TypeSerializerUtil.writeSerializerConfigSnapshots(out, nestedSerializerConfigSnapshots);
+	}
+
+	@Override
+	public void read(DataInputView in) throws IOException {
+		super.read(in);
+		nestedSerializerConfigSnapshots = TypeSerializerUtil.readSerializerConfigSnapshots(in, getUserCodeClassLoader());
+	}
+
+	public TypeSerializerConfigSnapshot[] getNestedSerializerConfigSnapshots() {
+		return nestedSerializerConfigSnapshots;
+	}
+
+	public TypeSerializerConfigSnapshot getSingleNestedSerializerConfigSnapshot() {
+		return nestedSerializerConfigSnapshots[0];
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj == this) {
+			return true;
+		}
+
+		if (obj == null) {
+			return false;
+		}
+
+		return (obj.getClass().equals(getClass()))
+				&& Arrays.equals(
+					nestedSerializerConfigSnapshots,
+					((CompositeTypeSerializerConfigSnapshot) obj).getNestedSerializerConfigSnapshots());
+	}
+
+	@Override
+	public int hashCode() {
+		return Arrays.hashCode(nestedSerializerConfigSnapshots);
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..4edfe12324134
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * Configuration snapshot for serializers for generic types.
+ *
+ * @param <T> The type to be instantiated.
+ */
+@Internal
+public abstract class GenericTypeSerializerConfigSnapshot<T> extends TypeSerializerConfigSnapshot {
+
+	private Class<T> typeClass;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public GenericTypeSerializerConfigSnapshot() {}
+
+	public GenericTypeSerializerConfigSnapshot(Class<T> typeClass) {
+		this.typeClass = Preconditions.checkNotNull(typeClass);
+	}
+
+	@Override
+	public void write(DataOutputView out) throws IOException {
+		super.write(out);
+
+		// write only the classname to avoid Java serialization
+		out.writeUTF(typeClass.getName());
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public void read(DataInputView in) throws IOException {
+		super.read(in);
+
+		String genericTypeClassname = in.readUTF();
+		try {
+			typeClass = (Class<T>) Class.forName(genericTypeClassname, true, getUserCodeClassLoader());
+		} catch (ClassNotFoundException e) {
+			throw new IOException("Could not find the requested class " + genericTypeClassname + " in classpath.", e);
+		}
+	}
+
+	public Class<T> getTypeClass() {
+		return typeClass;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj == this) {
+			return true;
+		}
+
+		if (obj == null) {
+			return false;
+		}
+
+		return (obj.getClass().equals(getClass()))
+				&& typeClass.equals(((GenericTypeSerializerConfigSnapshot) obj).getTypeClass());
+	}
+
+	@Override
+	public int hashCode() {
+		return typeClass.hashCode();
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java
new file mode 100644
index 0000000000000..7ba7dd452d609
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * A base class for {@link TypeSerializerConfigSnapshot}s that do not have any parameters.
+ */
+@Internal
+public final class ParameterlessTypeSerializerConfig extends TypeSerializerConfigSnapshot {
+
+	private static final int VERSION = 1;
+
+	/**
+	 * A string identifier that encodes the serialization format used by the serializer.
+	 *
+	 * TODO we might change this to a proper serialization format class in the future
+	 */
+	private String serializationFormatIdentifier;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public ParameterlessTypeSerializerConfig() {}
+
+	public ParameterlessTypeSerializerConfig(String serializationFormatIdentifier) {
+		this.serializationFormatIdentifier = Preconditions.checkNotNull(serializationFormatIdentifier);
+	}
+
+	@Override
+	public void write(DataOutputView out) throws IOException {
+		super.write(out);
+		out.writeUTF(serializationFormatIdentifier);
+	}
+
+	@Override
+	public void read(DataInputView in) throws IOException {
+		super.read(in);
+		serializationFormatIdentifier = in.readUTF();
+	}
+
+	@Override
+	public int getVersion() {
+		return VERSION;
+	}
+
+	public String getSerializationFormatIdentifier() {
+		return serializationFormatIdentifier;
+	}
+
+	@Override
+	public boolean equals(Object other) {
+		if (other == this) {
+			return true;
+		}
+
+		if (other == null) {
+			return false;
+		}
+
+		return (other instanceof ParameterlessTypeSerializerConfig)
+				&& serializationFormatIdentifier.equals(((ParameterlessTypeSerializerConfig) other).getSerializationFormatIdentifier());
+	}
+
+	@Override
+	public int hashCode() {
+		return serializationFormatIdentifier.hashCode();
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
index 6edaec6a44e2a..f0562d41b2752 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
@@ -161,7 +161,54 @@ public abstract class TypeSerializer<T> implements Serializable {
 
 	public abstract int hashCode();
 
-	public boolean canRestoreFrom(TypeSerializer<?> other) {
-		return equals(other);
-	}
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	/**
+	 * Create a snapshot of the serializer's current configuration to be stored along with the managed state it is
+	 * registered to (if any - this method is only relevant if this serializer is registered for serialization of
+	 * managed state).
+	 *
+	 * <p>The configuration snapshot should contain information about the serializer's parameter settings and its
+	 * serialization format. When a new serializer is registered to serialize the same managed state that this
+	 * serializer was registered to, the returned configuration snapshot can be used to ensure compatibility
+	 * of the new serializer and determine if state migration is required.
+	 *
+	 * @see TypeSerializerConfigSnapshot
+	 *
+	 * @return snapshot of the serializer's current configuration (cannot be {@code null}).
+	 */
+	public abstract TypeSerializerConfigSnapshot snapshotConfiguration();
+
+	/**
+	 * Ensure compatibility of this serializer with a preceding serializer that was registered for serialization of
+	 * the same managed state (if any - this method is only relevant if this serializer is registered for
+	 * serialization of managed state).
+	 *
+	 * The compatibility check in this method should be performed by inspecting the preceding serializer's configuration
+	 * snapshot. The method may reconfigure the serializer (if required and possible) so that it may be compatible,
+	 * or provide a signaling result that informs Flink that state migration is necessary before continuing to use
+	 * this serializer.
+	 *
+	 * <p>The result can be one of the following:
+	 * <ul>
+	 *     <li>{@link CompatibilityResult#compatible()}: this signals Flink that this serializer is compatible, or
+	 *     has been reconfigured to be compatible, to continue reading previous data, and that the
+	 *     serialization schema remains the same. No migration needs to be performed.</li>
+	 *
+	 *     <li>{@link CompatibilityResult#requiresMigration(TypeSerializer)}: this signals Flink that
+	 *     migration needs to be performed, because this serializer is not compatible, or cannot be reconfigured to be
+	 *     compatible, for previous data. Furthermore, in the case that the preceding serializer cannot be found or
+	 *     restored to read the previous data to perform the migration, the provided convert deserializer can be
+	 *     used (may be {@code null} if one cannot be provided).</li>
+	 * </ul>
+	 *
+	 * @see CompatibilityResult
+	 *
+	 * @param configSnapshot configuration snapshot of a preceding serializer for the same managed state
+	 *
+	 * @return the determined compatibility result.
+	 */
+	public abstract CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot);
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..27369b9245469
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.core.io.VersionMismatchException;
+import org.apache.flink.core.io.VersionedIOReadableWritable;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * A {@code TypeSerializerConfigSnapshot} is a point-in-time view of a {@link TypeSerializer's} configuration.
+ * The configuration snapshot of a serializer is persisted along with checkpoints of the managed state that the
+ * serializer is registered to.
+ *
+ * <p>The persisted configuration may later on be used by new serializers to ensure serialization compatibility
+ * for the same managed state. In order for new serializers to be able to ensure this, the configuration snapshot
+ * should encode sufficient information about:
+ *
+ * <ul>
+ *   <li><strong>Parameter settings of the serializer:</strong> parameters of the serializer include settings
+ *   required to setup the serializer, or the state of the serializer if it is stateful. If the serializer
+ *   has nested serializers, then the configuration snapshot should also contain the parameters of the nested
+ *   serializers.</li>
+ *
+ *   <li><strong>Serialization schema of the serializer:</strong> the data format used by the serializer.</li>
+ * </ul>
+ *
+ * <p>NOTE: Implementations must contain the default empty nullary constructor. This is required to be able to
+ * deserialize the configuration snapshot from its binary form.
+ */
+@PublicEvolving
+public abstract class TypeSerializerConfigSnapshot extends VersionedIOReadableWritable {
+
+	/** The user code class loader; only relevant if this configuration instance was deserialized from binary form. */
+	private ClassLoader userCodeClassLoader;
+
+	/** The snapshot version of this configuration. */
+	private Integer snapshotVersion;
+
+	/**
+	 * Returns the version of the configuration at the time its snapshot was taken.
+	 *
+	 * @return the snapshot configuration's version.
+	 */
+	public int getSnapshotVersion() {
+		if (snapshotVersion == null) {
+			return getVersion();
+		} else {
+			return snapshotVersion;
+		}
+	}
+
+	/**
+	 * Set the user code class loader.
+	 * Only relevant if this configuration instance was deserialized from binary form.
+	 *
+	 * <p>This method is not part of the public user-facing API, and cannot be overriden.
+	 *
+	 * @param userCodeClassLoader user code class loader.
+	 */
+	@Internal
+	public final void setUserCodeClassLoader(ClassLoader userCodeClassLoader) {
+		this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
+	}
+
+	/**
+	 * Returns the user code class loader.
+	 * Only relevant if this configuration instance was deserialized from binary form.
+	 *
+	 * @return the user code class loader
+	 */
+	@Internal
+	public final ClassLoader getUserCodeClassLoader() {
+		return userCodeClassLoader;
+	}
+
+	@Override
+	protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
+		super.resolveVersionRead(foundVersion);
+		this.snapshotVersion = foundVersion;
+	}
+
+	public abstract boolean equals(Object obj);
+
+	public abstract int hashCode();
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java
index c94124f0ac606..067a1ca116f0f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationProxy.java
@@ -143,7 +143,7 @@ public void setIgnoreClassNotFound(boolean ignoreClassNotFound) {
 	 * Dummy TypeSerializer to avoid that data is lost when checkpointing again a serializer for which we encountered
 	 * a {@link ClassNotFoundException}.
 	 */
-	static final class ClassNotFoundDummyTypeSerializer<T> extends TypeSerializer<T> {
+	public static final class ClassNotFoundDummyTypeSerializer<T> extends TypeSerializer<T> {
 
 		private static final long serialVersionUID = 2526330533671642711L;
 		private final byte[] actualBytes;
@@ -206,6 +206,16 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 			throw new UnsupportedOperationException("This object is a dummy TypeSerializer.");
 		}
 
+		@Override
+		public TypeSerializerConfigSnapshot snapshotConfiguration() {
+			throw new UnsupportedOperationException("This object is a dummy TypeSerializer.");
+		}
+
+		@Override
+		public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+			throw new UnsupportedOperationException("This object is a dummy TypeSerializer.");
+		}
+
 		@Override
 		public boolean canEqual(Object obj) {
 			return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtil.java
new file mode 100644
index 0000000000000..0a2148a5169d4
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtil.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.io.VersionedIOReadableWritable;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * Utility methods for {@link TypeSerializer} and {@link TypeSerializerConfigSnapshot}.
+ */
+@Internal
+public class TypeSerializerUtil {
+
+	/**
+	 * Creates an array of {@link TypeSerializerConfigSnapshot}s taken
+	 * from the provided array of {@link TypeSerializer}s.
+	 *
+	 * @param serializers array of type serializers.
+	 *
+	 * @return array of configuration snapshots taken from each serializer.
+	 */
+	public static TypeSerializerConfigSnapshot[] snapshotConfigurations(TypeSerializer<?>[] serializers) {
+		final TypeSerializerConfigSnapshot[] configSnapshots = new TypeSerializerConfigSnapshot[serializers.length];
+
+		for (int i = 0; i < serializers.length; i++) {
+			configSnapshots[i] = serializers[i].snapshotConfiguration();
+		}
+
+		return configSnapshots;
+	}
+
+	/**
+	 * Writes a {@link TypeSerializerConfigSnapshot} to the provided data output view.
+	 *
+	 * <p>It is written with a format that can be later read again using
+	 * {@link #readSerializerConfigSnapshot(DataInputView, ClassLoader)}.
+	 *
+	 * @param out the data output view
+	 * @param serializerConfigSnapshot the serializer configuration snapshot to write
+	 *
+	 * @throws IOException
+	 */
+	public static void writeSerializerConfigSnapshot(
+			DataOutputView out,
+			TypeSerializerConfigSnapshot serializerConfigSnapshot) throws IOException {
+
+		new TypeSerializerConfigSnapshotProxy(serializerConfigSnapshot).write(out);
+	}
+
+	/**
+	 * Reads from a data input view a {@link TypeSerializerConfigSnapshot} that was previously
+	 * written using {@link #writeSerializerConfigSnapshot(DataOutputView, TypeSerializerConfigSnapshot)}.
+	 *
+	 * @param in the data input view
+	 * @param userCodeClassLoader the user code class loader to use
+	 *
+	 * @return the read serializer configuration snapshot
+	 *
+	 * @throws IOException
+	 */
+	public static TypeSerializerConfigSnapshot readSerializerConfigSnapshot(
+			DataInputView in,
+			ClassLoader userCodeClassLoader) throws IOException {
+
+		final TypeSerializerConfigSnapshotProxy proxy = new TypeSerializerConfigSnapshotProxy(userCodeClassLoader);
+		proxy.read(in);
+
+		return proxy.getSerializerConfigSnapshot();
+	}
+
+	/**
+	 * Writes multiple {@link TypeSerializerConfigSnapshot}s to the provided data output view.
+	 *
+	 * <p>It is written with a format that can be later read again using
+	 * {@link #readSerializerConfigSnapshots(DataInputView, ClassLoader)}.
+	 *
+	 * @param out the data output view
+	 * @param serializerConfigSnapshots the serializer configuration snapshots to write
+	 *
+	 * @throws IOException
+	 */
+	public static void writeSerializerConfigSnapshots(
+			DataOutputView out,
+			TypeSerializerConfigSnapshot... serializerConfigSnapshots) throws IOException {
+
+		out.writeInt(serializerConfigSnapshots.length);
+
+		for (TypeSerializerConfigSnapshot snapshot : serializerConfigSnapshots) {
+			new TypeSerializerConfigSnapshotProxy(snapshot).write(out);
+		}
+	}
+
+	/**
+	 * Reads from a data input view multiple {@link TypeSerializerConfigSnapshot}s that was previously
+	 * written using {@link #writeSerializerConfigSnapshot(DataOutputView, TypeSerializerConfigSnapshot)}.
+	 *
+	 * @param in the data input view
+	 * @param userCodeClassLoader the user code class loader to use
+	 *
+	 * @return the read serializer configuration snapshots
+	 *
+	 * @throws IOException
+	 */
+	public static TypeSerializerConfigSnapshot[] readSerializerConfigSnapshots(
+			DataInputView in,
+			ClassLoader userCodeClassLoader) throws IOException {
+
+		int numFields = in.readInt();
+		final TypeSerializerConfigSnapshot[] serializerConfigSnapshots = new TypeSerializerConfigSnapshot[numFields];
+
+		TypeSerializerConfigSnapshotProxy proxy;
+		for (int i = 0; i < numFields; i++) {
+			proxy = new TypeSerializerConfigSnapshotProxy(userCodeClassLoader);
+			proxy.read(in);
+			serializerConfigSnapshots[i] = proxy.getSerializerConfigSnapshot();
+		}
+
+		return serializerConfigSnapshots;
+	}
+
+	/**
+	 * Utility serialization proxy for a {@link TypeSerializerConfigSnapshot}.
+	 */
+	static class TypeSerializerConfigSnapshotProxy extends VersionedIOReadableWritable {
+
+		private static final int VERSION = 1;
+
+		private ClassLoader userCodeClassLoader;
+		private TypeSerializerConfigSnapshot serializerConfigSnapshot;
+
+		TypeSerializerConfigSnapshotProxy(ClassLoader userCodeClassLoader) {
+			this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
+		}
+
+		TypeSerializerConfigSnapshotProxy(TypeSerializerConfigSnapshot serializerConfigSnapshot) {
+			this.serializerConfigSnapshot = serializerConfigSnapshot;
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+			super.write(out);
+
+			// config snapshot class, so that we can re-instantiate the
+			// correct type of config snapshot instance when deserializing
+			out.writeUTF(serializerConfigSnapshot.getClass().getName());
+
+			// the actual configuration parameters
+			serializerConfigSnapshot.write(out);
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		public void read(DataInputView in) throws IOException {
+			super.read(in);
+
+			String serializerConfigClassname = in.readUTF();
+			Class<? extends TypeSerializerConfigSnapshot> serializerConfigSnapshotClass;
+			try {
+				serializerConfigSnapshotClass = (Class<? extends TypeSerializerConfigSnapshot>)
+					Class.forName(serializerConfigClassname, true, userCodeClassLoader);
+			} catch (ClassNotFoundException e) {
+				throw new IOException(
+					"Could not find requested TypeSerializerConfigSnapshot class "
+						+ serializerConfigClassname +  " in classpath.", e);
+			}
+
+			serializerConfigSnapshot = InstantiationUtil.instantiate(serializerConfigSnapshotClass);
+			serializerConfigSnapshot.setUserCodeClassLoader(userCodeClassLoader);
+			serializerConfigSnapshot.read(in);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+
+		TypeSerializerConfigSnapshot getSerializerConfigSnapshot() {
+			return serializerConfigSnapshot;
+		}
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanSerializer.java
index 609f1849d595c..f275807b8b882 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanSerializer.java
@@ -82,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof BooleanSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(BooleanValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanValueSerializer.java
index 62c91dfb86922..4755549bbc3b4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BooleanValueSerializer.java
@@ -31,8 +31,7 @@ public final class BooleanValueSerializer extends TypeSerializerSingleton<Boolea
 	private static final long serialVersionUID = 1L;
 	
 	public static final BooleanValueSerializer INSTANCE = new BooleanValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -86,4 +85,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof BooleanValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(BooleanSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteSerializer.java
index 6ad7e4ee22d7d..bf2baf511b8d1 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteSerializer.java
@@ -33,7 +33,6 @@ public final class ByteSerializer extends TypeSerializerSingleton<Byte> {
 	
 	private static final Byte ZERO = Byte.valueOf((byte) 0);
 
-
 	@Override
 	public boolean isImmutableType() {
 		return true;
@@ -83,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof ByteSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(ByteValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteValueSerializer.java
index 848b01e0860f5..2547dda6f7ebf 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ByteValueSerializer.java
@@ -31,8 +31,7 @@ public final class ByteValueSerializer extends TypeSerializerSingleton<ByteValue
 	private static final long serialVersionUID = 1L;
 	
 	public static final ByteValueSerializer INSTANCE = new ByteValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -84,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof ByteValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+				|| identifier.equals(ByteSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharSerializer.java
index 21a32f147b1ca..dda3543d31f5e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharSerializer.java
@@ -33,7 +33,6 @@ public final class CharSerializer extends TypeSerializerSingleton<Character> {
 	
 	private static final Character ZERO = Character.valueOf((char)0);
 
-
 	@Override
 	public boolean isImmutableType() {
 		return true;
@@ -83,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof CharSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(CharValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharValueSerializer.java
index 84dc39a747999..e012b8c2f712b 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CharValueSerializer.java
@@ -31,8 +31,7 @@ public class CharValueSerializer extends TypeSerializerSingleton<CharValue> {
 	private static final long serialVersionUID = 1L;
 	
 	public static final CharValueSerializer INSTANCE = new CharValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -84,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof CharValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(CharSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..8fa2315dddd04
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils.base;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+
+/**
+ * Configuration snapshot of a serializer for collection types.
+ */
+@Internal
+public final class CollectionSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot {
+
+	private static final int VERSION = 1;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public CollectionSerializerConfigSnapshot() {}
+
+	public CollectionSerializerConfigSnapshot(TypeSerializerConfigSnapshot elementSerializerConfigSnapshot) {
+		super(elementSerializerConfigSnapshot);
+	}
+
+	@Override
+	public int getVersion() {
+		return VERSION;
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DateSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DateSerializer.java
index 3f27de2f8b1ff..28ed904208c29 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DateSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DateSerializer.java
@@ -101,4 +101,11 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof DateSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(SqlDateSerializer.class.getCanonicalName())
+			|| identifier.equals(SqlTimeSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleSerializer.java
index 375cb9c0738a0..92fe71d55a797 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleSerializer.java
@@ -33,7 +33,6 @@ public final class DoubleSerializer extends TypeSerializerSingleton<Double> {
 	
 	private static final Double ZERO = Double.valueOf(0);
 
-	
 	@Override
 	public boolean isImmutableType() {
 		return true;
@@ -83,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof DoubleSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(DoubleValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleValueSerializer.java
index 232ad6b962596..9e7e8d0e164fa 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/DoubleValueSerializer.java
@@ -31,8 +31,7 @@ public final class DoubleValueSerializer extends TypeSerializerSingleton<DoubleV
 	private static final long serialVersionUID = 1L;
 	
 	public static final DoubleValueSerializer INSTANCE = new DoubleValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -84,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof DoubleValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(DoubleSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
index c45487f2fbddc..2f74d84fc60bf 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
@@ -20,11 +20,24 @@
 
 import java.io.IOException;
 import java.io.ObjectInputStream;
+import java.lang.reflect.Array;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
 
 import static org.apache.flink.util.Preconditions.checkArgument;
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -36,7 +49,21 @@ public final class EnumSerializer<T extends Enum<T>> extends TypeSerializer<T> {
 
 	private final Class<T> enumClass;
 
-	private transient T[] values;
+	/**
+	 * Maintain our own map of enum value to their ordinal, instead of directly using {@link Enum#ordinal}.
+	 * This allows us to maintain backwards compatibility for previous serialized data in the case that the
+	 * order of enum constants was changed or new constants were added.
+	 *
+	 * <p>On a fresh start with no reconfiguration, the ordinals would simply be identical to the enum
+	 * constants actual ordinals. Ordinals may change after reconfiguration.
+	 */
+	private Map<T, Integer> valueToOrdinal;
+
+	/**
+	 * Array of enum constants with their indexes identical to their ordinals in the {@link #valueToOrdinal} map.
+	 * Serves as a bidirectional map to have fast access from ordinal to value. May be reordered after reconfiguration.
+	 */
+	private T[] values;
 
 	public EnumSerializer(Class<T> enumClass) {
 		this.enumClass = checkNotNull(enumClass);
@@ -44,6 +71,12 @@ public EnumSerializer(Class<T> enumClass) {
 
 		this.values = enumClass.getEnumConstants();
 		checkArgument(this.values.length > 0, "cannot use an empty enum");
+
+		this.valueToOrdinal = new HashMap<>(values.length);
+		int i = 0;
+		for (T value : values) {
+			this.valueToOrdinal.put(value, i++);
+		}
 	}
 
 	@Override
@@ -78,7 +111,8 @@ public int getLength() {
 
 	@Override
 	public void serialize(T record, DataOutputView target) throws IOException {
-		target.writeInt(record.ordinal());
+		// use our own maintained ordinals instead of the actual enum ordinal
+		target.writeInt(valueToOrdinal.get(record));
 	}
 
 	@Override
@@ -121,6 +155,141 @@ public int hashCode() {
 
 	private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
 		in.defaultReadObject();
-		this.values = enumClass.getEnumConstants();
+
+		// may be null if this serializer was deserialized from an older version
+		if (this.values == null) {
+			this.values = enumClass.getEnumConstants();
+
+			this.valueToOrdinal = new HashMap<>(values.length);
+			int i = 0;
+			for (T value : values) {
+				this.valueToOrdinal.put(value, i++);
+			}
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public EnumSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return new EnumSerializerConfigSnapshot<>(enumClass, values);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof EnumSerializerConfigSnapshot) {
+			final EnumSerializerConfigSnapshot<T> config = (EnumSerializerConfigSnapshot<T>) configSnapshot;
+
+			if (enumClass.equals(config.getTypeClass())) {
+
+				// reorder enum constants so that previously existing constants
+				// remain in the same order, and new
+				LinkedHashSet<T> reorderedEnumConstants = new LinkedHashSet<>();
+				reorderedEnumConstants.addAll(Arrays.asList(config.getEnumConstants()));
+				reorderedEnumConstants.addAll(Arrays.asList(enumClass.getEnumConstants()));
+
+				// regenerate enum constant to ordinal bidirectional map
+				this.values = (T[]) Array.newInstance(enumClass, reorderedEnumConstants.size());
+				this.valueToOrdinal.clear();
+				int i = 0;
+				for (T constant : reorderedEnumConstants) {
+					this.values[i] = constant;
+					this.valueToOrdinal.put(constant, i);
+					i++;
+				}
+
+				return CompatibilityResult.compatible();
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	/**
+	 * Configuration snapshot of a serializer for enumerations.
+	 *
+	 * Configuration contains the enum class, and an array of the enum's constants
+	 * that existed when the configuration snapshot was taken.
+	 *
+	 * @param <T> the enum type.
+	 */
+	public static final class EnumSerializerConfigSnapshot<T extends Enum<T>>
+			extends GenericTypeSerializerConfigSnapshot<T> {
+
+		private static final int VERSION = 1;
+
+		private T[] enumConstants;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public EnumSerializerConfigSnapshot() {}
+
+		public EnumSerializerConfigSnapshot(Class<T> enumClass, T[] enumConstants) {
+			super(enumClass);
+			this.enumConstants = Preconditions.checkNotNull(enumConstants);
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+			super.write(out);
+
+			try (final DataOutputViewStream outViewWrapper = new DataOutputViewStream(out)) {
+				InstantiationUtil.serializeObject(outViewWrapper, enumConstants);
+			}
+		}
+
+		@Override
+		public void read(DataInputView in) throws IOException {
+			super.read(in);
+
+			try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
+				try {
+					enumConstants = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader());
+				} catch (ClassNotFoundException e) {
+					throw new IOException("The requested enum class cannot be found in classpath.", e);
+				} catch (IllegalArgumentException e) {
+					throw new IOException("A previously existing enum constant of "
+						+ getTypeClass().getName() + " no longer exists.", e);
+				}
+			}
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+
+		public T[] getEnumConstants() {
+			return enumConstants;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return super.equals(obj)
+					&& Arrays.equals(
+						enumConstants,
+						((EnumSerializerConfigSnapshot) obj).getEnumConstants());
+		}
+
+		@Override
+		public int hashCode() {
+			return super.hashCode() * 31 + Arrays.hashCode(enumConstants);
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Test utilities
+	// --------------------------------------------------------------------------------------------
+
+	@VisibleForTesting
+	T[] getValues() {
+		return values;
+	}
+
+	@VisibleForTesting
+	Map<T, Integer> getValueToOrdinal() {
+		return valueToOrdinal;
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatSerializer.java
index b1550054832df..953c4fa979a5c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatSerializer.java
@@ -33,7 +33,6 @@ public final class FloatSerializer extends TypeSerializerSingleton<Float> {
 	
 	private static final Float ZERO = Float.valueOf(0);
 
-	
 	@Override
 	public boolean isImmutableType() {
 		return true;
@@ -83,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof FloatSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(FloatValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatValueSerializer.java
index 3ccd88a021734..2542be9e4f0e9 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/FloatValueSerializer.java
@@ -31,8 +31,7 @@ public class FloatValueSerializer extends TypeSerializerSingleton<FloatValue> {
 	private static final long serialVersionUID = 1L;
 	
 	public static final FloatValueSerializer INSTANCE = new FloatValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -84,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof FloatValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(FloatSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
index f35d71b7d7fa2..fe61ab37f7454 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
@@ -22,7 +22,9 @@
 import java.lang.reflect.Array;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -31,7 +33,7 @@
 /**
  * A serializer for arrays of objects.
  * 
- * @param <C> The component type
+ * @param <C> The component type.
  */
 @Internal
 public final class GenericArraySerializer<C> extends TypeSerializer<C[]> {
@@ -186,4 +188,36 @@ public boolean canEqual(Object obj) {
 	public String toString() {
 		return "Serializer " + componentClass.getName() + "[]";
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public GenericArraySerializerConfigSnapshot snapshotConfiguration() {
+		return new GenericArraySerializerConfigSnapshot<>(componentClass, componentSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<C[]> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof GenericArraySerializerConfigSnapshot) {
+			final GenericArraySerializerConfigSnapshot config = (GenericArraySerializerConfigSnapshot) configSnapshot;
+
+			if (componentClass.equals(config.getComponentClass())) {
+				CompatibilityResult<C> compatResult = componentSerializer.ensureCompatibility(
+					config.getSingleNestedSerializerConfigSnapshot());
+
+				if (!compatResult.requiresMigration()) {
+					return CompatibilityResult.requiresMigration(null);
+				} else if (compatResult.getConvertDeserializer() != null) {
+					return CompatibilityResult.requiresMigration(
+						new GenericArraySerializer<>(
+							componentClass,
+							compatResult.getConvertDeserializer()));
+				}
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..e78eb6c01ee93
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils.base;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * Point-in-time configuration of a {@link GenericArraySerializer}.
+ *
+ * @param <C> The component type.
+ */
+@Internal
+public final class GenericArraySerializerConfigSnapshot<C> extends CompositeTypeSerializerConfigSnapshot {
+
+	private static final int VERSION = 1;
+
+	private Class<C> componentClass;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public GenericArraySerializerConfigSnapshot() {}
+
+	public GenericArraySerializerConfigSnapshot(
+			Class<C> componentClass,
+			TypeSerializerConfigSnapshot componentSerializerConfigSnapshot) {
+
+		super(componentSerializerConfigSnapshot);
+
+		this.componentClass = Preconditions.checkNotNull(componentClass);
+	}
+
+	@Override
+	public void write(DataOutputView out) throws IOException {
+		super.write(out);
+
+		InstantiationUtil.serializeObject(new DataOutputViewStream(out), componentClass);
+	}
+
+	@Override
+	public void read(DataInputView in) throws IOException {
+		super.read(in);
+
+		try {
+			componentClass = InstantiationUtil.deserializeObject(new DataInputViewStream(in), getUserCodeClassLoader());
+		} catch (ClassNotFoundException e) {
+			throw new IOException("Could not find requested element class in classpath.", e);
+		}
+	}
+
+	@Override
+	public int getVersion() {
+		return VERSION;
+	}
+
+	public Class<C> getComponentClass() {
+		return componentClass;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		return super.equals(obj)
+			&& (obj instanceof GenericArraySerializerConfigSnapshot)
+			&& (componentClass.equals(((GenericArraySerializerConfigSnapshot) obj).getComponentClass()));
+	}
+
+	@Override
+	public int hashCode() {
+		return super.hashCode() * 31 + componentClass.hashCode();
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntSerializer.java
index de6b3261c9880..e046e8eb173ef 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntSerializer.java
@@ -33,7 +33,6 @@ public final class IntSerializer extends TypeSerializerSingleton<Integer> {
 	
 	private static final Integer ZERO = Integer.valueOf(0);
 
-
 	@Override
 	public boolean isImmutableType() {
 		return true;
@@ -83,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof IntSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(IntValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntValueSerializer.java
index 1a90313c30307..65f5e359bf283 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/IntValueSerializer.java
@@ -31,8 +31,7 @@ public final class IntValueSerializer extends TypeSerializerSingleton<IntValue>
 	private static final long serialVersionUID = 1L;
 	
 	public static final IntValueSerializer INSTANCE = new IntValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -84,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof IntValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(IntSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
index 5d5e8f83cf7d8..02d22de9b0b8a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
@@ -19,7 +19,9 @@
 package org.apache.flink.api.common.typeutils.base;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -39,7 +41,7 @@
  * @param <T> The type of element in the list.
  */
 @Internal
-public class ListSerializer<T> extends TypeSerializer<List<T>> {
+public final class ListSerializer<T> extends TypeSerializer<List<T>> {
 
 	private static final long serialVersionUID = 1119562170939152304L;
 
@@ -166,4 +168,30 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return elementSerializer.hashCode();
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public CollectionSerializerConfigSnapshot snapshotConfiguration() {
+		return new CollectionSerializerConfigSnapshot(elementSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<List<T>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof CollectionSerializerConfigSnapshot) {
+			CompatibilityResult<T> compatResult = elementSerializer.ensureCompatibility(
+				((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
+
+			if (!compatResult.requiresMigration()) {
+				return CompatibilityResult.compatible();
+			} else if (compatResult.getConvertDeserializer() != null) {
+				return CompatibilityResult.requiresMigration(
+					new ListSerializer<>(compatResult.getConvertDeserializer()));
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongSerializer.java
index 086a2fc2c7cec..cbdc3db747638 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongSerializer.java
@@ -33,7 +33,6 @@ public final class LongSerializer extends TypeSerializerSingleton<Long> {
 	
 	private static final Long ZERO = Long.valueOf(0);
 
-
 	@Override
 	public boolean isImmutableType() {
 		return true;
@@ -83,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof LongSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(LongValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongValueSerializer.java
index 5d943251edf69..dcf3805c07a02 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/LongValueSerializer.java
@@ -31,8 +31,7 @@ public final class LongValueSerializer extends TypeSerializerSingleton<LongValue
 	private static final long serialVersionUID = 1L;
 	
 	public static final LongValueSerializer INSTANCE = new LongValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -84,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof LongValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(LongSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
index 5e1a3bf11a00b..50900e43398a9 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
@@ -18,7 +18,10 @@
 
 package org.apache.flink.api.common.typeutils.base;
 
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.util.Preconditions;
@@ -38,7 +41,8 @@
  * @param <K> The type of the keys in the map.
  * @param <V> The type of the values in the map.
  */
-public class MapSerializer<K, V> extends TypeSerializer<Map<K, V>> {
+@Internal
+public final class MapSerializer<K, V> extends TypeSerializer<Map<K, V>> {
 
 	private static final long serialVersionUID = -6885593032367050078L;
 	
@@ -190,4 +194,37 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return keySerializer.hashCode() * 31 + valueSerializer.hashCode();
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public MapSerializerConfigSnapshot snapshotConfiguration() {
+		return new MapSerializerConfigSnapshot(
+				keySerializer.snapshotConfiguration(),
+				valueSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<Map<K, V>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof MapSerializerConfigSnapshot) {
+			TypeSerializerConfigSnapshot[] keyValueSerializerConfigSnapshots =
+				((MapSerializerConfigSnapshot) configSnapshot).getNestedSerializerConfigSnapshots();
+
+			CompatibilityResult<K> keyCompatResult = keySerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[0]);
+			CompatibilityResult<V> valueCompatResult = valueSerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[1]);
+
+			if (!keyCompatResult.requiresMigration() && !valueCompatResult.requiresMigration()) {
+				return CompatibilityResult.compatible();
+			} else if (keyCompatResult.getConvertDeserializer() != null && valueCompatResult.getConvertDeserializer() != null) {
+				return CompatibilityResult.requiresMigration(
+					new MapSerializer<>(
+						keyCompatResult.getConvertDeserializer(),
+						valueCompatResult.getConvertDeserializer()));
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..38e12545d8def
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils.base;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+
+/**
+ * Configuration snapshot for serializers of maps, containing the
+ * configuration snapshot of its key serializer and value serializer.
+ */
+@Internal
+public final class MapSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot {
+
+	private static final int VERSION = 1;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public MapSerializerConfigSnapshot() {}
+
+	public MapSerializerConfigSnapshot(
+			TypeSerializerConfigSnapshot keySerializerConfigSnapshot,
+			TypeSerializerConfigSnapshot valueSerializerConfigSnapshot) {
+
+		super(keySerializerConfigSnapshot, valueSerializerConfigSnapshot);
+	}
+
+	@Override
+	public int getVersion() {
+		return VERSION;
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortSerializer.java
index b94c0ac1e4aa9..322591d8563d9 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortSerializer.java
@@ -33,7 +33,6 @@ public final class ShortSerializer extends TypeSerializerSingleton<Short> {
 	
 	private static final Short ZERO = Short.valueOf((short)0);
 
-	
 	@Override
 	public boolean isImmutableType() {
 		return true;
@@ -83,4 +82,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof ShortSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(ShortValueSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortValueSerializer.java
index 51b234ef36b99..9e909279db366 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ShortValueSerializer.java
@@ -31,8 +31,7 @@ public final class ShortValueSerializer extends TypeSerializerSingleton<ShortVal
 	private static final long serialVersionUID = 1L;
 	
 	public static final ShortValueSerializer INSTANCE = new ShortValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -84,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof ShortValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(ShortSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlDateSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlDateSerializer.java
index 9e1bbfdd5e588..6e30a5f936a70 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlDateSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlDateSerializer.java
@@ -102,4 +102,11 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof SqlDateSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(DateSerializer.class.getCanonicalName())
+			|| identifier.equals(SqlTimeSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlTimeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlTimeSerializer.java
index 544cf0f3ebac7..3fa06c403d91a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlTimeSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/SqlTimeSerializer.java
@@ -101,4 +101,16 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof SqlTimeSerializer;
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & reconfiguring
+	// --------------------------------------------------------------------------------------------
+
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(DateSerializer.class.getCanonicalName())
+			|| identifier.equals(SqlDateSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringSerializer.java
index 034a8cc8e66ec..f2cd54f032a39 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringSerializer.java
@@ -83,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof StringSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+				|| identifier.equals(StringValue.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringValueSerializer.java
index 7f86b7607c68a..4e7ff9748cc76 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/StringValueSerializer.java
@@ -33,8 +33,7 @@ public final class StringValueSerializer extends TypeSerializerSingleton<StringV
 	private static final int HIGH_BIT = 0x1 << 7;
 	
 	public static final StringValueSerializer INSTANCE = new StringValueSerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -109,4 +108,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof StringValueSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(StringSerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java
index c091072c923d4..c5decc5c2658c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java
@@ -19,7 +19,10 @@
 package org.apache.flink.api.common.typeutils.base;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.ParameterlessTypeSerializerConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 
 @Internal
 public abstract class TypeSerializerSingleton<T> extends TypeSerializer<T>{
@@ -48,4 +51,33 @@ public boolean equals(Object obj) {
 			return false;
 		}
 	}
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		// type serializer singletons should always be parameter-less
+		return new ParameterlessTypeSerializerConfig(getSerializationFormatIdentifier());
+	}
+
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof ParameterlessTypeSerializerConfig
+				&& isCompatibleSerializationFormatIdentifier(
+						((ParameterlessTypeSerializerConfig) configSnapshot).getSerializationFormatIdentifier())) {
+
+			return CompatibilityResult.compatible();
+		} else {
+			return CompatibilityResult.requiresMigration(null);
+		}
+	}
+
+	/**
+	 * Subclasses can override this if they know that they are also compatible with identifiers of other formats.
+	 */
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return identifier.equals(getSerializationFormatIdentifier());
+	}
+
+	private String getSerializationFormatIdentifier() {
+		return getClass().getCanonicalName();
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BooleanPrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BooleanPrimitiveArraySerializer.java
index 371200c690bd2..bf14f726da2a0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BooleanPrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BooleanPrimitiveArraySerializer.java
@@ -36,8 +36,7 @@ public final class BooleanPrimitiveArraySerializer extends TypeSerializerSinglet
 	private static final boolean[] EMPTY = new boolean[0];
 
 	public static final BooleanPrimitiveArraySerializer INSTANCE = new BooleanPrimitiveArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BytePrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BytePrimitiveArraySerializer.java
index 0f56149f9d4b4..dbad1dfc821d2 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BytePrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/BytePrimitiveArraySerializer.java
@@ -36,7 +36,7 @@ public final class BytePrimitiveArraySerializer extends TypeSerializerSingleton<
 	private static final byte[] EMPTY = new byte[0];
 
 	public static final BytePrimitiveArraySerializer INSTANCE = new BytePrimitiveArraySerializer();
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializer.java
index c3655e999d121..ae3010911694d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializer.java
@@ -36,8 +36,7 @@ public final class CharPrimitiveArraySerializer extends TypeSerializerSingleton<
 	private static final char[] EMPTY = new char[0];
 
 	public static final CharPrimitiveArraySerializer INSTANCE = new CharPrimitiveArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/DoublePrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/DoublePrimitiveArraySerializer.java
index 2b07cfd3fa0fc..610a22b2023b7 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/DoublePrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/DoublePrimitiveArraySerializer.java
@@ -36,8 +36,7 @@ public final class DoublePrimitiveArraySerializer extends TypeSerializerSingleto
 	private static final double[] EMPTY = new double[0];
 
 	public static final DoublePrimitiveArraySerializer INSTANCE = new DoublePrimitiveArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/FloatPrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/FloatPrimitiveArraySerializer.java
index 1fa4715df69c2..50b2b2e9a8d71 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/FloatPrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/FloatPrimitiveArraySerializer.java
@@ -36,8 +36,7 @@ public final class FloatPrimitiveArraySerializer extends TypeSerializerSingleton
 	private static final float[] EMPTY = new float[0];
 
 	public static final FloatPrimitiveArraySerializer INSTANCE = new FloatPrimitiveArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/IntPrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/IntPrimitiveArraySerializer.java
index a5a3e568eba1b..f11b46eb893df 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/IntPrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/IntPrimitiveArraySerializer.java
@@ -36,8 +36,7 @@ public class IntPrimitiveArraySerializer extends TypeSerializerSingleton<int[]>{
 	private static final int[] EMPTY = new int[0];
 
 	public static final IntPrimitiveArraySerializer INSTANCE = new IntPrimitiveArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/LongPrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/LongPrimitiveArraySerializer.java
index 3ecae3648b427..b0788c9d62b3c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/LongPrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/LongPrimitiveArraySerializer.java
@@ -36,8 +36,7 @@ public final class LongPrimitiveArraySerializer extends TypeSerializerSingleton<
 	private static final long[] EMPTY = new long[0];
 
 	public static final LongPrimitiveArraySerializer INSTANCE = new LongPrimitiveArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/ShortPrimitiveArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/ShortPrimitiveArraySerializer.java
index 7f542c3bbf43b..107fe033567a4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/ShortPrimitiveArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/ShortPrimitiveArraySerializer.java
@@ -36,8 +36,7 @@ public final class ShortPrimitiveArraySerializer extends TypeSerializerSingleton
 	private static final short[] EMPTY = new short[0];
 
 	public static final ShortPrimitiveArraySerializer INSTANCE = new ShortPrimitiveArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/StringArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/StringArraySerializer.java
index ccf369a7163e9..61a654c506756 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/StringArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/array/StringArraySerializer.java
@@ -38,8 +38,7 @@ public final class StringArraySerializer extends TypeSerializerSingleton<String[
 	private static final String[] EMPTY = new String[0];
 	
 	public static final StringArraySerializer INSTANCE = new StringArraySerializer();
-	
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
index 8bcd1574cec9f..c9eeb342c65a0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
@@ -19,6 +19,9 @@
 package org.apache.flink.api.java.typeutils.runtime;
 
 import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.reflect.ReflectDatumReader;
@@ -26,13 +29,17 @@
 import org.apache.avro.util.Utf8;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.java.typeutils.runtime.kryo.Serializers;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.util.InstantiationUtil;
 
 import com.esotericsoftware.kryo.Kryo;
+import org.apache.flink.util.Preconditions;
 import org.objenesis.strategy.StdInstantiatorStrategy;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -51,6 +58,14 @@ public final class AvroSerializer<T> extends TypeSerializer<T> {
 	private final Class<T> type;
 	
 	private final Class<? extends T> typeToInstantiate;
+
+	/**
+	 * Map of class tag (using classname as tag) to their Kryo registration.
+	 *
+	 * <p>This map serves as a preview of the final registration result of
+	 * the Kryo instance, taking into account registration overwrites.
+	 */
+	private LinkedHashMap<String, KryoRegistration> kryoRegistrations;
 	
 	private transient ReflectDatumWriter<T> writer;
 	private transient ReflectDatumReader<T> reader;
@@ -73,6 +88,8 @@ public AvroSerializer(Class<T> type, Class<? extends T> typeToInstantiate) {
 		this.typeToInstantiate = checkNotNull(typeToInstantiate);
 		
 		InstantiationUtil.checkForInstantiation(typeToInstantiate);
+
+		this.kryoRegistrations = buildKryoRegistrations(type);
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -165,14 +182,9 @@ private void checkKryoInitialized() {
 			instantiatorStrategy.setFallbackInstantiatorStrategy(new StdInstantiatorStrategy());
 			kryo.setInstantiatorStrategy(instantiatorStrategy);
 
-			// register Avro types.
-			this.kryo.register(GenericData.Array.class, new Serializers.SpecificInstanceCollectionSerializerForArrayList());
-			this.kryo.register(Utf8.class);
-			this.kryo.register(GenericData.EnumSymbol.class);
-			this.kryo.register(GenericData.Fixed.class);
-			this.kryo.register(GenericData.StringType.class);
-			this.kryo.setAsmEnabled(true);
-			this.kryo.register(type);
+			kryo.setAsmEnabled(true);
+
+			KryoUtils.applyRegistrations(kryo, kryoRegistrations.values());
 		}
 	}
 	
@@ -201,4 +213,120 @@ public boolean equals(Object obj) {
 	public boolean canEqual(Object obj) {
 		return obj instanceof AvroSerializer;
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public AvroSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return new AvroSerializerConfigSnapshot<>(type, typeToInstantiate, kryoRegistrations);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof AvroSerializerConfigSnapshot) {
+			final AvroSerializerConfigSnapshot<T> config = (AvroSerializerConfigSnapshot<T>) configSnapshot;
+
+			if (type.equals(config.getTypeClass()) && typeToInstantiate.equals(config.getTypeToInstantiate())) {
+				// resolve Kryo registrations; currently, since the Kryo registrations in Avro
+				// are fixed, there shouldn't be a problem with the resolution here.
+
+				LinkedHashMap<String, KryoRegistration> oldRegistrations = config.getKryoRegistrations();
+				oldRegistrations.putAll(kryoRegistrations);
+
+				for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : kryoRegistrations.entrySet()) {
+					if (reconfiguredRegistrationEntry.getValue().isDummy()) {
+						return CompatibilityResult.requiresMigration(null);
+					}
+				}
+
+				this.kryoRegistrations = oldRegistrations;
+				return CompatibilityResult.compatible();
+			}
+		}
+
+		// ends up here if the preceding serializer is not
+		// the ValueSerializer, or serialized data type has changed
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	public static class AvroSerializerConfigSnapshot<T> extends KryoRegistrationSerializerConfigSnapshot<T> {
+
+		private static final int VERSION = 1;
+
+		private Class<? extends T> typeToInstantiate;
+
+		public AvroSerializerConfigSnapshot() {}
+
+		public AvroSerializerConfigSnapshot(
+				Class<T> baseType,
+				Class<? extends T> typeToInstantiate,
+				LinkedHashMap<String, KryoRegistration> kryoRegistrations) {
+
+			super(baseType, kryoRegistrations);
+			this.typeToInstantiate = Preconditions.checkNotNull(typeToInstantiate);
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+			super.write(out);
+
+			out.writeUTF(typeToInstantiate.getName());
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		public void read(DataInputView in) throws IOException {
+			super.read(in);
+
+			String classname = in.readUTF();
+			try {
+				typeToInstantiate = (Class<? extends T>) Class.forName(classname, true, getUserCodeClassLoader());
+			} catch (ClassNotFoundException e) {
+				throw new IOException("Cannot find requested class " + classname + " in classpath.", e);
+			}
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+
+		public Class<? extends T> getTypeToInstantiate() {
+			return typeToInstantiate;
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+
+	private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+		in.defaultReadObject();
+
+		// kryoRegistrations may be null if this Avro serializer is deserialized from an old version
+		if (kryoRegistrations == null) {
+			this.kryoRegistrations = buildKryoRegistrations(type);
+		}
+	}
+
+	private static <T> LinkedHashMap<String, KryoRegistration> buildKryoRegistrations(Class<T> serializedDataType) {
+		final LinkedHashMap<String, KryoRegistration> registrations = new LinkedHashMap<>();
+
+		// register Avro types.
+		registrations.put(
+				GenericData.Array.class.getName(),
+				new KryoRegistration(
+						GenericData.Array.class,
+						new ExecutionConfig.SerializableSerializer<>(new Serializers.SpecificInstanceCollectionSerializerForArrayList())));
+		registrations.put(Utf8.class.getName(), new KryoRegistration(Utf8.class));
+		registrations.put(GenericData.EnumSymbol.class.getName(), new KryoRegistration(GenericData.EnumSymbol.class));
+		registrations.put(GenericData.Fixed.class.getName(), new KryoRegistration(GenericData.Fixed.class));
+		registrations.put(GenericData.StringType.class.getName(), new KryoRegistration(GenericData.StringType.class));
+
+		// register the serialized data type
+		registrations.put(serializedDataType.getName(), new KryoRegistration(serializedDataType));
+
+		return registrations;
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
index 1f3fcbc5a85ac..46b93c2000fca 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
@@ -21,7 +21,10 @@
 import java.io.IOException;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.CopyableValue;
@@ -30,7 +33,7 @@
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 @Internal
-public class CopyableValueSerializer<T extends CopyableValue<T>> extends TypeSerializer<T> {
+public final class CopyableValueSerializer<T extends CopyableValue<T>> extends TypeSerializer<T> {
 
 	private static final long serialVersionUID = 1L;
 	
@@ -128,4 +131,41 @@ public boolean equals(Object obj) {
 	public boolean canEqual(Object obj) {
 		return obj instanceof CopyableValueSerializer;
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public CopyableValueSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return new CopyableValueSerializerConfigSnapshot<>(valueClass);
+	}
+
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof CopyableValueSerializerConfigSnapshot
+				&& valueClass.equals(((CopyableValueSerializerConfigSnapshot) configSnapshot).getTypeClass())) {
+			return CompatibilityResult.compatible();
+		} else {
+			return CompatibilityResult.requiresMigration(null);
+		}
+	}
+
+	public static final class CopyableValueSerializerConfigSnapshot<T extends CopyableValue<T>>
+			extends GenericTypeSerializerConfigSnapshot<T> {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public CopyableValueSerializerConfigSnapshot() {}
+
+		public CopyableValueSerializerConfigSnapshot(Class<T> copyableValueClass) {
+			super(copyableValueClass);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
index d9018da982c62..c025d61a42848 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
@@ -19,7 +19,9 @@
 package org.apache.flink.api.java.typeutils.runtime;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.Either;
@@ -183,4 +185,39 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return 17 * leftSerializer.hashCode() + rightSerializer.hashCode();
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public EitherSerializerConfigSnapshot snapshotConfiguration() {
+		return new EitherSerializerConfigSnapshot(
+				leftSerializer.snapshotConfiguration(),
+				rightSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<Either<L, R>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof EitherSerializerConfigSnapshot) {
+			TypeSerializerConfigSnapshot[] leftRightSerializerConfigSnapshots =
+				((EitherSerializerConfigSnapshot) configSnapshot).getNestedSerializerConfigSnapshots();
+
+			CompatibilityResult<L> leftCompatResult = leftSerializer.ensureCompatibility(leftRightSerializerConfigSnapshots[0]);
+			CompatibilityResult<R> rightCompatResult = rightSerializer.ensureCompatibility(leftRightSerializerConfigSnapshots[1]);
+
+			if (!leftCompatResult.requiresMigration() && !rightCompatResult.requiresMigration()) {
+				return CompatibilityResult.compatible();
+			} else {
+				if (leftCompatResult.getConvertDeserializer() != null && rightCompatResult.getConvertDeserializer() != null) {
+					return CompatibilityResult.requiresMigration(
+						new EitherSerializer<>(
+							leftCompatResult.getConvertDeserializer(),
+							rightCompatResult.getConvertDeserializer()));
+				}
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..473d4385dd15f
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.types.Either;
+
+/**
+ * Configuration snapshot for serializers of the {@link Either} type,
+ * containing configuration snapshots of the Left and Right serializers.
+ */
+@Internal
+public final class EitherSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot {
+
+	private static final int VERSION = 1;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public EitherSerializerConfigSnapshot() {}
+
+	public EitherSerializerConfigSnapshot(
+			TypeSerializerConfigSnapshot leftSerializerConfigSnapshot,
+			TypeSerializerConfigSnapshot rightSerializerConfigSnapshot) {
+
+		super(leftSerializerConfigSnapshot, rightSerializerConfigSnapshot);
+	}
+
+	@Override
+	public int getVersion() {
+		return VERSION;
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistration.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistration.java
new file mode 100644
index 0000000000000..882073d6e1a4f
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistration.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.factories.ReflectionSerializerFactory;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.java.typeutils.runtime.KryoRegistrationSerializerConfigSnapshot;
+import org.apache.flink.util.Preconditions;
+
+import java.io.Serializable;
+
+/**
+ * A {@code KryoRegistration} resembles a registered class and its serializer in Kryo.
+ */
+@Internal
+public class KryoRegistration implements Serializable {
+
+	private static final long serialVersionUID = 5375110512910892655L;
+
+	/** IMPORTANT: the order of the enumerations must not change, since their ordinals are used for serialization. */
+	public enum SerializerDefinitionType {
+		UNSPECIFIED, CLASS, INSTANCE
+	}
+
+	/**
+	 * The registered class.
+	 *
+	 * <p>This can be a dummy class {@link KryoRegistrationSerializerConfigSnapshot.DummyRegisteredClass} if
+	 * the class no longer exists when this registration instance was restored.
+	 */
+	private final Class<?> registeredClass;
+
+	/**
+	 * Class of the serializer to use for the registered class.
+	 * Exists only if the serializer definition type is {@link SerializerDefinitionType#CLASS}.
+	 *
+	 * <p>This can be a dummy serializer {@link KryoRegistrationSerializerConfigSnapshot.DummyKryoSerializerClass} if
+	 * the serializer class no longer exists when this registration instance was restored.
+	 */
+	private final Class<? extends Serializer<?>> serializerClass;
+
+	/**
+	 * A serializable instance of the serializer to use for the registered class.
+	 * Exists only if the serializer definition type is {@link SerializerDefinitionType#INSTANCE}.
+	 *
+	 * <p>This can be a dummy serializer {@link KryoRegistrationSerializerConfigSnapshot.DummyKryoSerializerClass} if
+	 * the serializer class no longer exists or is no longer valid when this registration instance was restored.
+	 */
+	private final ExecutionConfig.SerializableSerializer<? extends Serializer<?>> serializableSerializerInstance;
+
+	private SerializerDefinitionType serializerDefinitionType;
+
+	public KryoRegistration(Class<?> registeredClass) {
+		this.registeredClass = Preconditions.checkNotNull(registeredClass);
+
+		this.serializerClass = null;
+		this.serializableSerializerInstance = null;
+
+		this.serializerDefinitionType = SerializerDefinitionType.UNSPECIFIED;
+	}
+
+	public KryoRegistration(Class<?> registeredClass, Class<? extends Serializer<?>> serializerClass) {
+		this.registeredClass = Preconditions.checkNotNull(registeredClass);
+
+		this.serializerClass = Preconditions.checkNotNull(serializerClass);
+		this.serializableSerializerInstance = null;
+
+		this.serializerDefinitionType = SerializerDefinitionType.CLASS;
+	}
+
+	public KryoRegistration(
+			Class<?> registeredClass,
+			ExecutionConfig.SerializableSerializer<? extends Serializer<?>> serializableSerializerInstance) {
+		this.registeredClass = Preconditions.checkNotNull(registeredClass);
+
+		this.serializerClass = null;
+		this.serializableSerializerInstance = Preconditions.checkNotNull(serializableSerializerInstance);
+
+		this.serializerDefinitionType = SerializerDefinitionType.INSTANCE;
+	}
+
+	public Class<?> getRegisteredClass() {
+		return registeredClass;
+	}
+
+	public SerializerDefinitionType getSerializerDefinitionType() {
+		return serializerDefinitionType;
+	}
+
+	public Class<? extends Serializer<?>> getSerializerClass() {
+		return serializerClass;
+	}
+
+	public ExecutionConfig.SerializableSerializer<? extends Serializer<?>> getSerializableSerializerInstance() {
+		return serializableSerializerInstance;
+	}
+
+	public Serializer<?> getSerializer(Kryo kryo) {
+		switch (serializerDefinitionType) {
+			case UNSPECIFIED:
+				return null;
+			case CLASS:
+				return ReflectionSerializerFactory.makeSerializer(kryo, serializerClass, registeredClass);
+			case INSTANCE:
+				return serializableSerializerInstance.getSerializer();
+			default:
+				// this should not happen; adding as a guard for the future
+				throw new IllegalStateException(
+						"Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType);
+		}
+	}
+
+	public boolean isDummy() {
+		return registeredClass.equals(KryoRegistrationSerializerConfigSnapshot.DummyRegisteredClass.class)
+				|| (serializerClass != null
+						&& serializerClass.equals(KryoRegistrationSerializerConfigSnapshot.DummyKryoSerializerClass.class))
+				|| (serializableSerializerInstance != null
+						&& serializableSerializerInstance.getSerializer() instanceof KryoRegistrationSerializerConfigSnapshot.DummyKryoSerializerClass);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj == this) {
+			return true;
+		}
+
+		if (obj == null) {
+			return false;
+		}
+
+		if (obj instanceof KryoRegistration) {
+			KryoRegistration other = (KryoRegistration) obj;
+
+			// we cannot include the serializer instances here because they don't implement the equals method
+			return serializerDefinitionType == other.serializerDefinitionType
+				&& registeredClass == other.registeredClass
+				&& serializerClass == other.serializerClass;
+		} else {
+			return false;
+		}
+	}
+
+	@Override
+	public int hashCode() {
+		int result = serializerDefinitionType.hashCode();
+		result = 31 * result + registeredClass.hashCode();
+
+		if (serializerClass != null) {
+			result = 31 * result + serializerClass.hashCode();
+		}
+
+		return result;
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistrationSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistrationSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..3a42d6912c818
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoRegistrationSerializerConfigSnapshot.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
+import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InvalidClassException;
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Configuration snapshot base class for serializers that use Kryo for serialization.
+ *
+ * <p>The configuration captures the order of Kryo serializer registrations, so that new
+ * Kryo serializers can determine how to reconfigure their registration order to retain
+ * backwards compatibility.
+ *
+ * @param <T> the data type that the Kryo serializer handles.
+ */
+@Internal
+public abstract class KryoRegistrationSerializerConfigSnapshot<T> extends GenericTypeSerializerConfigSnapshot<T> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(KryoRegistrationSerializerConfigSnapshot.class);
+
+	/** Map of class tag to the registration, with ordering. */
+	private LinkedHashMap<String, KryoRegistration> kryoRegistrations;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public KryoRegistrationSerializerConfigSnapshot() {}
+
+	public KryoRegistrationSerializerConfigSnapshot(
+			Class<T> typeClass,
+			LinkedHashMap<String, KryoRegistration> kryoRegistrations) {
+
+		super(typeClass);
+
+		this.kryoRegistrations = Preconditions.checkNotNull(kryoRegistrations);
+	}
+
+	@Override
+	public void write(DataOutputView out) throws IOException {
+		super.write(out);
+
+		out.writeInt(kryoRegistrations.size());
+		for (Map.Entry<String, KryoRegistration> kryoRegistrationEntry : kryoRegistrations.entrySet()) {
+			out.writeUTF(kryoRegistrationEntry.getKey());
+			new KryoRegistrationSerializationProxy<>(kryoRegistrationEntry.getValue()).write(out);
+		}
+	}
+
+	@Override
+	public void read(DataInputView in) throws IOException {
+		super.read(in);
+
+		int numKryoRegistrations = in.readInt();
+		kryoRegistrations = new LinkedHashMap<>(numKryoRegistrations);
+
+		KryoRegistrationSerializationProxy proxy;
+		for (int i = 0; i < numKryoRegistrations; i++) {
+			String classTag = in.readUTF();
+
+			proxy = new KryoRegistrationSerializationProxy(getUserCodeClassLoader());
+			proxy.read(in);
+
+			kryoRegistrations.put(classTag, proxy.kryoRegistration);
+		}
+	}
+
+	public LinkedHashMap<String, KryoRegistration> getKryoRegistrations() {
+		return kryoRegistrations;
+	}
+
+	// --------------------------------------------------------------------------------------------
+
+	private static class KryoRegistrationSerializationProxy<RC> implements IOReadableWritable {
+
+		private ClassLoader userCodeClassLoader;
+
+		private KryoRegistration kryoRegistration;
+
+		public KryoRegistrationSerializationProxy(ClassLoader userCodeClassLoader) {
+			this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
+		}
+
+		public KryoRegistrationSerializationProxy(KryoRegistration kryoRegistration) {
+			this.kryoRegistration = Preconditions.checkNotNull(kryoRegistration);
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+			out.writeUTF(kryoRegistration.getRegisteredClass().getName());
+
+			final KryoRegistration.SerializerDefinitionType serializerDefinitionType = kryoRegistration.getSerializerDefinitionType();
+
+			out.writeInt(serializerDefinitionType.ordinal());
+			switch (serializerDefinitionType) {
+				case UNSPECIFIED:
+					// nothing else to write
+					break;
+				case CLASS:
+					out.writeUTF(kryoRegistration.getSerializerClass().getName());
+					break;
+				case INSTANCE:
+					InstantiationUtil.serializeObject(new DataOutputViewStream(out), kryoRegistration.getSerializableSerializerInstance());
+					break;
+				default:
+					// this should not happen; adding as a guard for the future
+					throw new IllegalStateException(
+							"Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType);
+			}
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		public void read(DataInputView in) throws IOException {
+			String registeredClassname = in.readUTF();
+
+			Class<RC> registeredClass;
+			try {
+				registeredClass = (Class<RC>) Class.forName(registeredClassname, true, userCodeClassLoader);
+			} catch (ClassNotFoundException e) {
+				LOG.warn("Cannot find registered class " + registeredClassname + " for Kryo serialization in classpath;" +
+					" using a dummy class as a placeholder.", e);
+
+				registeredClass = (Class) DummyRegisteredClass.class;
+			}
+
+			final KryoRegistration.SerializerDefinitionType serializerDefinitionType =
+				KryoRegistration.SerializerDefinitionType.values()[in.readInt()];
+
+			switch (serializerDefinitionType) {
+				case UNSPECIFIED:
+					kryoRegistration = new KryoRegistration(registeredClass);
+					break;
+
+				case CLASS:
+					String serializerClassname = in.readUTF();
+
+					Class serializerClass;
+					try {
+						serializerClass = Class.forName(serializerClassname, true, userCodeClassLoader);
+					} catch (ClassNotFoundException e) {
+						LOG.warn("Cannot find registered Kryo serializer class for class " + registeredClassname +
+								" in classpath; using a dummy Kryo serializer that should be replaced as soon as" +
+								" a new Kryo serializer for the class is present", e);
+
+						serializerClass = DummyKryoSerializerClass.class;
+					}
+
+					kryoRegistration = new KryoRegistration(registeredClass, serializerClass);
+					break;
+
+				case INSTANCE:
+					ExecutionConfig.SerializableSerializer<? extends Serializer<RC>> serializerInstance;
+					try {
+						serializerInstance = InstantiationUtil.deserializeObject(new DataInputViewStream(in), userCodeClassLoader);
+					} catch (ClassNotFoundException e) {
+						LOG.warn("Cannot find registered Kryo serializer class for class " + registeredClassname +
+								" in classpath; using a dummy Kryo serializer that should be replaced as soon as" +
+								" a new Kryo serializer for the class is present", e);
+
+						serializerInstance = new ExecutionConfig.SerializableSerializer<>(new DummyKryoSerializerClass<RC>());
+					} catch (InvalidClassException e) {
+						LOG.warn("The registered Kryo serializer class for class " + registeredClassname +
+								" has changed and is no longer valid; using a dummy Kryo serializer that should be replaced" +
+								" as soon as a new Kryo serializer for the class is present.", e);
+
+						serializerInstance = new ExecutionConfig.SerializableSerializer<>(new DummyKryoSerializerClass<RC>());
+					}
+
+					kryoRegistration = new KryoRegistration(registeredClass, serializerInstance);
+					break;
+
+				default:
+					// this should not happen; adding as a guard for the future
+					throw new IllegalStateException(
+							"Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType);
+			}
+		}
+	}
+
+	/**
+	 * Placeholder dummy for a previously registered class that can no longer be found in classpath on restore.
+	 */
+	public static class DummyRegisteredClass {}
+
+	/**
+	 * Placeholder dummmy for a previously registered Kryo serializer that is no longer valid or in classpath on restore.
+	 */
+	public static class DummyKryoSerializerClass<RC> extends Serializer<RC> implements Serializable {
+
+		private static final long serialVersionUID = -6172780797425739308L;
+
+		@Override
+		public void write(Kryo kryo, Output output, Object o) {
+			throw new UnsupportedOperationException(
+					"This exception indicates that you're trying to write a data type" +
+						" that no longer has a valid Kryo serializer registered for it.");
+		}
+
+		@Override
+		public Object read(Kryo kryo, Input input, Class aClass) {
+			throw new UnsupportedOperationException(
+					"This exception indicates that you're trying to read a data type" +
+						" that no longer has a valid Kryo serializer registered for it.");
+		}
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		return super.equals(obj)
+				&& kryoRegistrations.equals(((KryoSerializer.KryoSerializerConfigSnapshot) obj).getKryoRegistrations());
+	}
+
+	@Override
+	public int hashCode() {
+		return super.hashCode() + kryoRegistrations.hashCode();
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoUtils.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoUtils.java
index 50c46e4d54767..0937ac7f30436 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoUtils.java
@@ -20,11 +20,13 @@
 
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.KryoException;
+import com.esotericsoftware.kryo.Serializer;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.util.InstantiationUtil;
 
 import java.io.IOException;
+import java.util.Collection;
 
 /**
  * Convenience methods for Kryo
@@ -86,4 +88,29 @@ public static <T> T copy(T from, T reuse, Kryo kryo, TypeSerializer<T> serialize
 			}
 		}
 	}
+
+	/**
+	 * Apply a list of {@link KryoRegistration} to a Kryo instance. The list of registrations is
+	 * assumed to already be a final resolution of all possible registration overwrites.
+	 *
+	 * <p>The registrations are applied in the given order and always specify the registration id as
+	 * the next available id in the Kryo instance (providing the id just extra ensures nothing is
+	 * overwritten, and isn't strictly required);
+	 *
+	 * @param kryo the Kryo instance to apply the registrations
+	 * @param resolvedRegistrations the registrations, which should already be resolved of all possible registration overwrites
+	 */
+	public static void applyRegistrations(Kryo kryo, Collection<KryoRegistration> resolvedRegistrations) {
+
+		Serializer<?> serializer;
+		for (KryoRegistration registration : resolvedRegistrations) {
+			serializer = registration.getSerializer(kryo);
+
+			if (serializer != null) {
+				kryo.register(registration.getRegisteredClass(), serializer, kryo.getNextRegistrationId());
+			} else {
+				kryo.register(registration.getRegisteredClass(), kryo.getNextRegistrationId());
+			}
+		}
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index 1a9c8f992e768..08da49eb91d89 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -23,22 +23,25 @@
 import java.io.ObjectOutputStream;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.Preconditions;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
@@ -53,22 +56,55 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 
 	private static final long serialVersionUID = 1L;
 
-	private final Class<T> clazz;
+	// --------------------------------------------------------------------------------------------
+	// PojoSerializer parameters
+	// --------------------------------------------------------------------------------------------
 
-	private final TypeSerializer<Object>[] fieldSerializers;
+	/** The POJO type class. */
+	private final Class<T> clazz;
 
+	/**
+	 * Fields of the POJO and their serializers.
+	 *
+	 * <p>The fields are kept as a separate transient member, with their serialization
+	 * handled with the {@link #readObject(ObjectInputStream)} and {@link #writeObject(ObjectOutputStream)}
+	 * methods.
+	 *
+	 * <p>These may be reconfigured in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
+	 */
+	private transient Field[] fields;
+	private TypeSerializer<Object>[] fieldSerializers;
 	private final int numFields;
 
-	private final Map<Class<?>, Integer> registeredClasses;
-
-	private final TypeSerializer<?>[] registeredSerializers;
-
+	/**
+	 * Registered subclasses and their serializers.
+	 * Each subclass to their registered class tag is maintained as a separate map ordered by the class tag.
+	 *
+	 * <p>These may be reconfigured in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
+	 */
+	private LinkedHashMap<Class<?>, Integer> registeredClasses;
+	private TypeSerializer<?>[] registeredSerializers;
+
+	/**
+	 * Cache of non-registered subclasses to their serializers, created on-the-fly.
+	 *
+	 * <p>This cache is persisted and will be repopulated with reconfigured serializers
+	 * in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
+	 */
+	private transient HashMap<Class<?>, TypeSerializer<?>> subclassSerializerCache;
+
+	// --------------------------------------------------------------------------------------------
+
+	/**
+	 * Configuration of the current execution.
+	 *
+	 * <p>Nested serializers created using this will have the most up-to-date configuration,
+	 * and can be resolved for backwards compatibility with previous configuration
+	 * snapshots in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
+	 */
 	private final ExecutionConfig executionConfig;
 
-	private transient Map<Class<?>, TypeSerializer<?>> subclassSerializerCache;
 	private transient ClassLoader cl;
-	// We need to handle these ourselves in writeObject()/readObject()
-	private transient Field[] fields;
 
 	@SuppressWarnings("unchecked")
 	public PojoSerializer(
@@ -83,93 +119,20 @@ public PojoSerializer(
 		this.numFields = fieldSerializers.length;
 		this.executionConfig = checkNotNull(executionConfig);
 
-		LinkedHashSet<Class<?>> registeredPojoTypes = executionConfig.getRegisteredPojoTypes();
-
 		for (int i = 0; i < numFields; i++) {
 			this.fields[i].setAccessible(true);
 		}
 
 		cl = Thread.currentThread().getContextClassLoader();
 
-		subclassSerializerCache = new HashMap<Class<?>, TypeSerializer<?>>();
-
 		// We only want those classes that are not our own class and are actually sub-classes.
-		List<Class<?>> cleanedTaggedClasses = new ArrayList<Class<?>>(registeredPojoTypes.size());
-		for (Class<?> registeredClass: registeredPojoTypes) {
-			if (registeredClass.equals(clazz)) {
-				continue;
-			}
-			if (!clazz.isAssignableFrom(registeredClass)) {
-				continue;
-			}
-			cleanedTaggedClasses.add(registeredClass);
-
-		}
-		this.registeredClasses = new LinkedHashMap<Class<?>, Integer>(cleanedTaggedClasses.size());
-		registeredSerializers = new TypeSerializer[cleanedTaggedClasses.size()];
+		LinkedHashSet<Class<?>> registeredSubclasses =
+				getRegisteredSubclassesFromExecutionConfig(clazz, executionConfig);
 
-		int id = 0;
-		for (Class<?> registeredClass: cleanedTaggedClasses) {
-			this.registeredClasses.put(registeredClass, id);
-			TypeInformation<?> typeInfo = TypeExtractor.createTypeInfo(registeredClass);
-			registeredSerializers[id] = typeInfo.createSerializer(executionConfig);
+		this.registeredClasses = createRegisteredSubclassTags(registeredSubclasses);
+		this.registeredSerializers = createRegisteredSubclassSerializers(registeredSubclasses, executionConfig);
 
-			id++;
-		}
-	}
-
-	private void writeObject(ObjectOutputStream out)
-			throws IOException, ClassNotFoundException {
-		out.defaultWriteObject();
-		out.writeInt(fields.length);
-		for (Field field: fields) {
-			FieldSerializer.serializeField(field, out);
-		}
-	}
-
-	private void readObject(ObjectInputStream in)
-			throws IOException, ClassNotFoundException {
-		in.defaultReadObject();
-		int numFields = in.readInt();
-		fields = new Field[numFields];
-		for (int i = 0; i < numFields; i++) {
-			fields[i] = FieldSerializer.deserializeField(in);
-		}
-
-		cl = Thread.currentThread().getContextClassLoader();
-		subclassSerializerCache = new HashMap<Class<?>, TypeSerializer<?>>();
-	}
-
-	private TypeSerializer<?> getSubclassSerializer(Class<?> subclass) {
-		TypeSerializer<?> result = subclassSerializerCache.get(subclass);
-		if (result == null) {
-
-			TypeInformation<?> typeInfo = TypeExtractor.createTypeInfo(subclass);
-			result = typeInfo.createSerializer(executionConfig);
-			if (result instanceof PojoSerializer) {
-				PojoSerializer<?> subclassSerializer = (PojoSerializer<?>) result;
-				subclassSerializer.copyBaseFieldOrder(this);
-			}
-			subclassSerializerCache.put(subclass, result);
-
-		}
-		return result;
-	}
-
-	@SuppressWarnings("unused")
-	private boolean hasField(Field f) {
-		for (Field field: fields) {
-			if (f.equals(field)) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	private void copyBaseFieldOrder(PojoSerializer<?> baseSerializer) {
-		// do nothing for now, but in the future, adapt subclass serializer to have same
-		// ordering as base class serializer so that binary comparison on base class fields
-		// can work
+		this.subclassSerializerCache = new HashMap<>();
 	}
 	
 	@Override
@@ -296,7 +259,7 @@ public T copy(T from, T reuse) {
 					}
 				}
 			} catch (IllegalAccessException e) {
-				throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields" + "before.");
+				throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields before.", e);
 			}
 		} else {
 			TypeSerializer subclassSerializer = getSubclassSerializer(actualType);
@@ -341,13 +304,15 @@ public void serialize(T value, DataOutputView target) throws IOException {
 
 		target.writeByte(flags);
 
+		// if its a registered subclass, write the class tag id, otherwise write the full classname
 		if ((flags & IS_SUBCLASS) != 0) {
 			target.writeUTF(actualClass.getName());
 		} else if ((flags & IS_TAGGED_SUBCLASS) != 0) {
 			target.writeByte(subclassTag);
 		}
 
-
+		// if its a subclass, use the corresponding subclass serializer,
+		// otherwise serialize each field with our field serializers
 		if ((flags & NO_SUBCLASS) != 0) {
 			try {
 				for (int i = 0; i < numFields; i++) {
@@ -360,8 +325,7 @@ public void serialize(T value, DataOutputView target) throws IOException {
 					}
 				}
 			} catch (IllegalAccessException e) {
-				throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields" + "before.");
-
+				throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields before.", e);
 			}
 		} else {
 			// subclass
@@ -418,8 +382,7 @@ public T deserialize(DataInputView source) throws IOException {
 					}
 				}
 			} catch (IllegalAccessException e) {
-				throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields" + "before.");
-
+				throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields before.", e);
 			}
 		} else {
 			if (subclassSerializer != null) {
@@ -493,8 +456,7 @@ public T deserialize(T reuse, DataInputView source) throws IOException {
 					}
 				}
 			} catch (IllegalAccessException e) {
-				throw new RuntimeException(
-						"Error during POJO copy, this should not happen since we check the fields before.");
+				throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields before.", e);
 			}
 		} else {
 			if (subclassSerializer != null) {
@@ -574,4 +536,527 @@ public boolean equals(Object obj) {
 	public boolean canEqual(Object obj) {
 		return obj instanceof PojoSerializer;
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public PojoSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return buildConfigSnapshot(
+				clazz,
+				registeredClasses,
+				registeredSerializers,
+				fields,
+				fieldSerializers,
+				subclassSerializerCache);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof PojoSerializerConfigSnapshot) {
+			final PojoSerializerConfigSnapshot<T> config = (PojoSerializerConfigSnapshot<T>) configSnapshot;
+
+			if (clazz.equals(config.getTypeClass())) {
+				if (this.numFields == config.getFieldToSerializerConfigSnapshot().size()) {
+
+					CompatibilityResult<?> compatResult;
+
+					// ----------- check field order and compatibility of field serializers -----------
+
+					// reordered fields and their serializers;
+					// this won't be applied to this serializer until all compatibility checks have been completed
+					final Field[] reorderedFields = new Field[this.numFields];
+					final TypeSerializer<Object>[] reorderedFieldSerializers =
+						(TypeSerializer<Object>[]) new TypeSerializer<?>[this.numFields];
+
+					int i = 0;
+					for (Map.Entry<Field, TypeSerializerConfigSnapshot> fieldToConfigSnapshotEntry
+							: config.getFieldToSerializerConfigSnapshot().entrySet()) {
+
+						int fieldIndex = findField(fieldToConfigSnapshotEntry.getKey());
+						if (fieldIndex != -1) {
+							reorderedFields[i] = fieldToConfigSnapshotEntry.getKey();
+
+							compatResult = fieldSerializers[fieldIndex].ensureCompatibility(fieldToConfigSnapshotEntry.getValue());
+							if (compatResult.requiresMigration()) {
+								return CompatibilityResult.requiresMigration(null);
+							} else {
+								reorderedFieldSerializers[i] = fieldSerializers[fieldIndex];
+							}
+						} else {
+							return CompatibilityResult.requiresMigration(null);
+						}
+
+						i++;
+					}
+
+					// ---- check subclass registration order and compatibility of registered serializers ----
+
+					// reordered subclass registrations and their serializers;
+					// this won't be applied to this serializer until all compatibility checks have been completed
+					final LinkedHashMap<Class<?>, Integer> reorderedRegisteredSubclassesToClasstags;
+					final TypeSerializer<?>[] reorderedRegisteredSubclassSerializers;
+
+					final LinkedHashMap<Class<?>, TypeSerializerConfigSnapshot> previousRegistrations =
+						config.getRegisteredSubclassesToSerializerConfigSnapshots();
+
+					// the reconfigured list of registered subclasses will be the previous registered
+					// subclasses in the original order with new subclasses appended at the end
+					LinkedHashSet<Class<?>> reorderedRegisteredSubclasses = new LinkedHashSet<>();
+					reorderedRegisteredSubclasses.addAll(previousRegistrations.keySet());
+					reorderedRegisteredSubclasses.addAll(
+						getRegisteredSubclassesFromExecutionConfig(clazz, executionConfig));
+
+					// re-establish the registered class tags and serializers
+					reorderedRegisteredSubclassesToClasstags = createRegisteredSubclassTags(reorderedRegisteredSubclasses);
+					reorderedRegisteredSubclassSerializers = createRegisteredSubclassSerializers(
+						reorderedRegisteredSubclasses, executionConfig);
+
+					i = 0;
+					for (TypeSerializerConfigSnapshot previousRegisteredSerializerConfig : previousRegistrations.values()) {
+						// check compatibility of subclass serializer
+						compatResult = reorderedRegisteredSubclassSerializers[i].ensureCompatibility(previousRegisteredSerializerConfig);
+						if (compatResult.requiresMigration()) {
+							return CompatibilityResult.requiresMigration(null);
+						}
+
+						i++;
+					}
+
+					// ------------------ ensure compatibility of non-registered subclass serializers ------------------
+
+					// the rebuilt cache for non-registered subclass serializers;
+					// this won't be applied to this serializer until all compatibility checks have been completed
+					HashMap<Class<?>, TypeSerializer<?>> rebuiltCache = new HashMap<>();
+
+					for (Map.Entry<Class<?>, TypeSerializerConfigSnapshot> previousCachedEntry
+							: config.getNonRegisteredSubclassesToSerializerConfigSnapshots().entrySet()) {
+
+						TypeSerializer<?> cachedSerializer = createSubclassSerializer(previousCachedEntry.getKey());
+
+						// check compatibility of cached subclass serializer
+						compatResult = cachedSerializer.ensureCompatibility(previousCachedEntry.getValue());
+						if (compatResult.requiresMigration()) {
+							return CompatibilityResult.requiresMigration(null);
+						} else {
+							rebuiltCache.put(previousCachedEntry.getKey(), cachedSerializer);
+						}
+					}
+
+					// completed compatibility checks; up to this point, we can just reconfigure
+					// the serializer so that it is compatible and migration is not required
+
+					this.fields = reorderedFields;
+					this.fieldSerializers = reorderedFieldSerializers;
+
+					this.registeredClasses = reorderedRegisteredSubclassesToClasstags;
+					this.registeredSerializers = reorderedRegisteredSubclassSerializers;
+
+					this.subclassSerializerCache = rebuiltCache;
+
+					return CompatibilityResult.compatible();
+				}
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	public static final class PojoSerializerConfigSnapshot<T> extends GenericTypeSerializerConfigSnapshot<T> {
+
+		private static final int VERSION = 1;
+
+		/**
+		 * Ordered map of POJO fields to the configuration snapshots of their corresponding serializers.
+		 *
+		 * <p>Ordering of the fields is kept so that new Pojo serializers for previous data
+		 * may reorder the fields in case they are different. The order of the fields need to
+		 * stay the same for binary compatibility, as the field order is part of the serialization format.
+		 */
+		private LinkedHashMap<Field, TypeSerializerConfigSnapshot> fieldToSerializerConfigSnapshot;
+
+		/**
+		 * Ordered map of registered subclasses to the configuration snapshots of their corresponding serializers.
+		 *
+		 * <p>Ordering of the registered subclasses is kept so that new Pojo serializers for previous data
+		 * may retain the same class tag used for registered subclasses. Newly registered subclasses that
+		 * weren't present before should be appended with the next available class tag.
+		 */
+		private LinkedHashMap<Class<?>, TypeSerializerConfigSnapshot> registeredSubclassesToSerializerConfigSnapshots;
+
+		/**
+		 * Configuration snapshots of previously cached non-registered subclass serializers.
+		 *
+		 * <p>This is kept so that new Pojo serializers may eagerly repopulate their
+		 * cache with reconfigured subclass serializers.
+		 */
+		private HashMap<Class<?>, TypeSerializerConfigSnapshot> nonRegisteredSubclassesToSerializerConfigSnapshots;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public PojoSerializerConfigSnapshot() {}
+
+		public PojoSerializerConfigSnapshot(
+				Class<T> pojoType,
+				LinkedHashMap<Field, TypeSerializerConfigSnapshot> fieldToSerializerConfigSnapshot,
+				LinkedHashMap<Class<?>, TypeSerializerConfigSnapshot> registeredSubclassesToSerializerConfigSnapshots,
+				HashMap<Class<?>, TypeSerializerConfigSnapshot> nonRegisteredSubclassesToSerializerConfigSnapshots) {
+
+			super(pojoType);
+
+			this.fieldToSerializerConfigSnapshot =
+					Preconditions.checkNotNull(fieldToSerializerConfigSnapshot);
+			this.registeredSubclassesToSerializerConfigSnapshots =
+					Preconditions.checkNotNull(registeredSubclassesToSerializerConfigSnapshots);
+			this.nonRegisteredSubclassesToSerializerConfigSnapshots =
+					Preconditions.checkNotNull(nonRegisteredSubclassesToSerializerConfigSnapshots);
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+			super.write(out);
+
+			// --- write fields and their serializers, in order
+
+			out.writeInt(fieldToSerializerConfigSnapshot.size());
+			for (Map.Entry<Field, TypeSerializerConfigSnapshot> entry
+				: fieldToSerializerConfigSnapshot.entrySet()) {
+				out.writeUTF(entry.getKey().getName());
+				TypeSerializerUtil.writeSerializerConfigSnapshot(out, entry.getValue());
+			}
+
+			// --- write registered subclasses and their serializers, in registration order
+
+			out.writeInt(registeredSubclassesToSerializerConfigSnapshots.size());
+			for (Map.Entry<Class<?>, TypeSerializerConfigSnapshot> entry
+					: registeredSubclassesToSerializerConfigSnapshots.entrySet()) {
+				out.writeUTF(entry.getKey().getName());
+				TypeSerializerUtil.writeSerializerConfigSnapshot(out, entry.getValue());
+			}
+
+			// --- write snapshot of non-registered subclass serializer cache
+
+			out.writeInt(nonRegisteredSubclassesToSerializerConfigSnapshots.size());
+			for (Map.Entry<Class<?>, TypeSerializerConfigSnapshot> entry
+					: nonRegisteredSubclassesToSerializerConfigSnapshots.entrySet()) {
+				out.writeUTF(entry.getKey().getName());
+				TypeSerializerUtil.writeSerializerConfigSnapshot(out, entry.getValue());
+			}
+		}
+
+		@Override
+		public void read(DataInputView in) throws IOException {
+			super.read(in);
+
+			// --- read fields and their serializers, in order
+
+			int numFields = in.readInt();
+			this.fieldToSerializerConfigSnapshot = new LinkedHashMap<>(numFields);
+			String fieldName;
+			Field field;
+			for (int i = 0; i < numFields; i++) {
+				fieldName = in.readUTF();
+
+				// search all superclasses for the field
+				Class<?> clazz = getTypeClass();
+				field = null;
+				while (clazz != null) {
+					try {
+						field = clazz.getDeclaredField(fieldName);
+						field.setAccessible(true);
+						break;
+					} catch (NoSuchFieldException e) {
+						clazz = clazz.getSuperclass();
+					}
+				}
+
+				if (field == null) {
+					// the field no longer exists in the POJO
+					throw new IOException("Can't find field " + fieldName + " in POJO class " + getTypeClass().getName());
+				} else {
+					fieldToSerializerConfigSnapshot.put(
+							field,
+							TypeSerializerUtil.readSerializerConfigSnapshot(in, getUserCodeClassLoader()));
+				}
+			}
+
+			// --- read registered subclasses and their serializers, in registration order
+
+			int numRegisteredSubclasses = in.readInt();
+			this.registeredSubclassesToSerializerConfigSnapshots = new LinkedHashMap<>(numRegisteredSubclasses);
+			String registeredSubclassname;
+			Class<?> registeredSubclass;
+			for (int i = 0; i < numRegisteredSubclasses; i++) {
+				registeredSubclassname = in.readUTF();
+				try {
+					registeredSubclass = Class.forName(registeredSubclassname, true, getUserCodeClassLoader());
+				} catch (ClassNotFoundException e) {
+					throw new IOException("Cannot find requested class " + registeredSubclassname + " in classpath.", e);
+				}
+
+				this.registeredSubclassesToSerializerConfigSnapshots.put(
+						registeredSubclass,
+						TypeSerializerUtil.readSerializerConfigSnapshot(in, getUserCodeClassLoader()));
+			}
+
+			// --- read snapshot of non-registered subclass serializer cache
+
+			int numCachedSubclassSerializers = in.readInt();
+			this.nonRegisteredSubclassesToSerializerConfigSnapshots = new HashMap<>(numCachedSubclassSerializers);
+			String cachedSubclassname;
+			Class<?> cachedSubclass;
+			for (int i = 0; i < numCachedSubclassSerializers; i++) {
+				cachedSubclassname = in.readUTF();
+				try {
+					cachedSubclass = Class.forName(cachedSubclassname, true, getUserCodeClassLoader());
+				} catch (ClassNotFoundException e) {
+					throw new IOException("Cannot find requested class " + cachedSubclassname + " in classpath.", e);
+				}
+
+				this.nonRegisteredSubclassesToSerializerConfigSnapshots.put(
+						cachedSubclass,
+						TypeSerializerUtil.readSerializerConfigSnapshot(in, getUserCodeClassLoader()));
+			}
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+
+		public LinkedHashMap<Field, TypeSerializerConfigSnapshot> getFieldToSerializerConfigSnapshot() {
+			return fieldToSerializerConfigSnapshot;
+		}
+
+		public LinkedHashMap<Class<?>, TypeSerializerConfigSnapshot> getRegisteredSubclassesToSerializerConfigSnapshots() {
+			return registeredSubclassesToSerializerConfigSnapshots;
+		}
+
+		public HashMap<Class<?>, TypeSerializerConfigSnapshot> getNonRegisteredSubclassesToSerializerConfigSnapshots() {
+			return nonRegisteredSubclassesToSerializerConfigSnapshots;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return super.equals(obj)
+					&& (obj instanceof PojoSerializerConfigSnapshot)
+					&& fieldToSerializerConfigSnapshot.equals(((PojoSerializerConfigSnapshot) obj).getFieldToSerializerConfigSnapshot())
+					&& registeredSubclassesToSerializerConfigSnapshots.equals(((PojoSerializerConfigSnapshot) obj).getRegisteredSubclassesToSerializerConfigSnapshots())
+					&& nonRegisteredSubclassesToSerializerConfigSnapshots.equals(((PojoSerializerConfigSnapshot) obj).nonRegisteredSubclassesToSerializerConfigSnapshots);
+		}
+
+		@Override
+		public int hashCode() {
+			return super.hashCode()
+				+ Objects.hash(
+					fieldToSerializerConfigSnapshot,
+					registeredSubclassesToSerializerConfigSnapshots,
+					nonRegisteredSubclassesToSerializerConfigSnapshots);
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+
+	private void writeObject(ObjectOutputStream out)
+		throws IOException, ClassNotFoundException {
+		out.defaultWriteObject();
+		out.writeInt(fields.length);
+		for (Field field: fields) {
+			FieldSerializer.serializeField(field, out);
+		}
+	}
+
+	private void readObject(ObjectInputStream in)
+		throws IOException, ClassNotFoundException {
+		in.defaultReadObject();
+		int numFields = in.readInt();
+		fields = new Field[numFields];
+		for (int i = 0; i < numFields; i++) {
+			fields[i] = FieldSerializer.deserializeField(in);
+		}
+
+		cl = Thread.currentThread().getContextClassLoader();
+		subclassSerializerCache = new HashMap<Class<?>, TypeSerializer<?>>();
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Utilities
+	// --------------------------------------------------------------------------------------------
+
+	/**
+	 * Extracts the subclasses of the base POJO class registered in the execution config.
+	 */
+	private static LinkedHashSet<Class<?>> getRegisteredSubclassesFromExecutionConfig(
+			Class<?> basePojoClass,
+			ExecutionConfig executionConfig) {
+
+		LinkedHashSet<Class<?>> subclassesInRegistrationOrder = new LinkedHashSet<>(executionConfig.getRegisteredPojoTypes().size());
+		for (Class<?> registeredClass : executionConfig.getRegisteredPojoTypes()) {
+			if (registeredClass.equals(basePojoClass)) {
+				continue;
+			}
+			if (!basePojoClass.isAssignableFrom(registeredClass)) {
+				continue;
+			}
+			subclassesInRegistrationOrder.add(registeredClass);
+		}
+
+		return subclassesInRegistrationOrder;
+	}
+
+	/**
+	 * Builds map of registered subclasses to their class tags.
+	 * Class tags will be integers starting from 0, assigned incrementally with the order of provided subclasses.
+	 */
+	private static LinkedHashMap<Class<?>, Integer> createRegisteredSubclassTags(LinkedHashSet<Class<?>> registeredSubclasses) {
+		final LinkedHashMap<Class<?>, Integer> classToTag = new LinkedHashMap<>();
+
+		int id = 0;
+		for (Class<?> registeredClass : registeredSubclasses) {
+			classToTag.put(registeredClass, id);
+			id ++;
+		}
+
+		return classToTag;
+	}
+
+	/**
+	 * Creates an array of serializers for provided list of registered subclasses.
+	 * Order of returned serializers will correspond to order of provided subclasses.
+	 */
+	private static TypeSerializer<?>[] createRegisteredSubclassSerializers(
+			LinkedHashSet<Class<?>> registeredSubclasses,
+			ExecutionConfig executionConfig) {
+
+		final TypeSerializer<?>[] subclassSerializers = new TypeSerializer[registeredSubclasses.size()];
+
+		int i = 0;
+		for (Class<?> registeredClass : registeredSubclasses) {
+			subclassSerializers[i] = TypeExtractor.createTypeInfo(registeredClass).createSerializer(executionConfig);
+			i++;
+		}
+
+		return subclassSerializers;
+	}
+
+	/**
+	 * Fetches cached serializer for a non-registered subclass;
+	 * also creates the serializer if it doesn't exist yet.
+	 *
+	 * This method is also exposed to package-private access
+	 * for testing purposes.
+	 */
+	TypeSerializer<?> getSubclassSerializer(Class<?> subclass) {
+		TypeSerializer<?> result = subclassSerializerCache.get(subclass);
+		if (result == null) {
+			result = createSubclassSerializer(subclass);
+			subclassSerializerCache.put(subclass, result);
+		}
+		return result;
+	}
+
+	private TypeSerializer<?> createSubclassSerializer(Class<?> subclass) {
+		TypeSerializer<?> serializer = TypeExtractor.createTypeInfo(subclass).createSerializer(executionConfig);
+
+		if (serializer instanceof PojoSerializer) {
+			PojoSerializer<?> subclassSerializer = (PojoSerializer<?>) serializer;
+			subclassSerializer.copyBaseFieldOrder(this);
+		}
+
+		return serializer;
+	}
+
+	/**
+	 * Finds and returns the order (0-based) of a POJO field.
+	 * Returns -1 if the field does not exist for this POJO.
+	 */
+	private int findField(Field f) {
+		int foundIndex = 0;
+		for (Field field : fields) {
+			if (f.equals(field)) {
+				return foundIndex;
+			}
+
+			foundIndex++;
+		}
+
+		return -1;
+	}
+
+	private void copyBaseFieldOrder(PojoSerializer<?> baseSerializer) {
+		// do nothing for now, but in the future, adapt subclass serializer to have same
+		// ordering as base class serializer so that binary comparison on base class fields
+		// can work
+	}
+
+	/**
+	 * Build and return a snapshot of the serializer's parameters and currently cached serializers.
+	 */
+	private static <T> PojoSerializerConfigSnapshot<T> buildConfigSnapshot(
+			Class<T> pojoType,
+			LinkedHashMap<Class<?>, Integer> registeredSubclassesToTags,
+			TypeSerializer<?>[] registeredSubclassSerializers,
+			Field[] fields,
+			TypeSerializer<?>[] fieldSerializers,
+			HashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializerCache) {
+
+		final LinkedHashMap<Field, TypeSerializerConfigSnapshot> fieldToSerializerConfigSnapshots =
+			new LinkedHashMap<>(fields.length);
+
+		for (int i = 0; i < fields.length; i++) {
+			fieldToSerializerConfigSnapshots.put(fields[i], fieldSerializers[i].snapshotConfiguration());
+		}
+
+		final LinkedHashMap<Class<?>, TypeSerializerConfigSnapshot> registeredSubclassesToSerializerConfigSnapshots =
+				new LinkedHashMap<>(registeredSubclassesToTags.size());
+
+		for (Map.Entry<Class<?>, Integer> entry : registeredSubclassesToTags.entrySet()) {
+			registeredSubclassesToSerializerConfigSnapshots.put(
+					entry.getKey(),
+					registeredSubclassSerializers[entry.getValue()].snapshotConfiguration());
+		}
+
+		final HashMap<Class<?>, TypeSerializerConfigSnapshot> nonRegisteredSubclassesToSerializerConfigSnapshots =
+				new LinkedHashMap<>(nonRegisteredSubclassSerializerCache.size());
+
+		for (Map.Entry<Class<?>, TypeSerializer<?>> entry : nonRegisteredSubclassSerializerCache.entrySet()) {
+			nonRegisteredSubclassesToSerializerConfigSnapshots.put(entry.getKey(), entry.getValue().snapshotConfiguration());
+		}
+
+		return new PojoSerializerConfigSnapshot<>(
+				pojoType,
+				fieldToSerializerConfigSnapshots,
+				registeredSubclassesToSerializerConfigSnapshots,
+				nonRegisteredSubclassesToSerializerConfigSnapshots);
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Test utilities
+	// --------------------------------------------------------------------------------------------
+
+	@VisibleForTesting
+	Field[] getFields() {
+		return fields;
+	}
+
+	@VisibleForTesting
+	TypeSerializer<?>[] getFieldSerializers() {
+		return fieldSerializers;
+	}
+
+	@VisibleForTesting
+	LinkedHashMap<Class<?>, Integer> getRegisteredClasses() {
+		return registeredClasses;
+	}
+
+	@VisibleForTesting
+	TypeSerializer<?>[] getRegisteredSerializers() {
+		return registeredSerializers;
+	}
+
+	@VisibleForTesting
+	HashMap<Class<?>, TypeSerializer<?>> getSubclassSerializerCache() {
+		return subclassSerializerCache;
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
index dbd5d3a549925..5770dac9ee34d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
@@ -18,12 +18,17 @@
 package org.apache.flink.api.java.typeutils.runtime;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.Row;
 
 import java.io.IOException;
+import java.io.ObjectInputStream;
 import java.util.Arrays;
 
 import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.readIntoAndCopyNullMask;
@@ -35,12 +40,15 @@
  * Serializer for {@link Row}.
  */
 @Internal
-public class RowSerializer extends TypeSerializer<Row> {
+public final class RowSerializer extends TypeSerializer<Row> {
 
 	private static final long serialVersionUID = 1L;
-	private final boolean[] nullMask;
+
 	private final TypeSerializer<Object>[] fieldSerializers;
 
+	private transient boolean[] nullMask;
+
+	@SuppressWarnings("unchecked")
 	public RowSerializer(TypeSerializer<?>[] fieldSerializers) {
 		this.fieldSerializers = (TypeSerializer<Object>[]) checkNotNull(fieldSerializers);
 		this.nullMask = new boolean[fieldSerializers.length];
@@ -231,4 +239,73 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return Arrays.hashCode(fieldSerializers);
 	}
+
+	// --------------------------------------------------------------------------------------------
+
+	private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+		in.defaultReadObject();
+		this.nullMask = new boolean[fieldSerializers.length];
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public RowSerializerConfigSnapshot snapshotConfiguration() {
+		return new RowSerializerConfigSnapshot(TypeSerializerUtil.snapshotConfigurations(fieldSerializers));
+	}
+
+	@Override
+	public CompatibilityResult<Row> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof RowSerializerConfigSnapshot) {
+			TypeSerializerConfigSnapshot[] fieldSerializerConfigSnapshots =
+				((RowSerializerConfigSnapshot) configSnapshot).getNestedSerializerConfigSnapshots();
+
+			if (fieldSerializerConfigSnapshots.length == fieldSerializers.length) {
+				boolean requireMigration = false;
+				TypeSerializer<?>[] convertDeserializers = new TypeSerializer<?>[fieldSerializers.length];
+
+				CompatibilityResult<?> compatResult;
+				for (int i = 0; i < fieldSerializers.length; i++) {
+					compatResult = fieldSerializers[i].ensureCompatibility(fieldSerializerConfigSnapshots[i]);
+					if (compatResult.requiresMigration()) {
+						requireMigration = true;
+
+						if (compatResult.getConvertDeserializer() == null) {
+							// one of the field serializers cannot provide a fallback deserializer
+							return CompatibilityResult.requiresMigration(null);
+						} else {
+							convertDeserializers[i] = compatResult.getConvertDeserializer();
+						}
+					}
+				}
+
+				if (requireMigration) {
+					return CompatibilityResult.requiresMigration(new RowSerializer(convertDeserializers));
+				} else {
+					return CompatibilityResult.compatible();
+				}
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	public static final class RowSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public RowSerializerConfigSnapshot() {}
+
+		public RowSerializerConfigSnapshot(TypeSerializerConfigSnapshot[] fieldSerializerConfigSnapshots) {
+			super(fieldSerializerConfigSnapshots);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
index afc4aa24aa1c8..68d5aa8f73f3b 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
@@ -19,7 +19,10 @@
 package org.apache.flink.api.java.typeutils.runtime;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -115,4 +118,43 @@ public boolean equals(Object obj) {
 	public boolean canEqual(Object obj) {
 		return obj instanceof TupleSerializerBase;
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public TupleSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return new TupleSerializerConfigSnapshot<>(
+				tupleClass,
+				TypeSerializerUtil.snapshotConfigurations(fieldSerializers));
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof TupleSerializerConfigSnapshot) {
+			final TupleSerializerConfigSnapshot<T> config = (TupleSerializerConfigSnapshot<T>) configSnapshot;
+
+			if (tupleClass.equals(config.getTupleClass())) {
+				TypeSerializerConfigSnapshot[] fieldSerializerConfigSnapshots =
+					((TupleSerializerConfigSnapshot) configSnapshot).getNestedSerializerConfigSnapshots();
+
+				if (fieldSerializerConfigSnapshots.length == fieldSerializers.length) {
+
+					CompatibilityResult compatResult;
+					for (int i = 0; i < fieldSerializers.length; i++) {
+						compatResult = fieldSerializers[i].ensureCompatibility(fieldSerializerConfigSnapshots[i]);
+						if (compatResult.requiresMigration()) {
+							return CompatibilityResult.requiresMigration(null);
+						}
+					}
+
+					return CompatibilityResult.compatible();
+				}
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java
new file mode 100644
index 0000000000000..6d2bb5f201915
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * Snapshot of a tuple serializer's configuration.
+ */
+@Internal
+public final class TupleSerializerConfigSnapshot<T> extends CompositeTypeSerializerConfigSnapshot {
+
+	private static final int VERSION = 1;
+
+	private Class<T> tupleClass;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	public TupleSerializerConfigSnapshot() {}
+
+	public TupleSerializerConfigSnapshot(
+			Class<T> tupleClass,
+			TypeSerializerConfigSnapshot[] fieldSerializerConfigSnapshots) {
+
+		super(fieldSerializerConfigSnapshots);
+
+		this.tupleClass = Preconditions.checkNotNull(tupleClass);
+	}
+
+	@Override
+	public void write(DataOutputView out) throws IOException {
+		super.write(out);
+
+		InstantiationUtil.serializeObject(new DataOutputViewStream(out), tupleClass);
+	}
+
+	@Override
+	public void read(DataInputView in) throws IOException {
+		super.read(in);
+
+		try {
+			tupleClass = InstantiationUtil.deserializeObject(new DataInputViewStream(in), getUserCodeClassLoader());
+		} catch (ClassNotFoundException e) {
+			throw new IOException("Could not find requested tuple class in classpath.", e);
+		}
+	}
+
+	@Override
+	public int getVersion() {
+		return VERSION;
+	}
+
+	public Class<T> getTupleClass() {
+		return tupleClass;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		return super.equals(obj)
+			&& (obj instanceof TupleSerializerConfigSnapshot)
+			&& (tupleClass.equals(((TupleSerializerConfigSnapshot) obj).getTupleClass()));
+	}
+
+	@Override
+	public int hashCode() {
+		return super.hashCode() * 31 + tupleClass.hashCode();
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
index 56e204c9cecb8..10e233037b61e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
@@ -19,15 +19,20 @@
 package org.apache.flink.api.java.typeutils.runtime;
 
 import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.LinkedHashMap;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.Value;
 import org.apache.flink.util.InstantiationUtil;
 
 import com.esotericsoftware.kryo.Kryo;
+import org.apache.flink.util.Preconditions;
 import org.objenesis.strategy.StdInstantiatorStrategy;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -39,12 +44,23 @@
  * @param <T> The type serialized.
  */
 @Internal
-public class ValueSerializer<T extends Value> extends TypeSerializer<T> {
+public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 
 	private static final long serialVersionUID = 1L;
 	
 	private final Class<T> type;
-	
+
+	/**
+	 * Map of class tag (using classname as tag) to their Kryo registration.
+	 *
+	 * <p>This map serves as a preview of the final registration result of
+	 * the Kryo instance, taking into account registration overwrites.
+	 *
+	 * <p>Currently, we only have one single registration for the value type.
+	 * Nevertheless, we keep this information here for future compatibility.
+	 */
+	private LinkedHashMap<String, KryoRegistration> kryoRegistrations;
+
 	private transient Kryo kryo;
 	
 	private transient T copyInstance;
@@ -53,6 +69,7 @@ public class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 	
 	public ValueSerializer(Class<T> type) {
 		this.type = checkNotNull(type);
+		this.kryoRegistrations = asKryoRegistrations(type);
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -126,7 +143,8 @@ private void checkKryoInitialized() {
 			kryo.setInstantiatorStrategy(instantiatorStrategy);
 
 			this.kryo.setAsmEnabled(true);
-			this.kryo.register(type);
+
+			KryoUtils.applyRegistrations(this.kryo, kryoRegistrations.values());
 		}
 	}
 	
@@ -152,4 +170,66 @@ public boolean equals(Object obj) {
 	public boolean canEqual(Object obj) {
 		return obj instanceof ValueSerializer;
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public ValueSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return new ValueSerializerConfigSnapshot<>(type);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof ValueSerializerConfigSnapshot) {
+			final ValueSerializerConfigSnapshot<T> config = (ValueSerializerConfigSnapshot<T>) configSnapshot;
+
+			if (type.equals(config.getTypeClass())) {
+				// currently, simply checking the type of the value class is sufficient;
+				// in the future, if there are more Kryo registrations, we should try to resolve that
+				return CompatibilityResult.compatible();
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	public static class ValueSerializerConfigSnapshot<T extends Value> extends KryoRegistrationSerializerConfigSnapshot<T> {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public ValueSerializerConfigSnapshot() {}
+
+		public ValueSerializerConfigSnapshot(Class<T> valueTypeClass) {
+			super(valueTypeClass, asKryoRegistrations(valueTypeClass));
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+
+	private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+		in.defaultReadObject();
+
+		// kryoRegistrations may be null if this value serializer is deserialized from an old version
+		if (kryoRegistrations == null) {
+			this.kryoRegistrations = asKryoRegistrations(type);
+		}
+	}
+
+	private static LinkedHashMap<String, KryoRegistration> asKryoRegistrations(Class<?> type) {
+		Preconditions.checkNotNull(type);
+
+		LinkedHashMap<String, KryoRegistration> registration = new LinkedHashMap<>(1);
+		registration.put(type.getClass().getName(), new KryoRegistration(type));
+
+		return registration;
+	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
index cba0c8415f08e..a172b722f2a6b 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
@@ -21,18 +21,22 @@
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.KryoException;
 import com.esotericsoftware.kryo.Serializer;
-import com.esotericsoftware.kryo.factories.ReflectionSerializerFactory;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 
 import org.apache.avro.generic.GenericData;
 
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
 import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.KryoRegistration;
+import org.apache.flink.api.java.typeutils.runtime.KryoRegistrationSerializerConfigSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.KryoUtils;
 import org.apache.flink.api.java.typeutils.runtime.NoFetchingInput;
-import org.apache.flink.api.java.typeutils.runtime.kryo.Serializers.SpecificInstanceCollectionSerializerForArrayList;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -45,6 +49,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
 import java.io.IOException;
+import java.io.ObjectInputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
@@ -72,11 +77,16 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
 
 	// ------------------------------------------------------------------------
 
-	private final LinkedHashMap<Class<?>, ExecutionConfig.SerializableSerializer<?>> registeredTypesWithSerializers;
-	private final LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> registeredTypesWithSerializerClasses;
 	private final LinkedHashMap<Class<?>, ExecutionConfig.SerializableSerializer<?>> defaultSerializers;
 	private final LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> defaultSerializerClasses;
-	private final LinkedHashSet<Class<?>> registeredTypes;
+
+	/**
+	 * Map of class tag (using classname as tag) to their Kryo registration.
+	 *
+	 * <p>This map serves as a preview of the final registration result of
+	 * the Kryo instance, taking into account registration overwrites.
+	 */
+	private LinkedHashMap<String, KryoRegistration> kryoRegistrations;
 
 	private final Class<T> type;
 	
@@ -92,6 +102,13 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
 	private transient Input input;
 	private transient Output output;
 
+	// ------------------------------------------------------------------------
+	// legacy fields; these fields cannot yet be removed to retain backwards compatibility
+
+	private LinkedHashMap<Class<?>, ExecutionConfig.SerializableSerializer<?>> registeredTypesWithSerializers;
+	private LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> registeredTypesWithSerializerClasses;
+	private LinkedHashSet<Class<?>> registeredTypes;
+
 	// ------------------------------------------------------------------------
 
 	public KryoSerializer(Class<T> type, ExecutionConfig executionConfig){
@@ -99,20 +116,22 @@ public KryoSerializer(Class<T> type, ExecutionConfig executionConfig){
 
 		this.defaultSerializers = executionConfig.getDefaultKryoSerializers();
 		this.defaultSerializerClasses = executionConfig.getDefaultKryoSerializerClasses();
-		this.registeredTypesWithSerializers = executionConfig.getRegisteredTypesWithKryoSerializers();
-		this.registeredTypesWithSerializerClasses = executionConfig.getRegisteredTypesWithKryoSerializerClasses();
-		this.registeredTypes = executionConfig.getRegisteredKryoTypes();
+
+		this.kryoRegistrations = buildKryoRegistrations(
+				this.type,
+				executionConfig.getRegisteredKryoTypes(),
+				executionConfig.getRegisteredTypesWithKryoSerializerClasses(),
+				executionConfig.getRegisteredTypesWithKryoSerializers());
 	}
 
 	/**
 	 * Copy-constructor that does not copy transient fields. They will be initialized once required.
 	 */
 	protected KryoSerializer(KryoSerializer<T> toCopy) {
-		registeredTypesWithSerializers = toCopy.registeredTypesWithSerializers;
-		registeredTypesWithSerializerClasses = toCopy.registeredTypesWithSerializerClasses;
 		defaultSerializers = toCopy.defaultSerializers;
 		defaultSerializerClasses = toCopy.defaultSerializerClasses;
-		registeredTypes = toCopy.registeredTypes;
+
+		kryoRegistrations = toCopy.kryoRegistrations;
 
 		type = toCopy.type;
 		if(type == null){
@@ -272,11 +291,9 @@ public boolean equals(Object obj) {
 		if (obj instanceof KryoSerializer) {
 			KryoSerializer<?> other = (KryoSerializer<?>) obj;
 
-			// we cannot include the Serializers here because they don't implement the equals method
 			return other.canEqual(this) &&
 				type == other.type &&
-				registeredTypes.equals(other.registeredTypes) &&
-				registeredTypesWithSerializerClasses.equals(other.registeredTypesWithSerializerClasses) &&
+				kryoRegistrations.equals(other.kryoRegistrations) &&
 				defaultSerializerClasses.equals(other.defaultSerializerClasses);
 		} else {
 			return false;
@@ -334,7 +351,7 @@ private void checkKryoInitialized() {
 			//       This is due to a know issue with Kryo's JavaSerializer. See FLINK-6025 for details.
 			kryo.addDefaultSerializer(Throwable.class, new JavaSerializer());
 
-			// Add default serializers first, so that they type registrations without a serializer
+			// Add default serializers first, so that the type registrations without a serializer
 			// are registered with a default serializer
 			for (Map.Entry<Class<?>, ExecutionConfig.SerializableSerializer<?>> entry: defaultSerializers.entrySet()) {
 				kryo.addDefaultSerializer(entry.getKey(), entry.getValue().getSerializer());
@@ -344,59 +361,152 @@ private void checkKryoInitialized() {
 				kryo.addDefaultSerializer(entry.getKey(), entry.getValue());
 			}
 
-			// register the type of our class
-			kryo.register(type);
+			KryoUtils.applyRegistrations(this.kryo, kryoRegistrations.values());
 
-			// register given types. we do this first so that any registration of a
-			// more specific serializer overrides this
-			for (Class<?> type : registeredTypes) {
-				kryo.register(type);
-			}
+			kryo.setRegistrationRequired(false);
+			kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
+		}
+	}
 
-			// register given serializer classes
-			for (Map.Entry<Class<?>, Class<? extends Serializer<?>>> e : registeredTypesWithSerializerClasses.entrySet()) {
-				Class<?> typeClass = e.getKey();
-				Class<? extends Serializer<?>> serializerClass = e.getValue();
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
 
-				Serializer<?> serializer =
-						ReflectionSerializerFactory.makeSerializer(kryo, serializerClass, typeClass);
-				kryo.register(typeClass, serializer);
-			}
+	@Override
+	public KryoSerializerConfigSnapshot<T> snapshotConfiguration() {
+		return new KryoSerializerConfigSnapshot<>(type, kryoRegistrations);
+	}
 
-			// register given serializers
-			for (Map.Entry<Class<?>, ExecutionConfig.SerializableSerializer<?>> e : registeredTypesWithSerializers.entrySet()) {
-				kryo.register(e.getKey(), e.getValue().getSerializer());
+	@SuppressWarnings("unchecked")
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof KryoSerializerConfigSnapshot) {
+			final KryoSerializerConfigSnapshot<T> config = (KryoSerializerConfigSnapshot<T>) configSnapshot;
+
+			if (type.equals(config.getTypeClass())) {
+				LinkedHashMap<String, KryoRegistration> reconfiguredRegistrations = config.getKryoRegistrations();
+
+				// reconfigure by assuring that classes which were previously registered are registered
+				// again in the exact same order; new class registrations will be appended.
+				// this also overwrites any dummy placeholders that the restored old configuration has
+				reconfiguredRegistrations.putAll(kryoRegistrations);
+
+				// check if there is still any dummy placeholders even after reconfiguration;
+				// if so, then this new Kryo serializer cannot read old data and is therefore incompatible
+				for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : reconfiguredRegistrations.entrySet()) {
+					if (reconfiguredRegistrationEntry.getValue().isDummy()) {
+						LOG.warn("The Kryo registration for a previously registered class {} does not have a " +
+							"proper serializer, because its previous serializer cannot be loaded or is no " +
+							"longer valid but a new serializer is not available", reconfiguredRegistrationEntry.getKey());
+
+						return CompatibilityResult.requiresMigration(null);
+					}
+				}
+
+				// there's actually no way to tell if new Kryo serializers are compatible with
+				// the previous ones they overwrite; we can only signal compatibly and hope for the best
+				this.kryoRegistrations = reconfiguredRegistrations;
+				return CompatibilityResult.compatible();
 			}
-			// this is needed for Avro but can not be added on demand.
-			kryo.register(GenericData.Array.class, new SpecificInstanceCollectionSerializerForArrayList());
+		}
 
-			kryo.setRegistrationRequired(false);
-			kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	public static final class KryoSerializerConfigSnapshot<T> extends KryoRegistrationSerializerConfigSnapshot<T> {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public KryoSerializerConfigSnapshot() {}
+
+		public KryoSerializerConfigSnapshot(
+				Class<T> typeClass,
+				LinkedHashMap<String, KryoRegistration> kryoRegistrations) {
+
+			super(typeClass, kryoRegistrations);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
 		}
 	}
 
 	// --------------------------------------------------------------------------------------------
-	// For testing
+	// Utilities
 	// --------------------------------------------------------------------------------------------
-	
-	public Kryo getKryo() {
-		checkKryoInitialized();
-		return this.kryo;
+
+	/**
+	 * Utility method that takes lists of registered types and their serializers, and resolve
+	 * them into a single list such that the result will resemble the final registration
+	 * result in Kryo.
+	 */
+	private static LinkedHashMap<String, KryoRegistration> buildKryoRegistrations(
+			Class<?> serializedType,
+			LinkedHashSet<Class<?>> registeredTypes,
+			LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> registeredTypesWithSerializerClasses,
+			LinkedHashMap<Class<?>, ExecutionConfig.SerializableSerializer<?>> registeredTypesWithSerializers) {
+
+		final LinkedHashMap<String, KryoRegistration> kryoRegistrations = new LinkedHashMap<>();
+
+		kryoRegistrations.put(serializedType.getName(), new KryoRegistration(serializedType));
+
+		for (Class<?> registeredType : checkNotNull(registeredTypes)) {
+			kryoRegistrations.put(registeredType.getName(), new KryoRegistration(registeredType));
+		}
+
+		for (Map.Entry<Class<?>, Class<? extends Serializer<?>>> registeredTypeWithSerializerClassEntry :
+				checkNotNull(registeredTypesWithSerializerClasses).entrySet()) {
+
+			kryoRegistrations.put(
+					registeredTypeWithSerializerClassEntry.getKey().getName(),
+					new KryoRegistration(
+							registeredTypeWithSerializerClassEntry.getKey(),
+							registeredTypeWithSerializerClassEntry.getValue()));
+		}
+
+		for (Map.Entry<Class<?>, ExecutionConfig.SerializableSerializer<?>> registeredTypeWithSerializerEntry :
+				checkNotNull(registeredTypesWithSerializers).entrySet()) {
+
+			kryoRegistrations.put(
+					registeredTypeWithSerializerEntry.getKey().getName(),
+					new KryoRegistration(
+							registeredTypeWithSerializerEntry.getKey(),
+							registeredTypeWithSerializerEntry.getValue()));
+		}
+
+		kryoRegistrations.put(
+				GenericData.Array.class.getName(),
+				new KryoRegistration(
+						GenericData.Array.class,
+						new ExecutionConfig.SerializableSerializer<>(new Serializers.SpecificInstanceCollectionSerializerForArrayList())));
+
+		return kryoRegistrations;
 	}
 
-	@Override
-	public boolean canRestoreFrom(TypeSerializer<?> other) {
-		if (other instanceof KryoSerializer) {
-			KryoSerializer<?> otherKryo = (KryoSerializer<?>) other;
+	// --------------------------------------------------------------------------------------------
 
-			// we cannot include the Serializers here because they don't implement the equals method
-			return other.canEqual(this) &&
-					type == otherKryo.type &&
-					(registeredTypes.equals(otherKryo.registeredTypes) || otherKryo.registeredTypes.isEmpty()) &&
-					(registeredTypesWithSerializerClasses.equals(otherKryo.registeredTypesWithSerializerClasses) || otherKryo.registeredTypesWithSerializerClasses.isEmpty()) &&
-					(defaultSerializerClasses.equals(otherKryo.defaultSerializerClasses) || otherKryo.defaultSerializerClasses.isEmpty());
-		} else {
-			return false;
+	private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+		in.defaultReadObject();
+
+		// kryoRegistrations may be null if this Kryo serializer is deserialized from an old version
+		if (kryoRegistrations == null) {
+			this.kryoRegistrations = buildKryoRegistrations(
+					type,
+					registeredTypes,
+					registeredTypesWithSerializerClasses,
+					registeredTypesWithSerializers);
 		}
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// For testing
+	// --------------------------------------------------------------------------------------------
+
+	@VisibleForTesting
+	public Kryo getKryo() {
+		checkKryoInitialized();
+		return this.kryo;
+	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
index 91c6145d45d61..a84670349cb38 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
@@ -20,6 +20,7 @@
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -31,6 +32,9 @@
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.TestLogger;
 import org.junit.Assert;
 
@@ -85,6 +89,38 @@ public void testInstantiate() {
 			fail("Exception in test: " + e.getMessage());
 		}
 	}
+
+	@Test
+	public void testConfigSnapshotInstantiation() {
+		TypeSerializerConfigSnapshot configSnapshot = getSerializer().snapshotConfiguration();
+
+		InstantiationUtil.instantiate(configSnapshot.getClass());
+	}
+
+	@Test
+	public void testSnapshotConfigurationAndReconfigure() throws Exception {
+		final TypeSerializerConfigSnapshot configSnapshot = getSerializer().snapshotConfiguration();
+
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(
+				new DataOutputViewStreamWrapper(out), configSnapshot);
+			serializedConfig = out.toByteArray();
+		}
+
+		TypeSerializerConfigSnapshot restoredConfig;
+		try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			restoredConfig = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		CompatibilityResult strategy = getSerializer().ensureCompatibility(restoredConfig);
+		assertFalse(strategy.requiresMigration());
+
+		// also verify that the serializer's reconfigure implementation detects incompatibility
+		strategy = getSerializer().ensureCompatibility(new TestIncompatibleSerializerConfigSnapshot());
+		assertTrue(strategy.requiresMigration());
+	}
 	
 	@Test
 	public void testGetLength() {
@@ -477,4 +513,21 @@ public void skipBytesToRead(int numBytes) throws IOException {
 			}
 		}
 	}
+
+	public static final class TestIncompatibleSerializerConfigSnapshot extends TypeSerializerConfigSnapshot {
+		@Override
+		public int getVersion() {
+			return 0;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return obj instanceof TestIncompatibleSerializerConfigSnapshot;
+		}
+
+		@Override
+		public int hashCode() {
+			return getClass().hashCode();
+		}
+	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotTest.java
new file mode 100644
index 0000000000000..0783bb604e401
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Unit tests related to {@link TypeSerializerConfigSnapshot}.
+ */
+public class TypeSerializerConfigSnapshotTest {
+
+	/**
+	 * Verifies that reading and writing configuration snapshots work correctly.
+	 */
+	@Test
+	public void testSerializeConfigurationSnapshots() throws Exception {
+		TestConfigSnapshot configSnapshot1 = new TestConfigSnapshot(1, "foo");
+		TestConfigSnapshot configSnapshot2 = new TestConfigSnapshot(2, "bar");
+
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshots(
+				new DataOutputViewStreamWrapper(out),
+				configSnapshot1,
+				configSnapshot2);
+
+			serializedConfig = out.toByteArray();
+		}
+
+		TypeSerializerConfigSnapshot[] restoredConfigs;
+		try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			restoredConfigs = TypeSerializerUtil.readSerializerConfigSnapshots(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		assertEquals(2, restoredConfigs.length);
+		assertEquals(configSnapshot1, restoredConfigs[0]);
+		assertEquals(configSnapshot2, restoredConfigs[1]);
+	}
+
+	/**
+	 * Verifies that deserializing config snapshots fail if the config class could not be found.
+	 */
+	@Test
+	public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(
+				new DataOutputViewStreamWrapper(out), new TestConfigSnapshot(123, "foobar"));
+			serializedConfig = out.toByteArray();
+		}
+
+		try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			// read using a dummy classloader
+			TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null));
+			fail("Expected a ClassNotFoundException wrapped in IOException");
+		} catch (IOException expected) {
+			// test passes
+		}
+	}
+
+	public static class TestConfigSnapshot extends TypeSerializerConfigSnapshot {
+
+		static final int VERSION = 1;
+
+		private int val;
+		private String msg;
+
+		public TestConfigSnapshot() {}
+
+		public TestConfigSnapshot(int val, String msg) {
+			this.val = val;
+			this.msg = msg;
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+			super.write(out);
+			out.writeInt(val);
+			out.writeUTF(msg);
+		}
+
+		@Override
+		public void read(DataInputView in) throws IOException {
+			super.read(in);
+			val = in.readInt();
+			msg = in.readUTF();
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (obj == this) {
+				return true;
+			}
+
+			if (obj == null) {
+				return false;
+			}
+
+			if (obj instanceof TestConfigSnapshot) {
+				return val == ((TestConfigSnapshot) obj).val && msg.equals(((TestConfigSnapshot) obj).msg);
+			} else {
+				return false;
+			}
+		}
+
+		@Override
+		public int hashCode() {
+			return 31 * val + msg.hashCode();
+		}
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java
index 5e2e733a8c2e1..5c615de87c8e9 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java
@@ -18,11 +18,25 @@
 
 package org.apache.flink.api.common.typeutils.base;
 
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.SerializerTestInstance;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.TestLogger;
 
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
 public class EnumSerializerTest extends TestLogger {
 
 	@Test
@@ -41,6 +55,132 @@ public void testEmptyEnum() {
 		new EnumSerializer<>(EmptyEnum.class);
 	}
 
+	@Test
+	public void testReconfiguration() {
+		// mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL
+		PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL};
+
+		// now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order"
+		EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class);
+
+		// verify that the serializer is first using the "wrong order" (i.e., the initial new configuration)
+		assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
+		assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
+		assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
+		assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
+		assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
+		assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
+
+		// reconfigure and verify compatibility
+		CompatibilityResult<PublicEnum> compatResult = serializer.ensureCompatibility(
+			new EnumSerializer.EnumSerializerConfigSnapshot<>(PublicEnum.class, mockPreviousOrder));
+		assertFalse(compatResult.requiresMigration());
+
+		// after reconfiguration, the order should be first the original BAR, PAULA, NATHANIEL,
+		// followed by the "new enum constants" FOO, PETER, EMMA
+		PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA};
+
+		int i = 0;
+		for (PublicEnum constant : expectedOrder) {
+			assertEquals(i, serializer.getValueToOrdinal().get(constant).intValue());
+			i++;
+		}
+
+		assertTrue(Arrays.equals(expectedOrder, serializer.getValues()));
+	}
+
+	@Test
+	public void testConfigurationSnapshotSerialization() throws Exception {
+		EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class);
+
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(
+				new DataOutputViewStreamWrapper(out), serializer.snapshotConfiguration());
+			serializedConfig = out.toByteArray();
+		}
+
+		TypeSerializerConfigSnapshot restoredConfig;
+		try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			restoredConfig = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		CompatibilityResult<PublicEnum> compatResult = serializer.ensureCompatibility(restoredConfig);
+		assertFalse(compatResult.requiresMigration());
+
+		assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
+		assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
+		assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
+		assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
+		assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
+		assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
+		assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues()));
+	}
+
+	@Test
+	public void testSerializeEnumSerializer() throws Exception {
+		EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class);
+
+		// verify original transient parameters
+		assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
+		assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
+		assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
+		assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
+		assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
+		assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
+		assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues()));
+
+		byte[] serializedSerializer = InstantiationUtil.serializeObject(serializer);
+
+		// deserialize and re-verify transient parameters
+		serializer = InstantiationUtil.deserializeObject(serializedSerializer, Thread.currentThread().getContextClassLoader());
+		assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
+		assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
+		assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
+		assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
+		assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
+		assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
+		assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues()));
+	}
+
+	@Test
+	public void testSerializeReconfiguredEnumSerializer() throws Exception {
+		// mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL
+		PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL};
+
+		// now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order"
+		EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class);
+
+		// verify that the serializer is first using the "wrong order" (i.e., the initial new configuration)
+		assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
+		assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
+		assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
+		assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
+		assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
+		assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
+
+		// reconfigure and verify compatibility
+		CompatibilityResult<PublicEnum> compatResult = serializer.ensureCompatibility(
+			new EnumSerializer.EnumSerializerConfigSnapshot<>(PublicEnum.class, mockPreviousOrder));
+		assertFalse(compatResult.requiresMigration());
+
+		// serialize and deserialize again the serializer
+		byte[] serializedSerializer = InstantiationUtil.serializeObject(serializer);
+		serializer = InstantiationUtil.deserializeObject(serializedSerializer, Thread.currentThread().getContextClassLoader());
+
+		// verify that after the serializer was read, the reconfigured constant ordering is untouched
+		PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA};
+
+		int i = 0;
+		for (PublicEnum constant : expectedOrder) {
+			assertEquals(i, serializer.getValueToOrdinal().get(constant).intValue());
+			i++;
+		}
+
+		assertTrue(Arrays.equals(expectedOrder, serializer.getValues()));
+	}
+
 	@SafeVarargs
 	public final <T extends Enum<T>> void testEnumSerializer(T... data) {
 		@SuppressWarnings("unchecked")
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializerTest.java
index 9fda3d096ec4a..3301aa2071fc1 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/array/CharPrimitiveArraySerializerTest.java
@@ -20,8 +20,6 @@
 
 import org.apache.flink.api.common.typeutils.SerializerTestBase;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.base.array.CharPrimitiveArraySerializer;
-import org.apache.flink.api.common.typeutils.base.array.LongPrimitiveArraySerializer;
 
 /**
  * A test for the {@link LongPrimitiveArraySerializer}.
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
index 4712ed1ac04dd..5459d538d7053 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
@@ -18,29 +18,43 @@
 
 package org.apache.flink.api.java.typeutils.runtime;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Objects;
 import java.util.Random;
 
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.SerializerTestBase;
 import org.apache.flink.api.common.typeutils.TypeComparator;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
 import org.apache.flink.api.common.operators.Keys.ExpressionKeys;
 import org.apache.flink.api.common.operators.Keys.IncompatibleKeysException;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
 import org.apache.flink.api.java.tuple.Tuple1;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.typeutils.PojoTypeInfo;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
 /**
  * A test for the {@link PojoSerializer}.
  */
@@ -191,6 +205,20 @@ public boolean equals(Object other) {
 			return true;
 		}
 	}
+
+	public static class SubTestUserClassA extends TestUserClass {
+		public int subDumm1;
+		public String subDumm2;
+
+		public SubTestUserClassA() {}
+	}
+
+	public static class SubTestUserClassB extends TestUserClass {
+		public Double subDumm1;
+		public float subDumm2;
+
+		public SubTestUserClassB() {}
+	}
 	
 	/**
 	 * This tests if the hashes returned by the pojo and tuple comparators are the same
@@ -240,4 +268,244 @@ public void testTuplePojoTestEquality() {
 		Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable. Also for those with multiple key fields", multiPojoHash == multiTupleHash);
 		
 	}
-}	
+
+	// --------------------------------------------------------------------------------------------
+	// Configuration snapshotting & reconfiguring tests
+	// --------------------------------------------------------------------------------------------
+
+	/**
+	 * Verifies that reconfiguring with a config snapshot of a preceding POJO serializer
+	 * with different POJO type will result in INCOMPATIBLE.
+	 */
+	@Test
+	public void testReconfigureWithDifferentPojoType() throws Exception {
+		PojoSerializer<SubTestUserClassB> pojoSerializer1 = (PojoSerializer<SubTestUserClassB>)
+			TypeExtractor.getForClass(SubTestUserClassB.class).createSerializer(new ExecutionConfig());
+
+		// snapshot configuration and serialize to bytes
+		TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer1.snapshotConfiguration();
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot);
+			serializedConfig = out.toByteArray();
+		}
+
+		PojoSerializer<SubTestUserClassA> pojoSerializer2 = (PojoSerializer<SubTestUserClassA>)
+			TypeExtractor.getForClass(SubTestUserClassA.class).createSerializer(new ExecutionConfig());
+
+		// read configuration again from bytes
+		try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			pojoSerializerConfigSnapshot = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		CompatibilityResult<SubTestUserClassA> compatResult = pojoSerializer2.ensureCompatibility(pojoSerializerConfigSnapshot);
+		assertTrue(compatResult.requiresMigration());
+	}
+
+	/**
+	 * Tests that reconfiguration correctly reorders subclass registrations to their previous order.
+	 */
+	@Test
+	public void testReconfigureDifferentSubclassRegistrationOrder() throws Exception {
+		ExecutionConfig executionConfig = new ExecutionConfig();
+		executionConfig.registerPojoType(SubTestUserClassA.class);
+		executionConfig.registerPojoType(SubTestUserClassB.class);
+
+		PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(executionConfig);
+
+		// get original registration ids
+		int subClassATag = pojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class);
+		int subClassBTag = pojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class);
+
+		// snapshot configuration and serialize to bytes
+		TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot);
+			serializedConfig = out.toByteArray();
+		}
+
+		// use new config and instantiate new PojoSerializer
+		executionConfig = new ExecutionConfig();
+		executionConfig.registerPojoType(SubTestUserClassB.class); // test with B registered before A
+		executionConfig.registerPojoType(SubTestUserClassA.class);
+
+		pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(executionConfig);
+
+		// read configuration from bytes
+		try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			pojoSerializerConfigSnapshot = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot);
+		assertTrue(!compatResult.requiresMigration());
+
+		// reconfigure - check reconfiguration result and that registration ids remains the same
+		//assertEquals(ReconfigureResult.COMPATIBLE, pojoSerializer.reconfigure(pojoSerializerConfigSnapshot));
+		assertEquals(subClassATag, pojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class).intValue());
+		assertEquals(subClassBTag, pojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class).intValue());
+	}
+
+	/**
+	 * Tests that reconfiguration repopulates previously cached subclass serializers.
+	 */
+	@Test
+	public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() throws Exception {
+		// don't register any subclasses
+		PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
+
+		// create cached serializers for SubTestUserClassA and SubTestUserClassB
+		pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
+		pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);
+
+		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
+
+		// snapshot configuration and serialize to bytes
+		TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot);
+			serializedConfig = out.toByteArray();
+		}
+
+		// instantiate new PojoSerializer
+
+		pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
+
+		// read configuration from bytes
+		try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			pojoSerializerConfigSnapshot = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		// reconfigure - check reconfiguration result and that subclass serializer cache is repopulated
+		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot);
+		assertFalse(compatResult.requiresMigration());
+		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
+	}
+
+	/**
+	 * Tests that:
+	 *  - Previous Pojo serializer did not have registrations, and created cached serializers for subclasses
+	 *  - On restore, it had those subclasses registered
+	 *
+	 * In this case, after reconfiguration, the cache should be repopulated, and registrations should
+	 * also exist for the subclasses.
+	 *
+	 * Note: the cache still needs to be repopulated because previous data of those subclasses were
+	 * written with the cached serializers. In this case, the repopulated cache has reconfigured serializers
+	 * for the subclasses so that previous written data can be read, but the registered serializers
+	 * for the subclasses do not necessarily need to be reconfigured since they will only be used to
+	 * write new data.
+	 */
+	@Test
+	public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception {
+		// don't register any subclasses at first
+		PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
+
+		// create cached serializers for SubTestUserClassA and SubTestUserClassB
+		pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
+		pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);
+
+		// make sure serializers are in cache
+		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
+
+		// make sure that registrations are empty
+		assertTrue(pojoSerializer.getRegisteredClasses().isEmpty());
+		assertEquals(0, pojoSerializer.getRegisteredSerializers().length);
+
+		// snapshot configuration and serialize to bytes
+		TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot);
+			serializedConfig = out.toByteArray();
+		}
+
+		// instantiate new PojoSerializer, with new execution config that has the subclass registrations
+		ExecutionConfig newExecutionConfig = new ExecutionConfig();
+		newExecutionConfig.registerPojoType(SubTestUserClassA.class);
+		newExecutionConfig.registerPojoType(SubTestUserClassB.class);
+		pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig);
+
+		// read configuration from bytes
+		try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			pojoSerializerConfigSnapshot = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		// reconfigure - check reconfiguration result and that
+		// 1) subclass serializer cache is repopulated
+		// 2) registrations also contain the now registered subclasses
+		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot);
+		assertFalse(compatResult.requiresMigration());
+		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
+		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
+		assertEquals(2, pojoSerializer.getRegisteredClasses().size());
+		assertTrue(pojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
+		assertTrue(pojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
+	}
+
+	/**
+	 * Verifies that reconfiguration reorders the fields of the new Pojo serializer to remain the same.
+	 */
+	@Test
+	public void testReconfigureWithDifferentFieldOrder() throws Exception {
+		Field[] mockOriginalFieldOrder = {
+			TestUserClass.class.getField("dumm4"),
+			TestUserClass.class.getField("dumm3"),
+			TestUserClass.class.getField("nestedClass"),
+			TestUserClass.class.getField("dumm1"),
+			TestUserClass.class.getField("dumm2"),
+			TestUserClass.class.getField("dumm5"),
+		};
+
+		// creating this serializer just for generating config snapshots of the field serializers
+		PojoSerializer<TestUserClass> ser = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
+
+		LinkedHashMap<Field, TypeSerializerConfigSnapshot> mockOriginalFieldToSerializerConfigSnapshot =
+			new LinkedHashMap<>(mockOriginalFieldOrder.length);
+		mockOriginalFieldToSerializerConfigSnapshot.put(mockOriginalFieldOrder[0], ser.getFieldSerializers()[3].snapshotConfiguration());
+		mockOriginalFieldToSerializerConfigSnapshot.put(mockOriginalFieldOrder[1], ser.getFieldSerializers()[2].snapshotConfiguration());
+		mockOriginalFieldToSerializerConfigSnapshot.put(mockOriginalFieldOrder[2], ser.getFieldSerializers()[5].snapshotConfiguration());
+		mockOriginalFieldToSerializerConfigSnapshot.put(mockOriginalFieldOrder[3], ser.getFieldSerializers()[0].snapshotConfiguration());
+		mockOriginalFieldToSerializerConfigSnapshot.put(mockOriginalFieldOrder[4], ser.getFieldSerializers()[1].snapshotConfiguration());
+		mockOriginalFieldToSerializerConfigSnapshot.put(mockOriginalFieldOrder[5], ser.getFieldSerializers()[4].snapshotConfiguration());
+
+		PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
+
+		assertEquals(TestUserClass.class.getField("dumm1"), pojoSerializer.getFields()[0]);
+		assertEquals(TestUserClass.class.getField("dumm2"), pojoSerializer.getFields()[1]);
+		assertEquals(TestUserClass.class.getField("dumm3"), pojoSerializer.getFields()[2]);
+		assertEquals(TestUserClass.class.getField("dumm4"), pojoSerializer.getFields()[3]);
+		assertEquals(TestUserClass.class.getField("dumm5"), pojoSerializer.getFields()[4]);
+		assertEquals(TestUserClass.class.getField("nestedClass"), pojoSerializer.getFields()[5]);
+
+		PojoSerializer.PojoSerializerConfigSnapshot<TestUserClass> mockPreviousConfigSnapshot =
+			new PojoSerializer.PojoSerializerConfigSnapshot<>(
+				TestUserClass.class,
+				mockOriginalFieldToSerializerConfigSnapshot, // this mocks the previous field order
+				new LinkedHashMap<Class<?>, TypeSerializerConfigSnapshot>(), // empty; irrelevant for this test
+				new HashMap<Class<?>, TypeSerializerConfigSnapshot>()); // empty; irrelevant for this test
+
+		// reconfigure - check reconfiguration result and that fields are reordered to the previous order
+		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(
+
+			mockPreviousConfigSnapshot);
+		assertFalse(compatResult.requiresMigration());
+		int i = 0;
+		for (Field field : mockOriginalFieldOrder) {
+			assertEquals(field, pojoSerializer.getFields()[i]);
+			i++;
+		}
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
new file mode 100644
index 0000000000000..60c4dc45cfe7d
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests related to configuration snapshotting and reconfiguring for the {@link KryoSerializer}.
+ */
+public class KryoSerializerCompatibilityTest {
+
+	/**
+	 * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed.
+	 */
+	@Test
+	public void testMigrationStrategyWithDifferentKryoType() throws Exception {
+		KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig());
+
+		// snapshot configuration and serialize to bytes
+		TypeSerializerConfigSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration();
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot);
+			serializedConfig = out.toByteArray();
+		}
+
+		KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig());
+
+		// read configuration again from bytes
+		try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			kryoSerializerConfigSnapshot = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		CompatibilityResult<TestClassB> compatResult = kryoSerializerForB.ensureCompatibility(kryoSerializerConfigSnapshot);
+		assertTrue(compatResult.requiresMigration());
+	}
+
+	/**
+	 * Tests that after reconfiguration, registration ids are reconfigured to
+	 * remain the same as the preceding KryoSerializer.
+	 */
+	@Test
+	public void testMigrationStrategyForDifferentRegistrationOrder() throws Exception {
+
+		ExecutionConfig executionConfig = new ExecutionConfig();
+		executionConfig.registerKryoType(TestClassA.class);
+		executionConfig.registerKryoType(TestClassB.class);
+
+		KryoSerializer<TestClass> kryoSerializer = new KryoSerializer<>(TestClass.class, executionConfig);
+
+		// get original registration ids
+		int testClassId = kryoSerializer.getKryo().getRegistration(TestClass.class).getId();
+		int testClassAId = kryoSerializer.getKryo().getRegistration(TestClassA.class).getId();
+		int testClassBId = kryoSerializer.getKryo().getRegistration(TestClassB.class).getId();
+
+		// snapshot configuration and serialize to bytes
+		TypeSerializerConfigSnapshot kryoSerializerConfigSnapshot = kryoSerializer.snapshotConfiguration();
+		byte[] serializedConfig;
+		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+			TypeSerializerUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot);
+			serializedConfig = out.toByteArray();
+		}
+
+		// use new config and instantiate new KryoSerializer
+		executionConfig = new ExecutionConfig();
+		executionConfig.registerKryoType(TestClassB.class); // test with B registered before A
+		executionConfig.registerKryoType(TestClassA.class);
+
+		kryoSerializer = new KryoSerializer<>(TestClass.class, executionConfig);
+
+		// read configuration from bytes
+		try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
+			kryoSerializerConfigSnapshot = TypeSerializerUtil.readSerializerConfigSnapshot(
+				new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
+		}
+
+		// reconfigure - check reconfiguration result and that registration id remains the same
+		CompatibilityResult<TestClass> compatResult = kryoSerializer.ensureCompatibility(kryoSerializerConfigSnapshot);
+		assertFalse(compatResult.requiresMigration());
+		assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId());
+		assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId());
+		assertEquals(testClassBId, kryoSerializer.getKryo().getRegistration(TestClassB.class).getId());
+	}
+
+	private static class TestClass {}
+
+	private static class TestClassA {}
+
+	private static class TestClassB {}
+
+	private static class TestClassBSerializer extends Serializer {
+		@Override
+		public void write(Kryo kryo, Output output, Object o) {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public Object read(Kryo kryo, Input input, Class aClass) {
+			throw new UnsupportedOperationException();
+		}
+	}
+}
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java
index 3bb40ebcb2f38..4dabaca3a14f3 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java
@@ -26,7 +26,9 @@
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.core.io.GenericInputSplit;
 import org.apache.flink.core.memory.DataInputView;
@@ -386,5 +388,15 @@ public boolean canEqual(Object obj) {
 		public int hashCode() {
 			return Objects.hash(failOnRead, failOnWrite);
 		}
+
+		@Override
+		public TypeSerializerConfigSnapshot snapshotConfiguration() {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public CompatibilityResult<ElementType> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+			throw new UnsupportedOperationException();
+		}
 	}
 }
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/NonDuplicatingTypeSerializer.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/NonDuplicatingTypeSerializer.java
index 846b6c3133927..9e22fc2706d43 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/NonDuplicatingTypeSerializer.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/NonDuplicatingTypeSerializer.java
@@ -18,7 +18,10 @@
 
 package org.apache.flink.cep;
 
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -39,7 +42,8 @@
  *
  * @param <T> Type of the element to be serialized
  */
-public class NonDuplicatingTypeSerializer<T> extends TypeSerializer<T> {
+@Internal
+public final class NonDuplicatingTypeSerializer<T> extends TypeSerializer<T> {
 	private static final long serialVersionUID = -7633631762221447524L;
 
 	// underlying type serializer
@@ -192,4 +196,14 @@ private void readObject(ObjectInputStream ois) throws IOException, ClassNotFound
 		this.identityMap = new IdentityHashMap<>();
 		this.elementList = new ArrayList<>();
 	}
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		throw new UnsupportedOperationException("This serializer is not registered for managed state.");
+	}
+
+	@Override
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		throw new UnsupportedOperationException("This serializer is not registered for managed state.");
+	}
 }
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java
index b8c4e65c223c2..70755e553f904 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java
@@ -21,6 +21,22 @@
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.LinkedHashMultimap;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
+import org.apache.flink.cep.NonDuplicatingTypeSerializer;
+import org.apache.flink.cep.nfa.compiler.NFACompiler;
+import org.apache.flink.cep.pattern.conditions.IterativeCondition;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -43,20 +59,6 @@
 import java.util.Stack;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import javax.annotation.Nullable;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
-import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
-import org.apache.flink.cep.NonDuplicatingTypeSerializer;
-import org.apache.flink.cep.nfa.compiler.NFACompiler;
-import org.apache.flink.cep.pattern.conditions.IterativeCondition;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.util.Preconditions;
 
 /**
  * Non-deterministic finite automaton implementation.
@@ -859,7 +861,7 @@ private ComputationState<T> readComputationState(ObjectInputStream ois) throws I
 	/**
 	 * A {@link TypeSerializer} for {@link NFA} that uses Java Serialization.
 	 */
-	public static class Serializer<T> extends TypeSerializer<NFA<T>> {
+	public static class Serializer<T> extends TypeSerializerSingleton<NFA<T>> {
 
 		private static final long serialVersionUID = 1L;
 
@@ -868,11 +870,6 @@ public boolean isImmutableType() {
 			return false;
 		}
 
-		@Override
-		public TypeSerializer<NFA<T>> duplicate() {
-			return this;
-		}
-
 		@Override
 		public NFA<T> createInstance() {
 			return null;
@@ -943,19 +940,9 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 			target.write(source, size);
 		}
 
-		@Override
-		public boolean equals(Object obj) {
-			return obj instanceof Serializer && ((Serializer) obj).canEqual(this);
-		}
-
 		@Override
 		public boolean canEqual(Object obj) {
 			return obj instanceof Serializer;
 		}
-
-		@Override
-		public int hashCode() {
-			return getClass().hashCode();
-		}
 	}
 }
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
index b6374cdc441cc..14235dc48ffa4 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
@@ -21,7 +21,10 @@
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.base.CollectionSerializerConfigSnapshot;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.cep.nfa.NFA;
 import org.apache.flink.cep.nfa.compiler.NFACompiler;
@@ -472,11 +475,6 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 
 		}
 
-		@Override
-		public boolean canRestoreFrom(TypeSerializer<?> other) {
-			return equals(other) || other instanceof AbstractKeyedCEPPatternOperator.PriorityQueueSerializer;
-		}
-
 		@Override
 		public boolean equals(Object obj) {
 			if (obj instanceof PriorityQueueSerializer) {
@@ -498,6 +496,32 @@ public boolean canEqual(Object obj) {
 		public int hashCode() {
 			return Objects.hash(factory, elementSerializer);
 		}
+
+		// --------------------------------------------------------------------------------------------
+		// Serializer configuration snapshotting & compatibility
+		// --------------------------------------------------------------------------------------------
+
+		@Override
+		public TypeSerializerConfigSnapshot snapshotConfiguration() {
+			return new CollectionSerializerConfigSnapshot(elementSerializer.snapshotConfiguration());
+		}
+
+		@Override
+		public CompatibilityResult<PriorityQueue<T>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+			if (configSnapshot instanceof CollectionSerializerConfigSnapshot) {
+				CompatibilityResult<T> compatResult = elementSerializer.ensureCompatibility(
+						((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
+
+				if (!compatResult.requiresMigration()) {
+					return CompatibilityResult.compatible();
+				} else if (compatResult.getConvertDeserializer() != null) {
+					return CompatibilityResult.requiresMigration(
+						new PriorityQueueSerializer<>(compatResult.getConvertDeserializer(), factory));
+				}
+			}
+
+			return CompatibilityResult.requiresMigration(null);
+		}
 	}
 
 	private interface PriorityQueueFactory<T> extends Serializable {
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializer.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializer.java
index b86fe87ed427f..59841226314aa 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializer.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArraySerializer.java
@@ -19,6 +19,7 @@
 package org.apache.flink.graph.types.valuearray;
 
 import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
+import org.apache.flink.api.common.typeutils.base.array.IntPrimitiveArraySerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -82,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof IntValueArraySerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(IntPrimitiveArraySerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializer.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializer.java
index 95219b6824b98..e95a1a721b78a 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializer.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/LongValueArraySerializer.java
@@ -19,6 +19,7 @@
 package org.apache.flink.graph.types.valuearray;
 
 import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
+import org.apache.flink.api.common.typeutils.base.array.LongPrimitiveArraySerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -82,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof LongValueArraySerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+				|| identifier.equals(LongPrimitiveArraySerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializer.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializer.java
index 0e875e35d73a6..6dbe0e5e1d72b 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializer.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/StringValueArraySerializer.java
@@ -19,6 +19,7 @@
 package org.apache.flink.graph.types.valuearray;
 
 import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
+import org.apache.flink.api.common.typeutils.base.array.StringArraySerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -82,4 +83,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof StringValueArraySerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(StringArraySerializer.class.getCanonicalName());
+	}
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
index 1f56a985966c6..7ffa57ce7c488 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
@@ -18,7 +18,7 @@
 
 package org.apache.flink.table.runtime.types
 
-import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.common.typeutils._
 import org.apache.flink.core.memory.{DataInputView, DataOutputView}
 import org.apache.flink.types.Row
 
@@ -75,4 +75,55 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali
   }
 
   override def hashCode: Int = rowSerializer.hashCode() * 13
+
+  // --------------------------------------------------------------------------------------------
+  // Serializer configuration snapshotting & compatibility
+  // --------------------------------------------------------------------------------------------
+
+  override def snapshotConfiguration(): TypeSerializerConfigSnapshot = {
+    new CRowSerializer.CRowSerializerConfigSnapshot(
+      rowSerializer.snapshotConfiguration())
+  }
+
+  override def ensureCompatibility(
+      configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[CRow] = {
+
+    configSnapshot match {
+      case crowSerializerConfigSnapshot: CRowSerializer.CRowSerializerConfigSnapshot =>
+        val compatResult = rowSerializer.ensureCompatibility(
+            crowSerializerConfigSnapshot.getSingleNestedSerializerConfigSnapshot)
+
+        if (compatResult.requiresMigration()) {
+          if (compatResult.getConvertDeserializer != null) {
+            CompatibilityResult.requiresMigration(
+              new CRowSerializer(compatResult.getConvertDeserializer)
+            )
+          } else {
+            CompatibilityResult.requiresMigration(null)
+          }
+        } else {
+          CompatibilityResult.compatible()
+        }
+
+      case _ => CompatibilityResult.requiresMigration(null)
+    }
+  }
+}
+
+object CRowSerializer {
+
+  class CRowSerializerConfigSnapshot(
+      private var rowSerializerConfigSnapshot: TypeSerializerConfigSnapshot)
+    extends CompositeTypeSerializerConfigSnapshot(rowSerializerConfigSnapshot) {
+
+    /** This empty nullary constructor is required for deserializing the configuration. */
+    def this() = this(null)
+
+    override def getVersion: Int = CRowSerializerConfigSnapshot.VERSION
+  }
+
+  object CRowSerializerConfigSnapshot {
+    val VERSION = 1
+  }
+
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java b/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java
index c6813b6863ebe..c4e23ca27ffe4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java
@@ -18,7 +18,10 @@
 
 package org.apache.flink.migration;
 
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.ParameterlessTypeSerializerConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -51,8 +54,7 @@ public boolean isImmutableType() {
 
 	@Override
 	public TypeSerializer<Serializable> duplicate() {
-		throw new UnsupportedOperationException(
-				"This is just a proxy used during migration until the real type serializer is provided by the user.");
+		return this;
 	}
 
 	@Override
@@ -102,6 +104,17 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 				"This is just a proxy used during migration until the real type serializer is provided by the user.");
 	}
 
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		return new ParameterlessTypeSerializerConfig(getClass().getCanonicalName());
+	}
+
+	@Override
+	public CompatibilityResult<Serializable> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		// always assume compatibility since we're just a proxy for migration
+		return CompatibilityResult.compatible();
+	}
+
 	@Override
 	public boolean equals(Object obj) {
 		return obj instanceof MigrationNamespaceSerializerProxy;
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java
index f47989a7bf04e..f58070e564b92 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java
@@ -23,7 +23,7 @@
 import org.apache.flink.api.common.typeutils.base.VoidSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.VoidNamespace;
 import org.apache.flink.runtime.state.VoidNamespaceSerializer;
 import org.apache.flink.runtime.state.heap.HeapKeyedStateBackend;
@@ -85,14 +85,14 @@ public StateTable<K, N, S> deserialize(String stateName, HeapKeyedStateBackend<K
 			patchedNamespaceSerializer = (TypeSerializer<N>) VoidNamespaceSerializer.INSTANCE;
 		}
 
-		RegisteredBackendStateMetaInfo<N, S> registeredBackendStateMetaInfo =
-				new RegisteredBackendStateMetaInfo<>(
+		RegisteredKeyedBackendStateMetaInfo<N, S> registeredKeyedBackendStateMetaInfo =
+				new RegisteredKeyedBackendStateMetaInfo<>(
 						StateDescriptor.Type.UNKNOWN,
 						stateName,
 						patchedNamespaceSerializer,
 						stateSerializer);
 
-		final StateTable<K, N, S> stateTable = stateBackend.newStateTable(registeredBackendStateMetaInfo);
+		final StateTable<K, N, S> stateTable = stateBackend.newStateTable(registeredKeyedBackendStateMetaInfo);
 		final DataInputView inView = openDataInputView();
 		final int keyGroup = keyGroupRange.getStartKeyGroup();
 		final int numNamespaces = inView.readInt();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
index 0badb415f97b9..8fbc227d1b169 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
@@ -17,7 +17,10 @@
  */
 package org.apache.flink.runtime.state;
 
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.base.CollectionSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -132,4 +135,30 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return elementSerializer.hashCode();
 	}
-}
\ No newline at end of file
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		return new CollectionSerializerConfigSnapshot(elementSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<ArrayList<T>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof CollectionSerializerConfigSnapshot) {
+			CompatibilityResult<T> compatResult = elementSerializer.ensureCompatibility(
+				((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
+
+			if (!compatResult.requiresMigration()) {
+				return CompatibilityResult.compatible();
+			} else if (compatResult.getConvertDeserializer() != null) {
+				return CompatibilityResult.requiresMigration(
+					new ArrayListSerializer<>(compatResult.getConvertDeserializer()));
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
index e7ed26f67fd11..ec4aa81dfd62d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
@@ -207,26 +207,18 @@ public OperatorStateHandle performOperation() throws Exception {
 					final Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData =
 						new HashMap<>(registeredStatesDeepCopies.size());
 
-					List<OperatorBackendSerializationProxy.StateMetaInfo<?>> metaInfoList =
+					List<RegisteredOperatorBackendStateMetaInfo.Snapshot<?>> metaInfoSnapshots =
 						new ArrayList<>(registeredStatesDeepCopies.size());
 
-					for (Map.Entry<String, PartitionableListState<?>> entry :
-						registeredStatesDeepCopies.entrySet()) {
-
-						PartitionableListState<?> state = entry.getValue();
-						OperatorBackendSerializationProxy.StateMetaInfo<?> metaInfo =
-							new OperatorBackendSerializationProxy.StateMetaInfo<>(
-								state.getName(),
-								state.getPartitionStateSerializer(),
-								state.getAssignmentMode());
-						metaInfoList.add(metaInfo);
+					for (Map.Entry<String, PartitionableListState<?>> entry : registeredStatesDeepCopies.entrySet()) {
+						metaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
 					}
 
 					CheckpointStreamFactory.CheckpointStateOutputStream out = getIoHandle();
 					DataOutputView dov = new DataOutputViewStreamWrapper(out);
 
 					OperatorBackendSerializationProxy backendSerializationProxy =
-						new OperatorBackendSerializationProxy(metaInfoList);
+						new OperatorBackendSerializationProxy(metaInfoSnapshots);
 
 					backendSerializationProxy.write(dov);
 
@@ -237,7 +229,7 @@ public OperatorStateHandle performOperation() throws Exception {
 
 						PartitionableListState<?> value = entry.getValue();
 						long[] partitionOffsets = value.write(out);
-						OperatorStateHandle.Mode mode = value.getAssignmentMode();
+						OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
 						writtenStatesMetaData.put(
 							entry.getKey(),
 							new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
@@ -254,10 +246,7 @@ public OperatorStateHandle performOperation() throws Exception {
 						return null;
 					}
 
-					OperatorStateHandle operatorStateHandle =
-						new OperatorStateHandle(writtenStatesMetaData, stateHandle);
-
-					return operatorStateHandle;
+					return new OperatorStateHandle(writtenStatesMetaData, stateHandle);
 				}
 			};
 
@@ -298,25 +287,23 @@ public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exc
 
 				backendSerializationProxy.read(new DataInputViewStreamWrapper(in));
 
-				List<OperatorBackendSerializationProxy.StateMetaInfo<?>> metaInfoList =
-						backendSerializationProxy.getNamedStateSerializationProxies();
+				List<RegisteredOperatorBackendStateMetaInfo.Snapshot<?>> restoredMetaInfoSnapshots =
+						backendSerializationProxy.getStateMetaInfoSnapshots();
 
 				// Recreate all PartitionableListStates from the meta info
-				for (OperatorBackendSerializationProxy.StateMetaInfo<?> stateMetaInfo : metaInfoList) {
-					PartitionableListState<?> listState = registeredStates.get(stateMetaInfo.getName());
+				for (RegisteredOperatorBackendStateMetaInfo.Snapshot<?> restoredMetaInfo : restoredMetaInfoSnapshots) {
+					PartitionableListState<?> listState = registeredStates.get(restoredMetaInfo.getName());
 
 					if (null == listState) {
 						listState = new PartitionableListState<>(
-								stateMetaInfo.getName(),
-								stateMetaInfo.getStateSerializer(),
-								stateMetaInfo.getMode());
+								new RegisteredOperatorBackendStateMetaInfo<>(
+										restoredMetaInfo.getName(),
+										restoredMetaInfo.getPartitionStateSerializer(),
+										restoredMetaInfo.getAssignmentMode()));
 
-						registeredStates.put(listState.getName(), listState);
+						registeredStates.put(listState.getStateMetaInfo().getName(), listState);
 					} else {
-						Preconditions.checkState(listState.getPartitionStateSerializer().canRestoreFrom(
-								stateMetaInfo.getStateSerializer()), "Incompatible state serializers found: " +
-								listState.getPartitionStateSerializer() + " is not compatible with " +
-								stateMetaInfo.getStateSerializer());
+						// TODO with eager state registration in place, check here for serializer migration strategies
 					}
 				}
 
@@ -341,7 +328,6 @@ public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exc
 	}
 
 	/**
-	 *
 	 * Implementation of operator list state.
 	 *
 	 * @param <S> the type of an operator state partition.
@@ -349,19 +335,9 @@ public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exc
 	static final class PartitionableListState<S> implements ListState<S> {
 
 		/**
-		 * The name of the state, as registered by the user
-		 */
-		private final String name;
-
-		/**
-		 * The type serializer for the elements in the state list
-		 */
-		private final TypeSerializer<S> partitionStateSerializer;
-
-		/**
-		 * The mode how elements in this state are assigned to tasks during restore
+		 * Meta information of the state, including state name, assignment mode, and serializer
 		 */
-		private final OperatorStateHandle.Mode assignmentMode;
+		private final RegisteredOperatorBackendStateMetaInfo<S> stateMetaInfo;
 
 		/**
 		 * The internal list the holds the elements of the state
@@ -373,46 +349,26 @@ static final class PartitionableListState<S> implements ListState<S> {
 		 */
 		private final ArrayListSerializer<S> internalListCopySerializer;
 
-		public PartitionableListState(
-				String name,
-				TypeSerializer<S> partitionStateSerializer,
-				OperatorStateHandle.Mode assignmentMode) {
-
-			this(name, partitionStateSerializer, assignmentMode, new ArrayList<S>());
+		public PartitionableListState(RegisteredOperatorBackendStateMetaInfo<S> stateMetaInfo) {
+			this(stateMetaInfo, new ArrayList<S>());
 		}
 
 		private PartitionableListState(
-				String name,
-				TypeSerializer<S> partitionStateSerializer,
-				OperatorStateHandle.Mode assignmentMode,
+				RegisteredOperatorBackendStateMetaInfo<S> stateMetaInfo,
 				ArrayList<S> internalList) {
 
-			this.name = Preconditions.checkNotNull(name);
-			this.partitionStateSerializer = Preconditions.checkNotNull(partitionStateSerializer);
-			this.assignmentMode = Preconditions.checkNotNull(assignmentMode);
+			this.stateMetaInfo = Preconditions.checkNotNull(stateMetaInfo);
 			this.internalList = Preconditions.checkNotNull(internalList);
-			this.internalListCopySerializer = new ArrayListSerializer<>(partitionStateSerializer);
+			this.internalListCopySerializer = new ArrayListSerializer<>(stateMetaInfo.getPartitionStateSerializer());
 		}
 
 		private PartitionableListState(PartitionableListState<S> toCopy) {
 
-			this(
-					toCopy.name,
-					toCopy.partitionStateSerializer.duplicate(),
-					toCopy.assignmentMode,
-					toCopy.internalListCopySerializer.copy(toCopy.internalList));
-		}
-
-		public String getName() {
-			return name;
-		}
-
-		public OperatorStateHandle.Mode getAssignmentMode() {
-			return assignmentMode;
+			this(toCopy.stateMetaInfo, toCopy.internalListCopySerializer.copy(toCopy.internalList));
 		}
 
-		public TypeSerializer<S> getPartitionStateSerializer() {
-			return partitionStateSerializer;
+		public RegisteredOperatorBackendStateMetaInfo<S> getStateMetaInfo() {
+			return stateMetaInfo;
 		}
 
 		public List<S> getInternalList() {
@@ -441,8 +397,7 @@ public void add(S value) {
 		@Override
 		public String toString() {
 			return "PartitionableListState{" +
-					"name='" + name + '\'' +
-					", assignmentMode=" + assignmentMode +
+					"stateMetaInfo=" + stateMetaInfo +
 					", internalList=" + internalList +
 					'}';
 		}
@@ -456,7 +411,7 @@ public long[] write(FSDataOutputStream out) throws IOException {
 			for (int i = 0; i < internalList.size(); ++i) {
 				S element = internalList.get(i);
 				partitionOffsets[i] = out.getPos();
-				partitionStateSerializer.serialize(element, dov);
+				getStateMetaInfo().getPartitionStateSerializer().serialize(element, dov);
 			}
 
 			return partitionOffsets;
@@ -466,7 +421,6 @@ public long[] write(FSDataOutputStream out) throws IOException {
 	private <S> ListState<S> getListState(
 		ListStateDescriptor<S> stateDescriptor,
 		OperatorStateHandle.Mode mode) throws IOException {
-
 		Preconditions.checkNotNull(stateDescriptor);
 
 		stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
@@ -478,23 +432,27 @@ private <S> ListState<S> getListState(
 		PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredStates.get(name);
 
 		if (null == partitionableListState) {
-
 			partitionableListState = new PartitionableListState<>(
-				name,
-				partitionStateSerializer,
-				mode);
+				new RegisteredOperatorBackendStateMetaInfo<>(
+					name,
+					partitionStateSerializer,
+					mode));
 
 			registeredStates.put(name, partitionableListState);
 		} else {
+			// TODO with eager registration in place, these checks should be moved to restore()
+
 			Preconditions.checkState(
-				partitionableListState.getAssignmentMode().equals(mode),
-				"Incompatible assignment mode. Provided: " + mode + ", expected: " +
-					partitionableListState.getAssignmentMode());
+				partitionableListState.getStateMetaInfo().getName().equals(name),
+				"Incompatible state names. " +
+					"Was [" + partitionableListState.getStateMetaInfo().getName() + "], " +
+					"registered with [" + name + "].");
+
 			Preconditions.checkState(
-				stateDescriptor.getElementSerializer().
-					canRestoreFrom(partitionableListState.getPartitionStateSerializer()),
-				"Incompatible type serializers. Provided: " + stateDescriptor.getElementSerializer() +
-					", found: " + partitionableListState.getPartitionStateSerializer());
+				partitionableListState.getStateMetaInfo().getAssignmentMode().equals(mode),
+				"Incompatible state assignment modes. " +
+					"Was [" + partitionableListState.getStateMetaInfo().getAssignmentMode() + "], " +
+					"registered with [" + mode + "].");
 		}
 
 		return partitionableListState;
@@ -509,7 +467,7 @@ private static <S> void deserializeStateValues(
 			long[] offsets = metaInfo.getOffsets();
 			if (null != offsets) {
 				DataInputView div = new DataInputViewStreamWrapper(in);
-				TypeSerializer<S> serializer = stateListForName.getPartitionStateSerializer();
+				TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
 				for (long offset : offsets) {
 					in.seek(offset);
 					stateListForName.add(serializer.deserialize(div));
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
index 61cc58ca32886..d52c207b1bcc5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
@@ -18,7 +18,11 @@
 
 package org.apache.flink.runtime.state;
 
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.base.MapSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.util.Preconditions;
@@ -38,7 +42,8 @@
  * @param <K> The type of the keys in the map.
  * @param <V> The type of the values in the map.
  */
-public class HashMapSerializer<K, V> extends TypeSerializer<HashMap<K, V>> {
+@Internal
+public final class HashMapSerializer<K, V> extends TypeSerializer<HashMap<K, V>> {
 
 	private static final long serialVersionUID = -6885593032367050078L;
 	
@@ -190,4 +195,37 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return keySerializer.hashCode() * 31 + valueSerializer.hashCode();
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		return new MapSerializerConfigSnapshot(
+				keySerializer.snapshotConfiguration(),
+				valueSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<HashMap<K, V>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof MapSerializerConfigSnapshot) {
+			TypeSerializerConfigSnapshot[] keyValueSerializerConfigSnapshots =
+				((MapSerializerConfigSnapshot) configSnapshot).getNestedSerializerConfigSnapshots();
+
+			CompatibilityResult<K> keyCompatResult = keySerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[0]);
+			CompatibilityResult<V> valueCompatResult = valueSerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[1]);
+
+			if (!keyCompatResult.requiresMigration() && !valueCompatResult.requiresMigration()) {
+				return CompatibilityResult.compatible();
+			} else if (keyCompatResult.getConvertDeserializer() != null && valueCompatResult.getConvertDeserializer() != null) {
+				return CompatibilityResult.requiresMigration(
+					new HashMapSerializer<>(
+						keyCompatResult.getConvertDeserializer(),
+						valueCompatResult.getConvertDeserializer()));
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
index 512baf6cf10d0..d49b1d22112d6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
@@ -19,7 +19,7 @@
 package org.apache.flink.runtime.state;
 
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
 import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
 import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
 import org.apache.flink.core.memory.DataInputView;
@@ -31,7 +31,7 @@
 
 @SuppressWarnings("serial")
 @Internal
-final class JavaSerializer<T extends Serializable> extends TypeSerializer<T> {
+final class JavaSerializer<T extends Serializable> extends TypeSerializerSingleton<T> {
 
 	private static final long serialVersionUID = 5067491650263321234L;
 
@@ -40,11 +40,6 @@ public boolean isImmutableType() {
 		return false;
 	}
 
-	@Override
-	public TypeSerializer<T> duplicate() {
-		return this;
-	}
-
 	@Override
 	public T createInstance() {
 		return null;
@@ -97,18 +92,8 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 		target.write(source, size);
 	}
 
-	@Override
-	public boolean equals(Object obj) {
-		return obj instanceof JavaSerializer;
-	}
-
 	@Override
 	public boolean canEqual(Object obj) {
 		return obj instanceof JavaSerializer;
 	}
-
-	@Override
-	public int hashCode() {
-		return getClass().hashCode();
-	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java
index 5661c38d4a6fc..a389c4f7ef7f8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java
@@ -18,10 +18,8 @@
 
 package org.apache.flink.runtime.state;
 
-import org.apache.flink.api.common.state.StateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
-import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.core.io.VersionMismatchException;
 import org.apache.flink.core.io.VersionedIOReadableWritable;
 import org.apache.flink.core.memory.DataInputView;
@@ -33,15 +31,15 @@
 import java.util.List;
 
 /**
- * Serialization proxy for all meta data in keyed state backends. In the future we might also migrate the actual state
+ * Serialization proxy for all meta data in keyed state backends. In the future we might also requiresMigration the actual state
  * serialization logic here.
  */
 public class KeyedBackendSerializationProxy extends VersionedIOReadableWritable {
 
-	public static final int VERSION = 2;
+	public static final int VERSION = 3;
 
-	private TypeSerializerSerializationProxy<?> keySerializerProxy;
-	private List<StateMetaInfo<?, ?>> namedStateSerializationProxies;
+	private TypeSerializer<?> keySerializer;
+	private List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots;
 
 	private int restoredVersion;
 	private ClassLoader userCodeClassLoader;
@@ -50,19 +48,25 @@ public KeyedBackendSerializationProxy(ClassLoader userCodeClassLoader) {
 		this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
 	}
 
-	public KeyedBackendSerializationProxy(TypeSerializer<?> keySerializer, List<StateMetaInfo<?, ?>> namedStateSerializationProxies) {
-		this.keySerializerProxy = new TypeSerializerSerializationProxy<>(Preconditions.checkNotNull(keySerializer));
-		this.namedStateSerializationProxies = Preconditions.checkNotNull(namedStateSerializationProxies);
+	public KeyedBackendSerializationProxy(
+			TypeSerializer<?> keySerializer,
+			List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots) {
+
+		this.keySerializer = Preconditions.checkNotNull(keySerializer);
+
+		Preconditions.checkNotNull(stateMetaInfoSnapshots);
+		Preconditions.checkArgument(stateMetaInfoSnapshots.size() <= Short.MAX_VALUE);
+		this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
+
 		this.restoredVersion = VERSION;
-		Preconditions.checkArgument(namedStateSerializationProxies.size() <= Short.MAX_VALUE);
 	}
 
-	public List<StateMetaInfo<?, ?>> getNamedStateSerializationProxies() {
-		return namedStateSerializationProxies;
+	public List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> getStateMetaInfoSnapshots() {
+		return stateMetaInfoSnapshots;
 	}
 
-	public TypeSerializerSerializationProxy<?> getKeySerializerProxy() {
-		return keySerializerProxy;
+	public TypeSerializer<?> getKeySerializer() {
+		return keySerializer;
 	}
 
 	@Override
@@ -82,20 +86,22 @@ protected void resolveVersionRead(int foundVersion) throws VersionMismatchExcept
 
 	@Override
 	public boolean isCompatibleVersion(int version) {
-		// we are compatible with version 2 (Flink 1.3.x) and version 1 (Flink 1.2.x)
-		return super.isCompatibleVersion(version) || version == 1;
+		// we are compatible with version 3 (Flink 1.3.x) and version 1 & 2 (Flink 1.2.x)
+		return super.isCompatibleVersion(version) || version == 2 || version == 1;
 	}
 
 	@Override
 	public void write(DataOutputView out) throws IOException {
 		super.write(out);
 
-		keySerializerProxy.write(out);
+		new TypeSerializerSerializationProxy<>(keySerializer).write(out);
 
-		out.writeShort(namedStateSerializationProxies.size());
+		out.writeShort(stateMetaInfoSnapshots.size());
 
-		for (StateMetaInfo<?, ?> kvState : namedStateSerializationProxies) {
-			kvState.write(out);
+		for (RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> metaInfo : stateMetaInfoSnapshots) {
+			KeyedBackendStateMetaInfoSnapshotReaderWriters
+				.getWriterForVersion(VERSION, metaInfo)
+				.writeStateMetaInfo(out);
 		}
 	}
 
@@ -103,132 +109,18 @@ public void write(DataOutputView out) throws IOException {
 	public void read(DataInputView in) throws IOException {
 		super.read(in);
 
-		keySerializerProxy = new TypeSerializerSerializationProxy<>(userCodeClassLoader);
+		final TypeSerializerSerializationProxy<?> keySerializerProxy =
+			new TypeSerializerSerializationProxy<>(userCodeClassLoader);
 		keySerializerProxy.read(in);
+		this.keySerializer = keySerializerProxy.getTypeSerializer();
 
 		int numKvStates = in.readShort();
-		namedStateSerializationProxies = new ArrayList<>(numKvStates);
-		for (int i = 0; i < numKvStates; ++i) {
-			StateMetaInfo<?, ?> stateSerializationProxy = new StateMetaInfo<>(userCodeClassLoader);
-			stateSerializationProxy.read(in);
-			namedStateSerializationProxies.add(stateSerializationProxy);
-		}
-	}
-
-	//----------------------------------------------------------------------------------------------------------------------
-
-	/**
-	 * This is the serialization proxy for {@link RegisteredBackendStateMetaInfo} for a single registered state in a
-	 * keyed backend.
-	 */
-	public static class StateMetaInfo<N, S> implements IOReadableWritable {
-
-		private StateDescriptor.Type stateType;
-		private String stateName;
-		private TypeSerializerSerializationProxy<N> namespaceSerializerSerializationProxy;
-		private TypeSerializerSerializationProxy<S> stateSerializerSerializationProxy;
-
-		private ClassLoader userClassLoader;
-
-		StateMetaInfo(ClassLoader userClassLoader) {
-			this.userClassLoader = Preconditions.checkNotNull(userClassLoader);
-		}
-
-		public StateMetaInfo(
-				StateDescriptor.Type stateType,
-				String name,
-				TypeSerializer<N> namespaceSerializer,
-				TypeSerializer<S> stateSerializer) {
-
-			this.stateType = Preconditions.checkNotNull(stateType);
-			this.stateName = Preconditions.checkNotNull(name);
-			this.namespaceSerializerSerializationProxy = new TypeSerializerSerializationProxy<>(Preconditions.checkNotNull(namespaceSerializer));
-			this.stateSerializerSerializationProxy = new TypeSerializerSerializationProxy<>(Preconditions.checkNotNull(stateSerializer));
-		}
-
-		public StateDescriptor.Type getStateType() {
-			return stateType;
-		}
-
-		public void setStateType(StateDescriptor.Type stateType) {
-			this.stateType = stateType;
-		}
-
-		public String getStateName() {
-			return stateName;
-		}
-
-		public void setStateName(String stateName) {
-			this.stateName = stateName;
-		}
-
-		public TypeSerializerSerializationProxy<N> getNamespaceSerializerSerializationProxy() {
-			return namespaceSerializerSerializationProxy;
-		}
-
-		public void setNamespaceSerializerSerializationProxy(TypeSerializerSerializationProxy<N> namespaceSerializerSerializationProxy) {
-			this.namespaceSerializerSerializationProxy = namespaceSerializerSerializationProxy;
-		}
-
-		public TypeSerializerSerializationProxy<S> getStateSerializerSerializationProxy() {
-			return stateSerializerSerializationProxy;
-		}
-
-		public void setStateSerializerSerializationProxy(TypeSerializerSerializationProxy<S> stateSerializerSerializationProxy) {
-			this.stateSerializerSerializationProxy = stateSerializerSerializationProxy;
-		}
-
-		@Override
-		public void write(DataOutputView out) throws IOException {
-			out.writeInt(getStateType().ordinal());
-			out.writeUTF(getStateName());
-
-			getNamespaceSerializerSerializationProxy().write(out);
-			getStateSerializerSerializationProxy().write(out);
-		}
-
-		@Override
-		public void read(DataInputView in) throws IOException {
-			int enumOrdinal = in.readInt();
-			setStateType(StateDescriptor.Type.values()[enumOrdinal]);
-			setStateName(in.readUTF());
-
-			namespaceSerializerSerializationProxy = new TypeSerializerSerializationProxy<>(userClassLoader);
-			namespaceSerializerSerializationProxy.read(in);
-
-			stateSerializerSerializationProxy = new TypeSerializerSerializationProxy<>(userClassLoader);
-			stateSerializerSerializationProxy.read(in);
-		}
-
-		@Override
-		public boolean equals(Object o) {
-			if (this == o) {
-				return true;
-			}
-
-			if (o == null || getClass() != o.getClass()) {
-				return false;
-			}
-
-			StateMetaInfo<?, ?> that = (StateMetaInfo<?, ?>) o;
-
-			if (!getStateName().equals(that.getStateName())) {
-				return false;
-			}
-
-			if (!getNamespaceSerializerSerializationProxy().equals(that.getNamespaceSerializerSerializationProxy())) {
-				return false;
-			}
-
-			return getStateSerializerSerializationProxy().equals(that.getStateSerializerSerializationProxy());
-		}
-
-		@Override
-		public int hashCode() {
-			int result = getStateName().hashCode();
-			result = 31 * result + getNamespaceSerializerSerializationProxy().hashCode();
-			result = 31 * result + getStateSerializerSerializationProxy().hashCode();
-			return result;
+		stateMetaInfoSnapshots = new ArrayList<>(numKvStates);
+		for (int i = 0; i < numKvStates; i++) {
+			stateMetaInfoSnapshots.add(
+				KeyedBackendStateMetaInfoSnapshotReaderWriters
+					.getReaderForVersion(restoredVersion, userCodeClassLoader)
+					.readStateMetaInfo(in));
 		}
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java
new file mode 100644
index 0000000000000..83aa33576aded
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.state.StateDescriptor;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
+import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * Readers and writers for different versions of the {@link RegisteredKeyedBackendStateMetaInfo.Snapshot}.
+ * Outdated formats are also kept here for documentation of history backlog.
+ */
+public class KeyedBackendStateMetaInfoSnapshotReaderWriters {
+
+	// -------------------------------------------------------------------------------
+	//  Writers
+	//   - v1: Flink 1.2.x
+	//   - v2: Flink 1.3.x
+	// -------------------------------------------------------------------------------
+
+	public static <N, S> KeyedBackendStateMetaInfoWriter getWriterForVersion(
+		int version, RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> stateMetaInfo) {
+
+		switch (version) {
+			case 1:
+			case 2:
+				return new KeyedBackendStateMetaInfoWriterV1V2<>(stateMetaInfo);
+
+			// current version
+			case KeyedBackendSerializationProxy.VERSION:
+				return new KeyedBackendStateMetaInfoWriterV3<>(stateMetaInfo);
+
+			default:
+				// guard for future
+				throw new IllegalStateException(
+							"Unrecognized keyed backend state meta info writer version: " + version);
+		}
+	}
+
+	public interface KeyedBackendStateMetaInfoWriter {
+		void writeStateMetaInfo(DataOutputView out) throws IOException;
+	}
+
+	static abstract class AbstractKeyedBackendStateMetaInfoWriter<N, S> implements KeyedBackendStateMetaInfoWriter {
+
+		protected final RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> stateMetaInfo;
+
+		public AbstractKeyedBackendStateMetaInfoWriter(RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> stateMetaInfo) {
+			this.stateMetaInfo = Preconditions.checkNotNull(stateMetaInfo);
+		}
+
+	}
+
+	static class KeyedBackendStateMetaInfoWriterV1V2<N, S> extends AbstractKeyedBackendStateMetaInfoWriter<N, S> {
+
+		public KeyedBackendStateMetaInfoWriterV1V2(RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> stateMetaInfo) {
+			super(stateMetaInfo);
+		}
+
+		@Override
+		public void writeStateMetaInfo(DataOutputView out) throws IOException {
+			out.writeInt(stateMetaInfo.getStateType().ordinal());
+			out.writeUTF(stateMetaInfo.getName());
+
+			new TypeSerializerSerializationProxy<>(stateMetaInfo.getNamespaceSerializer()).write(out);
+			new TypeSerializerSerializationProxy<>(stateMetaInfo.getStateSerializer()).write(out);
+		}
+	}
+
+	static class KeyedBackendStateMetaInfoWriterV3<N, S> extends AbstractKeyedBackendStateMetaInfoWriter<N, S> {
+
+		public KeyedBackendStateMetaInfoWriterV3(RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> stateMetaInfo) {
+			super(stateMetaInfo);
+		}
+
+		@Override
+		public void writeStateMetaInfo(DataOutputView out) throws IOException {
+			out.writeInt(stateMetaInfo.getStateType().ordinal());
+			out.writeUTF(stateMetaInfo.getName());
+
+			// write in a way that allows us to be fault-tolerant and skip blocks in the case of java serialization failures
+			try (
+				ByteArrayOutputStreamWithPos outWithPos = new ByteArrayOutputStreamWithPos();
+				DataOutputViewStreamWrapper outViewWrapper = new DataOutputViewStreamWrapper(outWithPos)) {
+
+				new TypeSerializerSerializationProxy<>(stateMetaInfo.getNamespaceSerializer()).write(outViewWrapper);
+
+				// write current offset, which represents the start offset of the state serializer
+				out.writeInt(outWithPos.getPosition());
+				new TypeSerializerSerializationProxy<>(stateMetaInfo.getStateSerializer()).write(outViewWrapper);
+
+				// write current offset, which represents the start of the configuration snapshots
+				out.writeInt(outWithPos.getPosition());
+				TypeSerializerUtil.writeSerializerConfigSnapshot(outViewWrapper, stateMetaInfo.getNamespaceSerializerConfigSnapshot());
+				TypeSerializerUtil.writeSerializerConfigSnapshot(outViewWrapper, stateMetaInfo.getStateSerializerConfigSnapshot());
+
+				// write total number of bytes and then flush
+				out.writeInt(outWithPos.getPosition());
+				out.write(outWithPos.getBuf(), 0, outWithPos.getPosition());
+			}
+		}
+	}
+
+
+	// -------------------------------------------------------------------------------
+	//  Readers
+	//   - v1: Flink 1.2.x
+	//   - v2: Flink 1.3.x
+	// -------------------------------------------------------------------------------
+
+	public static KeyedBackendStateMetaInfoReader getReaderForVersion(
+			int version, ClassLoader userCodeClassLoader) {
+
+		switch (version) {
+			case 1:
+			case 2:
+				return new KeyedBackendStateMetaInfoReaderV1V2<>(userCodeClassLoader);
+
+			// current version
+			case KeyedBackendSerializationProxy.VERSION:
+				return new KeyedBackendStateMetaInfoReaderV3<>(userCodeClassLoader);
+
+			default:
+				// guard for future
+				throw new IllegalStateException(
+							"Unrecognized keyed backend state meta info reader version: " + version);
+		}
+	}
+
+	public interface KeyedBackendStateMetaInfoReader<N, S> {
+		RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> readStateMetaInfo(DataInputView in) throws IOException;
+	}
+
+	static abstract class AbstractKeyedBackendStateMetaInfoReader implements KeyedBackendStateMetaInfoReader {
+
+		protected final ClassLoader userCodeClassLoader;
+
+		public AbstractKeyedBackendStateMetaInfoReader(ClassLoader userCodeClassLoader) {
+			this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
+		}
+
+	}
+
+	static class KeyedBackendStateMetaInfoReaderV1V2<N, S> extends AbstractKeyedBackendStateMetaInfoReader {
+
+		public KeyedBackendStateMetaInfoReaderV1V2(ClassLoader userCodeClassLoader) {
+			super(userCodeClassLoader);
+		}
+
+		@Override
+		public RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> readStateMetaInfo(DataInputView in) throws IOException {
+			RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> metaInfo =
+				new RegisteredKeyedBackendStateMetaInfo.Snapshot<>();
+
+			metaInfo.setStateType(StateDescriptor.Type.values()[in.readInt()]);
+			metaInfo.setName(in.readUTF());
+
+			final TypeSerializerSerializationProxy<N> namespaceSerializerProxy =
+				new TypeSerializerSerializationProxy<>(userCodeClassLoader);
+			namespaceSerializerProxy.read(in);
+			metaInfo.setNamespaceSerializer(namespaceSerializerProxy.getTypeSerializer());
+
+			final TypeSerializerSerializationProxy<S> stateSerializerProxy =
+				new TypeSerializerSerializationProxy<>(userCodeClassLoader);
+			stateSerializerProxy.read(in);
+			metaInfo.setStateSerializer(stateSerializerProxy.getTypeSerializer());
+
+			// older versions do not contain the configuration snapshot
+			metaInfo.setNamespaceSerializerConfigSnapshot(null);
+			metaInfo.setStateSerializerConfigSnapshot(null);
+
+			return metaInfo;
+		}
+	}
+
+	static class KeyedBackendStateMetaInfoReaderV3<N, S> extends AbstractKeyedBackendStateMetaInfoReader {
+
+		public KeyedBackendStateMetaInfoReaderV3(ClassLoader userCodeClassLoader) {
+			super(userCodeClassLoader);
+		}
+
+		@Override
+		public RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> readStateMetaInfo(DataInputView in) throws IOException {
+			RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> metaInfo =
+				new RegisteredKeyedBackendStateMetaInfo.Snapshot<>();
+
+			metaInfo.setStateType(StateDescriptor.Type.values()[in.readInt()]);
+			metaInfo.setName(in.readUTF());
+
+			// read offsets
+			int stateSerializerStartOffset = in.readInt();
+			int configSnapshotsStartOffset = in.readInt();
+
+			int totalBytes = in.readInt();
+
+			byte[] buffer = new byte[totalBytes];
+			in.readFully(buffer);
+
+			ByteArrayInputStreamWithPos inWithPos = new ByteArrayInputStreamWithPos(buffer);
+			DataInputViewStreamWrapper inViewWrapper = new DataInputViewStreamWrapper(inWithPos);
+
+			try {
+				final TypeSerializerSerializationProxy<N> namespaceSerializerProxy =
+					new TypeSerializerSerializationProxy<>(userCodeClassLoader);
+				namespaceSerializerProxy.read(inViewWrapper);
+				metaInfo.setNamespaceSerializer(namespaceSerializerProxy.getTypeSerializer());
+			} catch (IOException e) {
+				metaInfo.setNamespaceSerializer(null);
+			}
+
+			// make sure we start from the state serializer bytes position
+			inWithPos.setPosition(stateSerializerStartOffset);
+			try {
+				final TypeSerializerSerializationProxy<S> stateSerializerProxy =
+					new TypeSerializerSerializationProxy<>(userCodeClassLoader);
+				stateSerializerProxy.read(inViewWrapper);
+				metaInfo.setStateSerializer(stateSerializerProxy.getTypeSerializer());
+			} catch (IOException e) {
+				metaInfo.setStateSerializer(null);
+			}
+
+			// make sure we start from the config snapshot bytes position
+			inWithPos.setPosition(configSnapshotsStartOffset);
+			metaInfo.setNamespaceSerializerConfigSnapshot(
+				TypeSerializerUtil.readSerializerConfigSnapshot(inViewWrapper, userCodeClassLoader));
+			metaInfo.setStateSerializerConfigSnapshot(
+				TypeSerializerUtil.readSerializerConfigSnapshot(inViewWrapper, userCodeClassLoader));
+
+			return metaInfo;
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java
index d571dccc8fc28..91d7aab10f5db 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java
@@ -18,15 +18,10 @@
 
 package org.apache.flink.runtime.state;
 
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
-import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
-import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.core.io.VersionMismatchException;
 import org.apache.flink.core.io.VersionedIOReadableWritable;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.Preconditions;
 
 import java.io.IOException;
@@ -34,23 +29,29 @@
 import java.util.List;
 
 /**
- * Serialization proxy for all meta data in operator state backends. In the future we might also migrate the actual state
+ * Serialization proxy for all meta data in operator state backends. In the future we might also requiresMigration the actual state
  * serialization logic here.
  */
 public class OperatorBackendSerializationProxy extends VersionedIOReadableWritable {
 
-	private static final int VERSION = 1;
+	public static final int VERSION = 2;
 
-	private List<StateMetaInfo<?>> namedStateSerializationProxies;
+	private List<RegisteredOperatorBackendStateMetaInfo.Snapshot<?>> stateMetaInfoSnapshots;
 	private ClassLoader userCodeClassLoader;
 
+	private int restoredVersion;
+
 	public OperatorBackendSerializationProxy(ClassLoader userCodeClassLoader) {
 		this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
 	}
 
-	public OperatorBackendSerializationProxy(List<StateMetaInfo<?>> namedStateSerializationProxies) {
-		this.namedStateSerializationProxies = Preconditions.checkNotNull(namedStateSerializationProxies);
-		Preconditions.checkArgument(namedStateSerializationProxies.size() <= Short.MAX_VALUE);
+	public OperatorBackendSerializationProxy(
+			List<RegisteredOperatorBackendStateMetaInfo.Snapshot<?>> stateMetaInfoSnapshots) {
+
+		this.stateMetaInfoSnapshots = Preconditions.checkNotNull(stateMetaInfoSnapshots);
+		Preconditions.checkArgument(stateMetaInfoSnapshots.size() <= Short.MAX_VALUE);
+
+		this.restoredVersion = VERSION;
 	}
 
 	@Override
@@ -59,129 +60,44 @@ public int getVersion() {
 	}
 
 	@Override
-	public void write(DataOutputView out) throws IOException {
-		super.write(out);
-
-		out.writeShort(namedStateSerializationProxies.size());
-
-		for (StateMetaInfo<?> kvState : namedStateSerializationProxies) {
-			kvState.write(out);
-		}
+	protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
+		super.resolveVersionRead(foundVersion);
+		this.restoredVersion = foundVersion;
 	}
 
 	@Override
-	public void read(DataInputView out) throws IOException {
-		super.read(out);
-
-		int numKvStates = out.readShort();
-		namedStateSerializationProxies = new ArrayList<>(numKvStates);
-		for (int i = 0; i < numKvStates; ++i) {
-			StateMetaInfo<?> stateSerializationProxy = new StateMetaInfo<>(userCodeClassLoader);
-			stateSerializationProxy.read(out);
-			namedStateSerializationProxies.add(stateSerializationProxy);
-		}
+	public boolean isCompatibleVersion(int version) {
+		// we are compatible with version 2 (Flink 1.3.x) and version 1 (Flink 1.2.x)
+		return super.isCompatibleVersion(version) || version == 1;
 	}
 
-	public List<StateMetaInfo<?>> getNamedStateSerializationProxies() {
-		return namedStateSerializationProxies;
-	}
-
-	//----------------------------------------------------------------------------------------------------------------------
-
-	public static class StateMetaInfo<S> implements IOReadableWritable {
-
-		private String name;
-		private TypeSerializer<S> stateSerializer;
-		private OperatorStateHandle.Mode mode;
-
-		private ClassLoader userClassLoader;
-
-		@VisibleForTesting
-		public StateMetaInfo(ClassLoader userClassLoader) {
-			this.userClassLoader = Preconditions.checkNotNull(userClassLoader);
-		}
-
-		public StateMetaInfo(String name, TypeSerializer<S> stateSerializer, OperatorStateHandle.Mode mode) {
-			this.name = Preconditions.checkNotNull(name);
-			this.stateSerializer = Preconditions.checkNotNull(stateSerializer);
-			this.mode = Preconditions.checkNotNull(mode);
-		}
-
-		public String getName() {
-			return name;
-		}
-
-		public void setName(String name) {
-			this.name = name;
-		}
-
-		public TypeSerializer<S> getStateSerializer() {
-			return stateSerializer;
-		}
-
-		public void setStateSerializer(TypeSerializer<S> stateSerializer) {
-			this.stateSerializer = stateSerializer;
-		}
-
-		public OperatorStateHandle.Mode getMode() {
-			return mode;
-		}
-
-		public void setMode(OperatorStateHandle.Mode mode) {
-			this.mode = mode;
-		}
-
-		@Override
-		public void write(DataOutputView out) throws IOException {
-			out.writeUTF(getName());
-			out.writeByte(getMode().ordinal());
-			DataOutputViewStream dos = new DataOutputViewStream(out);
-			InstantiationUtil.serializeObject(dos, getStateSerializer());
-		}
+	@Override
+	public void write(DataOutputView out) throws IOException {
+		super.write(out);
 
-		@Override
-		public void read(DataInputView in) throws IOException {
-			setName(in.readUTF());
-			setMode(OperatorStateHandle.Mode.values()[in.readByte()]);
-			DataInputViewStream dis = new DataInputViewStream(in);
-			try {
-				TypeSerializer<S> stateSerializer = InstantiationUtil.deserializeObject(dis, userClassLoader);
-				setStateSerializer(stateSerializer);
-			} catch (ClassNotFoundException exception) {
-				throw new IOException(exception);
-			}
+		out.writeShort(stateMetaInfoSnapshots.size());
+		for (RegisteredOperatorBackendStateMetaInfo.Snapshot<?> kvState : stateMetaInfoSnapshots) {
+			OperatorBackendStateMetaInfoSnapshotReaderWriters
+				.getWriterForVersion(VERSION, kvState)
+				.writeStateMetaInfo(out);
 		}
+	}
 
-		@Override
-		public boolean equals(Object o) {
-
-			if (this == o) {
-				return true;
-			}
-
-			if (o == null || getClass() != o.getClass()) {
-				return false;
-			}
-
-			StateMetaInfo<?> metaInfo = (StateMetaInfo<?>) o;
-
-			if (!getName().equals(metaInfo.getName())) {
-				return false;
-			}
-
-			if (!getStateSerializer().equals(metaInfo.getStateSerializer())) {
-				return false;
-			}
-
-			return getMode() == metaInfo.getMode();
+	@Override
+	public void read(DataInputView in) throws IOException {
+		super.read(in);
+
+		int numKvStates = in.readShort();
+		stateMetaInfoSnapshots = new ArrayList<>(numKvStates);
+		for (int i = 0; i < numKvStates; i++) {
+			stateMetaInfoSnapshots.add(
+				OperatorBackendStateMetaInfoSnapshotReaderWriters
+					.getReaderForVersion(restoredVersion, userCodeClassLoader)
+					.readStateMetaInfo(in));
 		}
+	}
 
-		@Override
-		public int hashCode() {
-			int result = getName().hashCode();
-			result = 31 * result + getStateSerializer().hashCode();
-			result = 31 * result + getMode().hashCode();
-			return result;
-		}
+	public List<RegisteredOperatorBackendStateMetaInfo.Snapshot<?>> getStateMetaInfoSnapshots() {
+		return stateMetaInfoSnapshots;
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
new file mode 100644
index 0000000000000..9ab106bc492db
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * Readers and writers for different versions of the {@link RegisteredOperatorBackendStateMetaInfo.Snapshot}.
+ * Outdated formats are also kept here for documentation of history backlog.
+ */
+public class OperatorBackendStateMetaInfoSnapshotReaderWriters {
+
+	// -------------------------------------------------------------------------------
+	//  Writers
+	//   - v1: Flink 1.2.x
+	//   - v2: Flink 1.3.x
+	// -------------------------------------------------------------------------------
+
+	public static <S> OperatorBackendStateMetaInfoWriter getWriterForVersion(
+			int version, RegisteredOperatorBackendStateMetaInfo.Snapshot<S> stateMetaInfo) {
+
+		switch (version) {
+			case 1:
+				return new OperatorBackendStateMetaInfoWriterV1<>(stateMetaInfo);
+
+			// current version
+			case OperatorBackendSerializationProxy.VERSION:
+				return new OperatorBackendStateMetaInfoWriterV2<>(stateMetaInfo);
+
+			default:
+				// guard for future
+				throw new IllegalStateException(
+						"Unrecognized operator backend state meta info writer version: " + version);
+		}
+	}
+
+	public interface OperatorBackendStateMetaInfoWriter {
+		void writeStateMetaInfo(DataOutputView out) throws IOException;
+	}
+
+	public static abstract class AbstractOperatorBackendStateMetaInfoWriter<S>
+			implements OperatorBackendStateMetaInfoWriter {
+
+		protected final RegisteredOperatorBackendStateMetaInfo.Snapshot<S> stateMetaInfo;
+
+		public AbstractOperatorBackendStateMetaInfoWriter(RegisteredOperatorBackendStateMetaInfo.Snapshot<S> stateMetaInfo) {
+			this.stateMetaInfo = Preconditions.checkNotNull(stateMetaInfo);
+		}
+	}
+
+	public static class OperatorBackendStateMetaInfoWriterV1<S> extends AbstractOperatorBackendStateMetaInfoWriter<S> {
+
+		public OperatorBackendStateMetaInfoWriterV1(RegisteredOperatorBackendStateMetaInfo.Snapshot<S> stateMetaInfo) {
+			super(stateMetaInfo);
+		}
+
+		@Override
+		public void writeStateMetaInfo(DataOutputView out) throws IOException {
+			out.writeUTF(stateMetaInfo.getName());
+			out.writeByte(stateMetaInfo.getAssignmentMode().ordinal());
+			new TypeSerializerSerializationProxy<>(stateMetaInfo.getPartitionStateSerializer()).write(out);
+		}
+	}
+
+	public static class OperatorBackendStateMetaInfoWriterV2<S> extends AbstractOperatorBackendStateMetaInfoWriter<S> {
+
+		public OperatorBackendStateMetaInfoWriterV2(RegisteredOperatorBackendStateMetaInfo.Snapshot<S> stateMetaInfo) {
+			super(stateMetaInfo);
+		}
+
+		@Override
+		public void writeStateMetaInfo(DataOutputView out) throws IOException {
+			out.writeUTF(stateMetaInfo.getName());
+			out.writeByte(stateMetaInfo.getAssignmentMode().ordinal());
+
+			// write in a way that allows us to be fault-tolerant and skip blocks in the case of java serialization failures
+			try (
+				ByteArrayOutputStreamWithPos outWithPos = new ByteArrayOutputStreamWithPos();
+				DataOutputViewStreamWrapper outViewWrapper = new DataOutputViewStreamWrapper(outWithPos)) {
+
+				new TypeSerializerSerializationProxy<>(stateMetaInfo.getPartitionStateSerializer()).write(outViewWrapper);
+
+				// write the start offset of the config snapshot
+				out.writeInt(outWithPos.getPosition());
+				TypeSerializerUtil.writeSerializerConfigSnapshot(
+					outViewWrapper,
+					stateMetaInfo.getPartitionStateSerializerConfigSnapshot());
+
+				// write the total number of bytes and flush
+				out.writeInt(outWithPos.getPosition());
+				out.write(outWithPos.getBuf(), 0, outWithPos.getPosition());
+			}
+		}
+	}
+
+	// -------------------------------------------------------------------------------
+	//  Readers
+	//   - v1: Flink 1.2.x
+	//   - v2: Flink 1.3.x
+	// -------------------------------------------------------------------------------
+
+	public static <S> OperatorBackendStateMetaInfoReader<S> getReaderForVersion(
+			int version, ClassLoader userCodeClassLoader) {
+
+		switch (version) {
+			case 1:
+				return new OperatorBackendStateMetaInfoReaderV1<>(userCodeClassLoader);
+
+			// current version
+			case OperatorBackendSerializationProxy.VERSION:
+				return new OperatorBackendStateMetaInfoReaderV2<>(userCodeClassLoader);
+
+			default:
+				// guard for future
+				throw new IllegalStateException(
+					"Unrecognized operator backend state meta info reader version: " + version);
+		}
+	}
+
+	public interface OperatorBackendStateMetaInfoReader<S> {
+		RegisteredOperatorBackendStateMetaInfo.Snapshot<S> readStateMetaInfo(DataInputView in) throws IOException;
+	}
+
+	public static abstract class AbstractOperatorBackendStateMetaInfoReader<S>
+		implements OperatorBackendStateMetaInfoReader<S> {
+
+		protected final ClassLoader userCodeClassLoader;
+
+		public AbstractOperatorBackendStateMetaInfoReader(ClassLoader userCodeClassLoader) {
+			this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
+		}
+	}
+
+	public static class OperatorBackendStateMetaInfoReaderV1<S> extends AbstractOperatorBackendStateMetaInfoReader<S> {
+
+		public OperatorBackendStateMetaInfoReaderV1(ClassLoader userCodeClassLoader) {
+			super(userCodeClassLoader);
+		}
+
+		@Override
+		public RegisteredOperatorBackendStateMetaInfo.Snapshot<S> readStateMetaInfo(DataInputView in) throws IOException {
+			RegisteredOperatorBackendStateMetaInfo.Snapshot<S> stateMetaInfo =
+				new RegisteredOperatorBackendStateMetaInfo.Snapshot<>();
+
+			stateMetaInfo.setName(in.readUTF());
+			stateMetaInfo.setAssignmentMode(OperatorStateHandle.Mode.values()[in.readByte()]);
+			DataInputViewStream dis = new DataInputViewStream(in);
+			try {
+				TypeSerializer<S> stateSerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader);
+				stateMetaInfo.setPartitionStateSerializer(stateSerializer);
+			} catch (ClassNotFoundException exception) {
+				throw new IOException(exception);
+			}
+
+			// old versions do not contain the partition state serializer's configuration snapshot
+			stateMetaInfo.setPartitionStateSerializerConfigSnapshot(null);
+
+			return stateMetaInfo;
+		}
+	}
+
+	public static class OperatorBackendStateMetaInfoReaderV2<S> extends AbstractOperatorBackendStateMetaInfoReader<S> {
+
+		public OperatorBackendStateMetaInfoReaderV2(ClassLoader userCodeClassLoader) {
+			super(userCodeClassLoader);
+		}
+
+		@Override
+		public RegisteredOperatorBackendStateMetaInfo.Snapshot<S> readStateMetaInfo(DataInputView in) throws IOException {
+			RegisteredOperatorBackendStateMetaInfo.Snapshot<S> stateMetaInfo =
+				new RegisteredOperatorBackendStateMetaInfo.Snapshot<>();
+
+			stateMetaInfo.setName(in.readUTF());
+			stateMetaInfo.setAssignmentMode(OperatorStateHandle.Mode.values()[in.readByte()]);
+
+			// read start offset of configuration snapshot
+			int configSnapshotStartOffset = in.readInt();
+
+			int totalBytes = in.readInt();
+
+			byte[] buffer = new byte[totalBytes];
+			in.readFully(buffer);
+
+			ByteArrayInputStreamWithPos inWithPos = new ByteArrayInputStreamWithPos(buffer);
+			DataInputViewStreamWrapper inViewWrapper = new DataInputViewStreamWrapper(inWithPos);
+
+			try {
+				final TypeSerializerSerializationProxy<S> partitionStateSerializerProxy =
+					new TypeSerializerSerializationProxy<>(userCodeClassLoader);
+				partitionStateSerializerProxy.read(inViewWrapper);
+				stateMetaInfo.setPartitionStateSerializer(partitionStateSerializerProxy.getTypeSerializer());
+			} catch (IOException e) {
+				stateMetaInfo.setPartitionStateSerializer(null);
+			}
+
+			// make sure we start from the partition state serializer bytes position
+			inWithPos.setPosition(configSnapshotStartOffset);
+			stateMetaInfo.setPartitionStateSerializerConfigSnapshot(
+				TypeSerializerUtil.readSerializerConfigSnapshot(inViewWrapper, userCodeClassLoader));
+
+			return stateMetaInfo;
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBackendStateMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBackendStateMetaInfo.java
deleted file mode 100644
index 0d4b3c82e1b90..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBackendStateMetaInfo.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.state;
-
-import org.apache.flink.api.common.state.StateDescriptor;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.migration.MigrationNamespaceSerializerProxy;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * Compound meta information for a registered state in a keyed state backend. This combines all serializers and the
- * state name.
- *
- * @param <N> Type of namespace
- * @param <S> Type of state value
- */
-public class RegisteredBackendStateMetaInfo<N, S> {
-
-	private final StateDescriptor.Type stateType;
-	private final String name;
-	private final TypeSerializer<N> namespaceSerializer;
-	private final TypeSerializer<S> stateSerializer;
-
-	public RegisteredBackendStateMetaInfo(KeyedBackendSerializationProxy.StateMetaInfo<N, S> metaInfoProxy) {
-		this(
-				metaInfoProxy.getStateType(),
-				metaInfoProxy.getStateName(),
-				metaInfoProxy.getNamespaceSerializerSerializationProxy().getTypeSerializer(),
-				metaInfoProxy.getStateSerializerSerializationProxy().getTypeSerializer());
-	}
-
-	public RegisteredBackendStateMetaInfo(
-			StateDescriptor.Type stateType,
-			String name,
-			TypeSerializer<N> namespaceSerializer,
-			TypeSerializer<S> stateSerializer) {
-
-		this.stateType = checkNotNull(stateType);
-		this.name = checkNotNull(name);
-		this.namespaceSerializer = checkNotNull(namespaceSerializer);
-		this.stateSerializer = checkNotNull(stateSerializer);
-	}
-
-	public StateDescriptor.Type getStateType() {
-		return stateType;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public TypeSerializer<N> getNamespaceSerializer() {
-		return namespaceSerializer;
-	}
-
-	public TypeSerializer<S> getStateSerializer() {
-		return stateSerializer;
-	}
-
-	public boolean canRestoreFrom(RegisteredBackendStateMetaInfo<?, ?> other) {
-
-		if (this == other) {
-			return true;
-		}
-
-		if (null == other) {
-			return false;
-		}
-
-		if (!stateType.equals(StateDescriptor.Type.UNKNOWN)
-				&& !other.stateType.equals(StateDescriptor.Type.UNKNOWN)
-				&& !stateType.equals(other.stateType)) {
-			return false;
-		}
-
-		if (!name.equals(other.getName())) {
-			return false;
-		}
-
-		return (stateSerializer.canRestoreFrom(other.stateSerializer)) &&
-				(namespaceSerializer.canRestoreFrom(other.namespaceSerializer)
-						// we also check if there is just a migration proxy that should be replaced by any real serializer
-						|| other.namespaceSerializer instanceof MigrationNamespaceSerializerProxy);
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o) {
-			return true;
-		}
-
-		if (o == null || getClass() != o.getClass()) {
-			return false;
-		}
-
-		RegisteredBackendStateMetaInfo<?, ?> that = (RegisteredBackendStateMetaInfo<?, ?>) o;
-
-		if (!stateType.equals(that.stateType)) {
-			return false;
-		}
-
-		if (!getName().equals(that.getName())) {
-			return false;
-		}
-
-		return getStateSerializer().equals(that.getStateSerializer())
-				&& getNamespaceSerializer().equals(that.getNamespaceSerializer());
-	}
-
-	@Override
-	public String toString() {
-		return "RegisteredBackendStateMetaInfo{" +
-				"stateType=" + stateType +
-				", name='" + name + '\'' +
-				", namespaceSerializer=" + namespaceSerializer +
-				", stateSerializer=" + stateSerializer +
-				'}';
-	}
-
-	@Override
-	public int hashCode() {
-		int result = getName().hashCode();
-		result = 31 * result + getStateType().hashCode();
-		result = 31 * result + (getNamespaceSerializer() != null ? getNamespaceSerializer().hashCode() : 0);
-		result = 31 * result + (getStateSerializer() != null ? getStateSerializer().hashCode() : 0);
-		return result;
-	}
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java
new file mode 100644
index 0000000000000..e1a7e067668ed
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.state.StateDescriptor;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.util.Preconditions;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Compound meta information for a registered state in a keyed state backend. This combines all serializers and the
+ * state name.
+ *
+ * @param <N> Type of namespace
+ * @param <S> Type of state value
+ */
+public class RegisteredKeyedBackendStateMetaInfo<N, S> {
+
+	private final StateDescriptor.Type stateType;
+	private final String name;
+	private final TypeSerializer<N> namespaceSerializer;
+	private final TypeSerializer<S> stateSerializer;
+
+	public RegisteredKeyedBackendStateMetaInfo(
+			StateDescriptor.Type stateType,
+			String name,
+			TypeSerializer<N> namespaceSerializer,
+			TypeSerializer<S> stateSerializer) {
+
+		this.stateType = checkNotNull(stateType);
+		this.name = checkNotNull(name);
+		this.namespaceSerializer = checkNotNull(namespaceSerializer);
+		this.stateSerializer = checkNotNull(stateSerializer);
+	}
+
+	public StateDescriptor.Type getStateType() {
+		return stateType;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public TypeSerializer<N> getNamespaceSerializer() {
+		return namespaceSerializer;
+	}
+
+	public TypeSerializer<S> getStateSerializer() {
+		return stateSerializer;
+	}
+
+	public Snapshot<N, S> snapshot() {
+		return new Snapshot<>(
+			stateType,
+			name,
+			namespaceSerializer.duplicate(),
+			stateSerializer.duplicate(),
+			namespaceSerializer.snapshotConfiguration(),
+			stateSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+
+		RegisteredKeyedBackendStateMetaInfo<?, ?> that = (RegisteredKeyedBackendStateMetaInfo<?, ?>) o;
+
+		if (!stateType.equals(that.stateType)) {
+			return false;
+		}
+
+		if (!getName().equals(that.getName())) {
+			return false;
+		}
+
+		return getStateSerializer().equals(that.getStateSerializer())
+				&& getNamespaceSerializer().equals(that.getNamespaceSerializer());
+	}
+
+	@Override
+	public String toString() {
+		return "RegisteredKeyedBackendStateMetaInfo{" +
+				"stateType=" + stateType +
+				", name='" + name + '\'' +
+				", namespaceSerializer=" + namespaceSerializer +
+				", stateSerializer=" + stateSerializer +
+				'}';
+	}
+
+	@Override
+	public int hashCode() {
+		int result = getName().hashCode();
+		result = 31 * result + getStateType().hashCode();
+		result = 31 * result + getNamespaceSerializer().hashCode();
+		result = 31 * result + getStateSerializer().hashCode();
+		return result;
+	}
+
+	/**
+	 * A consistent snapshot of a {@link RegisteredKeyedBackendStateMetaInfo}.
+	 */
+	public static class Snapshot<N, S> {
+
+		private StateDescriptor.Type stateType;
+		private String name;
+		private TypeSerializer<N> namespaceSerializer;
+		private TypeSerializer<S> stateSerializer;
+		private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot;
+		private TypeSerializerConfigSnapshot stateSerializerConfigSnapshot;
+
+		/** Empty constructor used when restoring the state meta info snapshot. */
+		Snapshot() {}
+
+		private Snapshot(
+				StateDescriptor.Type stateType,
+				String name,
+				TypeSerializer<N> namespaceSerializer,
+				TypeSerializer<S> stateSerializer,
+				TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot,
+				TypeSerializerConfigSnapshot stateSerializerConfigSnapshot) {
+
+			this.stateType = Preconditions.checkNotNull(stateType);
+			this.name = Preconditions.checkNotNull(name);
+			this.namespaceSerializer = Preconditions.checkNotNull(namespaceSerializer);
+			this.stateSerializer = Preconditions.checkNotNull(stateSerializer);
+			this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializerConfigSnapshot);
+			this.stateSerializerConfigSnapshot = Preconditions.checkNotNull(stateSerializerConfigSnapshot);
+		}
+
+		public StateDescriptor.Type getStateType() {
+			return stateType;
+		}
+
+		void setStateType(StateDescriptor.Type stateType) {
+			this.stateType = stateType;
+		}
+
+		public String getName() {
+			return name;
+		}
+
+		void setName(String name) {
+			this.name = name;
+		}
+
+		public TypeSerializer<N> getNamespaceSerializer() {
+			return namespaceSerializer;
+		}
+
+		void setNamespaceSerializer(TypeSerializer<N> namespaceSerializer) {
+			this.namespaceSerializer = namespaceSerializer;
+		}
+
+		public TypeSerializer<S> getStateSerializer() {
+			return stateSerializer;
+		}
+
+		void setStateSerializer(TypeSerializer<S> stateSerializer) {
+			this.stateSerializer = stateSerializer;
+		}
+
+		public TypeSerializerConfigSnapshot getNamespaceSerializerConfigSnapshot() {
+			return namespaceSerializerConfigSnapshot;
+		}
+
+		void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) {
+			this.namespaceSerializerConfigSnapshot = namespaceSerializerConfigSnapshot;
+		}
+
+		public TypeSerializerConfigSnapshot getStateSerializerConfigSnapshot() {
+			return stateSerializerConfigSnapshot;
+		}
+
+		void setStateSerializerConfigSnapshot(TypeSerializerConfigSnapshot stateSerializerConfigSnapshot) {
+			this.stateSerializerConfigSnapshot = stateSerializerConfigSnapshot;
+		}
+
+		@Override
+		public boolean equals(Object o) {
+			if (this == o) {
+				return true;
+			}
+
+			if (o == null || getClass() != o.getClass()) {
+				return false;
+			}
+
+			Snapshot<?, ?> that = (Snapshot<?, ?>) o;
+
+			if (!stateType.equals(that.stateType)) {
+				return false;
+			}
+
+			if (!getName().equals(that.getName())) {
+				return false;
+			}
+
+			// need to check for nulls because serializer and config snapshots may be null on restore
+			return
+				((getStateSerializer() == null && that.getStateSerializer() == null)
+					|| getStateSerializer().equals(that.getStateSerializer()))
+				&& ((getNamespaceSerializer() == null && that.getNamespaceSerializer() == null)
+					|| getNamespaceSerializer().equals(that.getNamespaceSerializer()))
+				&& ((getNamespaceSerializerConfigSnapshot() == null && that.getNamespaceSerializerConfigSnapshot() == null)
+					|| getNamespaceSerializerConfigSnapshot().equals(that.getNamespaceSerializerConfigSnapshot()))
+				&& ((getStateSerializerConfigSnapshot() == null && that.getStateSerializerConfigSnapshot() == null)
+					|| getStateSerializerConfigSnapshot().equals(that.getStateSerializerConfigSnapshot()));
+		}
+
+		@Override
+		public int hashCode() {
+			// need to check for nulls because serializer and config snapshots may be null on restore
+			int result = getName().hashCode();
+			result = 31 * result + getStateType().hashCode();
+			result = 31 * result + (getNamespaceSerializer() != null ? getNamespaceSerializer().hashCode() : 0);
+			result = 31 * result + (getStateSerializer() != null ? getStateSerializer().hashCode() : 0);
+			result = 31 * result + (getNamespaceSerializerConfigSnapshot() != null ? getNamespaceSerializerConfigSnapshot().hashCode() : 0);
+			result = 31 * result + (getStateSerializerConfigSnapshot() != null ? getStateSerializerConfigSnapshot().hashCode() : 0);
+			return result;
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java
new file mode 100644
index 0000000000000..b43fc9c537cc5
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * Compound meta information for a registered state in an operator state backend.
+ * This contains the state name, assignment mode, and state partition serializer.
+ *
+ * @param <S> Type of the state.
+ */
+public class RegisteredOperatorBackendStateMetaInfo<S> {
+
+	/**
+	 * The name of the state, as registered by the user
+	 */
+	private final String name;
+
+	/**
+	 * The mode how elements in this state are assigned to tasks during restore
+	 */
+	private final OperatorStateHandle.Mode assignmentMode;
+
+	/**
+	 * The type serializer for the elements in the state list
+	 */
+	private final TypeSerializer<S> partitionStateSerializer;
+
+	public RegisteredOperatorBackendStateMetaInfo(
+			String name,
+			TypeSerializer<S> partitionStateSerializer,
+			OperatorStateHandle.Mode assignmentMode) {
+
+		this.name = Preconditions.checkNotNull(name);
+		this.partitionStateSerializer = Preconditions.checkNotNull(partitionStateSerializer);
+		this.assignmentMode = Preconditions.checkNotNull(assignmentMode);
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public OperatorStateHandle.Mode getAssignmentMode() {
+		return assignmentMode;
+	}
+
+	public TypeSerializer<S> getPartitionStateSerializer() {
+		return partitionStateSerializer;
+	}
+
+	public Snapshot<S> snapshot() {
+		return new Snapshot<>(
+			name,
+			assignmentMode,
+			partitionStateSerializer.duplicate(),
+			partitionStateSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj == this) {
+			return true;
+		}
+
+		if (obj == null) {
+			return false;
+		}
+
+		return (obj instanceof RegisteredOperatorBackendStateMetaInfo)
+			&& name.equals(((RegisteredOperatorBackendStateMetaInfo) obj).getName())
+			&& assignmentMode.equals(((RegisteredOperatorBackendStateMetaInfo) obj).getAssignmentMode())
+			&& partitionStateSerializer.equals(((RegisteredOperatorBackendStateMetaInfo) obj).getPartitionStateSerializer());
+	}
+
+	@Override
+	public int hashCode() {
+		int result = getName().hashCode();
+		result = 31 * result + getAssignmentMode().hashCode();
+		result = 31 * result + getPartitionStateSerializer().hashCode();
+		return result;
+	}
+
+	@Override
+	public String toString() {
+		return "RegisteredOperatorBackendStateMetaInfo{" +
+			"name='" + name + "\'" +
+			", assignmentMode=" + assignmentMode +
+			", partitionStateSerializer=" + partitionStateSerializer +
+			'}';
+	}
+
+	/**
+	 * A consistent snapshot of a {@link RegisteredOperatorBackendStateMetaInfo}.
+	 */
+	public static class Snapshot<S> {
+
+		private String name;
+		private OperatorStateHandle.Mode assignmentMode;
+		private TypeSerializer<S> partitionStateSerializer;
+		private TypeSerializerConfigSnapshot partitionStateSerializerConfigSnapshot;
+
+		/** Empty constructor used when restoring the state meta info snapshot. */
+		Snapshot() {}
+
+		private Snapshot(
+				String name,
+				OperatorStateHandle.Mode assignmentMode,
+				TypeSerializer<S> partitionStateSerializer,
+				TypeSerializerConfigSnapshot partitionStateSerializerConfigSnapshot) {
+
+			this.name = Preconditions.checkNotNull(name);
+			this.assignmentMode = Preconditions.checkNotNull(assignmentMode);
+			this.partitionStateSerializer = Preconditions.checkNotNull(partitionStateSerializer);
+			this.partitionStateSerializerConfigSnapshot = Preconditions.checkNotNull(partitionStateSerializerConfigSnapshot);
+		}
+
+		public String getName() {
+			return name;
+		}
+
+		void setName(String name) {
+			this.name = name;
+		}
+
+		public OperatorStateHandle.Mode getAssignmentMode() {
+			return assignmentMode;
+		}
+
+		void setAssignmentMode(OperatorStateHandle.Mode assignmentMode) {
+			this.assignmentMode = assignmentMode;
+		}
+
+		public TypeSerializer<S> getPartitionStateSerializer() {
+			return partitionStateSerializer;
+		}
+
+		void setPartitionStateSerializer(TypeSerializer<S> partitionStateSerializer) {
+			this.partitionStateSerializer = partitionStateSerializer;
+		}
+
+		public TypeSerializerConfigSnapshot getPartitionStateSerializerConfigSnapshot() {
+			return partitionStateSerializerConfigSnapshot;
+		}
+
+		void setPartitionStateSerializerConfigSnapshot(TypeSerializerConfigSnapshot partitionStateSerializerConfigSnapshot) {
+			this.partitionStateSerializerConfigSnapshot = partitionStateSerializerConfigSnapshot;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (obj == this) {
+				return true;
+			}
+
+			if (obj == null) {
+				return false;
+			}
+
+			// need to check for nulls because serializer and config snapshots may be null on restore
+			return (obj instanceof Snapshot)
+				&& name.equals(((Snapshot) obj).getName())
+				&& assignmentMode.equals(((Snapshot) obj).getAssignmentMode())
+				&& ((partitionStateSerializer == null && ((Snapshot) obj).getPartitionStateSerializer() == null)
+					|| partitionStateSerializer.equals(((Snapshot) obj).getPartitionStateSerializer()))
+				&& ((partitionStateSerializerConfigSnapshot == null && ((Snapshot) obj).getPartitionStateSerializerConfigSnapshot() == null)
+					|| partitionStateSerializerConfigSnapshot.equals(((Snapshot) obj).getPartitionStateSerializerConfigSnapshot()));
+		}
+
+		@Override
+		public int hashCode() {
+			// need to check for nulls because serializer and config snapshots may be null on restore
+			int result = getName().hashCode();
+			result = 31 * result + getAssignmentMode().hashCode();
+			result = 31 * result + (getPartitionStateSerializer() != null ? getPartitionStateSerializer().hashCode() : 0);
+			result = 31 * result + (getPartitionStateSerializerConfigSnapshot() != null ? getPartitionStateSerializerConfigSnapshot().hashCode() : 0);
+			return result;
+		}
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java
new file mode 100644
index 0000000000000..978f28d15291d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+
+/**
+ * Utilities related to state migration, commonly used in the state backends.
+ */
+public class StateMigrationUtil {
+
+	/**
+	 * Resolves the final compatibility result of two serializers by taking into account compound information,
+	 * including the preceding serializer, the preceding serializer's configuration snapshot, and the new serializer.
+	 *
+	 * The final result is determined as follows:
+	 *   1. If there is no configuration snapshot of the preceding serializer,
+	 *      assumes the new serializer to be compatible.
+	 *   2. Confront the configuration snapshot with the new serializer.
+	 *   3. If the result is compatible, just return that as the result.
+	 *   4. If not compatible and requires migration, check if the preceding serializer is valid.
+	 *      If yes, use that as the convert deserializer for state migration.
+	 *   5. If the preceding serializer is not valid, check if the result came with a convert deserializer.
+	 *      If yes, use that for state migration and simply return the result.
+	 *   6. If all of above fails, state migration is required but could not be performed; throw exception.
+	 *
+	 * @param precedingSerializer the preceding serializer used to write the data
+	 * @param dummySerializerClassTag any class tags that identifies the preceding serializer as a dummy placeholder
+	 * @param precedingSerializerConfigSnapshot configuration snapshot of the preceding serializer
+	 * @param newSerializer the new serializer to ensure compatibility with
+	 *
+	 * @param <T> Type of the data handled by the serializers
+	 *
+	 * @return the final resolved compatiblity result
+	 */
+	public static <T> CompatibilityResult<T> resolveCompatibilityResult(
+			TypeSerializer<T> precedingSerializer,
+			Class<?> dummySerializerClassTag,
+			TypeSerializerConfigSnapshot precedingSerializerConfigSnapshot,
+			TypeSerializer<T> newSerializer) {
+
+		if (precedingSerializerConfigSnapshot != null) {
+			CompatibilityResult<T> initialResult = newSerializer.ensureCompatibility(precedingSerializerConfigSnapshot);
+
+			if (!initialResult.requiresMigration()) {
+				return initialResult;
+			} else {
+				if (precedingSerializer != null && !(precedingSerializer.getClass().equals(dummySerializerClassTag))) {
+					// if the preceding serializer exists and is not a dummy, use
+					// that for converting instead of the provided convert deserializer
+					return CompatibilityResult.requiresMigration(precedingSerializer);
+				} else if (initialResult.getConvertDeserializer() != null) {
+					return initialResult;
+				} else {
+					throw new RuntimeException(
+						"State migration required, but there is no available serializer capable of reading previous data.");
+				}
+			}
+		} else {
+			// if the configuration snapshot of the preceding serializer cannot be provided,
+			// we can only simply assume that the new serializer is compatible
+			return CompatibilityResult.compatible();
+		}
+	}
+
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/VoidNamespaceSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/VoidNamespaceSerializer.java
index 8b58891e36ba6..2800899280a21 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/VoidNamespaceSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/VoidNamespaceSerializer.java
@@ -21,6 +21,7 @@
 import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.migration.MigrationNamespaceSerializerProxy;
 
 import java.io.IOException;
 
@@ -89,4 +90,11 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 	public boolean canEqual(Object obj) {
 		return obj instanceof VoidNamespaceSerializer;
 	}
+
+	@Override
+	protected boolean isCompatibleSerializationFormatIdentifier(String identifier) {
+		// we might be replacing a migration namespace serializer, in which case we just assume compatibility
+		return super.isCompatibleSerializationFormatIdentifier(identifier)
+			|| identifier.equals(MigrationNamespaceSerializerProxy.class.getCanonicalName());
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTable.java
index d63b6d3d06acf..7b61da10138ca 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTable.java
@@ -20,7 +20,7 @@
 
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StateTransformationFunction;
 import org.apache.flink.util.MathUtils;
 import org.apache.flink.util.Preconditions;
@@ -196,7 +196,7 @@ public class CopyOnWriteStateTable<K, N, S> extends StateTable<K, N, S> implemen
 	 * @param keyContext the key context.
 	 * @param metaInfo   the meta information, including the type serializer for state copy-on-write.
 	 */
-	CopyOnWriteStateTable(InternalKeyContext<K> keyContext, RegisteredBackendStateMetaInfo<N, S> metaInfo) {
+	CopyOnWriteStateTable(InternalKeyContext<K> keyContext, RegisteredKeyedBackendStateMetaInfo<N, S> metaInfo) {
 		this(keyContext, metaInfo, 1024);
 	}
 
@@ -209,7 +209,7 @@ public class CopyOnWriteStateTable<K, N, S> extends StateTable<K, N, S> implemen
 	 * @throws IllegalArgumentException when the capacity is less than zero.
 	 */
 	@SuppressWarnings("unchecked")
-	private CopyOnWriteStateTable(InternalKeyContext<K> keyContext, RegisteredBackendStateMetaInfo<N, S> metaInfo, int capacity) {
+	private CopyOnWriteStateTable(InternalKeyContext<K> keyContext, RegisteredKeyedBackendStateMetaInfo<N, S> metaInfo, int capacity) {
 		super(keyContext, metaInfo);
 
 		// initialized tables to EMPTY_TABLE.
@@ -532,12 +532,12 @@ public TypeSerializer<N> getNamespaceSerializer() {
 	}
 
 	@Override
-	public RegisteredBackendStateMetaInfo<N, S> getMetaInfo() {
+	public RegisteredKeyedBackendStateMetaInfo<N, S> getMetaInfo() {
 		return metaInfo;
 	}
 
 	@Override
-	public void setMetaInfo(RegisteredBackendStateMetaInfo<N, S> metaInfo) {
+	public void setMetaInfo(RegisteredKeyedBackendStateMetaInfo<N, S> metaInfo) {
 		this.metaInfo = metaInfo;
 	}
 
@@ -1063,4 +1063,4 @@ public void remove() {
 			throw new UnsupportedOperationException("Read-only iterator");
 		}
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index aecc72e3bf869..866ed2830cd26 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -52,7 +52,7 @@
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
 import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.internal.InternalAggregatingState;
 import org.apache.flink.runtime.state.internal.InternalFoldingState;
@@ -132,8 +132,8 @@ private <N, V> StateTable<K, N, V> tryRegisterStateTable(
 			TypeSerializer<N> namespaceSerializer,
 			TypeSerializer<V> valueSerializer) {
 
-		final RegisteredBackendStateMetaInfo<N, V> newMetaInfo =
-				new RegisteredBackendStateMetaInfo<>(stateType, stateName, namespaceSerializer, valueSerializer);
+		final RegisteredKeyedBackendStateMetaInfo<N, V> newMetaInfo =
+				new RegisteredKeyedBackendStateMetaInfo<>(stateType, stateName, namespaceSerializer, valueSerializer);
 
 		@SuppressWarnings("unchecked")
 		StateTable<K, N, V> stateTable = (StateTable<K, N, V>) stateTables.get(stateName);
@@ -142,12 +142,27 @@ private <N, V> StateTable<K, N, V> tryRegisterStateTable(
 			stateTable = newStateTable(newMetaInfo);
 			stateTables.put(stateName, stateTable);
 		} else {
-			if (!newMetaInfo.canRestoreFrom(stateTable.getMetaInfo())) {
-				throw new RuntimeException("Trying to access state using incompatible meta info, was " +
-						stateTable.getMetaInfo() + " trying access with " + newMetaInfo);
+			// TODO with eager registration in place, these checks should be moved to restorePartitionedState()
+
+			Preconditions.checkState(
+				stateName.equals(stateTable.getMetaInfo().getName()),
+				"Incompatible state names. " +
+					"Was [" + stateTable.getMetaInfo().getName() + "], " +
+					"registered with [" + newMetaInfo.getName() + "].");
+
+			if (!newMetaInfo.getStateType().equals(StateDescriptor.Type.UNKNOWN)
+					&& !stateTable.getMetaInfo().getStateType().equals(StateDescriptor.Type.UNKNOWN)) {
+
+				Preconditions.checkState(
+					newMetaInfo.getStateType().equals(stateTable.getMetaInfo().getStateType()),
+					"Incompatible state types. " +
+						"Was [" + stateTable.getMetaInfo().getStateType() + "], " +
+						"registered with [" + newMetaInfo.getStateType() + "].");
 			}
+
 			stateTable.setMetaInfo(newMetaInfo);
 		}
+
 		return stateTable;
 	}
 
@@ -240,21 +255,14 @@ public  RunnableFuture<KeyedStateHandle> snapshot(
 				"Too many KV-States: " + stateTables.size() +
 						". Currently at most " + Short.MAX_VALUE + " states are supported");
 
-		List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> metaInfoProxyList = new ArrayList<>(stateTables.size());
+		List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> metaInfoSnapshots = new ArrayList<>(stateTables.size());
 
 		final Map<String, Integer> kVStateToId = new HashMap<>(stateTables.size());
 
 		final Map<StateTable<K, ?, ?>, StateTableSnapshot> cowStateStableSnapshots = new HashedMap(stateTables.size());
 
 		for (Map.Entry<String, StateTable<K, ?, ?>> kvState : stateTables.entrySet()) {
-			RegisteredBackendStateMetaInfo<?, ?> metaInfo = kvState.getValue().getMetaInfo();
-			KeyedBackendSerializationProxy.StateMetaInfo<?, ?> metaInfoProxy = new KeyedBackendSerializationProxy.StateMetaInfo(
-					metaInfo.getStateType(),
-					metaInfo.getName(),
-					metaInfo.getNamespaceSerializer(),
-					metaInfo.getStateSerializer());
-
-			metaInfoProxyList.add(metaInfoProxy);
+			metaInfoSnapshots.add(kvState.getValue().getMetaInfo().snapshot());
 			kVStateToId.put(kvState.getKey(), kVStateToId.size());
 			StateTable<K, ?, ?> stateTable = kvState.getValue();
 			if (null != stateTable) {
@@ -263,7 +271,7 @@ public  RunnableFuture<KeyedStateHandle> snapshot(
 		}
 
 		final KeyedBackendSerializationProxy serializationProxy =
-				new KeyedBackendSerializationProxy(keySerializer, metaInfoProxyList);
+				new KeyedBackendSerializationProxy(keySerializer, metaInfoSnapshots);
 
 		//--------------------------------------------------- this becomes the end of sync part
 
@@ -376,23 +384,29 @@ private void restorePartitionedState(Collection<KeyedStateHandle> state) throws
 
 				serializationProxy.read(inView);
 
-				List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> metaInfoList =
-						serializationProxy.getNamedStateSerializationProxies();
+				List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> restoredMetaInfos =
+						serializationProxy.getStateMetaInfoSnapshots();
 
-				for (KeyedBackendSerializationProxy.StateMetaInfo<?, ?> metaInfoSerializationProxy : metaInfoList) {
+				for (RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> restoredMetaInfo : restoredMetaInfos) {
 
-					StateTable<K, ?, ?> stateTable = stateTables.get(metaInfoSerializationProxy.getStateName());
+					StateTable<K, ?, ?> stateTable = stateTables.get(restoredMetaInfo.getName());
 
 					//important: only create a new table we did not already create it previously
 					if (null == stateTable) {
 
-						RegisteredBackendStateMetaInfo<?, ?> registeredBackendStateMetaInfo =
-								new RegisteredBackendStateMetaInfo<>(metaInfoSerializationProxy);
+						RegisteredKeyedBackendStateMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
+								new RegisteredKeyedBackendStateMetaInfo<>(
+									restoredMetaInfo.getStateType(),
+									restoredMetaInfo.getName(),
+									restoredMetaInfo.getNamespaceSerializer(),
+									restoredMetaInfo.getStateSerializer());
 
-						stateTable = newStateTable(registeredBackendStateMetaInfo);
-						stateTables.put(metaInfoSerializationProxy.getStateName(), stateTable);
-						kvStatesById.put(numRegisteredKvStates, metaInfoSerializationProxy.getStateName());
+						stateTable = newStateTable(registeredKeyedBackendStateMetaInfo);
+						stateTables.put(restoredMetaInfo.getName(), stateTable);
+						kvStatesById.put(numRegisteredKvStates, restoredMetaInfo.getName());
 						++numRegisteredKvStates;
+					} else {
+						// TODO with eager state registration in place, check here for serializer migration strategies
 					}
 				}
 
@@ -410,7 +424,7 @@ private void restorePartitionedState(Collection<KeyedStateHandle> state) throws
 					Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
 							"Unexpected key-group in restore.");
 
-					for (int i = 0; i < metaInfoList.size(); i++) {
+					for (int i = 0; i < restoredMetaInfos.size(); i++) {
 						int kvStateId = inView.readShort();
 						StateTable<K, ?, ?> stateTable = stateTables.get(kvStatesById.get(kvStateId));
 
@@ -509,7 +523,7 @@ public int numStateEntries(Object namespace) {
 		return sum;
 	}
 
-	public <N, V> StateTable<K, N, V> newStateTable(RegisteredBackendStateMetaInfo<N, V> newMetaInfo) {
+	public <N, V> StateTable<K, N, V> newStateTable(RegisteredKeyedBackendStateMetaInfo<N, V> newMetaInfo) {
 		return asynchronousSnapshots ?
 				new CopyOnWriteStateTable<>(this, newMetaInfo) :
 				new NestedMapsStateTable<>(this, newMetaInfo);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/NestedMapsStateTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/NestedMapsStateTable.java
index 22f344d7c618c..75c31db0f7b39 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/NestedMapsStateTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/NestedMapsStateTable.java
@@ -22,7 +22,7 @@
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StateTransformationFunction;
 import org.apache.flink.util.Preconditions;
 
@@ -63,7 +63,7 @@ public class NestedMapsStateTable<K, N, S> extends StateTable<K, N, S> {
 	 * @param keyContext the key context.
 	 * @param metaInfo the meta information for this state table.
 	 */
-	public NestedMapsStateTable(InternalKeyContext<K> keyContext, RegisteredBackendStateMetaInfo<N, S> metaInfo) {
+	public NestedMapsStateTable(InternalKeyContext<K> keyContext, RegisteredKeyedBackendStateMetaInfo<N, S> metaInfo) {
 		super(keyContext, metaInfo);
 		this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup();
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java
index 62fc869736439..c1cdcc39fb659 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java
@@ -20,7 +20,7 @@
 
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StateTransformationFunction;
 import org.apache.flink.util.Preconditions;
 
@@ -42,14 +42,14 @@ public abstract class StateTable<K, N, S> {
 	/**
 	 * Combined meta information such as name and serializers for this state
 	 */
-	protected RegisteredBackendStateMetaInfo<N, S> metaInfo;
+	protected RegisteredKeyedBackendStateMetaInfo<N, S> metaInfo;
 
 	/**
 	 *
 	 * @param keyContext the key context provides the key scope for all put/get/delete operations.
 	 * @param metaInfo the meta information, including the type serializer for state copy-on-write.
 	 */
-	public StateTable(InternalKeyContext<K> keyContext, RegisteredBackendStateMetaInfo<N, S> metaInfo) {
+	public StateTable(InternalKeyContext<K> keyContext, RegisteredKeyedBackendStateMetaInfo<N, S> metaInfo) {
 		this.keyContext = Preconditions.checkNotNull(keyContext);
 		this.metaInfo = Preconditions.checkNotNull(metaInfo);
 	}
@@ -168,11 +168,11 @@ public TypeSerializer<N> getNamespaceSerializer() {
 		return metaInfo.getNamespaceSerializer();
 	}
 
-	public RegisteredBackendStateMetaInfo<N, S> getMetaInfo() {
+	public RegisteredKeyedBackendStateMetaInfo<N, S> getMetaInfo() {
 		return metaInfo;
 	}
 
-	public void setMetaInfo(RegisteredBackendStateMetaInfo<N, S> metaInfo) {
+	public void setMetaInfo(RegisteredKeyedBackendStateMetaInfo<N, S> metaInfo) {
 		this.metaInfo = metaInfo;
 	}
 
@@ -186,4 +186,4 @@ public void setMetaInfo(RegisteredBackendStateMetaInfo<N, S> metaInfo) {
 
 	@VisibleForTesting
 	public abstract int sizeOfNamespace(Object namespace);
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTableByKeyGroupReaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTableByKeyGroupReaders.java
index 53ec3498129d2..d7bc94e27c607 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTableByKeyGroupReaders.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTableByKeyGroupReaders.java
@@ -47,7 +47,8 @@ static <K, N, S> StateTableByKeyGroupReader readerForVersion(StateTable<K, N, S>
 			case 1:
 				return new StateTableByKeyGroupReaderV1<>(table);
 			case 2:
-				return new StateTableByKeyGroupReaderV2<>(table);
+			case 3:
+				return new StateTableByKeyGroupReaderV2V3<>(table);
 			default:
 				throw new IllegalArgumentException("Unknown version: " + version);
 		}
@@ -110,10 +111,10 @@ public void readMappingsInKeyGroup(DataInputView inView, int keyGroupId) throws
 		}
 	}
 
-	private static final class StateTableByKeyGroupReaderV2<K, N, S>
+	private static final class StateTableByKeyGroupReaderV2V3<K, N, S>
 			extends AbstractStateTableByKeyGroupReader<K, N, S> {
 
-		StateTableByKeyGroupReaderV2(StateTable<K, N, S> stateTable) {
+		StateTableByKeyGroupReaderV2V3(StateTable<K, N, S> stateTable) {
 			super(stateTable);
 		}
 
@@ -133,4 +134,4 @@ public void readMappingsInKeyGroup(DataInputView inView, int keyGroupId) throws
 			}
 		}
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java
index e872526b5f593..4bdc5e85cdd31 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java
@@ -21,7 +21,9 @@
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -127,4 +129,14 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return IntListSerializer.class.hashCode();
 	}
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public CompatibilityResult<IntList> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		throw new UnsupportedOperationException();
+	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java
index e4a926440df8c..0ae5e71d42803 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java
@@ -21,7 +21,9 @@
 
 import java.io.IOException;
 
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.memory.DataInputView;
@@ -136,4 +138,14 @@ public boolean equals(Object obj) {
 			return obj.getClass() == IntPairSerializerFactory.class;
 		};
 	}
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public CompatibilityResult<IntPair> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		throw new UnsupportedOperationException();
+	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java
index b62b097c1cf43..17ee5f1530b4a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java
@@ -20,7 +20,9 @@
 
 import java.io.IOException;
 
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.StringValue;
@@ -104,4 +106,14 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return StringPairSerializer.class.hashCode();
 	}
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public CompatibilityResult<StringPair> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		throw new UnsupportedOperationException();
+	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/query/QueryableStateClientTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/query/QueryableStateClientTest.java
index b4d6eb7230bad..8c4e049eac473 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/query/QueryableStateClientTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/query/QueryableStateClientTest.java
@@ -34,7 +34,7 @@
 import org.apache.flink.runtime.query.netty.message.KvStateRequestSerializer;
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.VoidNamespace;
 import org.apache.flink.runtime.state.VoidNamespaceSerializer;
 import org.apache.flink.runtime.state.heap.HeapValueState;
@@ -270,7 +270,7 @@ public void testIntegrationWithKvStateServer() throws Exception {
 				ValueStateDescriptor<Integer> descriptor =
 						new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
 
-				RegisteredBackendStateMetaInfo<VoidNamespace, Integer> registeredBackendStateMetaInfo = new RegisteredBackendStateMetaInfo<>(
+				RegisteredKeyedBackendStateMetaInfo<VoidNamespace, Integer> registeredKeyedBackendStateMetaInfo = new RegisteredKeyedBackendStateMetaInfo<>(
 						descriptor.getType(),
 						descriptor.getName(),
 						VoidNamespaceSerializer.INSTANCE,
@@ -279,7 +279,7 @@ public void testIntegrationWithKvStateServer() throws Exception {
 				// Register state
 				HeapValueState<Integer, VoidNamespace, Integer> kvState = new HeapValueState<>(
 						descriptor,
-						new NestedMapsStateTable<Integer, VoidNamespace, Integer>(keyedStateBackend, registeredBackendStateMetaInfo),
+						new NestedMapsStateTable<Integer, VoidNamespace, Integer>(keyedStateBackend, registeredKeyedBackendStateMetaInfo),
 						IntSerializer.INSTANCE,
 						VoidNamespaceSerializer.INSTANCE);
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
index c04ed8cf39562..50ca159420dce 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
@@ -97,7 +97,7 @@ public void testRegisterStatesWithoutTypeSerializer() throws Exception {
 		assertEquals(2, operatorStateBackend.getRegisteredStateNames().size());
 
 		// make sure that type registrations are forwarded
-		TypeSerializer<?> serializer = ((PartitionableListState<?>) listState).getPartitionStateSerializer();
+		TypeSerializer<?> serializer = ((PartitionableListState<?>) listState).getStateMetaInfo().getPartitionStateSerializer();
 		assertTrue(serializer instanceof KryoSerializer);
 		assertTrue(((KryoSerializer<?>) serializer).getKryo().getSerializer(registeredType)
 				instanceof com.esotericsoftware.kryo.serializers.JavaSerializer);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java
index 0dbe2eb15855d..02b4d6216d909 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.api.common.state.StateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
 import org.apache.flink.api.common.typeutils.base.DoubleSerializer;
 import org.apache.flink.api.common.typeutils.base.IntSerializer;
 import org.apache.flink.api.common.typeutils.base.LongSerializer;
@@ -29,10 +30,21 @@
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({KeyedBackendStateMetaInfoSnapshotReaderWriters.class, OperatorBackendStateMetaInfoSnapshotReaderWriters.class})
 public class SerializationProxiesTest {
 
 	@Test
@@ -42,14 +54,14 @@ public void testKeyedBackendSerializationProxyRoundtrip() throws Exception {
 		TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
 		TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
 
-		List<KeyedBackendSerializationProxy.StateMetaInfo<?, ?>> stateMetaInfoList = new ArrayList<>();
+		List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoList = new ArrayList<>();
 
-		stateMetaInfoList.add(
-				new KeyedBackendSerializationProxy.StateMetaInfo<>(StateDescriptor.Type.VALUE, "a", namespaceSerializer, stateSerializer));
-		stateMetaInfoList.add(
-				new KeyedBackendSerializationProxy.StateMetaInfo<>(StateDescriptor.Type.VALUE, "b", namespaceSerializer, stateSerializer));
-		stateMetaInfoList.add(
-				new KeyedBackendSerializationProxy.StateMetaInfo<>(StateDescriptor.Type.VALUE, "c", namespaceSerializer, stateSerializer));
+		stateMetaInfoList.add(new RegisteredKeyedBackendStateMetaInfo<>(
+			StateDescriptor.Type.VALUE, "a", namespaceSerializer, stateSerializer).snapshot());
+		stateMetaInfoList.add(new RegisteredKeyedBackendStateMetaInfo<>(
+			StateDescriptor.Type.VALUE, "b", namespaceSerializer, stateSerializer).snapshot());
+		stateMetaInfoList.add(new RegisteredKeyedBackendStateMetaInfo<>(
+			StateDescriptor.Type.VALUE, "c", namespaceSerializer, stateSerializer).snapshot());
 
 		KeyedBackendSerializationProxy serializationProxy =
 				new KeyedBackendSerializationProxy(keySerializer, stateMetaInfoList);
@@ -67,8 +79,8 @@ public void testKeyedBackendSerializationProxyRoundtrip() throws Exception {
 			serializationProxy.read(new DataInputViewStreamWrapper(in));
 		}
 
-		Assert.assertEquals(keySerializer, serializationProxy.getKeySerializerProxy().getTypeSerializer());
-		Assert.assertEquals(stateMetaInfoList, serializationProxy.getNamedStateSerializationProxies());
+		Assert.assertEquals(keySerializer, serializationProxy.getKeySerializer());
+		Assert.assertEquals(stateMetaInfoList, serializationProxy.getStateMetaInfoSnapshots());
 	}
 
 	@Test
@@ -78,41 +90,79 @@ public void testKeyedStateMetaInfoSerialization() throws Exception {
 		TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
 		TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
 
-		KeyedBackendSerializationProxy.StateMetaInfo<?, ?> metaInfo =
-				new KeyedBackendSerializationProxy.StateMetaInfo<>(StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer);
+		RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> metaInfo = new RegisteredKeyedBackendStateMetaInfo<>(
+			StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();
 
 		byte[] serialized;
 		try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
-			metaInfo.write(new DataOutputViewStreamWrapper(out));
+			KeyedBackendStateMetaInfoSnapshotReaderWriters
+				.getWriterForVersion(KeyedBackendSerializationProxy.VERSION, metaInfo)
+				.writeStateMetaInfo(new DataOutputViewStreamWrapper(out));
+
 			serialized = out.toByteArray();
 		}
 
-		metaInfo = new KeyedBackendSerializationProxy.StateMetaInfo<>(Thread.currentThread().getContextClassLoader());
-
 		try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
-			metaInfo.read(new DataInputViewStreamWrapper(in));
+			metaInfo = KeyedBackendStateMetaInfoSnapshotReaderWriters
+				.getReaderForVersion(KeyedBackendSerializationProxy.VERSION, Thread.currentThread().getContextClassLoader())
+				.readStateMetaInfo(new DataInputViewStreamWrapper(in));
 		}
 
-		Assert.assertEquals(name, metaInfo.getStateName());
+		Assert.assertEquals(name, metaInfo.getName());
 	}
 
+	@Test
+	public void testKeyedStateMetaInfoReadSerializerFailureResilience() throws Exception {
+		String name = "test";
+		TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
+		TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
+
+		RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> metaInfo = new RegisteredKeyedBackendStateMetaInfo<>(
+			StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();
+
+		byte[] serialized;
+		try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
+			KeyedBackendStateMetaInfoSnapshotReaderWriters
+				.getWriterForVersion(KeyedBackendSerializationProxy.VERSION, metaInfo)
+				.writeStateMetaInfo(new DataOutputViewStreamWrapper(out));
+
+			serialized = out.toByteArray();
+		}
+
+		// mock failure when deserializing serializer
+		TypeSerializerSerializationProxy<?> mockProxy = mock(TypeSerializerSerializationProxy.class);
+		doThrow(new IOException()).when(mockProxy).read(any(DataInputViewStreamWrapper.class));
+		PowerMockito.whenNew(TypeSerializerSerializationProxy.class).withAnyArguments().thenReturn(mockProxy);
+
+		try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+			metaInfo = KeyedBackendStateMetaInfoSnapshotReaderWriters
+				.getReaderForVersion(KeyedBackendSerializationProxy.VERSION, Thread.currentThread().getContextClassLoader())
+				.readStateMetaInfo(new DataInputViewStreamWrapper(in));
+		}
+
+		Assert.assertEquals(name, metaInfo.getName());
+		Assert.assertEquals(null, metaInfo.getNamespaceSerializer());
+		Assert.assertEquals(null, metaInfo.getStateSerializer());
+		Assert.assertEquals(namespaceSerializer.snapshotConfiguration(), metaInfo.getNamespaceSerializerConfigSnapshot());
+		Assert.assertEquals(stateSerializer.snapshotConfiguration(), metaInfo.getStateSerializerConfigSnapshot());
+	}
 
 	@Test
 	public void testOperatorBackendSerializationProxyRoundtrip() throws Exception {
 
 		TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
 
-		List<OperatorBackendSerializationProxy.StateMetaInfo<?>> stateMetaInfoList = new ArrayList<>();
+		List<RegisteredOperatorBackendStateMetaInfo.Snapshot<?>> stateMetaInfoSnapshots = new ArrayList<>();
 
-		stateMetaInfoList.add(
-				new OperatorBackendSerializationProxy.StateMetaInfo<>("a", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
-		stateMetaInfoList.add(
-				new OperatorBackendSerializationProxy.StateMetaInfo<>("b", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
-		stateMetaInfoList.add(
-				new OperatorBackendSerializationProxy.StateMetaInfo<>("c", stateSerializer, OperatorStateHandle.Mode.BROADCAST));
+		stateMetaInfoSnapshots.add(new RegisteredOperatorBackendStateMetaInfo<>(
+			"a", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE).snapshot());
+		stateMetaInfoSnapshots.add(new RegisteredOperatorBackendStateMetaInfo<>(
+			"b", stateSerializer, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE).snapshot());
+		stateMetaInfoSnapshots.add(new RegisteredOperatorBackendStateMetaInfo<>(
+			"c", stateSerializer, OperatorStateHandle.Mode.BROADCAST).snapshot());
 
 		OperatorBackendSerializationProxy serializationProxy =
-				new OperatorBackendSerializationProxy(stateMetaInfoList);
+				new OperatorBackendSerializationProxy(stateMetaInfoSnapshots);
 
 		byte[] serialized;
 		try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
@@ -127,7 +177,7 @@ public void testOperatorBackendSerializationProxyRoundtrip() throws Exception {
 			serializationProxy.read(new DataInputViewStreamWrapper(in));
 		}
 
-		Assert.assertEquals(stateMetaInfoList, serializationProxy.getNamedStateSerializationProxies());
+		Assert.assertEquals(stateMetaInfoSnapshots, serializationProxy.getStateMetaInfoSnapshots());
 	}
 
 	@Test
@@ -136,22 +186,60 @@ public void testOperatorStateMetaInfoSerialization() throws Exception {
 		String name = "test";
 		TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
 
-		OperatorBackendSerializationProxy.StateMetaInfo<?> metaInfo =
-				new OperatorBackendSerializationProxy.StateMetaInfo<>(name, stateSerializer, OperatorStateHandle.Mode.BROADCAST);
+		RegisteredOperatorBackendStateMetaInfo.Snapshot<?> metaInfo =
+			new RegisteredOperatorBackendStateMetaInfo<>(
+				name, stateSerializer, OperatorStateHandle.Mode.BROADCAST).snapshot();
+
+		byte[] serialized;
+		try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
+			OperatorBackendStateMetaInfoSnapshotReaderWriters
+				.getWriterForVersion(OperatorBackendSerializationProxy.VERSION, metaInfo)
+				.writeStateMetaInfo(new DataOutputViewStreamWrapper(out));
+
+			serialized = out.toByteArray();
+		}
+
+		try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+			metaInfo = OperatorBackendStateMetaInfoSnapshotReaderWriters
+				.getReaderForVersion(OperatorBackendSerializationProxy.VERSION, Thread.currentThread().getContextClassLoader())
+				.readStateMetaInfo(new DataInputViewStreamWrapper(in));
+		}
+
+		Assert.assertEquals(name, metaInfo.getName());
+	}
+
+	@Test
+	public void testOperatorStateMetaInfoReadSerializerFailureResilience() throws Exception {
+		String name = "test";
+		TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;
+
+		RegisteredOperatorBackendStateMetaInfo.Snapshot<?> metaInfo =
+			new RegisteredOperatorBackendStateMetaInfo<>(
+				name, stateSerializer, OperatorStateHandle.Mode.BROADCAST).snapshot();
 
 		byte[] serialized;
 		try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
-			metaInfo.write(new DataOutputViewStreamWrapper(out));
+			OperatorBackendStateMetaInfoSnapshotReaderWriters
+				.getWriterForVersion(OperatorBackendSerializationProxy.VERSION, metaInfo)
+				.writeStateMetaInfo(new DataOutputViewStreamWrapper(out));
+
 			serialized = out.toByteArray();
 		}
 
-		metaInfo = new OperatorBackendSerializationProxy.StateMetaInfo<>(Thread.currentThread().getContextClassLoader());
+		// mock failure when deserializing serializer
+		TypeSerializerSerializationProxy<?> mockProxy = mock(TypeSerializerSerializationProxy.class);
+		doThrow(new IOException()).when(mockProxy).read(any(DataInputViewStreamWrapper.class));
+		PowerMockito.whenNew(TypeSerializerSerializationProxy.class).withAnyArguments().thenReturn(mockProxy);
 
 		try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
-			metaInfo.read(new DataInputViewStreamWrapper(in));
+			metaInfo = OperatorBackendStateMetaInfoSnapshotReaderWriters
+				.getReaderForVersion(OperatorBackendSerializationProxy.VERSION, Thread.currentThread().getContextClassLoader())
+				.readStateMetaInfo(new DataInputViewStreamWrapper(in));
 		}
 
 		Assert.assertEquals(name, metaInfo.getName());
+		Assert.assertEquals(null, metaInfo.getPartitionStateSerializer());
+		Assert.assertEquals(stateSerializer.snapshotConfiguration(), metaInfo.getPartitionStateSerializerConfigSnapshot());
 	}
 
 	/**
@@ -171,4 +259,4 @@ public void testFixTypeOrder() {
 		Assert.assertEquals(5, StateDescriptor.Type.AGGREGATING.ordinal());
 		Assert.assertEquals(6, StateDescriptor.Type.MAP.ordinal());
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
index 7152bfc9b8809..96025fe16552b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
@@ -43,6 +43,8 @@
 import org.apache.flink.api.common.typeutils.base.LongSerializer;
 import org.apache.flink.api.common.typeutils.base.StringSerializer;
 import org.apache.flink.api.java.typeutils.GenericTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.api.java.typeutils.runtime.PojoSerializer;
 import org.apache.flink.api.java.typeutils.runtime.kryo.JavaSerializer;
 import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
@@ -71,7 +73,6 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -642,6 +643,139 @@ public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throw
 		state.value();
 	}
 
+	@Test
+	public void testKryoRestoreResilienceWithDifferentRegistrationOrder() throws Exception {
+		CheckpointStreamFactory streamFactory = createStreamFactory();
+		Environment env = new DummyEnvironment("test", 1, 0);
+
+		// register A first then B
+		env.getExecutionConfig().registerKryoType(TestNestedPojoClassA.class);
+		env.getExecutionConfig().registerKryoType(TestNestedPojoClassB.class);
+
+		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
+
+		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
+
+		// make sure that we are in fact using the KryoSerializer
+		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
+
+		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
+		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
+
+		// ============== create snapshot of current configuration ==============
+
+		// make some more modifications
+		backend.setCurrentKey(1);
+		state.update(new TestPojo("u1", 1, new TestNestedPojoClassA(1.0, 2), new TestNestedPojoClassB(2.3, "foo")));
+
+		backend.setCurrentKey(2);
+		state.update(new TestPojo("u2", 2, new TestNestedPojoClassA(2.0, 5), new TestNestedPojoClassB(3.1, "bar")));
+
+		KeyedStateHandle snapshot = runSnapshot(backend.snapshot(
+			682375462378L,
+			2,
+			streamFactory,
+			CheckpointOptions.forFullCheckpoint()));
+
+		backend.dispose();
+
+		// ========== restore snapshot, with a different registration order in the configuration ==========
+
+		env = new DummyEnvironment("test", 1, 0);
+
+		env.getExecutionConfig().registerKryoType(TestNestedPojoClassB.class); // this time register B first
+		env.getExecutionConfig().registerKryoType(TestNestedPojoClassA.class);
+
+		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
+
+		snapshot.discardState();
+
+		// re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
+		// initializeSerializerUnlessSet would not pick up our new config
+		kvId = new ValueStateDescriptor<>("id", pojoType);
+		state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
+
+		backend.setCurrentKey(1);
+
+		// update to test state backends that eagerly serialize, such as RocksDB
+		state.update(new TestPojo("u1", 11, new TestNestedPojoClassA(22.1, 12), new TestNestedPojoClassB(1.23, "foobar")));
+
+		// this tests backends that lazily serialize, such as memory state backend
+		runSnapshot(backend.snapshot(
+			682375462378L,
+			2,
+			streamFactory,
+			CheckpointOptions.forFullCheckpoint()));
+
+		backend.dispose();
+	}
+
+	@Test
+	public void testPojoRestoreResilienceWithDifferentRegistrationOrder() throws Exception {
+		CheckpointStreamFactory streamFactory = createStreamFactory();
+		Environment env = new DummyEnvironment("test", 1, 0);
+
+		// register A first then B
+		env.getExecutionConfig().registerPojoType(TestNestedPojoClassA.class);
+		env.getExecutionConfig().registerPojoType(TestNestedPojoClassB.class);
+
+		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
+
+		TypeInformation<TestPojo> pojoType = TypeExtractor.getForClass(TestPojo.class);
+
+		// make sure that we are in fact using the PojoSerializer
+		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof PojoSerializer);
+
+		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
+		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
+
+		// ============== create snapshot of current configuration ==============
+
+		// make some more modifications
+		backend.setCurrentKey(1);
+		state.update(new TestPojo("u1", 1, new TestNestedPojoClassA(1.0, 2), new TestNestedPojoClassB(2.3, "foo")));
+
+		backend.setCurrentKey(2);
+		state.update(new TestPojo("u2", 2, new TestNestedPojoClassA(2.0, 5), new TestNestedPojoClassB(3.1, "bar")));
+
+		KeyedStateHandle snapshot = runSnapshot(backend.snapshot(
+			682375462378L,
+			2,
+			streamFactory,
+			CheckpointOptions.forFullCheckpoint()));
+
+		backend.dispose();
+
+		// ========== restore snapshot, with a different registration order in the configuration ==========
+
+		env = new DummyEnvironment("test", 1, 0);
+
+		env.getExecutionConfig().registerPojoType(TestNestedPojoClassB.class); // this time register B first
+		env.getExecutionConfig().registerPojoType(TestNestedPojoClassA.class);
+
+		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
+
+		snapshot.discardState();
+
+		// re-initialize to ensure that we create the PojoSerializer from scratch, otherwise
+		// initializeSerializerUnlessSet would not pick up our new config
+		kvId = new ValueStateDescriptor<>("id", pojoType);
+		state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
+
+		backend.setCurrentKey(1);
+
+		// update to test state backends that eagerly serialize, such as RocksDB
+		state.update(new TestPojo("u1", 11, new TestNestedPojoClassA(22.1, 12), new TestNestedPojoClassB(1.23, "foobar")));
+
+		// this tests backends that lazily serialize, such as memory state backend
+		runSnapshot(backend.snapshot(
+			682375462378L,
+			2,
+			streamFactory,
+			CheckpointOptions.forFullCheckpoint()));
+
+		backend.dispose();
+	}
 
 	@Test
 	@SuppressWarnings("unchecked")
@@ -1696,8 +1830,8 @@ public void testValueStateRestoreWithWrongSerializers() {
 				state.value();
 
 				fail("should recognize wrong serializers");
-			} catch (IOException e) {
-				if (!e.getMessage().contains("Trying to access state using wrong")) {
+			} catch (RuntimeException e) {
+				if (!e.getMessage().contains("State migration currently isn't supported")) {
 					fail("wrong exception " + e);
 				}
 				// expected
@@ -1747,8 +1881,8 @@ public void testListStateRestoreWithWrongSerializers() {
 				state.get();
 
 				fail("should recognize wrong serializers");
-			} catch (IOException e) {
-				if (!e.getMessage().contains("Trying to access state using wrong")) {
+			} catch (RuntimeException e) {
+				if (!e.getMessage().contains("State migration currently isn't supported")) {
 					fail("wrong exception " + e);
 				}
 				// expected
@@ -1800,8 +1934,8 @@ public void testReducingStateRestoreWithWrongSerializers() {
 				state.get();
 
 				fail("should recognize wrong serializers");
-			} catch (IOException e) {
-				if (!e.getMessage().contains("Trying to access state using wrong ")) {
+			} catch (RuntimeException e) {
+				if (!e.getMessage().contains("State migration currently isn't supported")) {
 					fail("wrong exception " + e);
 				}
 				// expected
@@ -1851,8 +1985,8 @@ public void testMapStateRestoreWithWrongSerializers() {
 				state.entries();
 
 				fail("should recognize wrong serializers");
-			} catch (IOException e) {
-				if (!e.getMessage().contains("Trying to access state using wrong ")) {
+			} catch (RuntimeException e) {
+				if (!e.getMessage().contains("State migration currently isn't supported")) {
 					fail("wrong exception " + e);
 				}
 				// expected
@@ -2382,15 +2516,27 @@ private KeyedStateHandle runSnapshot(RunnableFuture<KeyedStateHandle> snapshotRu
 		return snapshotRunnableFuture.get();
 	}
 
-	private static class TestPojo implements Serializable {
+	public static class TestPojo implements Serializable {
 		private String strField;
 		private Integer intField;
 
+		private TestNestedPojoClassA kryoClassAField;
+		private TestNestedPojoClassB kryoClassBField;
+
 		public TestPojo() {}
 
 		public TestPojo(String strField, Integer intField) {
 			this.strField = strField;
 			this.intField = intField;
+			this.kryoClassAField = null;
+			this.kryoClassBField = null;
+		}
+
+		public TestPojo(String strField, Integer intField, TestNestedPojoClassA classAField, TestNestedPojoClassB classBfield) {
+			this.strField = strField;
+			this.intField = intField;
+			this.kryoClassAField = classAField;
+			this.kryoClassBField = classBfield;
 		}
 
 		public String getStrField() {
@@ -2409,6 +2555,22 @@ public void setIntField(Integer intField) {
 			this.intField = intField;
 		}
 
+		public TestNestedPojoClassA getKryoClassAField() {
+			return kryoClassAField;
+		}
+
+		public void setKryoClassAField(TestNestedPojoClassA kryoClassAField) {
+			this.kryoClassAField = kryoClassAField;
+		}
+
+		public TestNestedPojoClassB getKryoClassBField() {
+			return kryoClassBField;
+		}
+
+		public void setKryoClassBField(TestNestedPojoClassB kryoClassBField) {
+			this.kryoClassBField = kryoClassBField;
+		}
+
 		@Override
 		public String toString() {
 			return "TestPojo{" +
@@ -2424,14 +2586,133 @@ public boolean equals(Object o) {
 
 			TestPojo testPojo = (TestPojo) o;
 
-			if (!strField.equals(testPojo.strField)) return false;
-			return intField.equals(testPojo.intField);
+			return strField.equals(testPojo.strField)
+				&& intField.equals(testPojo.intField)
+				&& ((kryoClassAField == null && testPojo.kryoClassAField == null) || kryoClassAField.equals(testPojo.kryoClassAField))
+				&& ((kryoClassBField == null && testPojo.kryoClassBField == null) || kryoClassBField.equals(testPojo.kryoClassBField));
 		}
 
 		@Override
 		public int hashCode() {
 			int result = strField.hashCode();
 			result = 31 * result + intField.hashCode();
+
+			if (kryoClassAField != null) {
+				result = 31 * result + kryoClassAField.hashCode();
+			}
+
+			if (kryoClassBField != null) {
+				result = 31 * result + kryoClassBField.hashCode();
+			}
+
+			return result;
+		}
+	}
+
+	public static class TestNestedPojoClassA implements Serializable {
+		private Double doubleField;
+		private Integer intField;
+
+		public TestNestedPojoClassA() {}
+
+		public TestNestedPojoClassA(Double doubleField, Integer intField) {
+			this.doubleField = doubleField;
+			this.intField = intField;
+		}
+
+		public Double getDoubleField() {
+			return doubleField;
+		}
+
+		public void setDoubleField(Double doubleField) {
+			this.doubleField = doubleField;
+		}
+
+		public Integer getIntField() {
+			return intField;
+		}
+
+		public void setIntField(Integer intField) {
+			this.intField = intField;
+		}
+
+		@Override
+		public String toString() {
+			return "TestNestedPojoClassA{" +
+				"doubleField='" + doubleField + '\'' +
+				", intField=" + intField +
+				'}';
+		}
+
+		@Override
+		public boolean equals(Object o) {
+			if (this == o) return true;
+			if (o == null || getClass() != o.getClass()) return false;
+
+			TestNestedPojoClassA testNestedPojoClassA = (TestNestedPojoClassA) o;
+
+			if (!doubleField.equals(testNestedPojoClassA.doubleField)) return false;
+			return intField.equals(testNestedPojoClassA.intField);
+		}
+
+		@Override
+		public int hashCode() {
+			int result = doubleField.hashCode();
+			result = 31 * result + intField.hashCode();
+			return result;
+		}
+	}
+
+	public static class TestNestedPojoClassB implements Serializable {
+		private Double doubleField;
+		private String strField;
+
+		public TestNestedPojoClassB() {}
+
+		public TestNestedPojoClassB(Double doubleField, String strField) {
+			this.doubleField = doubleField;
+			this.strField = strField;
+		}
+
+		public Double getDoubleField() {
+			return doubleField;
+		}
+
+		public void setDoubleField(Double doubleField) {
+			this.doubleField = doubleField;
+		}
+
+		public String getStrField() {
+			return strField;
+		}
+
+		public void setStrField(String strField) {
+			this.strField = strField;
+		}
+
+		@Override
+		public String toString() {
+			return "TestNestedPojoClassB{" +
+				"doubleField='" + doubleField + '\'' +
+				", strField=" + strField +
+				'}';
+		}
+
+		@Override
+		public boolean equals(Object o) {
+			if (this == o) return true;
+			if (o == null || getClass() != o.getClass()) return false;
+
+			TestNestedPojoClassB testNestedPojoClassB = (TestNestedPojoClassB) o;
+
+			if (!doubleField.equals(testNestedPojoClassB.doubleField)) return false;
+			return strField.equals(testNestedPojoClassB.strField);
+		}
+
+		@Override
+		public int hashCode() {
+			int result = doubleField.hashCode();
+			result = 31 * result + strField.hashCode();
 			return result;
 		}
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java
index 976b9aada116f..828982174abee 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java
@@ -19,7 +19,9 @@
 package org.apache.flink.runtime.state.heap;
 
 import org.apache.flink.api.common.state.StateDescriptor;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.base.IntSerializer;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
@@ -29,7 +31,7 @@
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.runtime.state.ArrayListSerializer;
 import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StateTransformationFunction;
 import org.apache.flink.util.TestLogger;
 import org.junit.Assert;
@@ -50,8 +52,8 @@ public class CopyOnWriteStateTableTest extends TestLogger {
 	 */
 	@Test
 	public void testPutGetRemoveContainsTransform() throws Exception {
-		RegisteredBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
-				new RegisteredBackendStateMetaInfo<>(
+		RegisteredKeyedBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
+				new RegisteredKeyedBackendStateMetaInfo<>(
 						StateDescriptor.Type.UNKNOWN,
 						"test",
 						IntSerializer.INSTANCE,
@@ -122,8 +124,8 @@ public ArrayList<Integer> apply(ArrayList<Integer> previousState, Integer value)
 	 */
 	@Test
 	public void testIncrementalRehash() {
-		RegisteredBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
-				new RegisteredBackendStateMetaInfo<>(
+		RegisteredKeyedBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
+				new RegisteredKeyedBackendStateMetaInfo<>(
 						StateDescriptor.Type.UNKNOWN,
 						"test",
 						IntSerializer.INSTANCE,
@@ -167,8 +169,8 @@ public void testIncrementalRehash() {
 	@Test
 	public void testRandomModificationsAndCopyOnWriteIsolation() throws Exception {
 
-		final RegisteredBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
-				new RegisteredBackendStateMetaInfo<>(
+		final RegisteredKeyedBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
+				new RegisteredKeyedBackendStateMetaInfo<>(
 						StateDescriptor.Type.UNKNOWN,
 						"test",
 						IntSerializer.INSTANCE,
@@ -322,8 +324,8 @@ public ArrayList<Integer> apply(ArrayList<Integer> previousState, Integer value)
 	 */
 	@Test
 	public void testCopyOnWriteContracts() {
-		RegisteredBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
-				new RegisteredBackendStateMetaInfo<>(
+		RegisteredKeyedBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
+				new RegisteredKeyedBackendStateMetaInfo<>(
 						StateDescriptor.Type.UNKNOWN,
 						"test",
 						IntSerializer.INSTANCE,
@@ -397,8 +399,8 @@ public void testSerializerDuplicationInSnapshot() throws IOException {
 		final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();;
 		final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();;
 
-		RegisteredBackendStateMetaInfo<Integer, Integer> metaInfo =
-			new RegisteredBackendStateMetaInfo<>(
+		RegisteredKeyedBackendStateMetaInfo<Integer, Integer> metaInfo =
+			new RegisteredKeyedBackendStateMetaInfo<>(
 				StateDescriptor.Type.VALUE,
 				"test",
 				namespaceSerializer,
@@ -649,5 +651,15 @@ public int hashCode() {
 		public void disable() {
 			this.disabled = true;
 		}
+
+		@Override
+		public TypeSerializerConfigSnapshot snapshotConfiguration() {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public CompatibilityResult<Integer> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+			throw new UnsupportedOperationException();
+		}
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/StateTableSnapshotCompatibilityTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/StateTableSnapshotCompatibilityTest.java
index 6fd94f774917e..85bc1774b58f9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/StateTableSnapshotCompatibilityTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/StateTableSnapshotCompatibilityTest.java
@@ -27,7 +27,7 @@
 import org.apache.flink.runtime.state.ArrayListSerializer;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
-import org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -44,8 +44,8 @@ public class StateTableSnapshotCompatibilityTest {
 	@Test
 	public void checkCompatibleSerializationFormats() throws IOException {
 		final Random r = new Random(42);
-		RegisteredBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
-				new RegisteredBackendStateMetaInfo<>(
+		RegisteredKeyedBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo =
+				new RegisteredKeyedBackendStateMetaInfo<>(
 						StateDescriptor.Type.UNKNOWN,
 						"test",
 						IntSerializer.INSTANCE,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java
index 146ccd054db90..ce23f30006be4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java
@@ -21,7 +21,9 @@
 
 import java.io.IOException;
 
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.Record;
@@ -141,4 +143,14 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return RecordSerializer.class.hashCode();
 	}
+
+	@Override
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public CompatibilityResult<Record> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		throw new UnsupportedOperationException();
+	}
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
index 6871159ca284e..468fddc3429ed 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
@@ -18,8 +18,9 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.annotation.Internal
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.core.memory.{DataOutputView, DataInputView}
+import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializer, TypeSerializerConfigSnapshot}
+import org.apache.flink.api.java.typeutils.runtime.EitherSerializerConfigSnapshot
+import org.apache.flink.core.memory.{DataInputView, DataOutputView}
 
 /**
  * Serializer for [[Either]].
@@ -104,4 +105,47 @@ class EitherSerializer[A, B, T <: Either[A, B]](
   override def hashCode(): Int = {
     31 * leftSerializer.hashCode() + rightSerializer.hashCode()
   }
+
+  // --------------------------------------------------------------------------------------------
+  // Serializer configuration snapshotting & compatibility
+  // --------------------------------------------------------------------------------------------
+
+  override def snapshotConfiguration(): EitherSerializerConfigSnapshot = {
+    new EitherSerializerConfigSnapshot(
+      leftSerializer.snapshotConfiguration(),
+      rightSerializer.snapshotConfiguration())
+  }
+
+  override def ensureCompatibility(
+      configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[T] = {
+
+    configSnapshot match {
+      case eitherSerializerConfig: EitherSerializerConfigSnapshot =>
+        val leftRightConfigs =
+          eitherSerializerConfig.getNestedSerializerConfigSnapshots
+
+        val leftCompatResult = leftSerializer.ensureCompatibility(leftRightConfigs(0))
+        val rightCompatResult = rightSerializer.ensureCompatibility(leftRightConfigs(1))
+
+        if (leftCompatResult.requiresMigration || rightCompatResult.requiresMigration) {
+          if (leftCompatResult.getConvertDeserializer != null
+              && rightCompatResult.getConvertDeserializer != null) {
+
+            CompatibilityResult.requiresMigration(
+              new EitherSerializer[A, B, T](
+                leftCompatResult.getConvertDeserializer,
+                rightCompatResult.getConvertDeserializer
+              )
+            )
+
+          } else {
+            CompatibilityResult.requiresMigration(null)
+          }
+        } else {
+          CompatibilityResult.compatible()
+        }
+
+      case _ => CompatibilityResult.requiresMigration(null)
+    }
+  }
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala
index 67c1445dfa473..dc96c9899cb9d 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala
@@ -17,10 +17,14 @@
  */
 package org.apache.flink.api.scala.typeutils
 
+import java.io.IOException
+
 import org.apache.flink.annotation.Internal
-import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializer, TypeSerializerConfigSnapshot}
 import org.apache.flink.api.common.typeutils.base.IntSerializer
+import org.apache.flink.api.java.typeutils.runtime.{DataInputViewStream, DataOutputViewStream}
 import org.apache.flink.core.memory.{DataInputView, DataOutputView}
+import org.apache.flink.util.InstantiationUtil
 
 /**
  * Serializer for [[Enumeration]] values.
@@ -67,4 +71,114 @@ class EnumValueSerializer[E <: Enumeration](val enum: E) extends TypeSerializer[
   override def canEqual(obj: scala.Any): Boolean = {
     obj.isInstanceOf[EnumValueSerializer[_]]
   }
+
+  // --------------------------------------------------------------------------------------------
+  // Serializer configuration snapshotting & compatibility
+  // --------------------------------------------------------------------------------------------
+
+  override def snapshotConfiguration(): EnumValueSerializer.ScalaEnumSerializerConfigSnapshot[E] = {
+    new EnumValueSerializer.ScalaEnumSerializerConfigSnapshot[E](
+      enum.getClass.asInstanceOf[Class[E]])
+  }
+
+  override def ensureCompatibility(
+      configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[E#Value] = {
+
+    configSnapshot match {
+      case enumSerializerConfigSnapshot: EnumValueSerializer.ScalaEnumSerializerConfigSnapshot[_] =>
+        val enumClass = enum.getClass.asInstanceOf[Class[E]]
+        if (enumClass.equals(enumSerializerConfigSnapshot.getEnumClass)) {
+          val currentEnumConstants = enumSerializerConfigSnapshot.getEnumClass.getEnumConstants
+
+          for ( i <- 0 to currentEnumConstants.length) {
+            // compatible only if new enum constants are only appended,
+            // and original constants must be in the exact same order
+
+            if (currentEnumConstants(i) != enumSerializerConfigSnapshot.getEnumConstants(i)) {
+              CompatibilityResult.requiresMigration(null)
+            }
+          }
+
+          CompatibilityResult.compatible()
+        } else {
+          CompatibilityResult.requiresMigration(null)
+        }
+
+      case _ => CompatibilityResult.requiresMigration(null)
+    }
+  }
+}
+
+object EnumValueSerializer {
+
+  class ScalaEnumSerializerConfigSnapshot[E <: Enumeration](private var enumClass: Class[E])
+      extends TypeSerializerConfigSnapshot {
+
+    var enumConstants: Array[E] = enumClass.getEnumConstants
+
+    /** This empty nullary constructor is required for deserializing the configuration. */
+    def this() = this(null)
+
+    override def write(out: DataOutputView): Unit = {
+      super.write(out)
+
+      try {
+        val outViewWrapper = new DataOutputViewStream(out)
+        try {
+          InstantiationUtil.serializeObject(outViewWrapper, enumClass)
+          InstantiationUtil.serializeObject(outViewWrapper, enumConstants)
+        } finally if (outViewWrapper != null) outViewWrapper.close()
+      }
+    }
+
+    override def read(in: DataInputView): Unit = {
+      super.read(in)
+
+      try {
+        val inViewWrapper = new DataInputViewStream(in)
+        try
+          try {
+            enumClass = InstantiationUtil.deserializeObject(
+              inViewWrapper, getUserCodeClassLoader)
+
+            enumConstants = InstantiationUtil.deserializeObject(
+              inViewWrapper, getUserCodeClassLoader)
+          } catch {
+            case e: ClassNotFoundException =>
+              throw new IOException("The requested enum class cannot be found in classpath.", e)
+          }
+          finally if (inViewWrapper != null) inViewWrapper.close()
+      }
+    }
+
+    override def getVersion: Int = ScalaEnumSerializerConfigSnapshot.VERSION
+
+    def getEnumClass: Class[E] = enumClass
+
+    def getEnumConstants: Array[E] = enumConstants
+
+    override def equals(obj: scala.Any): Boolean = {
+      if (obj == this) {
+        return true
+      }
+
+      if (obj == null) {
+        return false
+      }
+
+      obj.isInstanceOf[ScalaEnumSerializerConfigSnapshot[E]] &&
+        enumClass.equals(obj.asInstanceOf[ScalaEnumSerializerConfigSnapshot[E]].enumClass) &&
+        enumConstants.sameElements(
+          obj.asInstanceOf[ScalaEnumSerializerConfigSnapshot[E]].enumConstants)
+    }
+
+    override def hashCode(): Int = {
+      enumClass.hashCode() * 31 + enumConstants.toSeq.hashCode()
+    }
+  }
+
+  object ScalaEnumSerializerConfigSnapshot {
+    val VERSION = 1
+  }
+
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala
index fa5279e9bf0d8..01ca29594e2a2 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala
@@ -18,8 +18,8 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.annotation.Internal
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.core.memory.{DataOutputView, DataInputView}
+import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializer, TypeSerializerConfigSnapshot}
+import org.apache.flink.core.memory.{DataInputView, DataOutputView}
 
 /**
  * Serializer for cases where no serializer is required but the system still expects one. This
@@ -50,13 +50,19 @@ class NothingSerializer extends TypeSerializer[Any] {
   override def serialize(any: Any, target: DataOutputView): Unit =
     throw new RuntimeException("This must not be used. You encountered a bug.")
 
-
   override def deserialize(source: DataInputView): Any =
     throw new RuntimeException("This must not be used. You encountered a bug.")
 
   override def deserialize(reuse: Any, source: DataInputView): Any =
     throw new RuntimeException("This must not be used. You encountered a bug.")
 
+  override def snapshotConfiguration(): TypeSerializerConfigSnapshot =
+    throw new RuntimeException("This must not be used. You encountered a bug.")
+
+  override def ensureCompatibility(
+      configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[Any] =
+    throw new RuntimeException("This must not be used. You encountered a bug.")
+
   override def equals(obj: Any): Boolean = {
     obj match {
       case nothingSerializer: NothingSerializer => nothingSerializer.canEqual(this)
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
index a8b3a56325e08..d2bb0985a9849 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
@@ -18,8 +18,8 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.annotation.Internal
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.core.memory.{DataOutputView, DataInputView}
+import org.apache.flink.api.common.typeutils._
+import org.apache.flink.core.memory.{DataInputView, DataOutputView}
 
 /**
  * Serializer for [[Option]].
@@ -95,4 +95,52 @@ class OptionSerializer[A](val elemSerializer: TypeSerializer[A])
   override def hashCode(): Int = {
     elemSerializer.hashCode()
   }
+
+  // --------------------------------------------------------------------------------------------
+  // Serializer configuration snapshotting & compatibility
+  // --------------------------------------------------------------------------------------------
+
+  override def snapshotConfiguration(): OptionSerializer.OptionSerializerConfigSnapshot = {
+    new OptionSerializer.OptionSerializerConfigSnapshot(elemSerializer.snapshotConfiguration())
+  }
+
+  override def ensureCompatibility(
+      configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[Option[A]] = {
+    configSnapshot match {
+      case optionSerializerConfigSnapshot: OptionSerializer.OptionSerializerConfigSnapshot =>
+        val compatResult = elemSerializer.ensureCompatibility(
+          optionSerializerConfigSnapshot.getSingleNestedSerializerConfigSnapshot)
+
+        if (compatResult.requiresMigration()) {
+          if (compatResult.getConvertDeserializer != null) {
+            CompatibilityResult.requiresMigration(
+              new OptionSerializer[A](compatResult.getConvertDeserializer))
+          } else {
+            CompatibilityResult.requiresMigration(null)
+          }
+        } else {
+          CompatibilityResult.compatible()
+        }
+
+      case _ => CompatibilityResult.requiresMigration(null)
+    }
+  }
+}
+
+object OptionSerializer {
+
+  class OptionSerializerConfigSnapshot(
+      private var elemSerializerConfigSnapshot: TypeSerializerConfigSnapshot)
+    extends CompositeTypeSerializerConfigSnapshot(elemSerializerConfigSnapshot) {
+
+    /** This empty nullary constructor is required for deserializing the configuration. */
+    def this() = this(null)
+
+    override def getVersion: Int = OptionSerializerConfigSnapshot.VERSION
+  }
+
+  object OptionSerializerConfigSnapshot {
+    val VERSION = 1
+  }
+
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
index d1b908591f2ae..1ac46f9545fd0 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
@@ -20,8 +20,8 @@ package org.apache.flink.api.scala.typeutils
 import java.io.ObjectInputStream
 
 import org.apache.flink.annotation.Internal
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.core.memory.{DataOutputView, DataInputView}
+import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializer, TypeSerializerConfigSnapshot}
+import org.apache.flink.core.memory.{DataInputView, DataOutputView}
 
 import scala.collection.generic.CanBuildFrom
 
@@ -150,4 +150,13 @@ abstract class TraversableSerializer[T <: TraversableOnce[E], E](
   override def canEqual(obj: Any): Boolean = {
     obj.isInstanceOf[TraversableSerializer[_, _]]
   }
+
+  override def snapshotConfiguration(): TypeSerializerConfigSnapshot = {
+    throw new UnsupportedOperationException()
+  }
+
+  override def ensureCompatibility(
+      configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[T] = {
+    throw new UnsupportedOperationException()
+  }
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
index a5ec03a9d45b0..c864dc784c9e6 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
@@ -19,11 +19,12 @@ package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.annotation.Internal
 import org.apache.flink.api.common.ExecutionConfig
-import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.common.typeutils._
 import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer.KryoSerializerConfigSnapshot
 import org.apache.flink.core.memory.{DataInputView, DataOutputView}
 
-import scala.util.{Success, Try, Failure}
+import scala.util.{Failure, Success, Try}
 
 /**
  * Serializer for [[scala.util.Try]].
@@ -98,4 +99,57 @@ class TrySerializer[A](
   override def hashCode(): Int = {
     31 * elemSerializer.hashCode() + executionConfig.hashCode()
   }
+
+  // --------------------------------------------------------------------------------------------
+  // Serializer configuration snapshotting & compatibility
+  // --------------------------------------------------------------------------------------------
+
+  override def snapshotConfiguration(): TypeSerializerConfigSnapshot = {
+    new TrySerializer.TrySerializerConfigSnapshot(
+        elemSerializer.snapshotConfiguration(),
+        throwableSerializer.snapshotConfiguration())
+  }
+
+  override def ensureCompatibility(
+      configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[Try[A]] = {
+
+    configSnapshot match {
+      case trySerializerConfigSnapshot: TrySerializer.TrySerializerConfigSnapshot =>
+        val serializerConfigSnapshots =
+          trySerializerConfigSnapshot.getNestedSerializerConfigSnapshots
+
+        val elemCompatRes =
+          elemSerializer.ensureCompatibility(serializerConfigSnapshots(0))
+        val throwableCompatRes =
+          throwableSerializer.ensureCompatibility(serializerConfigSnapshots(1))
+
+        if (elemCompatRes.requiresMigration() || throwableCompatRes.requiresMigration()) {
+          CompatibilityResult.requiresMigration(null)
+        } else {
+          CompatibilityResult.compatible()
+        }
+
+      case _ => CompatibilityResult.requiresMigration(null)
+    }
+  }
+}
+
+object TrySerializer {
+
+  class TrySerializerConfigSnapshot(
+      private var elemSerializerConfigSnapshot: TypeSerializerConfigSnapshot,
+      private var throwableSerializerConfigSnapshot: KryoSerializerConfigSnapshot[Throwable])
+    extends CompositeTypeSerializerConfigSnapshot(
+      elemSerializerConfigSnapshot, throwableSerializerConfigSnapshot) {
+
+    /** This empty nullary constructor is required for deserializing the configuration. */
+    def this() = this(null, null)
+
+    override def getVersion: Int = TrySerializerConfigSnapshot.VERSION
+  }
+
+  object TrySerializerConfigSnapshot {
+    val VERSION = 1
+  }
+
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
index fdcd5b89ae18c..53fea46d9105f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
@@ -21,7 +21,11 @@
 import static java.util.Objects.requireNonNull;
 
 import java.io.IOException;
+
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.streaming.api.watermark.Watermark;
@@ -205,6 +209,53 @@ else if (tag == TAG_WATERMARK) {
 		}
 	}
 
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public MultiplexingStreamRecordSerializerConfigSnapshot snapshotConfiguration() {
+		return new MultiplexingStreamRecordSerializerConfigSnapshot(typeSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof MultiplexingStreamRecordSerializerConfigSnapshot) {
+			CompatibilityResult<T> compatResult = typeSerializer.ensureCompatibility(
+				((MultiplexingStreamRecordSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
+
+			if (!compatResult.requiresMigration()) {
+				return CompatibilityResult.compatible();
+			} else if (compatResult.getConvertDeserializer() != null) {
+				return CompatibilityResult.requiresMigration(
+					new MultiplexingStreamRecordSerializer<>(compatResult.getConvertDeserializer()));
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	/**
+	 * Configuration snapshot specific to the {@link MultiplexingStreamRecordSerializer}.
+	 */
+	public static final class MultiplexingStreamRecordSerializerConfigSnapshot
+			extends CompositeTypeSerializerConfigSnapshot {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public MultiplexingStreamRecordSerializerConfigSnapshot() {}
+
+		public MultiplexingStreamRecordSerializerConfigSnapshot(TypeSerializerConfigSnapshot typeSerializerConfigSnapshot) {
+			super(typeSerializerConfigSnapshot);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+	}
+
 	// ------------------------------------------------------------------------
 	//  Utilities
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
index 2c8dc4a328358..2a87f4e688030 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
@@ -20,7 +20,10 @@
 
 import java.io.IOException;
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -144,4 +147,50 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return typeSerializer.hashCode();
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public StreamRecordSerializerConfigSnapshot snapshotConfiguration() {
+		return new StreamRecordSerializerConfigSnapshot(typeSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<StreamRecord<T>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof StreamRecordSerializerConfigSnapshot) {
+			CompatibilityResult<T> compatResult = typeSerializer.ensureCompatibility(
+				((StreamRecordSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
+
+			if (!compatResult.requiresMigration()) {
+				return CompatibilityResult.requiresMigration(null);
+			} else if (compatResult.getConvertDeserializer() != null) {
+				return CompatibilityResult.requiresMigration(
+					new StreamRecordSerializer<>(compatResult.getConvertDeserializer()));
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	/**
+	 * Configuration snapshot specific to the {@link StreamRecordSerializer}.
+	 */
+	public static final class StreamRecordSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public StreamRecordSerializerConfigSnapshot() {}
+
+		public StreamRecordSerializerConfigSnapshot(TypeSerializerConfigSnapshot typeSerializerConfigSnapshot) {
+			super(typeSerializerConfigSnapshot);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
index 13a8a248d7d7e..f0c3dc2a6622a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java
@@ -29,7 +29,9 @@
 import org.apache.flink.api.common.functions.CoGroupFunction;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.operators.translation.WrappingFunction;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -547,6 +549,16 @@ public boolean equals(Object obj) {
 		public boolean canEqual(Object obj) {
 			return obj instanceof UnionSerializer;
 		}
+
+		@Override
+		public TypeSerializerConfigSnapshot snapshotConfiguration() {
+			throw new UnsupportedOperationException("This serializer is not registered for managed state.");
+		}
+
+		@Override
+		public CompatibilityResult<TaggedUnion<T1, T2>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+			throw new UnsupportedOperationException("This serializer is not registered for managed state.");
+		}
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java
index 7b6ba8da66eaa..14557129df945 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimer.java
@@ -19,7 +19,9 @@
 
 import java.io.IOException;
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.base.LongSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
@@ -184,5 +186,15 @@ public boolean canEqual(Object obj) {
 		public int hashCode() {
 			return getClass().hashCode();
 		}
+
+		@Override
+		public TypeSerializerConfigSnapshot snapshotConfiguration() {
+			throw new UnsupportedOperationException("This serializer is not registered for managed state.");
+		}
+
+		@Override
+		public CompatibilityResult<InternalTimer<K, N>> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+			throw new UnsupportedOperationException("This serializer is not registered for managed state.");
+		}
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
index b1e06f5e8d105..cf5c74c36846f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/GlobalWindow.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
@@ -61,7 +62,7 @@ public String toString() {
 	/**
 	 * A {@link TypeSerializer} for {@link GlobalWindow}.
 	 */
-	public static class Serializer extends TypeSerializer<GlobalWindow> {
+	public static class Serializer extends TypeSerializerSingleton<GlobalWindow> {
 		private static final long serialVersionUID = 1L;
 
 		@Override
@@ -69,11 +70,6 @@ public boolean isImmutableType() {
 			return true;
 		}
 
-		@Override
-		public TypeSerializer<GlobalWindow> duplicate() {
-			return this;
-		}
-
 		@Override
 		public GlobalWindow createInstance() {
 			return GlobalWindow.INSTANCE;
@@ -118,19 +114,9 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 			target.writeByte(0);
 		}
 
-		@Override
-		public boolean equals(Object obj) {
-			return obj instanceof Serializer;
-		}
-
 		@Override
 		public boolean canEqual(Object obj) {
 			return obj instanceof Serializer;
 		}
-
-		@Override
-		public int hashCode() {
-			return 0;
-		}
 	}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
index 6d896cb943dfe..a7ea244e2680f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/windows/TimeWindow.java
@@ -26,7 +26,7 @@
 import java.util.List;
 import java.util.Set;
 import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
@@ -130,7 +130,7 @@ public TimeWindow cover(TimeWindow other) {
 	/**
 	 * The serializer used to write the TimeWindow type.
 	 */
-	public static class Serializer extends TypeSerializer<TimeWindow> {
+	public static class Serializer extends TypeSerializerSingleton<TimeWindow> {
 		private static final long serialVersionUID = 1L;
 
 		@Override
@@ -138,11 +138,6 @@ public boolean isImmutableType() {
 			return true;
 		}
 
-		@Override
-		public TypeSerializer<TimeWindow> duplicate() {
-			return this;
-		}
-
 		@Override
 		public TimeWindow createInstance() {
 			return null;
@@ -187,20 +182,10 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 			target.writeLong(source.readLong());
 		}
 
-		@Override
-		public boolean equals(Object obj) {
-			return obj instanceof Serializer;
-		}
-
 		@Override
 		public boolean canEqual(Object obj) {
 			return obj instanceof Serializer;
 		}
-
-		@Override
-		public int hashCode() {
-			return 0;
-		}
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index 7fe088a599352..5c52fa6e4da72 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -22,7 +22,10 @@
 
 import java.io.IOException;
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.streaming.api.watermark.Watermark;
@@ -262,4 +265,54 @@ public boolean canEqual(Object obj) {
 	public int hashCode() {
 		return typeSerializer.hashCode();
 	}
+
+	// --------------------------------------------------------------------------------------------
+	// Serializer configuration snapshotting & compatibility
+	//
+	// This serializer may be used by Flink internal operators that need to checkpoint
+	// buffered records. Therefore, it may be part of managed state and need to implement
+	// the configuration snapshot and compatibility methods.
+	// --------------------------------------------------------------------------------------------
+
+	@Override
+	public StreamElementSerializerConfigSnapshot snapshotConfiguration() {
+		return new StreamElementSerializerConfigSnapshot(typeSerializer.snapshotConfiguration());
+	}
+
+	@Override
+	public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		if (configSnapshot instanceof StreamElementSerializerConfigSnapshot) {
+			CompatibilityResult<T> compatResult = typeSerializer.ensureCompatibility(
+				((StreamElementSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
+
+			if (!compatResult.requiresMigration()) {
+				return CompatibilityResult.compatible();
+			} else if (compatResult.getConvertDeserializer() != null) {
+				return CompatibilityResult.requiresMigration(
+					new StreamElementSerializer<>(compatResult.getConvertDeserializer()));
+			}
+		}
+
+		return CompatibilityResult.requiresMigration(null);
+	}
+
+	/**
+	 * Configuration snapshot specific to the {@link StreamElementSerializer}.
+	 */
+	public static final class StreamElementSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot {
+
+		private static final int VERSION = 1;
+
+		/** This empty nullary constructor is required for deserializing the configuration. */
+		public StreamElementSerializerConfigSnapshot() {}
+
+		public StreamElementSerializerConfigSnapshot(TypeSerializerConfigSnapshot typeSerializerConfigSnapshot) {
+			super(typeSerializerConfigSnapshot);
+		}
+
+		@Override
+		public int getVersion() {
+			return VERSION;
+		}
+	}
 }
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java
index a24a3a88571a7..2693bc1a43c1f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java
@@ -228,6 +228,5 @@ public void copy(DataInputView source, DataOutputView target) throws IOException
 		public boolean canEqual(Object obj) {
 			return obj instanceof CustomIntSerializer;
 		}
-
 	}
 }

From e0c8e4025f1fc00494ef3438fe07a210be2052ef Mon Sep 17 00:00:00 2001
From: "Tzu-Li (Gordon) Tai" <tzulitai@apache.org>
Date: Mon, 8 May 2017 02:42:02 +0800
Subject: [PATCH 184/253] [FLINK-6178] [core] Introduce TypeDeserializer
 interface for CompatibilityResult

Previously, the CompatibilityResult class accepts a full-blown
TypeSerializer for its convert deserializer, which will actually only
ever be used for deserialization.

This commit narrows down the interface by introducing a new
TypeDeserializer interface that contains only the read methods.

This closes #3834.
This closes #3804.
---
 .../common/typeutils/CompatibilityResult.java |   8 +-
 .../common/typeutils/TypeDeserializer.java    |  88 ++++++++++++
 .../typeutils/TypeDeserializerAdapter.java    | 127 ++++++++++++++++++
 .../api/common/typeutils/TypeSerializer.java  |   4 +-
 .../base/GenericArraySerializer.java          |   3 +-
 .../common/typeutils/base/ListSerializer.java |   3 +-
 .../common/typeutils/base/MapSerializer.java  |   5 +-
 .../typeutils/runtime/EitherSerializer.java   |   5 +-
 .../java/typeutils/runtime/RowSerializer.java |   4 +-
 .../AbstractKeyedCEPPatternOperator.java      |   4 +-
 .../table/runtime/types/CRowSerializer.scala  |   3 +-
 .../runtime/state/ArrayListSerializer.java    |   3 +-
 .../runtime/state/HashMapSerializer.java      |   5 +-
 .../scala/typeutils/EitherSerializer.scala    |   6 +-
 .../scala/typeutils/OptionSerializer.scala    |   3 +-
 .../MultiplexingStreamRecordSerializer.java   |   4 +-
 .../streamrecord/StreamRecordSerializer.java  |   4 +-
 .../streamrecord/StreamElementSerializer.java |   4 +-
 18 files changed, 258 insertions(+), 25 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializer.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
index cfbb516549230..891cfe0eebd9d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
@@ -38,7 +38,7 @@ public final class CompatibilityResult<T> {
 	 *
 	 * <p>This is only relevant if migration is required.
 	 */
-	private final TypeSerializer<T> convertDeserializer;
+	private final TypeDeserializer<T> convertDeserializer;
 
 	/**
 	 * Returns a strategy that signals that the new serializer is compatible and no migration is required.
@@ -61,16 +61,16 @@ public static <T> CompatibilityResult<T> compatible() {
 	 *
 	 * @return a result that signals migration is necessary, possibly providing a convert deserializer.
 	 */
-	public static <T> CompatibilityResult<T> requiresMigration(TypeSerializer<T> convertDeserializer) {
+	public static <T> CompatibilityResult<T> requiresMigration(TypeDeserializer<T> convertDeserializer) {
 		return new CompatibilityResult<>(true, convertDeserializer);
 	}
 
-	private CompatibilityResult(boolean requiresMigration, TypeSerializer<T> convertDeserializer) {
+	private CompatibilityResult(boolean requiresMigration, TypeDeserializer<T> convertDeserializer) {
 		this.requiresMigration = requiresMigration;
 		this.convertDeserializer = convertDeserializer;
 	}
 
-	public TypeSerializer<T> getConvertDeserializer() {
+	public TypeDeserializer<T> getConvertDeserializer() {
 		return convertDeserializer;
 	}
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializer.java
new file mode 100644
index 0000000000000..2ec064abd4503
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializer.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.core.memory.DataInputView;
+
+import java.io.IOException;
+
+/**
+ * This interface describes the methods that are required for a data type to be read by the Flink runtime.
+ * Specifically, this interface contains the deserialization methods. In contrast, the {@link TypeSerializer}
+ * interface contains the complete set of methods for both serialization and deserialization.
+ *
+ * <p>The methods in this class are assumed to be stateless, such that it is effectively thread safe. Stateful
+ * implementations of the methods may lead to unpredictable side effects and will compromise both stability and
+ * correctness of the program.
+ *
+ * @param <T> The data type that the deserializer deserializes.
+ */
+public interface TypeDeserializer<T> {
+
+	/**
+	 * Creates a deep copy of this deserializer if it is necessary, i.e. if it is stateful. This
+	 * can return itself if the serializer is not stateful.
+	 *
+	 * We need this because deserializers might be used in several threads. Stateless deserializers
+	 * are inherently thread-safe while stateful deserializers might not be thread-safe.
+	 */
+	TypeSerializer<T> duplicate();
+
+	/**
+	 * De-serializes a record from the given source input view.
+	 *
+	 * @param source The input view from which to read the data.
+	 * @return The deserialized element.
+	 *
+	 * @throws IOException Thrown, if the de-serialization encountered an I/O related error. Typically raised by the
+	 *                     input view, which may have an underlying I/O channel from which it reads.
+	 */
+	T deserialize(DataInputView source) throws IOException;
+
+	/**
+	 * De-serializes a record from the given source input view into the given reuse record instance if mutable.
+	 *
+	 * @param reuse The record instance into which to de-serialize the data.
+	 * @param source The input view from which to read the data.
+	 * @return The deserialized element.
+	 *
+	 * @throws IOException Thrown, if the de-serialization encountered an I/O related error. Typically raised by the
+	 *                     input view, which may have an underlying I/O channel from which it reads.
+	 */
+	T deserialize(T reuse, DataInputView source) throws IOException;
+
+	/**
+	 * Gets the length of the data type, if it is a fix length data type.
+	 *
+	 * @return The length of the data type, or <code>-1</code> for variable length data types.
+	 */
+	int getLength();
+
+	/**
+	 * Returns true if the given object can be equaled with this object. If not, it returns false.
+	 *
+	 * @param obj Object which wants to take part in the equality relation
+	 * @return true if obj can be equaled with this, otherwise false
+	 */
+	boolean canEqual(Object obj);
+
+	boolean equals(Object obj);
+
+	int hashCode();
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java
new file mode 100644
index 0000000000000..e02bed4dfb3a5
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.Preconditions;
+
+import java.io.IOException;
+
+/**
+ * A utility class that wraps a {@link TypeDeserializer} as a {@link TypeSerializer}.
+ *
+ * <p>Methods related to deserialization are directly forwarded to the wrapped deserializer,
+ * while serialization methods are masked and not intended for use.
+ *
+ * @param <T> The data type that the deserializer deserializes.
+ */
+public final class TypeDeserializerAdapter<T> extends TypeSerializer<T> {
+
+	private static final long serialVersionUID = 1L;
+
+	/** The actual wrapped deserializer instance */
+	private final TypeDeserializer<T> deserializer;
+
+	/**
+	 * Creates a {@link TypeSerializer} that wraps a {@link TypeDeserializer}.
+	 *
+	 * @param deserializer the actual deserializer to wrap.
+	 */
+	public TypeDeserializerAdapter(TypeDeserializer<T> deserializer) {
+		this.deserializer = Preconditions.checkNotNull(deserializer);
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Forwarded deserialization related methods
+	// --------------------------------------------------------------------------------------------
+
+	public T deserialize(DataInputView source) throws IOException {
+		return deserializer.deserialize(source);
+	}
+
+	public T deserialize(T reuse, DataInputView source) throws IOException {
+		return deserializer.deserialize(reuse, source);
+	}
+
+	public TypeSerializer<T> duplicate() {
+		return deserializer.duplicate();
+	}
+
+	public int getLength() {
+		return deserializer.getLength();
+	}
+
+	public boolean equals(Object obj) {
+		return deserializer.equals(obj);
+	}
+
+	public boolean canEqual(Object obj) {
+		return deserializer.canEqual(obj);
+	}
+
+	public int hashCode() {
+		return deserializer.hashCode();
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Irrelevant methods not intended for use
+	// --------------------------------------------------------------------------------------------
+
+	public boolean isImmutableType() {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+	public T createInstance() {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+	public T copy(T from) {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+	public T copy(T from, T reuse) {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+	public void serialize(T record, DataOutputView target) throws IOException {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+	public void copy(DataInputView source, DataOutputView target) throws IOException {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+	public TypeSerializerConfigSnapshot snapshotConfiguration() {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) {
+		throw new UnsupportedOperationException(
+			"This is a TypeDeserializerAdapter used only for deserialization; this method should not be used.");
+	}
+
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
index f0562d41b2752..0b5a08a5203ee 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
@@ -36,7 +36,7 @@
  * @param <T> The data type that the serializer serializes.
  */
 @PublicEvolving
-public abstract class TypeSerializer<T> implements Serializable {
+public abstract class TypeSerializer<T> implements TypeDeserializer<T>, Serializable {
 	
 	private static final long serialVersionUID = 1L;
 
@@ -197,7 +197,7 @@ public abstract class TypeSerializer<T> implements Serializable {
 	 *     has been reconfigured to be compatible, to continue reading previous data, and that the
 	 *     serialization schema remains the same. No migration needs to be performed.</li>
 	 *
-	 *     <li>{@link CompatibilityResult#requiresMigration(TypeSerializer)}: this signals Flink that
+	 *     <li>{@link CompatibilityResult#requiresMigration(TypeDeserializer)}: this signals Flink that
 	 *     migration needs to be performed, because this serializer is not compatible, or cannot be reconfigured to be
 	 *     compatible, for previous data. Furthermore, in the case that the preceding serializer cannot be found or
 	 *     restored to read the previous data to perform the migration, the provided convert deserializer can be
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
index fe61ab37f7454..3e592b48a4b52 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
@@ -23,6 +23,7 @@
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
@@ -213,7 +214,7 @@ public CompatibilityResult<C[]> ensureCompatibility(TypeSerializerConfigSnapshot
 					return CompatibilityResult.requiresMigration(
 						new GenericArraySerializer<>(
 							componentClass,
-							compatResult.getConvertDeserializer()));
+							new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer())));
 				}
 			}
 		}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
index 02d22de9b0b8a..1b6540c37ecd6 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
@@ -188,7 +189,7 @@ public CompatibilityResult<List<T>> ensureCompatibility(TypeSerializerConfigSnap
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
-					new ListSerializer<>(compatResult.getConvertDeserializer()));
+					new ListSerializer<>(new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer())));
 			}
 		}
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
index 50900e43398a9..182fff6929665 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
@@ -220,8 +221,8 @@ public CompatibilityResult<Map<K, V>> ensureCompatibility(TypeSerializerConfigSn
 			} else if (keyCompatResult.getConvertDeserializer() != null && valueCompatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
 					new MapSerializer<>(
-						keyCompatResult.getConvertDeserializer(),
-						valueCompatResult.getConvertDeserializer()));
+						new TypeDeserializerAdapter<>(keyCompatResult.getConvertDeserializer()),
+						new TypeDeserializerAdapter<>(valueCompatResult.getConvertDeserializer())));
 			}
 		}
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
index c025d61a42848..461dd8721ad8e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
@@ -212,8 +213,8 @@ public CompatibilityResult<Either<L, R>> ensureCompatibility(TypeSerializerConfi
 				if (leftCompatResult.getConvertDeserializer() != null && rightCompatResult.getConvertDeserializer() != null) {
 					return CompatibilityResult.requiresMigration(
 						new EitherSerializer<>(
-							leftCompatResult.getConvertDeserializer(),
-							rightCompatResult.getConvertDeserializer()));
+							new TypeDeserializerAdapter<>(leftCompatResult.getConvertDeserializer()),
+							new TypeDeserializerAdapter<>(rightCompatResult.getConvertDeserializer())));
 				}
 			}
 		}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
index 5770dac9ee34d..075c9d3483dcb 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
@@ -20,6 +20,7 @@
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializerUtil;
@@ -276,7 +277,8 @@ public CompatibilityResult<Row> ensureCompatibility(TypeSerializerConfigSnapshot
 							// one of the field serializers cannot provide a fallback deserializer
 							return CompatibilityResult.requiresMigration(null);
 						} else {
-							convertDeserializers[i] = compatResult.getConvertDeserializer();
+							convertDeserializers[i] =
+								new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer());
 						}
 					}
 				}
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
index 14235dc48ffa4..140e091382a6a 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
@@ -22,6 +22,7 @@
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.base.CollectionSerializerConfigSnapshot;
@@ -516,7 +517,8 @@ public CompatibilityResult<PriorityQueue<T>> ensureCompatibility(TypeSerializerC
 					return CompatibilityResult.compatible();
 				} else if (compatResult.getConvertDeserializer() != null) {
 					return CompatibilityResult.requiresMigration(
-						new PriorityQueueSerializer<>(compatResult.getConvertDeserializer(), factory));
+						new PriorityQueueSerializer<>(
+							new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer()), factory));
 				}
 			}
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
index 7ffa57ce7c488..122f4fba1b493 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
@@ -96,7 +96,8 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali
         if (compatResult.requiresMigration()) {
           if (compatResult.getConvertDeserializer != null) {
             CompatibilityResult.requiresMigration(
-              new CRowSerializer(compatResult.getConvertDeserializer)
+              new CRowSerializer(
+                new TypeDeserializerAdapter(compatResult.getConvertDeserializer))
             )
           } else {
             CompatibilityResult.requiresMigration(null)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
index 8fbc227d1b169..c39cb9bb3d4aa 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
@@ -18,6 +18,7 @@
 package org.apache.flink.runtime.state;
 
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.base.CollectionSerializerConfigSnapshot;
@@ -155,7 +156,7 @@ public CompatibilityResult<ArrayList<T>> ensureCompatibility(TypeSerializerConfi
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
-					new ArrayListSerializer<>(compatResult.getConvertDeserializer()));
+					new ArrayListSerializer<>(new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer())));
 			}
 		}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
index d52c207b1bcc5..925fe7883aa96 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
@@ -20,6 +20,7 @@
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.api.common.typeutils.base.MapSerializerConfigSnapshot;
@@ -221,8 +222,8 @@ public CompatibilityResult<HashMap<K, V>> ensureCompatibility(TypeSerializerConf
 			} else if (keyCompatResult.getConvertDeserializer() != null && valueCompatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
 					new HashMapSerializer<>(
-						keyCompatResult.getConvertDeserializer(),
-						valueCompatResult.getConvertDeserializer()));
+						new TypeDeserializerAdapter<>(keyCompatResult.getConvertDeserializer()),
+						new TypeDeserializerAdapter<>(valueCompatResult.getConvertDeserializer())));
 			}
 		}
 
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
index 468fddc3429ed..88b2041462e44 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
@@ -18,7 +18,7 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.annotation.Internal
-import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializer, TypeSerializerConfigSnapshot}
+import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeDeserializerAdapter, TypeSerializer, TypeSerializerConfigSnapshot}
 import org.apache.flink.api.java.typeutils.runtime.EitherSerializerConfigSnapshot
 import org.apache.flink.core.memory.{DataInputView, DataOutputView}
 
@@ -133,8 +133,8 @@ class EitherSerializer[A, B, T <: Either[A, B]](
 
             CompatibilityResult.requiresMigration(
               new EitherSerializer[A, B, T](
-                leftCompatResult.getConvertDeserializer,
-                rightCompatResult.getConvertDeserializer
+                new TypeDeserializerAdapter(leftCompatResult.getConvertDeserializer),
+                new TypeDeserializerAdapter(rightCompatResult.getConvertDeserializer)
               )
             )
 
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
index d2bb0985a9849..81b3bccf9f213 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
@@ -114,7 +114,8 @@ class OptionSerializer[A](val elemSerializer: TypeSerializer[A])
         if (compatResult.requiresMigration()) {
           if (compatResult.getConvertDeserializer != null) {
             CompatibilityResult.requiresMigration(
-              new OptionSerializer[A](compatResult.getConvertDeserializer))
+              new OptionSerializer[A](
+                new TypeDeserializerAdapter(compatResult.getConvertDeserializer)))
           } else {
             CompatibilityResult.requiresMigration(null)
           }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
index 53fea46d9105f..552ffd032534c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
@@ -24,6 +24,7 @@
 
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
@@ -228,7 +229,8 @@ public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConf
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
-					new MultiplexingStreamRecordSerializer<>(compatResult.getConvertDeserializer()));
+					new MultiplexingStreamRecordSerializer<>(
+						new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer())));
 			}
 		}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
index 2a87f4e688030..f7a661e32e883 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
@@ -22,6 +22,7 @@
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
@@ -167,7 +168,8 @@ public CompatibilityResult<StreamRecord<T>> ensureCompatibility(TypeSerializerCo
 				return CompatibilityResult.requiresMigration(null);
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
-					new StreamRecordSerializer<>(compatResult.getConvertDeserializer()));
+					new StreamRecordSerializer<>(
+						new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer())));
 			}
 		}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index 5c52fa6e4da72..e444ced44fd66 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -24,6 +24,7 @@
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
 import org.apache.flink.core.memory.DataInputView;
@@ -289,7 +290,8 @@ public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConf
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
-					new StreamElementSerializer<>(compatResult.getConvertDeserializer()));
+					new StreamElementSerializer<>(
+						new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer())));
 			}
 		}
 

From 913ad478b9b9dd27e2c7a0f91d4567db35fb757a Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Sun, 7 May 2017 15:09:05 +0200
Subject: [PATCH 185/253] [FLINK-6471] [checkpoint] Fix
 RocksDBStateBackendTest#testCancelRunningSnapshot failing sporadically

---
 .../state/RocksDBKeyedStateBackend.java       |   6 +-
 .../state/RocksDBStateBackendTest.java        |   4 +-
 .../memory/MemCheckpointStreamFactory.java    |  22 ++--
 .../BlockerCheckpointStreamFactory.java       | 112 ------------------
 .../state/OperatorStateBackendTest.java       |  12 +-
 .../util/BlockerCheckpointStreamFactory.java  |  48 +++++---
 6 files changed, 60 insertions(+), 144 deletions(-)
 delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 079ea1301e4b7..3cb21acf02215 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -799,7 +799,7 @@ private StreamStateHandle materializeMetaData() throws Exception {
 			try {
 				outputStream = checkpointStreamFactory
 					.createCheckpointStateOutputStream(checkpointId, checkpointTimestamp);
-				stateBackend.cancelStreamRegistry.registerClosable(outputStream);
+				closeableRegistry.registerClosable(outputStream);
 
 				KeyedBackendSerializationProxy serializationProxy =
 					new KeyedBackendSerializationProxy(stateBackend.keySerializer, stateMetaInfoSnapshots);
@@ -807,14 +807,14 @@ private StreamStateHandle materializeMetaData() throws Exception {
 
 				serializationProxy.write(out);
 
-				stateBackend.cancelStreamRegistry.unregisterClosable(outputStream);
+				closeableRegistry.unregisterClosable(outputStream);
 				StreamStateHandle result = outputStream.closeAndGetHandle();
 				outputStream = null;
 
 				return result;
 			} finally {
 				if (outputStream != null) {
-					stateBackend.cancelStreamRegistry.unregisterClosable(outputStream);
+					closeableRegistry.unregisterClosable(outputStream);
 					outputStream.close();
 				}
 			}
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
index 99b71c52386ca..9340455a3e744 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java
@@ -26,7 +26,7 @@
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeutils.base.IntSerializer;
 import org.apache.flink.core.testutils.OneShotLatch;
-import org.apache.flink.runtime.checkpoint.BlockerCheckpointStreamFactory;
+import org.apache.flink.runtime.util.BlockerCheckpointStreamFactory;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
@@ -308,8 +308,8 @@ public void testCancelRunningSnapshot() throws Exception {
 		waiter.await(); // wait for snapshot to run
 		waiter.reset();
 		runStateUpdates();
-		blocker.trigger(); // allow checkpointing to start writing
 		snapshot.cancel(true);
+		blocker.trigger(); // allow checkpointing to start writing
 		assertTrue(testStreamFactory.getLastCreatedStream().isClosed());
 		waiter.await(); // wait for snapshot stream writing to run
 		try {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemCheckpointStreamFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemCheckpointStreamFactory.java
index 9b2b46f20d3c4..3920ce88778cb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemCheckpointStreamFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/memory/MemCheckpointStreamFactory.java
@@ -24,6 +24,7 @@
 
 import java.io.IOException;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * {@link CheckpointStreamFactory} that produces streams that write to in-memory byte arrays.
@@ -78,12 +79,13 @@ public static class MemoryCheckpointOutputStream extends CheckpointStateOutputSt
 
 		private final int maxSize;
 
-		private boolean closed;
+		private AtomicBoolean closed;
 
 		boolean isEmpty = true;
 
 		public MemoryCheckpointOutputStream(int maxSize) {
 			this.maxSize = maxSize;
+			this.closed = new AtomicBoolean(false);
 		}
 
 		@Override
@@ -110,8 +112,9 @@ public void sync() throws IOException { }
 
 		@Override
 		public void close() {
-			closed = true;
-			os.reset();
+			if (closed.compareAndSet(false, true)) {
+				closeInternal();
+			}
 		}
 
 		@Override
@@ -128,7 +131,7 @@ public long getPos() throws IOException {
 		}
 
 		public boolean isClosed() {
-			return closed;
+			return closed.get();
 		}
 
 		/**
@@ -137,15 +140,18 @@ public boolean isClosed() {
 		 * @throws IOException Thrown if the size of the data exceeds the maximal
 		 */
 		public byte[] closeAndGetBytes() throws IOException {
-			if (!closed) {
+			if (closed.compareAndSet(false, true)) {
 				checkSize(os.size(), maxSize);
 				byte[] bytes = os.toByteArray();
-				close();
+				closeInternal();
 				return bytes;
-			}
-			else {
+			} else {
 				throw new IOException("stream has already been closed");
 			}
 		}
+
+		private void closeInternal() {
+			os.reset();
+		}
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java
deleted file mode 100644
index 6f892e2053980..0000000000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/BlockerCheckpointStreamFactory.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.checkpoint;
-
-import org.apache.flink.core.testutils.OneShotLatch;
-import org.apache.flink.runtime.state.CheckpointStreamFactory;
-import org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory;
-
-import java.io.IOException;
-
-/**
- * A {@link CheckpointStreamFactory} for tests that creates streams that block on a latch to test concurrency in
- * checkpointing.
- */
-public class BlockerCheckpointStreamFactory implements CheckpointStreamFactory {
-
-	private final int maxSize;
-	private int afterNumberInvocations;
-	private OneShotLatch blocker;
-	private OneShotLatch waiter;
-
-	MemCheckpointStreamFactory.MemoryCheckpointOutputStream lastCreatedStream;
-
-	public MemCheckpointStreamFactory.MemoryCheckpointOutputStream getLastCreatedStream() {
-		return lastCreatedStream;
-	}
-
-	public BlockerCheckpointStreamFactory(int maxSize) {
-		this.maxSize = maxSize;
-	}
-
-	public void setAfterNumberInvocations(int afterNumberInvocations) {
-		this.afterNumberInvocations = afterNumberInvocations;
-	}
-
-	public void setBlockerLatch(OneShotLatch latch) {
-		this.blocker = latch;
-	}
-
-	public void setWaiterLatch(OneShotLatch latch) {
-		this.waiter = latch;
-	}
-
-	@Override
-	public MemCheckpointStreamFactory.MemoryCheckpointOutputStream createCheckpointStateOutputStream(long checkpointID, long timestamp) throws Exception {
-		waiter.trigger();
-		this.lastCreatedStream = new MemCheckpointStreamFactory.MemoryCheckpointOutputStream(maxSize) {
-
-			private int afterNInvocations = afterNumberInvocations;
-			private final OneShotLatch streamBlocker = blocker;
-			private final OneShotLatch streamWaiter = waiter;
-
-			@Override
-			public void write(int b) throws IOException {
-
-				if (afterNInvocations > 0) {
-					--afterNInvocations;
-				}
-
-				if (0 == afterNInvocations && null != streamBlocker) {
-					try {
-						streamBlocker.await();
-					} catch (InterruptedException ignored) {
-					}
-				}
-				try {
-					super.write(b);
-				} catch (IOException ex) {
-					if (null != streamWaiter) {
-						streamWaiter.trigger();
-					}
-					throw ex;
-				}
-
-				if (0 == afterNInvocations && null != streamWaiter) {
-					streamWaiter.trigger();
-				}
-			}
-
-			@Override
-			public void close() {
-				super.close();
-				if (null != streamWaiter) {
-					streamWaiter.trigger();
-				}
-			}
-		};
-
-		return lastCreatedStream;
-	}
-
-	@Override
-	public void close() throws Exception {
-
-	}
-}
\ No newline at end of file
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
index 50ca159420dce..85b9eaf5f8cc3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
@@ -21,21 +21,21 @@
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.core.testutils.OneShotLatch;
-import org.apache.flink.runtime.checkpoint.BlockerCheckpointStreamFactory;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
+import org.apache.flink.core.testutils.OneShotLatch;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.state.DefaultOperatorStateBackend.PartitionableListState;
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.runtime.util.BlockerCheckpointStreamFactory;
 import org.apache.flink.util.FutureUtil;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
-import java.io.File;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.concurrent.CancellationException;
@@ -477,18 +477,20 @@ public void testSnapshotAsyncCancel() throws Exception {
 
 		executorService.submit(runnableFuture);
 
-		// wait until the async checkpoint is in the write code, then continue
+		// wait until the async checkpoint is in the stream's write code, then continue
 		waiterLatch.await();
 
+		// cancel the future, which should close the underlying stream
 		runnableFuture.cancel(true);
+		Assert.assertTrue(streamFactory.getLastCreatedStream().isClosed());
 
+		// we allow the stream under test to proceed
 		blockerLatch.trigger();
 
 		try {
 			runnableFuture.get(60, TimeUnit.SECONDS);
 			Assert.fail();
 		} catch (CancellationException ignore) {
-
 		}
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java
index 1e31490a38501..98e654f8d5682 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/BlockerCheckpointStreamFactory.java
@@ -71,31 +71,28 @@ public MemCheckpointStreamFactory.MemoryCheckpointOutputStream createCheckpointS
 			@Override
 			public void write(int b) throws IOException {
 
-				if (null != waiter) {
-					waiter.trigger();
-				}
+				unblockWaiter();
 
 				if (afterNInvocations > 0) {
 					--afterNInvocations;
+				} else {
+					awaitBlocker();
 				}
 
-				if (0 == afterNInvocations && null != streamBlocker) {
-					try {
-						streamBlocker.await();
-					} catch (InterruptedException ignored) {
-					}
-				}
 				try {
 					super.write(b);
 				} catch (IOException ex) {
-					if (null != streamWaiter) {
-						streamWaiter.trigger();
-					}
+					unblockWaiter();
 					throw ex;
 				}
 
-				if (0 == afterNInvocations && null != streamWaiter) {
-					streamWaiter.trigger();
+				if (0 == afterNInvocations) {
+					unblockWaiter();
+				}
+
+				// We also check for close here, in case the underlying stream does not do this
+				if (isClosed()) {
+					throw new IOException("Stream closed.");
 				}
 			}
 
@@ -110,10 +107,33 @@ public void write(byte[] b, int off, int len) throws IOException {
 			@Override
 			public void close() {
 				super.close();
+				// trigger all the latches, essentially all blocking ops on the stream should resume after close.
+				unblockAll();
+			}
+
+			private void unblockWaiter() {
 				if (null != streamWaiter) {
 					streamWaiter.trigger();
 				}
 			}
+
+			private void awaitBlocker() {
+				if (null != streamBlocker) {
+					try {
+						streamBlocker.await();
+					} catch (InterruptedException ignored) {
+					}
+				}
+			}
+
+			private void unblockAll() {
+				if (null != streamWaiter) {
+					streamWaiter.trigger();
+				}
+				if (null != streamBlocker) {
+					streamBlocker.trigger();
+				}
+			}
 		};
 
 		return lastCreatedStream;

From f71c0b68eb0fd4e8ab774f49ebe19d3835949ec4 Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Sun, 7 May 2017 20:22:34 +0200
Subject: [PATCH 186/253] [FLINK-6475] [checkpoint] Incremental snapshots in
 RocksDB should not hold lock during async file upload

---
 .../state/RocksDBKeyedStateBackend.java       | 63 +++++++++----------
 .../flink/util/AbstractCloseableRegistry.java |  4 ++
 .../state/AbstractKeyedStateBackend.java      |  4 ++
 .../state/DefaultOperatorStateBackend.java    |  1 +
 .../api/operators/AbstractStreamOperator.java |  3 -
 5 files changed, 37 insertions(+), 38 deletions(-)

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 3cb21acf02215..f5dddd6f2f0e4 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -844,54 +844,47 @@ void takeSnapshot() throws Exception {
 
 		KeyedStateHandle materializeSnapshot() throws Exception {
 
-			synchronized (stateBackend.asyncSnapshotLock) {
+			stateBackend.cancelStreamRegistry.registerClosable(closeableRegistry);
 
-				if (stateBackend.db == null) {
-					throw new IOException("RocksDB closed.");
-				}
-
-				stateBackend.cancelStreamRegistry.registerClosable(closeableRegistry);
+			// write meta data
+			metaStateHandle = materializeMetaData();
 
-				// write meta data
-				metaStateHandle = materializeMetaData();
+			// write state data
+			Preconditions.checkState(backupFileSystem.exists(backupPath));
 
-				// write state data
-				Preconditions.checkState(backupFileSystem.exists(backupPath));
+			FileStatus[] fileStatuses = backupFileSystem.listStatus(backupPath);
+			if (fileStatuses != null) {
+				for (FileStatus fileStatus : fileStatuses) {
+					Path filePath = fileStatus.getPath();
+					String fileName = filePath.getName();
 
-				FileStatus[] fileStatuses = backupFileSystem.listStatus(backupPath);
-				if (fileStatuses != null) {
-					for (FileStatus fileStatus : fileStatuses) {
-						Path filePath = fileStatus.getPath();
-						String fileName = filePath.getName();
+					if (fileName.endsWith(SST_FILE_SUFFIX)) {
+						StreamStateHandle fileHandle =
+							baseSstFiles == null ? null : baseSstFiles.get(fileName);
 
-						if (fileName.endsWith(SST_FILE_SUFFIX)) {
-							StreamStateHandle fileHandle =
-								baseSstFiles == null ? null : baseSstFiles.get(fileName);
+						if (fileHandle == null) {
+							fileHandle = materializeStateData(filePath);
 
-							if (fileHandle == null) {
-								fileHandle = materializeStateData(filePath);
-
-								newSstFiles.put(fileName, fileHandle);
-							} else {
-								oldSstFiles.put(fileName, fileHandle);
-							}
+							newSstFiles.put(fileName, fileHandle);
 						} else {
-							StreamStateHandle fileHandle = materializeStateData(filePath);
-							miscFiles.put(fileName, fileHandle);
+							oldSstFiles.put(fileName, fileHandle);
 						}
+					} else {
+						StreamStateHandle fileHandle = materializeStateData(filePath);
+						miscFiles.put(fileName, fileHandle);
 					}
 				}
+			}
 
-				Map<String, StreamStateHandle> sstFiles = new HashMap<>(newSstFiles.size() + oldSstFiles.size());
-				sstFiles.putAll(newSstFiles);
-				sstFiles.putAll(oldSstFiles);
+			Map<String, StreamStateHandle> sstFiles = new HashMap<>(newSstFiles.size() + oldSstFiles.size());
+			sstFiles.putAll(newSstFiles);
+			sstFiles.putAll(oldSstFiles);
 
-				stateBackend.materializedSstFiles.put(checkpointId, sstFiles);
+			stateBackend.materializedSstFiles.put(checkpointId, sstFiles);
 
-				return new RocksDBIncrementalKeyedStateHandle(stateBackend.jobId,
-					stateBackend.operatorIdentifier, stateBackend.keyGroupRange,
-					checkpointId, newSstFiles, oldSstFiles, miscFiles, metaStateHandle);
-			}
+			return new RocksDBIncrementalKeyedStateHandle(stateBackend.jobId,
+				stateBackend.operatorIdentifier, stateBackend.keyGroupRange,
+				checkpointId, newSstFiles, oldSstFiles, miscFiles, metaStateHandle);
 		}
 
 		void stop() {
diff --git a/flink-core/src/main/java/org/apache/flink/util/AbstractCloseableRegistry.java b/flink-core/src/main/java/org/apache/flink/util/AbstractCloseableRegistry.java
index 766ede9323460..f949779ff1831 100644
--- a/flink-core/src/main/java/org/apache/flink/util/AbstractCloseableRegistry.java
+++ b/flink-core/src/main/java/org/apache/flink/util/AbstractCloseableRegistry.java
@@ -90,6 +90,10 @@ public final void unregisterClosable(C closeable) {
 	public void close() throws IOException {
 		synchronized (getSynchronizationLock()) {
 
+			if (closed) {
+				return;
+			}
+
 			IOUtils.closeAllQuietly(closeableToRef.keySet());
 
 			closeableToRef.clear();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
index 4f3ed016ac036..47ebe3b73f558 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
@@ -46,6 +46,7 @@
 import org.apache.flink.runtime.state.internal.InternalMapState;
 import org.apache.flink.runtime.state.internal.InternalReducingState;
 import org.apache.flink.runtime.state.internal.InternalValueState;
+import org.apache.flink.util.IOUtils;
 import org.apache.flink.util.Preconditions;
 
 import java.io.Closeable;
@@ -122,6 +123,9 @@ public AbstractKeyedStateBackend(
 	 */
 	@Override
 	public void dispose() {
+
+		IOUtils.closeQuietly(this);
+
 		if (kvStateRegistry != null) {
 			kvStateRegistry.unregisterAll();
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
index ec4aa81dfd62d..ab0c1f0bf3ed2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
@@ -120,6 +120,7 @@ public void close() throws IOException {
 
 	@Override
 	public void dispose() {
+		IOUtils.closeQuietly(this);
 		registeredStates.clear();
 	}
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 8c1caee4dd2c3..057df2b9d8600 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -26,7 +26,6 @@
 import java.util.ConcurrentModificationException;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.annotation.VisibleForTesting;
@@ -358,12 +357,10 @@ public void close() throws Exception {}
 	public void dispose() throws Exception {
 
 		if (operatorStateBackend != null) {
-			IOUtils.closeQuietly(operatorStateBackend);
 			operatorStateBackend.dispose();
 		}
 
 		if (keyedStateBackend != null) {
-			IOUtils.closeQuietly(keyedStateBackend);
 			keyedStateBackend.dispose();
 		}
 	}

From 598de43b633f73252c3ff59f663240545224ad42 Mon Sep 17 00:00:00 2001
From: tedyu <yuzhihong@gmail.com>
Date: Sun, 7 May 2017 12:20:27 -0700
Subject: [PATCH 187/253] FLINK-6474 Potential loss of precision in 32 bit
 integer multiplication

---
 .../runtime/taskexecutor/TaskManagerServicesConfiguration.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
index 3fee689205dbb..d1cfc4ce2e239 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
@@ -308,7 +308,7 @@ private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfigurat
 
 		if (!hasNewNetworkBufConf(configuration)) {
 			// map old config to new one:
-			networkBufMin = networkBufMax = numNetworkBuffers * pageSize;
+			networkBufMin = networkBufMax = ((long)numNetworkBuffers) * pageSize;
 		} else {
 			if (configuration.contains(TaskManagerOptions.NETWORK_NUM_BUFFERS)) {
 				LOG.info("Ignoring old (but still present) network buffer configuration via {}.",

From fcc75a097ce4e2ed6eac03779abb29a0bb3a7ee2 Mon Sep 17 00:00:00 2001
From: Robert Metzger <rmetzger@apache.org>
Date: Mon, 8 May 2017 09:17:43 +0200
Subject: [PATCH 188/253] Update Flink version to 1.4-SNAPSHOT

---
 docs/_config.yml                                             | 5 +++--
 flink-annotations/pom.xml                                    | 2 +-
 flink-clients/pom.xml                                        | 2 +-
 flink-connectors/flink-avro/pom.xml                          | 2 +-
 flink-connectors/flink-connector-cassandra/pom.xml           | 2 +-
 flink-connectors/flink-connector-elasticsearch-base/pom.xml  | 2 +-
 flink-connectors/flink-connector-elasticsearch/pom.xml       | 2 +-
 flink-connectors/flink-connector-elasticsearch2/pom.xml      | 2 +-
 flink-connectors/flink-connector-elasticsearch5/pom.xml      | 2 +-
 flink-connectors/flink-connector-filesystem/pom.xml          | 2 +-
 flink-connectors/flink-connector-kafka-0.10/pom.xml          | 2 +-
 flink-connectors/flink-connector-kafka-0.8/pom.xml           | 2 +-
 flink-connectors/flink-connector-kafka-0.9/pom.xml           | 2 +-
 flink-connectors/flink-connector-kafka-base/pom.xml          | 2 +-
 flink-connectors/flink-connector-kinesis/pom.xml             | 2 +-
 flink-connectors/flink-connector-nifi/pom.xml                | 2 +-
 flink-connectors/flink-connector-rabbitmq/pom.xml            | 2 +-
 flink-connectors/flink-connector-twitter/pom.xml             | 2 +-
 flink-connectors/flink-hadoop-compatibility/pom.xml          | 2 +-
 flink-connectors/flink-hbase/pom.xml                         | 2 +-
 flink-connectors/flink-hcatalog/pom.xml                      | 2 +-
 flink-connectors/flink-jdbc/pom.xml                          | 2 +-
 flink-connectors/pom.xml                                     | 2 +-
 flink-contrib/flink-connector-wikiedits/pom.xml              | 2 +-
 flink-contrib/flink-statebackend-rocksdb/pom.xml             | 2 +-
 flink-contrib/flink-storm-examples/pom.xml                   | 2 +-
 flink-contrib/flink-storm/pom.xml                            | 2 +-
 flink-contrib/flink-streaming-contrib/pom.xml                | 2 +-
 flink-contrib/flink-tweet-inputformat/pom.xml                | 2 +-
 flink-contrib/pom.xml                                        | 2 +-
 flink-core/pom.xml                                           | 2 +-
 flink-dist/pom.xml                                           | 2 +-
 flink-examples/flink-examples-batch/pom.xml                  | 2 +-
 flink-examples/flink-examples-streaming/pom.xml              | 2 +-
 flink-examples/flink-examples-table/pom.xml                  | 2 +-
 flink-examples/pom.xml                                       | 2 +-
 flink-fs-tests/pom.xml                                       | 2 +-
 flink-java/pom.xml                                           | 2 +-
 flink-java8/pom.xml                                          | 2 +-
 flink-libraries/flink-cep-scala/pom.xml                      | 2 +-
 flink-libraries/flink-cep/pom.xml                            | 2 +-
 flink-libraries/flink-gelly-examples/pom.xml                 | 2 +-
 flink-libraries/flink-gelly-scala/pom.xml                    | 2 +-
 flink-libraries/flink-gelly/pom.xml                          | 2 +-
 flink-libraries/flink-ml/pom.xml                             | 2 +-
 flink-libraries/flink-python/pom.xml                         | 2 +-
 flink-libraries/flink-table/pom.xml                          | 2 +-
 flink-libraries/pom.xml                                      | 2 +-
 flink-mesos/pom.xml                                          | 2 +-
 flink-metrics/flink-metrics-core/pom.xml                     | 2 +-
 flink-metrics/flink-metrics-dropwizard/pom.xml               | 2 +-
 flink-metrics/flink-metrics-ganglia/pom.xml                  | 2 +-
 flink-metrics/flink-metrics-graphite/pom.xml                 | 2 +-
 flink-metrics/flink-metrics-jmx/pom.xml                      | 2 +-
 flink-metrics/flink-metrics-statsd/pom.xml                   | 2 +-
 flink-metrics/pom.xml                                        | 2 +-
 flink-optimizer/pom.xml                                      | 2 +-
 flink-quickstart/flink-quickstart-java/pom.xml               | 2 +-
 .../src/main/resources/archetype-resources/pom.xml           | 2 +-
 flink-quickstart/flink-quickstart-scala/pom.xml              | 2 +-
 .../src/main/resources/archetype-resources/pom.xml           | 2 +-
 flink-quickstart/pom.xml                                     | 2 +-
 flink-runtime-web/pom.xml                                    | 2 +-
 flink-runtime/pom.xml                                        | 2 +-
 flink-scala-shell/pom.xml                                    | 2 +-
 flink-scala/pom.xml                                          | 2 +-
 flink-shaded-curator/flink-shaded-curator-recipes/pom.xml    | 2 +-
 flink-shaded-curator/flink-shaded-curator-test/pom.xml       | 2 +-
 flink-shaded-curator/pom.xml                                 | 2 +-
 flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml             | 2 +-
 flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml  | 2 +-
 flink-shaded-hadoop/pom.xml                                  | 2 +-
 flink-streaming-java/pom.xml                                 | 2 +-
 flink-streaming-scala/pom.xml                                | 2 +-
 flink-test-utils-parent/flink-test-utils-junit/pom.xml       | 2 +-
 flink-test-utils-parent/flink-test-utils/pom.xml             | 2 +-
 flink-test-utils-parent/pom.xml                              | 2 +-
 flink-tests/pom.xml                                          | 2 +-
 flink-yarn-tests/pom.xml                                     | 2 +-
 flink-yarn/pom.xml                                           | 2 +-
 pom.xml                                                      | 4 ++--
 tools/change-version.sh                                      | 4 ++--
 tools/force-shading/pom.xml                                  | 2 +-
 83 files changed, 87 insertions(+), 86 deletions(-)

diff --git a/docs/_config.yml b/docs/_config.yml
index 9167aceee9358..cf5ee10b986e0 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -26,8 +26,8 @@
 # to reference a specific Flink version, because this is the only place where
 # we change the version for the complete docs when forking of a release branch
 # etc.
-version: "1.3-SNAPSHOT"
-version_short: "1.3" # Used for the top navbar w/o snapshot suffix
+version: "1.4-SNAPSHOT"
+version_short: "1.4" # Used for the top navbar w/o snapshot suffix
 is_snapshot_version: true
 
 # This suffix is appended to the Scala-dependent Maven artifact names
@@ -46,6 +46,7 @@ is_latest: true
 is_stable: false
 
 previous_docs:
+  1.3: http://ci.apache.org/projects/flink/flink-docs-release-1.3
   1.2: http://ci.apache.org/projects/flink/flink-docs-release-1.2
   1.1: http://ci.apache.org/projects/flink/flink-docs-release-1.1
   1.0: http://ci.apache.org/projects/flink/flink-docs-release-1.0
diff --git a/flink-annotations/pom.xml b/flink-annotations/pom.xml
index f3013a24d92c4..b375611f5b325 100644
--- a/flink-annotations/pom.xml
+++ b/flink-annotations/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 0e0c1464d4b27..8dde2e9f67e28 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-avro/pom.xml b/flink-connectors/flink-avro/pom.xml
index fb44a21ca824b..468a2baabe965 100644
--- a/flink-connectors/flink-avro/pom.xml
+++ b/flink-connectors/flink-avro/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-cassandra/pom.xml b/flink-connectors/flink-connector-cassandra/pom.xml
index 308093827a3a7..d01c76984ebcf 100644
--- a/flink-connectors/flink-connector-cassandra/pom.xml
+++ b/flink-connectors/flink-connector-cassandra/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-elasticsearch-base/pom.xml b/flink-connectors/flink-connector-elasticsearch-base/pom.xml
index 32327ff147a0b..e183354568941 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch-base/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-elasticsearch/pom.xml b/flink-connectors/flink-connector-elasticsearch/pom.xml
index eebdb4b9027ec..07028dfde03d4 100644
--- a/flink-connectors/flink-connector-elasticsearch/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml
index 30396ddf38013..ec1b7da32c3d9 100644
--- a/flink-connectors/flink-connector-elasticsearch2/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml
index a0bd32880f7bd..93d7bbe37c7fc 100644
--- a/flink-connectors/flink-connector-elasticsearch5/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-filesystem/pom.xml b/flink-connectors/flink-connector-filesystem/pom.xml
index a37cb6866bcf3..28d996288c8f9 100644
--- a/flink-connectors/flink-connector-filesystem/pom.xml
+++ b/flink-connectors/flink-connector-filesystem/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml
index 9fb0e613bc18b..c17386d31821d 100644
--- a/flink-connectors/flink-connector-kafka-0.10/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-kafka-0.8/pom.xml b/flink-connectors/flink-connector-kafka-0.8/pom.xml
index ff35300203526..ceccb3e400b73 100644
--- a/flink-connectors/flink-connector-kafka-0.8/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.8/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml
index aa79a1db3305d..de4dedbc3a7ec 100644
--- a/flink-connectors/flink-connector-kafka-0.9/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml
index 263eb9aaed7e1..a6b23499482d4 100644
--- a/flink-connectors/flink-connector-kafka-base/pom.xml
+++ b/flink-connectors/flink-connector-kafka-base/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-kinesis/pom.xml b/flink-connectors/flink-connector-kinesis/pom.xml
index 080626f053ed8..7515f3160a985 100644
--- a/flink-connectors/flink-connector-kinesis/pom.xml
+++ b/flink-connectors/flink-connector-kinesis/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-nifi/pom.xml b/flink-connectors/flink-connector-nifi/pom.xml
index 7e9949f8134ea..67bfb29928040 100644
--- a/flink-connectors/flink-connector-nifi/pom.xml
+++ b/flink-connectors/flink-connector-nifi/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-rabbitmq/pom.xml b/flink-connectors/flink-connector-rabbitmq/pom.xml
index 6973a0ed5221a..f26e57c99ca93 100644
--- a/flink-connectors/flink-connector-rabbitmq/pom.xml
+++ b/flink-connectors/flink-connector-rabbitmq/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-connector-twitter/pom.xml b/flink-connectors/flink-connector-twitter/pom.xml
index 3e68541aa3dfb..c683da13820a5 100644
--- a/flink-connectors/flink-connector-twitter/pom.xml
+++ b/flink-connectors/flink-connector-twitter/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
index 46e20ef656841..8c5b673225f27 100644
--- a/flink-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml
index 4fc05568d3fe6..6a295a8ef345a 100644
--- a/flink-connectors/flink-hbase/pom.xml
+++ b/flink-connectors/flink-hbase/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml
index ba0e142692b45..a975e53776128 100644
--- a/flink-connectors/flink-hcatalog/pom.xml
+++ b/flink-connectors/flink-hcatalog/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/flink-jdbc/pom.xml b/flink-connectors/flink-jdbc/pom.xml
index e96ab69867003..38a0163559c9b 100644
--- a/flink-connectors/flink-jdbc/pom.xml
+++ b/flink-connectors/flink-jdbc/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml
index 5d8ca700f951d..5af85b834646b 100644
--- a/flink-connectors/pom.xml
+++ b/flink-connectors/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-contrib/flink-connector-wikiedits/pom.xml b/flink-contrib/flink-connector-wikiedits/pom.xml
index 976e0dcfc5fbc..fed8656ef3fee 100644
--- a/flink-contrib/flink-connector-wikiedits/pom.xml
+++ b/flink-contrib/flink-connector-wikiedits/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-contrib</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-contrib/flink-statebackend-rocksdb/pom.xml b/flink-contrib/flink-statebackend-rocksdb/pom.xml
index bd890d90dae54..2148706807f65 100644
--- a/flink-contrib/flink-statebackend-rocksdb/pom.xml
+++ b/flink-contrib/flink-statebackend-rocksdb/pom.xml
@@ -27,7 +27,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-contrib</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml
index d042ddea25a73..4d9d76cae5eea 100644
--- a/flink-contrib/flink-storm-examples/pom.xml
+++ b/flink-contrib/flink-storm-examples/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-contrib</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml
index b6cc61cdf6422..ec34ba991585c 100644
--- a/flink-contrib/flink-storm/pom.xml
+++ b/flink-contrib/flink-storm/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-contrib</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-contrib/flink-streaming-contrib/pom.xml b/flink-contrib/flink-streaming-contrib/pom.xml
index a218f120de4c5..2df0231a2501c 100644
--- a/flink-contrib/flink-streaming-contrib/pom.xml
+++ b/flink-contrib/flink-streaming-contrib/pom.xml
@@ -27,7 +27,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-contrib</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-contrib/flink-tweet-inputformat/pom.xml b/flink-contrib/flink-tweet-inputformat/pom.xml
index 95b6bd88d77e3..5e0d79ce68e17 100644
--- a/flink-contrib/flink-tweet-inputformat/pom.xml
+++ b/flink-contrib/flink-tweet-inputformat/pom.xml
@@ -27,7 +27,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-contrib</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-contrib/pom.xml b/flink-contrib/pom.xml
index 75770a43ccf2f..2277465352bb8 100644
--- a/flink-contrib/pom.xml
+++ b/flink-contrib/pom.xml
@@ -27,7 +27,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index 1efcb290ddf22..6b0d7781d94ee 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 5f6a39bcc47ca..97739917cae1a 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-examples/flink-examples-batch/pom.xml b/flink-examples/flink-examples-batch/pom.xml
index 20dc95119163d..269f5a60bdbd8 100644
--- a/flink-examples/flink-examples-batch/pom.xml
+++ b/flink-examples/flink-examples-batch/pom.xml
@@ -24,7 +24,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-examples_2.10</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index adc819edb2d0a..2bacf252e107d 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-examples_2.10</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml
index 684d883f3d599..ce1c701da3e88 100644
--- a/flink-examples/flink-examples-table/pom.xml
+++ b/flink-examples/flink-examples-table/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-examples_2.10</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml
index a5939f5d8d3c8..409c8f1bda87b 100644
--- a/flink-examples/pom.xml
+++ b/flink-examples/pom.xml
@@ -24,7 +24,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml
index 84bba0a7c349b..c354c801a0557 100644
--- a/flink-fs-tests/pom.xml
+++ b/flink-fs-tests/pom.xml
@@ -23,7 +23,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index 55b3975c6da2c..24fc0f6d2e289 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-java8/pom.xml b/flink-java8/pom.xml
index fbc5b3a9d32de..8de3b92ccac4b 100644
--- a/flink-java8/pom.xml
+++ b/flink-java8/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml
index 57c594e6037d1..ca5bbced5c749 100644
--- a/flink-libraries/flink-cep-scala/pom.xml
+++ b/flink-libraries/flink-cep-scala/pom.xml
@@ -26,7 +26,7 @@ under the License.
     <parent>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-libraries</artifactId>
-        <version>1.3-SNAPSHOT</version>
+        <version>1.4-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
     
diff --git a/flink-libraries/flink-cep/pom.xml b/flink-libraries/flink-cep/pom.xml
index 1c9f8be80d6e2..7f89c24539734 100644
--- a/flink-libraries/flink-cep/pom.xml
+++ b/flink-libraries/flink-cep/pom.xml
@@ -26,7 +26,7 @@ under the License.
     <parent>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-libraries</artifactId>
-        <version>1.3-SNAPSHOT</version>
+        <version>1.4-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index e95aa37080a55..6fb0979d65744 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -23,7 +23,7 @@
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-libraries</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
index dad7237b5a7fc..256fc53cd33ae 100644
--- a/flink-libraries/flink-gelly-scala/pom.xml
+++ b/flink-libraries/flink-gelly-scala/pom.xml
@@ -23,7 +23,7 @@ under the License.
     <parent>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-libraries</artifactId>
-        <version>1.3-SNAPSHOT</version>
+        <version>1.4-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
diff --git a/flink-libraries/flink-gelly/pom.xml b/flink-libraries/flink-gelly/pom.xml
index fa091023ebac4..d656bb36131dc 100644
--- a/flink-libraries/flink-gelly/pom.xml
+++ b/flink-libraries/flink-gelly/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-libraries</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-libraries/flink-ml/pom.xml b/flink-libraries/flink-ml/pom.xml
index bea956e87d48c..8aa84d22478c7 100644
--- a/flink-libraries/flink-ml/pom.xml
+++ b/flink-libraries/flink-ml/pom.xml
@@ -24,7 +24,7 @@
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-libraries</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-libraries/flink-python/pom.xml b/flink-libraries/flink-python/pom.xml
index 8f75f2550ea8f..9d2f0c3c8b20a 100644
--- a/flink-libraries/flink-python/pom.xml
+++ b/flink-libraries/flink-python/pom.xml
@@ -23,7 +23,7 @@ under the License.
     <parent>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-libraries</artifactId>
-        <version>1.3-SNAPSHOT</version>
+        <version>1.4-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml
index c7440ab456786..8fa2ed27178d1 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -23,7 +23,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-libraries</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-libraries/pom.xml b/flink-libraries/pom.xml
index 6dbc692fe33d7..5b4cf7932810f 100644
--- a/flink-libraries/pom.xml
+++ b/flink-libraries/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-mesos/pom.xml b/flink-mesos/pom.xml
index eea5da7792bba..d03144fe8b1b4 100644
--- a/flink-mesos/pom.xml
+++ b/flink-mesos/pom.xml
@@ -23,7 +23,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 	
diff --git a/flink-metrics/flink-metrics-core/pom.xml b/flink-metrics/flink-metrics-core/pom.xml
index 658847f366beb..5f3a7a50ae23e 100644
--- a/flink-metrics/flink-metrics-core/pom.xml
+++ b/flink-metrics/flink-metrics-core/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-metrics</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-metrics/flink-metrics-dropwizard/pom.xml b/flink-metrics/flink-metrics-dropwizard/pom.xml
index de57729282c26..2ba12e65aa2a3 100644
--- a/flink-metrics/flink-metrics-dropwizard/pom.xml
+++ b/flink-metrics/flink-metrics-dropwizard/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-metrics</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-metrics/flink-metrics-ganglia/pom.xml b/flink-metrics/flink-metrics-ganglia/pom.xml
index 8f39edf5109bf..244dee7693197 100644
--- a/flink-metrics/flink-metrics-ganglia/pom.xml
+++ b/flink-metrics/flink-metrics-ganglia/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-metrics</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-metrics/flink-metrics-graphite/pom.xml b/flink-metrics/flink-metrics-graphite/pom.xml
index f8a21a022c9f6..b7d02bdad09de 100644
--- a/flink-metrics/flink-metrics-graphite/pom.xml
+++ b/flink-metrics/flink-metrics-graphite/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-metrics</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-metrics/flink-metrics-jmx/pom.xml b/flink-metrics/flink-metrics-jmx/pom.xml
index 8063deb1751f9..d738fc4133d7d 100644
--- a/flink-metrics/flink-metrics-jmx/pom.xml
+++ b/flink-metrics/flink-metrics-jmx/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-metrics</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-metrics/flink-metrics-statsd/pom.xml b/flink-metrics/flink-metrics-statsd/pom.xml
index a243b921d9d1e..80cfa324664ac 100644
--- a/flink-metrics/flink-metrics-statsd/pom.xml
+++ b/flink-metrics/flink-metrics-statsd/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-metrics</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-metrics/pom.xml b/flink-metrics/pom.xml
index 09fb990d4445c..317dde8fb8c30 100644
--- a/flink-metrics/pom.xml
+++ b/flink-metrics/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-optimizer/pom.xml b/flink-optimizer/pom.xml
index 11e2e16b20061..c347082777b0a 100644
--- a/flink-optimizer/pom.xml
+++ b/flink-optimizer/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-quickstart/flink-quickstart-java/pom.xml b/flink-quickstart/flink-quickstart-java/pom.xml
index 31b45f7cb2157..94574dc4adb20 100644
--- a/flink-quickstart/flink-quickstart-java/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/pom.xml
@@ -27,7 +27,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-quickstart</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 1e525fd8ca61d..2fa1835e88dd6 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -30,7 +30,7 @@ under the License.
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<flink.version>1.3-SNAPSHOT</flink.version>
+		<flink.version>1.4-SNAPSHOT</flink.version>
 		<slf4j.version>1.7.7</slf4j.version>
 		<log4j.version>1.2.17</log4j.version>
 	</properties>
diff --git a/flink-quickstart/flink-quickstart-scala/pom.xml b/flink-quickstart/flink-quickstart-scala/pom.xml
index 4a5bc97c7254b..875fcca286ae9 100644
--- a/flink-quickstart/flink-quickstart-scala/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/pom.xml
@@ -27,7 +27,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-quickstart</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 5abe496b54b73..189bbceea4f91 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -45,7 +45,7 @@ under the License.
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<flink.version>1.3-SNAPSHOT</flink.version>
+		<flink.version>1.4-SNAPSHOT</flink.version>
 		<slf4j.version>1.7.7</slf4j.version>
 		<log4j.version>1.2.17</log4j.version>
 	</properties>
diff --git a/flink-quickstart/pom.xml b/flink-quickstart/pom.xml
index 068bbc9f9291c..7e1b460b846da 100644
--- a/flink-quickstart/pom.xml
+++ b/flink-quickstart/pom.xml
@@ -23,7 +23,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-runtime-web/pom.xml b/flink-runtime-web/pom.xml
index a50e01f88902d..e7870f393f188 100644
--- a/flink-runtime-web/pom.xml
+++ b/flink-runtime-web/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index a6b9513803ccf..b7ccace69e216 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-scala-shell/pom.xml b/flink-scala-shell/pom.xml
index cf637d4afcbc9..0aadfc6a664e4 100644
--- a/flink-scala-shell/pom.xml
+++ b/flink-scala-shell/pom.xml
@@ -23,7 +23,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index f4c8246ddca3b..cb9b538dd6380 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -24,7 +24,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-shaded-curator/flink-shaded-curator-recipes/pom.xml b/flink-shaded-curator/flink-shaded-curator-recipes/pom.xml
index 5295a1acbfd33..fcc275941fd5b 100644
--- a/flink-shaded-curator/flink-shaded-curator-recipes/pom.xml
+++ b/flink-shaded-curator/flink-shaded-curator-recipes/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-shaded-curator</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-shaded-curator/flink-shaded-curator-test/pom.xml b/flink-shaded-curator/flink-shaded-curator-test/pom.xml
index fd9f1ce05d196..751b590efe0c8 100644
--- a/flink-shaded-curator/flink-shaded-curator-test/pom.xml
+++ b/flink-shaded-curator/flink-shaded-curator-test/pom.xml
@@ -24,7 +24,7 @@
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-shaded-curator</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-shaded-curator/pom.xml b/flink-shaded-curator/pom.xml
index be1da9c261395..d08320d364851 100644
--- a/flink-shaded-curator/pom.xml
+++ b/flink-shaded-curator/pom.xml
@@ -24,7 +24,7 @@
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
index c750bbda80584..9ac033e64cbb4 100644
--- a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
+++ b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-shaded-hadoop</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
index b7b2229d563fc..fe8733101188d 100644
--- a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
+++ b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-shaded-hadoop</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml
index 3df07566bdf62..1d957ab3b220e 100644
--- a/flink-shaded-hadoop/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml
index 013e958a7c1fd..ea987af0a9894 100644
--- a/flink-streaming-java/pom.xml
+++ b/flink-streaming-java/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml
index f95ad791295f2..4ec12bdb56971 100644
--- a/flink-streaming-scala/pom.xml
+++ b/flink-streaming-scala/pom.xml
@@ -24,7 +24,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-test-utils-parent/flink-test-utils-junit/pom.xml b/flink-test-utils-parent/flink-test-utils-junit/pom.xml
index aefd62d8865de..c8469cd24d058 100644
--- a/flink-test-utils-parent/flink-test-utils-junit/pom.xml
+++ b/flink-test-utils-parent/flink-test-utils-junit/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-test-utils-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml
index 14fca07da43b0..36bdcebc597e6 100644
--- a/flink-test-utils-parent/flink-test-utils/pom.xml
+++ b/flink-test-utils-parent/flink-test-utils/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-test-utils-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-test-utils-parent/pom.xml b/flink-test-utils-parent/pom.xml
index 92d6be3f19853..48e4550e15f5d 100644
--- a/flink-test-utils-parent/pom.xml
+++ b/flink-test-utils-parent/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index e5e3e891cf924..ff16349310005 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index 4b975960932ea..f296e8d76237f 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -23,7 +23,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
diff --git a/flink-yarn/pom.xml b/flink-yarn/pom.xml
index 11d238ace301e..5946094dbd651 100644
--- a/flink-yarn/pom.xml
+++ b/flink-yarn/pom.xml
@@ -23,7 +23,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>1.3-SNAPSHOT</version>
+		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 	
diff --git a/pom.xml b/pom.xml
index 71ad7792ae1cb..10677a4b3c585 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@ under the License.
 
 	<groupId>org.apache.flink</groupId>
 	<artifactId>flink-parent</artifactId>
-	<version>1.3-SNAPSHOT</version>
+	<version>1.4-SNAPSHOT</version>
 
 	<name>flink</name>
 	<packaging>pom</packaging>
@@ -121,7 +121,7 @@ under the License.
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>force-shading</artifactId>
-			<version>1.3-SNAPSHOT</version>
+			<version>1.4-SNAPSHOT</version>
 		</dependency>
 
 		<!-- Root dependencies for all projects -->
diff --git a/tools/change-version.sh b/tools/change-version.sh
index 45ed2faf3705d..cafefd0b88717 100755
--- a/tools/change-version.sh
+++ b/tools/change-version.sh
@@ -17,8 +17,8 @@
 # limitations under the License.
 ################################################################################
 
-OLD="1.2-SNAPSHOT"
-NEW="1.3-SNAPSHOT"
+OLD="1.3-SNAPSHOT"
+NEW="1.4-SNAPSHOT"
 
 
 HERE=` basename "$PWD"`
diff --git a/tools/force-shading/pom.xml b/tools/force-shading/pom.xml
index 3de6a5a7be219..bbad337197ba8 100644
--- a/tools/force-shading/pom.xml
+++ b/tools/force-shading/pom.xml
@@ -38,7 +38,7 @@ under the License.
 
 	<groupId>org.apache.flink</groupId>
 	<artifactId>force-shading</artifactId>
-	<version>1.3-SNAPSHOT</version>
+	<version>1.4-SNAPSHOT</version>
 
 	<packaging>jar</packaging>
 

From 7a57785c6dd91e3fbe5c0a076b528278bc6ffff4 Mon Sep 17 00:00:00 2001
From: Kurt Young <ykt836@gmail.com>
Date: Sun, 30 Apr 2017 16:56:00 +0800
Subject: [PATCH 189/253] [FLINK-6394] [runtime] Respect object reuse
 configuration when executing group combining function

This closes #3803.
---
 .../sort/CombiningUnilateralSortMerger.java   | 48 +++++++++----
 .../util/NonReusingKeyGroupedIterator.java    |  1 +
 .../CombiningUnilateralSortMergerITCase.java  | 71 ++++++++++++++++++-
 3 files changed, 107 insertions(+), 13 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
index a02ced2adabbd..5500f37f10bce 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
@@ -33,6 +33,7 @@
 import org.apache.flink.runtime.memory.MemoryAllocationException;
 import org.apache.flink.runtime.memory.MemoryManager;
 import org.apache.flink.runtime.util.EmptyMutableObjectIterator;
+import org.apache.flink.runtime.util.NonReusingKeyGroupedIterator;
 import org.apache.flink.runtime.util.ReusingKeyGroupedIterator;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.MutableObjectIterator;
@@ -162,7 +163,8 @@ protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptio
 		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
 	{
 		return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
-			memoryManager, ioManager, serializerFactory.getSerializer(), comparator, sortReadMemory, writeMemory, maxFileHandles);
+				memoryManager, ioManager, serializerFactory.getSerializer(),
+				comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
 	}
 
 	// ------------------------------------------------------------------------
@@ -172,16 +174,20 @@ protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptio
 	protected class CombiningSpillingThread extends SpillingThread {
 		
 		private final TypeComparator<E> comparator2;
-		
+
+		private final boolean objectReuseEnabled;
+
 		public CombiningSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
 				AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
 				TypeSerializer<E> serializer, TypeComparator<E> comparator, 
-				List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
+				List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles,
+				boolean objectReuseEnabled)
 		{
 			super(exceptionHandler, queues, parentTask, memManager, ioManager, serializer, comparator, 
 				sortReadMemory, writeMemory, maxNumFileHandles);
 			
 			this.comparator2 = comparator.duplicate();
+			this.objectReuseEnabled = objectReuseEnabled;
 		}
 
 		/**
@@ -315,7 +321,8 @@ else if (element == endMarker()) {
 
 				// set up the combining helpers
 				final InMemorySorter<E> buffer = element.buffer;
-				final CombineValueIterator<E> iter = new CombineValueIterator<E>(buffer, this.serializer.createInstance());
+				final CombineValueIterator<E> iter = new CombineValueIterator<E>(
+						buffer, this.serializer.createInstance(), this.objectReuseEnabled);
 				final WriterCollector<E> collector = new WriterCollector<E>(output, this.serializer);
 
 				int i = 0;
@@ -454,7 +461,6 @@ protected ChannelWithBlockCount mergeChannels(List<ChannelWithBlockCount> channe
 
 			// the list with the target iterators
 			final MergeIterator<E> mergeIterator = getMergingIterator(channelIDs, readBuffers, channelAccesses, null);
-			final ReusingKeyGroupedIterator<E> groupedIter = new ReusingKeyGroupedIterator<E>(mergeIterator, this.serializer, this.comparator2);
 
 			// create a new channel writer
 			final FileIOChannel.ID mergedChannelID = this.ioManager.createChannel();
@@ -469,8 +475,18 @@ protected ChannelWithBlockCount mergeChannels(List<ChannelWithBlockCount> channe
 
 			// combine and write to disk
 			try {
-				while (groupedIter.nextKey()) {
-					combineStub.combine(groupedIter.getValues(), collector);
+				if (objectReuseEnabled) {
+					final ReusingKeyGroupedIterator<E> groupedIter = new ReusingKeyGroupedIterator<>(
+							mergeIterator, this.serializer, this.comparator2);
+					while (groupedIter.nextKey()) {
+						combineStub.combine(groupedIter.getValues(), collector);
+					}
+				} else {
+					final NonReusingKeyGroupedIterator<E> groupedIter = new NonReusingKeyGroupedIterator<>(
+							mergeIterator, this.comparator2);
+					while (groupedIter.nextKey()) {
+						combineStub.combine(groupedIter.getValues(), collector);
+					}
 				}
 			}
 			catch (Exception e) {
@@ -505,7 +521,9 @@ private static final class CombineValueIterator<E> implements Iterator<E>, Itera
 		
 		private final InMemorySorter<E> buffer; // the buffer from which values are returned
 		
-		private E record;
+		private E recordReuse;
+
+		private final boolean objectReuseEnabled;
 
 		private int last; // the position of the last value to be returned
 
@@ -519,9 +537,10 @@ private static final class CombineValueIterator<E> implements Iterator<E>, Itera
 		 * @param buffer
 		 *        The buffer to get the values from.
 		 */
-		public CombineValueIterator(InMemorySorter<E> buffer, E instance) {
+		public CombineValueIterator(InMemorySorter<E> buffer, E instance, boolean objectReuseEnabled) {
 			this.buffer = buffer;
-			this.record = instance;
+			this.recordReuse = instance;
+			this.objectReuseEnabled = objectReuseEnabled;
 		}
 
 		/**
@@ -547,9 +566,14 @@ public boolean hasNext() {
 		public E next() {
 			if (this.position <= this.last) {
 				try {
-					this.record = this.buffer.getRecord(this.record, this.position);
+					E record;
+					if (objectReuseEnabled) {
+						record = this.buffer.getRecord(this.recordReuse, this.position);
+					} else {
+						record = this.buffer.getRecord(this.position);
+					}
 					this.position++;
-					return this.record;
+					return record;
 				}
 				catch (IOException ioex) {
 					LOG.error("Error retrieving a value from a buffer.", ioex);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/NonReusingKeyGroupedIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/NonReusingKeyGroupedIterator.java
index 6f4448c9b33bc..221cf243d40a2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/NonReusingKeyGroupedIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/NonReusingKeyGroupedIterator.java
@@ -164,6 +164,7 @@ public TypeComparator<E> getComparatorWithCurrentReference() {
 	 * 
 	 * @return Iterator over all values that belong to the current key.
 	 */
+	@Override
 	public ValuesIterator getValues() {
 		return this.valuesIterator;
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
index fba0a6fd7f8f3..fa675f69e24c3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.operators.sort;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Hashtable;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
@@ -183,6 +184,42 @@ public void testCombineSpilling() throws Exception {
 		Assert.assertTrue(comb.opened == comb.closed);
 	}
 
+	@Test
+	public void testCombineSpillingDisableObjectReuse() throws Exception {
+		int noKeys = 100;
+		int noKeyCnt = 10000;
+
+		TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();
+
+		LOG.debug("initializing sortmerger");
+
+		MaterializedCountCombiner comb = new MaterializedCountCombiner();
+
+		// set maxNumFileHandles = 2 to trigger multiple channel merging
+		Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
+				this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
+				0.01, 2, 0.005f, true /* use large record handler */, false);
+
+		final Tuple2<Integer, Integer> rec = new Tuple2<>();
+
+		for (int i = 0; i < noKeyCnt; i++) {
+			rec.setField(i, 0);
+			for (int j = 0; j < noKeys; j++) {
+				rec.setField(j, 1);
+				reader.emit(rec);
+			}
+		}
+		reader.close();
+
+		MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();
+		Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
+		while (result.hasNext()) {
+			Assert.assertEquals(4950, result.next().intValue());
+		}
+
+		merger.close();
+	}
+
 	@Test
 	public void testSortAndValidate() throws Exception
 	{
@@ -332,7 +369,39 @@ public void close() throws Exception {
 			closed = true;
 		}
 	}
-	
+
+	// --------------------------------------------------------------------------------------------
+
+	public static class MaterializedCountCombiner
+			extends RichGroupReduceFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>
+			implements GroupCombineFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>
+	{
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public void combine(Iterable<Tuple2<Integer, Integer>> values, Collector<Tuple2<Integer, Integer>> out) {
+			ArrayList<Tuple2<Integer, Integer>> valueList = new ArrayList<>();
+			for (Tuple2<Integer, Integer> next : values) {
+				valueList.add(next);
+			}
+
+			int count = 0;
+			Tuple2<Integer, Integer> rec = new Tuple2<>();
+			for (Tuple2<Integer, Integer> tuple : valueList) {
+				rec.setField(tuple.f0, 0);
+				count += tuple.f1;
+			}
+			rec.setField(count, 1);
+			out.collect(rec);
+		}
+
+		@Override
+		public void reduce(Iterable<Tuple2<Integer, Integer>> values,
+				Collector<Tuple2<Integer, Integer>> out) throws Exception
+		{
+		}
+	}
+
 	private static Iterator<Integer> getReducingIterator(MutableObjectIterator<Tuple2<Integer, Integer>> data, TypeSerializer<Tuple2<Integer, Integer>> serializer, TypeComparator<Tuple2<Integer, Integer>>  comparator) {
 		
 		final ReusingKeyGroupedIterator<Tuple2<Integer, Integer>>  groupIter = new ReusingKeyGroupedIterator<> (data, serializer, comparator);

From 0d66b42d3f32eda4ff0238c465a2071ca03427e2 Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Mon, 8 May 2017 12:09:22 +0200
Subject: [PATCH 190/253] [FLINK-6506] [build] Tests in flink-tests exceed
 memory resources on containerized Travis

---
 flink-tests/pom.xml          | 3 +++
 tools/travis_mvn_watchdog.sh | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index ff16349310005..ebee6ef471f3b 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -342,6 +342,9 @@ under the License.
 						<classpathDependencyExclude>org.apache.curator:curator-client</classpathDependencyExclude>
 						<classpathDependencyExclude>org.apache.curator:curator-framework</classpathDependencyExclude>
 					</classpathDependencyExcludes>
+					<!-- We override the fork behaviour for those expensive tests to avoid process
+					kills due to container limits on travis -->
+					<forkCount>${flink.forkCountTestPackage}</forkCount>
 					<reuseForks>false</reuseForks>
 					<environmentVariables>
 						<!-- Make sure external hadoop environment will not affect maven building -->
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 20afa87a83195..d747f911471de 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -45,7 +45,7 @@ LOG4J_PROPERTIES=${HERE}/log4j-travis.properties
 
 # Maven command to run. We set the forkCount manually, because otherwise Maven sees too many cores
 # on the Travis VMs.
-MVN="mvn -Dflink.forkCount=2 -B $PROFILE -Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES clean install"
+MVN="mvn -Dflink.forkCount=2 -Dflink.forkCountTestPackage=1 -B $PROFILE -Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES clean install"
 
 MVN_PID="${ARTIFACTS_DIR}/watchdog.mvn.pid"
 MVN_EXIT="${ARTIFACTS_DIR}/watchdog.mvn.exit"
@@ -75,6 +75,7 @@ upload_artifacts_s3() {
 	echo "COMPRESSING build artifacts."
 
 	cd $ARTIFACTS_DIR
+	dmesg > container.log
 	tar -zcvf $ARTIFACTS_FILE *
 
 	# Upload to secured S3

From caa230c2369fcb2e8ebacf9a6f6f223feba80267 Mon Sep 17 00:00:00 2001
From: David Anderson <david@alpinegizmo.com>
Date: Thu, 4 May 2017 13:51:36 +0200
Subject: [PATCH 191/253] [FLINK-5742] [docs] allow sidenav to work for widths
 down to 992

This closes #3821
---
 docs/_layouts/base.html | 4 ++--
 docs/page/css/flink.css | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/_layouts/base.html b/docs/_layouts/base.html
index 35499cd2867b9..88df5c974487a 100644
--- a/docs/_layouts/base.html
+++ b/docs/_layouts/base.html
@@ -69,10 +69,10 @@
       the _layouts directory goes here.
       {% endcomment %}
       <div class="row">
-        <div class="col-lg-3">
+        <div class="col-lg-3" id="sidenavcol">
           {% include sidenav.html %}
         </div>
-        <div class="col-lg-9 content">
+        <div class="col-lg-9 content" id="contentcol">
           {% if page.mathjax %}
           {% include latex_commands.html %}
           {% endif %}
diff --git a/docs/page/css/flink.css b/docs/page/css/flink.css
index b6d49472535f1..c0a7917e8fe39 100644
--- a/docs/page/css/flink.css
+++ b/docs/page/css/flink.css
@@ -30,6 +30,11 @@ body {
 	background: #f8f8f8;
 }
 
+@media (min-width: 992px) and (max-width: 1200px) {
+    #contentcol { float: left; width: 70%; }
+    #sidenavcol { float: left; width: 30%; }
+}
+
 /*=============================================================================
  Per page TOC
 =============================================================================*/

From 7ff46c2815e6ca5306ffb60cbf7f2ea18953ece5 Mon Sep 17 00:00:00 2001
From: David Anderson <david@alpinegizmo.com>
Date: Wed, 3 May 2017 16:44:03 +0200
Subject: [PATCH 192/253] [FLINK-6438] [docs] Added a few links to the docs
 home page, and made some other small adjustments.

This closes #3823
---
 docs/index.md | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/docs/index.md b/docs/index.md
index a74f88028465c..ef9317fa2eef5 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -33,9 +33,13 @@ Apache Flink is an open source platform for distributed stream and batch data pr
 
 - **Concepts**: Start with the basic concepts of Flink's [Dataflow Programming Model](concepts/programming-model.html) and [Distributed Runtime Environment](concepts/runtime.html). This will help you to fully understand the other parts of the documentation, including the setup and programming guides. It is highly recommended to read these sections first.
 
-- **Quickstarts**: [Run an example program](quickstart/setup_quickstart.html) on your local machine or [write a simple program](quickstart/run_example_quickstart.html) working on live Wikipedia edits.
+- **Quickstarts**: [Run an example program](quickstart/setup_quickstart.html) on your local machine or [study some examples](examples/index.html).
 
-- **Programming Guides**: You can check out our guides about [basic concepts](dev/api_concepts.html) and the [DataStream API](dev/datastream_api.html) or [DataSet API](dev/batch/index.html) to learn how to write your first Flink programs.
+- **Programming Guides**: You can check out our guides about [basic API concepts](dev/api_concepts.html) and the [DataStream API](dev/datastream_api.html) or [DataSet API](dev/batch/index.html) to learn how to write your first Flink programs.
+
+## Deployment
+
+Before putting your Flink job into production, be sure to read the [Production Readiness Checklist](ops/production_ready.html).
 
 ## Migration Guide
 
@@ -44,3 +48,12 @@ While all parts of the API that were marked as public and stable are still suppo
 newer interfaces where applicable.
 
 For users that plan to upgrade a Flink system in production, we recommend reading the guide on [upgrading Apache Flink](ops/upgrading.html).
+
+## External Resources
+
+- **Flink Forward**: Talks from past conferences are available at the [Flink Forward](http://flink-forward.org/) website and on [YouTube](https://www.youtube.com/channel/UCY8_lgiZLZErZPF47a2hXMA). [Robust Stream Processing with Apache Flink](http://berlin.flink-forward.org/kb_sessions/robust-stream-processing-with-apache-flink/) is a good place to start.
+
+- **Training**: The [training materials](http://dataartisans.github.io/flink-training/) from data Artisans include slides, exercises, and sample solutions.
+
+- **Blogs**: The [Apache Flink](https://flink.apache.org/blog/) and [data Artisans](https://data-artisans.com/blog/) blogs publish frequent,
+in-depth technical articles about Flink.

From b94a8a17c011498b724498c10ff163de85d7d18a Mon Sep 17 00:00:00 2001
From: sunjincheng121 <sunjincheng121@gmail.com>
Date: Mon, 8 May 2017 16:41:31 +0800
Subject: [PATCH 193/253] [FLINK-6479] [table] Fix IOOBE in
 DataStreamGroupWindowAggregate.

This closes #3841.
---
 .../DataStreamGroupWindowAggregate.scala      |  2 +-
 .../table/GroupWindowAggregationsITCase.scala | 32 +++++++++++++++
 .../scala/stream/table/GroupWindowTest.scala  | 41 ++++++++++++++++++-
 3 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index 1be1896a96fdf..c38e5af560768 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -134,7 +134,7 @@ class DataStreamGroupWindowAggregate(
       namedAggregates,
       namedProperties)
 
-    val keyedAggOpName = s"groupBy: (${groupingToString(schema.logicalType, grouping)}), " +
+    val keyedAggOpName = s"groupBy: (${groupingToString(inputSchema.logicalType, grouping)}), " +
       s"window: ($window), " +
       s"select: ($aggString)"
     val nonKeyedAggOpName = s"window: ($window), select: ($aggString)"
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
index 2c027a9046d4d..846fe3e83991c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
@@ -175,6 +175,38 @@ class GroupWindowAggregationsITCase extends StreamingMultipleProgramsTestBase {
       "Hi,1,1,1,1,1,1,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
+
+  @Test
+  def testGroupWindowWithoutKeyInProjection(): Unit = {
+    val data = List(
+      (1L, 1, "Hi", 1, 1),
+      (2L, 2, "Hello", 2, 2),
+      (4L, 2, "Hello", 2, 2),
+      (8L, 3, "Hello world", 3, 3),
+      (16L, 3, "Hello world", 3, 3))
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setParallelism(1)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env.fromCollection(data)
+    val table = stream.toTable(tEnv, 'long, 'int, 'string, 'int2, 'int3, 'proctime.proctime)
+
+    val weightAvgFun = new WeightedAvg
+
+    val windowedTable = table
+      .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
+      .groupBy('w, 'int2, 'int3, 'string)
+      .select(weightAvgFun('long, 'int))
+
+    val results = windowedTable.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq("12", "8", "2", "3", "1")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
 }
 
 object GroupWindowAggregationsITCase {
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index 0573ff3323159..ef071b777f8aa 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -19,7 +19,7 @@
 package org.apache.flink.table.api.scala.stream.table
 
 import org.apache.flink.api.scala._
-import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge}
 import org.apache.flink.table.api.ValidationException
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.expressions.WindowReference
@@ -791,6 +791,45 @@ class GroupWindowTest extends TableTestBase {
     util.verifyTable(windowedTable, expected)
   }
 
+  @Test
+  def testSlidingWindowWithUDAF(): Unit = {
+    val util = streamTestUtil()
+    val table = util.addTable[(Long, Int, String, Int, Int)](
+      'long,
+      'int,
+      'string,
+      'int2,
+      'int3,
+      'proctime.proctime)
+
+    val weightAvgFun = new WeightedAvg
+
+    val windowedTable = table
+      .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
+      .groupBy('w, 'int2, 'int3, 'string)
+      .select(weightAvgFun('long, 'int))
+
+    val expected =
+      unaryNode(
+        "DataStreamCalc",
+        unaryNode(
+          "DataStreamGroupWindowAggregate",
+          streamTableNode(0),
+          term("groupBy", "string, int2, int3"),
+          term("window", SlidingGroupWindow(WindowReference("w"), 'proctime,  2.rows, 1.rows)),
+          term(
+            "select",
+            "string",
+            "int2",
+            "int3",
+            "WeightedAvg(long, int) AS TMP_0")
+        ),
+        term("select","TMP_0")
+      )
+
+    util.verifyTable(windowedTable, expected)
+  }
+
   @Test
   def testSlideWindowStartEnd(): Unit = {
     val util = streamTestUtil()

From 10be40009e8395dac0113f0e31c42afe58146576 Mon Sep 17 00:00:00 2001
From: godfreyhe <godfreyhe@163.com>
Date: Wed, 3 May 2017 20:59:34 +0800
Subject: [PATCH 194/253] [FLINK-6436] [table] Fix code-gen bug when using a
 scalar UDF in a UDTF join condition.

This closes #3815.
---
 .../table/plan/nodes/CommonCorrelate.scala    | 13 +++++++++-
 .../utils/UserDefinedScalarFunctions.scala    |  9 +++++--
 .../DataSetUserDefinedFunctionITCase.scala    | 24 ++++++++++++++++---
 .../DataStreamUserDefinedFunctionITCase.scala | 22 +++++++++++++++--
 4 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
index 44a109e3b2b02..c95f2f78d29c1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
@@ -18,7 +18,7 @@
 package org.apache.flink.table.plan.nodes
 
 import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rex.{RexCall, RexNode}
+import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode, RexShuttle}
 import org.apache.calcite.sql.SemiJoinType
 import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.api.common.typeinfo.TypeInformation
@@ -143,6 +143,17 @@ trait CommonCorrelate[T] {
          |getCollector().collect(${crossResultExpr.resultTerm});
          |""".stripMargin
     } else {
+
+      // adjust indicies of InputRefs to adhere to schema expected by generator
+      val changeInputRefIndexShuttle = new RexShuttle {
+        override def visitInputRef(inputRef: RexInputRef): RexNode = {
+          new RexInputRef(inputSchema.physicalArity + inputRef.getIndex, inputRef.getType)
+        }
+      }
+      // Run generateExpression to add init statements (ScalarFunctions) of condition to generator.
+      //   The generated expression is discarded.
+      generator.generateExpression(condition.get.accept(changeInputRefIndexShuttle))
+
       val filterGenerator = new CodeGenerator(config, false, udtfTypeInfo, None, pojoFieldMapping)
       filterGenerator.input1Term = filterGenerator.input2Term
       val filterCondition = filterGenerator.generateExpression(condition.get)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
index 8972a7719ca5f..528556968bf56 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
@@ -25,11 +25,10 @@ import org.apache.flink.table.api.Types
 import org.apache.flink.table.functions.{ScalarFunction, FunctionContext}
 import org.junit.Assert
 
+import scala.annotation.varargs
 import scala.collection.mutable
 import scala.io.Source
 
-import scala.annotation.varargs
-
 case class SimplePojo(name: String, age: Int)
 
 object Func0 extends ScalarFunction {
@@ -263,3 +262,9 @@ object Func17 extends ScalarFunction {
     a.mkString(", ")
   }
 }
+
+object Func18 extends ScalarFunction {
+  def eval(str: String, prefix: String): Boolean = {
+    str.startsWith(prefix)
+  }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala
index 20bbf8b3aa47f..b69dd494f8f74 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala
@@ -24,9 +24,9 @@ import org.apache.flink.api.scala.util.CollectionDataSets
 import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.api.java.utils.UserDefinedTableFunctions.JavaTableFunc0
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
 import org.apache.flink.table.api.scala.batch.utils.TableProgramsClusterTestBase
-import org.apache.flink.table.expressions.utils.{Func13, RichFunc2}
+import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
+import org.apache.flink.table.expressions.utils.{Func1, Func13, Func18, RichFunc2}
 import org.apache.flink.table.utils._
 import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
 import org.apache.flink.test.util.TestBaseUtils
@@ -143,7 +143,7 @@ class DataSetUserDefinedFunctionITCase(
     val pojo = new PojoTableFunc()
     val result = in
       .join(pojo('c))
-      .where(('age > 20))
+      .where('age > 20)
       .select('c, 'name, 'age)
       .toDataSet[Row]
 
@@ -170,6 +170,24 @@ class DataSetUserDefinedFunctionITCase(
     TestBaseUtils.compareResultAsText(results.asJava, expected)
   }
 
+  @Test
+  def testUserDefinedTableFunctionWithScalarFunctionInCondition(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tableEnv = TableEnvironment.getTableEnvironment(env, config)
+    val in = testData(env).toTable(tableEnv).as('a, 'b, 'c)
+    val func0 = new TableFunc0
+
+    val result = in
+      .join(func0('c))
+      .where(Func18('name, "J") && (Func1('a) < 3) && Func1('age) > 20)
+      .select('c, 'name, 'age)
+      .toDataSet[Row]
+
+    val results = result.collect()
+    val expected = "Jack#22,Jack,22"
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
   @Test
   def testLongAndTemporalTypes(): Unit = {
     val env = ExecutionEnvironment.getExecutionEnvironment
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala
index 2e8a065e79c42..b3d9c6f7e95e6 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala
@@ -23,7 +23,7 @@ import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
 import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData}
-import org.apache.flink.table.expressions.utils.{Func13, RichFunc2}
+import org.apache.flink.table.expressions.utils.{Func13, Func18, RichFunc2}
 import org.apache.flink.table.utils._
 import org.apache.flink.types.Row
 import org.junit.Assert._
@@ -51,7 +51,7 @@ class DataStreamUserDefinedFunctionITCase extends StreamingMultipleProgramsTestB
       .join(func0('c) as('d, 'e))
       .select('c, 'd, 'e)
       .join(pojoFunc0('c))
-      .where(('age > 20))
+      .where('age > 20)
       .select('c, 'name, 'age)
       .toDataStream[Row]
 
@@ -81,6 +81,24 @@ class DataStreamUserDefinedFunctionITCase extends StreamingMultipleProgramsTestB
     assertEquals(expected.sorted, StreamITCase.testResults.sorted)
   }
 
+  @Test
+  def testUserDefinedTableFunctionWithScalarFunction(): Unit = {
+    val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
+    val func0 = new TableFunc0
+
+    val result = t
+      .join(func0('c) as('d, 'e))
+      .where(Func18('d, "J"))
+      .select('c, 'd, 'e)
+      .toDataStream[Row]
+
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = mutable.MutableList("Jack#22,Jack,22", "John#19,John,19")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
   @Test
   def testUserDefinedTableFunctionWithParameter(): Unit = {
     val tableFunc1 = new RichTableFunc1

From f9fd38d58cfdce65356b847e26e1c1fa172d11e7 Mon Sep 17 00:00:00 2001
From: Hequn Cheng <chenghequn@gmail.com>
Date: Mon, 8 May 2017 20:55:51 +0800
Subject: [PATCH 195/253] [FLINK-6486] [table] Pass RowTypeInfo to
 CodeGenerator instead of CRowTypeInfo.

This closes #3850.
---
 .../table/plan/nodes/datastream/DataStreamGroupAggregate.scala  | 2 +-
 .../plan/nodes/datastream/DataStreamGroupWindowAggregate.scala  | 2 +-
 .../table/plan/nodes/datastream/DataStreamOverAggregate.scala   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index 18f1fc89b76ed..506c0cb930c86 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -115,7 +115,7 @@ class DataStreamGroupAggregate(
     val generator = new CodeGenerator(
       tableEnv.getConfig,
       false,
-      inputDS.getType)
+      inputSchema.physicalTypeInfo)
 
     val aggString = aggregationToString(
       inputSchema.logicalType,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index c38e5af560768..ef207b0adba25 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -142,7 +142,7 @@ class DataStreamGroupWindowAggregate(
     val generator = new CodeGenerator(
       tableEnv.getConfig,
       false,
-      inputDS.getType)
+      inputSchema.physicalTypeInfo)
 
     val needMerge = window match {
       case SessionGroupWindow(_, _, _) => true
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index e823cd6a5b9b9..40612421a65fb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -116,7 +116,7 @@ class DataStreamOverAggregate(
     val generator = new CodeGenerator(
       tableEnv.getConfig,
       false,
-      inputDS.getType)
+      inputSchema.physicalTypeInfo)
 
     val timeType = schema.logicalType
       .getFieldList

From f55f4b2f2dfbc8080db4f405d3b3bd9778b09110 Mon Sep 17 00:00:00 2001
From: rtudoran <tudoranradu@ymail.com>
Date: Mon, 8 May 2017 20:30:57 +0200
Subject: [PATCH 196/253] [FLINK-6476] [table] Add support to convert
 DataSet[Row] and DataStream[Row] to Table.

This closes #3849.
---
 .../flink/table/api/TableEnvironment.scala    | 18 ++++++++
 .../table/api/java/stream/sql/SqlITCase.java  | 42 +++++++++++++++++++
 .../flink/table/TableEnvironmentTest.scala    | 22 ++++++++++
 .../api/scala/stream/sql/SqlITCase.scala      | 36 ++++++++++++++++
 4 files changed, 118 insertions(+)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index bb0de3e529037..bf4a8e072037a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -63,6 +63,7 @@ import org.apache.flink.table.sinks.TableSink
 import org.apache.flink.table.sources.{DefinedFieldNames, TableSource}
 import org.apache.flink.table.validate.FunctionCatalog
 import org.apache.flink.types.Row
+import org.apache.flink.api.java.typeutils.RowTypeInfo
 
 import _root_.scala.collection.JavaConverters._
 import _root_.scala.collection.mutable.HashMap
@@ -677,6 +678,23 @@ abstract class TableEnvironment(val config: TableConfig) {
           case _ => throw new TableException(
             "Field reference expression or alias on field expression expected.")
         }
+      case r: RowTypeInfo => {
+        exprs.zipWithIndex flatMap {
+          case (UnresolvedFieldReference(name), idx) =>
+            Some((idx, name))
+          case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
+            val idx = r.getFieldIndex(origName)
+            if (idx < 0) {
+              throw new TableException(s"$origName is not a field of type $r")
+            }
+            Some((idx, name))
+          case (_: TimeAttribute, _) =>
+            None
+          case _ => throw new TableException(
+            "Field reference expression or alias on field expression expected.")
+        }
+        
+      }
       case tpe => throw new TableException(
         s"Source of type $tpe cannot be converted into Table.")
     }
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java
index 7c01d2bbfe618..0c0b37e32acdd 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java
@@ -30,11 +30,53 @@
 import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
 import org.apache.flink.table.api.java.stream.utils.StreamTestData;
 import org.junit.Test;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
 
 import java.util.ArrayList;
 import java.util.List;
 
 public class SqlITCase extends StreamingMultipleProgramsTestBase {
+	
+	@Test
+	public void testRowRegisterRowWithNames() throws Exception {
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+		StreamITCase.clear();
+
+		List<Row> data = new ArrayList<>();
+		data.add(Row.of(1, 1L, "Hi"));
+		data.add(Row.of(2, 2L, "Hello"));
+		data.add(Row.of(3, 2L, "Hello world"));
+		
+		TypeInformation<?>[] types = {
+				BasicTypeInfo.INT_TYPE_INFO,
+				BasicTypeInfo.LONG_TYPE_INFO,
+				BasicTypeInfo.STRING_TYPE_INFO};
+		String names[] = {"a","b","c"};
+		
+		RowTypeInfo typeInfo = new RowTypeInfo(types, names);
+		
+		DataStream<Row> ds = env.fromCollection(data).returns(typeInfo);
+		
+		Table in = tableEnv.fromDataStream(ds, "a,b,c");
+		tableEnv.registerTable("MyTableRow", in);
+
+		String sqlQuery = "SELECT a,c FROM MyTableRow";
+		Table result = tableEnv.sql(sqlQuery);
+
+		DataStream<Row> resultSet = tableEnv.toDataStream(result, Row.class);
+		resultSet.addSink(new StreamITCase.StringSink());
+		env.execute();
+
+		List<String> expected = new ArrayList<>();
+		expected.add("1,Hi");
+		expected.add("2,Hello");
+		expected.add("3,Hello world");
+
+		StreamITCase.compareWithList(expected);
+	}
 
 	@Test
 	public void testSelect() throws Exception {
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index ba3b5918f5958..98a8edbd6f5a6 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -53,6 +53,28 @@ class TableEnvironmentTest extends TableTestBase {
 
   val genericRowType = new GenericTypeInfo[Row](classOf[Row])
 
+  @Test
+  def testGetFieldInfoRow(): Unit = {
+    val fieldInfo = tEnv.getFieldInfo(rowType)
+
+    fieldInfo._1.zip(Array("f0", "f1", "f2")).foreach(x => assertEquals(x._2, x._1))
+    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
+  }
+  
+  @Test
+  def testGetFieldInfoRowNames(): Unit = {
+    val fieldInfo = tEnv.getFieldInfo(
+      rowType,
+      Array(
+        UnresolvedFieldReference("name1"),
+        UnresolvedFieldReference("name2"),
+        UnresolvedFieldReference("name3")
+      ))
+
+    fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
+    fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
+  }
+  
   @Test
   def testGetFieldInfoTuple(): Unit = {
     val fieldInfo = tEnv.getFieldInfo(tupleType)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
index 4147358a7c959..ba8c1850d1485 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
@@ -23,12 +23,48 @@ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.types.Row
 import org.junit.Assert._
 import org.junit._
 
 class SqlITCase extends StreamingWithStateTestBase {
 
+   /** test row stream registered table **/
+  @Test
+  def testRowRegister(): Unit = {
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.clear
+
+    val sqlQuery = "SELECT * FROM MyTableRow WHERE c < 3"
+
+    val data = List(
+      Row.of("Hello", "Worlds", Int.box(1)),
+      Row.of("Hello", "Hiden", Int.box(5)),
+      Row.of("Hello again", "Worlds", Int.box(2)))
+        
+    implicit val tpe: TypeInformation[Row] = new RowTypeInfo(
+      BasicTypeInfo.STRING_TYPE_INFO,
+      BasicTypeInfo.STRING_TYPE_INFO,
+      BasicTypeInfo.INT_TYPE_INFO) // tpe is automatically 
+    
+    val ds = env.fromCollection(data)
+    
+    val t = ds.toTable(tEnv).as('a, 'b, 'c)
+    tEnv.registerTable("MyTableRow", t)
+
+    val result = tEnv.sql(sqlQuery).toDataStream[Row]
+    result.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = List("Hello,Worlds,1","Hello again,Worlds,2")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+    
   /** test unbounded groupby (without window) **/
   @Test
   def testUnboundedGroupby(): Unit = {

From 01cb821dcace6fc6d3dfcd2b9814a341b559c67d Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Wed, 15 Feb 2017 11:05:38 +0100
Subject: [PATCH 197/253] [FLINK-5070] [types] Unable to use Scala's
 BeanProperty with classes

This closes #3318
---
 .../api/java/typeutils/TypeExtractor.java     | 12 ++------
 .../scala/typeutils/TypeExtractionTest.scala  | 28 +++++++++++++++----
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 2b9eed9e013fa..a5f236ff62bae 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -1711,9 +1711,6 @@ private boolean isValidPojoField(Field f, Class<?> clazz, ArrayList<Type> typeHi
 					// return type is same as field type (or the generic variant of it)
 					(m.getGenericReturnType().equals( fieldType ) || (fieldTypeWrapper != null && m.getReturnType().equals( fieldTypeWrapper )) || (fieldTypeGeneric != null && m.getGenericReturnType().equals(fieldTypeGeneric)) )
 				) {
-					if(hasGetter) {
-						throw new IllegalStateException("Detected more than one getter");
-					}
 					hasGetter = true;
 				}
 				// check for setters (<FieldName>_$eq for scala)
@@ -1723,9 +1720,6 @@ private boolean isValidPojoField(Field f, Class<?> clazz, ArrayList<Type> typeHi
 					// return type is void.
 					m.getReturnType().equals(Void.TYPE)
 				) {
-					if(hasSetter) {
-						throw new IllegalStateException("Detected more than one setter");
-					}
 					hasSetter = true;
 				}
 			}
@@ -1733,10 +1727,10 @@ private boolean isValidPojoField(Field f, Class<?> clazz, ArrayList<Type> typeHi
 				return true;
 			} else {
 				if(!hasGetter) {
-					LOG.debug(clazz+" does not contain a getter for field "+f.getName() );
+					LOG.info(clazz+" does not contain a getter for field "+f.getName() );
 				}
 				if(!hasSetter) {
-					LOG.debug(clazz+" does not contain a setter for field "+f.getName() );
+					LOG.info(clazz+" does not contain a setter for field "+f.getName() );
 				}
 				return false;
 			}
@@ -1771,7 +1765,7 @@ else if (typeHierarchy.size() <= 1) {
 		for (Field field : fields) {
 			Type fieldType = field.getGenericType();
 			if(!isValidPojoField(field, clazz, typeHierarchy)) {
-				LOG.info(clazz + " is not a valid POJO type");
+				LOG.info(clazz + " is not a valid POJO type because not all fields are valid POJO fields.");
 				return null;
 			}
 			try {
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TypeExtractionTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TypeExtractionTest.scala
index 0462ffa5329cc..d5b7867a20010 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TypeExtractionTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TypeExtractionTest.scala
@@ -20,22 +20,34 @@ package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.api.common.io.FileInputFormat
 import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable
+import org.apache.flink.api.java.typeutils.{PojoTypeInfo, ResultTypeQueryable}
 import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.typeutils.TypeExtractionTest.CustomTypeInputFormat
+import org.apache.flink.api.scala.typeutils.TypeExtractionTest.{CustomBeanClass, CustomTypeInputFormat}
 import org.apache.flink.util.TestLogger
-import org.junit.Assert.assertEquals
+import org.junit.Assert._
 import org.junit.Test
 import org.scalatest.junit.JUnitSuiteLike
 
+import scala.beans.BeanProperty
+
 
 class TypeExtractionTest extends TestLogger with JUnitSuiteLike {
 
   @Test
   def testResultTypeQueryable(): Unit = {
     val env = ExecutionEnvironment.getExecutionEnvironment
-    val productedType = env.createInput(new CustomTypeInputFormat).getType()
-    assertEquals(productedType, BasicTypeInfo.LONG_TYPE_INFO)
+    val producedType = env.createInput(new CustomTypeInputFormat).getType()
+    assertEquals(producedType, BasicTypeInfo.LONG_TYPE_INFO)
+  }
+
+  @Test
+  def testBeanPropertyClass(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val producedType = env.fromElements(new CustomBeanClass()).getType()
+    assertTrue(producedType.isInstanceOf[PojoTypeInfo[_]])
+    val pojoTypeInfo = producedType.asInstanceOf[PojoTypeInfo[_]]
+    assertEquals(pojoTypeInfo.getTypeAt(0), BasicTypeInfo.INT_TYPE_INFO)
+    assertEquals(pojoTypeInfo.getTypeAt(1), BasicTypeInfo.LONG_TYPE_INFO)
   }
 
 }
@@ -50,4 +62,10 @@ object TypeExtractionTest {
 
     override def nextRecord(reuse: String): String = throw new UnsupportedOperationException()
   }
+
+  class CustomBeanClass(
+      @BeanProperty var prop: Int,
+      var prop2: Long) {
+    def this() = this(0, 0L)
+  }
 }

From 99ce3b34fb2d3ce132e4639e336a3231fd2f3fce Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Mon, 27 Mar 2017 10:33:58 +0200
Subject: [PATCH 198/253] [FLINK-6157] [core] Make TypeInformation fully
 serializable

This closes #3619.
---
 flink-connectors/flink-avro/pom.xml           |  15 +-
 .../api/java/typeutils/AvroTypeInfoTest.java  |  37 +++++
 .../flink-hadoop-compatibility/pom.xml        |  13 ++
 .../java/typeutils/WritableTypeInfoTest.java  |  37 ++---
 .../typeinfo/BasicArrayTypeInfoTest.java      |  42 +++++
 .../common/typeinfo/BasicTypeInfoTest.java    |  46 ++++++
 .../typeinfo/FractionalTypeInfoTest.java      |  35 ++++
 .../common/typeinfo/IntegerTypeInfoTest.java  |  37 +++++
 .../common/typeinfo/NothingTypeInfoTest.java  |  34 ++++
 .../common/typeinfo/NumericTypeInfoTest.java  |  39 +++++
 .../typeinfo/PrimitiveArrayTypeInfoTest.java  |  41 +++++
 .../common/typeinfo/SqlTimeTypeInfoTest.java  |  36 ++++
 .../typeutils/TypeInformationTestBase.java    | 155 ++++++++++++++++++
 .../java/typeutils/EitherTypeInfoTest.java    |  42 ++---
 .../api/java/typeutils/EnumTypeInfoTest.java  |  35 ++--
 .../java/typeutils/GenericTypeInfoTest.java   |  34 ++--
 .../java/typeutils/ListTypeInfoTest.java}     |  38 ++---
 .../api/java/typeutils/MapTypeInfoTest.java   |  37 +++++
 .../java/typeutils/MissingTypeInfoTest.java   |  29 ++--
 .../typeutils/ObjectArrayTypeInfoTest.java    |  46 ++----
 .../api/java/typeutils/PojoTypeInfoTest.java  |  82 ++-------
 .../api/java/typeutils/RowTypeInfoTest.java   |  60 ++-----
 .../api/java/typeutils/TupleTypeInfoTest.java |  49 ++----
 .../api/java/typeutils/ValueTypeInfoTest.java |  46 +++---
 .../flink/types/BasicArrayTypeInfoTest.java   |  56 -------
 .../types/PrimitiveArrayTypeInfoTest.java     |  56 -------
 flink-scala/pom.xml                           |  13 ++
 .../typeutils/CaseClassTypeInfoTest.scala     |  51 ++----
 .../scala/typeutils/EitherTypeInfoTest.scala  |  39 ++---
 .../typeutils/EnumValueTypeInfoTest.scala     |  36 ++--
 .../scala/typeutils/OptionTypeInfoTest.scala  |  38 +----
 .../typeutils/ScalaNothingTypeInfoTest.scala  |  31 ++++
 .../typeutils/TraversableTypeInfoTest.scala   |  46 ++----
 .../api/scala/typeutils/TryTypeInfoTest.scala |  40 +----
 .../scala/typeutils/UnitTypeInfoTest.scala    |  31 ++++
 flink-tests/pom.xml                           |  33 +++-
 .../TypeInfoTestCoverageTest.java             |  62 +++++++
 37 files changed, 958 insertions(+), 639 deletions(-)
 create mode 100644 flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/typeutils/AvroTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NothingTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NumericTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeinfo/SqlTimeTypeInfoTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeInformationTestBase.java
 rename flink-core/src/test/java/org/apache/flink/{types/NothingTypeInfoTest.java => api/java/typeutils/ListTypeInfoTest.java} (53%)
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/MapTypeInfoTest.java
 delete mode 100644 flink-core/src/test/java/org/apache/flink/types/BasicArrayTypeInfoTest.java
 delete mode 100644 flink-core/src/test/java/org/apache/flink/types/PrimitiveArrayTypeInfoTest.java
 create mode 100644 flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/ScalaNothingTypeInfoTest.scala
 create mode 100644 flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/UnitTypeInfoTest.scala
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/completeness/TypeInfoTestCoverageTest.java

diff --git a/flink-connectors/flink-avro/pom.xml b/flink-connectors/flink-avro/pom.xml
index 468a2baabe965..170b3445987d1 100644
--- a/flink-connectors/flink-avro/pom.xml
+++ b/flink-connectors/flink-avro/pom.xml
@@ -154,8 +154,21 @@ under the License.
 					</execution>
 				</executions>
 			</plugin>
+
+			<!-- Add Avro test classes to test jar in order to test AvroTypeInfo. -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>test-jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
 		</plugins>
-		
+
 		<pluginManagement>
 			<plugins>
 				<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
diff --git a/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/typeutils/AvroTypeInfoTest.java b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/typeutils/AvroTypeInfoTest.java
new file mode 100644
index 0000000000000..e0bb1a1355965
--- /dev/null
+++ b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/typeutils/AvroTypeInfoTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+import org.apache.flink.api.io.avro.generated.Address;
+import org.apache.flink.api.io.avro.generated.User;
+
+/**
+ * Test for {@link AvroTypeInfo}.
+ */
+public class AvroTypeInfoTest extends TypeInformationTestBase<AvroTypeInfo<?>> {
+
+	@Override
+	protected AvroTypeInfo<?>[] getTestData() {
+		return new AvroTypeInfo<?>[] {
+			new AvroTypeInfo<>(Address.class),
+			new AvroTypeInfo<>(User.class),
+		};
+	}
+}
diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
index 8c5b673225f27..003225b3057e3 100644
--- a/flink-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -176,6 +176,19 @@ under the License.
 				</configuration>
 			</plugin>
 
+			<!-- Add test classes to test jar in order to test WritableTypeInfo. -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>test-jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-surefire-plugin</artifactId>
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/WritableTypeInfoTest.java b/flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/WritableTypeInfoTest.java
index eb9cdf0297b00..666ab8439c1c6 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/WritableTypeInfoTest.java
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/WritableTypeInfoTest.java
@@ -18,34 +18,23 @@
 
 package org.apache.flink.api.java.typeutils;
 
-import org.apache.flink.util.TestLogger;
-import org.apache.hadoop.io.Writable;
-import org.junit.Test;
-
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+import org.apache.hadoop.io.Writable;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
-public class WritableTypeInfoTest extends TestLogger {
-	
-	@Test
-	public void testWritableTypeInfoEquality() {
-		WritableTypeInfo<TestClass> tpeInfo1 = new WritableTypeInfo<>(TestClass.class);
-		WritableTypeInfo<TestClass> tpeInfo2 = new WritableTypeInfo<>(TestClass.class);
-
-		assertEquals(tpeInfo1, tpeInfo2);
-		assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
-	}
-
-	@Test
-	public void testWritableTypeInfoInequality() {
-		WritableTypeInfo<TestClass> tpeInfo1 = new WritableTypeInfo<>(TestClass.class);
-		WritableTypeInfo<AlternateClass> tpeInfo2 = new WritableTypeInfo<>(AlternateClass.class);
-
-		assertNotEquals(tpeInfo1, tpeInfo2);
+/**
+ * Test for {@link WritableTypeInfo}.
+ */
+public class WritableTypeInfoTest extends TypeInformationTestBase<WritableTypeInfo<?>> {
+
+	@Override
+	protected WritableTypeInfo<?>[] getTestData() {
+		return new WritableTypeInfo<?>[] {
+			new WritableTypeInfo<>(TestClass.class),
+			new WritableTypeInfo<>(AlternateClass.class)
+		};
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfoTest.java
new file mode 100644
index 0000000000000..7a51808ad65f8
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfoTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link BasicArrayTypeInfo}.
+ */
+public class BasicArrayTypeInfoTest extends TypeInformationTestBase<BasicArrayTypeInfo<?, ?>> {
+
+	@Override
+	protected BasicArrayTypeInfo<?, ?>[] getTestData() {
+		return new BasicArrayTypeInfo<?, ?>[] {
+			BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.BYTE_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.SHORT_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.INT_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.LONG_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.FLOAT_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO,
+			BasicArrayTypeInfo.CHAR_ARRAY_TYPE_INFO
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicTypeInfoTest.java
new file mode 100644
index 0000000000000..cd4511aeb2dc6
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/BasicTypeInfoTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link BasicTypeInfo}.
+ */
+public class BasicTypeInfoTest extends TypeInformationTestBase<BasicTypeInfo<?>> {
+
+	@Override
+	protected BasicTypeInfo<?>[] getTestData() {
+		return new BasicTypeInfo<?>[] {
+			BasicTypeInfo.STRING_TYPE_INFO,
+			BasicTypeInfo.BOOLEAN_TYPE_INFO,
+			BasicTypeInfo.BYTE_TYPE_INFO,
+			BasicTypeInfo.SHORT_TYPE_INFO,
+			BasicTypeInfo.INT_TYPE_INFO,
+			BasicTypeInfo.LONG_TYPE_INFO,
+			BasicTypeInfo.FLOAT_TYPE_INFO,
+			BasicTypeInfo.DOUBLE_TYPE_INFO,
+			BasicTypeInfo.CHAR_TYPE_INFO,
+			BasicTypeInfo.DATE_TYPE_INFO,
+			BasicTypeInfo.VOID_TYPE_INFO,
+			BasicTypeInfo.BIG_INT_TYPE_INFO,
+			BasicTypeInfo.BIG_DEC_TYPE_INFO
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfoTest.java
new file mode 100644
index 0000000000000..87c4db5600dcf
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfoTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link FractionalTypeInfo}.
+ */
+public class FractionalTypeInfoTest extends TypeInformationTestBase<FractionalTypeInfo<?>> {
+
+	@Override
+	protected FractionalTypeInfo<?>[] getTestData() {
+		return new FractionalTypeInfo<?>[] {
+			(FractionalTypeInfo<?>) BasicTypeInfo.FLOAT_TYPE_INFO,
+			(FractionalTypeInfo<?>) BasicTypeInfo.DOUBLE_TYPE_INFO
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfoTest.java
new file mode 100644
index 0000000000000..b5fd41f5e5898
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfoTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link IntegerTypeInfo}.
+ */
+public class IntegerTypeInfoTest extends TypeInformationTestBase<IntegerTypeInfo<?>> {
+
+	@Override
+	protected IntegerTypeInfo<?>[] getTestData() {
+		return new IntegerTypeInfo<?>[] {
+			(IntegerTypeInfo<?>) BasicTypeInfo.BYTE_TYPE_INFO,
+			(IntegerTypeInfo<?>) BasicTypeInfo.SHORT_TYPE_INFO,
+			(IntegerTypeInfo<?>) BasicTypeInfo.INT_TYPE_INFO,
+			(IntegerTypeInfo<?>) BasicTypeInfo.LONG_TYPE_INFO
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NothingTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NothingTypeInfoTest.java
new file mode 100644
index 0000000000000..a4771db6cd4b6
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NothingTypeInfoTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link NothingTypeInfo}.
+ */
+public class NothingTypeInfoTest extends TypeInformationTestBase<NothingTypeInfo> {
+
+	@Override
+	protected NothingTypeInfo[] getTestData() {
+		return new NothingTypeInfo[] {
+			new NothingTypeInfo()
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NumericTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NumericTypeInfoTest.java
new file mode 100644
index 0000000000000..49dd722275c10
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/NumericTypeInfoTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link NumericTypeInfo}.
+ */
+public class NumericTypeInfoTest extends TypeInformationTestBase<NumericTypeInfo<?>> {
+
+	@Override
+	protected NumericTypeInfo<?>[] getTestData() {
+		return new NumericTypeInfo<?>[] {
+			(NumericTypeInfo<?>) BasicTypeInfo.BYTE_TYPE_INFO,
+			(NumericTypeInfo<?>) BasicTypeInfo.SHORT_TYPE_INFO,
+			(NumericTypeInfo<?>) BasicTypeInfo.INT_TYPE_INFO,
+			(NumericTypeInfo<?>) BasicTypeInfo.LONG_TYPE_INFO,
+			(NumericTypeInfo<?>) BasicTypeInfo.FLOAT_TYPE_INFO,
+			(NumericTypeInfo<?>) BasicTypeInfo.DOUBLE_TYPE_INFO
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfoTest.java
new file mode 100644
index 0000000000000..867ccad1a6c76
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfoTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link PrimitiveArrayTypeInfoTest}.
+ */
+public class PrimitiveArrayTypeInfoTest extends TypeInformationTestBase<PrimitiveArrayTypeInfo<?>> {
+
+	@Override
+	protected PrimitiveArrayTypeInfo<?>[] getTestData() {
+		return new PrimitiveArrayTypeInfo<?>[] {
+			PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO,
+			PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO,
+			PrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO,
+			PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO,
+			PrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO,
+			PrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO,
+			PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO,
+			PrimitiveArrayTypeInfo.CHAR_PRIMITIVE_ARRAY_TYPE_INFO
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/SqlTimeTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/SqlTimeTypeInfoTest.java
new file mode 100644
index 0000000000000..892ab2b7fe0af
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeinfo/SqlTimeTypeInfoTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link SqlTimeTypeInfo}.
+ */
+public class SqlTimeTypeInfoTest extends TypeInformationTestBase<SqlTimeTypeInfo<?>> {
+
+	@Override
+	protected SqlTimeTypeInfo<?>[] getTestData() {
+		return new SqlTimeTypeInfo<?>[] {
+			SqlTimeTypeInfo.DATE,
+			SqlTimeTypeInfo.TIME,
+			SqlTimeTypeInfo.TIMESTAMP
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeInformationTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeInformationTestBase.java
new file mode 100644
index 0000000000000..bd35070a0d522
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeInformationTestBase.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import java.io.IOException;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.TestLogger;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import org.junit.Test;
+
+/**
+ * Abstract test base for type information.
+ */
+public abstract class TypeInformationTestBase<T extends TypeInformation<?>> extends TestLogger {
+
+	protected abstract T[] getTestData();
+
+	@Test
+	public void testHashcodeAndEquals() throws Exception {
+		final T[] testData = getTestData();
+		final TypeInformation<?> unrelatedTypeInfo = new UnrelatedTypeInfo();
+
+		for (T typeInfo : testData) {
+			// check for implemented hashCode and equals
+			if (typeInfo.getClass().getMethod("hashCode").getDeclaringClass() == Object.class) {
+				throw new AssertionError("Type information does not implement own hashCode method: " +
+					typeInfo.getClass().getCanonicalName());
+			}
+			if (typeInfo.getClass().getMethod("equals", Object.class).getDeclaringClass() == Object.class) {
+				throw new AssertionError("Type information does not implement own equals method: " +
+					typeInfo.getClass().getCanonicalName());
+			}
+
+			// compare among test data
+			for (T otherTypeInfo : testData) {
+				assertTrue("canEqual() returns inconsistent results.", typeInfo.canEqual(otherTypeInfo));
+				// test equality
+				if (typeInfo == otherTypeInfo) {
+					assertTrue("hashCode() returns inconsistent results.", typeInfo.hashCode() == otherTypeInfo.hashCode());
+					assertEquals("equals() is false for same object.", typeInfo, otherTypeInfo);
+				}
+				// test inequality
+				else {
+					assertNotEquals("equals() returned true for different objects.", typeInfo, otherTypeInfo);
+				}
+			}
+
+			// compare with unrelated type
+			assertFalse("Type information allows to compare with unrelated type.", typeInfo.canEqual(unrelatedTypeInfo));
+			assertNotEquals(typeInfo, unrelatedTypeInfo);
+		}
+	}
+
+	@Test
+	public void testSerialization() {
+		final T[] testData = getTestData();
+
+		for (T typeInfo : testData) {
+			final byte[] serialized;
+			try {
+				serialized = InstantiationUtil.serializeObject(typeInfo);
+			} catch (IOException e) {
+				throw new AssertionError("Could not serialize type information: " + typeInfo, e);
+			}
+			final T deserialized;
+			try {
+				deserialized = InstantiationUtil.deserializeObject(serialized, getClass().getClassLoader());
+			} catch (IOException | ClassNotFoundException e) {
+				throw new AssertionError("Could not deserialize type information: " + typeInfo, e);
+			}
+			if (typeInfo.hashCode() != deserialized.hashCode() || !typeInfo.equals(deserialized)) {
+				throw new AssertionError("Deserialized type information differs from original one.");
+			}
+		}
+	}
+
+	private static class UnrelatedTypeInfo extends TypeInformation<Object> {
+
+		@Override
+		public boolean isBasicType() {
+			return false;
+		}
+
+		@Override
+		public boolean isTupleType() {
+			return false;
+		}
+
+		@Override
+		public int getArity() {
+			return 0;
+		}
+
+		@Override
+		public int getTotalFields() {
+			return 0;
+		}
+
+		@Override
+		public Class<Object> getTypeClass() {
+			return null;
+		}
+
+		@Override
+		public boolean isKeyType() {
+			return false;
+		}
+
+		@Override
+		public TypeSerializer<Object> createSerializer(ExecutionConfig config) {
+			return null;
+		}
+
+		@Override
+		public String toString() {
+			return null;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return false;
+		}
+
+		@Override
+		public int hashCode() {
+			return 0;
+		}
+
+		@Override
+		public boolean canEqual(Object obj) {
+			return false;
+		}
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EitherTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EitherTypeInfoTest.java
index 13009eef69933..a5e340e86d2aa 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EitherTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EitherTypeInfoTest.java
@@ -18,37 +18,21 @@
 
 package org.apache.flink.api.java.typeutils;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-
-public class EitherTypeInfoTest extends TestLogger {
-
-	@Test
-	public void testEitherTypeEquality() {
-		EitherTypeInfo<Integer, String> eitherInfo1 = new EitherTypeInfo<Integer, String>(
-				BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
 
-		EitherTypeInfo<Integer, String> eitherInfo2 = new EitherTypeInfo<Integer, String>(
-				BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
-
-		assertEquals(eitherInfo1, eitherInfo2);
-		assertEquals(eitherInfo1.hashCode(), eitherInfo2.hashCode());
-	}
-
-	@Test
-	public void testEitherTypeInequality() {
-		EitherTypeInfo<Integer, String> eitherInfo1 = new EitherTypeInfo<Integer, String>(
-				BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
-
-		EitherTypeInfo<Integer, Tuple2<Double, Long>> eitherInfo2 = new EitherTypeInfo<Integer, Tuple2<Double, Long>>(
-				BasicTypeInfo.INT_TYPE_INFO, new TupleTypeInfo<Tuple2<Double, Long>>(
-				TypeExtractor.getForClass(Double.class), TypeExtractor.getForClass(String.class)));
-
-		assertNotEquals(eitherInfo1, eitherInfo2);
+/**
+ * Test for {@link EitherTypeInfo}.
+ */
+public class EitherTypeInfoTest extends TypeInformationTestBase<EitherTypeInfo<?, ?>> {
+
+	@Override
+	protected EitherTypeInfo<?, ?>[] getTestData() {
+		return new EitherTypeInfo<?, ?>[] {
+			new EitherTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
+			new EitherTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO,
+				new TupleTypeInfo<Tuple2<Double, Long>>(TypeExtractor.getForClass(Double.class), TypeExtractor.getForClass(String.class)))
+		};
 	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EnumTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EnumTypeInfoTest.java
index b200566e78225..2023f7fdb3c4d 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EnumTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/EnumTypeInfoTest.java
@@ -18,11 +18,21 @@
 
 package org.apache.flink.api.java.typeutils;
 
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-import static org.junit.Assert.*;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 
-public class EnumTypeInfoTest extends TestLogger {
+/**
+ * Test for {@link EnumTypeInfo}.
+ */
+public class EnumTypeInfoTest extends TypeInformationTestBase<EnumTypeInfo<?>> {
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected EnumTypeInfo<?>[] getTestData() {
+		return new EnumTypeInfo<?>[] {
+			(EnumTypeInfo<?>) new EnumTypeInfo(TestEnum.class),
+			(EnumTypeInfo<?>) new EnumTypeInfo(AlternativeEnum.class)
+		};
+	}
 
 	enum TestEnum {
 		ONE, TWO
@@ -31,21 +41,4 @@ enum TestEnum {
 	enum AlternativeEnum {
 		ONE, TWO
 	}
-
-	@Test
-	public void testEnumTypeEquality() {
-		EnumTypeInfo<TestEnum> enumTypeInfo1 = new EnumTypeInfo<TestEnum>(TestEnum.class);
-		EnumTypeInfo<TestEnum> enumTypeInfo2 = new EnumTypeInfo<TestEnum>(TestEnum.class);
-
-		assertEquals(enumTypeInfo1, enumTypeInfo2);
-		assertEquals(enumTypeInfo1.hashCode(), enumTypeInfo2.hashCode());
-	}
-
-	@Test
-	public void testEnumTypeInequality() {
-		EnumTypeInfo<TestEnum> enumTypeInfo1 = new EnumTypeInfo<TestEnum>(TestEnum.class);
-		EnumTypeInfo<AlternativeEnum> enumTypeInfo2 = new EnumTypeInfo<AlternativeEnum>(AlternativeEnum.class);
-
-		assertNotEquals(enumTypeInfo1, enumTypeInfo2);
-	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/GenericTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/GenericTypeInfoTest.java
index fad43df362ac7..8dbcc99cdc457 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/GenericTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/GenericTypeInfoTest.java
@@ -18,30 +18,22 @@
 
 package org.apache.flink.api.java.typeutils;
 
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-import static org.junit.Assert.*;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 
-public class GenericTypeInfoTest extends TestLogger {
+/**
+ * Test for {@link GenericTypeInfo}.
+ */
+public class GenericTypeInfoTest extends TypeInformationTestBase<GenericTypeInfo<?>> {
+
+	@Override
+	protected GenericTypeInfo<?>[] getTestData() {
+		return new GenericTypeInfo<?>[] {
+			new GenericTypeInfo<>(TestClass.class),
+			new GenericTypeInfo<>(AlternativeClass.class)
+		};
+	}
 
 	static class TestClass {}
 	static class AlternativeClass {}
 
-	@Test
-	public void testGenericTypeInfoEquality() {
-		GenericTypeInfo<TestClass> tpeInfo1 = new GenericTypeInfo<>(TestClass.class);
-		GenericTypeInfo<TestClass> tpeInfo2 = new GenericTypeInfo<>(TestClass.class);
-
-		assertEquals(tpeInfo1, tpeInfo2);
-		assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
-	}
-
-	@Test
-	public void testGenericTypeInfoInequality() {
-		GenericTypeInfo<TestClass> tpeInfo1 = new GenericTypeInfo<>(TestClass.class);
-		GenericTypeInfo<AlternativeClass> tpeInfo2 = new GenericTypeInfo<>(AlternativeClass.class);
-
-		assertNotEquals(tpeInfo1, tpeInfo2);
-	}
-
 }
diff --git a/flink-core/src/test/java/org/apache/flink/types/NothingTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ListTypeInfoTest.java
similarity index 53%
rename from flink-core/src/test/java/org/apache/flink/types/NothingTypeInfoTest.java
rename to flink-core/src/test/java/org/apache/flink/api/java/typeutils/ListTypeInfoTest.java
index a7976eef96e64..58cabfd22d190 100644
--- a/flink-core/src/test/java/org/apache/flink/types/NothingTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ListTypeInfoTest.java
@@ -16,32 +16,22 @@
  * limitations under the License.
  */
 
-package org.apache.flink.types;
+package org.apache.flink.api.java.typeutils;
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.NothingTypeInfo;
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 
-import static org.junit.Assert.*;
-
-public class NothingTypeInfoTest extends TestLogger {
-
-	@Test
-	public void testNothingTypeInfoEquality() {
-		NothingTypeInfo tpeInfo1 = new NothingTypeInfo();
-		NothingTypeInfo tpeInfo2 = new NothingTypeInfo();
-
-		assertEquals(tpeInfo1, tpeInfo2);
-		assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
-	}
-
-	@Test
-	public void testNothingTypeInfoInequality() {
-		NothingTypeInfo tpeInfo1 = new NothingTypeInfo();
-		BasicTypeInfo<Integer> tpeInfo2 = BasicTypeInfo.getInfoFor(Integer.class);
-
-		assertNotEquals(tpeInfo1, tpeInfo2);
-		assertNotEquals(tpeInfo2, tpeInfo1);
+/**
+ * Test for {@link ListTypeInfo}.
+ */
+public class ListTypeInfoTest extends TypeInformationTestBase<ListTypeInfo<?>> {
+
+	@Override
+	protected ListTypeInfo<?>[] getTestData() {
+		return new ListTypeInfo<?>[] {
+			new ListTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO),
+			new ListTypeInfo<>(BasicTypeInfo.BOOLEAN_TYPE_INFO),
+			new ListTypeInfo<>(Object.class),
+		};
 	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MapTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MapTypeInfoTest.java
new file mode 100644
index 0000000000000..5f3fc06dbfdf4
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MapTypeInfoTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+
+/**
+ * Test for {@link MapTypeInfo}.
+ */
+public class MapTypeInfoTest extends TypeInformationTestBase<MapTypeInfo<?, ?>> {
+
+	@Override
+	protected MapTypeInfo<?, ?>[] getTestData() {
+		return new MapTypeInfo<?, ?>[] {
+			new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO),
+			new MapTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
+			new MapTypeInfo<>(String.class, Boolean.class)
+		};
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java
index ee57475f4389a..cccc14df5130e 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java
@@ -19,29 +19,26 @@
 package org.apache.flink.api.java.typeutils;
 
 import org.apache.flink.api.common.functions.InvalidTypesException;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
 
-public class MissingTypeInfoTest extends TestLogger {
-	static final String functionName = "foobar";
-	static final InvalidTypesException testException = new InvalidTypesException("Test exception.");
+public class MissingTypeInfoTest extends TypeInformationTestBase<MissingTypeInfo> {
+	private static final String functionName = "foobar";
+	private static final InvalidTypesException testException = new InvalidTypesException("Test exception.");
 
-	@Test
-	public void testMissingTypeInfoEquality() {
-		MissingTypeInfo tpeInfo1 = new MissingTypeInfo(functionName, testException);
-		MissingTypeInfo tpeInfo2 = new MissingTypeInfo(functionName, testException);
-
-		assertEquals(tpeInfo1, tpeInfo2);
-		assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
+	@Override
+	protected MissingTypeInfo[] getTestData() {
+		return new MissingTypeInfo[] {
+			new MissingTypeInfo(functionName, testException),
+			new MissingTypeInfo("alt" + functionName, testException)
+		};
 	}
 
-	@Test
-	public void testMissingTypeInfoInequality() {
-		MissingTypeInfo tpeInfo1 = new MissingTypeInfo(functionName, testException);
-		MissingTypeInfo tpeInfo2 = new MissingTypeInfo("alt" + functionName, testException);
-
-		assertNotEquals(tpeInfo1, tpeInfo2);
+	@Override
+	public void testSerialization() {
+		// this class is not intended to be serialized
 	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfoTest.java
index f3b39c0327a35..655c28ad8e7a7 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfoTest.java
@@ -18,41 +18,21 @@
 
 package org.apache.flink.api.java.typeutils;
 
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-
 import java.util.ArrayList;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 
-import static org.junit.Assert.*;
-
-public class ObjectArrayTypeInfoTest extends TestLogger {
-
-	public static class TestClass{}
-
-	@Test
-	public void testObjectArrayTypeInfoEquality() {
-		ObjectArrayTypeInfo<TestClass[], TestClass> tpeInfo1 = ObjectArrayTypeInfo.getInfoFor(
-			TestClass[].class,
-			new GenericTypeInfo<TestClass>(TestClass.class));
-
-		ObjectArrayTypeInfo<TestClass[], TestClass> tpeInfo2 = ObjectArrayTypeInfo.getInfoFor(
-			TestClass[].class,
-			new GenericTypeInfo<TestClass>(TestClass.class));
-
-		assertEquals(tpeInfo1, tpeInfo2);
-		assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
+/**
+ * Test for {@link ObjectArrayTypeInfo}.
+ */
+public class ObjectArrayTypeInfoTest extends TypeInformationTestBase<ObjectArrayTypeInfo<?, ?>> {
+
+	@Override
+	protected ObjectArrayTypeInfo<?, ?>[] getTestData() {
+		return new ObjectArrayTypeInfo<?, ?>[] {
+			ObjectArrayTypeInfo.getInfoFor(TestClass[].class, new GenericTypeInfo<>(TestClass.class)),
+			ObjectArrayTypeInfo.getInfoFor(TestClass[].class, new PojoTypeInfo<>(TestClass.class, new ArrayList<PojoField>()))
+		};
 	}
 
-	@Test
-	public void testObjectArrayTypeInfoInequality() {
-		ObjectArrayTypeInfo<TestClass[], TestClass> tpeInfo1 = ObjectArrayTypeInfo.getInfoFor(
-			TestClass[].class,
-			new GenericTypeInfo<TestClass>(TestClass.class));
-
-		ObjectArrayTypeInfo<TestClass[], TestClass> tpeInfo2 = ObjectArrayTypeInfo.getInfoFor(
-			TestClass[].class,
-			new PojoTypeInfo<TestClass>(TestClass.class, new ArrayList<PojoField>()));
-
-		assertNotEquals(tpeInfo1, tpeInfo2);
-	}
+	public static class TestClass {}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeInfoTest.java
index dbe51152edaf6..0af4ed5fe5eea 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeInfoTest.java
@@ -18,75 +18,21 @@
 
 package org.apache.flink.api.java.typeutils;
 
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.util.InstantiationUtil;
-import org.junit.Test;
-
-import java.io.IOException;
-
-public class PojoTypeInfoTest {
-
-	@Test
-	public void testPojoTypeInfoEquality() {
-		try {
-			TypeInformation<TestPojo> info1 = TypeExtractor.getForClass(TestPojo.class);
-			TypeInformation<TestPojo> info2 = TypeExtractor.getForClass(TestPojo.class);
-			
-			assertTrue(info1 instanceof PojoTypeInfo);
-			assertTrue(info2 instanceof PojoTypeInfo);
-			
-			assertTrue(info1.equals(info2));
-			assertTrue(info1.hashCode() == info2.hashCode());
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
-	}
-
-	@Test
-	public void testPojoTypeInfoInequality() {
-		try {
-			TypeInformation<TestPojo> info1 = TypeExtractor.getForClass(TestPojo.class);
-			TypeInformation<AlternatePojo> info2 = TypeExtractor.getForClass(AlternatePojo.class);
-
-			assertTrue(info1 instanceof PojoTypeInfo);
-			assertTrue(info2 instanceof PojoTypeInfo);
-
-			assertFalse(info1.equals(info2));
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			fail(e.getMessage());
-		}
-	}
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 
-	@Test
-	public void testSerializabilityOfPojoTypeInfo() throws IOException, ClassNotFoundException {
-		PojoTypeInfo<TestPojo> pojoTypeInfo = (PojoTypeInfo<TestPojo>)TypeExtractor.getForClass(TestPojo.class);
-
-		byte[] serializedPojoTypeInfo = InstantiationUtil.serializeObject(pojoTypeInfo);
-		PojoTypeInfo<TestPojo> deserializedPojoTypeInfo = (PojoTypeInfo<TestPojo>)InstantiationUtil.deserializeObject(
-			serializedPojoTypeInfo,
-			getClass().getClassLoader());
-
-		assertEquals(pojoTypeInfo, deserializedPojoTypeInfo);
-	}
-
-	@Test
-	public void testPrimitivePojo() {
-		TypeInformation<PrimitivePojo> info1 = TypeExtractor.getForClass(PrimitivePojo.class);
-
-		assertTrue(info1 instanceof PojoTypeInfo);
-	}
-
-	@Test
-	public void testUnderscorePojo() {
-		TypeInformation<UnderscorePojo> info1 = TypeExtractor.getForClass(UnderscorePojo.class);
-
-		assertTrue(info1 instanceof PojoTypeInfo);
+/**
+ * Test for {@link PojoTypeInfo}.
+ */
+public class PojoTypeInfoTest extends TypeInformationTestBase<PojoTypeInfo<?>>{
+
+	@Override
+	protected PojoTypeInfo<?>[] getTestData() {
+		return new PojoTypeInfo<?>[] {
+			(PojoTypeInfo<?>) TypeExtractor.getForClass(TestPojo.class),
+			(PojoTypeInfo<?>) TypeExtractor.getForClass(AlternatePojo.class),
+			(PojoTypeInfo<?>) TypeExtractor.getForClass(PrimitivePojo.class),
+			(PojoTypeInfo<?>) TypeExtractor.getForClass(UnderscorePojo.class)
+		};
 	}
 
 	public static final class TestPojo {
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/RowTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/RowTypeInfoTest.java
index 45f616c275ea0..03d1e0484b60b 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/RowTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/RowTypeInfoTest.java
@@ -20,6 +20,7 @@
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -29,7 +30,10 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 
-public class RowTypeInfoTest {
+/**
+ * Test for {@link RowTypeInfo}.
+ */
+public class RowTypeInfoTest extends TypeInformationTestBase<RowTypeInfo> {
 	private static TypeInformation<?>[] typeList = new TypeInformation<?>[]{
 		BasicTypeInfo.INT_TYPE_INFO,
 		new RowTypeInfo(
@@ -38,6 +42,15 @@ public class RowTypeInfoTest {
 		BasicTypeInfo.STRING_TYPE_INFO
 	};
 
+	@Override
+	protected RowTypeInfo[] getTestData() {
+		return new RowTypeInfo[] {
+			new RowTypeInfo(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
+			new RowTypeInfo(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO),
+			new RowTypeInfo(typeList)
+		};
+	}
+
 	@Test(expected = IllegalArgumentException.class)
 	public void testWrongNumberOfFieldNames() {
 		new RowTypeInfo(typeList, new String[]{"int", "string"});
@@ -92,13 +105,7 @@ public void testGetFlatFields() {
 
 	@Test
 	public void testGetTypeAt() {
-		RowTypeInfo typeInfo = new RowTypeInfo(
-			BasicTypeInfo.INT_TYPE_INFO,
-			new RowTypeInfo(
-				BasicTypeInfo.SHORT_TYPE_INFO,
-				BasicTypeInfo.BIG_DEC_TYPE_INFO
-			),
-			BasicTypeInfo.STRING_TYPE_INFO);
+		RowTypeInfo typeInfo = new RowTypeInfo(typeList);
 
 
 		assertArrayEquals(new String[]{"f0", "f1", "f2"}, typeInfo.getFieldNames());
@@ -108,45 +115,12 @@ public void testGetTypeAt() {
 		assertEquals(BasicTypeInfo.BIG_DEC_TYPE_INFO, typeInfo.getTypeAt("f1.1"));
 	}
 
-	@Test
-	public void testRowTypeInfoEquality() {
-		RowTypeInfo typeInfo1 = new RowTypeInfo(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.STRING_TYPE_INFO);
-
-		RowTypeInfo typeInfo2 = new RowTypeInfo(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.STRING_TYPE_INFO);
-
-		assertEquals(typeInfo1, typeInfo2);
-		assertEquals(typeInfo1.hashCode(), typeInfo2.hashCode());
-	}
-
-	@Test
-	public void testRowTypeInfoInequality() {
-		RowTypeInfo typeInfo1 = new RowTypeInfo(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.STRING_TYPE_INFO);
-
-		RowTypeInfo typeInfo2 = new RowTypeInfo(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.BOOLEAN_TYPE_INFO);
-
-		assertNotEquals(typeInfo1, typeInfo2);
-		assertNotEquals(typeInfo1.hashCode(), typeInfo2.hashCode());
-	}
-
 	@Test
 	public void testNestedRowTypeInfo() {
-		RowTypeInfo typeInfo = new RowTypeInfo(
-			BasicTypeInfo.INT_TYPE_INFO,
-			new RowTypeInfo(
-				BasicTypeInfo.SHORT_TYPE_INFO,
-				BasicTypeInfo.BIG_DEC_TYPE_INFO
-			),
-			BasicTypeInfo.STRING_TYPE_INFO);
+		RowTypeInfo typeInfo = new RowTypeInfo(typeList);
 
 		assertEquals("Row(f0: Short, f1: BigDecimal)", typeInfo.getTypeAt("f1").toString());
 		assertEquals("Short", typeInfo.getTypeAt("f1.f0").toString());
 	}
+
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/TupleTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/TupleTypeInfoTest.java
index b6cff34caeab8..3a9887713657a 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/TupleTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/TupleTypeInfoTest.java
@@ -21,21 +21,31 @@
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.TestLogger;
-import org.junit.Assert;
+import static org.junit.Assert.assertTrue;
 import org.junit.Test;
 
-public class TupleTypeInfoTest extends TestLogger {
+/**
+ * Test for {@link TupleTypeInfo}.
+ */
+public class TupleTypeInfoTest extends TypeInformationTestBase<TupleTypeInfo<?>> {
+
+	@Override
+	protected TupleTypeInfo<?>[] getTestData() {
+		return new TupleTypeInfo<?>[] {
+			new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
+			new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO)
+		};
+	}
 
 	@Test
 	public void testTupleTypeInfoSymmetricEqualityRelation() {
 		TupleTypeInfo<Tuple1<Integer>> tupleTypeInfo = new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO);
 
 		TupleTypeInfoBase<Tuple1> anonymousTupleTypeInfo = new TupleTypeInfoBase<Tuple1>(
-			(Class<Tuple1>)Tuple1.class,
+			Tuple1.class,
 			(TypeInformation<?>)BasicTypeInfo.INT_TYPE_INFO) {
 
 			private static final long serialVersionUID = -7985593598027660836L;
@@ -64,33 +74,6 @@ public int getFieldIndex(String fieldName) {
 		boolean tupleVsAnonymous = tupleTypeInfo.equals(anonymousTupleTypeInfo);
 		boolean anonymousVsTuple = anonymousTupleTypeInfo.equals(tupleTypeInfo);
 
-		Assert.assertTrue("Equality relation should be symmetric", tupleVsAnonymous == anonymousVsTuple);
-	}
-
-	@Test
-	public void testTupleTypeInfoEquality() {
-		TupleTypeInfo<Tuple2<Integer, String>> tupleTypeInfo1 = new TupleTypeInfo<>(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.STRING_TYPE_INFO);
-
-		TupleTypeInfo<Tuple2<Integer, String>> tupleTypeInfo2 = new TupleTypeInfo<>(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.STRING_TYPE_INFO);
-
-		Assert.assertEquals(tupleTypeInfo1, tupleTypeInfo2);
-		Assert.assertEquals(tupleTypeInfo1.hashCode(), tupleTypeInfo2.hashCode());
-	}
-
-	@Test
-	public void testTupleTypeInfoInequality() {
-		TupleTypeInfo<Tuple2<Integer, String>> tupleTypeInfo1 = new TupleTypeInfo<>(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.STRING_TYPE_INFO);
-
-		TupleTypeInfo<Tuple2<Integer, Boolean>> tupleTypeInfo2 = new TupleTypeInfo<>(
-			BasicTypeInfo.INT_TYPE_INFO,
-			BasicTypeInfo.BOOLEAN_TYPE_INFO);
-
-		Assert.assertNotEquals(tupleTypeInfo1, tupleTypeInfo2);
+		assertTrue("Equality relation should be symmetric", tupleVsAnonymous == anonymousVsTuple);
 	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java
index 4a579c86bec93..b67d7545b3c34 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.api.java.typeutils;
 
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.Record;
@@ -30,7 +31,26 @@
 
 import static org.junit.Assert.*;
 
-public class ValueTypeInfoTest extends TestLogger {
+/**
+ * Test for {@link ListTypeInfo}.
+ */
+public class ValueTypeInfoTest extends TypeInformationTestBase<ValueTypeInfo<?>> {
+
+	@Override
+	protected ValueTypeInfo<?>[] getTestData() {
+		return new ValueTypeInfo<?>[] {
+			new ValueTypeInfo<>(TestClass.class),
+			new ValueTypeInfo<>(AlternativeClass.class),
+			new ValueTypeInfo<>(Record.class),
+		};
+	}
+
+	@Test
+	public void testValueTypeEqualsWithNull() throws Exception {
+		ValueTypeInfo<Record> tpeInfo = new ValueTypeInfo<>(Record.class);
+
+		Assert.assertFalse(tpeInfo.equals(null));
+	}
 
 	public static class TestClass implements Value {
 		private static final long serialVersionUID = -492760806806568285L;
@@ -60,28 +80,4 @@ public void read(DataInputView in) throws IOException {
 
 		}
 	}
-
-	@Test
-	public void testValueTypeInfoEquality() {
-		ValueTypeInfo<TestClass> tpeInfo1 = new ValueTypeInfo<>(TestClass.class);
-		ValueTypeInfo<TestClass> tpeInfo2 = new ValueTypeInfo<>(TestClass.class);
-
-		assertEquals(tpeInfo1, tpeInfo2);
-		assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
-	}
-
-	@Test
-	public void testValueTyepInfoInequality() {
-		ValueTypeInfo<TestClass> tpeInfo1 = new ValueTypeInfo<>(TestClass.class);
-		ValueTypeInfo<AlternativeClass> tpeInfo2 = new ValueTypeInfo<>(AlternativeClass.class);
-
-		assertNotEquals(tpeInfo1, tpeInfo2);
-	}
-
-	@Test
-	public void testValueTypeEqualsWithNull() throws Exception {
-		ValueTypeInfo<Record> tpeInfo = new ValueTypeInfo<>(Record.class);
-
-		Assert.assertFalse(tpeInfo.equals(null));
-	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/types/BasicArrayTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/types/BasicArrayTypeInfoTest.java
deleted file mode 100644
index 3e086ff823d55..0000000000000
--- a/flink-core/src/test/java/org/apache/flink/types/BasicArrayTypeInfoTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.types;
-
-import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
-public class BasicArrayTypeInfoTest extends TestLogger {
-
-	static Class<?>[] classes = {String[].class, Integer[].class, Boolean[].class, Byte[].class,
-		Short[].class, Long[].class, Float[].class, Double[].class, Character[].class};
-
-	@Test
-	public void testBasicArrayTypeInfoEquality() {
-		for (Class<?> clazz: classes) {
-			BasicArrayTypeInfo<?, ?> tpeInfo1 = BasicArrayTypeInfo.getInfoFor(clazz);
-			BasicArrayTypeInfo<?, ?> tpeInfo2 = BasicArrayTypeInfo.getInfoFor(clazz);
-
-			assertEquals(tpeInfo1, tpeInfo2);
-			assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
-		}
-	}
-
-	@Test
-	public void testBasicArrayTypeInfoInequality() {
-		for (Class<?> clazz1: classes) {
-			for (Class<?> clazz2: classes) {
-				if (!clazz1.equals(clazz2)) {
-					BasicArrayTypeInfo<?, ?> tpeInfo1 = BasicArrayTypeInfo.getInfoFor(clazz1);
-					BasicArrayTypeInfo<?, ?> tpeInfo2 = BasicArrayTypeInfo.getInfoFor(clazz2);
-					assertNotEquals(tpeInfo1, tpeInfo2);
-				}
-			}
-		}
-	}
-}
diff --git a/flink-core/src/test/java/org/apache/flink/types/PrimitiveArrayTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/types/PrimitiveArrayTypeInfoTest.java
deleted file mode 100644
index 7ef14f934d927..0000000000000
--- a/flink-core/src/test/java/org/apache/flink/types/PrimitiveArrayTypeInfoTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.types;
-
-import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
-public class PrimitiveArrayTypeInfoTest extends TestLogger {
-
-	static Class<?>[] classes = {int[].class, boolean[].class, byte[].class,
-		short[].class, long[].class, float[].class, double[].class, char[].class};
-
-	@Test
-	public void testPrimitiveArrayTypeInfoEquality() {
-		for (Class<?> clazz: classes) {
-			PrimitiveArrayTypeInfo<?> tpeInfo1 = PrimitiveArrayTypeInfo.getInfoFor(clazz);
-			PrimitiveArrayTypeInfo<?> tpeInfo2 = PrimitiveArrayTypeInfo.getInfoFor(clazz);
-
-			assertEquals(tpeInfo1, tpeInfo2);
-			assertEquals(tpeInfo1.hashCode(), tpeInfo2.hashCode());
-		}
-	}
-
-	@Test
-	public void testBasicArrayTypeInfoInequality() {
-		for (Class<?> clazz1: classes) {
-			for (Class<?> clazz2: classes) {
-				if (!clazz1.equals(clazz2)) {
-					PrimitiveArrayTypeInfo<?> tpeInfo1 = PrimitiveArrayTypeInfo.getInfoFor(clazz1);
-					PrimitiveArrayTypeInfo<?> tpeInfo2 = PrimitiveArrayTypeInfo.getInfoFor(clazz2);
-					assertNotEquals(tpeInfo1, tpeInfo2);
-				}
-			}
-		}
-	}
-}
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index cb9b538dd6380..7be1e292327bc 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -220,6 +220,19 @@ under the License.
 				</configuration>
 			</plugin>
 
+			<!-- Add Scala test classes to test jar in order to test Scala type information. -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>test-jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+
 		</plugins>
 	</build>
 
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfoTest.scala
index 479483f6c7787..088cc705a92b4 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfoTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfoTest.scala
@@ -20,54 +20,27 @@ package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.api.common.ExecutionConfig
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.util.TestLogger
-import org.junit.Test
-import org.scalatest.junit.JUnitSuiteLike
+import org.apache.flink.api.common.typeutils.{TypeInformationTestBase, TypeSerializer}
 
-class CaseClassTypeInfoTest extends TestLogger with JUnitSuiteLike {
+/**
+  * Test for [[CaseClassTypeInfo]].
+  */
+class CaseClassTypeInfoTest extends TypeInformationTestBase[CaseClassTypeInfo[_]] {
 
-  @Test
-  def testCaseClassTypeInfoEquality(): Unit = {
-    val tpeInfo1 = new CaseClassTypeInfo[Tuple2[Int, String]](
-      classOf[Tuple2[Int, String]],
+  override protected def getTestData: Array[CaseClassTypeInfo[_]] = Array(
+    new CaseClassTypeInfo[(Int, String)](
+      classOf[(Int, String)],
       Array(),
       Array(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
       Array("_1", "_2")) {
       override def createSerializer(config: ExecutionConfig): TypeSerializer[(Int, String)] = ???
-    }
-
-    val tpeInfo2 = new CaseClassTypeInfo[Tuple2[Int, String]](
-      classOf[Tuple2[Int, String]],
-      Array(),
-      Array(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
-      Array("_1", "_2")) {
-      override def createSerializer(config: ExecutionConfig): TypeSerializer[(Int, String)] = ???
-    }
-
-    assert(tpeInfo1.equals(tpeInfo2))
-    assert(tpeInfo1.hashCode() == tpeInfo2.hashCode())
-  }
-
-  @Test
-  def testCaseClassTypeInfoInequality(): Unit = {
-    val tpeInfo1 = new CaseClassTypeInfo[Tuple2[Int, String]](
-      classOf[Tuple2[Int, String]],
-      Array(),
-      Array(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
-      Array("_1", "_2")) {
-      override def createSerializer(config: ExecutionConfig): TypeSerializer[(Int, String)] = ???
-    }
-
-    val tpeInfo2 = new CaseClassTypeInfo[Tuple2[Int, Boolean]](
-      classOf[Tuple2[Int, Boolean]],
+    },
+    new CaseClassTypeInfo[(Int, Boolean)](
+      classOf[(Int, Boolean)],
       Array(),
       Array(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO),
       Array("_1", "_2")) {
       override def createSerializer(config: ExecutionConfig): TypeSerializer[(Int, Boolean)] = ???
     }
-
-    assert(!tpeInfo1.equals(tpeInfo2))
-  }
-
+  )
 }
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EitherTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EitherTypeInfoTest.scala
index e23a6a04937ed..fc9811e5a9cd8 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EitherTypeInfoTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EitherTypeInfoTest.scala
@@ -19,41 +19,24 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.util.TestLogger
-import org.junit.Test
-import org.scalatest.junit.JUnitSuiteLike
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase
 
-class EitherTypeInfoTest extends TestLogger with JUnitSuiteLike {
+/**
+  * Test for [[EitherTypeInfo]].
+  */
+class EitherTypeInfoTest extends TypeInformationTestBase[EitherTypeInfo[_, _, _]] {
 
-  @Test
-  def testEitherTypeEquality(): Unit = {
-    val eitherTypeInfo1 = new EitherTypeInfo[Integer, String, Either[Integer, String]](
-      classOf[Either[Integer,String]],
-      BasicTypeInfo.INT_TYPE_INFO,
-      BasicTypeInfo.STRING_TYPE_INFO
-    )
-    val eitherTypeInfo2 = new EitherTypeInfo[Integer, String, Either[Integer, String]](
-      classOf[Either[Integer,String]],
-      BasicTypeInfo.INT_TYPE_INFO,
-      BasicTypeInfo.STRING_TYPE_INFO
-    )
-
-    assert(eitherTypeInfo1.equals(eitherTypeInfo2))
-    assert(eitherTypeInfo1.hashCode() == eitherTypeInfo2.hashCode())
-  }
-
-  @Test
-  def testEitherTypeInequality(): Unit = {
-    val eitherTypeInfo1 = new EitherTypeInfo[Integer, Integer, Either[Integer, Integer]](
+  override protected def getTestData: Array[EitherTypeInfo[_, _, _]] = Array(
+    new EitherTypeInfo[Integer, Integer, Either[Integer, Integer]](
       classOf[Either[Integer,Integer]],
       BasicTypeInfo.INT_TYPE_INFO,
       BasicTypeInfo.INT_TYPE_INFO
-    )
-    val eitherTypeInfo2 = new EitherTypeInfo[Integer, String, Either[Integer, String]](
+    ),
+    new EitherTypeInfo[Integer, String, Either[Integer, String]](
       classOf[Either[Integer,String]],
       BasicTypeInfo.INT_TYPE_INFO,
       BasicTypeInfo.STRING_TYPE_INFO
     )
-    assert(!eitherTypeInfo1.equals(eitherTypeInfo2))
-  }
+  )
+
 }
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueTypeInfoTest.scala
index acd6a39400b2f..14ca0e8abd134 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueTypeInfoTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueTypeInfoTest.scala
@@ -18,39 +18,23 @@
 
 package org.apache.flink.api.scala.typeutils
 
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase
 import org.apache.flink.api.scala.typeutils.AlternateEnumeration.AlternateEnumeration
 import org.apache.flink.api.scala.typeutils.TestEnumeration.TestEnumeration
-import org.apache.flink.util.TestLogger
-import org.junit.Test
-import org.scalatest.junit.JUnitSuiteLike
 
-class EnumValueTypeInfoTest extends TestLogger with JUnitSuiteLike {
+/**
+  * Test for [[EnumValueTypeInfo]].
+  */
+class EnumValueTypeInfoTest extends TypeInformationTestBase[EnumValueTypeInfo[_]] {
 
-  @Test
-  def testEnumValueTypeInfoEquality(): Unit = {
-    val enumTypeInfo1 = new EnumValueTypeInfo[TestEnumeration.type](
+  override protected def getTestData: Array[EnumValueTypeInfo[_]] = Array(
+    new EnumValueTypeInfo[TestEnumeration.type](
       TestEnumeration,
-      classOf[TestEnumeration])
-    val enumTypeInfo2 = new EnumValueTypeInfo[TestEnumeration.type](
-      TestEnumeration,
-      classOf[TestEnumeration])
-
-    assert(enumTypeInfo1.equals(enumTypeInfo2))
-    assert(enumTypeInfo1.hashCode() == enumTypeInfo2.hashCode())
-  }
-
-  @Test
-  def testEnumValueTypeInfoInequality(): Unit = {
-    val enumTypeInfo1 = new EnumValueTypeInfo[TestEnumeration.type](
-      TestEnumeration,
-      classOf[TestEnumeration])
-    val enumTypeInfo2 = new EnumValueTypeInfo[AlternateEnumeration.type](
+      classOf[TestEnumeration]),
+    new EnumValueTypeInfo[AlternateEnumeration.type](
       AlternateEnumeration,
       classOf[AlternateEnumeration])
-
-    assert(!enumTypeInfo1.equals(enumTypeInfo2))
-  }
-
+  )
 }
 
 object TestEnumeration extends Enumeration {
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/OptionTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/OptionTypeInfoTest.scala
index b7656581bbe51..3effb0d36fe23 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/OptionTypeInfoTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/OptionTypeInfoTest.scala
@@ -19,36 +19,16 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.api.java.typeutils.GenericTypeInfo
-import org.apache.flink.util.TestLogger
-import org.junit.Test
-import org.scalatest.junit.{JUnitSuiteLike, JUnitSuite}
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase
 
-class OptionTypeInfoTest extends TestLogger with JUnitSuiteLike {
+/**
+  * Test for [[OptionTypeInfo]].
+  */
+class OptionTypeInfoTest extends TypeInformationTestBase[OptionTypeInfo[_, _]] {
 
-  @Test
-  def testOptionTypeEquality: Unit = {
-    val optionTypeInfo1 = new OptionTypeInfo[Integer, Option[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-    val optionTypeInfo2 = new OptionTypeInfo[Integer, Option[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-
-    assert(optionTypeInfo1.equals(optionTypeInfo2))
-    assert(optionTypeInfo1.hashCode == optionTypeInfo2.hashCode)
-  }
-
-  @Test
-  def testOptionTypeInequality: Unit = {
-    val optionTypeInfo1 = new OptionTypeInfo[Integer, Option[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-    val optionTypeInfo2 = new OptionTypeInfo[String, Option[String]](BasicTypeInfo.STRING_TYPE_INFO)
-
-    assert(!optionTypeInfo1.equals(optionTypeInfo2))
-  }
-
-  @Test
-  def testOptionTypeInequalityWithDifferentType: Unit = {
-    val optionTypeInfo = new OptionTypeInfo[Integer, Option[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-    val genericTypeInfo = new GenericTypeInfo[Double](Double.getClass.asInstanceOf[Class[Double]])
-
-    assert(!optionTypeInfo.equals(genericTypeInfo))
-  }
+  override protected def getTestData: Array[OptionTypeInfo[_, _]] = Array(
+    new OptionTypeInfo[Integer, Option[Integer]](BasicTypeInfo.INT_TYPE_INFO),
+    new OptionTypeInfo[String, Option[String]](BasicTypeInfo.STRING_TYPE_INFO)
+  )
 }
 
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/ScalaNothingTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/ScalaNothingTypeInfoTest.scala
new file mode 100644
index 0000000000000..0193de9e41fbe
--- /dev/null
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/ScalaNothingTypeInfoTest.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.typeutils
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase
+
+/**
+  * Test for [[ScalaNothingTypeInfo]].
+  */
+class ScalaNothingTypeInfoTest extends TypeInformationTestBase[ScalaNothingTypeInfo] {
+
+  override protected def getTestData: Array[ScalaNothingTypeInfo] = Array(
+    new ScalaNothingTypeInfo
+  )
+}
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TraversableTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TraversableTypeInfoTest.scala
index e83b3261b922a..f337de1334650 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TraversableTypeInfoTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TraversableTypeInfoTest.scala
@@ -19,50 +19,26 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.api.common.ExecutionConfig
-import org.apache.flink.api.common.typeinfo.{TypeInformation, BasicTypeInfo}
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.util.TestLogger
-import org.junit.Test
-import org.scalatest.junit.JUnitSuiteLike
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeutils.{TypeInformationTestBase, TypeSerializer}
 
-class TraversableTypeInfoTest extends TestLogger with JUnitSuiteLike {
+/**
+  * Test for [[TraversableTypeInfo]].
+  */
+class TraversableTypeInfoTest extends TypeInformationTestBase[TraversableTypeInfo[_, _]] {
 
-  @Test
-  def testTraversableTypeInfoEquality(): Unit = {
-    val tpeInfo1 = new TraversableTypeInfo[Seq[Int], Int](
+  override protected def getTestData: Array[TraversableTypeInfo[_, _]] = Array(
+    new TraversableTypeInfo[Seq[Int], Int](
       classOf[Seq[Int]],
       BasicTypeInfo.INT_TYPE_INFO.asInstanceOf[TypeInformation[Int]]) {
       override def createSerializer(executionConfig: ExecutionConfig): TypeSerializer[Seq[Int]] =
         ???
-    }
-
-    val tpeInfo2 = new TraversableTypeInfo[Seq[Int], Int](
-      classOf[Seq[Int]],
-      BasicTypeInfo.INT_TYPE_INFO.asInstanceOf[TypeInformation[Int]]) {
-      override def createSerializer(executionConfig: ExecutionConfig): TypeSerializer[Seq[Int]] =
-        ???
-    }
-
-    assert(tpeInfo1.equals(tpeInfo2))
-  }
-
-  @Test
-  def testTraversableTypeInfoInequality(): Unit = {
-    val tpeInfo1 = new TraversableTypeInfo[Seq[Int], Int](
-      classOf[Seq[Int]],
-      BasicTypeInfo.INT_TYPE_INFO.asInstanceOf[TypeInformation[Int]]) {
-      override def createSerializer(executionConfig: ExecutionConfig): TypeSerializer[Seq[Int]] =
-        ???
-    }
-
-    val tpeInfo2 = new TraversableTypeInfo[List[Int], Int](
+    },
+    new TraversableTypeInfo[List[Int], Int](
       classOf[List[Int]],
       BasicTypeInfo.INT_TYPE_INFO.asInstanceOf[TypeInformation[Int]]) {
       override def createSerializer(executionConfig: ExecutionConfig): TypeSerializer[List[Int]] =
         ???
     }
-
-    assert(!tpeInfo1.equals(tpeInfo2))
-  }
-
+  )
 }
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TryTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TryTypeInfoTest.scala
index 5e689512587aa..05d18212065b9 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TryTypeInfoTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/TryTypeInfoTest.scala
@@ -19,39 +19,17 @@
 package org.apache.flink.api.scala.typeutils
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.api.java.typeutils.GenericTypeInfo
-import org.apache.flink.util.TestLogger
-import org.junit.Test
-import org.scalatest.junit.JUnitSuiteLike
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase
 
 import scala.util.Try
 
-class TryTypeInfoTest extends TestLogger with JUnitSuiteLike {
+/**
+  * Test for [[TryTypeInfo]].
+  */
+class TryTypeInfoTest extends TypeInformationTestBase[TryTypeInfo[_, _]] {
 
-  @Test
-  def testTryTypeEquality(): Unit = {
-    val TryTypeInfo1 = new TryTypeInfo[Integer, Try[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-    val TryTypeInfo2 = new TryTypeInfo[Integer, Try[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-
-    assert(TryTypeInfo1.equals(TryTypeInfo2))
-    assert(TryTypeInfo1.hashCode == TryTypeInfo2.hashCode)
-  }
-
-  @Test
-  def testTryTypeInequality(): Unit = {
-    val TryTypeInfo1 = new TryTypeInfo[Integer, Try[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-    val TryTypeInfo2 = new TryTypeInfo[String, Try[String]](BasicTypeInfo.STRING_TYPE_INFO)
-
-    //noinspection ComparingUnrelatedTypes
-    assert(!TryTypeInfo1.equals(TryTypeInfo2))
-  }
-
-  @Test
-  def testTryTypeInequalityWithDifferentType(): Unit = {
-    val TryTypeInfo = new TryTypeInfo[Integer, Try[Integer]](BasicTypeInfo.INT_TYPE_INFO)
-    val genericTypeInfo = new GenericTypeInfo[Double](Double.getClass.asInstanceOf[Class[Double]])
-
-    //noinspection ComparingUnrelatedTypes
-    assert(!TryTypeInfo.equals(genericTypeInfo))
-  }
+  override protected def getTestData: Array[TryTypeInfo[_, _]] = Array(
+    new TryTypeInfo[Integer, Try[Integer]](BasicTypeInfo.INT_TYPE_INFO),
+    new TryTypeInfo[String, Try[String]](BasicTypeInfo.STRING_TYPE_INFO)
+  )
 }
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/UnitTypeInfoTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/UnitTypeInfoTest.scala
new file mode 100644
index 0000000000000..1f22c5ac10d42
--- /dev/null
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/UnitTypeInfoTest.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.typeutils
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase
+
+/**
+  * Test for [[UnitTypeInfo]].
+  */
+class UnitTypeInfoTest extends TypeInformationTestBase[UnitTypeInfo] {
+
+  override protected def getTestData: Array[UnitTypeInfo] = Array(
+    new UnitTypeInfo
+  )
+}
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index ebee6ef471f3b..b67edbbe9e75d 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -94,11 +94,19 @@ under the License.
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
-		
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-scala_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-scala_2.10</artifactId>
 			<version>${project.version}</version>
+			<type>test-jar</type>
 			<scope>test</scope>
 		</dependency>
 
@@ -138,6 +146,29 @@ under the License.
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-hadoop-compatibility_2.10</artifactId>
+			<version>${project.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-avro_2.10</artifactId>
+			<version>${project.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-avro_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-optimizer_2.10</artifactId>
diff --git a/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeInfoTestCoverageTest.java b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeInfoTestCoverageTest.java
new file mode 100644
index 0000000000000..b4549a84b1e32
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeInfoTestCoverageTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.completeness;
+
+import java.lang.reflect.Modifier;
+import java.util.Set;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+import org.apache.flink.util.TestLogger;
+import static org.junit.Assert.assertTrue;
+import org.junit.Test;
+import org.reflections.Reflections;
+
+/**
+ * Scans the class path for type information and checks if there is a test for it.
+ */
+public class TypeInfoTestCoverageTest extends TestLogger {
+
+	@Test
+	public void testTypeInfoTestCoverage() {
+		Reflections reflections = new Reflections("org.apache.flink");
+
+		Set<Class<? extends TypeInformation>> typeInfos = reflections.getSubTypesOf(TypeInformation.class);
+		Set<Class<? extends TypeInformationTestBase>> typeInfoTests = reflections.getSubTypesOf(TypeInformationTestBase.class);
+
+		// check if a test exists for each type information
+		for (Class<? extends TypeInformation> typeInfo : typeInfos) {
+			// we skip abstract classes and inner classes to skip type information defined in test classes
+			if (Modifier.isAbstract(typeInfo.getModifiers()) ||
+					Modifier.isPrivate(typeInfo.getModifiers()) ||
+					typeInfo.getName().contains("Test$") ||
+					typeInfo.getName().contains("TestBase$") ||
+					typeInfo.getName().contains("ITCase$") ||
+					typeInfo.getName().contains("$$anon")) {
+				continue;
+			}
+			boolean found = false;
+			for (Class<? extends TypeInformationTestBase> typeInfoTest : typeInfoTests) {
+				if (typeInfoTest.getName().equals(typeInfo.getName() + "Test")) {
+					found = true;
+				}
+			}
+			assertTrue("Could not find test that corresponds to " + typeInfo.getName(), found);
+		}
+	}
+}

From 8df7af384af594996befdad7306fd57c1cd55568 Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Tue, 9 May 2017 15:12:57 -0400
Subject: [PATCH 199/253] Close hanging GitHub PRs

This closes #1033
This closes #1944
This closes #1979
This closes #2029
This closes #2809
This closes #3043
This closes #3627

From fca75506e17bf13ae19887d2376bdd8a034bdd95 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Wed, 3 May 2017 15:57:05 +0200
Subject: [PATCH 200/253] [FLINK-6164] Make ProcessWindowFunction a
 RichFunction

This closes #3824.
---
 .../FoldApplyProcessAllWindowFunction.java    |  2 +-
 .../FoldApplyProcessWindowFunction.java       |  2 +-
 .../windowing/ProcessAllWindowFunction.java   |  4 +-
 .../windowing/ProcessWindowFunction.java      |  4 +-
 .../ReduceApplyProcessAllWindowFunction.java  |  3 +-
 .../ReduceApplyProcessWindowFunction.java     |  2 +-
 .../RichProcessAllWindowFunction.java         | 53 ++-----------------
 .../windowing/RichProcessWindowFunction.java  | 53 ++-----------------
 .../functions/InternalWindowFunctionTest.java | 11 ++--
 ...ignedProcessingTimeWindowOperatorTest.java |  4 +-
 .../function/ProcessAllWindowFunction.scala   |  7 ++-
 .../function/ProcessWindowFunction.scala      |  6 ++-
 .../RichProcessAllWindowFunction.scala        | 53 +------------------
 .../function/RichProcessWindowFunction.scala  | 53 +------------------
 .../ScalaProcessWindowFunctionWrapper.scala   | 20 +++----
 ...IdentityRichProcessAllWindowFunction.scala |  4 +-
 ...ingIdentityRichProcessWindowFunction.scala |  4 +-
 17 files changed, 45 insertions(+), 240 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
index 38244dd0a25dc..b96a8ffaed50a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
@@ -42,7 +42,7 @@
  */
 @Internal
 public class FoldApplyProcessAllWindowFunction<W extends Window, T, ACC, R>
-	extends RichProcessAllWindowFunction<T, R, W>
+	extends ProcessAllWindowFunction<T, R, W>
 	implements OutputTypeConfigurable<R> {
 
 	private static final long serialVersionUID = 1L;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
index 1b2c2e252fb6a..98f5622fde5ee 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
@@ -42,7 +42,7 @@
  */
 @Internal
 public class FoldApplyProcessWindowFunction<K, W extends Window, T, ACC, R>
-	extends RichProcessWindowFunction<T, R, K, W>
+	extends ProcessWindowFunction<T, R, K, W>
 	implements OutputTypeConfigurable<R> {
 
 	private static final long serialVersionUID = 1L;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java
index 4d247a71f76b9..34a37bfcfea67 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessAllWindowFunction.java
@@ -19,7 +19,7 @@
 package org.apache.flink.streaming.api.functions.windowing;
 
 import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
 import org.apache.flink.api.common.state.KeyedStateStore;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
@@ -33,7 +33,7 @@
  * @param <W> The type of {@code Window} that this window function can be applied on.
  */
 @PublicEvolving
-public abstract class ProcessAllWindowFunction<IN, OUT, W extends Window> implements Function {
+public abstract class ProcessAllWindowFunction<IN, OUT, W extends Window> extends AbstractRichFunction {
 
 	private static final long serialVersionUID = 1L;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
index 2c80e9e7d9d4e..506b6109f5519 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ProcessWindowFunction.java
@@ -19,7 +19,7 @@
 package org.apache.flink.streaming.api.functions.windowing;
 
 import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
 import org.apache.flink.api.common.state.KeyedStateStore;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 import org.apache.flink.util.Collector;
@@ -34,7 +34,7 @@
  * @param <W> The type of {@code Window} that this window function can be applied on.
  */
 @PublicEvolving
-public abstract class ProcessWindowFunction<IN, OUT, KEY, W extends Window> implements Function {
+public abstract class ProcessWindowFunction<IN, OUT, KEY, W extends Window> extends AbstractRichFunction {
 
 	private static final long serialVersionUID = 1L;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
index d1f9ccdf3a969..e7e66099a7520 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessAllWindowFunction.java
@@ -32,8 +32,7 @@
  * {@link ReduceFunction}.
  */
 @Internal
-public class ReduceApplyProcessAllWindowFunction<W extends Window, T, R>
-	extends RichProcessAllWindowFunction<T, R, W> {
+public class ReduceApplyProcessAllWindowFunction<W extends Window, T, R> extends ProcessAllWindowFunction<T, R, W> {
 
 	private static final long serialVersionUID = 1L;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
index 836726d850874..18037b77a4e8b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/ReduceApplyProcessWindowFunction.java
@@ -33,7 +33,7 @@
  */
 @Internal
 public class ReduceApplyProcessWindowFunction<K, W extends Window, T, R>
-	extends RichProcessWindowFunction<T, R, K, W> {
+	extends ProcessWindowFunction<T, R, K, W> {
 
 	private static final long serialVersionUID = 1L;
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java
index 1130fa5ef8630..a800870ff0590 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessAllWindowFunction.java
@@ -19,10 +19,6 @@
 package org.apache.flink.streaming.api.functions.windowing;
 
 import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.functions.IterationRuntimeContext;
-import org.apache.flink.api.common.functions.RichFunction;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 
 /**
@@ -32,53 +28,12 @@
  * @param <IN> The type of the input value.
  * @param <OUT> The type of the output value.
  * @param <W> The type of {@code Window} that this window function can be applied on.
+ *
+ * @deprecated use {@link ProcessAllWindowFunction} instead
  */
 @PublicEvolving
-public abstract class RichProcessAllWindowFunction<IN, OUT, W extends Window>
-		extends ProcessAllWindowFunction<IN, OUT, W>
-		implements RichFunction {
+@Deprecated
+public abstract class RichProcessAllWindowFunction<IN, OUT, W extends Window> extends ProcessAllWindowFunction<IN, OUT, W> {
 
 	private static final long serialVersionUID = 1L;
-
-
-	// --------------------------------------------------------------------------------------------
-	//  Runtime context access
-	// --------------------------------------------------------------------------------------------
-
-	private transient RuntimeContext runtimeContext;
-
-	@Override
-	public void setRuntimeContext(RuntimeContext t) {
-		this.runtimeContext = t;
-	}
-
-	@Override
-	public RuntimeContext getRuntimeContext() {
-		if (this.runtimeContext != null) {
-			return this.runtimeContext;
-		} else {
-			throw new IllegalStateException("The runtime context has not been initialized.");
-		}
-	}
-
-	@Override
-	public IterationRuntimeContext getIterationRuntimeContext() {
-		if (this.runtimeContext == null) {
-			throw new IllegalStateException("The runtime context has not been initialized.");
-		} else if (this.runtimeContext instanceof IterationRuntimeContext) {
-			return (IterationRuntimeContext) this.runtimeContext;
-		} else {
-			throw new IllegalStateException("This stub is not part of an iteration step function.");
-		}
-	}
-
-	// --------------------------------------------------------------------------------------------
-	//  Default life cycle methods
-	// --------------------------------------------------------------------------------------------
-
-	@Override
-	public void open(Configuration parameters) throws Exception {}
-
-	@Override
-	public void close() throws Exception {}
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessWindowFunction.java
index ac55bc6c786e3..83da0659169dd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/RichProcessWindowFunction.java
@@ -19,10 +19,6 @@
 package org.apache.flink.streaming.api.functions.windowing;
 
 import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.functions.IterationRuntimeContext;
-import org.apache.flink.api.common.functions.RichFunction;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.windowing.windows.Window;
 
 /**
@@ -33,53 +29,12 @@
  * @param <OUT> The type of the output value.
  * @param <KEY> The type of the key.
  * @param <W> The type of {@code Window} that this window function can be applied on.
+ *
+ * @deprecated use {@link ProcessWindowFunction} instead
  */
 @PublicEvolving
-public abstract class RichProcessWindowFunction<IN, OUT, KEY, W extends Window>
-		extends ProcessWindowFunction<IN, OUT, KEY, W>
-		implements RichFunction {
+@Deprecated
+public abstract class RichProcessWindowFunction<IN, OUT, KEY, W extends Window> extends ProcessWindowFunction<IN, OUT, KEY, W> {
 
 	private static final long serialVersionUID = 1L;
-
-
-	// --------------------------------------------------------------------------------------------
-	//  Runtime context access
-	// --------------------------------------------------------------------------------------------
-
-	private transient RuntimeContext runtimeContext;
-
-	@Override
-	public void setRuntimeContext(RuntimeContext t) {
-		this.runtimeContext = t;
-	}
-
-	@Override
-	public RuntimeContext getRuntimeContext() {
-		if (this.runtimeContext != null) {
-			return this.runtimeContext;
-		} else {
-			throw new IllegalStateException("The runtime context has not been initialized.");
-		}
-	}
-
-	@Override
-	public IterationRuntimeContext getIterationRuntimeContext() {
-		if (this.runtimeContext == null) {
-			throw new IllegalStateException("The runtime context has not been initialized.");
-		} else if (this.runtimeContext instanceof IterationRuntimeContext) {
-			return (IterationRuntimeContext) this.runtimeContext;
-		} else {
-			throw new IllegalStateException("This stub is not part of an iteration step function.");
-		}
-	}
-
-	// --------------------------------------------------------------------------------------------
-	//  Default life cycle methods
-	// --------------------------------------------------------------------------------------------
-
-	@Override
-	public void open(Configuration parameters) throws Exception {}
-
-	@Override
-	public void close() throws Exception {}
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java
index 4b8057f5a6bcb..4b0f5ab2d5c07 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/windowing/functions/InternalWindowFunctionTest.java
@@ -24,11 +24,10 @@
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 import org.apache.flink.streaming.util.functions.StreamingFunctionUtils;
 import org.apache.flink.streaming.api.functions.windowing.RichAllWindowFunction;
-import org.apache.flink.streaming.api.functions.windowing.RichProcessAllWindowFunction;
-import org.apache.flink.streaming.api.functions.windowing.RichProcessWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
 import org.apache.flink.streaming.api.operators.OutputTypeConfigurable;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
@@ -612,7 +611,7 @@ public Set<Long> merge(Set<Long> a, Set<Long> b) {
 	}
 
 	public static class ProcessWindowFunctionMock
-		extends RichProcessWindowFunction<Long, String, Long, TimeWindow>
+		extends ProcessWindowFunction<Long, String, Long, TimeWindow>
 		implements OutputTypeConfigurable<String> {
 
 		private static final long serialVersionUID = 1L;
@@ -626,7 +625,7 @@ public void process(Long aLong, ProcessWindowFunction<Long, String, Long, TimeWi
 	}
 
 	public static class AggregateProcessWindowFunctionMock
-			extends RichProcessWindowFunction<Map<Long, Long>, String, Long, TimeWindow>
+			extends ProcessWindowFunction<Map<Long, Long>, String, Long, TimeWindow>
 			implements OutputTypeConfigurable<String> {
 
 		private static final long serialVersionUID = 1L;
@@ -640,7 +639,7 @@ public void process(Long aLong, ProcessWindowFunction<Map<Long, Long>, String, L
 	}
 
 	public static class AggregateProcessAllWindowFunctionMock
-			extends RichProcessAllWindowFunction<Map<Long, Long>, String, TimeWindow>
+			extends ProcessAllWindowFunction<Map<Long, Long>, String, TimeWindow>
 			implements OutputTypeConfigurable<String> {
 
 		private static final long serialVersionUID = 1L;
@@ -679,7 +678,7 @@ public void apply(TimeWindow window, Iterable<Long> values, Collector<String> ou
 	}
 
 	public static class ProcessAllWindowFunctionMock
-		extends RichProcessAllWindowFunction<Long, String, TimeWindow>
+		extends ProcessAllWindowFunction<Long, String, TimeWindow>
 		implements OutputTypeConfigurable<String> {
 
 		private static final long serialVersionUID = 1L;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
index a8d31548aefdd..2f7e3020bd5a8 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
@@ -35,9 +35,7 @@
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
 import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
-import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
-import org.apache.flink.streaming.api.functions.windowing.RichProcessWindowFunction;
 import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableProcessWindowFunction;
@@ -1038,7 +1036,7 @@ private void assertInvalidParameter(long windowSize, long windowSlide) {
 
 	// ------------------------------------------------------------------------
 
-	private static class StatefulFunction extends RichProcessWindowFunction<Integer, Integer, Integer, TimeWindow> {
+	private static class StatefulFunction extends ProcessWindowFunction<Integer, Integer, Integer, TimeWindow> {
 
 		// we use a concurrent map here even though there is no concurrency, to
 		// get "volatile" style access to entries
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala
index 2f0e48e0b5cd8..49911e4a165c7 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessAllWindowFunction.scala
@@ -18,10 +18,8 @@
 
 package org.apache.flink.streaming.api.scala.function
 
-import java.io.Serializable
-
 import org.apache.flink.annotation.PublicEvolving
-import org.apache.flink.api.common.functions.Function
+import org.apache.flink.api.common.functions.AbstractRichFunction
 import org.apache.flink.api.common.state.KeyedStateStore
 import org.apache.flink.streaming.api.windowing.windows.Window
 import org.apache.flink.util.Collector
@@ -35,7 +33,8 @@ import org.apache.flink.util.Collector
   * @tparam W The type of the window.
   */
 @PublicEvolving
-abstract class ProcessAllWindowFunction[IN, OUT, W <: Window] extends Function with Serializable {
+abstract class ProcessAllWindowFunction[IN, OUT, W <: Window]
+    extends AbstractRichFunction {
   /**
     * Evaluates the window and outputs none or several elements.
     *
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala
index bc79a267d6eb2..d2075db836aef 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/ProcessWindowFunction.scala
@@ -21,7 +21,7 @@ package org.apache.flink.streaming.api.scala.function
 import java.io.Serializable
 
 import org.apache.flink.annotation.PublicEvolving
-import org.apache.flink.api.common.functions.Function
+import org.apache.flink.api.common.functions.AbstractRichFunction
 import org.apache.flink.api.common.state.KeyedStateStore
 import org.apache.flink.streaming.api.windowing.windows.Window
 import org.apache.flink.util.Collector
@@ -36,7 +36,9 @@ import org.apache.flink.util.Collector
   * @tparam W The type of the window.
   */
 @PublicEvolving
-abstract class ProcessWindowFunction[IN, OUT, KEY, W <: Window] extends Function with Serializable {
+abstract class ProcessWindowFunction[IN, OUT, KEY, W <: Window]
+    extends AbstractRichFunction {
+
   /**
     * Evaluates the window and outputs none or several elements.
     *
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala
index 22d64a83698d3..6edc1e610aa03 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessAllWindowFunction.scala
@@ -18,11 +18,7 @@
 
 package org.apache.flink.streaming.api.scala.function
 
-import java.beans.Transient
-
 import org.apache.flink.annotation.Public
-import org.apache.flink.api.common.functions.{IterationRuntimeContext, RichFunction, RuntimeContext}
-import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.windowing.windows.Window
 
 /**
@@ -34,53 +30,8 @@ import org.apache.flink.streaming.api.windowing.windows.Window
   * @tparam W The type of the window.
   */
 @Public
+@deprecated("use [[ProcessAllWindowFunction]] instead")
 abstract class RichProcessAllWindowFunction[IN, OUT, W <: Window]
-    extends ProcessAllWindowFunction[IN, OUT, W]
-    with RichFunction {
-
-  @Transient
-  private var runtimeContext: RuntimeContext = null
-
-  // --------------------------------------------------------------------------------------------
-  //  Runtime context access
-  // --------------------------------------------------------------------------------------------
-
-  override def setRuntimeContext(t: RuntimeContext) {
-    this.runtimeContext = t
-  }
-
-  override def getRuntimeContext: RuntimeContext = {
-    if (this.runtimeContext != null) {
-      this.runtimeContext
-    }
-    else {
-      throw new IllegalStateException("The runtime context has not been initialized.")
-    }
-  }
-
-  override def getIterationRuntimeContext: IterationRuntimeContext = {
-    if (this.runtimeContext == null) {
-      throw new IllegalStateException("The runtime context has not been initialized.")
-    }
-    else {
-      this.runtimeContext match {
-        case iterationRuntimeContext: IterationRuntimeContext => iterationRuntimeContext
-        case _ =>
-          throw new IllegalStateException("This stub is not part of an iteration step function.")
-      }
-    }
-  }
-
-  // --------------------------------------------------------------------------------------------
-  //  Default life cycle methods
-  // --------------------------------------------------------------------------------------------
-
-  @throws[Exception]
-  override def open(parameters: Configuration) {
-  }
-
-  @throws[Exception]
-  override def close() {
-  }
+    extends ProcessAllWindowFunction[IN, OUT, W] {
 }
 
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessWindowFunction.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessWindowFunction.scala
index 320685a9427d2..d9cd27506591b 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessWindowFunction.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/RichProcessWindowFunction.scala
@@ -18,11 +18,7 @@
 
 package org.apache.flink.streaming.api.scala.function
 
-import java.beans.Transient
-
 import org.apache.flink.annotation.Public
-import org.apache.flink.api.common.functions.{IterationRuntimeContext, RichFunction, RuntimeContext}
-import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.windowing.windows.Window
 
 /**
@@ -35,53 +31,8 @@ import org.apache.flink.streaming.api.windowing.windows.Window
   * @tparam W The type of the window.
   */
 @Public
+@deprecated("use [[ProcessWindowFunction]] instead")
 abstract class RichProcessWindowFunction[IN, OUT, KEY, W <: Window]
-    extends ProcessWindowFunction[IN, OUT, KEY, W]
-    with RichFunction {
-
-  @Transient
-  private var runtimeContext: RuntimeContext = null
-
-  // --------------------------------------------------------------------------------------------
-  //  Runtime context access
-  // --------------------------------------------------------------------------------------------
-
-  override def setRuntimeContext(t: RuntimeContext) {
-    this.runtimeContext = t
-  }
-
-  override def getRuntimeContext: RuntimeContext = {
-    if (this.runtimeContext != null) {
-      this.runtimeContext
-    }
-    else {
-      throw new IllegalStateException("The runtime context has not been initialized.")
-    }
-  }
-
-  override def getIterationRuntimeContext: IterationRuntimeContext = {
-    if (this.runtimeContext == null) {
-      throw new IllegalStateException("The runtime context has not been initialized.")
-    }
-    else {
-      this.runtimeContext match {
-        case iterationRuntimeContext: IterationRuntimeContext => iterationRuntimeContext
-        case _ =>
-          throw new IllegalStateException("This stub is not part of an iteration step function.")
-      }
-    }
-  }
-
-  // --------------------------------------------------------------------------------------------
-  //  Default life cycle methods
-  // --------------------------------------------------------------------------------------------
-  
-  @throws[Exception]
-  override def open(parameters: Configuration) {
-  }
-
-  @throws[Exception]
-  override def close() {
-  }
+    extends ProcessWindowFunction[IN, OUT, KEY, W] {
 }
 
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala
index 263373eeac7f3..bc4b7ddd54c1f 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/function/util/ScalaProcessWindowFunctionWrapper.scala
@@ -21,13 +21,9 @@ package org.apache.flink.streaming.api.scala.function.util
 import org.apache.flink.api.common.functions.RuntimeContext
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.windowing.{ProcessWindowFunction => JProcessWindowFunction}
-import org.apache.flink.streaming.api.functions.windowing.{RichProcessWindowFunction => JRichProcessWindowFunction}
-import org.apache.flink.streaming.api.functions.windowing.{RichProcessAllWindowFunction => JRichProcessAllWindowFunction}
 import org.apache.flink.streaming.api.functions.windowing.{ProcessAllWindowFunction => JProcessAllWindowFunction}
 import org.apache.flink.streaming.api.scala.function.{ProcessWindowFunction => ScalaProcessWindowFunction}
 import org.apache.flink.streaming.api.scala.function.{ProcessAllWindowFunction => ScalaProcessAllWindowFunction}
-import org.apache.flink.streaming.api.scala.function.{RichProcessWindowFunction => ScalaRichProcessWindowFunction}
-import org.apache.flink.streaming.api.scala.function.{RichProcessAllWindowFunction => ScalaRichProcessAllWindowFunction}
 import org.apache.flink.streaming.api.windowing.windows.Window
 import org.apache.flink.util.Collector
 
@@ -43,7 +39,7 @@ import scala.collection.JavaConverters._
   */
 final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window](
     private[this] val func: ScalaProcessWindowFunction[IN, OUT, KEY, W])
-    extends JRichProcessWindowFunction[IN, OUT, KEY, W] {
+    extends JProcessWindowFunction[IN, OUT, KEY, W] {
 
   override def process(
       key: KEY,
@@ -82,7 +78,7 @@ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window](
   override def setRuntimeContext(t: RuntimeContext): Unit = {
     super.setRuntimeContext(t)
     func match {
-      case rfunc: ScalaRichProcessWindowFunction[IN, OUT, KEY, W] => rfunc.setRuntimeContext(t)
+      case rfunc: ScalaProcessWindowFunction[IN, OUT, KEY, W] => rfunc.setRuntimeContext(t)
       case _ =>
     }
   }
@@ -90,7 +86,7 @@ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window](
   override def open(parameters: Configuration): Unit = {
     super.open(parameters)
     func match {
-      case rfunc: ScalaRichProcessWindowFunction[IN, OUT, KEY, W] => rfunc.open(parameters)
+      case rfunc: ScalaProcessWindowFunction[IN, OUT, KEY, W] => rfunc.open(parameters)
       case _ =>
     }
   }
@@ -98,7 +94,7 @@ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window](
   override def close(): Unit = {
     super.close()
     func match {
-      case rfunc: ScalaRichProcessWindowFunction[IN, OUT, KEY, W] => rfunc.close()
+      case rfunc: ScalaProcessWindowFunction[IN, OUT, KEY, W] => rfunc.close()
       case _ =>
     }
   }
@@ -114,7 +110,7 @@ final class ScalaProcessWindowFunctionWrapper[IN, OUT, KEY, W <: Window](
   */
 final class ScalaProcessAllWindowFunctionWrapper[IN, OUT, W <: Window](
     private[this] val func: ScalaProcessAllWindowFunction[IN, OUT, W])
-    extends JRichProcessAllWindowFunction[IN, OUT, W] {
+    extends JProcessAllWindowFunction[IN, OUT, W] {
 
   override def process(
       context: JProcessAllWindowFunction[IN, OUT, W]#Context,
@@ -145,7 +141,7 @@ final class ScalaProcessAllWindowFunctionWrapper[IN, OUT, W <: Window](
   override def setRuntimeContext(t: RuntimeContext): Unit = {
     super.setRuntimeContext(t)
     func match {
-      case rfunc : ScalaRichProcessAllWindowFunction[IN, OUT, W] => rfunc.setRuntimeContext(t)
+      case rfunc : ScalaProcessAllWindowFunction[IN, OUT, W] => rfunc.setRuntimeContext(t)
       case _ =>
     }
   }
@@ -153,7 +149,7 @@ final class ScalaProcessAllWindowFunctionWrapper[IN, OUT, W <: Window](
   override def open(parameters: Configuration): Unit = {
     super.open(parameters)
     func match {
-      case rfunc : ScalaRichProcessAllWindowFunction[IN, OUT, W] => rfunc.open(parameters)
+      case rfunc : ScalaProcessAllWindowFunction[IN, OUT, W] => rfunc.open(parameters)
       case _ =>
     }
   }
@@ -161,7 +157,7 @@ final class ScalaProcessAllWindowFunctionWrapper[IN, OUT, W <: Window](
   override def close(): Unit = {
     super.close()
     func match {
-      case rfunc : ScalaRichProcessAllWindowFunction[IN, OUT, W] => rfunc.close()
+      case rfunc : ScalaProcessAllWindowFunction[IN, OUT, W] => rfunc.close()
       case _ =>
     }
   }
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala
index df005fa63a83a..146452b78c857 100644
--- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessAllWindowFunction.scala
@@ -20,13 +20,13 @@ package org.apache.flink.streaming.api.scala.testutils
 
 import org.apache.flink.api.common.functions.RuntimeContext
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.scala.function.RichProcessAllWindowFunction
+import org.apache.flink.streaming.api.scala.function.ProcessAllWindowFunction
 import org.apache.flink.streaming.api.windowing.windows.Window
 import org.apache.flink.util.Collector
 
 
 class CheckingIdentityRichProcessAllWindowFunction[T, W <: Window]
-  extends RichProcessAllWindowFunction[T, T, W] {
+  extends ProcessAllWindowFunction[T, T, W] {
 
   override def process(context: Context, input: Iterable[T], out: Collector[T]): Unit = {
     for (value <- input) {
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessWindowFunction.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessWindowFunction.scala
index d62f2d3465a64..2ec179a90d3fa 100644
--- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessWindowFunction.scala
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/testutils/CheckingIdentityRichProcessWindowFunction.scala
@@ -20,13 +20,13 @@ package org.apache.flink.streaming.api.scala.testutils
 
 import org.apache.flink.api.common.functions.RuntimeContext
 import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.scala.function.RichProcessWindowFunction
+import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction
 import org.apache.flink.streaming.api.windowing.windows.Window
 import org.apache.flink.util.Collector
 
 
 class CheckingIdentityRichProcessWindowFunction[T, K, W <: Window]
-  extends RichProcessWindowFunction[T, T, K, W] {
+  extends ProcessWindowFunction[T, T, K, W] {
 
   override def process(key: K, context: Context, input: Iterable[T], out: Collector[T]): Unit = {
     for (value <- input) {

From 57835f26df29f2c055338a423410b479ba7cc094 Mon Sep 17 00:00:00 2001
From: zjureel <zjureel@gmail.com>
Date: Fri, 5 May 2017 12:11:20 +0200
Subject: [PATCH 201/253] [FLINK-5978] Move JM WebFrontend address ConfigOption
 to JMOptions

This closes #3552.
---
 .../org/apache/flink/configuration/ConfigConstants.java    | 7 ++++++-
 .../org/apache/flink/configuration/JobManagerOptions.java  | 7 +++++++
 .../apache/flink/runtime/webmonitor/WebMonitorConfig.java  | 3 ++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 975a3d49cf052..4897854f8bc35 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -1328,7 +1328,12 @@ public final class ConfigConstants {
 	
 	// ------------------------- JobManager Web Frontend ----------------------
 
-	/** The config key for the address of the JobManager web frontend. */
+	/**
+	 * The config key for the address of the JobManager web frontend.
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_FRONTEND_ADDRESS} instead
+	 */
+	@Deprecated
 	public static final ConfigOption<String> DEFAULT_JOB_MANAGER_WEB_FRONTEND_ADDRESS =
 		key("jobmanager.web.address")
 			.noDefaultValue();
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index 5481d7a32f5c3..b0b1eee066aff 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -93,6 +93,13 @@ public class JobManagerOptions {
 	//  JobManager web UI
 	// ------------------------------------------------------------------------
 
+	/**
+	 * Config parameter defining the runtime monitor web-frontend server address.
+	 */
+	public static final ConfigOption<String> WEB_FRONTEND_ADDRESS =
+		key("jobmanager.web.address")
+			.noDefaultValue();
+
 	/**
 	 * The port for the runtime monitor web-frontend server.
 	 */
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
index 11e94b0a32e75..1807e7d31dc0f 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
@@ -21,6 +21,7 @@
 
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 
 public class WebMonitorConfig {
 
@@ -62,7 +63,7 @@ public WebMonitorConfig(Configuration config) {
 	}
 
 	public String getWebFrontendAddress() {
-		return config.getValue(ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_ADDRESS);
+		return config.getValue(JobManagerOptions.WEB_FRONTEND_ADDRESS);
 	}
 
 	public int getWebFrontendPort() {

From 06a29312a24a2fadf4b69ab2775f3823ad3216e3 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Fri, 5 May 2017 12:13:10 +0200
Subject: [PATCH 202/253] [FLINK-6459] Move origin header ConfigOption to
 JMOptions

---
 .../org/apache/flink/configuration/ConfigConstants.java   | 6 ------
 .../org/apache/flink/configuration/JobManagerOptions.java | 8 ++++++++
 .../apache/flink/runtime/webmonitor/WebMonitorConfig.java | 2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 4897854f8bc35..92e6b5d018efb 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -1338,12 +1338,6 @@ public final class ConfigConstants {
 		key("jobmanager.web.address")
 			.noDefaultValue();
 
-	/** The config parameter defining the Access-Control-Allow-Origin header for all
-	 * responses from the web-frontend. */
-	public static final ConfigOption<String> JOB_MANAGER_WEB_ACCESS_CONTROL_ALLOW_ORIGIN =
-		key("jobmanager.web.access-control-allow-origin")
-			.defaultValue("*");
-
 	/** The config key for the port of the JobManager web frontend.
 	 * Setting this value to {@code -1} disables the web frontend. */
 	public static final int DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT = 8081;
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index b0b1eee066aff..b924e8e23a314 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -107,6 +107,14 @@ public class JobManagerOptions {
 		key("jobmanager.web.port")
 		.defaultValue(8081);
 
+	/**
+	 * The config parameter defining the Access-Control-Allow-Origin header for all
+	 * responses from the web-frontend.
+	 */
+	public static final ConfigOption<String> WEB_ACCESS_CONTROL_ALLOW_ORIGIN =
+		key("jobmanager.web.access-control-allow-origin")
+			.defaultValue("*");
+
 	/**
 	 * Config parameter to override SSL support for the JobManager Web UI
 	 */
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
index 1807e7d31dc0f..dba21459c0af8 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
@@ -81,6 +81,6 @@ public boolean isProgramSubmitEnabled() {
 	}
 
 	public String getAllowOrigin() {
-		return config.getString(ConfigConstants.JOB_MANAGER_WEB_ACCESS_CONTROL_ALLOW_ORIGIN);
+		return config.getString(JobManagerOptions.WEB_ACCESS_CONTROL_ALLOW_ORIGIN);
 	}
 }

From becfc1fe64484a8f2227b0f56a88987c47ec5b33 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Fri, 5 May 2017 12:39:55 +0200
Subject: [PATCH 203/253] [FLINK-6461] Deprecate web config defaults in
 ConfigConstants

This closes #3831.
---
 .../program/StandaloneClusterClient.java      |  5 +-
 .../flink/configuration/ConfigConstants.java  | 65 ++++++++++++++++---
 .../configuration/JobManagerOptions.java      |  7 ++
 .../flink/api/java/ExecutionEnvironment.java  |  4 --
 .../runtime/webmonitor/WebMonitorConfig.java  | 36 +---------
 .../runtime/webmonitor/WebRuntimeMonitor.java | 32 +++------
 .../webmonitor/WebRuntimeMonitorITCase.java   | 13 ++--
 .../executiongraph/ExecutionGraphBuilder.java |  5 +-
 .../runtime/webmonitor/WebMonitorUtils.java   |  6 +-
 .../flink/runtime/jobmanager/JobManager.scala |  3 +-
 .../StreamExecutionEnvironment.java           |  4 --
 .../apache/flink/test/util/TestBaseUtils.java |  5 +-
 .../flink/test/web/WebFrontendITCase.java     |  5 +-
 .../yarn/YarnApplicationMasterRunner.java     |  3 +-
 14 files changed, 98 insertions(+), 95 deletions(-)

diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java
index fd179c0e0a966..7517504ba08ef 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java
@@ -18,8 +18,8 @@
 package org.apache.flink.client.program;
 
 import org.apache.flink.api.common.JobSubmissionResult;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.runtime.clusterframework.messages.GetClusterStatus;
 import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
@@ -53,8 +53,7 @@ public void waitForClusterToBeReady() {}
 	@Override
 	public String getWebInterfaceURL() {
 		String host = this.getJobManagerAddress().getHostString();
-		int port = getFlinkConfiguration().getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY,
-			ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT);
+		int port = getFlinkConfiguration().getInteger(JobManagerOptions.WEB_PORT);
 		return "http://" +  host + ":" + port;
 	}
 
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 92e6b5d018efb..c3704be43e29f 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -1338,36 +1338,81 @@ public final class ConfigConstants {
 		key("jobmanager.web.address")
 			.noDefaultValue();
 
-	/** The config key for the port of the JobManager web frontend.
-	 * Setting this value to {@code -1} disables the web frontend. */
+	/**
+	 * The config key for the port of the JobManager web frontend.
+	 * Setting this value to {@code -1} disables the web frontend.
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_PORT} instead
+	 */
+	@Deprecated
 	public static final int DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT = 8081;
 
-	/** Default value to override SSL support for the JobManager web UI */
+	/**
+	 * Default value to override SSL support for the JobManager web UI
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_SSL_ENABLED} instead
+	 */
+	@Deprecated
 	public static final boolean DEFAULT_JOB_MANAGER_WEB_SSL_ENABLED = true;
 
-	/** The default number of archived jobs for the jobmanager */
+	/**
+	 * The default number of archived jobs for the jobmanager
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_ARCHIVE_COUNT} instead
+	 */
+	@Deprecated
 	public static final int DEFAULT_JOB_MANAGER_WEB_ARCHIVE_COUNT = 5;
 
-	/** By default, submitting jobs from the web-frontend is allowed. */
+	/**
+	 * By default, submitting jobs from the web-frontend is allowed.
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_SUBMIT_ENABLE} instead
+	 */
+	@Deprecated
 	public static final boolean DEFAULT_JOB_MANAGER_WEB_SUBMIT_ENABLED = true;
 
 	/** @deprecated Config key has been deprecated. Therefore, no default value required. */
 	@Deprecated
 	public static final boolean DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_DISABLE = false;
 
-	/** Default number of checkpoints to remember for recent history. */
+	/**
+	 * Default number of checkpoints to remember for recent history.
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_CHECKPOINTS_HISTORY_SIZE} instead
+	 */
+	@Deprecated
 	public static final int DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE = 10;
 
-	/** Time after which cached stats are cleaned up. */
+	/**
+	 * Time after which cached stats are cleaned up.
+	 *
+	 * @@deprecated use {@link JobManagerOptions#WEB_BACKPRESSURE_CLEANUP_INTERVAL} instead
+	 */
+	@Deprecated
 	public static final int DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_CLEAN_UP_INTERVAL = 10 * 60 * 1000;
 
-	/** Time after which available stats are deprecated and need to be refreshed (by resampling). */
+	/**
+	 * Time after which available stats are deprecated and need to be refreshed (by resampling).
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_BACKPRESSURE_REFRESH_INTERVAL} instead
+	 */
+	@Deprecated
 	public static final int DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_REFRESH_INTERVAL = 60 * 1000;
 
-	/** Number of samples to take to determine back pressure. */
+	/**
+	 * Number of samples to take to determine back pressure.
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_BACKPRESSURE_NUM_SAMPLES} instead
+	 */
+	@Deprecated
 	public static final int DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_NUM_SAMPLES = 100;
 
-	/** Delay between samples to determine back pressure. */
+	/**
+	 * Delay between samples to determine back pressure.
+	 *
+	 * @deprecated use {@link JobManagerOptions#WEB_BACKPRESSURE_DELAY} instead
+	 */
+	@Deprecated
 	public static final int DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_DELAY = 50;
 
 	// ------------------------------ Akka Values ------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index b924e8e23a314..76b6bed590b18 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -115,6 +115,13 @@ public class JobManagerOptions {
 		key("jobmanager.web.access-control-allow-origin")
 			.defaultValue("*");
 
+	/**
+	 * The config parameter defining the refresh interval for the web-frontend.
+	 */
+	public static final ConfigOption<Long> WEB_REFRESH_INTERVAL =
+		key("jobmanager.web.refresh-interval")
+			.defaultValue(3000L);
+	
 	/**
 	 * Config parameter to override SSL support for the JobManager Web UI
 	 */
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index 709ef0940180a..3d8a38471c990 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -1235,10 +1235,6 @@ public static LocalEnvironment createLocalEnvironment(Configuration customConfig
 	public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
 		checkNotNull(conf, "conf");
 
-		if (!conf.containsKey(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY)) {
-			int port = ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT;
-			conf.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, port);
-		}
 		conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);
 
 		LocalEnvironment localEnv = new LocalEnvironment(conf);
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
index dba21459c0af8..77537a2d2d43c 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorConfig.java
@@ -18,39 +18,11 @@
 
 package org.apache.flink.runtime.webmonitor;
 
-
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.JobManagerOptions;
 
 public class WebMonitorConfig {
 
-	// ------------------------------------------------------------------------
-	//  Config Keys
-	// ------------------------------------------------------------------------
-
-	/** The port for the runtime monitor web-frontend server. */
-	public static final String JOB_MANAGER_WEB_PORT_KEY = ConfigConstants.JOB_MANAGER_WEB_PORT_KEY;
-
-	/** The initial refresh interval for the web dashboard */
-	public static final String JOB_MANAGER_WEB_REFRESH_INTERVAL_KEY = "jobmanager.web.refresh-interval";
-
-
-	// ------------------------------------------------------------------------
-	//  Default values
-	// ------------------------------------------------------------------------
-
-	/** Default port for the web dashboard (= 8081) */
-	public static final int DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT = ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT;
-
-	/** Default refresh interval for the web dashboard (= 3000 msecs) */
-	public static final long DEFAULT_JOB_MANAGER_WEB_REFRESH_INTERVAL = 3000;
-
-
-	// ------------------------------------------------------------------------
-	//  Config
-	// ------------------------------------------------------------------------
-
 	/** The configuration queried by this config object */
 	private final Configuration config;
 
@@ -67,17 +39,15 @@ public String getWebFrontendAddress() {
 	}
 
 	public int getWebFrontendPort() {
-		return config.getInteger(JOB_MANAGER_WEB_PORT_KEY, DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT);
+		return config.getInteger(JobManagerOptions.WEB_PORT);
 	}
 
 	public long getRefreshInterval() {
-		return config.getLong(JOB_MANAGER_WEB_REFRESH_INTERVAL_KEY, DEFAULT_JOB_MANAGER_WEB_REFRESH_INTERVAL);
+		return config.getLong(JobManagerOptions.WEB_REFRESH_INTERVAL);
 	}
 	
 	public boolean isProgramSubmitEnabled() {
-		return config.getBoolean(
-			ConfigConstants.JOB_MANAGER_WEB_SUBMIT_ENABLED_KEY,
-			ConfigConstants.DEFAULT_JOB_MANAGER_WEB_SUBMIT_ENABLED);
+		return config.getBoolean(JobManagerOptions.WEB_SUBMIT_ENABLE);
 	}
 
 	public String getAllowOrigin() {
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
index f83fa272ab5b1..03b53ad245792 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
@@ -25,6 +25,7 @@
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
@@ -191,21 +192,13 @@ public WebRuntimeMonitor(
 		stackTraceSamples = new StackTraceSampleCoordinator(actorSystem.dispatcher(), 60000);
 
 		// Back pressure stats tracker config
-		int cleanUpInterval = config.getInteger(
-				ConfigConstants.JOB_MANAGER_WEB_BACK_PRESSURE_CLEAN_UP_INTERVAL,
-				ConfigConstants.DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_CLEAN_UP_INTERVAL);
+		int cleanUpInterval = config.getInteger(JobManagerOptions.WEB_BACKPRESSURE_CLEANUP_INTERVAL);
 
-		int refreshInterval = config.getInteger(
-				ConfigConstants.JOB_MANAGER_WEB_BACK_PRESSURE_REFRESH_INTERVAL,
-				ConfigConstants.DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_REFRESH_INTERVAL);
+		int refreshInterval = config.getInteger(JobManagerOptions.WEB_BACKPRESSURE_REFRESH_INTERVAL);
 
-		int numSamples = config.getInteger(
-				ConfigConstants.JOB_MANAGER_WEB_BACK_PRESSURE_NUM_SAMPLES,
-				ConfigConstants.DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_NUM_SAMPLES);
+		int numSamples = config.getInteger(JobManagerOptions.WEB_BACKPRESSURE_NUM_SAMPLES);
 
-		int delay = config.getInteger(
-				ConfigConstants.JOB_MANAGER_WEB_BACK_PRESSURE_DELAY,
-				ConfigConstants.DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_DELAY);
+		int delay = config.getInteger(JobManagerOptions.WEB_BACKPRESSURE_DELAY);
 
 		Time delayBetweenSamples = Time.milliseconds(delay);
 
@@ -219,10 +212,7 @@ public WebRuntimeMonitor(
 		ExecutionContextExecutor context = ExecutionContext$.MODULE$.fromExecutor(executorService);
 
 		// Config to enable https access to the web-ui
-		boolean enableSSL = config.getBoolean(
-				ConfigConstants.JOB_MANAGER_WEB_SSL_ENABLED,
-				ConfigConstants.DEFAULT_JOB_MANAGER_WEB_SSL_ENABLED) &&
-			SSLUtils.getSSLEnabled(config);
+		boolean enableSSL = config.getBoolean(JobManagerOptions.WEB_SSL_ENABLED) &&	SSLUtils.getSSLEnabled(config);
 
 		if (enableSSL) {
 			LOG.info("Enabling ssl for the web frontend");
@@ -310,9 +300,7 @@ public WebRuntimeMonitor(
 		// DELETE is the preferred way of stopping a job (Rest-conform)
 		DELETE(router, new JobStoppingHandler());
 
-		int maxCachedEntries = config.getInteger(
-				ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE,
-			ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE);
+		int maxCachedEntries = config.getInteger(JobManagerOptions.WEB_CHECKPOINTS_HISTORY_SIZE);
 		CheckpointStatsCache cache = new CheckpointStatsCache(maxCachedEntries);
 
 		// Register the checkpoint stats handlers
@@ -525,14 +513,14 @@ File getBaseDir(Configuration configuration) {
 	}
 
 	private String getBaseDirStr(Configuration configuration) {
-		return configuration.getString(ConfigConstants.JOB_MANAGER_WEB_TMPDIR_KEY, System.getProperty("java.io.tmpdir"));
+		return configuration.getString(JobManagerOptions.WEB_TMP_DIR);
 	}
 
 	private File getUploadDir(Configuration configuration) {
-		File baseDir = new File(configuration.getString(ConfigConstants.JOB_MANAGER_WEB_UPLOAD_DIR_KEY,
+		File baseDir = new File(configuration.getString(JobManagerOptions.WEB_UPLOAD_DIR,
 			getBaseDirStr(configuration)));
 
-		boolean uploadDirSpecified = configuration.containsKey(ConfigConstants.JOB_MANAGER_WEB_UPLOAD_DIR_KEY);
+		boolean uploadDirSpecified = configuration.contains(JobManagerOptions.WEB_UPLOAD_DIR);
 		return uploadDirSpecified ? baseDir : new File(baseDir, "flink-web-" + UUID.randomUUID());
 	}
 }
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
index 5ccfe90921804..a51a2340b985f 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
@@ -25,6 +25,7 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
@@ -137,8 +138,8 @@ public void testRedirectToLeader() throws Exception {
 			Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath());
 			Files.createFile(new File(logDir, "jobmanager.out").toPath());
 
-			config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0);
-			config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
+			config.setInteger(JobManagerOptions.WEB_PORT, 0);
+			config.setString(JobManagerOptions.WEB_LOG_PATH, logFile.toString());
 
 			highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
 				config,
@@ -286,8 +287,8 @@ public void testLeaderNotAvailable() throws Exception {
 			Files.createFile(new File(logDir, "jobmanager.out").toPath());
 
 			final Configuration config = new Configuration();
-			config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0);
-			config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
+			config.setInteger(JobManagerOptions.WEB_PORT, 0);
+			config.setString(JobManagerOptions.WEB_LOG_PATH, logFile.toString());
 			config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
 			config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeper.getConnectString());
 
@@ -463,8 +464,8 @@ private WebRuntimeMonitor startWebRuntimeMonitor(
 
 		// Web frontend on random port
 		Configuration config = new Configuration();
-		config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0);
-		config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
+		config.setInteger(JobManagerOptions.WEB_PORT, 0);
+		config.setString(JobManagerOptions.WEB_LOG_PATH, logFile.toString());
 
 		WebRuntimeMonitor webMonitor = new WebRuntimeMonitor(
 			config,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
index 88863e485197e..aa28fbcae24d3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
@@ -25,6 +25,7 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.runtime.JobException;
 import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
@@ -204,9 +205,7 @@ public static ExecutionGraph buildGraph(
 			}
 
 			// Maximum number of remembered checkpoints
-			int historySize = jobManagerConfig.getInteger(
-					ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE,
-					ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE);
+			int historySize = jobManagerConfig.getInteger(JobManagerOptions.WEB_CHECKPOINTS_HISTORY_SIZE);
 
 			CheckpointStatsTracker checkpointStatsTracker = new CheckpointStatsTracker(
 					historySize,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
index 2baadb50b2fa7..dd9527ea328f0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
@@ -25,8 +25,8 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 
 import java.net.URI;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
@@ -80,14 +80,14 @@ public static LogFileLocation find(Configuration config) {
 			
 			if (logFilePath == null) {
 				LOG.warn("Log file environment variable '{}' is not set.", logEnv);
-				logFilePath = config.getString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, null);
+				logFilePath = config.getString(JobManagerOptions.WEB_LOG_PATH);
 			}
 			
 			// not configured, cannot serve log files
 			if (logFilePath == null || logFilePath.length() < 4) {
 				LOG.warn("JobManager log files are unavailable in the web dashboard. " +
 					"Log file location not found in environment variable '{}' or configuration key '{}'.",
-					logEnv, ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY);
+					logEnv, JobManagerOptions.WEB_LOG_PATH.key());
 				return new LogFileLocation(null, null);
 			}
 			
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 5092643c490e5..57a6415c31975 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -2532,8 +2532,7 @@ object JobManager {
 
     val restartStrategy = RestartStrategyFactory.createRestartStrategyFactory(configuration)
 
-    val archiveCount = configuration.getInteger(ConfigConstants.JOB_MANAGER_WEB_ARCHIVE_COUNT,
-      ConfigConstants.DEFAULT_JOB_MANAGER_WEB_ARCHIVE_COUNT)
+    val archiveCount = configuration.getInteger(JobManagerOptions.WEB_ARCHIVE_COUNT)
 
     val archiveDir = configuration.getString(JobManagerOptions.ARCHIVE_DIR)
 
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index aad3a4b74c33b..97117d214eb40 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -1661,10 +1661,6 @@ public static LocalStreamEnvironment createLocalEnvironment(int parallelism, Con
 	public static StreamExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
 		checkNotNull(conf, "conf");
 
-		if (!conf.containsKey(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY)) {
-			int port = ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT;
-			conf.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, port);
-		}
 		conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);
 
 		LocalStreamEnvironment localEnv = new LocalStreamEnvironment(conf);
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
index f96ab3de47d7b..437dd5f1deb07 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
@@ -30,6 +30,7 @@
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.HighAvailabilityOptions;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.testutils.CommonTestUtils;
 import org.apache.flink.runtime.messages.TaskManagerMessages;
@@ -148,8 +149,8 @@ public static LocalFlinkMiniCluster startCluster(
 		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, DEFAULT_AKKA_ASK_TIMEOUT + "s");
 		config.setString(ConfigConstants.AKKA_STARTUP_TIMEOUT, DEFAULT_AKKA_STARTUP_TIMEOUT);
 
-		config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 8081);
-		config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
+		config.setInteger(JobManagerOptions.WEB_PORT, 8081);
+		config.setString(JobManagerOptions.WEB_LOG_PATH, logFile.toString());
 
 		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.toString());
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java b/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java
index 003eb0c57e71e..538ac98370f7a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java
@@ -24,12 +24,15 @@
 
 import org.apache.commons.io.FileUtils;
 
+import org.apache.commons.math3.optim.nonlinear.vector.JacobianMultivariateVectorOptimizer;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobmanager.JobManagerCliOptions;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.runtime.testutils.StoppableInvokable;
 import org.apache.flink.runtime.webmonitor.WebMonitorUtils;
@@ -84,7 +87,7 @@ public static void initialize() throws Exception {
 		Files.createFile(logFile.toPath());
 		Files.createFile(outFile.toPath());
 		
-		config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.getAbsolutePath());
+		config.setString(JobManagerOptions.WEB_LOG_PATH, logFile.getAbsolutePath());
 		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
 
 		cluster = new LocalFlinkMiniCluster(config, false);
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
index b62f957363bc0..64417f64acacf 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
@@ -372,8 +372,7 @@ protected int runApplicationMaster(Configuration config) {
 				LOG);
 
 			String protocol = "http://";
-			if (config.getBoolean(ConfigConstants.JOB_MANAGER_WEB_SSL_ENABLED,
-				ConfigConstants.DEFAULT_JOB_MANAGER_WEB_SSL_ENABLED) && SSLUtils.getSSLEnabled(config)) {
+			if (config.getBoolean(JobManagerOptions.WEB_SSL_ENABLED) && SSLUtils.getSSLEnabled(config)) {
 				protocol = "https://";
 			}
 			final String webMonitorURL = webMonitor == null ? null :

From 39ed6d2efef70c983396b7c2a849fa05710e979c Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 27 Apr 2017 15:56:48 +0200
Subject: [PATCH 204/253] [FLINK-6396] Fix FsSavepointStreamFactoryTest on
 Windows

This closes #3789.
---
 .../state/filesystem/FsSavepointStreamFactoryTest.java    | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/filesystem/FsSavepointStreamFactoryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/filesystem/FsSavepointStreamFactoryTest.java
index a29d29cb7a0d9..3095a09590f04 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/filesystem/FsSavepointStreamFactoryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/filesystem/FsSavepointStreamFactoryTest.java
@@ -23,6 +23,7 @@
 
 import java.io.File;
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.core.fs.FileStatus;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream;
 import org.junit.Rule;
@@ -48,7 +49,8 @@ public void testSavepointStreamDirectoryLayout() throws Exception {
 				jobId,
 				0);
 
-		File[] listed = testRoot.listFiles();
+		Path root = new Path(testRoot.getAbsolutePath());
+		FileStatus[] listed = root.getFileSystem().listStatus(root);
 		assertNotNull(listed);
 		assertEquals(0, listed.length);
 
@@ -59,9 +61,9 @@ public void testSavepointStreamDirectoryLayout() throws Exception {
 
 		FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
 
-		listed = testRoot.listFiles();
+		listed = root.getFileSystem().listStatus(root);
 		assertNotNull(listed);
 		assertEquals(1, listed.length);
-		assertEquals(handle.getFilePath().getPath(), listed[0].getPath());
+		assertEquals(handle.getFilePath().getPath(), listed[0].getPath().getPath());
 	}
 }

From ae2b8d7a94c348c2ed9e632bc7f2f76a4b333eba Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Wed, 3 May 2017 15:49:03 +0200
Subject: [PATCH 205/253] [FLINK-5720] Deprecate DataStream#fold()

This closes #3816.
---
 .../streaming/state/RocksDBFoldingState.java   |  3 +++
 .../api/common/functions/FoldFunction.java     |  3 +++
 .../api/common/functions/RichFoldFunction.java |  3 +++
 .../api/common/functions/RuntimeContext.java   |  3 +++
 .../flink/api/common/state/FoldingState.java   |  3 +++
 .../common/state/FoldingStateDescriptor.java   |  3 +++
 .../api/common/state/KeyedStateStore.java      |  5 ++++-
 .../flink/api/common/state/StateBinder.java    |  3 +++
 .../api/java/typeutils/TypeExtractor.java      |  8 ++++++++
 .../state/AbstractKeyedStateBackend.java       |  5 ++++-
 .../runtime/state/heap/HeapFoldingState.java   |  7 +++++--
 .../state/internal/InternalFoldingState.java   |  3 +++
 .../api/datastream/AllWindowedStream.java      | 18 ++++++++++++++++++
 .../streaming/api/datastream/KeyedStream.java  |  6 ++++++
 .../api/datastream/WindowedStream.java         | 18 ++++++++++++++++++
 .../windowing/FoldApplyAllWindowFunction.java  |  3 +++
 .../FoldApplyProcessAllWindowFunction.java     |  3 +++
 .../FoldApplyProcessWindowFunction.java        |  3 +++
 .../windowing/FoldApplyWindowFunction.java     |  3 +++
 .../api/operators/StreamGroupedFold.java       |  3 +++
 .../api/scala/AllWindowedStream.scala          | 10 ++++++++--
 .../streaming/api/scala/KeyedStream.scala      |  3 +++
 .../streaming/api/scala/WindowedStream.scala   |  8 +++++++-
 23 files changed, 120 insertions(+), 7 deletions(-)

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java
index 26dc3ddb2bf29..d5d9fce0189b3 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java
@@ -39,7 +39,10 @@
  * @param <N> The type of the namespace.
  * @param <T> The type of the values that can be folded into the state.
  * @param <ACC> The type of the value in the folding state.
+ *
+ * @deprecated will be removed in a future version
  */
+@Deprecated
 public class RocksDBFoldingState<K, N, T, ACC>
 	extends AbstractRocksDBState<K, N, FoldingState<T, ACC>, FoldingStateDescriptor<T, ACC>, ACC>
 	implements InternalFoldingState<N, T, ACC> {
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FoldFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FoldFunction.java
index b52828eb30e44..b3fd700c748aa 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/FoldFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FoldFunction.java
@@ -38,8 +38,11 @@
  *
  * @param <T> Type of the initial input and the returned element
  * @param <O> Type of the elements that the group/list/stream contains
+ *
+ * @deprecated use {@link AggregateFunction} instead
  */
 @Public
+@Deprecated
 public interface FoldFunction<O, T> extends Function, Serializable {
 	/**
 	 * The core method of FoldFunction, combining two values into one value of the same type.
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFoldFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFoldFunction.java
index 245550d8f47c6..516e1b4ac98de 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFoldFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFoldFunction.java
@@ -28,8 +28,11 @@
  *
  * @param <T> Type of the initial input and the returned element
  * @param <O> Type of the elements that the group/list/stream contains
+ *
+ *@deprecated use {@link RichAggregateFunction} instead
  */
 @Public
+@Deprecated
 public abstract class RichFoldFunction<O, T> extends AbstractRichFunction implements FoldFunction<O, T> {
 
 	private static final long serialVersionUID = 1L;
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
index 2978f3abe6a5c..38155f6e37552 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
@@ -394,8 +394,11 @@ public interface RuntimeContext {
 	 *
 	 * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the
 	 *                                       function (function is not part of a KeyedStream).
+	 *
+	 * @deprecated will be removed in a future version
 	 */
 	@PublicEvolving
+	@Deprecated
 	<T, ACC> FoldingState<T, ACC> getFoldingState(FoldingStateDescriptor<T, ACC> stateProperties);
 	
 	/**
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java
index 684a6121afedc..7e45399986044 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java
@@ -35,6 +35,9 @@
  * 
  * @param <T> Type of the values folded into the state
  * @param <ACC> Type of the value in the state
+ *
+ * @deprecated will be removed in a future version
  */
 @PublicEvolving
+@Deprecated
 public interface FoldingState<T, ACC> extends AppendingState<T, ACC> {}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java
index 73bfaa88ae0cf..f7609c39bf134 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java
@@ -32,8 +32,11 @@
  *
  * @param <T> Type of the values folded int othe state
  * @param <ACC> Type of the value in the state
+ *
+ * @deprecated will be removed in a future version in favor of {@link AggregatingStateDescriptor}
  */
 @PublicEvolving
+@Deprecated
 public class FoldingStateDescriptor<T, ACC> extends StateDescriptor<FoldingState<T, ACC>, ACC> {
 	private static final long serialVersionUID = 1L;
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java b/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java
index 2187f6c5073a3..ea9ac41d71c6c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java
@@ -193,8 +193,11 @@ public interface KeyedStateStore {
 	 *
 	 * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the
 	 *                                       function (function is not part of a KeyedStream).
+	 *
+	 * @deprecated will be removed in a future version
 	 */
 	@PublicEvolving
+	@Deprecated
 	<T, ACC> FoldingState<T, ACC> getFoldingState(FoldingStateDescriptor<T, ACC> stateProperties);
 
 	/**
@@ -236,4 +239,4 @@ public interface KeyedStateStore {
 	 */
 	@PublicEvolving
 	<UK, UV> MapState<UK,UV> getMapState(MapStateDescriptor<UK, UV> stateProperties);
-}
\ No newline at end of file
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java
index 9df7a47c53cd9..a37392394a4f2 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java
@@ -68,7 +68,10 @@ <IN, ACC, OUT> AggregatingState<IN, OUT> createAggregatingState(
 	 *
 	 * @param <T> Type of the values folded into the state
 	 * @param <ACC> Type of the value in the state
+	 *
+	 * @deprecated will be removed in a future version
 	 */
+	@Deprecated
 	<T, ACC> FoldingState<T, ACC> createFoldingState(FoldingStateDescriptor<T, ACC> stateDesc) throws Exception;
 
 	/**
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index a5f236ff62bae..f1bf9575bb41f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -177,13 +177,21 @@ public static <IN, OUT> TypeInformation<OUT> getFlatMapReturnTypes(FlatMapFuncti
 		return getUnaryOperatorReturnType((Function) flatMapInterface, FlatMapFunction.class, false, true, inType, functionName, allowMissing);
 	}
 
+	/**
+	 * @deprecated will be removed in a future version
+	 */
 	@PublicEvolving
+	@Deprecated
 	public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType)
 	{
 		return getFoldReturnTypes(foldInterface, inType, null, false);
 	}
 
+	/**
+	 * @deprecated will be removed in a future version
+	 */
 	@PublicEvolving
+	@Deprecated
 	public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType, String functionName, boolean allowMissing)
 	{
 		return getUnaryOperatorReturnType((Function) foldInterface, FoldFunction.class, false, false, inType, functionName, allowMissing);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
index 47ebe3b73f558..2b225df39de2e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java
@@ -195,8 +195,11 @@ protected abstract <N, T, ACC, R> InternalAggregatingState<N, T, R> createAggreg
 	 *
 	 * @param <N> The type of the namespace.
 	 * @param <T> Type of the values folded into the state
-	 * @param <ACC> Type of the value in the state	 *
+	 * @param <ACC> Type of the value in the state
+	 *
+	 * @deprecated will be removed in a future version
 	 */
+	@Deprecated
 	protected abstract <N, T, ACC> InternalFoldingState<N, T, ACC> createFoldingState(
 			TypeSerializer<N> namespaceSerializer,
 			FoldingStateDescriptor<T, ACC> stateDesc) throws Exception;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapFoldingState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapFoldingState.java
index dad6d0d48d6cc..3a77ccafb2f53 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapFoldingState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapFoldingState.java
@@ -36,7 +36,10 @@
  * @param <N> The type of the namespace.
  * @param <T> The type of the values that can be folded into the state.
  * @param <ACC> The type of the value in the folding state.
+ *
+ * @deprecated will be removed in a future version
  */
+@Deprecated
 public class HeapFoldingState<K, N, T, ACC>
 		extends AbstractHeapState<K, N, ACC, FoldingState<T, ACC>, FoldingStateDescriptor<T, ACC>>
 		implements InternalFoldingState<N, T, ACC> {
@@ -84,7 +87,7 @@ public void add(T value) throws IOException {
 		}
 	}
 
-	static final class FoldTransformation<T, ACC> implements StateTransformationFunction<ACC, T> {
+	private static final class FoldTransformation<T, ACC> implements StateTransformationFunction<ACC, T> {
 
 		private final FoldingStateDescriptor<T, ACC> stateDescriptor;
 		private final FoldFunction<T, ACC> foldFunction;
@@ -99,4 +102,4 @@ public ACC apply(ACC previousState, T value) throws Exception {
 			return foldFunction.fold((previousState != null) ? previousState : stateDescriptor.getDefaultValue(), value);
 		}
 	}
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/internal/InternalFoldingState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/internal/InternalFoldingState.java
index eb58ce50f4a95..4ef258f930580 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/internal/InternalFoldingState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/internal/InternalFoldingState.java
@@ -28,5 +28,8 @@
  * @param <N> The type of the namespace
  * @param <T> Type of the values folded into the state
  * @param <ACC> Type of the value in the state
+ *
+ * @deprecated will be removed in a future version
  */
+@Deprecated
 public interface InternalFoldingState<N, T, ACC> extends InternalAppendingState<N, T, ACC>, FoldingState<T, ACC> {}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
index 0d953a9fdc9a9..7ea65fc007060 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
@@ -754,7 +754,10 @@ public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
 	 *
 	 * @param function The fold function.
 	 * @return The data stream that is the result of applying the fold function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction)} instead
 	 */
+	@Deprecated
 	public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R> function) {
 		if (function instanceof RichFunction) {
 			throw new UnsupportedOperationException("FoldFunction of fold can not be a RichFunction. " +
@@ -774,7 +777,10 @@ public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R>
 	 *
 	 * @param function The fold function.
 	 * @return The data stream that is the result of applying the fold function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, TypeInformation, TypeInformation)} instead
 	 */
+	@Deprecated
 	public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R> function, TypeInformation<R> resultType) {
 		if (function instanceof RichFunction) {
 			throw new UnsupportedOperationException("FoldFunction of fold can not be a RichFunction. " +
@@ -795,8 +801,11 @@ public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R>
 	 * @param foldFunction The fold function that is used for incremental aggregation.
 	 * @param function The window function.
 	 * @return The data stream that is the result of applying the window function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, ProcessAllWindowFunction)} instead
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunction<T, ACC> foldFunction, AllWindowFunction<ACC, R, W> function) {
 
 		TypeInformation<ACC> foldAccumulatorType = TypeExtractor.getFoldReturnTypes(foldFunction, input.getType(),
@@ -821,8 +830,11 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunctio
 	 * @param foldAccumulatorType Type information for the result type of the fold function
 	 * @param resultType Type information for the result type of the window function
 	 * @return The data stream that is the result of applying the window function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, AllWindowFunction, TypeInformation, TypeInformation, TypeInformation)} instead
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue,
 			FoldFunction<T, ACC> foldFunction,
 			AllWindowFunction<ACC, R, W> function,
@@ -901,8 +913,11 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue,
 	 * @param foldFunction The fold function that is used for incremental aggregation.
 	 * @param function The window function.
 	 * @return The data stream that is the result of applying the window function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, ProcessAllWindowFunction)} instead
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunction<T, ACC> foldFunction, ProcessAllWindowFunction<ACC, R, W> function) {
 
 		TypeInformation<ACC> foldAccumulatorType = TypeExtractor.getFoldReturnTypes(foldFunction, input.getType(),
@@ -927,8 +942,11 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunctio
 	 * @param foldAccumulatorType Type information for the result type of the fold function
 	 * @param resultType Type information for the result type of the window function
 	 * @return The data stream that is the result of applying the window function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, ProcessAllWindowFunction, TypeInformation, TypeInformation, TypeInformation)} instead
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue,
 			FoldFunction<T, ACC> foldFunction,
 			ProcessAllWindowFunction<ACC, R, W> function,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
index 9334c660f6282..e3171c37631dc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
@@ -416,7 +416,10 @@ public SingleOutputStreamOperator<T> reduce(ReduceFunction<T> reducer) {
 	 * @param initialValue
 	 *            The initialValue passed to the folders for each key.
 	 * @return The transformed DataStream.
+	 *
+	 * @deprecated will be removed in a future version
 	 */
+	@Deprecated
 	public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R> folder) {
 
 		TypeInformation<R> outType = TypeExtractor.getFoldReturnTypes(
@@ -748,8 +751,11 @@ public QueryableStateStream<KEY, T> asQueryableState(
 	 * @param queryableStateName Name under which to the publish the queryable state instance
 	 * @param stateDescriptor State descriptor to create state instance from
 	 * @return Queryable state instance
+	 *
+	 * @deprecated will be removed in a future version
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <ACC> QueryableStateStream<KEY, ACC> asQueryableState(
 			String queryableStateName,
 			FoldingStateDescriptor<T, ACC> stateDescriptor) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
index 2d7dafef316a2..7913e956060bb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
@@ -487,7 +487,10 @@ public <R> SingleOutputStreamOperator<R> reduce(ReduceFunction<T> reduceFunction
 	 *
 	 * @param function The fold function.
 	 * @return The data stream that is the result of applying the fold function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregationFunction)} instead
 	 */
+	@Deprecated
 	public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R> function) {
 		if (function instanceof RichFunction) {
 			throw new UnsupportedOperationException("FoldFunction can not be a RichFunction. " +
@@ -507,7 +510,10 @@ public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R>
 	 *
 	 * @param function The fold function.
 	 * @return The data stream that is the result of applying the fold function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, TypeInformation, TypeInformation)} instead
 	 */
+	@Deprecated
 	public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R> function, TypeInformation<R> resultType) {
 		if (function instanceof RichFunction) {
 			throw new UnsupportedOperationException("FoldFunction can not be a RichFunction. " +
@@ -528,8 +534,11 @@ public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R>
 	 * @param foldFunction The fold function that is used for incremental aggregation.
 	 * @param function The window function.
 	 * @return The data stream that is the result of applying the window function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, WindowFunction)} instead
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunction<T, ACC> foldFunction, WindowFunction<ACC, R, K, W> function) {
 
 		TypeInformation<ACC> foldAccumulatorType = TypeExtractor.getFoldReturnTypes(foldFunction, input.getType(),
@@ -554,8 +563,11 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunctio
 	 * @param foldAccumulatorType Type information for the result type of the fold function
 	 * @param resultType Type information for the result type of the window function
 	 * @return The data stream that is the result of applying the window function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, ProcessWindowFunction, TypeInformation, TypeInformation, TypeInformation)} instead
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue,
 			FoldFunction<T, ACC> foldFunction,
 			WindowFunction<ACC, R, K, W> function,
@@ -638,8 +650,11 @@ public <ACC, R> SingleOutputStreamOperator<R> fold(ACC initialValue,
 	 * @param foldFunction The fold function that is used for incremental aggregation.
 	 * @param windowFunction The window function.
 	 * @return The data stream that is the result of applying the window function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, WindowFunction)} instead
 	 */
 	@PublicEvolving
+	@Deprecated
 	public <R, ACC> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunction<T, ACC> foldFunction, ProcessWindowFunction<ACC, R, K, W> windowFunction) {
 		if (foldFunction instanceof RichFunction) {
 			throw new UnsupportedOperationException("FoldFunction can not be a RichFunction.");
@@ -667,7 +682,10 @@ public <R, ACC> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunctio
 	 * @param windowFunction The process window function.
 	 * @param windowResultType The process window function result type.
 	 * @return The data stream that is the result of applying the fold function to the window.
+	 *
+	 * @deprecated use {@link #aggregate(AggregateFunction, WindowFunction, TypeInformation, TypeInformation, TypeInformation)} instead
 	 */
+	@Deprecated
 	@Internal
 	public <R, ACC> SingleOutputStreamOperator<R> fold(
 			ACC initialValue,
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
index 30662f002d716..2069f7a4626cd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyAllWindowFunction.java
@@ -37,8 +37,11 @@
 /**
  * Internal {@link AllWindowFunction} that is used for implementing a fold on a window configuration
  * that only allows {@link AllWindowFunction} and cannot directly execute a {@link FoldFunction}.
+ *
+ * @deprecated will be removed in a future version
  */
 @Internal
+@Deprecated
 public class FoldApplyAllWindowFunction<W extends Window, T, ACC, R>
 	extends WrappingFunction<AllWindowFunction<ACC, R, W>>
 	implements AllWindowFunction<T, R, W>, OutputTypeConfigurable<R> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
index b96a8ffaed50a..1d392523a0c42 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessAllWindowFunction.java
@@ -39,8 +39,11 @@
  * Internal {@link ProcessAllWindowFunction} that is used for implementing a fold on a window
  * configuration that only allows {@link ProcessAllWindowFunction} and cannot directly execute a
  * {@link FoldFunction}.
+ *
+ * @deprecated will be removed in a future version
  */
 @Internal
+@Deprecated
 public class FoldApplyProcessAllWindowFunction<W extends Window, T, ACC, R>
 	extends ProcessAllWindowFunction<T, R, W>
 	implements OutputTypeConfigurable<R> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
index 98f5622fde5ee..fa4fe8608e6df 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyProcessWindowFunction.java
@@ -39,8 +39,11 @@
  * Internal {@link ProcessWindowFunction} that is used for implementing a fold on a window
  * configuration that only allows {@link ProcessWindowFunction} and cannot directly execute a
  * {@link FoldFunction}.
+ *
+ * @deprecated will be removed in a future version
  */
 @Internal
+@Deprecated
 public class FoldApplyProcessWindowFunction<K, W extends Window, T, ACC, R>
 	extends ProcessWindowFunction<T, R, K, W>
 	implements OutputTypeConfigurable<R> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
index 770deb0e49387..865dbc9548bed 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/windowing/FoldApplyWindowFunction.java
@@ -37,8 +37,11 @@
 /**
  * Internal {@link WindowFunction} that is used for implementing a fold on a window configuration
  * that only allows {@link WindowFunction} and cannot directly execute a {@link FoldFunction}.
+ *
+ * @deprecated will be removed in a future version
  */
 @Internal
+@Deprecated
 public class FoldApplyWindowFunction<K, W extends Window, T, ACC, R>
 	extends WrappingFunction<WindowFunction<ACC, R, K, W>>
 	implements WindowFunction<T, R, K, W>, OutputTypeConfigurable<R> {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
index 1ed717812a497..07c5c900bf954 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamGroupedFold.java
@@ -35,8 +35,11 @@
 /**
  * A {@link StreamOperator} for executing a {@link FoldFunction} on a
  * {@link org.apache.flink.streaming.api.datastream.KeyedStream}.
+ *
+ * @deprecated will be removed in a future version
  */
 @Internal
+@Deprecated
 public class StreamGroupedFold<IN, OUT, KEY>
 		extends AbstractUdfStreamOperator<OUT, FoldFunction<IN, OUT>>
 		implements OneInputStreamOperator<IN, OUT>, OutputTypeConfigurable<OUT> {
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala
index 757e45f8d4fc3..bbdcf4a9322f2 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala
@@ -401,7 +401,8 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) {
    *
    * @param function The fold function.
    * @return The data stream that is the result of applying the fold function to the window.
-   */
+    */
+  @deprecated("use [[aggregate()]] instead")
   def fold[R: TypeInformation](
       initialValue: R,
       function: FoldFunction[T,R]): DataStream[R] = {
@@ -421,7 +422,8 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) {
    *
    * @param function The fold function.
    * @return The data stream that is the result of applying the fold function to the window.
-   */
+    */
+  @deprecated("use [[aggregate()]] instead")
   def fold[R: TypeInformation](initialValue: R)(function: (R, T) => R): DataStream[R] = {
     if (function == null) {
       throw new NullPointerException("Fold function must not be null.")
@@ -444,6 +446,7 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) {
     * @param windowFunction The window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   def fold[ACC: TypeInformation, R: TypeInformation](
       initialValue: ACC,
       preAggregator: FoldFunction[T, ACC],
@@ -474,6 +477,7 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) {
     * @param windowFunction The process window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   @PublicEvolving
   def fold[ACC: TypeInformation, R: TypeInformation](
       initialValue: ACC,
@@ -505,6 +509,7 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) {
     * @param windowFunction The window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   def fold[ACC: TypeInformation, R: TypeInformation](
       initialValue: ACC,
       preAggregator: (ACC, T) => ACC,
@@ -540,6 +545,7 @@ class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) {
     * @param windowFunction The window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   @PublicEvolving
   def fold[ACC: TypeInformation, R: TypeInformation](
       initialValue: ACC,
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala
index d5ef89f9e3413..aaeb1ec3eab36 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala
@@ -184,6 +184,7 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T]
    * using an associative fold function and an initial value. An independent 
    * aggregate is kept per key.
    */
+  @deprecated("will be removed in a future version")
   def fold[R: TypeInformation](initialValue: R, folder: FoldFunction[T,R]): 
       DataStream[R] = {
     if (folder == null) {
@@ -201,6 +202,7 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T]
    * using an associative fold function and an initial value. An independent 
    * aggregate is kept per key.
    */
+  @deprecated("will be removed in a future version")
   def fold[R: TypeInformation](initialValue: R)(fun: (R,T) => R): DataStream[R] = {
     if (fun == null) {
       throw new NullPointerException("Fold function must not be null.")
@@ -507,6 +509,7 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T]
     * @return Queryable state instance
     */
   @PublicEvolving
+  @deprecated("will be removed in a future version")
   def asQueryableState[ACC](
       queryableStateName: String,
       stateDescriptor: FoldingStateDescriptor[T, ACC]) : QueryableStateStream[K, ACC] =  {
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala
index 4e0e1a490de57..0f8a6e0f35877 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala
@@ -382,6 +382,7 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) {
    * @param function The fold function.
    * @return The data stream that is the result of applying the fold function to the window.
    */
+  @deprecated("use [[aggregate()]] instead")
   def fold[R: TypeInformation](
       initialValue: R,
       function: FoldFunction[T,R]): DataStream[R] = {
@@ -401,7 +402,8 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) {
    *
    * @param function The fold function.
    * @return The data stream that is the result of applying the fold function to the window.
-   */
+    */
+  @deprecated("use [[aggregate()]] instead")
   def fold[R: TypeInformation](initialValue: R)(function: (R, T) => R): DataStream[R] = {
     if (function == null) {
       throw new NullPointerException("Fold function must not be null.")
@@ -423,6 +425,7 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) {
     * @param function The window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   def fold[ACC: TypeInformation, R: TypeInformation](
       initialValue: ACC,
       foldFunction: FoldFunction[T, ACC],
@@ -452,6 +455,7 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) {
     * @param windowFunction The window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   def fold[ACC: TypeInformation, R: TypeInformation](
       initialValue: ACC,
       foldFunction: (ACC, T) => ACC,
@@ -486,6 +490,7 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) {
     * @param function The process window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   @PublicEvolving
   def fold[R: TypeInformation, ACC: TypeInformation](
       initialValue: ACC,
@@ -516,6 +521,7 @@ class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) {
     * @param function The process window function.
     * @return The data stream that is the result of applying the window function to the window.
     */
+  @deprecated("use [[aggregate()]] instead")
   @PublicEvolving
   def fold[R: TypeInformation, ACC: TypeInformation](
       initialValue: ACC,

From cf8128805f22b8bce7650db79c9d3a86ffbff982 Mon Sep 17 00:00:00 2001
From: Bowen Li <bowenli86@gmail.com>
Date: Tue, 18 Apr 2017 10:27:17 -0700
Subject: [PATCH 206/253] [FLINK-6013][metrics] Add Datadog HTTP metrics
 reporter

This closes #3736.
---
 docs/monitoring/metrics.md                    |  24 ++
 flink-dist/pom.xml                            |   7 +
 flink-dist/src/main/assemblies/opt.xml        |   7 +
 flink-metrics/flink-metrics-datadog/pom.xml   | 108 +++++++++
 .../flink/metrics/datadog/DCounter.java       |  44 ++++
 .../apache/flink/metrics/datadog/DGauge.java  |  45 ++++
 .../apache/flink/metrics/datadog/DMeter.java  |  42 ++++
 .../apache/flink/metrics/datadog/DMetric.java |  84 +++++++
 .../apache/flink/metrics/datadog/DSeries.java |  45 ++++
 .../metrics/datadog/DatadogHttpClient.java    |  97 ++++++++
 .../metrics/datadog/DatadogHttpReporter.java  | 210 ++++++++++++++++++
 .../flink/metrics/datadog/MetricType.java     |  30 +++
 .../datadog/DatadogHttpClientTest.java        | 199 +++++++++++++++++
 .../src/test/resources/log4j-test.properties  |  27 +++
 flink-metrics/pom.xml                         |   1 +
 15 files changed, 970 insertions(+)
 create mode 100644 flink-metrics/flink-metrics-datadog/pom.xml
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DGauge.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMeter.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/MetricType.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/test/java/org/apache/flink/metrics/datadog/DatadogHttpClientTest.java
 create mode 100644 flink-metrics/flink-metrics-datadog/src/test/resources/log4j-test.properties

diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md
index 290a4528c6989..2bc65a6eb45d3 100644
--- a/docs/monitoring/metrics.md
+++ b/docs/monitoring/metrics.md
@@ -436,6 +436,30 @@ metrics.reporter.stsd.port: 8125
 
 {% endhighlight %}
 
+### Datadog (org.apache.flink.metrics.datadog.DatadogHttpReporter)
+
+In order to use this reporter you must copy `/opt/flink-metrics-datadog-{{site.version}}.jar` into the `/lib` folder
+of your Flink distribution.
+
+Note any variables in Flink metrics, such as `<host>`, `<job_name>`, `<tm_id>`, `<subtask_index>`, `<task_name>`, and `<operator_name>`,
+will be sent to Datadog as tags. Tags will look like `host:localhost` and `job_name:myjobname`.
+
+Parameters:
+
+- `apikey` - the Datadog API key
+- `tags` - (optional) the global tags that will be applied to metrics when sending to Datadog. Tags should be separated by comma only
+
+Example configuration:
+
+{% highlight yaml %}
+
+metrics.reporters: dghttp
+metrics.reporter.dghttp.class: org.apache.flink.metrics.datadog.DatadogHttpReporter
+metrics.reporter.dghttp.apikey: xxx
+metrics.reporter.dghttp.tags: myflinkapp,prod
+
+{% endhighlight %}
+
 ## System metrics
 
 By default Flink gathers several metrics that provide deep insights on the current state.
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 97739917cae1a..6d8debfdba217 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -202,6 +202,13 @@ under the License.
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-metrics-datadog</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
 		<!-- end optional Flink metrics reporters -->
 
 		<!-- start optional Flink libraries -->
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 95218d73997ce..0386b92807c8b 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -104,6 +104,13 @@
 			<fileMode>0644</fileMode>
 		</file>
 
+		<file>
+			<source>../flink-metrics/flink-metrics-datadog/target/flink-metrics-datadog-${project.version}-shaded.jar</source>
+			<outputDirectory>opt/</outputDirectory>
+			<destName>flink-metrics-datadog-${project.version}.jar</destName>
+			<fileMode>0644</fileMode>
+		</file>
+
 		<file>
 			<source>../flink-shaded-hadoop/flink-shaded-hadoop2/target/flink-shaded-hadoop2-${project.version}.jar</source>
 			<outputDirectory>opt/</outputDirectory>
diff --git a/flink-metrics/flink-metrics-datadog/pom.xml b/flink-metrics/flink-metrics-datadog/pom.xml
new file mode 100644
index 0000000000000..0d473fc0743ce
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/pom.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>org.apache.flink</groupId>
+		<artifactId>flink-metrics</artifactId>
+		<version>1.4-SNAPSHOT</version>
+		<relativePath>..</relativePath>
+	</parent>
+
+	<artifactId>flink-metrics-datadog</artifactId>
+	<name>flink-metrics-datadog</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-metrics-core</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+			<scope>provided</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>com.squareup.okhttp3</groupId>
+			<artifactId>okhttp</artifactId>
+			<version>3.7.0</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.squareup.okio</groupId>
+			<artifactId>okio</artifactId>
+			<version>1.12.0</version>
+		</dependency>
+
+
+		<!-- test dependencies -->
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-runtime_2.10</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-test-utils-junit</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<shadedArtifactAttached>true</shadedArtifactAttached>
+							<relocations combine.children="append">
+								<relocation>
+									<pattern>okhttp3</pattern>
+									<shadedPattern>org.apache.flink.shaded.okhttp3</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>okio</pattern>
+									<shadedPattern>org.apache.flink.shaded.okio</shadedPattern>
+								</relocation>
+							</relocations>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java
new file mode 100644
index 0000000000000..58abbd6a4daec
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+import org.apache.flink.metrics.Counter;
+
+import java.util.List;
+
+/**
+ * Mapping of counter between Flink and Datadog
+ * */
+public class DCounter extends DMetric {
+	private final Counter counter;
+
+	public DCounter(Counter c, String metricName, String host, List<String> tags) {
+		super(MetricType.counter, metricName, host, tags);
+		counter = c;
+	}
+
+	/**
+	 * Visibility of this method must not be changed
+	 * since we deliberately not map it to json object in a Datadog-defined format
+	 * */
+	@Override
+	public Number getMetricValue() {
+		return counter.getCount();
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DGauge.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DGauge.java
new file mode 100644
index 0000000000000..8deb11758bb06
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DGauge.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+
+import org.apache.flink.metrics.Gauge;
+
+import java.util.List;
+
+/**
+ * Mapping of gauge between Flink and Datadog
+ * */
+public class DGauge extends DMetric {
+	private final Gauge<Number> gauge;
+
+	public DGauge(Gauge<Number> g, String metricName, String host, List<String> tags) {
+		super(MetricType.gauge, metricName, host, tags);
+		gauge = g;
+	}
+
+	/**
+	 * Visibility of this method must not be changed
+	 * since we deliberately not map it to json object in a Datadog-defined format
+	 * */
+	@Override
+	public Number getMetricValue() {
+		return gauge.getValue();
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMeter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMeter.java
new file mode 100644
index 0000000000000..181a00c4c0c99
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMeter.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+import org.apache.flink.metrics.Meter;
+
+import java.util.List;
+
+/**
+ * Mapping of meter between Flink and Datadog
+ *
+ * Only consider rate of the meter, due to Datadog HTTP API's limited support of meter
+ * */
+public class DMeter extends DMetric {
+	private final Meter meter;
+
+	public DMeter(Meter m, String metricName, String host, List<String> tags) {
+		super(MetricType.gauge, metricName, host, tags);
+		meter = m;
+	}
+
+	@Override
+	public Number getMetricValue() {
+		return meter.getRate();
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java
new file mode 100644
index 0000000000000..3f9d6ffba592c
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Abstract metric of Datadog for serialization
+ * */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public abstract class DMetric {
+	private static final long MILLIS_TO_SEC = 1000L;
+
+	/**
+	 * Names of metric/type/tags field and their getters must not be changed
+	 * since they are mapped to json objects in a Datadog-defined format
+	 * */
+	private final String metric; // Metric name
+	private final MetricType type;
+	private final String host;
+	private final List<String> tags;
+
+	public DMetric(MetricType metricType, String metric, String host, List<String> tags) {
+		this.type = metricType;
+		this.metric = metric;
+		this.host = host;
+		this.tags = tags;
+	}
+
+	public MetricType getType() {
+		return type;
+	}
+
+	public String getMetric() {
+		return metric;
+	}
+
+	public String getHost() {
+		return host;
+	}
+
+	public List<String> getTags() {
+		return tags;
+	}
+
+	public List<List<Number>> getPoints() {
+		// One single data point
+		List<Number> point = new ArrayList<>();
+		point.add(getUnixEpochTimestamp());
+		point.add(getMetricValue());
+
+		List<List<Number>> points = new ArrayList<>();
+		points.add(point);
+
+		return points;
+	}
+
+	@JsonIgnore
+	public abstract Number getMetricValue();
+
+	public static long getUnixEpochTimestamp() {
+		return (System.currentTimeMillis() / MILLIS_TO_SEC);
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java
new file mode 100644
index 0000000000000..fb0bb09ec6851
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Json serialization between Flink and Datadog
+ **/
+public class DSeries {
+	/**
+	 * Names of series field and its getters must not be changed
+	 * since they are mapped to json objects in a Datadog-defined format
+	 * */
+	private List<DMetric> series;
+
+	public DSeries() {
+		series = new ArrayList<>();
+	}
+
+	public void addMetric(DMetric metric) {
+		series.add(metric);
+	}
+
+	public List<DMetric> getSeries() {
+		return series;
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java
new file mode 100644
index 0000000000000..dfbcee138ce53
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.RequestBody;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Http client talking to Datadog
+ * */
+public class DatadogHttpClient{
+	private static final String SERIES_URL_FORMAT = "https://app.datadoghq.com/api/v1/series?api_key=%s";
+	private static final String VALIDATE_URL_FORMAT = "https://app.datadoghq.com/api/v1/validate?api_key=%s";
+	private static final MediaType MEDIA_TYPE = MediaType.parse("application/json; charset=utf-8");
+	private static final int TIMEOUT = 3;
+	private static final ObjectMapper MAPPER = new ObjectMapper();
+
+	private final String seriesUrl;
+	private final String validateUrl;
+	private final OkHttpClient client;
+	private final String apiKey;
+
+	public DatadogHttpClient(String dgApiKey) {
+		if (dgApiKey == null || dgApiKey.isEmpty()) {
+			throw new IllegalArgumentException("Invalid API key:" + dgApiKey);
+		}
+
+		apiKey = dgApiKey;
+		client = new OkHttpClient.Builder()
+			.connectTimeout(TIMEOUT, TimeUnit.SECONDS)
+			.writeTimeout(TIMEOUT, TimeUnit.SECONDS)
+			.readTimeout(TIMEOUT, TimeUnit.SECONDS)
+			.build();
+
+		seriesUrl = String.format(SERIES_URL_FORMAT, apiKey);
+		validateUrl = String.format(VALIDATE_URL_FORMAT, apiKey);
+		validateApiKey();
+	}
+
+	private void validateApiKey() {
+		Request r = new Request.Builder().url(validateUrl).get().build();
+
+		try {
+			Response response = client.newCall(r).execute();
+			if (!response.isSuccessful()) {
+				throw new IllegalArgumentException(
+					String.format("API key: %s is invalid", apiKey));
+			}
+		} catch(IOException e) {
+			throw new IllegalStateException("Failed contacting Datadog to validate API key", e);
+		}
+	}
+
+	public void send(DatadogHttpReporter.DatadogHttpRequest request) throws Exception {
+		String postBody = serialize(request.getSeries());
+
+		Request r = new Request.Builder()
+			.url(seriesUrl)
+			.post(RequestBody.create(MEDIA_TYPE, postBody))
+			.build();
+
+		client.newCall(r).execute().close();
+	}
+
+	public static String serialize(Object obj) throws JsonProcessingException {
+		return MAPPER.writeValueAsString(obj);
+	}
+
+	public void close() {
+		client.dispatcher().executorService().shutdown();
+		client.connectionPool().evictAll();
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
new file mode 100644
index 0000000000000..fcb5c4b92b9c5
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Meter;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricConfig;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.reporter.MetricReporter;
+import org.apache.flink.metrics.reporter.Scheduled;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+/**
+ * Metric Reporter for Datadog
+ *
+ * Variables in metrics scope will be sent to Datadog as tags
+ * */
+public class DatadogHttpReporter implements MetricReporter, Scheduled {
+	private static final Logger LOGGER = LoggerFactory.getLogger(DatadogHttpReporter.class);
+	private static final String HOST_VARIABLE = "<host>";
+
+	// Both Flink's Gauge and Meter values are taken as gauge in Datadog
+	private final Map<Gauge, DGauge> gauges = new ConcurrentHashMap<>();
+	private final Map<Counter, DCounter> counters = new ConcurrentHashMap<>();
+	private final Map<Meter, DMeter> meters = new ConcurrentHashMap<>();
+
+	private DatadogHttpClient client;
+	private List<String> configTags;
+
+	public static final String API_KEY = "apikey";
+	public static final String TAGS = "tags";
+
+	@Override
+	public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
+		final String name = group.getMetricIdentifier(metricName);
+
+		List<String> tags = new ArrayList<>(configTags);
+		tags.addAll(getTagsFromMetricGroup(group));
+		String host = getHostFromMetricGroup(group);
+
+		if (metric instanceof Counter) {
+			Counter c = (Counter) metric;
+			counters.put(c, new DCounter(c, name, host, tags));
+		} else if (metric instanceof Gauge) {
+			Gauge g = (Gauge) metric;
+			gauges.put(g, new DGauge(g, name, host, tags));
+		} else if (metric instanceof Meter) {
+			Meter m = (Meter) metric;
+			// Only consider rate
+			meters.put(m, new DMeter(m, name, host, tags));
+		} else if (metric instanceof Histogram) {
+			LOGGER.warn("Cannot add {} because Datadog HTTP API doesn't support Histogram", metricName);
+		} else {
+			LOGGER.warn("Cannot add unknown metric type {}. This indicates that the reporter " +
+				"does not support this metric type.", metric.getClass().getName());
+		}
+	}
+
+	@Override
+	public void notifyOfRemovedMetric(Metric metric, String metricName, MetricGroup group) {
+		if (metric instanceof Counter) {
+			counters.remove(metric);
+		} else if (metric instanceof Gauge) {
+			gauges.remove(metric);
+		} else if (metric instanceof Meter) {
+			meters.remove(metric);
+		} else if (metric instanceof Histogram) {
+			// No Histogram is registered
+		} else {
+			LOGGER.warn("Cannot remove unknown metric type {}. This indicates that the reporter " +
+				"does not support this metric type.", metric.getClass().getName());
+		}
+	}
+
+	@Override
+	public void open(MetricConfig config) {
+		client = new DatadogHttpClient(config.getString(API_KEY, null));
+		LOGGER.info("Configured DatadogHttpReporter");
+
+		configTags = getTagsFromConfig(config.getString(TAGS, ""));
+	}
+
+	@Override
+	public void close() {
+		client.close();
+		LOGGER.info("Shut down DatadogHttpReporter");
+	}
+
+	@Override
+	public void report() {
+		DatadogHttpRequest request = new DatadogHttpRequest();
+
+		for (Map.Entry<Gauge, DGauge> entry : gauges.entrySet()) {
+			DGauge g = entry.getValue();
+			try {
+				// Will throw exception if the Gauge is not of Number type
+				// Flink uses Gauge to store many types other than Number
+				g.getMetricValue();
+				request.addGauge(g);
+			} catch (Exception e) {
+				// Remove that Gauge if it's not of Number type
+				gauges.remove(entry.getKey());
+			}
+		}
+
+		for (DCounter c : counters.values()) {
+			request.addCounter(c);
+		}
+
+		for (DMeter m : meters.values()) {
+			request.addMeter(m);
+		}
+
+		try {
+			client.send(request);
+		} catch (Exception e) {
+			LOGGER.warn("Failed reporting metrics to Datadog.", e);
+		}
+	}
+
+	/**
+	 * Get config tags from config 'metrics.reporter.dghttp.tags'
+	 * */
+	private List<String> getTagsFromConfig(String str) {
+		return Arrays.asList(str.split(","));
+	}
+
+	/**
+	 * Get tags from MetricGroup#getAllVariables(), excluding 'host'
+	 * */
+	private List<String> getTagsFromMetricGroup(MetricGroup metricGroup) {
+		List<String> tags = new ArrayList<>();
+
+		for (Map.Entry<String, String> entry: metricGroup.getAllVariables().entrySet()) {
+			if(!entry.getKey().equals(HOST_VARIABLE)) {
+				tags.add(getVariableName(entry.getKey()) + ":" + entry.getValue());
+			}
+		}
+
+		return tags;
+	}
+
+	/**
+	 * Get host from MetricGroup#getAllVariables() if it exists; returns Null otherwise
+	 * */
+	private String getHostFromMetricGroup(MetricGroup metricGroup) {
+		return metricGroup.getAllVariables().get(HOST_VARIABLE);
+	}
+
+	/**
+	 * Given "<xxx>", return "xxx"
+	 * */
+	private String getVariableName(String str) {
+		return str.substring(1, str.length() - 1);
+	}
+
+	/**
+	 * Compact metrics in batch, serialize them, and send to Datadog via HTTP
+	 * */
+	static class DatadogHttpRequest {
+		private final DSeries series;
+
+		public DatadogHttpRequest() {
+			series = new DSeries();
+		}
+
+		public void addGauge(DGauge gauge) {
+			series.addMetric(gauge);
+		}
+
+		public void addCounter(DCounter counter) {
+			series.addMetric(counter);
+		}
+
+		public void addMeter(DMeter meter) {
+			series.addMetric(meter);
+		}
+
+		public DSeries getSeries() {
+			return series;
+		}
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/MetricType.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/MetricType.java
new file mode 100644
index 0000000000000..97f9b29ede309
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/MetricType.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+/**
+ * Metric types supported by Datadog
+ * */
+public enum MetricType {
+	/**
+	 * Names of 'gauge' and 'counter' must not be changed
+	 * since they are mapped to json objects in a Datadog-defined format
+	 * */
+	gauge, counter
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/test/java/org/apache/flink/metrics/datadog/DatadogHttpClientTest.java b/flink-metrics/flink-metrics-datadog/src/test/java/org/apache/flink/metrics/datadog/DatadogHttpClientTest.java
new file mode 100644
index 0000000000000..bda5d4749bd7f
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/test/java/org/apache/flink/metrics/datadog/DatadogHttpClientTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.datadog;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Meter;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Enclosed.class)
+public class DatadogHttpClientTest {
+	public static class TestApiKey {
+		@Test(expected = IllegalArgumentException.class)
+		public void testClientWithEmptyKey() {
+			new DatadogHttpClient("");
+		}
+
+		@Test(expected = IllegalArgumentException.class)
+		public void testClientWithNullKey() {
+			new DatadogHttpClient(null);
+		}
+	}
+
+	@RunWith(PowerMockRunner.class)
+	@PrepareForTest(DMetric.class)
+	public static class TestSerialization {
+		private static List<String> tags = Arrays.asList("tag1", "tag2");
+
+		private static final long MOCKED_SYSTEM_MILLIS = 123L;
+
+		@Before
+		public void mockSystemMillis() {
+			PowerMockito.mockStatic(DMetric.class);
+			PowerMockito.when(DMetric.getUnixEpochTimestamp()).thenReturn(MOCKED_SYSTEM_MILLIS);
+		}
+
+		@Test
+		public void serializeGauge() throws JsonProcessingException {
+
+			DGauge g = new DGauge(new Gauge<Number>() {
+				@Override
+				public Number getValue() {
+					return 1;
+				}
+			}, "testCounter", "localhost", tags);
+
+			assertEquals(
+				"{\"metric\":\"testCounter\",\"type\":\"gauge\",\"host\":\"localhost\",\"tags\":[\"tag1\",\"tag2\"],\"points\":[[123,1]]}",
+				DatadogHttpClient.serialize(g));
+		}
+
+		@Test
+		public void serializeGaugeWithoutHost() throws JsonProcessingException {
+
+			DGauge g = new DGauge(new Gauge<Number>() {
+				@Override
+				public Number getValue() {
+					return 1;
+				}
+			}, "testCounter", null, tags);
+
+			assertEquals(
+				"{\"metric\":\"testCounter\",\"type\":\"gauge\",\"tags\":[\"tag1\",\"tag2\"],\"points\":[[123,1]]}",
+				DatadogHttpClient.serialize(g));
+		}
+
+		@Test
+		public void serializeCounter() throws JsonProcessingException {
+			DCounter c = new DCounter(new Counter() {
+				@Override
+				public void inc() {}
+
+				@Override
+				public void inc(long n) {}
+
+				@Override
+				public void dec() {}
+
+				@Override
+				public void dec(long n) {}
+
+				@Override
+				public long getCount() {
+					return 1;
+				}
+			}, "testCounter", "localhost", tags);
+
+			assertEquals(
+				"{\"metric\":\"testCounter\",\"type\":\"counter\",\"host\":\"localhost\",\"tags\":[\"tag1\",\"tag2\"],\"points\":[[123,1]]}",
+				DatadogHttpClient.serialize(c));
+		}
+
+		@Test
+		public void serializeCounterWithoutHost() throws JsonProcessingException {
+			DCounter c = new DCounter(new Counter() {
+				@Override
+				public void inc() {}
+
+				@Override
+				public void inc(long n) {}
+
+				@Override
+				public void dec() {}
+
+				@Override
+				public void dec(long n) {}
+
+				@Override
+				public long getCount() {
+					return 1;
+				}
+			}, "testCounter", null, tags);
+
+			assertEquals(
+				"{\"metric\":\"testCounter\",\"type\":\"counter\",\"tags\":[\"tag1\",\"tag2\"],\"points\":[[123,1]]}",
+				DatadogHttpClient.serialize(c));
+		}
+
+		@Test
+		public void serializeMeter() throws JsonProcessingException {
+
+			DMeter m = new DMeter(new Meter() {
+				@Override
+				public void markEvent() {}
+
+				@Override
+				public void markEvent(long n) {}
+
+				@Override
+				public double getRate() {
+					return 1;
+				}
+
+				@Override
+				public long getCount() {
+					return 0;
+				}
+			}, "testMeter","localhost", tags);
+
+			assertEquals(
+				"{\"metric\":\"testMeter\",\"type\":\"gauge\",\"host\":\"localhost\",\"tags\":[\"tag1\",\"tag2\"],\"points\":[[123,1.0]]}",
+				DatadogHttpClient.serialize(m));
+		}
+
+		@Test
+		public void serializeMeterWithoutHost() throws JsonProcessingException {
+
+			DMeter m = new DMeter(new Meter() {
+				@Override
+				public void markEvent() {}
+
+				@Override
+				public void markEvent(long n) {}
+
+				@Override
+				public double getRate() {
+					return 1;
+				}
+
+				@Override
+				public long getCount() {
+					return 0;
+				}
+			}, "testMeter", null, tags);
+
+			assertEquals(
+				"{\"metric\":\"testMeter\",\"type\":\"gauge\",\"tags\":[\"tag1\",\"tag2\"],\"points\":[[123,1.0]]}",
+				DatadogHttpClient.serialize(m));
+		}
+	}
+}
diff --git a/flink-metrics/flink-metrics-datadog/src/test/resources/log4j-test.properties b/flink-metrics/flink-metrics-datadog/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000000000..2226f68653181
--- /dev/null
+++ b/flink-metrics/flink-metrics-datadog/src/test/resources/log4j-test.properties
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set root logger level to OFF to not flood build logs
+# set manually to INFO for debugging purposes
+log4j.rootLogger=OFF, testlogger
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
+log4j.appender.testlogger.target = System.err
+log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
+log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
diff --git a/flink-metrics/pom.xml b/flink-metrics/pom.xml
index 317dde8fb8c30..e1b66c23ff19b 100644
--- a/flink-metrics/pom.xml
+++ b/flink-metrics/pom.xml
@@ -40,6 +40,7 @@ under the License.
 		<module>flink-metrics-graphite</module>
 		<module>flink-metrics-jmx</module>
 		<module>flink-metrics-statsd</module>
+		<module>flink-metrics-datadog</module>
 	</modules>
 
 	<!-- override these root dependencies as 'provided', so they don't end up

From ae650c4b0cf9cdbf946fd9c3ab2ae0d97dcd7327 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Tue, 9 May 2017 13:13:02 +0200
Subject: [PATCH 207/253] [FLINK-6509] [tests] Perform
 TestingListener#waitForNewLeader under lock

Performin TestingListener#waitForNewLeader under the lock which is also hold when
updating the leader information makes sure that leader changes won't go unnoticed.
This led before to failing test cases due to timeouts.

This closes #3853.
---
 .../flink/runtime/leaderelection/TestingListener.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingListener.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingListener.java
index 87decc7bddfd8..85715054d173f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingListener.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingListener.java
@@ -51,11 +51,11 @@ public String waitForNewLeader(long timeout) throws Exception {
 		long start = System.currentTimeMillis();
 		long curTimeout;
 
-		while (
+		synchronized (lock) {
+			while (
 				exception == null &&
-				(address == null || address.equals(oldAddress)) &&
-				(curTimeout = timeout - System.currentTimeMillis() + start) > 0) {
-			synchronized (lock) {
+					(address == null || address.equals(oldAddress)) &&
+					(curTimeout = timeout - System.currentTimeMillis() + start) > 0) {
 				try {
 					lock.wait(curTimeout);
 				} catch (InterruptedException e) {

From bb4afb99032190eba7d8a69e7c3d5301fd32db16 Mon Sep 17 00:00:00 2001
From: paul <nellboy@gmail.com>
Date: Mon, 20 Feb 2017 18:20:45 +0100
Subject: [PATCH 208/253] [FLINK-5819] [webui] implements numeric option on
 metrics graphs

This closes #3367
---
 .../jobs/job.plan.node-list.metrics.jade      |  2 +-
 .../app/scripts/common/filters.coffee         | 34 +++++++++++++++++++
 .../app/scripts/modules/jobs/jobs.ctrl.coffee |  4 +++
 .../scripts/modules/jobs/metrics.dir.coffee   | 24 +++++++++----
 .../scripts/modules/jobs/metrics.svc.coffee   | 15 ++++++--
 .../web-dashboard/app/styles/metric.styl      |  5 +++
 .../web-dashboard/web/css/index.css           |  2 +-
 .../web-dashboard/web/js/hs/index.js          |  4 +--
 .../web-dashboard/web/js/index.js             |  4 +--
 .../web-dashboard/web/js/vendor.js            | 10 +++---
 .../jobs/job.plan.node-list.metrics.html      |  2 +-
 11 files changed, 85 insertions(+), 21 deletions(-)

diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade
index fd7382f50b5e0..f3ec8dc021733 100644
--- a/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade
+++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade
@@ -41,7 +41,7 @@ nav.navbar.navbar-default.navbar-secondary-additional.navbar-secondary-additiona
 
 ul.metric-row(ng-if="nodeid && metrics.length > 0" dnd-list="metrics" dnd-drop="dropped(event, index, item, external, type, external)")
   li.metric-col(ng-repeat="metric in metrics track by metric.id" dnd-draggable="metric" dnd-dragstart="dragStart()" dnd-dragend="dragEnd()" dnd-canceled="dragEnd()" ng-class="{big: metric.size != 'small'}")
-    metrics-graph(metric="metric" window="window" get-values="getValues(metric.id)" remove-metric="removeMetric(metric)" set-metric-size="setMetricSize")
+    metrics-graph(metric="metric" window="window" get-values="getValues(metric.id)" remove-metric="removeMetric(metric)" set-metric-size="setMetricSize" set-metric-view="setMetricView")
 
 .clearfix
 
diff --git a/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee b/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee
index a3ce508cc5c52..e2b83391e8975 100644
--- a/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee
+++ b/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee
@@ -98,3 +98,37 @@ angular.module('flinkApp')
 .filter "increment", ->
   (number) ->
     parseInt(number) + 1
+
+.filter "humanizeChartNumeric", ['humanizeBytesFilter', 'humanizeDurationFilter', (humanizeBytesFilter, humanizeDurationFilter)->
+  (value, metric)->
+    return_val = ''
+    if value != null
+      if /bytes/i.test(metric.id) && /persecond/i.test(metric.id)
+        return_val = humanizeBytesFilter(value) + ' / s'
+      else if /bytes/i.test(metric.id)
+        return_val = humanizeBytesFilter(value)
+      else if /persecond/i.test(metric.id)
+        return_val = value + ' / s'
+      else if /time/i.test(metric.id) || /latency/i.test(metric.id)
+        return_val = humanizeDurationFilter(value, true)
+      else
+        return_val = value
+    return return_val
+]
+
+.filter "humanizeChartNumericTitle", ['humanizeDurationFilter', (humanizeDurationFilter)->
+  (value, metric)->
+    return_val = ''
+    if value != null
+      if /bytes/i.test(metric.id) && /persecond/i.test(metric.id)
+        return_val = value + ' Bytes / s'
+      else if /bytes/i.test(metric.id)
+        return_val = value + ' Bytes'
+      else if /persecond/i.test(metric.id)
+        return_val = value + ' / s'
+      else if /time/i.test(metric.id) || /latency/i.test(metric.id)
+        return_val = humanizeDurationFilter(value, false)
+      else
+        return_val = value
+    return return_val
+]
diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee
index f25c94d1173fc..d4315ea2499b4 100644
--- a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee
+++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee
@@ -385,6 +385,10 @@ angular.module('flinkApp')
     MetricsService.setMetricSize($scope.jobid, $scope.nodeid, metric, size)
     loadMetrics()
 
+  $scope.setMetricView = (metric, view) ->
+    MetricsService.setMetricView($scope.jobid, $scope.nodeid, metric, view)
+    loadMetrics()
+
   $scope.getValues = (metric) ->
     MetricsService.getValues($scope.jobid, $scope.nodeid, metric)
 
diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.dir.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.dir.coffee
index adfc09f17944e..a3787488f5e7e 100644
--- a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.dir.coffee
+++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.dir.coffee
@@ -33,8 +33,16 @@ angular.module('flinkApp')
                  </div>
                </div>
                <div class="panel-body">
-                  <svg />
+                  <svg ng-if="metric.view == \'chart\'"/>
+                  <div ng-if="metric.view != \'chart\'">
+                      <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div>
+                  </div>
                </div>
+               <div class="buttons">
+                 <div class="btn-group">
+                   <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button>
+                   <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button>
+                 </div>
              </div>'
   replace: true
   scope:
@@ -42,6 +50,7 @@ angular.module('flinkApp')
     window: "="
     removeMetric: "&"
     setMetricSize: "="
+    setMetricView: "="
     getValues: "&"
 
   link: (scope, element, attrs) ->
@@ -106,10 +115,13 @@ angular.module('flinkApp')
     scope.setSize = (size) ->
       scope.setMetricSize(scope.metric, size)
 
-    scope.showChart()
+    scope.setView = (view) ->
+      scope.setMetricView(scope.metric, view)
+      scope.showChart() if view == 'chart'
+
+    scope.showChart() if scope.metric.view == 'chart'
 
     scope.$on 'metrics:data:update', (event, timestamp, data) ->
-#      scope.value = parseInt(data[scope.metric.id])
       scope.value = parseFloat(data[scope.metric.id])
 
       scope.data[0].values.push {
@@ -120,8 +132,8 @@ angular.module('flinkApp')
       if scope.data[0].values.length > scope.window
         scope.data[0].values.shift()
 
-      scope.showChart()
-      scope.chart.clearHighlights()
+      scope.showChart() if scope.metric.view == 'chart'
+      scope.chart.clearHighlights() if scope.metric.view == 'chart'
       scope.chart.tooltip.hidden(true)
 
     element.find(".metric-title").qtip({
@@ -135,4 +147,4 @@ angular.module('flinkApp')
       style: {
         classes: 'qtip-light qtip-timeline-bar'
       }
-    });
+    });
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.svc.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.svc.coffee
index 9b9fab77827dc..fbe3d6d8e5cdf 100644
--- a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.svc.coffee
+++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/metrics.svc.coffee
@@ -110,7 +110,7 @@ angular.module('flinkApp')
   @addMetric = (jobid, nodeid, metricid) ->
     @setupLSFor(jobid, nodeid)
 
-    @metrics[jobid][nodeid].push({id: metricid, size: 'small'})
+    @metrics[jobid][nodeid].push({id: metricid, size: 'small', view: 'chart'})
 
     @saveSetup()
 
@@ -128,7 +128,16 @@ angular.module('flinkApp')
       i = @metrics[jobid][nodeid].indexOf(metric.id)
       i = _.findIndex(@metrics[jobid][nodeid], { id: metric.id }) if i == -1
 
-      @metrics[jobid][nodeid][i] = { id: metric.id, size: size } if i != -1
+      @metrics[jobid][nodeid][i] = { id: metric.id, size: size, view: metric.view } if i != -1
+
+      @saveSetup()
+
+  @setMetricView = (jobid, nodeid, metric, view) =>
+    if @metrics[jobid][nodeid]?
+      i = @metrics[jobid][nodeid].indexOf(metric.id)
+      i = _.findIndex(@metrics[jobid][nodeid], { id: metric.id }) if i == -1
+
+      @metrics[jobid][nodeid][i] = { id: metric.id, size: metric.size, view: view } if i != -1
 
       @saveSetup()
 
@@ -148,7 +157,7 @@ angular.module('flinkApp')
   @getMetricsSetup = (jobid, nodeid) =>
     {
       names: _.map(@metrics[jobid][nodeid], (value) =>
-        if _.isString(value) then { id: value, size: "small" } else value
+        if _.isString(value) then { id: value, size: "small", view: "chart" } else value
       )
     }
 
diff --git a/flink-runtime-web/web-dashboard/app/styles/metric.styl b/flink-runtime-web/web-dashboard/app/styles/metric.styl
index 972352f1c6f6c..94c494838a9bf 100644
--- a/flink-runtime-web/web-dashboard/app/styles/metric.styl
+++ b/flink-runtime-web/web-dashboard/app/styles/metric.styl
@@ -67,6 +67,11 @@ $metric-row-height = 180px + 85px
         background-color: transparent
         height: $metric-row-height
         position: relative
+        .metric-numeric
+          text-align: center;
+          margin-top: 75px;
+          font-size: 40px;
+          font-weight: bold;
 
       .panel-heading
         padding: 0px 10px
diff --git a/flink-runtime-web/web-dashboard/web/css/index.css b/flink-runtime-web/web-dashboard/web/css/index.css
index 7f2099879be12..db615b1fb0081 100644
--- a/flink-runtime-web/web-dashboard/web/css/index.css
+++ b/flink-runtime-web/web-dashboard/web/css/index.css
@@ -1 +1 @@
-#main,#sidebar,body,html{height:100%}#content,#sidebar{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s}.gutter{background-color:transparent;background-repeat:no-repeat;background-position:50%}.gutter-vertical{cursor:row-resize;background-image:url(../images/grips/horizontal.png)}#sidebar{overflow:hidden;position:fixed;left:-250px;top:0;bottom:0;width:250px;background:#151515;transition:.4s;-webkit-box-shadow:inset -10px 0 10px rgba(0,0,0,.2);box-shadow:inset -10px 0 10px rgba(0,0,0,.2)}#sidebar.sidebar-visible{left:0}#sidebar .logo{width:auto;height:22px}#sidebar .logo img{display:inline-block}#sidebar .navbar-static-top{overflow:hidden;height:51px}#sidebar .navbar-static-top .navbar-header{width:100%}#sidebar .navbar-brand.navbar-brand-text{font-size:14px;font-weight:700;color:#fff;padding-left:0}#sidebar .nav>li>a{color:#aaa;margin-bottom:1px}#sidebar .nav>li>a:focus,#sidebar .nav>li>a:hover{background-color:rgba(40,40,40,.5)}#sidebar .nav>li>a.active{background-color:rgba(100,100,100,.5)}#content{background-color:#fff;margin-left:0;padding-top:70px;height:100%;transition:.4s}.table .table,.table.table-inner{background-color:transparent}#content .navbar-main,#content .navbar-main-additional{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s;transition:.4s}#content .navbar-main-additional{margin-top:51px;border-bottom:none;padding:0 20px}#content .navbar-main-additional .nav-tabs{margin:0 -20px;padding:0 20px}#content .navbar-secondary-additional{border:none;padding:0 20px;margin-bottom:0}#content .navbar-secondary-additional .nav-tabs{margin:0 -20px}#content.sidebar-visible{margin-left:250px}#content.sidebar-visible .navbar-main,#content.sidebar-visible .navbar-main-additional{left:250px}#content #fold-button{display:inline-block;margin-left:20px}#content #content-inner{padding:0 20px 20px}#content #content-inner.has-navbar-main-additional{padding-top:42px}.table#add-file-table span.btn,.table#job-submit-table td>span.btn{padding:2px 4px;font-size:14px}.page-header{margin:0 0 20px}.nav>li>a,.nav>li>a:focus,.nav>li>a:hover{color:#aaa;background-color:transparent;border-bottom:2px solid transparent}.nav>li.active>a,.nav>li.active>a:focus,.nav>li.active>a:hover{color:#000;border-bottom:2px solid #000}.nav.nav-tabs{margin-bottom:20px}.table th{font-weight:400;color:#999}.table td.td-long{width:20%;white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.table.table-clickable tr{cursor:pointer}.table.table-properties{table-layout:fixed;white-space:nowrap}.table.table-properties td{width:50%;white-space:nowrap;overflow:hidden;-o-text-overflow:ellipsis;text-overflow:ellipsis}.table.table-body-hover>tbody{border-top:none;border-left:2px solid transparent}.table.table-body-hover>tbody.active{border-left:2px solid #000}.table.table-body-hover>tbody.active td.tab-column li.active,.table.table-body-hover>tbody.active td:not(.tab-column),.table.table-body-hover>tbody:hover td.tab-column li.active,.table.table-body-hover>tbody:hover td:not(.tab-column){background-color:#f0f0f0}.table.table-activable td.tab-column,.table.table-activable th.tab-column{border-top:none;width:47px}.table.table-activable td.tab-column{border-right:1px solid #ddd}.table.table-activable td{position:relative}.table.table-no-border td,.table.table-no-border th{border-top:none!important}.table#job-submit-table{table-layout:fixed;white-space:nowrap}.table#job-submit-table td.td-large{width:40%}.table#job-submit-table td{width:15%}.table#job-submit-table td>input{height:28px;font-size:14px}.table#add-file-table{table-layout:fixed}.table#add-file-table span.btn{position:relative;overflow:hidden;border-radius:2px;margin-top:-3px}.table#add-file-table td#add-file-button{width:100px}.table#add-file-table td#add-file-button input[type=file]{position:absolute;top:0;right:0;min-width:100%;min-height:100%;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);outline:0;cursor:inherit;display:block}.timeline-canvas .timeline-insidelabel,.timeline-canvas .timeline-series,svg.graph .node{cursor:pointer}.table#add-file-table td#add-file-name{width:250px;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;white-space:nowrap}.table#add-file-table td#add-file-status{width:100%}.table#add-file-table td#add-file-status span.btn-progress-bar{padding:0!important;width:100%;background-color:#f5f5f5;text-align:left}.table#add-file-table td#add-file-status span.btn-progress{padding:2px;font-size:10px}.table span.error-area{color:red}.table span.row-button{padding:1px 2px;margin:0;border:none!important}.table .small-label{text-transform:uppercase;font-size:13px;color:#999}span.icon-wrapper{width:1.2em;display:inline-block}.panel.panel-dashboard .huge{font-size:28px}.panel.panel-lg{font-size:16px}.panel.panel-lg .badge{font-size:14px}.navbar-secondary{overflow:auto}.navbar-main .navbar-title,.navbar-main .navbar-title-job,.navbar-main .panel-title,.navbar-main-additional .navbar-title,.navbar-main-additional .navbar-title-job,.navbar-main-additional .panel-title,.navbar-secondary .navbar-title,.navbar-secondary .navbar-title-job,.navbar-secondary .panel-title,.navbar-secondary-additional .navbar-title,.navbar-secondary-additional .navbar-title-job,.navbar-secondary-additional .panel-title,.panel.panel-multi .navbar-title,.panel.panel-multi .navbar-title-job,.panel.panel-multi .panel-title{float:left;font-size:18px;padding:12px 20px 13px 10px;color:#333;display:inline-block}.navbar-main .navbar-info,.navbar-main .panel-info,.navbar-main-additional .navbar-info,.navbar-main-additional .panel-info,.navbar-secondary .navbar-info,.navbar-secondary .panel-info,.navbar-secondary-additional .navbar-info,.navbar-secondary-additional .panel-info,.panel.panel-multi .navbar-info,.panel.panel-multi .panel-info{float:left;font-size:14px;padding:15px;color:#999;display:inline-block;border-right:1px solid #e7e7e7;overflow:hidden}.navbar-main .navbar-info .overflow,.navbar-main .panel-info .overflow,.navbar-main-additional .navbar-info .overflow,.navbar-main-additional .panel-info .overflow,.navbar-secondary .navbar-info .overflow,.navbar-secondary .panel-info .overflow,.navbar-secondary-additional .navbar-info .overflow,.navbar-secondary-additional .panel-info .overflow,.panel.panel-multi .navbar-info .overflow,.panel.panel-multi .panel-info .overflow{position:absolute;display:block;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;height:22px;line-height:22px;vertical-align:middle}.navbar-main .navbar-info.first,.navbar-main .panel-info.first,.navbar-main-additional .navbar-info.first,.navbar-main-additional .panel-info.first,.navbar-secondary .navbar-info.first,.navbar-secondary .panel-info.first,.navbar-secondary-additional .navbar-info.first,.navbar-secondary-additional .panel-info.first,.panel.panel-multi .navbar-info.first,.panel.panel-multi .panel-info.first{border-left:1px solid #e7e7e7}.navbar-main .navbar-info.last,.navbar-main .panel-info.last,.navbar-main-additional .navbar-info.last,.navbar-main-additional .panel-info.last,.navbar-secondary .navbar-info.last,.navbar-secondary .panel-info.last,.navbar-secondary-additional .navbar-info.last,.navbar-secondary-additional .panel-info.last,.panel.panel-multi .navbar-info.last,.panel.panel-multi .panel-info.last{border-right:none}.panel.panel-multi .panel-heading{padding:0}.panel.panel-multi .panel-heading .panel-info.thin{padding:8px 10px}.panel.panel-multi .panel-body{padding:10px;background-color:#fdfdfd;color:#999;font-size:13px}.panel.panel-multi .panel-body.clean{color:inherit;font-size:inherit}.navbar-main-additional,.navbar-secondary-additional{min-height:40px;background-color:#fdfdfd}.navbar-main-additional .navbar-info,.navbar-secondary-additional .navbar-info{font-size:13px;padding:10px 15px}.nav-top-affix.affix{width:100%;top:50px;margin-left:-20px;padding-left:20px;margin-right:-20px;padding-right:20px;background-color:#fff;z-index:1}.badge-default[href]:focus,.badge-default[href]:hover{background-color:grey}.badge-primary{background-color:#428bca}.badge-primary[href]:focus,.badge-primary[href]:hover{background-color:#3071a9}.badge-success{background-color:#5cb85c}.badge-success[href]:focus,.badge-success[href]:hover{background-color:#449d44}.badge-info{background-color:#5bc0de}.badge-info[href]:focus,.badge-info[href]:hover{background-color:#31b0d5}.badge-warning{background-color:#f0ad4e}.badge-warning[href]:focus,.badge-warning[href]:hover{background-color:#ec971f}.badge-danger{background-color:#d9534f}.badge-danger[href]:focus,.badge-danger[href]:hover{background-color:#c9302c}.indicator{display:inline-block;margin-right:15px}.indicator.indicator-primary{color:#428bca}.indicator.indicator-success{color:#5cb85c}.indicator.indicator-info{color:#5bc0de}.indicator.indicator-warning{color:#f0ad4e}.indicator.indicator-danger{color:#d9534f}pre.exception{border:none;background-color:transparent;padding:0;margin:0}pre{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.nav-tabs.tabs-vertical{position:absolute;left:0;top:0;border-bottom:none;z-index:100}.nav-tabs.tabs-vertical li{float:none;margin-bottom:0;margin-right:-1px}.nav-tabs.tabs-vertical li>a{margin-right:0;border-radius:0;border-bottom:none;border-left:2px solid transparent}.nav-tabs.tabs-vertical li.active>a,.nav-tabs.tabs-vertical li>a:focus,.nav-tabs.tabs-vertical li>a:hover{border-bottom:none;border-left:2px solid #000}.navbar-main .navbar-title,.navbar-main-additional .navbar-title,.navbar-secondary .navbar-title,.navbar-secondary-additional .navbar-title{padding:12px 20px 13px}.navbar-main .navbar-title-job{padding:8px 20px}.navbar-main .navbar-title-job .indicator-primary{padding:8px 0 0}.navbar-main .navbar-title-job .no-padding{padding:0}.navbar-main .navbar-title-job .no-margin{margin:0}.navbar-main .navbar-title-job .job-name{font-size:14px}.navbar-main .navbar-title-job .job-id{color:#999;font-size:11px}.checkpoint-overview a,svg.graph .node h4{color:#000}livechart{width:30%;height:30%;text-align:center}.canvas-wrapper{border:1px solid #ddd;position:relative;margin-bottom:20px;height:100%}.canvas-wrapper .main-canvas{height:100%;overflow:hidden}.canvas-wrapper .main-canvas .zoom-buttons{position:absolute;top:10px;right:10px}.label-group .label{display:inline-block;padding-left:.4em;padding-right:.4em;margin:0;border-right:1px solid #fff;border-radius:0}.label-group .label.label-black{background-color:#000}.navbar-info-button{padding:3px 4px;font-size:12px;font-family:inherit;margin-top:-2px}svg.graph .edge-label,svg.graph text{font-size:14px}.checkpoints-view{padding-top:1em}.subtask-details .blank{height:2em}.checkpoint-overview td span{padding-left:2em}svg.graph{overflow:hidden;height:100%;width:100%!important}svg.graph g.type-TK>rect{fill:#00ffd0}svg.graph text{font-weight:300}svg.graph .node>rect{stroke:#999;stroke-width:5px;fill:#fff;margin:0;padding:0}svg.graph .node[active]>rect{fill:#eee}svg.graph .node.node-mirror>rect{stroke:#a8a8a8}svg.graph .node.node-iteration>rect{stroke:#cd3333}svg.graph .node.node-source>rect{stroke:#4ce199}svg.graph .node.node-sink>rect{stroke:#e6ec8b}svg.graph .node.node-normal>rect{stroke:#3fb6d8}svg.graph .node h5{color:#999}svg.graph .edgeLabel rect{fill:#fff}svg.graph .edgePath path{stroke:#333;stroke-width:2px;fill:#333}svg.graph .label{color:#777;margin:0}svg.graph .node-label{display:block;margin:0;text-decoration:none}.timeline{overflow:hidden}.timeline-canvas{overflow:hidden;padding:10px}.timeline-canvas .bar-container{overflow:hidden}.timeline-canvas.secondary .timeline-insidelabel,.timeline-canvas.secondary .timeline-series{cursor:auto}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics a,.show-pointer{cursor:pointer}.qtip-timeline-bar{font-size:14px;line-height:1.4}#content .navbar-secondary-additional.navbar-secondary-additional-2{margin:-10px -10px 10px;padding:0;border-bottom:1px solid #e4e4e4}#content .navbar-secondary-additional.navbar-secondary-additional-2 .navbar-info{padding-top:12px;padding-bottom:12px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics{margin-right:15px;float:right}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics .btn{margin-top:5px;margin-bottom:5px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .metric-menu{max-height:300px;max-width:900px;overflow-y:scroll;text-align:right}.metric-row{margin:0;min-height:275px;padding:0;list-style-type:none}.metric-row .metric-col{background-color:transparent;width:33.33%;float:left}.metric-row .metric-col.big{width:100%}.metric-row .metric-col .panel{margin-left:5px;margin-right:5px;min-height:265px;margin-bottom:10px}.metric-row .metric-col .panel .panel-body{background-color:transparent;height:265px;position:relative}.metric-row .metric-col .panel .panel-heading{padding:0 10px;background-color:transparent;height:41px;line-height:41px;position:relative;overflow:hidden;cursor:pointer}.metric-row .metric-col .panel .panel-heading .metric-title{padding:10px 0}.metric-row .metric-col .panel .panel-heading .buttons{position:absolute;top:0;right:0;padding:0 10px;background-color:#fff}.metric-row .metric-col.dndDraggingSource{display:none}.metric-row .dndPlaceholder{position:relative;background-color:#f0f0f0;min-height:305px;display:block;width:33.33%;float:left;margin-bottom:10px;border-radius:5px}.p-info{padding-left:5px;padding-right:5px}@media (min-width:1024px) and (max-width:1279px){#content #fold-button,#sidebar .navbar-static-top .navbar-brand-text{display:none}#sidebar{left:0;width:160px}#content{margin-left:160px}#content .navbar-main,#content .navbar-main-additional{left:160px}.table td.td-long{width:20%}}@media (min-width:1280px){#sidebar{left:0}#content{margin-left:250px}#content #fold-button{display:none}#content .navbar-main,#content .navbar-main-additional{left:250px}.table td.td-long{width:30%}}.legend-box{font-size:10px;width:2em}#total-mem{background-color:#7cb5ec}#heap-mem{background-color:#434348}#non-heap-mem{background-color:#90ed7d}#fetch-plan,#job-submit{width:100px}#content-inner,#details,#node-details{height:100%}#job-panel{overflow-y:auto}
\ No newline at end of file
+#main,#sidebar,body,html{height:100%}#content,#sidebar{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s}.gutter{background-color:transparent;background-repeat:no-repeat;background-position:50%}.gutter-vertical{cursor:row-resize;background-image:url(../images/grips/horizontal.png)}#sidebar{overflow:hidden;position:fixed;left:-250px;top:0;bottom:0;width:250px;background:#151515;transition:.4s;-webkit-box-shadow:inset -10px 0 10px rgba(0,0,0,.2);box-shadow:inset -10px 0 10px rgba(0,0,0,.2)}#sidebar.sidebar-visible{left:0}#sidebar .logo{width:auto;height:22px}#sidebar .logo img{display:inline-block}#sidebar .navbar-static-top{overflow:hidden;height:51px}#sidebar .navbar-static-top .navbar-header{width:100%}#sidebar .navbar-brand.navbar-brand-text{font-size:14px;font-weight:700;color:#fff;padding-left:0}#sidebar .nav>li>a{color:#aaa;margin-bottom:1px}#sidebar .nav>li>a:focus,#sidebar .nav>li>a:hover{background-color:rgba(40,40,40,.5)}#sidebar .nav>li>a.active{background-color:rgba(100,100,100,.5)}#content{background-color:#fff;margin-left:0;padding-top:70px;height:100%;transition:.4s}.table .table,.table.table-inner{background-color:transparent}#content .navbar-main,#content .navbar-main-additional{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s;transition:.4s}#content .navbar-main-additional{margin-top:51px;border-bottom:none;padding:0 20px}#content .navbar-main-additional .nav-tabs{margin:0 -20px;padding:0 20px}#content .navbar-secondary-additional{border:none;padding:0 20px;margin-bottom:0}#content .navbar-secondary-additional .nav-tabs{margin:0 -20px}#content.sidebar-visible{margin-left:250px}#content.sidebar-visible .navbar-main,#content.sidebar-visible .navbar-main-additional{left:250px}#content #fold-button{display:inline-block;margin-left:20px}#content #content-inner{padding:0 20px 20px}#content #content-inner.has-navbar-main-additional{padding-top:42px}.table#add-file-table span.btn,.table#job-submit-table td>span.btn{padding:2px 4px;font-size:14px}.page-header{margin:0 0 20px}.nav>li>a,.nav>li>a:focus,.nav>li>a:hover{color:#aaa;background-color:transparent;border-bottom:2px solid transparent}.nav>li.active>a,.nav>li.active>a:focus,.nav>li.active>a:hover{color:#000;border-bottom:2px solid #000}.nav.nav-tabs{margin-bottom:20px}.table th{font-weight:400;color:#999}.table td.td-long{width:20%;white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.table.table-clickable tr{cursor:pointer}.table.table-properties{table-layout:fixed;white-space:nowrap}.table.table-properties td{width:50%;white-space:nowrap;overflow:hidden;-o-text-overflow:ellipsis;text-overflow:ellipsis}.table.table-body-hover>tbody{border-top:none;border-left:2px solid transparent}.table.table-body-hover>tbody.active{border-left:2px solid #000}.table.table-body-hover>tbody.active td.tab-column li.active,.table.table-body-hover>tbody.active td:not(.tab-column),.table.table-body-hover>tbody:hover td.tab-column li.active,.table.table-body-hover>tbody:hover td:not(.tab-column){background-color:#f0f0f0}.table.table-activable td.tab-column,.table.table-activable th.tab-column{border-top:none;width:47px}.table.table-activable td.tab-column{border-right:1px solid #ddd}.table.table-activable td{position:relative}.table.table-no-border td,.table.table-no-border th{border-top:none!important}.table#job-submit-table{table-layout:fixed;white-space:nowrap}.table#job-submit-table td.td-large{width:40%}.table#job-submit-table td{width:15%}.table#job-submit-table td>input{height:28px;font-size:14px}.table#add-file-table{table-layout:fixed}.table#add-file-table span.btn{position:relative;overflow:hidden;border-radius:2px;margin-top:-3px}.table#add-file-table td#add-file-button{width:100px}.table#add-file-table td#add-file-button input[type=file]{position:absolute;top:0;right:0;min-width:100%;min-height:100%;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);outline:0;cursor:inherit;display:block}.timeline-canvas .timeline-insidelabel,.timeline-canvas .timeline-series,svg.graph .node{cursor:pointer}.table#add-file-table td#add-file-name{width:250px;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;white-space:nowrap}.table#add-file-table td#add-file-status{width:100%}.table#add-file-table td#add-file-status span.btn-progress-bar{padding:0!important;width:100%;background-color:#f5f5f5;text-align:left}.table#add-file-table td#add-file-status span.btn-progress{padding:2px;font-size:10px}.table span.error-area{color:red}.table span.row-button{padding:1px 2px;margin:0;border:none!important}.table .small-label{text-transform:uppercase;font-size:13px;color:#999}span.icon-wrapper{width:1.2em;display:inline-block}.panel.panel-dashboard .huge{font-size:28px}.panel.panel-lg{font-size:16px}.panel.panel-lg .badge{font-size:14px}.navbar-secondary{overflow:auto}.navbar-main .navbar-title,.navbar-main .navbar-title-job,.navbar-main .panel-title,.navbar-main-additional .navbar-title,.navbar-main-additional .navbar-title-job,.navbar-main-additional .panel-title,.navbar-secondary .navbar-title,.navbar-secondary .navbar-title-job,.navbar-secondary .panel-title,.navbar-secondary-additional .navbar-title,.navbar-secondary-additional .navbar-title-job,.navbar-secondary-additional .panel-title,.panel.panel-multi .navbar-title,.panel.panel-multi .navbar-title-job,.panel.panel-multi .panel-title{float:left;font-size:18px;padding:12px 20px 13px 10px;color:#333;display:inline-block}.navbar-main .navbar-info,.navbar-main .panel-info,.navbar-main-additional .navbar-info,.navbar-main-additional .panel-info,.navbar-secondary .navbar-info,.navbar-secondary .panel-info,.navbar-secondary-additional .navbar-info,.navbar-secondary-additional .panel-info,.panel.panel-multi .navbar-info,.panel.panel-multi .panel-info{float:left;font-size:14px;padding:15px;color:#999;display:inline-block;border-right:1px solid #e7e7e7;overflow:hidden}.navbar-main .navbar-info .overflow,.navbar-main .panel-info .overflow,.navbar-main-additional .navbar-info .overflow,.navbar-main-additional .panel-info .overflow,.navbar-secondary .navbar-info .overflow,.navbar-secondary .panel-info .overflow,.navbar-secondary-additional .navbar-info .overflow,.navbar-secondary-additional .panel-info .overflow,.panel.panel-multi .navbar-info .overflow,.panel.panel-multi .panel-info .overflow{position:absolute;display:block;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;height:22px;line-height:22px;vertical-align:middle}.navbar-main .navbar-info.first,.navbar-main .panel-info.first,.navbar-main-additional .navbar-info.first,.navbar-main-additional .panel-info.first,.navbar-secondary .navbar-info.first,.navbar-secondary .panel-info.first,.navbar-secondary-additional .navbar-info.first,.navbar-secondary-additional .panel-info.first,.panel.panel-multi .navbar-info.first,.panel.panel-multi .panel-info.first{border-left:1px solid #e7e7e7}.navbar-main .navbar-info.last,.navbar-main .panel-info.last,.navbar-main-additional .navbar-info.last,.navbar-main-additional .panel-info.last,.navbar-secondary .navbar-info.last,.navbar-secondary .panel-info.last,.navbar-secondary-additional .navbar-info.last,.navbar-secondary-additional .panel-info.last,.panel.panel-multi .navbar-info.last,.panel.panel-multi .panel-info.last{border-right:none}.panel.panel-multi .panel-heading{padding:0}.panel.panel-multi .panel-heading .panel-info.thin{padding:8px 10px}.panel.panel-multi .panel-body{padding:10px;background-color:#fdfdfd;color:#999;font-size:13px}.panel.panel-multi .panel-body.clean{color:inherit;font-size:inherit}.navbar-main-additional,.navbar-secondary-additional{min-height:40px;background-color:#fdfdfd}.navbar-main-additional .navbar-info,.navbar-secondary-additional .navbar-info{font-size:13px;padding:10px 15px}.nav-top-affix.affix{width:100%;top:50px;margin-left:-20px;padding-left:20px;margin-right:-20px;padding-right:20px;background-color:#fff;z-index:1}.badge-default[href]:focus,.badge-default[href]:hover{background-color:grey}.badge-primary{background-color:#428bca}.badge-primary[href]:focus,.badge-primary[href]:hover{background-color:#3071a9}.badge-success{background-color:#5cb85c}.badge-success[href]:focus,.badge-success[href]:hover{background-color:#449d44}.badge-info{background-color:#5bc0de}.badge-info[href]:focus,.badge-info[href]:hover{background-color:#31b0d5}.badge-warning{background-color:#f0ad4e}.badge-warning[href]:focus,.badge-warning[href]:hover{background-color:#ec971f}.badge-danger{background-color:#d9534f}.badge-danger[href]:focus,.badge-danger[href]:hover{background-color:#c9302c}.indicator{display:inline-block;margin-right:15px}.indicator.indicator-primary{color:#428bca}.indicator.indicator-success{color:#5cb85c}.indicator.indicator-info{color:#5bc0de}.indicator.indicator-warning{color:#f0ad4e}.indicator.indicator-danger{color:#d9534f}pre.exception{border:none;background-color:transparent;padding:0;margin:0}pre{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.nav-tabs.tabs-vertical{position:absolute;left:0;top:0;border-bottom:none;z-index:100}.nav-tabs.tabs-vertical li{float:none;margin-bottom:0;margin-right:-1px}.nav-tabs.tabs-vertical li>a{margin-right:0;border-radius:0;border-bottom:none;border-left:2px solid transparent}.nav-tabs.tabs-vertical li.active>a,.nav-tabs.tabs-vertical li>a:focus,.nav-tabs.tabs-vertical li>a:hover{border-bottom:none;border-left:2px solid #000}.navbar-main .navbar-title,.navbar-main-additional .navbar-title,.navbar-secondary .navbar-title,.navbar-secondary-additional .navbar-title{padding:12px 20px 13px}.navbar-main .navbar-title-job{padding:8px 20px}.navbar-main .navbar-title-job .indicator-primary{padding:8px 0 0}.navbar-main .navbar-title-job .no-padding{padding:0}.navbar-main .navbar-title-job .no-margin{margin:0}.navbar-main .navbar-title-job .job-name{font-size:14px}.navbar-main .navbar-title-job .job-id{color:#999;font-size:11px}.checkpoint-overview a,svg.graph .node h4{color:#000}livechart{width:30%;height:30%;text-align:center}.canvas-wrapper{border:1px solid #ddd;position:relative;margin-bottom:20px;height:100%}.canvas-wrapper .main-canvas{height:100%;overflow:hidden}.canvas-wrapper .main-canvas .zoom-buttons{position:absolute;top:10px;right:10px}.label-group .label{display:inline-block;padding-left:.4em;padding-right:.4em;margin:0;border-right:1px solid #fff;border-radius:0}.label-group .label.label-black{background-color:#000}.navbar-info-button{padding:3px 4px;font-size:12px;font-family:inherit;margin-top:-2px}svg.graph .edge-label,svg.graph text{font-size:14px}.checkpoints-view{padding-top:1em}.subtask-details .blank{height:2em}.checkpoint-overview td span{padding-left:2em}svg.graph{overflow:hidden;height:100%;width:100%!important}svg.graph g.type-TK>rect{fill:#00ffd0}svg.graph text{font-weight:300}svg.graph .node>rect{stroke:#999;stroke-width:5px;fill:#fff;margin:0;padding:0}svg.graph .node[active]>rect{fill:#eee}svg.graph .node.node-mirror>rect{stroke:#a8a8a8}svg.graph .node.node-iteration>rect{stroke:#cd3333}svg.graph .node.node-source>rect{stroke:#4ce199}svg.graph .node.node-sink>rect{stroke:#e6ec8b}svg.graph .node.node-normal>rect{stroke:#3fb6d8}svg.graph .node h5{color:#999}svg.graph .edgeLabel rect{fill:#fff}svg.graph .edgePath path{stroke:#333;stroke-width:2px;fill:#333}svg.graph .label{color:#777;margin:0}svg.graph .node-label{display:block;margin:0;text-decoration:none}.timeline{overflow:hidden}.timeline-canvas{overflow:hidden;padding:10px}.timeline-canvas .bar-container{overflow:hidden}.timeline-canvas.secondary .timeline-insidelabel,.timeline-canvas.secondary .timeline-series{cursor:auto}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics a,.show-pointer{cursor:pointer}.qtip-timeline-bar{font-size:14px;line-height:1.4}#content .navbar-secondary-additional.navbar-secondary-additional-2{margin:-10px -10px 10px;padding:0;border-bottom:1px solid #e4e4e4}#content .navbar-secondary-additional.navbar-secondary-additional-2 .navbar-info{padding-top:12px;padding-bottom:12px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics{margin-right:15px;float:right}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics .btn{margin-top:5px;margin-bottom:5px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .metric-menu{max-height:300px;max-width:900px;overflow-y:scroll;text-align:right}.metric-row{margin:0;min-height:275px;padding:0;list-style-type:none}.metric-row .metric-col{background-color:transparent;width:33.33%;float:left}.metric-row .metric-col.big{width:100%}.metric-row .metric-col .panel{margin-left:5px;margin-right:5px;min-height:265px;margin-bottom:10px}.metric-row .metric-col .panel .panel-body{background-color:transparent;height:265px;position:relative}.metric-row .metric-col .panel .panel-body .metric-numeric{text-align:center;margin-top:75px;font-size:40px;font-weight:700}.metric-row .metric-col .panel .panel-heading{padding:0 10px;background-color:transparent;height:41px;line-height:41px;position:relative;overflow:hidden;cursor:pointer}.metric-row .metric-col .panel .panel-heading .metric-title{padding:10px 0}.metric-row .metric-col .panel .panel-heading .buttons{position:absolute;top:0;right:0;padding:0 10px;background-color:#fff}.metric-row .metric-col.dndDraggingSource{display:none}.metric-row .dndPlaceholder{position:relative;background-color:#f0f0f0;min-height:305px;display:block;width:33.33%;float:left;margin-bottom:10px;border-radius:5px}.p-info{padding-left:5px;padding-right:5px}@media (min-width:1024px) and (max-width:1279px){#content #fold-button,#sidebar .navbar-static-top .navbar-brand-text{display:none}#sidebar{left:0;width:160px}#content{margin-left:160px}#content .navbar-main,#content .navbar-main-additional{left:160px}.table td.td-long{width:20%}}@media (min-width:1280px){#sidebar{left:0}#content{margin-left:250px}#content #fold-button{display:none}#content .navbar-main,#content .navbar-main-additional{left:250px}.table td.td-long{width:30%}}.legend-box{font-size:10px;width:2em}#total-mem{background-color:#7cb5ec}#heap-mem{background-color:#434348}#non-heap-mem{background-color:#90ed7d}#fetch-plan,#job-submit{width:100px}#content-inner,#details,#node-details{height:100%}#job-panel{overflow-y:auto}
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/web/js/hs/index.js b/flink-runtime-web/web-dashboard/web/js/hs/index.js
index 148d9dd01f798..766120610c899 100644
--- a/flink-runtime-web/web-dashboard/web/js/hs/index.js
+++ b/flink-runtime-web/web-dashboard/web/js/hs/index.js
@@ -1,2 +1,2 @@
-angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,o){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}),t.otherwise("/completed-jobs")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,o,i,s,a;return"undefined"==typeof e||null===e?"":(i=e%1e3,a=Math.floor(e/1e3),s=a%60,a=Math.floor(a/60),o=a%60,a=Math.floor(a/60),n=a%24,a=Math.floor(a/24),r=a,0===r?0===n?0===o?0===s?i+"ms":s+"s ":o+"m "+s+"s":t?n+"h "+o+"m":n+"h "+o+"m "+s+"s":t?r+"d "+n+"h":r+"d "+n+"h "+o+"m "+s+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var o;return o=Math.pow(1024,n),e<o?(e/o).toFixed(2)+" "+r[n]:e<1e3*o?(e/o).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,o){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,o){return o=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var o;return o=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,i){return n=e,o.resolve(e)}),o.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var o;return o=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,i){return n=e,o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,o){var i;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),i=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},o["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(i)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var o;return o=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,i){return n=e,o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,o,i,s,a,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=a(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},s["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,a.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,o.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,i,s;return i=function(t){return function(t){var r,n,i,s;return r=l.defer(),i=e.job.jid,s=function(){var e,r,o;for(o=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)o.push(n+".currentLowWatermark");return o}(),o.getMetrics(i,t.id,s).then(function(e){var t,n,o,i,s,a,l;o=NaN,l={},i=e.values;for(t in i)a=i[t],s=t.replace(".currentLowWatermark",""),l[s]=a,(isNaN(o)||a<o)&&(o=a);return n=!isNaN(o)&&o>u.noWatermark?o:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),s={},n=t.length,angular.forEach(t,function(e){return function(e,t){var o;return o=e.id,i(e).then(function(e){if(s[o]=e,t>=n-1)return r.resolve(s)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,o){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=o.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var o;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),o=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},o(),e.$on("reload",function(e){return o()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var o,i;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,o=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},i=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},o(r.checkpointId),e.nodeid&&i(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(o(r.checkpointId),e.nodeid)return i(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var o;return o=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},o(),e.$on("reload",function(e){return o()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),n=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t,e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},e.dropped=function(t,o,i,s,a){return r.orderMetrics(e.jobid,e.nodeid,i,o),e.$broadcast("metrics:refresh",i),n(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),n()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),n()},e.setMetricSize=function(t,o){return r.setMetricSize(e.jobid,e.nodeid,t,o),n()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return n()}),e.nodeid)return n()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,o,i;i=t.children()[0],o=t.width(),angular.element(i).attr("width",o),(n=function(e){var t,r,n;return d3.select(i).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(i).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var o,i,s,a;s=r.children()[0],i=r.width(),angular.element(s).attr("width",i),a=function(e){return e.replace("&gt;",">")},o=function(r){var n,o,i;return d3.select(s).selectAll("*").remove(),i=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?i.push({times:[{label:a(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):i.push({times:[{label:a(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,o){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),o=d3.select(s).datum(i).call(n)},t.$watch(n.vertices,function(e){if(e)return o(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,o,i,s,a,l,u,c,d,f,p,g,m,h,b,v,k,j,S,w,C,$,J,M,y;p=null,C=d3.behavior.zoom(),y=[],h=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},i=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},m=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},g=function(e,t){return"mirror"===t?"node-mirror":m(t)?"node-iteration":"node-normal"},s=function(e,t,r,n){var o,i;return o="<div href='#/jobs/"+h+"/vertex/"+e.id+"' class='node-label "+g(e,t)+"'>",o+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?o+="":(i=e.description,i=M(i),o+="<h4 class='step-name'>"+i+"</h4>"),null!=e.step_function?o+=f(e.id,r,n):(m(t)&&(o+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(o+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(o+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(o+="<h5>Operation: "+M(e.operator_strategy)+"</h5>")),o+="</div>"},f=function(e,t,r){var n,o;return o="svg-"+e,n="<svg class='"+o+"' width="+t+" height="+r+"><g /></svg>"},M=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},a=function(e,t,r,n,o,i){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:s(r,"partialSolution",o,i),labelType:"html","class":g(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:s(r,"nextPartialSolution",o,i),labelType:"html","class":g(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:s(r,"workset",o,i),labelType:"html","class":g(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:s(r,"nextWorkset",o,i),labelType:"html","class":g(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:s(r,"solutionSet",o,i),labelType:"html","class":g(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:s(r,"solutionDelta",o,i),labelType:"html","class":g(r,"solutionDelta")}):e.setNode(r.id,{label:s(r,"",o,i),labelType:"html","class":g(r,"")})},o=function(e,t,r,n,o){return e.setEdge(o.id,r.id,{label:i(o),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,i,s,l,u,d,f,p,g,m,h,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,i=!0),s=0,u=v.length;s<u;s++)if(r=v[s],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),y[r.id]=b,k(b,r),m=new dagreD3.render,c.select("g").call(m,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),a(e,t,r,i,p,f),n.push(r.id),null!=r.inputs)for(h=r.inputs,l=0,d=h.length;l<d;l++)g=h[l],o(e,t,r,n,g);return e},J=function(e,t){var r,n,o;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(o in r.step_function)if(r.step_function[o].id===t)return r.step_function[o]}},$=function(e,t){var r,n,o,i;if(!_.isEmpty(t))for(i=e.nodes,r=0,n=i.length;r<n;r++)o=i[r],t[o.id]&&!isNaN(t[o.id].lowWatermark)&&(o.lowWatermark=t[o.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,o,i,s;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in y)o=y[t],l.select("svg.svg-"+t+" g").call(n,o);return r=.5,i=Math.floor((angular.element(S).width()-p.graph().width*r)/2),s=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([i,s]),u.attr("transform","translate("+i+", "+s+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,o,i){var s,a,l,u,c,d;return s=null,a=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=o.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,o,i){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return s=null,l.job=o.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(o,i,a,u){return n.setEndTimes(o.vertices),n.processVertices(o),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return o=angular.extend(o,e),s=o,l.job.resolve(s)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,o,i,s;for(n=0,o=t.length;n<o;n++){if(i=t[n],i.id===e)return i;if(i.step_function&&(s=r(e,i.step_function)),s)return s}return null},t=o.defer(),l.job.promise.then(function(n){return function(o){var i;return i=r(e,s.plan.nodes),i.vertex=n.seekVertex(e),t.resolve(i)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,o;for(n=s.vertices,t=0,r=n.length;t<r;t++)if(o=n[t],o.id===e)return o;return null},this.getVertex=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(i){var a;return a=o.seekVertex(r),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return a.subtasks=e.subtasks,n.resolve(a)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return console.log(s.jid),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/accumulators").success(function(o){var i;return i=o["user-accumulators"],e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:i,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=o.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=o.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints").success(function(e,t,n,o){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var i;return i=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?i.resolve(null):i.resolve(e)})}}(this)),i.promise},this.getOperatorBackPressure=function(r){var n;return n=o.defer(),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=o.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/exceptions").success(function(e){return s.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg /> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,o;for(r=!1,n=0,o=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+o)?r=!0:n+=o;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),
-e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),e.showChart(),e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var o;if(o=[],angular.forEach(t,function(e,t){return o.push(e.id)}),o.length>0)return e.getMetrics(r,n,o).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n})),o!==-1&&e.metrics[t][r].splice(o,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,o){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n.id),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n.id})),i!==-1&&(e.metrics[t][r][i]={id:n.id,size:o}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(o){return function(i,s){if(i.id===r.id&&(o.metrics[e][t].splice(s,1),s<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(o,i){var s;return n.setupLSFor(o,i),s=t.defer(),e.get(r.jobServer+"jobs/"+o+"/vertices/"+i+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var s;if(s=n.metrics[o][i].indexOf(e.id),s===-1&&(s=_.findIndex(n.metrics[o][i],{id:e.id})),s===-1)return t.push(e)}),s.resolve(t)}),s.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,o){var i;return i=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+o+"/metrics").success(function(e){return i.resolve(e)}),i.promise}}(this),this.getMetrics=function(n,o,i){var s,a;return s=t.defer(),a=i.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+o+"/metrics?get="+a).success(function(e){return function(t){var r,i;return i={},angular.forEach(t,function(e,t){return i[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:i},e.saveValue(n,o,r),s.resolve(r)}}(this)),s.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,o,i){var s;return e.yarn=i.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),s=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(s)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return o.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,o){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var o;return o=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.getPlan=function(n,o){var i;return i=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:o}).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.runJob=function(n,o){var i;return i=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:o}).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var o;return t.loadManagers().then(function(t){return e.managers=t}),o=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(o)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,o){var i;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),i=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},o["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(i)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,o){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,o){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,o){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var o;return o=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return o.resolve(e.taskmanagers)}),o.promise},this.loadLogs=function(n){var o;return o=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.loadStdout=function(n){var o;return o=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]);
\ No newline at end of file
+angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}),t.otherwise("/completed-jobs")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,s,a;return"undefined"==typeof e||null===e?"":(o=e%1e3,a=Math.floor(e/1e3),s=a%60,a=Math.floor(a/60),i=a%60,a=Math.floor(a/60),n=a%24,a=Math.floor(a/24),r=a,0===r?0===n?0===i?0===s?o+"ms":s+"s ":i+"m "+s+"s":t?n+"h "+i+"m":n+"h "+i+"m "+s+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+s+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,s,a,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=a(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},s["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,a.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,s;return o=function(t){return function(t){var r,n,o,s;return r=l.defer(),o=e.job.jid,s=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,s).then(function(e){var t,n,i,o,s,a,l;i=NaN,l={},o=e.values;for(t in o)a=o[t],s=t.replace(".currentLowWatermark",""),l[s]=a,(isNaN(i)||a<i)&&(i=a);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),s={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(s[i]=e,t>=n-1)return r.resolve(s)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),n=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t,e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},e.dropped=function(t,i,o,s,a){return r.orderMetrics(e.jobid,e.nodeid,o,i),e.$broadcast("metrics:refresh",o),n(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),n()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),n()},e.setMetricSize=function(t,i){return r.setMetricSize(e.jobid,e.nodeid,t,i),n()},e.setMetricView=function(t,i){return r.setMetricView(e.jobid,e.nodeid,t,i),n()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return n()}),e.nodeid)return n()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,s,a;s=r.children()[0],o=r.width(),angular.element(s).attr("width",o),a=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(s).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:a(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:a(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(s).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,s,a,l,u,c,d,f,p,m,h,g,b,v,k,j,S,w,C,$,y,J,M;p=null,C=d3.behavior.zoom(),M=[],g=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},h=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":h(t)?"node-iteration":"node-normal"},s=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+g+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=J(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(h(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+J(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},J=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},a=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:s(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:s(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:s(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:s(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:s(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:s(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:s(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,s,l,u,d,f,p,m,h,g,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),s=0,u=v.length;s<u;s++)if(r=v[s],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),M[r.id]=b,k(b,r),h=new dagreD3.render,c.select("g").call(h,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),a(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(g=r.inputs,l=0,d=g.length;l<d;l++)m=g[l],i(e,t,r,n,m);return e},y=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,s;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in M)i=M[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),s=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,s]),u.attr("transform","translate("+o+", "+s+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var s,a,l,u,c,d;return s=null,a=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,i,o){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return s=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,a,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),s=i,l.job.resolve(s)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,s;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(s=r(e,o.step_function)),s)return s}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,s.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=s.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var a;return a=i.seekVertex(r),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return a.subtasks=e.subtasks,n.resolve(a)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(s.jid),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/exceptions").success(function(e){return s.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;
+return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,s){if(o.id===r.id&&(i.metrics[e][t].splice(s,1),s<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var s;return n.setupLSFor(i,o),s=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var s;if(s=n.metrics[i][o].indexOf(e.id),s===-1&&(s=_.findIndex(n.metrics[i][o],{id:e.id})),s===-1)return t.push(e)}),s.resolve(t)}),s.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var s,a;return s=t.defer(),a=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+a).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),s.resolve(r)}}(this)),s.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var s;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),s=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(s)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e.taskmanagers)}),i.promise},this.loadLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/web/js/index.js b/flink-runtime-web/web-dashboard/web/js/index.js
index 08eb129de2225..9dfb735ac6e87 100644
--- a/flink-runtime-web/web-dashboard/web/js/index.js
+++ b/flink-runtime-web/web-dashboard/web/js/index.js
@@ -1,2 +1,2 @@
-angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,o){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("overview",{url:"/overview",views:{main:{templateUrl:"partials/overview.html",controller:"OverviewController"}}}).state("running-jobs",{url:"/running-jobs",views:{main:{templateUrl:"partials/jobs/running-jobs.html",controller:"RunningJobsController"}}}).state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}).state("all-manager",{url:"/taskmanagers",views:{main:{templateUrl:"partials/taskmanager/index.html",controller:"AllTaskManagersController"}}}).state("single-manager",{url:"/taskmanager/{taskmanagerid}","abstract":!0,views:{main:{templateUrl:"partials/taskmanager/taskmanager.html",controller:"SingleTaskManagerController"}}}).state("single-manager.metrics",{url:"/metrics",views:{details:{templateUrl:"partials/taskmanager/taskmanager.metrics.html"}}}).state("single-manager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/taskmanager/taskmanager.stdout.html",controller:"SingleTaskManagerStdoutController"}}}).state("single-manager.log",{url:"/log",views:{details:{templateUrl:"partials/taskmanager/taskmanager.log.html",controller:"SingleTaskManagerLogsController"}}}).state("jobmanager",{url:"/jobmanager",views:{main:{templateUrl:"partials/jobmanager/index.html"}}}).state("jobmanager.config",{url:"/config",views:{details:{templateUrl:"partials/jobmanager/config.html",controller:"JobManagerConfigController"}}}).state("jobmanager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/jobmanager/stdout.html",controller:"JobManagerStdoutController"}}}).state("jobmanager.log",{url:"/log",views:{details:{templateUrl:"partials/jobmanager/log.html",controller:"JobManagerLogsController"}}}).state("submit",{url:"/submit",views:{main:{templateUrl:"partials/submit.html",controller:"JobSubmitController"}}}),t.otherwise("/overview")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,o,a,i,s;return"undefined"==typeof e||null===e?"":(a=e%1e3,s=Math.floor(e/1e3),i=s%60,s=Math.floor(s/60),o=s%60,s=Math.floor(s/60),n=s%24,s=Math.floor(s/24),r=s,0===r?0===n?0===o?0===i?a+"ms":i+"s ":o+"m "+i+"s":t?n+"h "+o+"m":n+"h "+o+"m "+i+"s":t?r+"d "+n+"h":r+"d "+n+"h "+o+"m "+i+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var o;return o=Math.pow(1024,n),e<o?(e/o).toFixed(2)+" "+r[n]:e<1e3*o?(e/o).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,o){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,o){return o=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var o;return o=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,a){return n=e,o.resolve(e)}),o.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var o;return o=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,a){return n=e,o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,o){var a;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),a=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},o["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(a)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var o;return o=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,a){return n=e,o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,o,a,i,s,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=s(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},i["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,s.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,o.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,a,i;return a=function(t){return function(t){var r,n,a,i;return r=l.defer(),a=e.job.jid,i=function(){var e,r,o;for(o=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)o.push(n+".currentLowWatermark");return o}(),o.getMetrics(a,t.id,i).then(function(e){var t,n,o,a,i,s,l;o=NaN,l={},a=e.values;for(t in a)s=a[t],i=t.replace(".currentLowWatermark",""),l[i]=s,(isNaN(o)||s<o)&&(o=s);return n=!isNaN(o)&&o>u.noWatermark?o:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),i={},n=t.length,angular.forEach(t,function(e){return function(e,t){var o;return o=e.id,a(e).then(function(e){if(i[o]=e,t>=n-1)return r.resolve(i)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,o){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=o.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var o;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),o=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},o(),e.$on("reload",function(e){return o()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var o,a;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,o=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},a=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},o(r.checkpointId),e.nodeid&&a(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(o(r.checkpointId),e.nodeid)return a(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var o;return o=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},o(),e.$on("reload",function(e){return o()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),n=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t,e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},e.dropped=function(t,o,a,i,s){return r.orderMetrics(e.jobid,e.nodeid,a,o),e.$broadcast("metrics:refresh",a),n(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),n()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),n()},e.setMetricSize=function(t,o){return r.setMetricSize(e.jobid,e.nodeid,t,o),n()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return n()}),e.nodeid)return n()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,o,a;a=t.children()[0],o=t.width(),angular.element(a).attr("width",o),(n=function(e){var t,r,n;return d3.select(a).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(a).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var o,a,i,s;i=r.children()[0],a=r.width(),angular.element(i).attr("width",a),s=function(e){return e.replace("&gt;",">")},o=function(r){var n,o,a;return d3.select(i).selectAll("*").remove(),a=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?a.push({times:[{label:s(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):a.push({times:[{label:s(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,o){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),o=d3.select(i).datum(a).call(n)},t.$watch(n.vertices,function(e){if(e)return o(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,o,a,i,s,l,u,c,d,f,p,m,g,b,h,v,k,j,S,w,C,$,J,M,y;p=null,C=d3.behavior.zoom(),y=[],b=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,h=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),h=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),h=C.translate()},a=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},g=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":g(t)?"node-iteration":"node-normal"},i=function(e,t,r,n){var o,a;return o="<div href='#/jobs/"+b+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",o+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?o+="":(a=e.description,a=M(a),o+="<h4 class='step-name'>"+a+"</h4>"),null!=e.step_function?o+=f(e.id,r,n):(g(t)&&(o+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(o+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(o+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(o+="<h5>Operation: "+M(e.operator_strategy)+"</h5>")),o+="</div>"},f=function(e,t,r){var n,o;return o="svg-"+e,n="<svg class='"+o+"' width="+t+" height="+r+"><g /></svg>"},M=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},s=function(e,t,r,n,o,a){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:i(r,"partialSolution",o,a),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:i(r,"nextPartialSolution",o,a),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:i(r,"workset",o,a),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:i(r,"nextWorkset",o,a),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:i(r,"solutionSet",o,a),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:i(r,"solutionDelta",o,a),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:i(r,"",o,a),labelType:"html","class":m(r,"")})},o=function(e,t,r,n,o){return e.setEdge(o.id,r.id,{label:a(o),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,a,i,l,u,d,f,p,m,g,b,h,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,a=!0),i=0,u=v.length;i<u;i++)if(r=v[i],p=0,f=0,r.step_function&&(h=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),y[r.id]=h,k(h,r),g=new dagreD3.render,c.select("g").call(g,h),p=h.graph().width,f=h.graph().height,angular.element(w).empty()),s(e,t,r,a,p,f),n.push(r.id),null!=r.inputs)for(b=r.inputs,l=0,d=b.length;l<d;l++)m=b[l],o(e,t,r,n,m);return e},J=function(e,t){var r,n,o;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(o in r.step_function)if(r.step_function[o].id===t)return r.step_function[o]}},$=function(e,t){var r,n,o,a;if(!_.isEmpty(t))for(a=e.nodes,r=0,n=a.length;r<n;r++)o=a[r],t[o.id]&&!isNaN(t[o.id].lowWatermark)&&(o.lowWatermark=t[o.id].lowWatermark);return e},h=0,v=0,d=function(){var t,r,n,o,a,i;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in y)o=y[t],l.select("svg.svg-"+t+" g").call(n,o);return r=.5,a=Math.floor((angular.element(S).width()-p.graph().width*r)/2),i=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==h?(C.scale(v).translate(h),u.attr("transform","translate("+h+") scale("+v+")")):(C.scale(r).translate([a,i]),u.attr("transform","translate("+a+", "+i+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,h=e.translate,u.attr("transform","translate("+h+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,o,a){var i,s,l,u,c,d;return i=null,s=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=o.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,o,a){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return i=null,l.job=o.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(o,a,s,u){return n.setEndTimes(o.vertices),n.processVertices(o),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return o=angular.extend(o,e),i=o,l.job.resolve(i)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,o,a,i;for(n=0,o=t.length;n<o;n++){if(a=t[n],a.id===e)return a;if(a.step_function&&(i=r(e,a.step_function)),i)return i}return null},t=o.defer(),l.job.promise.then(function(n){return function(o){var a;return a=r(e,i.plan.nodes),a.vertex=n.seekVertex(e),t.resolve(a)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,o;for(n=i.vertices,t=0,r=n.length;t<r;t++)if(o=n[t],o.id===e)return o;return null},this.getVertex=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(a){var s;return s=o.seekVertex(r),e.get(t.jobServer+"jobs/"+i.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return s.subtasks=e.subtasks,n.resolve(s)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+i.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+i.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return console.log(i.jid),e.get(t.jobServer+"jobs/"+i.jid+"/vertices/"+r+"/accumulators").success(function(o){var a;return a=o["user-accumulators"],e.get(t.jobServer+"jobs/"+i.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:a,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=o.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+i.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=o.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+i.jid+"/checkpoints").success(function(e,t,n,o){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+i.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var a;return a=o.defer(),l.job.promise.then(function(o){return function(o){return e.get(t.jobServer+"jobs/"+i.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?a.resolve(null):a.resolve(e)})}}(this)),a.promise},this.getOperatorBackPressure=function(r){var n;return n=o.defer(),e.get(t.jobServer+"jobs/"+i.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=o.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+i.jid+"/exceptions").success(function(e){
-return i.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg /> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,o;for(r=!1,n=0,o=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+o)?r=!0:n+=o;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),e.showChart(),e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var o;if(o=[],angular.forEach(t,function(e,t){return o.push(e.id)}),o.length>0)return e.getMetrics(r,n,o).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n})),o!==-1&&e.metrics[t][r].splice(o,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,o){var a;if(null!=e.metrics[t][r])return a=e.metrics[t][r].indexOf(n.id),a===-1&&(a=_.findIndex(e.metrics[t][r],{id:n.id})),a!==-1&&(e.metrics[t][r][a]={id:n.id,size:o}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(o){return function(a,i){if(a.id===r.id&&(o.metrics[e][t].splice(i,1),i<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(o,a){var i;return n.setupLSFor(o,a),i=t.defer(),e.get(r.jobServer+"jobs/"+o+"/vertices/"+a+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var i;if(i=n.metrics[o][a].indexOf(e.id),i===-1&&(i=_.findIndex(n.metrics[o][a],{id:e.id})),i===-1)return t.push(e)}),i.resolve(t)}),i.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,o){var a;return a=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+o+"/metrics").success(function(e){return a.resolve(e)}),a.promise}}(this),this.getMetrics=function(n,o,a){var i,s;return i=t.defer(),s=a.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+o+"/metrics?get="+s).success(function(e){return function(t){var r,a;return a={},angular.forEach(t,function(e,t){return a[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:a},e.saveValue(n,o,r),i.resolve(r)}}(this)),i.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,o,a){var i;return e.yarn=a.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),i=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return o.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,o){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var o;return o=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.getPlan=function(n,o){var a;return a=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:o}).success(function(e,t,r,n){return a.resolve(e)}),a.promise},this.runJob=function(n,o){var a;return a=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:o}).success(function(e,t,r,n){return a.resolve(e)}),a.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var o;return t.loadManagers().then(function(t){return e.managers=t}),o=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(o)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,o){var a;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),a=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},o["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(a)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,o){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,o){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,o){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var o;return o=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return o.resolve(e.taskmanagers)}),o.promise},this.loadLogs=function(n){var o;return o=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.loadStdout=function(n){var o;return o=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]);
\ No newline at end of file
+angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("overview",{url:"/overview",views:{main:{templateUrl:"partials/overview.html",controller:"OverviewController"}}}).state("running-jobs",{url:"/running-jobs",views:{main:{templateUrl:"partials/jobs/running-jobs.html",controller:"RunningJobsController"}}}).state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}).state("all-manager",{url:"/taskmanagers",views:{main:{templateUrl:"partials/taskmanager/index.html",controller:"AllTaskManagersController"}}}).state("single-manager",{url:"/taskmanager/{taskmanagerid}","abstract":!0,views:{main:{templateUrl:"partials/taskmanager/taskmanager.html",controller:"SingleTaskManagerController"}}}).state("single-manager.metrics",{url:"/metrics",views:{details:{templateUrl:"partials/taskmanager/taskmanager.metrics.html"}}}).state("single-manager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/taskmanager/taskmanager.stdout.html",controller:"SingleTaskManagerStdoutController"}}}).state("single-manager.log",{url:"/log",views:{details:{templateUrl:"partials/taskmanager/taskmanager.log.html",controller:"SingleTaskManagerLogsController"}}}).state("jobmanager",{url:"/jobmanager",views:{main:{templateUrl:"partials/jobmanager/index.html"}}}).state("jobmanager.config",{url:"/config",views:{details:{templateUrl:"partials/jobmanager/config.html",controller:"JobManagerConfigController"}}}).state("jobmanager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/jobmanager/stdout.html",controller:"JobManagerStdoutController"}}}).state("jobmanager.log",{url:"/log",views:{details:{templateUrl:"partials/jobmanager/log.html",controller:"JobManagerLogsController"}}}).state("submit",{url:"/submit",views:{main:{templateUrl:"partials/submit.html",controller:"JobSubmitController"}}}),t.otherwise("/overview")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,a,s;return"undefined"==typeof e||null===e?"":(o=e%1e3,s=Math.floor(e/1e3),a=s%60,s=Math.floor(s/60),i=s%60,s=Math.floor(s/60),n=s%24,s=Math.floor(s/24),r=s,0===r?0===n?0===i?0===a?o+"ms":a+"s ":i+"m "+a+"s":t?n+"h "+i+"m":n+"h "+i+"m "+a+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+a+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,a,s,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=s(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},a["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,s.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,a;return o=function(t){return function(t){var r,n,o,a;return r=l.defer(),o=e.job.jid,a=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,a).then(function(e){var t,n,i,o,a,s,l;i=NaN,l={},o=e.values;for(t in o)s=o[t],a=t.replace(".currentLowWatermark",""),l[a]=s,(isNaN(i)||s<i)&&(i=s);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),a={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(a[i]=e,t>=n-1)return r.resolve(a)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),n=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t,e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},e.dropped=function(t,i,o,a,s){return r.orderMetrics(e.jobid,e.nodeid,o,i),e.$broadcast("metrics:refresh",o),n(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),n()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),n()},e.setMetricSize=function(t,i){return r.setMetricSize(e.jobid,e.nodeid,t,i),n()},e.setMetricView=function(t,i){return r.setMetricView(e.jobid,e.nodeid,t,i),n()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return n()}),e.nodeid)return n()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,a,s;a=r.children()[0],o=r.width(),angular.element(a).attr("width",o),s=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(a).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:s(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:s(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(a).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,a,s,l,u,c,d,f,p,m,g,h,b,v,k,j,S,w,C,$,J,M,y;p=null,C=d3.behavior.zoom(),y=[],h=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},g=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":g(t)?"node-iteration":"node-normal"},a=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+h+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=M(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(g(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+M(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},M=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},s=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:a(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:a(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:a(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:a(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:a(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:a(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:a(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,a,l,u,d,f,p,m,g,h,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),a=0,u=v.length;a<u;a++)if(r=v[a],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),y[r.id]=b,k(b,r),g=new dagreD3.render,c.select("g").call(g,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),s(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(h=r.inputs,l=0,d=h.length;l<d;l++)m=h[l],i(e,t,r,n,m);return e},J=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,a;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in y)i=y[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),a=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,a]),u.attr("transform","translate("+o+", "+a+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var a,s,l,u,c,d;return a=null,s=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,i,o){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return a=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,s,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),a=i,l.job.resolve(a)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,a;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(a=r(e,o.step_function)),a)return a}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,a.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=a.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var s;return s=i.seekVertex(r),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return s.subtasks=e.subtasks,n.resolve(s)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(a.jid),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/exceptions").success(function(e){return a.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){
+return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,a){if(o.id===r.id&&(i.metrics[e][t].splice(a,1),a<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var a;return n.setupLSFor(i,o),a=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var a;if(a=n.metrics[i][o].indexOf(e.id),a===-1&&(a=_.findIndex(n.metrics[i][o],{id:e.id})),a===-1)return t.push(e)}),a.resolve(t)}),a.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var a,s;return a=t.defer(),s=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+s).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),a.resolve(r)}}(this)),a.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var a;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),a=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(a)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e.taskmanagers)}),i.promise},this.loadLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/web/js/vendor.js b/flink-runtime-web/web-dashboard/web/js/vendor.js
index ee2e3e3f37e93..d5a877b691606 100644
--- a/flink-runtime-web/web-dashboard/web/js/vendor.js
+++ b/flink-runtime-web/web-dashboard/web/js/vendor.js
@@ -22,9 +22,9 @@ n.left=void 0!==t.left?t.left:n.left}},color:{get:function(){return s},set:funct
 g.scale(o)._ticks(t.utils.calcTicksY(Z/36,w)).tickSize(-X,0),v.scale(a)._ticks(t.utils.calcTicksY(Z/36,w)),R?v.tickSize(tt.length?0:-X,0):v.tickSize(J.length?0:-X,0);var u=J.length?1:0,c=tt.length&&!Y(tt)?1:0,d=R?c:u,p=R?u:c;ot.select(".nv-focus .nv-y1.nv-axis").style("opacity",d),ot.select(".nv-focus .nv-y2.nv-axis").style("opacity",p).attr("transform","translate("+r.range()[1]+",0)"),ot.select(".nv-focus .nv-y1.nv-axis").transition().duration(P).call(g),ot.select(".nv-focus .nv-y2.nv-axis").transition().duration(P).call(v)}var G=d3.select(this);t.utils.initSVG(G);var X=t.utils.availableWidth(_,G,$),Z=t.utils.availableHeight(M,G,$)-(T?O:0),Q=O-k.top-k.bottom;if(e.update=function(){G.transition().duration(P).call(e)},e.container=this,F.setter(U(w),e.update).getter(H(w)).update(),F.disabled=w.map(function(t){return!!t.disabled}),!q){var K;q={};for(K in F)F[K]instanceof Array?q[K]=F[K].slice(0):q[K]=F[K]}if(!(w&&w.length&&w.filter(function(t){return t.values.length}).length))return t.utils.noData(e,G),e;G.selectAll(".nv-noData").remove();var J=w.filter(function(t){return!t.disabled&&t.bar}),tt=w.filter(function(t){return!t.bar});r=J.length&&!R?f.xScale():l.xScale(),i=p.scale(),o=R?l.yScale():f.yScale(),a=R?f.yScale():l.yScale(),s=R?c.yScale():d.yScale(),u=R?d.yScale():c.yScale();var et=w.filter(function(t){return!t.disabled&&(R?!t.bar:t.bar)}).map(function(t){return t.values.map(function(t,e){return{x:C(t,e),y:S(t,e)}})}),nt=w.filter(function(t){return!t.disabled&&(R?t.bar:!t.bar)}).map(function(t){return t.values.map(function(t,e){return{x:C(t,e),y:S(t,e)}})});r.range([0,X]),i.domain(d3.extent(d3.merge(et.concat(nt)),function(t){return t.x})).range([0,X]);var rt=G.selectAll("g.nv-wrap.nv-linePlusBar").data([w]),it=rt.enter().append("g").attr("class","nvd3 nv-wrap nv-linePlusBar").append("g"),ot=rt.select("g");it.append("g").attr("class","nv-legendWrap");var at=it.append("g").attr("class","nv-focus");at.append("g").attr("class","nv-x nv-axis"),at.append("g").attr("class","nv-y1 nv-axis"),at.append("g").attr("class","nv-y2 nv-axis"),at.append("g").attr("class","nv-barsWrap"),at.append("g").attr("class","nv-linesWrap");var st=it.append("g").attr("class","nv-context");if(st.append("g").attr("class","nv-x nv-axis"),st.append("g").attr("class","nv-y1 nv-axis"),st.append("g").attr("class","nv-y2 nv-axis"),st.append("g").attr("class","nv-barsWrap"),st.append("g").attr("class","nv-linesWrap"),st.append("g").attr("class","nv-brushBackground"),st.append("g").attr("class","nv-x nv-brush"),A){var ut=b.align()?X/2:X,lt=b.align()?ut:0;b.width(ut),ot.select(".nv-legendWrap").datum(w.map(function(t){return t.originalKey=void 0===t.originalKey?t.key:t.originalKey,R?t.key=t.originalKey+(t.bar?W:z):t.key=t.originalKey+(t.bar?z:W),t})).call(b),b.height()>$.top&&($.top=b.height(),Z=t.utils.availableHeight(M,G,$)-O),ot.select(".nv-legendWrap").attr("transform","translate("+lt+","+-$.top+")")}else ot.select(".nv-legendWrap").selectAll("*").remove();rt.attr("transform","translate("+$.left+","+$.top+")"),ot.select(".nv-context").style("display",T?"initial":"none"),d.width(X).height(Q).color(w.map(function(t,e){return t.color||E(t,e)}).filter(function(t,e){return!w[e].disabled&&w[e].bar})),c.width(X).height(Q).color(w.map(function(t,e){return t.color||E(t,e)}).filter(function(t,e){return!w[e].disabled&&!w[e].bar}));var ct=ot.select(".nv-context .nv-barsWrap").datum(J.length?J:[{values:[]}]),ft=ot.select(".nv-context .nv-linesWrap").datum(Y(tt)?[{values:[]}]:tt.filter(function(t){return!t.disabled}));ot.select(".nv-context").attr("transform","translate(0,"+(Z+$.bottom+k.top)+")"),ct.transition().call(d),ft.transition().call(c),D&&(p._ticks(t.utils.calcTicksX(X/100,w)).tickSize(-Q,0),ot.select(".nv-context .nv-x.nv-axis").attr("transform","translate(0,"+s.range()[0]+")"),ot.select(".nv-context .nv-x.nv-axis").transition().call(p)),N&&(m.scale(s)._ticks(Q/36).tickSize(-X,0),y.scale(u)._ticks(Q/36).tickSize(J.length?0:-X,0),ot.select(".nv-context .nv-y3.nv-axis").style("opacity",J.length?1:0).attr("transform","translate(0,"+i.range()[0]+")"),ot.select(".nv-context .nv-y2.nv-axis").style("opacity",tt.length?1:0).attr("transform","translate("+i.range()[1]+",0)"),ot.select(".nv-context .nv-y1.nv-axis").transition().call(m),ot.select(".nv-context .nv-y2.nv-axis").transition().call(y)),x.x(i).on("brush",V),j&&x.extent(j);var dt=ot.select(".nv-brushBackground").selectAll("g").data([j||x.extent()]),ht=dt.enter().append("g");ht.append("rect").attr("class","left").attr("x",0).attr("y",0).attr("height",Q),ht.append("rect").attr("class","right").attr("x",0).attr("y",0).attr("height",Q);var pt=ot.select(".nv-x.nv-brush").call(x);pt.selectAll("rect").attr("height",Q),pt.selectAll(".resize").append("path").attr("d",I),b.dispatch.on("stateChange",function(t){for(var n in t)F[n]=t[n];L.stateChange(F),e.update()}),L.on("changeState",function(t){"undefined"!=typeof t.disabled&&(w.forEach(function(e,n){e.disabled=t.disabled[n]}),F.disabled=t.disabled),e.update()}),V()}),e}var n,r,i,o,a,s,u,l=t.models.line(),c=t.models.line(),f=t.models.historicalBar(),d=t.models.historicalBar(),h=t.models.axis(),p=t.models.axis(),g=t.models.axis(),v=t.models.axis(),m=t.models.axis(),y=t.models.axis(),b=t.models.legend(),x=d3.svg.brush(),w=t.models.tooltip(),$={top:30,right:30,bottom:30,left:60},k={top:0,right:30,bottom:20,left:60},_=null,M=null,C=function(t){return t.x},S=function(t){return t.y},E=t.utils.defaultColor(),A=!0,T=!0,N=!1,D=!0,O=50,j=null,I=null,L=d3.dispatch("brush","stateChange","changeState"),P=0,F=t.utils.state(),q=null,z=" (left axis)",W=" (right axis)",R=!1;l.clipEdge(!0),c.interactive(!1),c.pointActive(function(t){return!1}),h.orient("bottom").tickPadding(5),g.orient("left"),v.orient("right"),p.orient("bottom").tickPadding(5),m.orient("left"),y.orient("right"),w.headerEnabled(!0).headerFormatter(function(t,e){return h.tickFormat()(t,e)});var B=function(){return R?{main:v,focus:y}:{main:g,focus:m}},V=function(){return R?{main:g,focus:m}:{main:v,focus:y}},H=function(t){return function(){return{active:t.map(function(t){return!t.disabled})}}},U=function(t){return function(e){void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}},Y=function(t){return t.every(function(t){return t.disabled})};return l.dispatch.on("elementMouseover.tooltip",function(t){w.duration(100).valueFormatter(function(t,e){return V().main.tickFormat()(t,e)}).data(t).hidden(!1)}),l.dispatch.on("elementMouseout.tooltip",function(t){w.hidden(!0)}),f.dispatch.on("elementMouseover.tooltip",function(t){t.value=e.x()(t.data),t.series={value:e.y()(t.data),color:t.color},w.duration(0).valueFormatter(function(t,e){return B().main.tickFormat()(t,e)}).data(t).hidden(!1)}),f.dispatch.on("elementMouseout.tooltip",function(t){w.hidden(!0)}),f.dispatch.on("elementMousemove.tooltip",function(t){w()}),e.dispatch=L,e.legend=b,e.lines=l,e.lines2=c,e.bars=f,e.bars2=d,e.xAxis=h,e.x2Axis=p,e.y1Axis=g,e.y2Axis=v,e.y3Axis=m,e.y4Axis=y,e.tooltip=w,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return _},set:function(t){_=t}},height:{get:function(){return M},set:function(t){M=t}},showLegend:{get:function(){return A},set:function(t){A=t}},brushExtent:{get:function(){return j},set:function(t){j=t}},noData:{get:function(){return I},set:function(t){I=t}},focusEnable:{get:function(){return T},set:function(t){T=t}},focusHeight:{get:function(){return O},set:function(t){O=t}},focusShowAxisX:{get:function(){return D},set:function(t){D=t}},focusShowAxisY:{get:function(){return N},set:function(t){N=t}},legendLeftAxisHint:{get:function(){return z},set:function(t){z=t}},legendRightAxisHint:{get:function(){return W},set:function(t){W=t}},margin:{get:function(){return $},set:function(t){$.top=void 0!==t.top?t.top:$.top,$.right=void 0!==t.right?t.right:$.right,$.bottom=void 0!==t.bottom?t.bottom:$.bottom,$.left=void 0!==t.left?t.left:$.left}},focusMargin:{get:function(){return k},set:function(t){k.top=void 0!==t.top?t.top:k.top,k.right=void 0!==t.right?t.right:k.right,k.bottom=void 0!==t.bottom?t.bottom:k.bottom,k.left=void 0!==t.left?t.left:k.left}},duration:{get:function(){return P},set:function(t){P=t}},color:{get:function(){return E},set:function(e){E=t.utils.getColor(e),b.color(E)}},x:{get:function(){return C},set:function(t){C=t,l.x(t),c.x(t),f.x(t),d.x(t)}},y:{get:function(){return S},set:function(t){S=t,l.y(t),c.y(t),f.y(t),d.y(t)}},switchYAxisOrder:{get:function(){return R},set:function(t){if(R!==t){var e=g;g=v,v=e;var n=m;m=y,y=n}R=t,g.orient("left"),v.orient("right"),m.orient("left"),y.orient("right")}}}),t.utils.inheritOptions(e,l),t.utils.initOptions(e),e},t.models.multiBar=function(){"use strict";function e(N){return A.reset(),N.each(function(e){var N=c-l.left-l.right,D=f-l.top-l.bottom;g=d3.select(this),t.utils.initSVG(g);var O=0;if(k&&e.length&&(k=[{values:e[0].values.map(function(t){return{x:t.x,y:0,series:t.series,size:.01}})}]),x){var j=d3.layout.stack().offset(w).values(function(t){return t.values}).y(m)(!e.length&&k?k:e);j.forEach(function(t,n){t.nonStackable?(e[n].nonStackableSeries=O++,j[n]=e[n]):n>0&&j[n-1].nonStackable&&j[n].values.map(function(t,e){t.y0-=j[n-1].values[e].y,t.y1=t.y0+t.y})}),e=j}e.forEach(function(t,e){t.values.forEach(function(n){n.series=e,n.key=t.key})}),x&&e.length>0&&e[0].values.map(function(t,n){var r=0,i=0;e.map(function(t,o){if(!e[o].nonStackable){var a=t.values[n];a.size=Math.abs(a.y),a.y<0?(a.y1=i,i-=a.size):(a.y1=a.size+r,r+=a.size)}})});var I=r&&i?[]:e.map(function(t,e){return t.values.map(function(t,n){return{x:v(t,n),y:m(t,n),y0:t.y0,y1:t.y1,idx:e}})});d.domain(r||d3.merge(I).map(function(t){return t.x})).rangeBands(o||[0,N],C),h.domain(i||d3.extent(d3.merge(I).map(function(t){var n=t.y;return x&&!e[t.idx].nonStackable&&(n=t.y>0?t.y1:t.y1+t.y),n}).concat(y))).range(a||[D,0]),d.domain()[0]===d.domain()[1]&&(d.domain()[0]?d.domain([d.domain()[0]-.01*d.domain()[0],d.domain()[1]+.01*d.domain()[1]]):d.domain([-1,1])),h.domain()[0]===h.domain()[1]&&(h.domain()[0]?h.domain([h.domain()[0]+.01*h.domain()[0],h.domain()[1]-.01*h.domain()[1]]):h.domain([-1,1])),s=s||d,u=u||h;var L=g.selectAll("g.nv-wrap.nv-multibar").data([e]),P=L.enter().append("g").attr("class","nvd3 nv-wrap nv-multibar"),F=P.append("defs"),q=P.append("g"),z=L.select("g");q.append("g").attr("class","nv-groups"),L.attr("transform","translate("+l.left+","+l.top+")"),F.append("clipPath").attr("id","nv-edge-clip-"+p).append("rect"),L.select("#nv-edge-clip-"+p+" rect").attr("width",N).attr("height",D),z.attr("clip-path",b?"url(#nv-edge-clip-"+p+")":"");var W=L.select(".nv-groups").selectAll(".nv-group").data(function(t){return t},function(t,e){return e});W.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6);var R=A.transition(W.exit().selectAll("rect.nv-bar"),"multibarExit",Math.min(100,M)).attr("y",function(t,n,r){var i=u(0)||0;return x&&e[t.series]&&!e[t.series].nonStackable&&(i=u(t.y0)),i}).attr("height",0).remove();R.delay&&R.delay(function(t,e){var n=e*(M/(T+1))-e;return n}),W.attr("class",function(t,e){return"nv-group nv-series-"+e}).classed("hover",function(t){return t.hover}).style("fill",function(t,e){return $(t,e)}).style("stroke",function(t,e){return $(t,e)}),W.style("stroke-opacity",1).style("fill-opacity",S);var B=W.selectAll("rect.nv-bar").data(function(t){return k&&!e.length?k.values:t.values});B.exit().remove();B.enter().append("rect").attr("class",function(t,e){return m(t,e)<0?"nv-bar negative":"nv-bar positive"}).attr("x",function(t,n,r){return x&&!e[r].nonStackable?0:r*d.rangeBand()/e.length}).attr("y",function(t,n,r){return u(x&&!e[r].nonStackable?t.y0:0)||0}).attr("height",0).attr("width",function(t,n,r){return d.rangeBand()/(x&&!e[r].nonStackable?1:e.length)}).attr("transform",function(t,e){return"translate("+d(v(t,e))+",0)"});B.style("fill",function(t,e,n){return $(t,n,e)}).style("stroke",function(t,e,n){return $(t,n,e)}).on("mouseover",function(t,e){d3.select(this).classed("hover",!0),E.elementMouseover({data:t,index:e,color:d3.select(this).style("fill")})}).on("mouseout",function(t,e){d3.select(this).classed("hover",!1),E.elementMouseout({data:t,index:e,color:d3.select(this).style("fill")})}).on("mousemove",function(t,e){E.elementMousemove({data:t,index:e,color:d3.select(this).style("fill")})}).on("click",function(t,e){var n=this;E.elementClick({data:t,index:e,color:d3.select(this).style("fill"),event:d3.event,element:n}),d3.event.stopPropagation()}).on("dblclick",function(t,e){E.elementDblClick({data:t,index:e,color:d3.select(this).style("fill")}),d3.event.stopPropagation()}),B.attr("class",function(t,e){return m(t,e)<0?"nv-bar negative":"nv-bar positive"}).attr("transform",function(t,e){return"translate("+d(v(t,e))+",0)"}),_&&(n||(n=e.map(function(){return!0})),B.style("fill",function(t,e,r){return d3.rgb(_(t,e)).darker(n.map(function(t,e){return e}).filter(function(t,e){return!n[e]})[r]).toString()}).style("stroke",function(t,e,r){return d3.rgb(_(t,e)).darker(n.map(function(t,e){return e}).filter(function(t,e){return!n[e]})[r]).toString()}));var V=B.watchTransition(A,"multibar",Math.min(250,M)).delay(function(t,n){return n*M/e[0].values.length});x?V.attr("y",function(t,n,r){var i=0;return i=e[r].nonStackable?m(t,n)<0?h(0):h(0)-h(m(t,n))<-1?h(0)-1:h(m(t,n))||0:h(t.y1)}).attr("height",function(t,n,r){return e[r].nonStackable?Math.max(Math.abs(h(m(t,n))-h(0)),0)||0:Math.max(Math.abs(h(t.y+t.y0)-h(t.y0)),0)}).attr("x",function(t,n,r){var i=0;return e[r].nonStackable&&(i=t.series*d.rangeBand()/e.length,e.length!==O&&(i=e[r].nonStackableSeries*d.rangeBand()/(2*O))),i}).attr("width",function(t,n,r){if(e[r].nonStackable){var i=d.rangeBand()/O;return e.length!==O&&(i=d.rangeBand()/(2*O)),i}return d.rangeBand()}):V.attr("x",function(t,n){return t.series*d.rangeBand()/e.length}).attr("width",d.rangeBand()/e.length).attr("y",function(t,e){return m(t,e)<0?h(0):h(0)-h(m(t,e))<1?h(0)-1:h(m(t,e))||0}).attr("height",function(t,e){return Math.max(Math.abs(h(m(t,e))-h(0)),1)||0}),s=d.copy(),u=h.copy(),e[0]&&e[0].values&&(T=e[0].values.length)}),A.renderEnd("multibar immediate"),e}var n,r,i,o,a,s,u,l={top:0,right:0,bottom:0,left:0},c=960,f=500,d=d3.scale.ordinal(),h=d3.scale.linear(),p=Math.floor(1e4*Math.random()),g=null,v=function(t){return t.x},m=function(t){return t.y},y=[0],b=!0,x=!1,w="zero",$=t.utils.defaultColor(),k=!1,_=null,M=500,C=.1,S=.75,E=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),A=t.utils.renderWatch(E,M),T=0;return e.dispatch=E,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return c},set:function(t){c=t}},height:{get:function(){return f},set:function(t){f=t}},x:{get:function(){return v},set:function(t){v=t}},y:{get:function(){return m},set:function(t){m=t}},xScale:{get:function(){return d},set:function(t){d=t}},yScale:{get:function(){return h},set:function(t){h=t}},xDomain:{get:function(){return r},set:function(t){r=t}},yDomain:{get:function(){return i},set:function(t){i=t}},xRange:{get:function(){return o},set:function(t){o=t}},yRange:{get:function(){return a},set:function(t){a=t}},forceY:{get:function(){return y},set:function(t){y=t}},stacked:{get:function(){return x},set:function(t){x=t}},stackOffset:{get:function(){return w},set:function(t){w=t}},clipEdge:{get:function(){return b},set:function(t){b=t}},disabled:{get:function(){return n},set:function(t){n=t}},id:{get:function(){return p},set:function(t){p=t}},hideable:{get:function(){return k},set:function(t){k=t}},groupSpacing:{get:function(){return C},set:function(t){C=t}},fillOpacity:{get:function(){return S},set:function(t){S=t}},margin:{get:function(){return l},set:function(t){l.top=void 0!==t.top?t.top:l.top,l.right=void 0!==t.right?t.right:l.right,l.bottom=void 0!==t.bottom?t.bottom:l.bottom,l.left=void 0!==t.left?t.left:l.left}},duration:{get:function(){return M},set:function(t){M=t,A.reset(M)}},color:{get:function(){return $},set:function(e){$=t.utils.getColor(e)}},barColor:{get:function(){return _},set:function(e){_=e?t.utils.getColor(e):null}}}),t.utils.initOptions(e),e},t.models.multiBarChart=function(){"use strict";function e(S){return D.reset(),D.models(i),y&&D.models(o),b&&D.models(a),S.each(function(S){var D=d3.select(this);t.utils.initSVG(D);var L=t.utils.availableWidth(d,D,f),P=t.utils.availableHeight(h,D,f);if(e.update=function(){0===T?D.call(e):D.transition().duration(T).call(e)},e.container=this,M.setter(I(S),e.update).getter(j(S)).update(),M.disabled=S.map(function(t){return!!t.disabled}),!C){var F;C={};for(F in M)M[F]instanceof Array?C[F]=M[F].slice(0):C[F]=M[F]}if(!(S&&S.length&&S.filter(function(t){return t.values.length}).length))return t.utils.noData(e,D),e;D.selectAll(".nv-noData").remove(),n=i.xScale(),r=i.yScale();var q=D.selectAll("g.nv-wrap.nv-multiBarWithLegend").data([S]),z=q.enter().append("g").attr("class","nvd3 nv-wrap nv-multiBarWithLegend").append("g"),W=q.select("g");if(z.append("g").attr("class","nv-x nv-axis"),z.append("g").attr("class","nv-y nv-axis"),z.append("g").attr("class","nv-barsWrap"),z.append("g").attr("class","nv-legendWrap"),z.append("g").attr("class","nv-controlsWrap"),z.append("g").attr("class","nv-interactive"),m?(u.width(L-A()),W.select(".nv-legendWrap").datum(S).call(u),u.height()>f.top&&(f.top=u.height(),P=t.utils.availableHeight(h,D,f)),W.select(".nv-legendWrap").attr("transform","translate("+A()+","+-f.top+")")):W.select(".nv-legendWrap").selectAll("*").remove(),g){var R=[{key:v.grouped||"Grouped",disabled:i.stacked()},{key:v.stacked||"Stacked",disabled:!i.stacked()}];l.width(A()).color(["#444","#444","#444"]),W.select(".nv-controlsWrap").datum(R).attr("transform","translate(0,"+-f.top+")").call(l)}else W.select(".nv-controlsWrap").selectAll("*").remove();q.attr("transform","translate("+f.left+","+f.top+")"),x&&W.select(".nv-y.nv-axis").attr("transform","translate("+L+",0)"),i.disabled(S.map(function(t){return t.disabled})).width(L).height(P).color(S.map(function(t,e){return t.color||p(t,e)}).filter(function(t,e){return!S[e].disabled}));var B=W.select(".nv-barsWrap").datum(S.filter(function(t){return!t.disabled}));if(B.call(i),y){o.scale(n)._ticks(t.utils.calcTicksX(L/100,S)).tickSize(-P,0),W.select(".nv-x.nv-axis").attr("transform","translate(0,"+r.range()[0]+")"),W.select(".nv-x.nv-axis").call(o);var V=W.select(".nv-x.nv-axis > g").selectAll("g");if(V.selectAll("line, text").style("opacity",1),$){var H=function(t,e){return"translate("+t+","+e+")"},U=5,Y=17;V.selectAll("text").attr("transform",function(t,e,n){return H(0,n%2==0?U:Y)});var G=d3.selectAll(".nv-x.nv-axis .nv-wrap g g text")[0].length;W.selectAll(".nv-x.nv-axis .nv-axisMaxMin text").attr("transform",function(t,e){return H(0,0===e||G%2!==0?Y:U)})}k&&W.selectAll(".tick text").call(t.utils.wrapTicks,e.xAxis.rangeBand()),w&&V.filter(function(t,e){return e%Math.ceil(S[0].values.length/(L/100))!==0}).selectAll("text, line").style("opacity",0),_&&V.selectAll(".tick text").attr("transform","rotate("+_+" 0,0)").style("text-anchor",_>0?"start":"end"),W.select(".nv-x.nv-axis").selectAll("g.nv-axisMaxMin text").style("opacity",1)}b&&(a.scale(r)._ticks(t.utils.calcTicksY(P/36,S)).tickSize(-L,0),W.select(".nv-y.nv-axis").call(a)),N&&(s.width(L).height(P).margin({left:f.left,top:f.top}).svgContainer(D).xScale(n),q.select(".nv-interactive").call(s)),u.dispatch.on("stateChange",function(t){for(var n in t)M[n]=t[n];E.stateChange(M),e.update()}),l.dispatch.on("legendClick",function(t,n){if(t.disabled){switch(R=R.map(function(t){return t.disabled=!0,t}),t.disabled=!1,t.key){case"Grouped":case v.grouped:i.stacked(!1);break;case"Stacked":case v.stacked:i.stacked(!0)}M.stacked=i.stacked(),E.stateChange(M),e.update()}}),E.on("changeState",function(t){"undefined"!=typeof t.disabled&&(S.forEach(function(e,n){e.disabled=t.disabled[n]}),M.disabled=t.disabled),"undefined"!=typeof t.stacked&&(i.stacked(t.stacked),M.stacked=t.stacked,O=t.stacked),e.update()}),N?(s.dispatch.on("elementMousemove",function(t){if(void 0!=t.pointXValue){var r,i,o,a,u=[];S.filter(function(t,e){return t.seriesIndex=e,!t.disabled}).forEach(function(s,l){i=n.domain().indexOf(t.pointXValue);var c=s.values[i];void 0!==c&&(a=c.x,void 0===r&&(r=c),void 0===o&&(o=t.mouseX),u.push({key:s.key,value:e.y()(c,i),color:p(s,s.seriesIndex),data:s.values[i]}))}),s.tooltip.data({value:a,index:i,series:u})(),s.renderGuideLine(o)}}),s.dispatch.on("elementMouseout",function(t){s.tooltip.hidden(!0)})):(i.dispatch.on("elementMouseover.tooltip",function(t){t.value=e.x()(t.data),t.series={key:t.data.key,value:e.y()(t.data),color:t.color},c.data(t).hidden(!1)}),i.dispatch.on("elementMouseout.tooltip",function(t){c.hidden(!0)}),i.dispatch.on("elementMousemove.tooltip",function(t){c()}))}),D.renderEnd("multibarchart immediate"),e}var n,r,i=t.models.multiBar(),o=t.models.axis(),a=t.models.axis(),s=t.interactiveGuideline(),u=t.models.legend(),l=t.models.legend(),c=t.models.tooltip(),f={top:30,right:20,bottom:50,left:60},d=null,h=null,p=t.utils.defaultColor(),g=!0,v={},m=!0,y=!0,b=!0,x=!1,w=!0,$=!1,k=!1,_=0,M=t.utils.state(),C=null,S=null,E=d3.dispatch("stateChange","changeState","renderEnd"),A=function(){return g?180:0},T=250,N=!1;M.stacked=!1,i.stacked(!1),o.orient("bottom").tickPadding(7).showMaxMin(!1).tickFormat(function(t){return t}),a.orient(x?"right":"left").tickFormat(d3.format(",.1f")),c.duration(0).valueFormatter(function(t,e){return a.tickFormat()(t,e)}).headerFormatter(function(t,e){return o.tickFormat()(t,e)}),l.updateState(!1);var D=t.utils.renderWatch(E),O=!1,j=function(t){return function(){return{active:t.map(function(t){return!t.disabled}),stacked:O}}},I=function(t){return function(e){void 0!==e.stacked&&(O=e.stacked),void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}};return e.dispatch=E,e.multibar=i,e.legend=u,e.controls=l,e.xAxis=o,e.yAxis=a,e.state=M,e.tooltip=c,e.interactiveLayer=s,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return d},set:function(t){d=t}},height:{get:function(){return h},set:function(t){h=t}},showLegend:{get:function(){return m},set:function(t){m=t}},showControls:{get:function(){return g},set:function(t){g=t}},controlLabels:{get:function(){return v},set:function(t){v=t}},showXAxis:{get:function(){return y},set:function(t){y=t}},showYAxis:{get:function(){return b},set:function(t){b=t}},defaultState:{get:function(){return C},set:function(t){C=t}},noData:{get:function(){return S},set:function(t){S=t}},reduceXTicks:{get:function(){return w},set:function(t){w=t}},rotateLabels:{get:function(){return _},set:function(t){_=t}},staggerLabels:{get:function(){return $},set:function(t){$=t}},wrapLabels:{get:function(){return k},set:function(t){k=!!t}},margin:{get:function(){return f},set:function(t){f.top=void 0!==t.top?t.top:f.top,f.right=void 0!==t.right?t.right:f.right,f.bottom=void 0!==t.bottom?t.bottom:f.bottom,f.left=void 0!==t.left?t.left:f.left}},duration:{get:function(){return T},set:function(t){T=t,i.duration(T),o.duration(T),a.duration(T),D.reset(T)}},color:{get:function(){return p},set:function(e){p=t.utils.getColor(e),u.color(p)}},rightAlignYAxis:{get:function(){return x},set:function(t){x=t,a.orient(x?"right":"left")}},useInteractiveGuideline:{get:function(){return N},set:function(t){N=t}},barColor:{get:function(){return i.barColor},set:function(t){i.barColor(t),u.color(function(t,e){return d3.rgb("#ccc").darker(1.5*e).toString()})}}}),t.utils.inheritOptions(e,i),t.utils.initOptions(e),e},t.models.multiBarHorizontal=function(){"use strict";function e(d){return N.reset(),d.each(function(e){var d=c-l.left-l.right,A=f-l.top-l.bottom;h=d3.select(this),t.utils.initSVG(h),$&&(e=d3.layout.stack().offset("zero").values(function(t){return t.values}).y(m)(e)),e.forEach(function(t,e){t.values.forEach(function(n){n.series=e,n.key=t.key})}),$&&e[0].values.map(function(t,n){var r=0,i=0;e.map(function(t){var e=t.values[n];e.size=Math.abs(e.y),e.y<0?(e.y1=i-e.size,i-=e.size):(e.y1=r,r+=e.size)})});var D=r&&i?[]:e.map(function(t){return t.values.map(function(t,e){return{x:v(t,e),y:m(t,e),y0:t.y0,y1:t.y1}})});p.domain(r||d3.merge(D).map(function(t){return t.x})).rangeBands(o||[0,A],C),g.domain(i||d3.extent(d3.merge(D).map(function(t){return $?t.y>0?t.y1+t.y:t.y1:t.y}).concat(b))),k&&!$?g.range(a||[g.domain()[0]<0?M:0,d-(g.domain()[1]>0?M:0)]):g.range(a||[0,d]),s=s||p,u=u||d3.scale.linear().domain(g.domain()).range([g(0),g(0)]);var O=d3.select(this).selectAll("g.nv-wrap.nv-multibarHorizontal").data([e]),j=O.enter().append("g").attr("class","nvd3 nv-wrap nv-multibarHorizontal"),I=(j.append("defs"),j.append("g"));O.select("g");I.append("g").attr("class","nv-groups"),O.attr("transform","translate("+l.left+","+l.top+")");var L=O.select(".nv-groups").selectAll(".nv-group").data(function(t){return t},function(t,e){return e});L.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),L.exit().watchTransition(N,"multibarhorizontal: exit groups").style("stroke-opacity",1e-6).style("fill-opacity",1e-6).remove(),L.attr("class",function(t,e){return"nv-group nv-series-"+e}).classed("hover",function(t){return t.hover}).style("fill",function(t,e){return x(t,e)}).style("stroke",function(t,e){return x(t,e)}),L.watchTransition(N,"multibarhorizontal: groups").style("stroke-opacity",1).style("fill-opacity",S);var P=L.selectAll("g.nv-bar").data(function(t){return t.values});P.exit().remove();var F=P.enter().append("g").attr("transform",function(t,n,r){return"translate("+u($?t.y0:0)+","+($?0:r*p.rangeBand()/e.length+p(v(t,n)))+")"});F.append("rect").attr("width",0).attr("height",p.rangeBand()/($?1:e.length)),P.on("mouseover",function(t,e){d3.select(this).classed("hover",!0),T.elementMouseover({data:t,index:e,color:d3.select(this).style("fill")})}).on("mouseout",function(t,e){d3.select(this).classed("hover",!1),T.elementMouseout({data:t,index:e,color:d3.select(this).style("fill")})}).on("mouseout",function(t,e){T.elementMouseout({data:t,index:e,color:d3.select(this).style("fill")})}).on("mousemove",function(t,e){T.elementMousemove({data:t,index:e,color:d3.select(this).style("fill")})}).on("click",function(t,e){var n=this;T.elementClick({data:t,index:e,color:d3.select(this).style("fill"),event:d3.event,element:n}),d3.event.stopPropagation()}).on("dblclick",function(t,e){T.elementDblClick({data:t,index:e,color:d3.select(this).style("fill")}),d3.event.stopPropagation()}),y(e[0],0)&&(F.append("polyline"),P.select("polyline").attr("fill","none").attr("points",function(t,n){var r=y(t,n),i=.8*p.rangeBand()/(2*($?1:e.length));r=r.length?r:[-Math.abs(r),Math.abs(r)],r=r.map(function(t){return g(t)-g(0)});var o=[[r[0],-i],[r[0],i],[r[0],0],[r[1],0],[r[1],-i],[r[1],i]];return o.map(function(t){return t.join(",")}).join(" ")}).attr("transform",function(t,n){var r=p.rangeBand()/(2*($?1:e.length));return"translate("+(m(t,n)<0?0:g(m(t,n))-g(0))+", "+r+")"})),F.append("text"),k&&!$?(P.select("text").attr("text-anchor",function(t,e){return m(t,e)<0?"end":"start"}).attr("y",p.rangeBand()/(2*e.length)).attr("dy",".32em").text(function(t,e){var n=E(m(t,e)),r=y(t,e);return void 0===r?n:r.length?n+"+"+E(Math.abs(r[1]))+"-"+E(Math.abs(r[0])):n+"±"+E(Math.abs(r))}),P.watchTransition(N,"multibarhorizontal: bars").select("text").attr("x",function(t,e){return m(t,e)<0?-4:g(m(t,e))-g(0)+4})):P.selectAll("text").text(""),_&&!$?(F.append("text").classed("nv-bar-label",!0),P.select("text.nv-bar-label").attr("text-anchor",function(t,e){return m(t,e)<0?"start":"end"}).attr("y",p.rangeBand()/(2*e.length)).attr("dy",".32em").text(function(t,e){return v(t,e)}),P.watchTransition(N,"multibarhorizontal: bars").select("text.nv-bar-label").attr("x",function(t,e){return m(t,e)<0?g(0)-g(m(t,e))+4:-4})):P.selectAll("text.nv-bar-label").text(""),P.attr("class",function(t,e){return m(t,e)<0?"nv-bar negative":"nv-bar positive"}),w&&(n||(n=e.map(function(){return!0})),P.style("fill",function(t,e,r){return d3.rgb(w(t,e)).darker(n.map(function(t,e){return e}).filter(function(t,e){return!n[e]})[r]).toString()}).style("stroke",function(t,e,r){return d3.rgb(w(t,e)).darker(n.map(function(t,e){return e}).filter(function(t,e){return!n[e]})[r]).toString()})),$?P.watchTransition(N,"multibarhorizontal: bars").attr("transform",function(t,e){return"translate("+g(t.y1)+","+p(v(t,e))+")"}).select("rect").attr("width",function(t,e){return Math.abs(g(m(t,e)+t.y0)-g(t.y0))||0}).attr("height",p.rangeBand()):P.watchTransition(N,"multibarhorizontal: bars").attr("transform",function(t,n){return"translate("+g(m(t,n)<0?m(t,n):0)+","+(t.series*p.rangeBand()/e.length+p(v(t,n)))+")"}).select("rect").attr("height",p.rangeBand()/e.length).attr("width",function(t,e){return Math.max(Math.abs(g(m(t,e))-g(0)),1)||0}),s=p.copy(),u=g.copy()}),N.renderEnd("multibarHorizontal immediate"),e}var n,r,i,o,a,s,u,l={top:0,right:0,bottom:0,left:0},c=960,f=500,d=Math.floor(1e4*Math.random()),h=null,p=d3.scale.ordinal(),g=d3.scale.linear(),v=function(t){return t.x},m=function(t){return t.y},y=function(t){return t.yErr},b=[0],x=t.utils.defaultColor(),w=null,$=!1,k=!1,_=!1,M=60,C=.1,S=.75,E=d3.format(",.2f"),A=250,T=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),N=t.utils.renderWatch(T,A);return e.dispatch=T,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return c},set:function(t){c=t}},height:{get:function(){return f},set:function(t){f=t}},x:{get:function(){return v},set:function(t){v=t}},y:{get:function(){return m},set:function(t){m=t}},yErr:{get:function(){return y},set:function(t){y=t}},xScale:{get:function(){return p},set:function(t){p=t}},yScale:{get:function(){return g},set:function(t){g=t}},xDomain:{get:function(){return r},set:function(t){r=t}},yDomain:{get:function(){return i},set:function(t){i=t}},xRange:{get:function(){return o},set:function(t){o=t}},yRange:{get:function(){return a},set:function(t){a=t}},forceY:{get:function(){return b},set:function(t){b=t}},stacked:{get:function(){return $},set:function(t){$=t}},showValues:{get:function(){return k},set:function(t){k=t}},disabled:{get:function(){return n},set:function(t){n=t}},id:{get:function(){return d},set:function(t){d=t}},valueFormat:{get:function(){return E},set:function(t){E=t}},valuePadding:{get:function(){return M},set:function(t){M=t}},groupSpacing:{get:function(){return C},set:function(t){C=t}},fillOpacity:{get:function(){return S},set:function(t){S=t}},margin:{get:function(){return l},set:function(t){l.top=void 0!==t.top?t.top:l.top,l.right=void 0!==t.right?t.right:l.right,l.bottom=void 0!==t.bottom?t.bottom:l.bottom,l.left=void 0!==t.left?t.left:l.left}},duration:{get:function(){return A},set:function(t){A=t,N.reset(A)}},color:{get:function(){return x},set:function(e){x=t.utils.getColor(e)}},barColor:{get:function(){return w},set:function(e){w=e?t.utils.getColor(e):null}}}),t.utils.initOptions(e),e},t.models.multiBarHorizontalChart=function(){"use strict";function e(l){return E.reset(),E.models(i),m&&E.models(o),y&&E.models(a),l.each(function(l){var $=d3.select(this);t.utils.initSVG($);var E=t.utils.availableWidth(f,$,c),A=t.utils.availableHeight(d,$,c);if(e.update=function(){$.transition().duration(M).call(e)},e.container=this,b=i.stacked(),x.setter(S(l),e.update).getter(C(l)).update(),x.disabled=l.map(function(t){return!!t.disabled}),!w){var T;w={};for(T in x)x[T]instanceof Array?w[T]=x[T].slice(0):w[T]=x[T]}if(!(l&&l.length&&l.filter(function(t){return t.values.length}).length))return t.utils.noData(e,$),e;$.selectAll(".nv-noData").remove(),n=i.xScale(),r=i.yScale().clamp(!0);var N=$.selectAll("g.nv-wrap.nv-multiBarHorizontalChart").data([l]),D=N.enter().append("g").attr("class","nvd3 nv-wrap nv-multiBarHorizontalChart").append("g"),O=N.select("g");if(D.append("g").attr("class","nv-x nv-axis"),D.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),
 D.append("g").attr("class","nv-barsWrap"),D.append("g").attr("class","nv-legendWrap"),D.append("g").attr("class","nv-controlsWrap"),v?(s.width(E-_()),O.select(".nv-legendWrap").datum(l).call(s),s.height()>c.top&&(c.top=s.height(),A=t.utils.availableHeight(d,$,c)),O.select(".nv-legendWrap").attr("transform","translate("+_()+","+-c.top+")")):O.select(".nv-legendWrap").selectAll("*").remove(),p){var j=[{key:g.grouped||"Grouped",disabled:i.stacked()},{key:g.stacked||"Stacked",disabled:!i.stacked()}];u.width(_()).color(["#444","#444","#444"]),O.select(".nv-controlsWrap").datum(j).attr("transform","translate(0,"+-c.top+")").call(u)}else O.select(".nv-controlsWrap").selectAll("*").remove();N.attr("transform","translate("+c.left+","+c.top+")"),i.disabled(l.map(function(t){return t.disabled})).width(E).height(A).color(l.map(function(t,e){return t.color||h(t,e)}).filter(function(t,e){return!l[e].disabled}));var I=O.select(".nv-barsWrap").datum(l.filter(function(t){return!t.disabled}));if(I.transition().call(i),m){o.scale(n)._ticks(t.utils.calcTicksY(A/24,l)).tickSize(-E,0),O.select(".nv-x.nv-axis").call(o);var L=O.select(".nv-x.nv-axis").selectAll("g");L.selectAll("line, text")}y&&(a.scale(r)._ticks(t.utils.calcTicksX(E/100,l)).tickSize(-A,0),O.select(".nv-y.nv-axis").attr("transform","translate(0,"+A+")"),O.select(".nv-y.nv-axis").call(a)),O.select(".nv-zeroLine line").attr("x1",r(0)).attr("x2",r(0)).attr("y1",0).attr("y2",-A),s.dispatch.on("stateChange",function(t){for(var n in t)x[n]=t[n];k.stateChange(x),e.update()}),u.dispatch.on("legendClick",function(t,n){if(t.disabled){switch(j=j.map(function(t){return t.disabled=!0,t}),t.disabled=!1,t.key){case"Grouped":case g.grouped:i.stacked(!1);break;case"Stacked":case g.stacked:i.stacked(!0)}x.stacked=i.stacked(),k.stateChange(x),b=i.stacked(),e.update()}}),k.on("changeState",function(t){"undefined"!=typeof t.disabled&&(l.forEach(function(e,n){e.disabled=t.disabled[n]}),x.disabled=t.disabled),"undefined"!=typeof t.stacked&&(i.stacked(t.stacked),x.stacked=t.stacked,b=t.stacked),e.update()})}),E.renderEnd("multibar horizontal chart immediate"),e}var n,r,i=t.models.multiBarHorizontal(),o=t.models.axis(),a=t.models.axis(),s=t.models.legend().height(30),u=t.models.legend().height(30),l=t.models.tooltip(),c={top:30,right:20,bottom:50,left:60},f=null,d=null,h=t.utils.defaultColor(),p=!0,g={},v=!0,m=!0,y=!0,b=!1,x=t.utils.state(),w=null,$=null,k=d3.dispatch("stateChange","changeState","renderEnd"),_=function(){return p?180:0},M=250;x.stacked=!1,i.stacked(b),o.orient("left").tickPadding(5).showMaxMin(!1).tickFormat(function(t){return t}),a.orient("bottom").tickFormat(d3.format(",.1f")),l.duration(0).valueFormatter(function(t,e){return a.tickFormat()(t,e)}).headerFormatter(function(t,e){return o.tickFormat()(t,e)}),u.updateState(!1);var C=function(t){return function(){return{active:t.map(function(t){return!t.disabled}),stacked:b}}},S=function(t){return function(e){void 0!==e.stacked&&(b=e.stacked),void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}},E=t.utils.renderWatch(k,M);return i.dispatch.on("elementMouseover.tooltip",function(t){t.value=e.x()(t.data),t.series={key:t.data.key,value:e.y()(t.data),color:t.color},l.data(t).hidden(!1)}),i.dispatch.on("elementMouseout.tooltip",function(t){l.hidden(!0)}),i.dispatch.on("elementMousemove.tooltip",function(t){l()}),e.dispatch=k,e.multibar=i,e.legend=s,e.controls=u,e.xAxis=o,e.yAxis=a,e.state=x,e.tooltip=l,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return f},set:function(t){f=t}},height:{get:function(){return d},set:function(t){d=t}},showLegend:{get:function(){return v},set:function(t){v=t}},showControls:{get:function(){return p},set:function(t){p=t}},controlLabels:{get:function(){return g},set:function(t){g=t}},showXAxis:{get:function(){return m},set:function(t){m=t}},showYAxis:{get:function(){return y},set:function(t){y=t}},defaultState:{get:function(){return w},set:function(t){w=t}},noData:{get:function(){return $},set:function(t){$=t}},margin:{get:function(){return c},set:function(t){c.top=void 0!==t.top?t.top:c.top,c.right=void 0!==t.right?t.right:c.right,c.bottom=void 0!==t.bottom?t.bottom:c.bottom,c.left=void 0!==t.left?t.left:c.left}},duration:{get:function(){return M},set:function(t){M=t,E.reset(M),i.duration(M),o.duration(M),a.duration(M)}},color:{get:function(){return h},set:function(e){h=t.utils.getColor(e),s.color(h)}},barColor:{get:function(){return i.barColor},set:function(t){i.barColor(t),s.color(function(t,e){return d3.rgb("#ccc").darker(1.5*e).toString()})}}}),t.utils.inheritOptions(e,i),t.utils.initOptions(e),e},t.models.multiChart=function(){"use strict";function e(l){return l.each(function(l){function h(t){var e=2===l[t.seriesIndex].yAxis?T:A;t.value=t.point.x,t.series={value:t.point.y,color:t.point.color,key:t.series.key},D.duration(0).headerFormatter(function(t,e){return E.tickFormat()(t,e)}).valueFormatter(function(t,n){return e.tickFormat()(t,n)}).data(t).hidden(!1)}function O(t){var e=2===l[t.seriesIndex].yAxis?T:A;t.value=t.point.x,t.series={value:t.point.y,color:t.point.color,key:t.series.key},D.duration(100).headerFormatter(function(t,e){return E.tickFormat()(t,e)}).valueFormatter(function(t,n){return e.tickFormat()(t,n)}).data(t).hidden(!1)}function I(t){var e=2===l[t.seriesIndex].yAxis?T:A;t.point.x=C.x()(t.point),t.point.y=C.y()(t.point),D.duration(0).headerFormatter(function(t,e){return E.tickFormat()(t,e)}).valueFormatter(function(t,n){return e.tickFormat()(t,n)}).data(t).hidden(!1)}function L(t){var e=2===l[t.data.series].yAxis?T:A;t.value=_.x()(t.data),t.series={value:_.y()(t.data),color:t.color,key:t.data.key},D.duration(0).headerFormatter(function(t,e){return E.tickFormat()(t,e)}).valueFormatter(function(t,n){return e.tickFormat()(t,n)}).data(t).hidden(!1)}function P(){for(var t=0,e=j.length;t<e;t++){var n=j[t];try{n.clearHighlights()}catch(r){}}}function F(t,e,n){for(var r=0,i=j.length;r<i;r++){var o=j[r];try{o.highlightPoint(t,e,n)}catch(a){}}}var q=d3.select(this);t.utils.initSVG(q),e.update=function(){q.transition().call(e)},e.container=this;var z=t.utils.availableWidth(a,q,i),W=t.utils.availableHeight(s,q,i),R=l.filter(function(t){return"line"==t.type&&1==t.yAxis}),B=l.filter(function(t){return"line"==t.type&&2==t.yAxis}),V=l.filter(function(t){return"scatter"==t.type&&1==t.yAxis}),H=l.filter(function(t){return"scatter"==t.type&&2==t.yAxis}),U=l.filter(function(t){return"bar"==t.type&&1==t.yAxis}),Y=l.filter(function(t){return"bar"==t.type&&2==t.yAxis}),G=l.filter(function(t){return"area"==t.type&&1==t.yAxis}),X=l.filter(function(t){return"area"==t.type&&2==t.yAxis});if(!(l&&l.length&&l.filter(function(t){return t.values.length}).length))return t.utils.noData(e,q),e;q.selectAll(".nv-noData").remove();var Z=l.filter(function(t){return!t.disabled&&1==t.yAxis}).map(function(t){return t.values.map(function(t,e){return{x:c(t),y:f(t)}})}),Q=l.filter(function(t){return!t.disabled&&2==t.yAxis}).map(function(t){return t.values.map(function(t,e){return{x:c(t),y:f(t)}})});m.domain(d3.extent(d3.merge(Z.concat(Q)),function(t){return t.x})).range([0,z]);var K=q.selectAll("g.wrap.multiChart").data([l]),J=K.enter().append("g").attr("class","wrap nvd3 multiChart").append("g");J.append("g").attr("class","nv-x nv-axis"),J.append("g").attr("class","nv-y1 nv-axis"),J.append("g").attr("class","nv-y2 nv-axis"),J.append("g").attr("class","stack1Wrap"),J.append("g").attr("class","stack2Wrap"),J.append("g").attr("class","bars1Wrap"),J.append("g").attr("class","bars2Wrap"),J.append("g").attr("class","scatters1Wrap"),J.append("g").attr("class","scatters2Wrap"),J.append("g").attr("class","lines1Wrap"),J.append("g").attr("class","lines2Wrap"),J.append("g").attr("class","legendWrap"),J.append("g").attr("class","nv-interactive");var tt=K.select("g"),et=l.map(function(t,e){return l[e].color||o(t,e)});if(u){var nt=N.align()?z/2:z,rt=N.align()?nt:0;N.width(nt),N.color(et),tt.select(".legendWrap").datum(l.map(function(t){return t.originalKey=void 0===t.originalKey?t.key:t.originalKey,t.key=t.originalKey+(1==t.yAxis?"":v),t})).call(N),N.height()>i.top&&(i.top=N.height(),W=t.utils.availableHeight(s,q,i)),tt.select(".legendWrap").attr("transform","translate("+rt+","+-i.top+")")}else tt.select(".legendWrap").selectAll("*").remove();x.width(z).height(W).interpolate(d).color(et.filter(function(t,e){return!l[e].disabled&&1==l[e].yAxis&&"line"==l[e].type})),w.width(z).height(W).interpolate(d).color(et.filter(function(t,e){return!l[e].disabled&&2==l[e].yAxis&&"line"==l[e].type})),$.width(z).height(W).color(et.filter(function(t,e){return!l[e].disabled&&1==l[e].yAxis&&"scatter"==l[e].type})),k.width(z).height(W).color(et.filter(function(t,e){return!l[e].disabled&&2==l[e].yAxis&&"scatter"==l[e].type})),_.width(z).height(W).color(et.filter(function(t,e){return!l[e].disabled&&1==l[e].yAxis&&"bar"==l[e].type})),M.width(z).height(W).color(et.filter(function(t,e){return!l[e].disabled&&2==l[e].yAxis&&"bar"==l[e].type})),C.width(z).height(W).interpolate(d).color(et.filter(function(t,e){return!l[e].disabled&&1==l[e].yAxis&&"area"==l[e].type})),S.width(z).height(W).interpolate(d).color(et.filter(function(t,e){return!l[e].disabled&&2==l[e].yAxis&&"area"==l[e].type})),tt.attr("transform","translate("+i.left+","+i.top+")");var it=tt.select(".lines1Wrap").datum(R.filter(function(t){return!t.disabled})),ot=tt.select(".scatters1Wrap").datum(V.filter(function(t){return!t.disabled})),at=tt.select(".bars1Wrap").datum(U.filter(function(t){return!t.disabled})),st=tt.select(".stack1Wrap").datum(G.filter(function(t){return!t.disabled})),ut=tt.select(".lines2Wrap").datum(B.filter(function(t){return!t.disabled})),lt=tt.select(".scatters2Wrap").datum(H.filter(function(t){return!t.disabled})),ct=tt.select(".bars2Wrap").datum(Y.filter(function(t){return!t.disabled})),ft=tt.select(".stack2Wrap").datum(X.filter(function(t){return!t.disabled})),dt=G.length?G.map(function(t){return t.values}).reduce(function(t,e){return t.map(function(t,n){return{x:t.x,y:t.y+e[n].y}})}).concat([{x:0,y:0}]):[],ht=X.length?X.map(function(t){return t.values}).reduce(function(t,e){return t.map(function(t,n){return{x:t.x,y:t.y+e[n].y}})}).concat([{x:0,y:0}]):[];y.domain(n||d3.extent(d3.merge(Z).concat(dt),function(t){return t.y})).range([0,W]),b.domain(r||d3.extent(d3.merge(Q).concat(ht),function(t){return t.y})).range([0,W]),x.yDomain(y.domain()),$.yDomain(y.domain()),_.yDomain(y.domain()),C.yDomain(y.domain()),w.yDomain(b.domain()),k.yDomain(b.domain()),M.yDomain(b.domain()),S.yDomain(b.domain()),G.length&&d3.transition(st).call(C),X.length&&d3.transition(ft).call(S),U.length&&d3.transition(at).call(_),Y.length&&d3.transition(ct).call(M),R.length&&d3.transition(it).call(x),B.length&&d3.transition(ut).call(w),V.length&&d3.transition(ot).call($),H.length&&d3.transition(lt).call(k),E._ticks(t.utils.calcTicksX(z/100,l)).tickSize(-W,0),tt.select(".nv-x.nv-axis").attr("transform","translate(0,"+W+")"),d3.transition(tt.select(".nv-x.nv-axis")).call(E),A._ticks(t.utils.calcTicksY(W/36,l)).tickSize(-z,0),d3.transition(tt.select(".nv-y1.nv-axis")).call(A),T._ticks(t.utils.calcTicksY(W/36,l)).tickSize(-z,0),d3.transition(tt.select(".nv-y2.nv-axis")).call(T),tt.select(".nv-y1.nv-axis").classed("nv-disabled",!Z.length).attr("transform","translate("+m.range()[0]+",0)"),tt.select(".nv-y2.nv-axis").classed("nv-disabled",!Q.length).attr("transform","translate("+m.range()[1]+",0)"),N.dispatch.on("stateChange",function(t){e.update()}),g&&(p.width(z).height(W).margin({left:i.left,top:i.top}).svgContainer(q).xScale(m),K.select(".nv-interactive").call(p)),g?(p.dispatch.on("elementMousemove",function(n){P();var r,i,a,s=[];l.filter(function(t,e){return t.seriesIndex=e,!t.disabled}).forEach(function(u,l){var c=m.domain(),f=u.values.filter(function(t,n){return e.x()(t,n)>=c[0]&&e.x()(t,n)<=c[1]});i=t.interactiveBisect(f,n.pointXValue,e.x());var d=f[i],h=e.y()(d,i);null!==h&&F(l,i,!0),void 0!==d&&(void 0===r&&(r=d),void 0===a&&(a=m(e.x()(d,i))),s.push({key:u.key,value:h,color:o(u,u.seriesIndex),data:d,yAxis:2==u.yAxis?T:A}))});var u=function(t,e){var n=s[e].yAxis;return null==t?"N/A":n.tickFormat()(t)};p.tooltip.headerFormatter(function(t,e){return E.tickFormat()(t,e)}).valueFormatter(p.tooltip.valueFormatter()||u).data({value:e.x()(r,i),index:i,series:s})(),p.renderGuideLine(a)}),p.dispatch.on("elementMouseout",function(t){P()})):(x.dispatch.on("elementMouseover.tooltip",h),w.dispatch.on("elementMouseover.tooltip",h),x.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),w.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),$.dispatch.on("elementMouseover.tooltip",O),k.dispatch.on("elementMouseover.tooltip",O),$.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),k.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),C.dispatch.on("elementMouseover.tooltip",I),S.dispatch.on("elementMouseover.tooltip",I),C.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),S.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),_.dispatch.on("elementMouseover.tooltip",L),M.dispatch.on("elementMouseover.tooltip",L),_.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),M.dispatch.on("elementMouseout.tooltip",function(t){D.hidden(!0)}),_.dispatch.on("elementMousemove.tooltip",function(t){D()}),M.dispatch.on("elementMousemove.tooltip",function(t){D()}))}),e}var n,r,i={top:30,right:20,bottom:50,left:60},o=t.utils.defaultColor(),a=null,s=null,u=!0,l=null,c=function(t){return t.x},f=function(t){return t.y},d="linear",h=!0,p=t.interactiveGuideline(),g=!1,v=" (right axis)",m=d3.scale.linear(),y=d3.scale.linear(),b=d3.scale.linear(),x=t.models.line().yScale(y),w=t.models.line().yScale(b),$=t.models.scatter().yScale(y),k=t.models.scatter().yScale(b),_=t.models.multiBar().stacked(!1).yScale(y),M=t.models.multiBar().stacked(!1).yScale(b),C=t.models.stackedArea().yScale(y),S=t.models.stackedArea().yScale(b),E=t.models.axis().scale(m).orient("bottom").tickPadding(5),A=t.models.axis().scale(y).orient("left"),T=t.models.axis().scale(b).orient("right"),N=t.models.legend().height(30),D=t.models.tooltip(),O=d3.dispatch(),j=[x,w,$,k,_,M,C,S];return e.dispatch=O,e.legend=N,e.lines1=x,e.lines2=w,e.scatters1=$,e.scatters2=k,e.bars1=_,e.bars2=M,e.stack1=C,e.stack2=S,e.xAxis=E,e.yAxis1=A,e.yAxis2=T,e.tooltip=D,e.interactiveLayer=p,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return a},set:function(t){a=t}},height:{get:function(){return s},set:function(t){s=t}},showLegend:{get:function(){return u},set:function(t){u=t}},yDomain1:{get:function(){return n},set:function(t){n=t}},yDomain2:{get:function(){return r},set:function(t){r=t}},noData:{get:function(){return l},set:function(t){l=t}},interpolate:{get:function(){return d},set:function(t){d=t}},legendRightAxisHint:{get:function(){return v},set:function(t){v=t}},margin:{get:function(){return i},set:function(t){i.top=void 0!==t.top?t.top:i.top,i.right=void 0!==t.right?t.right:i.right,i.bottom=void 0!==t.bottom?t.bottom:i.bottom,i.left=void 0!==t.left?t.left:i.left}},color:{get:function(){return o},set:function(e){o=t.utils.getColor(e)}},x:{get:function(){return c},set:function(t){c=t,x.x(t),w.x(t),$.x(t),k.x(t),_.x(t),M.x(t),C.x(t),S.x(t)}},y:{get:function(){return f},set:function(t){f=t,x.y(t),w.y(t),$.y(t),k.y(t),C.y(t),S.y(t),_.y(t),M.y(t)}},useVoronoi:{get:function(){return h},set:function(t){h=t,x.useVoronoi(t),w.useVoronoi(t),C.useVoronoi(t),S.useVoronoi(t)}},useInteractiveGuideline:{get:function(){return g},set:function(t){g=t,g&&(x.interactive(!1),x.useVoronoi(!1),w.interactive(!1),w.useVoronoi(!1),C.interactive(!1),C.useVoronoi(!1),S.interactive(!1),S.useVoronoi(!1),$.interactive(!1),k.interactive(!1))}}}),t.utils.initOptions(e),e},t.models.ohlcBar=function(){"use strict";function e(_){return _.each(function(e){c=d3.select(this);var _=t.utils.availableWidth(s,c,a),C=t.utils.availableHeight(u,c,a);t.utils.initSVG(c);var S=_/e[0].values.length*.9;f.domain(n||d3.extent(e[0].values.map(h).concat(b))),w?f.range(i||[.5*_/e[0].values.length,_*(e[0].values.length-.5)/e[0].values.length]):f.range(i||[5+S/2,_-S/2-5]),d.domain(r||[d3.min(e[0].values.map(y).concat(x)),d3.max(e[0].values.map(m).concat(x))]).range(o||[C,0]),f.domain()[0]===f.domain()[1]&&(f.domain()[0]?f.domain([f.domain()[0]-.01*f.domain()[0],f.domain()[1]+.01*f.domain()[1]]):f.domain([-1,1])),d.domain()[0]===d.domain()[1]&&(d.domain()[0]?d.domain([d.domain()[0]+.01*d.domain()[0],d.domain()[1]-.01*d.domain()[1]]):d.domain([-1,1]));var E=d3.select(this).selectAll("g.nv-wrap.nv-ohlcBar").data([e[0].values]),A=E.enter().append("g").attr("class","nvd3 nv-wrap nv-ohlcBar"),T=A.append("defs"),N=A.append("g"),D=E.select("g");N.append("g").attr("class","nv-ticks"),E.attr("transform","translate("+a.left+","+a.top+")"),c.on("click",function(t,e){M.chartClick({data:t,index:e,pos:d3.event,id:l})}),T.append("clipPath").attr("id","nv-chart-clip-path-"+l).append("rect"),E.select("#nv-chart-clip-path-"+l+" rect").attr("width",_).attr("height",C),D.attr("clip-path",$?"url(#nv-chart-clip-path-"+l+")":"");var O=E.select(".nv-ticks").selectAll(".nv-tick").data(function(t){return t});O.exit().remove(),O.enter().append("path").attr("class",function(t,e,n){return(g(t,e)>v(t,e)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+n+"-"+e}).attr("d",function(t,e){return"m0,0l0,"+(d(g(t,e))-d(m(t,e)))+"l"+-S/2+",0l"+S/2+",0l0,"+(d(y(t,e))-d(g(t,e)))+"l0,"+(d(v(t,e))-d(y(t,e)))+"l"+S/2+",0l"+-S/2+",0z"}).attr("transform",function(t,e){return"translate("+f(h(t,e))+","+d(m(t,e))+")"}).attr("fill",function(t,e){return k[0]}).attr("stroke",function(t,e){return k[0]}).attr("x",0).attr("y",function(t,e){return d(Math.max(0,p(t,e)))}).attr("height",function(t,e){return Math.abs(d(p(t,e))-d(0))}),O.attr("class",function(t,e,n){return(g(t,e)>v(t,e)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+n+"-"+e}),d3.transition(O).attr("transform",function(t,e){return"translate("+f(h(t,e))+","+d(m(t,e))+")"}).attr("d",function(t,n){var r=_/e[0].values.length*.9;return"m0,0l0,"+(d(g(t,n))-d(m(t,n)))+"l"+-r/2+",0l"+r/2+",0l0,"+(d(y(t,n))-d(g(t,n)))+"l0,"+(d(v(t,n))-d(y(t,n)))+"l"+r/2+",0l"+-r/2+",0z"})}),e}var n,r,i,o,a={top:0,right:0,bottom:0,left:0},s=null,u=null,l=Math.floor(1e4*Math.random()),c=null,f=d3.scale.linear(),d=d3.scale.linear(),h=function(t){return t.x},p=function(t){return t.y},g=function(t){return t.open},v=function(t){return t.close},m=function(t){return t.high},y=function(t){return t.low},b=[],x=[],w=!1,$=!0,k=t.utils.defaultColor(),_=!1,M=d3.dispatch("stateChange","changeState","renderEnd","chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove");return e.highlightPoint=function(t,n){e.clearHighlights(),c.select(".nv-ohlcBar .nv-tick-0-"+t).classed("hover",n)},e.clearHighlights=function(){c.select(".nv-ohlcBar .nv-tick.hover").classed("hover",!1)},e.dispatch=M,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return s},set:function(t){s=t}},height:{get:function(){return u},set:function(t){u=t}},xScale:{get:function(){return f},set:function(t){f=t}},yScale:{get:function(){return d},set:function(t){d=t}},xDomain:{get:function(){return n},set:function(t){n=t}},yDomain:{get:function(){return r},set:function(t){r=t}},xRange:{get:function(){return i},set:function(t){i=t}},yRange:{get:function(){return o},set:function(t){o=t}},forceX:{get:function(){return b},set:function(t){b=t}},forceY:{get:function(){return x},set:function(t){x=t}},padData:{get:function(){return w},set:function(t){w=t}},clipEdge:{get:function(){return $},set:function(t){$=t}},id:{get:function(){return l},set:function(t){l=t}},interactive:{get:function(){return _},set:function(t){_=t}},x:{get:function(){return h},set:function(t){h=t}},y:{get:function(){return p},set:function(t){p=t}},open:{get:function(){return g()},set:function(t){g=t}},close:{get:function(){return v()},set:function(t){v=t}},high:{get:function(){return m},set:function(t){m=t}},low:{get:function(){return y},set:function(t){y=t}},margin:{get:function(){return a},set:function(t){a.top=void 0!=t.top?t.top:a.top,a.right=void 0!=t.right?t.right:a.right,a.bottom=void 0!=t.bottom?t.bottom:a.bottom,a.left=void 0!=t.left?t.left:a.left}},color:{get:function(){return k},set:function(e){k=t.utils.getColor(e)}}}),t.utils.initOptions(e),e},t.models.parallelCoordinates=function(){"use strict";function e(S){return C.reset(),S.each(function(e){function C(t){return k(p.map(function(e){if(isNaN(t.values[e.key])||isNaN(parseFloat(t.values[e.key]))||z){var n=f[e.key].domain(),r=f[e.key].range(),i=n[0]-(n[1]-n[0])/9;if(w.indexOf(e.key)<0){var o=d3.scale.linear().domain([i,n[1]]).range([l-12,r[1]]);f[e.key].brush.y(o),w.push(e.key)}if(isNaN(t.values[e.key])||isNaN(parseFloat(t.values[e.key])))return[c(e.key),f[e.key](i)]}return void 0!==U&&(w.length>0||z?(U.style("display","inline"),Y.style("display","inline")):(U.style("display","none"),Y.style("display","none"))),[c(e.key),f[e.key](t.values[e.key])]}))}function S(t){y.forEach(function(e){var n=f[e.dimension].brush.y().domain();e.hasOnlyNaN&&(e.extent[1]=(f[e.dimension].domain()[1]-n[0])*(e.extent[1]-e.extent[0])/(q[e.dimension]-e.extent[0])+n[0]),e.hasNaN&&(e.extent[0]=n[0]),t&&f[e.dimension].brush.extent(e.extent)}),i.select(".nv-brushBackground").each(function(t){d3.select(this).call(f[t.key].brush)}).selectAll("rect").attr("x",-8).attr("width",16),N()}function E(){v===!1&&(v=!0,S(!0))}function A(){K=g.filter(function(t){return!f[t].brush.empty()}),J=K.map(function(t){return f[t].brush.extent()}),y=[],K.forEach(function(t,e){y[e]={dimension:t,extent:J[e],hasNaN:!1,hasOnlyNaN:!1}}),b=[],n.style("display",function(t){var e=K.every(function(e,n){return!(!isNaN(t.values[e])&&!isNaN(parseFloat(t.values[e]))||J[n][0]!=f[e].brush.y().domain()[0])||J[n][0]<=t.values[e]&&t.values[e]<=J[n][1]&&!isNaN(parseFloat(t.values[e]))});return e&&b.push(t),e?null:"none"}),N(),M.brush({filters:y,active:b})}function T(){var t=K.length>0;y.forEach(function(t){t.extent[0]===f[t.dimension].brush.y().domain()[0]&&w.indexOf(t.dimension)>=0&&(t.hasNaN=!0),t.extent[1]<f[t.dimension].domain()[0]&&(t.hasOnlyNaN=!0)}),M.brushEnd(b,t)}function N(){i.select(".nv-axis").each(function(t,e){var n=y.filter(function(e){return e.dimension==t.key});W[t.key]=f[t.key].domain(),0!=n.length&&v&&(W[t.key]=[],n[0].extent[1]>f[t.key].domain()[0]&&(W[t.key]=[n[0].extent[1]]),n[0].extent[0]>=f[t.key].domain()[0]&&W[t.key].push(n[0].extent[0])),d3.select(this).call(_.scale(f[t.key]).tickFormat(t.format).tickValues(W[t.key]))})}function D(t){x[t.key]=this.parentNode.__origin__=c(t.key),r.attr("visibility","hidden")}function O(t){x[t.key]=Math.min(u,Math.max(0,this.parentNode.__origin__+=d3.event.x)),n.attr("d",C),p.sort(function(t,e){return I(t.key)-I(e.key)}),p.forEach(function(t,e){return t.currentPosition=e}),c.domain(p.map(function(t){return t.key})),i.attr("transform",function(t){return"translate("+I(t.key)+")"})}function j(t,e){delete this.parentNode.__origin__,delete x[t.key],d3.select(this.parentNode).attr("transform","translate("+c(t.key)+")"),n.attr("d",C),r.attr("d",C).attr("visibility",null),M.dimensionsOrder(p)}function I(t){var e=x[t];return null==e?c(t):e}var L=d3.select(this);if(u=t.utils.availableWidth(a,L,o),l=t.utils.availableHeight(s,L,o),t.utils.initSVG(L),void 0===e[0].values){var P=[];e.forEach(function(t){var e={},n=Object.keys(t);n.forEach(function(n){"name"!==n&&(e[n]=t[n])}),P.push({key:t.name,values:e})}),e=P}var F=e.map(function(t){return t.values});0===b.length&&(b=e),g=h.sort(function(t,e){return t.currentPosition-e.currentPosition}).map(function(t){return t.key}),p=h.filter(function(t){return!t.disabled}),c.rangePoints([0,u],1).domain(p.map(function(t){return t.key}));var q={},z=!1,W=[];g.forEach(function(t){var e=d3.extent(F,function(e){return+e[t]}),n=e[0],r=e[1],i=!1;(isNaN(n)||isNaN(r))&&(i=!0,n=0,r=0),n===r&&(n-=1,r+=1);var o=y.filter(function(e){return e.dimension==t});0!==o.length&&(i?(n=f[t].domain()[0],r=f[t].domain()[1]):!o[0].hasOnlyNaN&&v?(n=n>o[0].extent[0]?o[0].extent[0]:n,r=r<o[0].extent[1]?o[0].extent[1]:r):o[0].hasNaN&&(r=r<o[0].extent[1]?o[0].extent[1]:r,q[t]=f[t].domain()[1],z=!0)),f[t]=d3.scale.linear().domain([n,r]).range([.9*(l-12),0]),w=[],f[t].brush=d3.svg.brush().y(f[t]).on("brushstart",E).on("brush",A).on("brushend",T)});var R=L.selectAll("g.nv-wrap.nv-parallelCoordinates").data([e]),B=R.enter().append("g").attr("class","nvd3 nv-wrap nv-parallelCoordinates"),V=B.append("g"),H=R.select("g");V.append("g").attr("class","nv-parallelCoordinates background"),V.append("g").attr("class","nv-parallelCoordinates foreground"),V.append("g").attr("class","nv-parallelCoordinates missingValuesline"),R.attr("transform","translate("+o.left+","+o.top+")"),k.interpolate("cardinal").tension($),_.orient("left");var U,Y,G=d3.behavior.drag().on("dragstart",D).on("drag",O).on("dragend",j),X=c.range()[1]-c.range()[0];if(!isNaN(X)){var Z=[0+X/2,l-12,u-X/2,l-12];U=R.select(".missingValuesline").selectAll("line").data([Z]),U.enter().append("line"),U.exit().remove(),U.attr("x1",function(t){return t[0]}).attr("y1",function(t){return t[1]}).attr("x2",function(t){return t[2]}).attr("y2",function(t){return t[3]}),Y=R.select(".missingValuesline").selectAll("text").data([d]),Y.append("text").data([d]),Y.enter().append("text"),Y.exit().remove(),Y.attr("y",l).attr("x",u-92-X/2).text(function(t){return t})}r=R.select(".background").selectAll("path").data(e),r.enter().append("path"),r.exit().remove(),r.attr("d",C),n=R.select(".foreground").selectAll("path").data(e),n.enter().append("path"),n.exit().remove(),n.attr("d",C).style("stroke-width",function(t,e){return isNaN(t.strokeWidth)&&(t.strokeWidth=1),t.strokeWidth}).attr("stroke",function(t,e){return t.color||m(t,e)}),n.on("mouseover",function(t,e){d3.select(this).classed("hover",!0).style("stroke-width",t.strokeWidth+2+"px").style("stroke-opacity",1),M.elementMouseover({label:t.name,color:t.color||m(t,e),values:t.values,dimensions:p})}),n.on("mouseout",function(t,e){d3.select(this).classed("hover",!1).style("stroke-width",t.strokeWidth+"px").style("stroke-opacity",.7),M.elementMouseout({label:t.name,index:e})}),n.on("mousemove",function(t,e){M.elementMousemove()}),n.on("click",function(t){M.elementClick({id:t.id})}),i=H.selectAll(".dimension").data(p);var Q=i.enter().append("g").attr("class","nv-parallelCoordinates dimension");i.attr("transform",function(t){return"translate("+c(t.key)+",0)"}),Q.append("g").attr("class","nv-axis"),Q.append("text").attr("class","nv-label").style("cursor","move").attr("dy","-1em").attr("text-anchor","middle").on("mouseover",function(t,e){M.elementMouseover({label:t.tooltip||t.key,color:t.color})}).on("mouseout",function(t,e){M.elementMouseout({label:t.tooltip})}).on("mousemove",function(t,e){M.elementMousemove()}).call(G),Q.append("g").attr("class","nv-brushBackground"),i.exit().remove(),i.select(".nv-label").text(function(t){return t.key}),S(v);var K=g.filter(function(t){return!f[t].brush.empty()}),J=K.map(function(t){return f[t].brush.extent()}),tt=b.slice(0);b=[],n.style("display",function(t){var e=K.every(function(e,n){return!(!isNaN(t.values[e])&&!isNaN(parseFloat(t.values[e]))||J[n][0]!=f[e].brush.y().domain()[0])||J[n][0]<=t.values[e]&&t.values[e]<=J[n][1]&&!isNaN(parseFloat(t.values[e]))});return e&&b.push(t),e?null:"none"}),(y.length>0||!t.utils.arrayEquals(b,tt))&&M.activeChanged(b)}),e}var n,r,i,o={top:30,right:0,bottom:10,left:0},a=null,s=null,u=null,l=null,c=d3.scale.ordinal(),f={},d="undefined values",h=[],p=[],g=[],v=!0,m=t.utils.defaultColor(),y=[],b=[],x=[],w=[],$=1,k=d3.svg.line(),_=d3.svg.axis(),M=d3.dispatch("brushstart","brush","brushEnd","dimensionsOrder","stateChange","elementClick","elementMouseover","elementMouseout","elementMousemove","renderEnd","activeChanged"),C=t.utils.renderWatch(M);return e.dispatch=M,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return a},set:function(t){a=t}},height:{get:function(){return s},set:function(t){s=t}},dimensionData:{get:function(){return h},set:function(t){h=t}},displayBrush:{get:function(){return v},set:function(t){v=t}},filters:{get:function(){return y},set:function(t){y=t}},active:{get:function(){return b},set:function(t){b=t}},lineTension:{get:function(){return $},set:function(t){$=t}},undefinedValuesLabel:{get:function(){return d},set:function(t){d=t}},dimensions:{get:function(){return h.map(function(t){return t.key})},set:function(e){t.deprecated("dimensions","use dimensionData instead"),0===h.length?e.forEach(function(t){h.push({key:t})}):e.forEach(function(t,e){h[e].key=t})}},dimensionNames:{get:function(){return h.map(function(t){return t.key})},set:function(e){t.deprecated("dimensionNames","use dimensionData instead"),g=[],0===h.length?e.forEach(function(t){h.push({key:t})}):e.forEach(function(t,e){h[e].key=t})}},dimensionFormats:{get:function(){return h.map(function(t){return t.format})},set:function(e){t.deprecated("dimensionFormats","use dimensionData instead"),0===h.length?e.forEach(function(t){h.push({format:t})}):e.forEach(function(t,e){h[e].format=t})}},margin:{get:function(){return o},set:function(t){o.top=void 0!==t.top?t.top:o.top,o.right=void 0!==t.right?t.right:o.right,o.bottom=void 0!==t.bottom?t.bottom:o.bottom,o.left=void 0!==t.left?t.left:o.left}},color:{get:function(){return m},set:function(e){m=t.utils.getColor(e)}}}),t.utils.initOptions(e),e},t.models.parallelCoordinatesChart=function(){"use strict";function e(i){return m.reset(),m.models(n),i.each(function(i){var l=d3.select(this);t.utils.initSVG(l);var p=t.utils.availableWidth(a,l,o),g=t.utils.availableHeight(s,l,o);if(e.update=function(){l.call(e)},e.container=this,c.setter(b(f),e.update).getter(y(f)).update(),c.disabled=f.map(function(t){return!!t.disabled}),f=f.map(function(t){return t.disabled=!!t.disabled,t}),f.forEach(function(t,e){t.originalPosition=isNaN(t.originalPosition)?e:t.originalPosition,t.currentPosition=isNaN(t.currentPosition)?e:t.currentPosition}),!h){var m;h={};for(m in c)c[m]instanceof Array?h[m]=c[m].slice(0):h[m]=c[m]}if(!i||!i.length)return t.utils.noData(e,l),e;l.selectAll(".nv-noData").remove();var x=l.selectAll("g.nv-wrap.nv-parallelCoordinatesChart").data([i]),w=x.enter().append("g").attr("class","nvd3 nv-wrap nv-parallelCoordinatesChart").append("g"),$=x.select("g");w.append("g").attr("class","nv-parallelCoordinatesWrap"),w.append("g").attr("class","nv-legendWrap"),$.select("rect").attr("width",p).attr("height",g>0?g:0),u?(r.width(p).color(function(t){return"rgb(188,190,192)"}),$.select(".nv-legendWrap").datum(f.sort(function(t,e){return t.originalPosition-e.originalPosition})).call(r),r.height()>o.top&&(o.top=r.height(),g=t.utils.availableHeight(s,l,o)),x.select(".nv-legendWrap").attr("transform","translate( 0 ,"+-o.top+")")):$.select(".nv-legendWrap").selectAll("*").remove(),x.attr("transform","translate("+o.left+","+o.top+")"),n.width(p).height(g).dimensionData(f).displayBrush(d);var k=$.select(".nv-parallelCoordinatesWrap ").datum(i);k.transition().call(n),n.dispatch.on("brushEnd",function(t,e){e?(d=!0,v.brushEnd(t)):d=!1}),r.dispatch.on("stateChange",function(t){for(var n in t)c[n]=t[n];v.stateChange(c),e.update()}),n.dispatch.on("dimensionsOrder",function(t){f.sort(function(t,e){return t.currentPosition-e.currentPosition});var e=!1;f.forEach(function(t,n){t.currentPosition=n,t.currentPosition!==t.originalPosition&&(e=!0)}),v.dimensionsOrder(f,e)}),v.on("changeState",function(t){"undefined"!=typeof t.disabled&&(f.forEach(function(e,n){e.disabled=t.disabled[n]}),c.disabled=t.disabled),e.update()})}),m.renderEnd("parraleleCoordinateChart immediate"),e}var n=t.models.parallelCoordinates(),r=t.models.legend(),i=t.models.tooltip(),o=(t.models.tooltip(),{top:0,right:0,bottom:0,left:0}),a=null,s=null,u=!0,l=t.utils.defaultColor(),c=t.utils.state(),f=[],d=!0,h=null,p=null,g="undefined",v=d3.dispatch("dimensionsOrder","brushEnd","stateChange","changeState","renderEnd"),m=t.utils.renderWatch(v),y=function(t){
 return function(){return{active:t.map(function(t){return!t.disabled})}}},b=function(t){return function(e){void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}};return i.contentGenerator(function(t){var e='<table><thead><tr><td class="legend-color-guide"><div style="background-color:'+t.color+'"></div></td><td><strong>'+t.key+"</strong></td></tr></thead>";return 0!==t.series.length&&(e+='<tbody><tr><td height ="10px"></td></tr>',t.series.forEach(function(t){e=e+'<tr><td class="legend-color-guide"><div style="background-color:'+t.color+'"></div></td><td class="key">'+t.key+'</td><td class="value">'+t.value+"</td></tr>"}),e+="</tbody>"),e+="</table>"}),n.dispatch.on("elementMouseover.tooltip",function(t){var e={key:t.label,color:t.color,series:[]};t.values&&(Object.keys(t.values).forEach(function(n){var r=t.dimensions.filter(function(t){return t.key===n})[0];if(r){var i;i=isNaN(t.values[n])||isNaN(parseFloat(t.values[n]))?g:r.format(t.values[n]),e.series.push({idx:r.currentPosition,key:n,value:i,color:r.color})}}),e.series.sort(function(t,e){return t.idx-e.idx})),i.data(e).hidden(!1)}),n.dispatch.on("elementMouseout.tooltip",function(t){i.hidden(!0)}),n.dispatch.on("elementMousemove.tooltip",function(){i()}),e.dispatch=v,e.parallelCoordinates=n,e.legend=r,e.tooltip=i,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return a},set:function(t){a=t}},height:{get:function(){return s},set:function(t){s=t}},showLegend:{get:function(){return u},set:function(t){u=t}},defaultState:{get:function(){return h},set:function(t){h=t}},dimensionData:{get:function(){return f},set:function(t){f=t}},displayBrush:{get:function(){return d},set:function(t){d=t}},noData:{get:function(){return p},set:function(t){p=t}},nanValue:{get:function(){return g},set:function(t){g=t}},margin:{get:function(){return o},set:function(t){o.top=void 0!==t.top?t.top:o.top,o.right=void 0!==t.right?t.right:o.right,o.bottom=void 0!==t.bottom?t.bottom:o.bottom,o.left=void 0!==t.left?t.left:o.left}},color:{get:function(){return l},set:function(e){l=t.utils.getColor(e),r.color(l),n.color(l)}}}),t.utils.inheritOptions(e,n),t.utils.initOptions(e),e},t.models.pie=function(){"use strict";function e(N){return T.reset(),N.each(function(e){function N(t,e){t.endAngle=isNaN(t.endAngle)?0:t.endAngle,t.startAngle=isNaN(t.startAngle)?0:t.startAngle,g||(t.innerRadius=0);var n=d3.interpolate(this._current,t);return this._current=n(0),function(t){return E[e](n(t))}}var D=r-n.left-n.right,O=i-n.top-n.bottom,j=Math.min(D,O)/2,I=[],L=[];if(u=d3.select(this),0===C.length)for(var P=j-j/5,F=_*j,q=0;q<e[0].length;q++)I.push(P),L.push(F);else m?(I=C.map(function(t){return(t.outer-t.outer/5)*j}),L=C.map(function(t){return(t.inner-t.inner/5)*j}),_=d3.min(C.map(function(t){return t.inner-t.inner/5}))):(I=C.map(function(t){return t.outer*j}),L=C.map(function(t){return t.inner*j}),_=d3.min(C.map(function(t){return t.inner})));t.utils.initSVG(u);var z=u.selectAll(".nv-wrap.nv-pie").data(e),W=z.enter().append("g").attr("class","nvd3 nv-wrap nv-pie nv-chart-"+s),R=W.append("g"),B=z.select("g"),V=R.append("g").attr("class","nv-pie");R.append("g").attr("class","nv-pieLabels"),z.attr("transform","translate("+n.left+","+n.top+")"),B.select(".nv-pie").attr("transform","translate("+D/2+","+O/2+")"),B.select(".nv-pieLabels").attr("transform","translate("+D/2+","+O/2+")"),u.on("click",function(t,e){S.chartClick({data:t,index:e,pos:d3.event,id:s})}),E=[],A=[];for(var q=0;q<e[0].length;q++){var H=d3.svg.arc().outerRadius(I[q]),U=d3.svg.arc().outerRadius(I[q]+5);x!==!1&&(H.startAngle(x),U.startAngle(x)),$!==!1&&(H.endAngle($),U.endAngle($)),g&&(H.innerRadius(L[q]),U.innerRadius(L[q])),H.cornerRadius&&k&&(H.cornerRadius(k),U.cornerRadius(k)),E.push(H),A.push(U)}var Y=d3.layout.pie().sort(null).value(function(t){return t.disabled?0:a(t)});Y.padAngle&&w&&Y.padAngle(w),g&&v&&(V.append("text").attr("class","nv-pie-title"),z.select(".nv-pie-title").style("text-anchor","middle").text(function(t){return v}).style("font-size",Math.min(D,O)*_*2/(v.length+2)+"px").attr("dy","0.35em").attr("transform",function(t,e){return"translate(0, "+y+")"}));var G=z.select(".nv-pie").selectAll(".nv-slice").data(Y),X=z.select(".nv-pieLabels").selectAll(".nv-label").data(Y);G.exit().remove(),X.exit().remove();var Z=G.enter().append("g");Z.attr("class","nv-slice"),Z.on("mouseover",function(t,e){d3.select(this).classed("hover",!0),m&&d3.select(this).select("path").transition().duration(70).attr("d",A[e]),S.elementMouseover({data:t.data,index:e,color:d3.select(this).style("fill"),percent:(t.endAngle-t.startAngle)/(2*Math.PI)})}),Z.on("mouseout",function(t,e){d3.select(this).classed("hover",!1),m&&d3.select(this).select("path").transition().duration(50).attr("d",E[e]),S.elementMouseout({data:t.data,index:e})}),Z.on("mousemove",function(t,e){S.elementMousemove({data:t.data,index:e})}),Z.on("click",function(t,e){var n=this;S.elementClick({data:t.data,index:e,color:d3.select(this).style("fill"),event:d3.event,element:n})}),Z.on("dblclick",function(t,e){S.elementDblClick({data:t.data,index:e,color:d3.select(this).style("fill")})}),G.attr("fill",function(t,e){return l(t.data,e)}),G.attr("stroke",function(t,e){return l(t.data,e)});Z.append("path").each(function(t){this._current=t});if(G.select("path").transition().duration(M).attr("d",function(t,e){return E[e](t)}).attrTween("d",N),f){for(var Q=[],q=0;q<e[0].length;q++)Q.push(E[q]),d?g&&(Q[q]=d3.svg.arc().outerRadius(E[q].outerRadius()),x!==!1&&Q[q].startAngle(x),$!==!1&&Q[q].endAngle($)):g||Q[q].innerRadius(0);X.enter().append("g").classed("nv-label",!0).each(function(t,e){var n=d3.select(this);n.attr("transform",function(t,e){if(b){t.outerRadius=I[e]+10,t.innerRadius=I[e]+15;var n=(t.startAngle+t.endAngle)/2*(180/Math.PI);return(t.startAngle+t.endAngle)/2<Math.PI?n-=90:n+=90,"translate("+Q[e].centroid(t)+") rotate("+n+")"}return t.outerRadius=j+10,t.innerRadius=j+15,"translate("+Q[e].centroid(t)+")"}),n.append("rect").style("stroke","#fff").style("fill","#fff").attr("rx",3).attr("ry",3),n.append("text").style("text-anchor",b?(t.startAngle+t.endAngle)/2<Math.PI?"start":"end":"middle").style("fill","#000")});var K={},J=14,tt=140,et=function(t){return Math.floor(t[0]/tt)*tt+","+Math.floor(t[1]/J)*J},nt=function(t){return(t.endAngle-t.startAngle)/(2*Math.PI)};X.watchTransition(T,"pie labels").attr("transform",function(t,e){if(b){t.outerRadius=I[e]+10,t.innerRadius=I[e]+15;var n=(t.startAngle+t.endAngle)/2*(180/Math.PI);return(t.startAngle+t.endAngle)/2<Math.PI?n-=90:n+=90,"translate("+Q[e].centroid(t)+") rotate("+n+")"}t.outerRadius=j+10,t.innerRadius=j+15;var r=Q[e].centroid(t),i=nt(t);if(t.value&&i>=p){var o=et(r);K[o]&&(r[1]-=J),K[et(r)]=!0}return"translate("+r+")"}),X.select(".nv-label text").style("text-anchor",function(t,e){return b?(t.startAngle+t.endAngle)/2<Math.PI?"start":"end":"middle"}).text(function(t,e){var n=nt(t),r="";if(!t.value||n<p)return"";if("function"==typeof h)r=h(t,e,{key:o(t.data),value:a(t.data),percent:c(n)});else switch(h){case"key":r=o(t.data);break;case"value":r=c(a(t.data));break;case"percent":r=d3.format("%")(n)}return r})}}),T.renderEnd("pie immediate"),e}var n={top:0,right:0,bottom:0,left:0},r=500,i=500,o=function(t){return t.x},a=function(t){return t.y},s=Math.floor(1e4*Math.random()),u=null,l=t.utils.defaultColor(),c=d3.format(",.2f"),f=!0,d=!1,h="key",p=.02,g=!1,v=!1,m=!0,y=0,b=!1,x=!1,w=!1,$=!1,k=0,_=.5,M=250,C=[],S=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),E=[],A=[],T=t.utils.renderWatch(S);return e.dispatch=S,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{arcsRadius:{get:function(){return C},set:function(t){C=t}},width:{get:function(){return r},set:function(t){r=t}},height:{get:function(){return i},set:function(t){i=t}},showLabels:{get:function(){return f},set:function(t){f=t}},title:{get:function(){return v},set:function(t){v=t}},titleOffset:{get:function(){return y},set:function(t){y=t}},labelThreshold:{get:function(){return p},set:function(t){p=t}},valueFormat:{get:function(){return c},set:function(t){c=t}},x:{get:function(){return o},set:function(t){o=t}},id:{get:function(){return s},set:function(t){s=t}},endAngle:{get:function(){return $},set:function(t){$=t}},startAngle:{get:function(){return x},set:function(t){x=t}},padAngle:{get:function(){return w},set:function(t){w=t}},cornerRadius:{get:function(){return k},set:function(t){k=t}},donutRatio:{get:function(){return _},set:function(t){_=t}},labelsOutside:{get:function(){return d},set:function(t){d=t}},labelSunbeamLayout:{get:function(){return b},set:function(t){b=t}},donut:{get:function(){return g},set:function(t){g=t}},growOnHover:{get:function(){return m},set:function(t){m=t}},pieLabelsOutside:{get:function(){return d},set:function(e){d=e,t.deprecated("pieLabelsOutside","use labelsOutside instead")}},donutLabelsOutside:{get:function(){return d},set:function(e){d=e,t.deprecated("donutLabelsOutside","use labelsOutside instead")}},labelFormat:{get:function(){return c},set:function(e){c=e,t.deprecated("labelFormat","use valueFormat instead")}},margin:{get:function(){return n},set:function(t){n.top="undefined"!=typeof t.top?t.top:n.top,n.right="undefined"!=typeof t.right?t.right:n.right,n.bottom="undefined"!=typeof t.bottom?t.bottom:n.bottom,n.left="undefined"!=typeof t.left?t.left:n.left}},duration:{get:function(){return M},set:function(t){M=t,T.reset(M)}},y:{get:function(){return a},set:function(t){a=d3.functor(t)}},color:{get:function(){return l},set:function(e){l=t.utils.getColor(e)}},labelType:{get:function(){return h},set:function(t){h=t||"key"}}}),t.utils.initOptions(e),e},t.models.pieChart=function(){"use strict";function e(i){return m.reset(),m.models(n),i.each(function(i){var u=d3.select(this);t.utils.initSVG(u);var f=t.utils.availableWidth(a,u,o),p=t.utils.availableHeight(s,u,o);if(e.update=function(){u.transition().call(e)},e.container=this,d.setter(b(i),e.update).getter(y(i)).update(),d.disabled=i.map(function(t){return!!t.disabled}),!h){var g;h={};for(g in d)d[g]instanceof Array?h[g]=d[g].slice(0):h[g]=d[g]}if(!i||!i.length)return t.utils.noData(e,u),e;u.selectAll(".nv-noData").remove();var m=u.selectAll("g.nv-wrap.nv-pieChart").data([i]),x=m.enter().append("g").attr("class","nvd3 nv-wrap nv-pieChart").append("g"),w=m.select("g");if(x.append("g").attr("class","nv-pieWrap"),x.append("g").attr("class","nv-legendWrap"),l){if("top"===c)r.width(f).key(n.x()),m.select(".nv-legendWrap").datum(i).call(r),r.height()>o.top&&(o.top=r.height(),p=t.utils.availableHeight(s,u,o)),m.select(".nv-legendWrap").attr("transform","translate(0,"+-o.top+")");else if("right"===c){var $=t.models.legend().width();f/2<$&&($=f/2),r.height(p).key(n.x()),r.width($),f-=r.width(),m.select(".nv-legendWrap").datum(i).call(r).attr("transform","translate("+f+",0)")}}else w.select(".nv-legendWrap").selectAll("*").remove();m.attr("transform","translate("+o.left+","+o.top+")"),n.width(f).height(p);var k=w.select(".nv-pieWrap").datum([i]);d3.transition(k).call(n),r.dispatch.on("stateChange",function(t){for(var n in t)d[n]=t[n];v.stateChange(d),e.update()}),v.on("changeState",function(t){"undefined"!=typeof t.disabled&&(i.forEach(function(e,n){e.disabled=t.disabled[n]}),d.disabled=t.disabled),e.update()})}),m.renderEnd("pieChart immediate"),e}var n=t.models.pie(),r=t.models.legend(),i=t.models.tooltip(),o={top:30,right:20,bottom:20,left:20},a=null,s=null,u=!1,l=!0,c="top",f=t.utils.defaultColor(),d=t.utils.state(),h=null,p=null,g=250,v=d3.dispatch("stateChange","changeState","renderEnd");i.duration(0).headerEnabled(!1).valueFormatter(function(t,e){return n.valueFormat()(t,e)});var m=t.utils.renderWatch(v),y=function(t){return function(){return{active:t.map(function(t){return!t.disabled})}}},b=function(t){return function(e){void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}};return n.dispatch.on("elementMouseover.tooltip",function(t){t.series={key:e.x()(t.data),value:e.y()(t.data),color:t.color,percent:t.percent},u||(delete t.percent,delete t.series.percent),i.data(t).hidden(!1)}),n.dispatch.on("elementMouseout.tooltip",function(t){i.hidden(!0)}),n.dispatch.on("elementMousemove.tooltip",function(t){i()}),e.legend=r,e.dispatch=v,e.pie=n,e.tooltip=i,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return a},set:function(t){a=t}},height:{get:function(){return s},set:function(t){s=t}},noData:{get:function(){return p},set:function(t){p=t}},showTooltipPercent:{get:function(){return u},set:function(t){u=t}},showLegend:{get:function(){return l},set:function(t){l=t}},legendPosition:{get:function(){return c},set:function(t){c=t}},defaultState:{get:function(){return h},set:function(t){h=t}},color:{get:function(){return f},set:function(t){f=t,r.color(f),n.color(f)}},duration:{get:function(){return g},set:function(t){g=t,m.reset(g),n.duration(g)}},margin:{get:function(){return o},set:function(t){o.top=void 0!==t.top?t.top:o.top,o.right=void 0!==t.right?t.right:o.right,o.bottom=void 0!==t.bottom?t.bottom:o.bottom,o.left=void 0!==t.left?t.left:o.left}}}),t.utils.inheritOptions(e,n),t.utils.initOptions(e),e},t.models.scatter=function(){"use strict";function e(t){var e,n;return e=u=u||{},n=t[0].series,e=e[n]=e[n]||{},n=t[1],e=e[n]=e[n]||{}}function n(t){var n,r,i=t[0],o=e(t),a=!1;for(n=1;n<arguments.length;n++)r=arguments[n],o[r]===i[r]&&o.hasOwnProperty(r)||(o[r]=i[r],a=!0);return a}function r(e){return U.reset(),e.each(function(e){function u(){if(H=!1,!M)return!1;if(W===!0){var n=d3.merge(e.map(function(e,n){return e.values.map(function(e,r){var i=y(e,r),o=b(e,r);return[t.utils.NaNtoZero(g(i))+1e-4*Math.random(),t.utils.NaNtoZero(v(o))+1e-4*Math.random(),n,r,e]}).filter(function(t,e){return C(t[4],e)})}));if(0==n.length)return!1;n.length<3&&(n.push([g.range()[0]-20,v.range()[0]-20,null,null]),n.push([g.range()[1]+20,v.range()[1]+20,null,null]),n.push([g.range()[0]-20,v.range()[0]+20,null,null]),n.push([g.range()[1]+20,v.range()[1]-20,null,null]));var r=d3.geom.polygon([[-10,-10],[-10,f+10],[c+10,f+10],[c+10,-10]]),i=d3.geom.voronoi(n).map(function(t,e){return{data:r.clip(t),series:n[e][2],point:n[e][3]}});J.select(".nv-point-paths").selectAll("path").remove();var o=J.select(".nv-point-paths").selectAll("path").data(i),a=o.enter().append("svg:path").attr("d",function(t){return t&&t.data&&0!==t.data.length?"M"+t.data.join(",")+"Z":"M 0 0"}).attr("id",function(t,e){return"nv-path-"+e}).attr("clip-path",function(t,e){return"url(#nv-clip-"+h+"-"+e+")"});if(N&&a.style("fill",d3.rgb(230,230,230)).style("fill-opacity",.4).style("stroke-opacity",1).style("stroke",d3.rgb(200,200,200)),T){J.select(".nv-point-clips").selectAll("*").remove();var s=J.select(".nv-point-clips").selectAll("clipPath").data(n);s.enter().append("svg:clipPath").attr("id",function(t,e){return"nv-clip-"+h+"-"+e}).append("svg:circle").attr("cx",function(t){return t[0]}).attr("cy",function(t){return t[1]}).attr("r",D)}var u=function(t,n){if(H)return 0;var r=e[t.series];if(void 0!==r){var i=r.values[t.point];i.color=d(r,t.series),i.x=y(i),i.y=b(i);var o=p.node().getBoundingClientRect(),a=window.pageYOffset||document.documentElement.scrollTop,s=window.pageXOffset||document.documentElement.scrollLeft,u={left:g(y(i,t.point))+o.left+s+l.left+10,top:v(b(i,t.point))+o.top+a+l.top+10};n({point:i,series:r,pos:u,relativePos:[g(y(i,t.point))+l.left,v(b(i,t.point))+l.top],seriesIndex:t.series,pointIndex:t.point})}};o.on("click",function(t){u(t,z.elementClick)}).on("dblclick",function(t){u(t,z.elementDblClick)}).on("mouseover",function(t){u(t,z.elementMouseover)}).on("mouseout",function(t,e){u(t,z.elementMouseout)})}else J.select(".nv-groups").selectAll(".nv-group").selectAll(".nv-point").on("click",function(t,n){if(H||!e[t.series])return 0;var r=e[t.series],i=r.values[n],o=this;z.elementClick({point:i,series:r,pos:[g(y(i,n))+l.left,v(b(i,n))+l.top],relativePos:[g(y(i,n))+l.left,v(b(i,n))+l.top],seriesIndex:t.series,pointIndex:n,event:d3.event,element:o})}).on("dblclick",function(t,n){if(H||!e[t.series])return 0;var r=e[t.series],i=r.values[n];z.elementDblClick({point:i,series:r,pos:[g(y(i,n))+l.left,v(b(i,n))+l.top],relativePos:[g(y(i,n))+l.left,v(b(i,n))+l.top],seriesIndex:t.series,pointIndex:n})}).on("mouseover",function(t,n){if(H||!e[t.series])return 0;var r=e[t.series],i=r.values[n];z.elementMouseover({point:i,series:r,pos:[g(y(i,n))+l.left,v(b(i,n))+l.top],relativePos:[g(y(i,n))+l.left,v(b(i,n))+l.top],seriesIndex:t.series,pointIndex:n,color:d(t,n)})}).on("mouseout",function(t,n){if(H||!e[t.series])return 0;var r=e[t.series],i=r.values[n];z.elementMouseout({point:i,series:r,pos:[g(y(i,n))+l.left,v(b(i,n))+l.top],relativePos:[g(y(i,n))+l.left,v(b(i,n))+l.top],seriesIndex:t.series,pointIndex:n,color:d(t,n)})})}p=d3.select(this);var R=t.utils.availableWidth(c,p,l),G=t.utils.availableHeight(f,p,l);t.utils.initSVG(p),e.forEach(function(t,e){t.values.forEach(function(t){t.series=e})});var X=r.yScale().name===d3.scale.log().name,Z=O&&j&&P?[]:d3.merge(e.map(function(t){return t.values.map(function(t,e){return{x:y(t,e),y:b(t,e),size:x(t,e)}})}));if(g.domain(O||d3.extent(Z.map(function(t){return t.x}).concat($))),S&&e[0]?g.range(I||[(R*E+R)/(2*e[0].values.length),R-R*(1+E)/(2*e[0].values.length)]):g.range(I||[0,R]),X){var Q=d3.min(Z.map(function(t){if(0!==t.y)return t.y}));v.clamp(!0).domain(j||d3.extent(Z.map(function(t){return 0!==t.y?t.y:.1*Q}).concat(k))).range(L||[G,0])}else v.domain(j||d3.extent(Z.map(function(t){return t.y}).concat(k))).range(L||[G,0]);m.domain(P||d3.extent(Z.map(function(t){return t.size}).concat(_))).range(F||Y),q=g.domain()[0]===g.domain()[1]||v.domain()[0]===v.domain()[1],g.domain()[0]===g.domain()[1]&&(g.domain()[0]?g.domain([g.domain()[0]-.01*g.domain()[0],g.domain()[1]+.01*g.domain()[1]]):g.domain([-1,1])),v.domain()[0]===v.domain()[1]&&(v.domain()[0]?v.domain([v.domain()[0]-.01*v.domain()[0],v.domain()[1]+.01*v.domain()[1]]):v.domain([-1,1])),isNaN(g.domain()[0])&&g.domain([-1,1]),isNaN(v.domain()[0])&&v.domain([-1,1]),i=i||g,o=o||v,a=a||m;var K=g(1)!==i(1)||v(1)!==o(1)||m(1)!==a(1),J=p.selectAll("g.nv-wrap.nv-scatter").data([e]),tt=J.enter().append("g").attr("class","nvd3 nv-wrap nv-scatter nv-chart-"+h),et=tt.append("defs"),nt=tt.append("g"),rt=J.select("g");J.classed("nv-single-point",q),nt.append("g").attr("class","nv-groups"),nt.append("g").attr("class","nv-point-paths"),tt.append("g").attr("class","nv-point-clips"),J.attr("transform","translate("+l.left+","+l.top+")"),et.append("clipPath").attr("id","nv-edge-clip-"+h).append("rect"),J.select("#nv-edge-clip-"+h+" rect").attr("width",R).attr("height",G>0?G:0),rt.attr("clip-path",A?"url(#nv-edge-clip-"+h+")":""),H=!0;var it=J.select(".nv-groups").selectAll(".nv-group").data(function(t){return t},function(t){return t.key});it.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),it.exit().remove(),it.attr("class",function(t,e){return(t.classed||"")+" nv-group nv-series-"+e}).classed("nv-noninteractive",!M).classed("hover",function(t){return t.hover}),it.watchTransition(U,"scatter: groups").style("fill",function(t,e){return d(t,e)}).style("stroke",function(t,e){return d(t,e)}).style("stroke-opacity",1).style("fill-opacity",.5);var ot=it.selectAll("path.nv-point").data(function(t){return t.values.map(function(t,e){return[t,e]}).filter(function(t,e){return C(t[0],e)})});if(ot.enter().append("path").attr("class",function(t){return"nv-point nv-point-"+t[1]}).style("fill",function(t){return t.color}).style("stroke",function(t){return t.color}).attr("transform",function(e){return"translate("+t.utils.NaNtoZero(i(y(e[0],e[1])))+","+t.utils.NaNtoZero(o(b(e[0],e[1])))+")"}).attr("d",t.utils.symbol().type(function(t){return w(t[0])}).size(function(t){return m(x(t[0],t[1]))})),ot.exit().remove(),it.exit().selectAll("path.nv-point").watchTransition(U,"scatter exit").attr("transform",function(e){return"translate("+t.utils.NaNtoZero(g(y(e[0],e[1])))+","+t.utils.NaNtoZero(v(b(e[0],e[1])))+")"}).remove(),ot.filter(function(t){return K||n(t,"x","y")}).watchTransition(U,"scatter points").attr("transform",function(e){return"translate("+t.utils.NaNtoZero(g(y(e[0],e[1])))+","+t.utils.NaNtoZero(v(b(e[0],e[1])))+")"}),ot.filter(function(t){return K||n(t,"shape","size")}).watchTransition(U,"scatter points").attr("d",t.utils.symbol().type(function(t){return w(t[0])}).size(function(t){return m(x(t[0],t[1]))})),V){var at=it.selectAll(".nv-label").data(function(t){return t.values.map(function(t,e){return[t,e]}).filter(function(t,e){return C(t[0],e)})});at.enter().append("text").style("fill",function(t,e){return t.color}).style("stroke-opacity",0).style("fill-opacity",1).attr("transform",function(e){var n=t.utils.NaNtoZero(i(y(e[0],e[1])))+Math.sqrt(m(x(e[0],e[1]))/Math.PI)+2;return"translate("+n+","+t.utils.NaNtoZero(o(b(e[0],e[1])))+")"}).text(function(t,e){return t[0].label}),at.exit().remove(),it.exit().selectAll("path.nv-label").watchTransition(U,"scatter exit").attr("transform",function(e){var n=t.utils.NaNtoZero(g(y(e[0],e[1])))+Math.sqrt(m(x(e[0],e[1]))/Math.PI)+2;return"translate("+n+","+t.utils.NaNtoZero(v(b(e[0],e[1])))+")"}).remove(),at.each(function(t){d3.select(this).classed("nv-label",!0).classed("nv-label-"+t[1],!1).classed("hover",!1)}),at.watchTransition(U,"scatter labels").attr("transform",function(e){var n=t.utils.NaNtoZero(g(y(e[0],e[1])))+Math.sqrt(m(x(e[0],e[1]))/Math.PI)+2;return"translate("+n+","+t.utils.NaNtoZero(v(b(e[0],e[1])))+")"})}B?(clearTimeout(s),s=setTimeout(u,B)):u(),i=g.copy(),o=v.copy(),a=m.copy()}),U.renderEnd("scatter immediate"),r}var i,o,a,s,u,l={top:0,right:0,bottom:0,left:0},c=null,f=null,d=t.utils.defaultColor(),h=Math.floor(1e5*Math.random()),p=null,g=d3.scale.linear(),v=d3.scale.linear(),m=d3.scale.linear(),y=function(t){return t.x},b=function(t){return t.y},x=function(t){return t.size||1},w=function(t){return t.shape||"circle"},$=[],k=[],_=[],M=!0,C=function(t){return!t.notActive},S=!1,E=.1,A=!1,T=!0,N=!1,D=function(){return 25},O=null,j=null,I=null,L=null,P=null,F=null,q=!1,z=d3.dispatch("elementClick","elementDblClick","elementMouseover","elementMouseout","renderEnd"),W=!0,R=250,B=300,V=!1,H=!1,U=t.utils.renderWatch(z,R),Y=[16,256];return r.dispatch=z,r.options=t.utils.optionsFunc.bind(r),r._calls=new function(){this.clearHighlights=function(){return t.dom.write(function(){p.selectAll(".nv-point.hover").classed("hover",!1)}),null},this.highlightPoint=function(e,n,r){t.dom.write(function(){p.select(".nv-groups").selectAll(".nv-series-"+e).selectAll(".nv-point-"+n).classed("hover",r)})}},z.on("elementMouseover.point",function(t){M&&r._calls.highlightPoint(t.seriesIndex,t.pointIndex,!0)}),z.on("elementMouseout.point",function(t){M&&r._calls.highlightPoint(t.seriesIndex,t.pointIndex,!1)}),r._options=Object.create({},{width:{get:function(){return c},set:function(t){c=t}},height:{get:function(){return f},set:function(t){f=t}},xScale:{get:function(){return g},set:function(t){g=t}},yScale:{get:function(){return v},set:function(t){v=t}},pointScale:{get:function(){return m},set:function(t){m=t}},xDomain:{get:function(){return O},set:function(t){O=t}},yDomain:{get:function(){return j},set:function(t){j=t}},pointDomain:{get:function(){return P},set:function(t){P=t}},xRange:{get:function(){return I},set:function(t){I=t}},yRange:{get:function(){return L},set:function(t){L=t}},pointRange:{get:function(){return F},set:function(t){F=t}},forceX:{get:function(){return $},set:function(t){$=t}},forceY:{get:function(){return k},set:function(t){k=t}},forcePoint:{get:function(){return _},set:function(t){_=t}},interactive:{get:function(){return M},set:function(t){M=t}},pointActive:{get:function(){return C},set:function(t){C=t}},padDataOuter:{get:function(){return E},set:function(t){E=t}},padData:{get:function(){return S},set:function(t){S=t}},clipEdge:{get:function(){return A},set:function(t){A=t}},clipVoronoi:{get:function(){return T},set:function(t){T=t}},clipRadius:{get:function(){return D},set:function(t){D=t}},showVoronoi:{get:function(){return N},set:function(t){N=t}},id:{get:function(){return h},set:function(t){h=t}},interactiveUpdateDelay:{get:function(){return B},set:function(t){B=t}},showLabels:{get:function(){return V},set:function(t){V=t}},x:{get:function(){return y},set:function(t){y=d3.functor(t)}},y:{get:function(){return b},set:function(t){b=d3.functor(t)}},pointSize:{get:function(){return x},set:function(t){x=d3.functor(t)}},pointShape:{get:function(){return w},set:function(t){w=d3.functor(t)}},margin:{get:function(){return l},set:function(t){l.top=void 0!==t.top?t.top:l.top,l.right=void 0!==t.right?t.right:l.right,l.bottom=void 0!==t.bottom?t.bottom:l.bottom,l.left=void 0!==t.left?t.left:l.left}},duration:{get:function(){return R},set:function(t){R=t,U.reset(R)}},color:{get:function(){return d},set:function(e){d=t.utils.getColor(e)}},useVoronoi:{get:function(){return W},set:function(t){W=t,W===!1&&(T=!1)}}}),t.utils.initOptions(r),r},t.models.scatterChart=function(){"use strict";function e(M){return T.reset(),T.models(n),b&&T.models(r),x&&T.models(i),v&&T.models(a),m&&T.models(s),M.each(function(M){d=d3.select(this),t.utils.initSVG(d);var O=t.utils.availableWidth(c,d,l),j=t.utils.availableHeight(f,d,l);if(e.update=function(){0===C?d.call(e):d.transition().duration(C).call(e)},e.container=this,$.setter(D(M),e.update).getter(N(M)).update(),$.disabled=M.map(function(t){return!!t.disabled}),!k){var I;k={};for(I in $)$[I]instanceof Array?k[I]=$[I].slice(0):k[I]=$[I]}if(!(M&&M.length&&M.filter(function(t){return t.values.length}).length))return t.utils.noData(e,d),T.renderEnd("scatter immediate"),e;d.selectAll(".nv-noData").remove(),p=n.xScale(),g=n.yScale();var L=d.selectAll("g.nv-wrap.nv-scatterChart").data([M]),P=L.enter().append("g").attr("class","nvd3 nv-wrap nv-scatterChart nv-chart-"+n.id()),F=P.append("g"),q=L.select("g");if(F.append("rect").attr("class","nvd3 nv-background").style("pointer-events","none"),F.append("g").attr("class","nv-x nv-axis"),F.append("g").attr("class","nv-y nv-axis"),F.append("g").attr("class","nv-scatterWrap"),F.append("g").attr("class","nv-regressionLinesWrap"),F.append("g").attr("class","nv-distWrap"),F.append("g").attr("class","nv-legendWrap"),w&&q.select(".nv-y.nv-axis").attr("transform","translate("+O+",0)"),y){var z=O;o.width(z),L.select(".nv-legendWrap").datum(M).call(o),o.height()>l.top&&(l.top=o.height(),j=t.utils.availableHeight(f,d,l)),L.select(".nv-legendWrap").attr("transform","translate(0,"+-l.top+")")}else q.select(".nv-legendWrap").selectAll("*").remove();L.attr("transform","translate("+l.left+","+l.top+")"),n.width(O).height(j).color(M.map(function(t,e){return t.color=t.color||h(t,e),t.color}).filter(function(t,e){return!M[e].disabled})).showLabels(S),L.select(".nv-scatterWrap").datum(M.filter(function(t){return!t.disabled})).call(n),L.select(".nv-regressionLinesWrap").attr("clip-path","url(#nv-edge-clip-"+n.id()+")");var W=L.select(".nv-regressionLinesWrap").selectAll(".nv-regLines").data(function(t){return t});W.enter().append("g").attr("class","nv-regLines");var R=W.selectAll(".nv-regLine").data(function(t){return[t]});R.enter().append("line").attr("class","nv-regLine").style("stroke-opacity",0),R.filter(function(t){return t.intercept&&t.slope}).watchTransition(T,"scatterPlusLineChart: regline").attr("x1",p.range()[0]).attr("x2",p.range()[1]).attr("y1",function(t,e){return g(p.domain()[0]*t.slope+t.intercept)}).attr("y2",function(t,e){return g(p.domain()[1]*t.slope+t.intercept)}).style("stroke",function(t,e,n){return h(t,n)}).style("stroke-opacity",function(t,e){return t.disabled||"undefined"==typeof t.slope||"undefined"==typeof t.intercept?0:1}),b&&(r.scale(p)._ticks(t.utils.calcTicksX(O/100,M)).tickSize(-j,0),q.select(".nv-x.nv-axis").attr("transform","translate(0,"+g.range()[0]+")").call(r)),x&&(i.scale(g)._ticks(t.utils.calcTicksY(j/36,M)).tickSize(-O,0),q.select(".nv-y.nv-axis").call(i)),v&&(a.getData(n.x()).scale(p).width(O).color(M.map(function(t,e){return t.color||h(t,e)}).filter(function(t,e){return!M[e].disabled})),F.select(".nv-distWrap").append("g").attr("class","nv-distributionX"),q.select(".nv-distributionX").attr("transform","translate(0,"+g.range()[0]+")").datum(M.filter(function(t){return!t.disabled})).call(a)),m&&(s.getData(n.y()).scale(g).width(j).color(M.map(function(t,e){return t.color||h(t,e)}).filter(function(t,e){return!M[e].disabled})),F.select(".nv-distWrap").append("g").attr("class","nv-distributionY"),q.select(".nv-distributionY").attr("transform","translate("+(w?O:-s.size())+",0)").datum(M.filter(function(t){return!t.disabled})).call(s)),o.dispatch.on("stateChange",function(t){for(var n in t)$[n]=t[n];_.stateChange($),e.update()}),_.on("changeState",function(t){"undefined"!=typeof t.disabled&&(M.forEach(function(e,n){e.disabled=t.disabled[n]}),$.disabled=t.disabled),e.update()}),n.dispatch.on("elementMouseout.tooltip",function(t){u.hidden(!0),d.select(".nv-chart-"+n.id()+" .nv-series-"+t.seriesIndex+" .nv-distx-"+t.pointIndex).attr("y1",0),d.select(".nv-chart-"+n.id()+" .nv-series-"+t.seriesIndex+" .nv-disty-"+t.pointIndex).attr("x2",s.size())}),n.dispatch.on("elementMouseover.tooltip",function(t){d.select(".nv-series-"+t.seriesIndex+" .nv-distx-"+t.pointIndex).attr("y1",t.relativePos[1]-j),d.select(".nv-series-"+t.seriesIndex+" .nv-disty-"+t.pointIndex).attr("x2",t.relativePos[0]+a.size()),u.data(t).hidden(!1)}),E=p.copy(),A=g.copy()}),T.renderEnd("scatter with line immediate"),e}var n=t.models.scatter(),r=t.models.axis(),i=t.models.axis(),o=t.models.legend(),a=t.models.distribution(),s=t.models.distribution(),u=t.models.tooltip(),l={top:30,right:20,bottom:50,left:75},c=null,f=null,d=null,h=t.utils.defaultColor(),p=n.xScale(),g=n.yScale(),v=!1,m=!1,y=!0,b=!0,x=!0,w=!1,$=t.utils.state(),k=null,_=d3.dispatch("stateChange","changeState","renderEnd"),M=null,C=250,S=!1;n.xScale(p).yScale(g),r.orient("bottom").tickPadding(10),i.orient(w?"right":"left").tickPadding(10),a.axis("x"),s.axis("y"),u.headerFormatter(function(t,e){return r.tickFormat()(t,e)}).valueFormatter(function(t,e){return i.tickFormat()(t,e)});var E,A,T=t.utils.renderWatch(_,C),N=function(t){return function(){return{active:t.map(function(t){return!t.disabled})}}},D=function(t){return function(e){void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}};return e.dispatch=_,e.scatter=n,e.legend=o,e.xAxis=r,e.yAxis=i,e.distX=a,e.distY=s,e.tooltip=u,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return c},set:function(t){c=t}},height:{get:function(){return f},set:function(t){f=t}},container:{get:function(){return d},set:function(t){d=t}},showDistX:{get:function(){return v},set:function(t){v=t}},showDistY:{get:function(){return m},set:function(t){m=t}},showLegend:{get:function(){return y},set:function(t){y=t}},showXAxis:{get:function(){return b},set:function(t){b=t}},showYAxis:{get:function(){return x},set:function(t){x=t}},defaultState:{get:function(){return k},set:function(t){k=t}},noData:{get:function(){return M},set:function(t){M=t}},duration:{get:function(){return C},set:function(t){C=t}},showLabels:{get:function(){return S},set:function(t){S=t}},margin:{get:function(){return l},set:function(t){l.top=void 0!==t.top?t.top:l.top,l.right=void 0!==t.right?t.right:l.right,l.bottom=void 0!==t.bottom?t.bottom:l.bottom,l.left=void 0!==t.left?t.left:l.left}},rightAlignYAxis:{get:function(){return w},set:function(t){w=t,i.orient(t?"right":"left")}},color:{get:function(){return h},set:function(e){h=t.utils.getColor(e),o.color(h),a.color(h),s.color(h)}}}),t.utils.inheritOptions(e,n),t.utils.initOptions(e),e},t.models.sparkline=function(){"use strict";function e(c){return b.reset(),c.each(function(e){var c=s-a.left-a.right,y=u-a.top-a.bottom;l=d3.select(this),t.utils.initSVG(l),f.domain(n||d3.extent(e,h)).range(i||[0,c]),d.domain(r||d3.extent(e,p)).range(o||[y,0]);
-var b=l.selectAll("g.nv-wrap.nv-sparkline").data([e]),x=b.enter().append("g").attr("class","nvd3 nv-wrap nv-sparkline");x.append("g"),b.select("g");b.attr("transform","translate("+a.left+","+a.top+")");var w=b.selectAll("path").data(function(t){return[t]});w.enter().append("path"),w.exit().remove(),w.style("stroke",function(t,e){return t.color||g(t,e)}).attr("d",d3.svg.line().x(function(t,e){return f(h(t,e))}).y(function(t,e){return d(p(t,e))}));var $=b.selectAll("circle.nv-point").data(function(t){function e(e){if(e!=-1){var n=t[e];return n.pointIndex=e,n}return null}var n=t.map(function(t,e){return p(t,e)}),r=e(n.lastIndexOf(d.domain()[1])),i=e(n.indexOf(d.domain()[0])),o=e(n.length-1);return[v?i:null,v?r:null,m?o:null].filter(function(t){return null!=t})});$.enter().append("circle"),$.exit().remove(),$.attr("cx",function(t,e){return f(h(t,t.pointIndex))}).attr("cy",function(t,e){return d(p(t,t.pointIndex))}).attr("r",2).attr("class",function(t,e){return h(t,t.pointIndex)==f.domain()[1]?"nv-point nv-currentValue":p(t,t.pointIndex)==d.domain()[0]?"nv-point nv-noWatermark":"nv-point nv-maxValue"})}),b.renderEnd("sparkline immediate"),e}var n,r,i,o,a={top:2,right:0,bottom:2,left:0},s=400,u=32,l=null,c=!0,f=d3.scale.linear(),d=d3.scale.linear(),h=function(t){return t.x},p=function(t){return t.y},g=t.utils.getColor(["#000"]),v=!0,m=!0,y=d3.dispatch("renderEnd"),b=t.utils.renderWatch(y);return e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return s},set:function(t){s=t}},height:{get:function(){return u},set:function(t){u=t}},xDomain:{get:function(){return n},set:function(t){n=t}},yDomain:{get:function(){return r},set:function(t){r=t}},xRange:{get:function(){return i},set:function(t){i=t}},yRange:{get:function(){return o},set:function(t){o=t}},xScale:{get:function(){return f},set:function(t){f=t}},yScale:{get:function(){return d},set:function(t){d=t}},animate:{get:function(){return c},set:function(t){c=t}},showMinMaxPoints:{get:function(){return v},set:function(t){v=t}},showCurrentPoint:{get:function(){return m},set:function(t){m=t}},x:{get:function(){return h},set:function(t){h=d3.functor(t)}},y:{get:function(){return p},set:function(t){p=d3.functor(t)}},margin:{get:function(){return a},set:function(t){a.top=void 0!==t.top?t.top:a.top,a.right=void 0!==t.right?t.right:a.right,a.bottom=void 0!==t.bottom?t.bottom:a.bottom,a.left=void 0!==t.left?t.left:a.left}},color:{get:function(){return g},set:function(e){g=t.utils.getColor(e)}}}),e.dispatch=y,t.utils.initOptions(e),e},t.models.sparklinePlus=function(){"use strict";function e(g){return m.reset(),m.models(i),g.each(function(g){function v(){if(!l){var t=M.selectAll(".nv-hoverValue").data(u),e=t.enter().append("g").attr("class","nv-hoverValue").style("stroke-opacity",0).style("fill-opacity",0);t.exit().transition().duration(250).style("stroke-opacity",0).style("fill-opacity",0).remove(),t.attr("transform",function(t){return"translate("+n(i.x()(g[t],t))+",0)"}).transition().duration(250).style("stroke-opacity",1).style("fill-opacity",1),u.length&&(e.append("line").attr("x1",0).attr("y1",-o.top).attr("x2",0).attr("y2",x),e.append("text").attr("class","nv-xValue").attr("x",-6).attr("y",-o.top).attr("text-anchor","end").attr("dy",".9em"),M.select(".nv-hoverValue .nv-xValue").text(c(i.x()(g[u[0]],u[0]))),e.append("text").attr("class","nv-yValue").attr("x",6).attr("y",-o.top).attr("text-anchor","start").attr("dy",".9em"),M.select(".nv-hoverValue .nv-yValue").text(f(i.y()(g[u[0]],u[0]))))}}function m(){function t(t,e){for(var n=Math.abs(i.x()(t[0],0)-e),r=0,o=0;o<t.length;o++)Math.abs(i.x()(t[o],o)-e)<n&&(n=Math.abs(i.x()(t[o],o)-e),r=o);return r}if(!l){var e=d3.mouse(this)[0]-o.left;u=[t(g,Math.round(n.invert(e)))],v()}}var y=d3.select(this);t.utils.initSVG(y);var b=t.utils.availableWidth(a,y,o),x=t.utils.availableHeight(s,y,o);if(e.update=function(){y.call(e)},e.container=this,!g||!g.length)return t.utils.noData(e,y),e;y.selectAll(".nv-noData").remove();var w=i.y()(g[g.length-1],g.length-1);n=i.xScale(),r=i.yScale();var $=y.selectAll("g.nv-wrap.nv-sparklineplus").data([g]),k=$.enter().append("g").attr("class","nvd3 nv-wrap nv-sparklineplus"),_=k.append("g"),M=$.select("g");_.append("g").attr("class","nv-sparklineWrap"),_.append("g").attr("class","nv-valueWrap"),_.append("g").attr("class","nv-hoverArea"),$.attr("transform","translate("+o.left+","+o.top+")");var C=M.select(".nv-sparklineWrap");if(i.width(b).height(x),C.call(i),d){var S=M.select(".nv-valueWrap"),E=S.selectAll(".nv-currentValue").data([w]);E.enter().append("text").attr("class","nv-currentValue").attr("dx",p?-8:8).attr("dy",".9em").style("text-anchor",p?"end":"start"),E.attr("x",b+(p?o.right:0)).attr("y",h?function(t){return r(t)}:0).style("fill",i.color()(g[g.length-1],g.length-1)).text(f(w))}_.select(".nv-hoverArea").append("rect").on("mousemove",m).on("click",function(){l=!l}).on("mouseout",function(){u=[],v()}),M.select(".nv-hoverArea rect").attr("transform",function(t){return"translate("+-o.left+","+-o.top+")"}).attr("width",b+o.left+o.right).attr("height",x+o.top)}),m.renderEnd("sparklinePlus immediate"),e}var n,r,i=t.models.sparkline(),o={top:15,right:100,bottom:10,left:50},a=null,s=null,u=[],l=!1,c=d3.format(",r"),f=d3.format(",.2f"),d=!0,h=!0,p=!1,g=null,v=d3.dispatch("renderEnd"),m=t.utils.renderWatch(v);return e.dispatch=v,e.sparkline=i,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return a},set:function(t){a=t}},height:{get:function(){return s},set:function(t){s=t}},xTickFormat:{get:function(){return c},set:function(t){c=t}},yTickFormat:{get:function(){return f},set:function(t){f=t}},showLastValue:{get:function(){return d},set:function(t){d=t}},alignValue:{get:function(){return h},set:function(t){h=t}},rightAlignValue:{get:function(){return p},set:function(t){p=t}},noData:{get:function(){return g},set:function(t){g=t}},margin:{get:function(){return o},set:function(t){o.top=void 0!==t.top?t.top:o.top,o.right=void 0!==t.right?t.right:o.right,o.bottom=void 0!==t.bottom?t.bottom:o.bottom,o.left=void 0!==t.left?t.left:o.left}}}),t.utils.inheritOptions(e,i),t.utils.initOptions(e),e},t.models.stackedArea=function(){"use strict";function e(h){return w.reset(),w.models(y),h.each(function(h){var b=o-i.left-i.right,$=a-i.top-i.bottom;l=d3.select(this),t.utils.initSVG(l),n=y.xScale(),r=y.yScale();var k=h;h.forEach(function(t,e){t.seriesIndex=e,t.values=t.values.map(function(t,n){return t.index=n,t.seriesIndex=e,t})});var _=h.filter(function(t){return!t.disabled});h=d3.layout.stack().order(g).offset(p).values(function(t){return t.values}).x(c).y(f).out(function(t,e,n){t.display={y:n,y0:e}})(_);var M=l.selectAll("g.nv-wrap.nv-stackedarea").data([h]),C=M.enter().append("g").attr("class","nvd3 nv-wrap nv-stackedarea"),S=C.append("defs"),E=C.append("g"),A=M.select("g");E.append("g").attr("class","nv-areaWrap"),E.append("g").attr("class","nv-scatterWrap"),M.attr("transform","translate("+i.left+","+i.top+")"),0==y.forceY().length&&y.forceY().push(0),y.width(b).height($).x(c).y(function(t){if(void 0!==t.display)return t.display.y+t.display.y0}).color(h.map(function(t,e){return t.color=t.color||s(t,t.seriesIndex),t.color}));var T=A.select(".nv-scatterWrap").datum(h);T.call(y),S.append("clipPath").attr("id","nv-edge-clip-"+u).append("rect"),M.select("#nv-edge-clip-"+u+" rect").attr("width",b).attr("height",$),A.attr("clip-path",m?"url(#nv-edge-clip-"+u+")":"");var N=d3.svg.area().defined(d).x(function(t,e){return n(c(t,e))}).y0(function(t){return r(t.display.y0)}).y1(function(t){return r(t.display.y+t.display.y0)}).interpolate(v),D=d3.svg.area().defined(d).x(function(t,e){return n(c(t,e))}).y0(function(t){return r(t.display.y0)}).y1(function(t){return r(t.display.y0)}),O=A.select(".nv-areaWrap").selectAll("path.nv-area").data(function(t){return t});O.enter().append("path").attr("class",function(t,e){return"nv-area nv-area-"+e}).attr("d",function(t,e){return D(t.values,t.seriesIndex)}).on("mouseover",function(t,e){d3.select(this).classed("hover",!0),x.areaMouseover({point:t,series:t.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:t.seriesIndex})}).on("mouseout",function(t,e){d3.select(this).classed("hover",!1),x.areaMouseout({point:t,series:t.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:t.seriesIndex})}).on("click",function(t,e){d3.select(this).classed("hover",!1),x.areaClick({point:t,series:t.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:t.seriesIndex})}),O.exit().remove(),O.style("fill",function(t,e){return t.color||s(t,t.seriesIndex)}).style("stroke",function(t,e){return t.color||s(t,t.seriesIndex)}),O.watchTransition(w,"stackedArea path").attr("d",function(t,e){return N(t.values,e)}),y.dispatch.on("elementMouseover.area",function(t){A.select(".nv-chart-"+u+" .nv-area-"+t.seriesIndex).classed("hover",!0)}),y.dispatch.on("elementMouseout.area",function(t){A.select(".nv-chart-"+u+" .nv-area-"+t.seriesIndex).classed("hover",!1)}),e.d3_stackedOffset_stackPercent=function(t){var e,n,r,i=t.length,o=t[0].length,a=[];for(n=0;n<o;++n){for(e=0,r=0;e<k.length;e++)r+=f(k[e].values[n]);if(r)for(e=0;e<i;e++)t[e][n][1]/=r;else for(e=0;e<i;e++)t[e][n][1]=0}for(n=0;n<o;++n)a[n]=0;return a}}),w.renderEnd("stackedArea immediate"),e}var n,r,i={top:0,right:0,bottom:0,left:0},o=960,a=500,s=t.utils.defaultColor(),u=Math.floor(1e5*Math.random()),l=null,c=function(t){return t.x},f=function(t){return t.y},d=function(t,e){return!isNaN(f(t,e))&&null!==f(t,e)},h="stack",p="zero",g="default",v="linear",m=!1,y=t.models.scatter(),b=250,x=d3.dispatch("areaClick","areaMouseover","areaMouseout","renderEnd","elementClick","elementMouseover","elementMouseout");y.pointSize(2.2).pointDomain([2.2,2.2]);var w=t.utils.renderWatch(x,b);return e.dispatch=x,e.scatter=y,y.dispatch.on("elementClick",function(){x.elementClick.apply(this,arguments)}),y.dispatch.on("elementMouseover",function(){x.elementMouseover.apply(this,arguments)}),y.dispatch.on("elementMouseout",function(){x.elementMouseout.apply(this,arguments)}),e.interpolate=function(t){return arguments.length?(v=t,e):v},e.duration=function(t){return arguments.length?(b=t,w.reset(b),y.duration(b),e):b},e.dispatch=x,e.scatter=y,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return o},set:function(t){o=t}},height:{get:function(){return a},set:function(t){a=t}},defined:{get:function(){return d},set:function(t){d=t}},clipEdge:{get:function(){return m},set:function(t){m=t}},offset:{get:function(){return p},set:function(t){p=t}},order:{get:function(){return g},set:function(t){g=t}},interpolate:{get:function(){return v},set:function(t){v=t}},x:{get:function(){return c},set:function(t){c=d3.functor(t)}},y:{get:function(){return f},set:function(t){f=d3.functor(t)}},margin:{get:function(){return i},set:function(t){i.top=void 0!==t.top?t.top:i.top,i.right=void 0!==t.right?t.right:i.right,i.bottom=void 0!==t.bottom?t.bottom:i.bottom,i.left=void 0!==t.left?t.left:i.left}},color:{get:function(){return s},set:function(e){s=t.utils.getColor(e)}},style:{get:function(){return h},set:function(t){switch(h=t){case"stack":e.offset("zero"),e.order("default");break;case"stream":e.offset("wiggle"),e.order("inside-out");break;case"stream-center":e.offset("silhouette"),e.order("inside-out");break;case"expand":e.offset("expand"),e.order("default");break;case"stack_percent":e.offset(e.d3_stackedOffset_stackPercent),e.order("default")}}},duration:{get:function(){return b},set:function(t){b=t,w.reset(b),y.duration(b)}}}),t.utils.inheritOptions(e,y),t.utils.initOptions(e),e},t.models.stackedAreaChart=function(){"use strict";function e(c){return I.reset(),I.models(i),y&&I.models(o),b&&I.models(a),c.each(function(c){function S(){y&&Y.select(".nv-focus .nv-x.nv-axis").attr("transform","translate(0,"+B+")").transition().duration(D).call(o)}function I(){if(b){if("expand"===i.style()||"stack_percent"===i.style()){var t=a.tickFormat();O&&t===q||(O=t),a.tickFormat(q)}else O&&(a.tickFormat(O),O=null);Y.select(".nv-focus .nv-y.nv-axis").transition().duration(0).call(a)}}function z(t){var e=Y.select(".nv-focus .nv-stackedWrap").datum(c.filter(function(t){return!t.disabled}).map(function(e,n){return{key:e.key,area:e.area,classed:e.classed,values:e.values.filter(function(e,n){return i.x()(e,n)>=t[0]&&i.x()(e,n)<=t[1]}),disableTooltip:e.disableTooltip}}));e.transition().duration(D).call(i),S(),I()}var W=d3.select(this);t.utils.initSVG(W);var R=t.utils.availableWidth(h,W,d),B=t.utils.availableHeight(p,W,d)-(w?f.height():0);if(e.update=function(){W.transition().duration(D).call(e)},e.container=this,M.setter(F(c),e.update).getter(P(c)).update(),M.disabled=c.map(function(t){return!!t.disabled}),!C){var V;C={};for(V in M)M[V]instanceof Array?C[V]=M[V].slice(0):C[V]=M[V]}if(!(c&&c.length&&c.filter(function(t){return t.values.length}).length))return t.utils.noData(e,W),e;W.selectAll(".nv-noData").remove(),n=i.xScale(),r=i.yScale();var H=W.selectAll("g.nv-wrap.nv-stackedAreaChart").data([c]),U=H.enter().append("g").attr("class","nvd3 nv-wrap nv-stackedAreaChart").append("g"),Y=H.select("g");U.append("g").attr("class","nv-legendWrap"),U.append("g").attr("class","nv-controlsWrap");var G=U.append("g").attr("class","nv-focus");G.append("g").attr("class","nv-background").append("rect"),G.append("g").attr("class","nv-x nv-axis"),G.append("g").attr("class","nv-y nv-axis"),G.append("g").attr("class","nv-stackedWrap"),G.append("g").attr("class","nv-interactive");U.append("g").attr("class","nv-focusWrap");if(m){var X=v?R-A:R;s.width(X),Y.select(".nv-legendWrap").datum(c).call(s),s.height()>d.top&&(d.top=s.height(),B=t.utils.availableHeight(p,W,d)-(w?f.height():0)),Y.select(".nv-legendWrap").attr("transform","translate("+(R-X)+","+-d.top+")")}else Y.select(".nv-legendWrap").selectAll("*").remove();if(v){var Z=[{key:N.stacked||"Stacked",metaKey:"Stacked",disabled:"stack"!=i.style(),style:"stack"},{key:N.stream||"Stream",metaKey:"Stream",disabled:"stream"!=i.style(),style:"stream"},{key:N.expanded||"Expanded",metaKey:"Expanded",disabled:"expand"!=i.style(),style:"expand"},{key:N.stack_percent||"Stack %",metaKey:"Stack_Percent",disabled:"stack_percent"!=i.style(),style:"stack_percent"}];A=T.length/3*260,Z=Z.filter(function(t){return T.indexOf(t.metaKey)!==-1}),u.width(A).color(["#444","#444","#444"]),Y.select(".nv-controlsWrap").datum(Z).call(u),Math.max(u.height(),s.height())>d.top&&(d.top=Math.max(u.height(),s.height()),B=t.utils.availableHeight(p,W,d)),Y.select(".nv-controlsWrap").attr("transform","translate(0,"+-d.top+")")}else Y.select(".nv-controlsWrap").selectAll("*").remove();H.attr("transform","translate("+d.left+","+d.top+")"),x&&Y.select(".nv-y.nv-axis").attr("transform","translate("+R+",0)"),$&&(l.width(R).height(B).margin({left:d.left,top:d.top}).svgContainer(W).xScale(n),H.select(".nv-interactive").call(l)),Y.select(".nv-focus .nv-background rect").attr("width",R).attr("height",B),i.width(R).height(B).color(c.map(function(t,e){return t.color||g(t,e)}).filter(function(t,e){return!c[e].disabled}));var Q=Y.select(".nv-focus .nv-stackedWrap").datum(c.filter(function(t){return!t.disabled}));if(y&&o.scale(n)._ticks(t.utils.calcTicksX(R/100,c)).tickSize(-B,0),b){var K;K="wiggle"===i.offset()?0:t.utils.calcTicksY(B/36,c),a.scale(r)._ticks(K).tickSize(-R,0)}if(w){f.width(R),Y.select(".nv-focusWrap").attr("transform","translate(0,"+(B+d.bottom+f.margin().top)+")").datum(c.filter(function(t){return!t.disabled})).call(f);var J=f.brush.empty()?f.xDomain():f.brush.extent();null!==J&&z(J)}else Q.transition().call(i),S(),I();i.dispatch.on("areaClick.toggle",function(t){1===c.filter(function(t){return!t.disabled}).length?c.forEach(function(t){t.disabled=!1}):c.forEach(function(e,n){e.disabled=n!=t.seriesIndex}),M.disabled=c.map(function(t){return!!t.disabled}),E.stateChange(M),e.update()}),s.dispatch.on("stateChange",function(t){for(var n in t)M[n]=t[n];E.stateChange(M),e.update()}),u.dispatch.on("legendClick",function(t,n){t.disabled&&(Z=Z.map(function(t){return t.disabled=!0,t}),t.disabled=!1,i.style(t.style),M.style=i.style(),E.stateChange(M),e.update())}),l.dispatch.on("elementMousemove",function(n){i.clearHighlights();var r,o,a,s=[],u=0,f=!0;if(c.filter(function(t,e){return t.seriesIndex=e,!t.disabled}).forEach(function(l,c){o=t.interactiveBisect(l.values,n.pointXValue,e.x());var d=l.values[o],h=e.y()(d,o);if(null!=h&&i.highlightPoint(c,o,!0),"undefined"!=typeof d){"undefined"==typeof r&&(r=d),"undefined"==typeof a&&(a=e.xScale()(e.x()(d,o)));var p="expand"==i.style()?d.display.y:e.y()(d,o);s.push({key:l.key,value:p,color:g(l,l.seriesIndex),point:d}),k&&"expand"!=i.style()&&null!=p&&(u+=p,f=!1)}}),s.reverse(),s.length>2){var d=e.yScale().invert(n.mouseY),h=null;s.forEach(function(t,e){d=Math.abs(d);var n=Math.abs(t.point.display.y0),r=Math.abs(t.point.display.y);if(d>=n&&d<=r+n)return void(h=e)}),null!=h&&(s[h].highlight=!0)}k&&"expand"!=i.style()&&s.length>=2&&!f&&s.push({key:_,value:u,total:!0});var p=e.x()(r,o),v=l.tooltip.valueFormatter();"expand"===i.style()||"stack_percent"===i.style()?(j||(j=v),v=d3.format(".1%")):j&&(v=j,j=null),l.tooltip.valueFormatter(v).data({value:p,series:s})(),l.renderGuideLine(a)}),l.dispatch.on("elementMouseout",function(t){i.clearHighlights()}),f.dispatch.on("onBrush",function(t){z(t)}),E.on("changeState",function(t){"undefined"!=typeof t.disabled&&c.length===t.disabled.length&&(c.forEach(function(e,n){e.disabled=t.disabled[n]}),M.disabled=t.disabled),"undefined"!=typeof t.style&&(i.style(t.style),L=t.style),e.update()})}),I.renderEnd("stacked Area chart immediate"),e}var n,r,i=t.models.stackedArea(),o=t.models.axis(),a=t.models.axis(),s=t.models.legend(),u=t.models.legend(),l=t.interactiveGuideline(),c=t.models.tooltip(),f=t.models.focus(t.models.stackedArea()),d={top:30,right:25,bottom:50,left:60},h=null,p=null,g=t.utils.defaultColor(),v=!0,m=!0,y=!0,b=!0,x=!1,w=!1,$=!1,k=!0,_="TOTAL",M=t.utils.state(),C=null,S=null,E=d3.dispatch("stateChange","changeState","renderEnd"),A=250,T=["Stacked","Stream","Expanded"],N={},D=250;M.style=i.style(),o.orient("bottom").tickPadding(7),a.orient(x?"right":"left"),c.headerFormatter(function(t,e){return o.tickFormat()(t,e)}).valueFormatter(function(t,e){return a.tickFormat()(t,e)}),l.tooltip.headerFormatter(function(t,e){return o.tickFormat()(t,e)}).valueFormatter(function(t,e){return null==t?"N/A":a.tickFormat()(t,e)});var O=null,j=null;u.updateState(!1);var I=t.utils.renderWatch(E),L=i.style(),P=function(t){return function(){return{active:t.map(function(t){return!t.disabled}),style:i.style()}}},F=function(t){return function(e){void 0!==e.style&&(L=e.style),void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}},q=d3.format("%");return i.dispatch.on("elementMouseover.tooltip",function(t){t.point.x=i.x()(t.point),t.point.y=i.y()(t.point),c.data(t).hidden(!1)}),i.dispatch.on("elementMouseout.tooltip",function(t){c.hidden(!0)}),e.dispatch=E,e.stacked=i,e.legend=s,e.controls=u,e.xAxis=o,e.x2Axis=f.xAxis,e.yAxis=a,e.y2Axis=f.yAxis,e.interactiveLayer=l,e.tooltip=c,e.focus=f,e.dispatch=E,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return h},set:function(t){h=t}},height:{get:function(){return p},set:function(t){p=t}},showLegend:{get:function(){return m},set:function(t){m=t}},showXAxis:{get:function(){return y},set:function(t){y=t}},showYAxis:{get:function(){return b},set:function(t){b=t}},defaultState:{get:function(){return C},set:function(t){C=t}},noData:{get:function(){return S},set:function(t){S=t}},showControls:{get:function(){return v},set:function(t){v=t}},controlLabels:{get:function(){return N},set:function(t){N=t}},controlOptions:{get:function(){return T},set:function(t){T=t}},showTotalInTooltip:{get:function(){return k},set:function(t){k=t}},totalLabel:{get:function(){return _},set:function(t){_=t}},focusEnable:{get:function(){return w},set:function(t){w=t}},focusHeight:{get:function(){return f.height()},set:function(t){f.height(t)}},brushExtent:{get:function(){return f.brushExtent()},set:function(t){f.brushExtent(t)}},margin:{get:function(){return d},set:function(t){d.top=void 0!==t.top?t.top:d.top,d.right=void 0!==t.right?t.right:d.right,d.bottom=void 0!==t.bottom?t.bottom:d.bottom,d.left=void 0!==t.left?t.left:d.left}},focusMargin:{get:function(){return f.margin},set:function(t){f.margin.top=void 0!==t.top?t.top:f.margin.top,f.margin.right=void 0!==t.right?t.right:f.margin.right,f.margin.bottom=void 0!==t.bottom?t.bottom:f.margin.bottom,f.margin.left=void 0!==t.left?t.left:f.margin.left}},duration:{get:function(){return D},set:function(t){D=t,I.reset(D),i.duration(D),o.duration(D),a.duration(D)}},color:{get:function(){return g},set:function(e){g=t.utils.getColor(e),s.color(g),i.color(g),f.color(g)}},x:{get:function(){return i.x()},set:function(t){i.x(t),f.x(t)}},y:{get:function(){return i.y()},set:function(t){i.y(t),f.y(t)}},rightAlignYAxis:{get:function(){return x},set:function(t){x=t,a.orient(x?"right":"left")}},useInteractiveGuideline:{get:function(){return $},set:function(t){$=!!t,e.interactive(!t),e.useVoronoi(!t),i.scatter.interactive(!t)}}}),t.utils.inheritOptions(e,i),t.utils.initOptions(e),e},t.models.stackedAreaWithFocusChart=function(){return t.models.stackedAreaChart().margin({bottom:30}).focusEnable(!0)},t.models.sunburst=function(){"use strict";function e(t){var e=n(t);return e>90?180:0}function n(t){var e=Math.max(0,Math.min(2*Math.PI,N(t.x))),n=Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx))),r=(e+n)/2*(180/Math.PI)-90;return r}function r(t){var e=Math.max(0,Math.min(2*Math.PI,N(t.x))),n=Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx)));return(n-e)/(2*Math.PI)}function i(t){var e=Math.max(0,Math.min(2*Math.PI,N(t.x))),n=Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx))),r=n-e;return r>M}function o(t,e){var n=d3.interpolate(N.domain(),[f.x,f.x+f.dx]),r=d3.interpolate(D.domain(),[f.y,1]),i=d3.interpolate(D.range(),[f.y?20:0,p]);return 0===e?function(){return I(t)}:function(e){return N.domain(n(e)),D.domain(r(e)).range(i(e)),I(t)}}function a(t){var e=d3.interpolate({x:t.x0,dx:t.dx0,y:t.y0,dy:t.dy0},t);return function(n){var r=e(n);return t.x0=r.x,t.dx0=r.dx,t.y0=r.y,t.dy0=r.dy,I(r)}}function s(t){var e=S(t);j[e]||(j[e]={});var n=j[e];n.dx=t.dx,n.x=t.x,n.dy=t.dy,n.y=t.y}function u(t){t.forEach(function(t){var e=S(t),n=j[e];n?(t.dx0=n.dx,t.x0=n.x,t.dy0=n.dy,t.y0=n.y):(t.dx0=t.dx,t.x0=t.x,t.dy0=t.dy,t.y0=t.y),s(t)})}function l(t){var r=w.selectAll("text"),a=w.selectAll("path");r.transition().attr("opacity",0),f=t,a.transition().duration(A).attrTween("d",o).each("end",function(r){if(r.x>=t.x&&r.x<t.x+t.dx&&r.depth>=t.depth){var o=d3.select(this.parentNode),a=o.select("text");a.transition().duration(A).text(function(t){return _(t)}).attr("opacity",function(t){return i(t)?1:0}).attr("transform",function(){var i=this.getBBox().width;if(0===r.depth)return"translate("+i/2*-1+",0)";if(r.depth===t.depth)return"translate("+(D(r.y)+5)+",0)";var o=n(r),a=e(r);return 0===a?"rotate("+o+")translate("+(D(r.y)+5)+",0)":"rotate("+o+")translate("+(D(r.y)+i+5)+",0)rotate("+a+")"})}})}function c(o){return L.reset(),o.each(function(o){w=d3.select(this),d=t.utils.availableWidth(v,w,g),h=t.utils.availableHeight(m,w,g),p=Math.min(d,h)/2,D.range([0,p]);var s=w.select("g.nvd3.nv-wrap.nv-sunburst");s[0][0]?s.attr("transform","translate("+(d/2+g.left+g.right)+","+(h/2+g.top+g.bottom)+")"):s=w.append("g").attr("class","nvd3 nv-wrap nv-sunburst nv-chart-"+x).attr("transform","translate("+(d/2+g.left+g.right)+","+(h/2+g.top+g.bottom)+")"),w.on("click",function(t,e){T.chartClick({data:t,index:e,pos:d3.event,id:x})}),O.value(b[y]||b.count);var c=O.nodes(o[0]).reverse();u(c);var f=s.selectAll(".arc-container").data(c,S),M=f.enter().append("g").attr("class","arc-container");M.append("path").attr("d",I).style("fill",function(t){return t.color?t.color:$(E?(t.children?t:t.parent).name:t.name)}).style("stroke","#FFF").on("click",l).on("mouseover",function(t,e){d3.select(this).classed("hover",!0).style("opacity",.8),T.elementMouseover({data:t,color:d3.select(this).style("fill"),percent:r(t)})}).on("mouseout",function(t,e){d3.select(this).classed("hover",!1).style("opacity",1),T.elementMouseout({data:t})}).on("mousemove",function(t,e){T.elementMousemove({data:t})}),f.each(function(t){d3.select(this).select("path").transition().duration(A).attrTween("d",a)}),k&&(f.selectAll("text").remove(),f.append("text").text(function(t){return _(t)}).transition().duration(A).attr("opacity",function(t){return i(t)?1:0}).attr("transform",function(t){var r=this.getBBox().width;if(0===t.depth)return"rotate(0)translate("+r/2*-1+",0)";var i=n(t),o=e(t);return 0===o?"rotate("+i+")translate("+(D(t.y)+5)+",0)":"rotate("+i+")translate("+(D(t.y)+r+5)+",0)rotate("+o+")"})),l(c[c.length-1]),f.exit().transition().duration(A).attr("opacity",0).each("end",function(t){var e=S(t);j[e]=void 0}).remove()}),L.renderEnd("sunburst immediate"),c}var f,d,h,p,g={top:0,right:0,bottom:0,left:0},v=600,m=600,y="count",b={count:function(t){return 1},value:function(t){return t.value||t.size},size:function(t){return t.value||t.size}},x=Math.floor(1e4*Math.random()),w=null,$=t.utils.defaultColor(),k=!1,_=function(t){return"count"===y?t.name+" #"+t.value:t.name+" "+(t.value||t.size)},M=.02,C=function(t,e){return t.name>e.name},S=function(t,e){return t.name},E=!0,A=500,T=d3.dispatch("chartClick","elementClick","elementDblClick","elementMousemove","elementMouseover","elementMouseout","renderEnd"),N=d3.scale.linear().range([0,2*Math.PI]),D=d3.scale.sqrt(),O=d3.layout.partition().sort(C),j={},I=d3.svg.arc().startAngle(function(t){return Math.max(0,Math.min(2*Math.PI,N(t.x)))}).endAngle(function(t){return Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx)))}).innerRadius(function(t){return Math.max(0,D(t.y))}).outerRadius(function(t){return Math.max(0,D(t.y+t.dy))}),L=t.utils.renderWatch(T);return c.dispatch=T,c.options=t.utils.optionsFunc.bind(c),c._options=Object.create({},{width:{get:function(){return v},set:function(t){v=t}},height:{get:function(){return m},set:function(t){m=t}},mode:{get:function(){return y},set:function(t){y=t}},id:{get:function(){return x},set:function(t){x=t}},duration:{get:function(){return A},set:function(t){A=t}},groupColorByParent:{get:function(){return E},set:function(t){E=!!t}},showLabels:{get:function(){return k},set:function(t){k=!!t}},labelFormat:{get:function(){return _},set:function(t){_=t}},labelThreshold:{get:function(){return M},set:function(t){M=t}},sort:{get:function(){return C},set:function(t){C=t}},key:{get:function(){return S},set:function(t){S=t}},margin:{get:function(){return g},set:function(t){g.top=void 0!=t.top?t.top:g.top,g.right=void 0!=t.right?t.right:g.right,g.bottom=void 0!=t.bottom?t.bottom:g.bottom,g.left=void 0!=t.left?t.left:g.left}},color:{get:function(){return $},set:function(e){$=t.utils.getColor(e)}}}),t.utils.initOptions(c),c},t.models.sunburstChart=function(){"use strict";function e(r){return h.reset(),h.models(n),r.each(function(r){var s=d3.select(this);t.utils.initSVG(s);var u=t.utils.availableWidth(o,s,i),l=t.utils.availableHeight(a,s,i);return e.update=function(){0===f?s.call(e):s.transition().duration(f).call(e)},e.container=s,r&&r.length?(s.selectAll(".nv-noData").remove(),n.width(u).height(l).margin(i),void s.call(n)):(t.utils.noData(e,s),e)}),h.renderEnd("sunburstChart immediate"),e}var n=t.models.sunburst(),r=t.models.tooltip(),i={top:30,right:20,bottom:20,left:20},o=null,a=null,s=t.utils.defaultColor(),u=!1,l=(Math.round(1e5*Math.random()),null),c=null,f=250,d=d3.dispatch("stateChange","changeState","renderEnd"),h=t.utils.renderWatch(d);return r.duration(0).headerEnabled(!1).valueFormatter(function(t){return t}),n.dispatch.on("elementMouseover.tooltip",function(t){t.series={key:t.data.name,value:t.data.value||t.data.size,color:t.color,percent:t.percent},u||(delete t.percent,delete t.series.percent),r.data(t).hidden(!1)}),n.dispatch.on("elementMouseout.tooltip",function(t){r.hidden(!0)}),n.dispatch.on("elementMousemove.tooltip",function(t){r()}),e.dispatch=d,e.sunburst=n,e.tooltip=r,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{noData:{get:function(){return c},set:function(t){c=t}},defaultState:{get:function(){return l},set:function(t){l=t}},showTooltipPercent:{get:function(){return u},set:function(t){u=t}},color:{get:function(){return s},set:function(t){s=t,n.color(s)}},duration:{get:function(){return f},set:function(t){f=t,h.reset(f),n.duration(f)}},margin:{get:function(){return i},set:function(t){i.top=void 0!==t.top?t.top:i.top,i.right=void 0!==t.right?t.right:i.right,i.bottom=void 0!==t.bottom?t.bottom:i.bottom,i.left=void 0!==t.left?t.left:i.left,n.margin(i)}}}),t.utils.inheritOptions(e,n),t.utils.initOptions(e),e},t.version="1.8.4"}(),function(){var t=this,e="addEventListener",n="removeEventListener",r="getBoundingClientRect",i=t.attachEvent&&!t[e],o=t.document,a=function(){for(var t,e=["","-webkit-","-moz-","-o-"],n=0;n<e.length;n++)if(t=o.createElement("div"),t.style.cssText="width:"+e[n]+"calc(9px)",t.style.length)return e[n]+"calc"}(),s=function(t){return"string"==typeof t||t instanceof String?o.querySelector(t):t},u=function(u,l){var c,f,d,h,p,g,v,m,y=[];l="undefined"!=typeof l?l:{},l.gutterSize||(l.gutterSize=10),l.minSize||(l.minSize=100),l.snapOffset||(l.snapOffset=30),l.direction||(l.direction="horizontal"),"horizontal"==l.direction?(c="width",d="clientWidth",h="clientX",p="left",g="gutter gutter-horizontal",v="paddingLeft",m="paddingRight",l.cursor||(l.cursor="ew-resize")):"vertical"==l.direction&&(c="height",d="clientHeight",h="clientY",p="top",g="gutter gutter-vertical",v="paddingTop",m="paddingBottom",l.cursor||(l.cursor="ns-resize"));var b=function(n){var r=this,i=r.a,o=r.b;!r.dragging&&l.onDragStart&&l.onDragStart(),n.preventDefault(),r.dragging=!0,r.move=w.bind(r),r.stop=x.bind(r),t[e]("mouseup",r.stop),t[e]("touchend",r.stop),t[e]("touchcancel",r.stop),r.parent[e]("mousemove",r.move),r.parent[e]("touchmove",r.move),i[e]("selectstart",S),i[e]("dragstart",S),o[e]("selectstart",S),o[e]("dragstart",S),i.style.userSelect="none",i.style.webkitUserSelect="none",i.style.MozUserSelect="none",i.style.pointerEvents="none",o.style.userSelect="none",o.style.webkitUserSelect="none",o.style.MozUserSelect="none",o.style.pointerEvents="none",r.gutter.style.cursor=l.cursor,r.parent.style.cursor=l.cursor,$.call(r)},x=function(){var e=this,r=e.a,i=e.b;e.dragging&&l.onDragEnd&&l.onDragEnd(),e.dragging=!1,t[n]("mouseup",e.stop),t[n]("touchend",e.stop),t[n]("touchcancel",e.stop),e.parent[n]("mousemove",e.move),e.parent[n]("touchmove",e.move),delete e.stop,delete e.move,r[n]("selectstart",S),r[n]("dragstart",S),i[n]("selectstart",S),i[n]("dragstart",S),r.style.userSelect="",r.style.webkitUserSelect="",r.style.MozUserSelect="",r.style.pointerEvents="",i.style.userSelect="",i.style.webkitUserSelect="",i.style.MozUserSelect="",i.style.pointerEvents="",e.gutter.style.cursor="",e.parent.style.cursor=""},w=function(t){var e;this.dragging&&(e="touches"in t?t.touches[0][h]-this.start:t[h]-this.start,e<=this.aMin+l.snapOffset?e=this.aMin:e>=this.size-this.bMin-l.snapOffset&&(e=this.size-this.bMin),k.call(this,e),l.onDrag&&l.onDrag())},$=function(){var e=t.getComputedStyle(this.parent),n=this.parent[d]-parseFloat(e[v])-parseFloat(e[m]);this.size=this.a[r]()[c]+this.b[r]()[c]+this.aGutterSize+this.bGutterSize,this.percentage=Math.min(this.size/n*100,100),this.start=this.a[r]()[p]},k=function(t){this.a.style[c]=a+"("+t/this.size*this.percentage+"% - "+this.aGutterSize+"px)",this.b.style[c]=a+"("+(this.percentage-t/this.size*this.percentage)+"% - "+this.bGutterSize+"px)"},_=function(){var t=this,e=t.a,n=t.b;e[r]()[c]<t.aMin?(e.style[c]=t.aMin-t.aGutterSize+"px",n.style[c]=t.size-t.aMin-t.aGutterSize+"px"):n[r]()[c]<t.bMin&&(e.style[c]=t.size-t.bMin-t.bGutterSize+"px",n.style[c]=t.bMin-t.bGutterSize+"px")},M=function(){var t=this,e=t.a,n=t.b;n[r]()[c]<t.bMin?(e.style[c]=t.size-t.bMin-t.bGutterSize+"px",
-n.style[c]=t.bMin-t.bGutterSize+"px"):e[r]()[c]<t.aMin&&(e.style[c]=t.aMin-t.aGutterSize+"px",n.style[c]=t.size-t.aMin-t.aGutterSize+"px")},C=function(t){for(var e=0;e<t.length;e++)$.call(t[e]),_.call(t[e]);for(e=t.length-1;e>=0;e--)$.call(t[e]),M.call(t[e])},S=function(){return!1},E=s(u[0]).parentNode;if(!l.sizes){var A=100/u.length;for(l.sizes=[],f=0;f<u.length;f++)l.sizes.push(A)}if(!Array.isArray(l.minSize)){var T=[];for(f=0;f<u.length;f++)T.push(l.minSize);l.minSize=T}for(f=0;f<u.length;f++){var N,D,O=s(u[f]),j=1==f,I=f==u.length-1,L=l.gutterSize;if(f>0&&(D={a:s(u[f-1]),b:O,aMin:l.minSize[f-1],bMin:l.minSize[f],dragging:!1,parent:E,isFirst:j,isLast:I,direction:l.direction},D.aGutterSize=l.gutterSize,D.bGutterSize=l.gutterSize,j&&(D.aGutterSize=l.gutterSize/2),I&&(D.bGutterSize=l.gutterSize/2)),i)N="string"==typeof l.sizes[f]||l.sizes[f]instanceof String?l.sizes[f]:l.sizes[f]+"%";else{if(f>0){var P=o.createElement("div");P.className=g,P.style[c]=l.gutterSize+"px",P[e]("mousedown",b.bind(D)),P[e]("touchstart",b.bind(D)),E.insertBefore(P,O),D.gutter=P}0!==f&&f!=u.length-1||(L=l.gutterSize/2),N="string"==typeof l.sizes[f]||l.sizes[f]instanceof String?l.sizes[f]:a+"("+l.sizes[f]+"% - "+L+"px)"}O.style[c]=N,f>0&&y.push(D)}C(y)};"undefined"!=typeof exports?("undefined"!=typeof module&&module.exports&&(exports=module.exports=u),exports.Split=u):t.Split=u}.call(window),function(){"use strict";function t(t,e){return t.module("angularMoment",[]).constant("angularMomentConfig",{preprocess:null,timezone:"",format:null,statefulFilters:!0}).constant("moment",e).constant("amTimeAgoConfig",{withoutSuffix:!1,serverTime:null,titleFormat:null,fullDateThreshold:null,fullDateFormat:null}).directive("amTimeAgo",["$window","moment","amMoment","amTimeAgoConfig","angularMomentConfig",function(e,n,r,i,o){return function(a,s,u){function l(){var t;if(g)t=g;else if(i.serverTime){var e=(new Date).getTime(),r=e-$+i.serverTime;t=n(r)}else t=n();return t}function c(){v&&(e.clearTimeout(v),v=null)}function f(t){var n=l().diff(t,"day"),r=x&&n>=x;if(r?s.text(t.format(w)):s.text(t.from(l(),y)),b&&!s.attr("title")&&s.attr("title",t.local().format(b)),!r){var i=Math.abs(l().diff(t,"minute")),o=3600;i<1?o=1:i<60?o=30:i<180&&(o=300),v=e.setTimeout(function(){f(t)},1e3*o)}}function d(t){M&&s.attr("datetime",t)}function h(){if(c(),p){var t=r.preprocessDate(p,k,m);f(t),d(t.toISOString())}}var p,g,v=null,m=o.format,y=i.withoutSuffix,b=i.titleFormat,x=i.fullDateThreshold,w=i.fullDateFormat,$=(new Date).getTime(),k=o.preprocess,_=u.amTimeAgo,M="TIME"===s[0].nodeName.toUpperCase();a.$watch(_,function(t){return"undefined"==typeof t||null===t||""===t?(c(),void(p&&(s.text(""),d(""),p=null))):(p=t,void h())}),t.isDefined(u.amFrom)&&a.$watch(u.amFrom,function(t){g="undefined"==typeof t||null===t||""===t?null:n(t),h()}),t.isDefined(u.amWithoutSuffix)&&a.$watch(u.amWithoutSuffix,function(t){"boolean"==typeof t?(y=t,h()):y=i.withoutSuffix}),u.$observe("amFormat",function(t){"undefined"!=typeof t&&(m=t,h())}),u.$observe("amPreprocess",function(t){k=t,h()}),u.$observe("amFullDateThreshold",function(t){x=t,h()}),u.$observe("amFullDateFormat",function(t){w=t,h()}),a.$on("$destroy",function(){c()}),a.$on("amMoment:localeChanged",function(){h()})}}]).service("amMoment",["moment","$rootScope","$log","angularMomentConfig",function(e,n,r,i){this.preprocessors={utc:e.utc,unix:e.unix},this.changeLocale=function(r,i){var o=e.locale(r,i);return t.isDefined(r)&&n.$broadcast("amMoment:localeChanged"),o},this.changeTimezone=function(t){i.timezone=t,n.$broadcast("amMoment:timezoneChanged")},this.preprocessDate=function(n,o,a){return t.isUndefined(o)&&(o=i.preprocess),this.preprocessors[o]?this.preprocessors[o](n,a):(o&&r.warn("angular-moment: Ignoring unsupported value for preprocess: "+o),!isNaN(parseFloat(n))&&isFinite(n)?e(parseInt(n,10)):e(n,a))},this.applyTimezone=function(t,e){return(e=e||i.timezone)?(e.match(/^Z|[+-]\d\d:?\d\d$/i)?t=t.utcOffset(e):t.tz?t=t.tz(e):r.warn("angular-moment: named timezone specified but moment.tz() is undefined. Did you forget to include moment-timezone.js?"),t):t}}]).filter("amCalendar",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(n,r,i){if("undefined"==typeof n||null===n)return"";n=e.preprocessDate(n,r);var o=t(n);return o.isValid()?e.applyTimezone(o,i).calendar():""}return r.$stateful=n.statefulFilters,r}]).filter("amDifference",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(n,r,i,o,a,s){if("undefined"==typeof n||null===n)return"";n=e.preprocessDate(n,a);var u=t(n);if(!u.isValid())return"";var l;if("undefined"==typeof r||null===r)l=t();else if(r=e.preprocessDate(r,s),l=t(r),!l.isValid())return"";return e.applyTimezone(u).diff(e.applyTimezone(l),i,o)}return r.$stateful=n.statefulFilters,r}]).filter("amDateFormat",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(r,i,o,a,s){var u=s||n.format;if("undefined"==typeof r||null===r)return"";r=e.preprocessDate(r,o,u);var l=t(r);return l.isValid()?e.applyTimezone(l,a).format(i):""}return r.$stateful=n.statefulFilters,r}]).filter("amDurationFormat",["moment","angularMomentConfig",function(t,e){function n(e,n,r){return"undefined"==typeof e||null===e?"":t.duration(e,n).humanize(r)}return n.$stateful=e.statefulFilters,n}]).filter("amTimeAgo",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(n,r,i,o){var a,s;return"undefined"==typeof n||null===n?"":(n=e.preprocessDate(n,r),a=t(n),a.isValid()?(s=t(o),"undefined"!=typeof o&&s.isValid()?e.applyTimezone(a).from(s,i):e.applyTimezone(a).fromNow(i)):"")}return r.$stateful=n.statefulFilters,r}]).filter("amSubtract",["moment","angularMomentConfig",function(t,e){function n(e,n,r){return"undefined"==typeof e||null===e?"":t(e).subtract(parseInt(n,10),r)}return n.$stateful=e.statefulFilters,n}]).filter("amAdd",["moment","angularMomentConfig",function(t,e){function n(e,n,r){return"undefined"==typeof e||null===e?"":t(e).add(parseInt(n,10),r)}return n.$stateful=e.statefulFilters,n}])}"function"==typeof define&&define.amd?define(["angular","moment"],t):"undefined"!=typeof module&&module&&module.exports?(t(angular,require("moment")),module.exports="angularMoment"):t(angular,("undefined"!=typeof global?global:window).moment)}(),function t(e,n,r){function i(a,s){if(!n[a]){if(!e[a]){var u="function"==typeof require&&require;if(!s&&u)return u(a,!0);if(o)return o(a,!0);var l=new Error("Cannot find module '"+a+"'");throw l.code="MODULE_NOT_FOUND",l}var c=n[a]={exports:{}};e[a][0].call(c.exports,function(t){var n=e[a][1][t];return i(n?n:t)},c,c.exports,t,e,n,r)}return n[a].exports}for(var o="function"==typeof require&&require,a=0;a<r.length;a++)i(r[a]);return i}({1:[function(t,e,n){(function(e){e.graphlib=t("./index")}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./index":2}],2:[function(t,e,n){var r=t("./lib");e.exports={Graph:r.Graph,json:t("./lib/json"),alg:t("./lib/alg"),version:r.version}},{"./lib":18,"./lib/alg":9,"./lib/json":19}],3:[function(t,e,n){function r(t){function e(o){i.has(r,o)||(r[o]=!0,n.push(o),i.each(t.successors(o),e),i.each(t.predecessors(o),e))}var n,r={},o=[];return i.each(t.nodes(),function(t){n=[],e(t),n.length&&o.push(n)}),o}var i=t("../lodash");e.exports=r},{"../lodash":20}],4:[function(t,e,n){function r(t,e,n){o.isArray(e)||(e=[e]);var r=[],a={};return o.each(e,function(e){if(!t.hasNode(e))throw new Error("Graph does not have node: "+e);i(t,e,"post"===n,a,r)}),r}function i(t,e,n,r,a){o.has(r,e)||(r[e]=!0,n||a.push(e),o.each(t.neighbors(e),function(e){i(t,e,n,r,a)}),n&&a.push(e))}var o=t("../lodash");e.exports=r},{"../lodash":20}],5:[function(t,e,n){function r(t,e,n){return o.transform(t.nodes(),function(r,o){r[o]=i(t,o,e,n)},{})}var i=t("./dijkstra"),o=t("../lodash");e.exports=r},{"../lodash":20,"./dijkstra":6}],6:[function(t,e,n){function r(t,e,n,r){return i(t,String(e),n||s,r||function(e){return t.outEdges(e)})}function i(t,e,n,r){var i,o,s={},u=new a,l=function(t){var e=t.v!==i?t.v:t.w,r=s[e],a=n(t),l=o.distance+a;if(a<0)throw new Error("dijkstra does not allow negative edge weights. Bad edge: "+t+" Weight: "+a);l<r.distance&&(r.distance=l,r.predecessor=i,u.decrease(e,l))};for(t.nodes().forEach(function(t){var n=t===e?0:Number.POSITIVE_INFINITY;s[t]={distance:n},u.add(t,n)});u.size()>0&&(i=u.removeMin(),o=s[i],o.distance!==Number.POSITIVE_INFINITY);)r(i).forEach(l);return s}var o=t("../lodash"),a=t("../data/priority-queue");e.exports=r;var s=o.constant(1)},{"../data/priority-queue":16,"../lodash":20}],7:[function(t,e,n){function r(t){return i.filter(o(t),function(e){return e.length>1||1===e.length&&t.hasEdge(e[0],e[0])})}var i=t("../lodash"),o=t("./tarjan");e.exports=r},{"../lodash":20,"./tarjan":14}],8:[function(t,e,n){function r(t,e,n){return i(t,e||a,n||function(e){return t.outEdges(e)})}function i(t,e,n){var r={},i=t.nodes();return i.forEach(function(t){r[t]={},r[t][t]={distance:0},i.forEach(function(e){t!==e&&(r[t][e]={distance:Number.POSITIVE_INFINITY})}),n(t).forEach(function(n){var i=n.v===t?n.w:n.v,o=e(n);r[t][i]={distance:o,predecessor:t}})}),i.forEach(function(t){var e=r[t];i.forEach(function(n){var o=r[n];i.forEach(function(n){var r=o[t],i=e[n],a=o[n],s=r.distance+i.distance;s<a.distance&&(a.distance=s,a.predecessor=i.predecessor)})})}),r}var o=t("../lodash");e.exports=r;var a=o.constant(1)},{"../lodash":20}],9:[function(t,e,n){e.exports={components:t("./components"),dijkstra:t("./dijkstra"),dijkstraAll:t("./dijkstra-all"),findCycles:t("./find-cycles"),floydWarshall:t("./floyd-warshall"),isAcyclic:t("./is-acyclic"),postorder:t("./postorder"),preorder:t("./preorder"),prim:t("./prim"),tarjan:t("./tarjan"),topsort:t("./topsort")}},{"./components":3,"./dijkstra":6,"./dijkstra-all":5,"./find-cycles":7,"./floyd-warshall":8,"./is-acyclic":10,"./postorder":11,"./preorder":12,"./prim":13,"./tarjan":14,"./topsort":15}],10:[function(t,e,n){function r(t){try{i(t)}catch(e){if(e instanceof i.CycleException)return!1;throw e}return!0}var i=t("./topsort");e.exports=r},{"./topsort":15}],11:[function(t,e,n){function r(t,e){return i(t,e,"post")}var i=t("./dfs");e.exports=r},{"./dfs":4}],12:[function(t,e,n){function r(t,e){return i(t,e,"pre")}var i=t("./dfs");e.exports=r},{"./dfs":4}],13:[function(t,e,n){function r(t,e){function n(t){var n=t.v===r?t.w:t.v,i=l.priority(n);if(void 0!==i){var o=e(t);o<i&&(u[n]=r,l.decrease(n,o))}}var r,s=new o,u={},l=new a;if(0===t.nodeCount())return s;i.each(t.nodes(),function(t){l.add(t,Number.POSITIVE_INFINITY),s.setNode(t)}),l.decrease(t.nodes()[0],0);for(var c=!1;l.size()>0;){if(r=l.removeMin(),i.has(u,r))s.setEdge(r,u[r]);else{if(c)throw new Error("Input graph is not connected: "+t);c=!0}t.nodeEdges(r).forEach(n)}return s}var i=t("../lodash"),o=t("../graph"),a=t("../data/priority-queue");e.exports=r},{"../data/priority-queue":16,"../graph":17,"../lodash":20}],14:[function(t,e,n){function r(t){function e(s){var u=o[s]={onStack:!0,lowlink:n,index:n++};if(r.push(s),t.successors(s).forEach(function(t){i.has(o,t)?o[t].onStack&&(u.lowlink=Math.min(u.lowlink,o[t].index)):(e(t),u.lowlink=Math.min(u.lowlink,o[t].lowlink))}),u.lowlink===u.index){var l,c=[];do l=r.pop(),o[l].onStack=!1,c.push(l);while(s!==l);a.push(c)}}var n=0,r=[],o={},a=[];return t.nodes().forEach(function(t){i.has(o,t)||e(t)}),a}var i=t("../lodash");e.exports=r},{"../lodash":20}],15:[function(t,e,n){function r(t){function e(s){if(o.has(r,s))throw new i;o.has(n,s)||(r[s]=!0,n[s]=!0,o.each(t.predecessors(s),e),delete r[s],a.push(s))}var n={},r={},a=[];if(o.each(t.sinks(),e),o.size(n)!==t.nodeCount())throw new i;return a}function i(){}var o=t("../lodash");e.exports=r,r.CycleException=i},{"../lodash":20}],16:[function(t,e,n){function r(){this._arr=[],this._keyIndices={}}var i=t("../lodash");e.exports=r,r.prototype.size=function(){return this._arr.length},r.prototype.keys=function(){return this._arr.map(function(t){return t.key})},r.prototype.has=function(t){return i.has(this._keyIndices,t)},r.prototype.priority=function(t){var e=this._keyIndices[t];if(void 0!==e)return this._arr[e].priority},r.prototype.min=function(){if(0===this.size())throw new Error("Queue underflow");return this._arr[0].key},r.prototype.add=function(t,e){var n=this._keyIndices;if(t=String(t),!i.has(n,t)){var r=this._arr,o=r.length;return n[t]=o,r.push({key:t,priority:e}),this._decrease(o),!0}return!1},r.prototype.removeMin=function(){this._swap(0,this._arr.length-1);var t=this._arr.pop();return delete this._keyIndices[t.key],this._heapify(0),t.key},r.prototype.decrease=function(t,e){var n=this._keyIndices[t];if(e>this._arr[n].priority)throw new Error("New priority is greater than current priority. Key: "+t+" Old: "+this._arr[n].priority+" New: "+e);this._arr[n].priority=e,this._decrease(n)},r.prototype._heapify=function(t){var e=this._arr,n=2*t,r=n+1,i=t;n<e.length&&(i=e[n].priority<e[i].priority?n:i,r<e.length&&(i=e[r].priority<e[i].priority?r:i),i!==t&&(this._swap(t,i),this._heapify(i)))},r.prototype._decrease=function(t){for(var e,n=this._arr,r=n[t].priority;0!==t&&(e=t>>1,!(n[e].priority<r));)this._swap(t,e),t=e},r.prototype._swap=function(t,e){var n=this._arr,r=this._keyIndices,i=n[t],o=n[e];n[t]=o,n[e]=i,r[o.key]=t,r[i.key]=e}},{"../lodash":20}],17:[function(t,e,n){"use strict";function r(t){this._isDirected=!l.has(t,"directed")||t.directed,this._isMultigraph=!!l.has(t,"multigraph")&&t.multigraph,this._isCompound=!!l.has(t,"compound")&&t.compound,this._label=void 0,this._defaultNodeLabelFn=l.constant(void 0),this._defaultEdgeLabelFn=l.constant(void 0),this._nodes={},this._isCompound&&(this._parent={},this._children={},this._children[f]={}),this._in={},this._preds={},this._out={},this._sucs={},this._edgeObjs={},this._edgeLabels={}}function i(t,e){t[e]?t[e]++:t[e]=1}function o(t,e){--t[e]||delete t[e]}function a(t,e,n,r){var i=""+e,o=""+n;if(!t&&i>o){var a=i;i=o,o=a}return i+d+o+d+(l.isUndefined(r)?c:r)}function s(t,e,n,r){var i=""+e,o=""+n;if(!t&&i>o){var a=i;i=o,o=a}var s={v:i,w:o};return r&&(s.name=r),s}function u(t,e){return a(t,e.v,e.w,e.name)}var l=t("./lodash");e.exports=r;var c="\0",f="\0",d="
";r.prototype._nodeCount=0,r.prototype._edgeCount=0,r.prototype.isDirected=function(){return this._isDirected},r.prototype.isMultigraph=function(){return this._isMultigraph},r.prototype.isCompound=function(){return this._isCompound},r.prototype.setGraph=function(t){return this._label=t,this},r.prototype.graph=function(){return this._label},r.prototype.setDefaultNodeLabel=function(t){return l.isFunction(t)||(t=l.constant(t)),this._defaultNodeLabelFn=t,this},r.prototype.nodeCount=function(){return this._nodeCount},r.prototype.nodes=function(){return l.keys(this._nodes)},r.prototype.sources=function(){return l.filter(this.nodes(),function(t){return l.isEmpty(this._in[t])},this)},r.prototype.sinks=function(){return l.filter(this.nodes(),function(t){return l.isEmpty(this._out[t])},this)},r.prototype.setNodes=function(t,e){var n=arguments;return l.each(t,function(t){n.length>1?this.setNode(t,e):this.setNode(t)},this),this},r.prototype.setNode=function(t,e){return l.has(this._nodes,t)?(arguments.length>1&&(this._nodes[t]=e),this):(this._nodes[t]=arguments.length>1?e:this._defaultNodeLabelFn(t),this._isCompound&&(this._parent[t]=f,this._children[t]={},this._children[f][t]=!0),this._in[t]={},this._preds[t]={},this._out[t]={},this._sucs[t]={},++this._nodeCount,this)},r.prototype.node=function(t){return this._nodes[t]},r.prototype.hasNode=function(t){return l.has(this._nodes,t)},r.prototype.removeNode=function(t){var e=this;if(l.has(this._nodes,t)){var n=function(t){e.removeEdge(e._edgeObjs[t])};delete this._nodes[t],this._isCompound&&(this._removeFromParentsChildList(t),delete this._parent[t],l.each(this.children(t),function(t){this.setParent(t)},this),delete this._children[t]),l.each(l.keys(this._in[t]),n),delete this._in[t],delete this._preds[t],l.each(l.keys(this._out[t]),n),delete this._out[t],delete this._sucs[t],--this._nodeCount}return this},r.prototype.setParent=function(t,e){if(!this._isCompound)throw new Error("Cannot set parent in a non-compound graph");if(l.isUndefined(e))e=f;else{e+="";for(var n=e;!l.isUndefined(n);n=this.parent(n))if(n===t)throw new Error("Setting "+e+" as parent of "+t+" would create create a cycle");this.setNode(e)}return this.setNode(t),this._removeFromParentsChildList(t),this._parent[t]=e,this._children[e][t]=!0,this},r.prototype._removeFromParentsChildList=function(t){delete this._children[this._parent[t]][t]},r.prototype.parent=function(t){if(this._isCompound){var e=this._parent[t];if(e!==f)return e}},r.prototype.children=function(t){if(l.isUndefined(t)&&(t=f),this._isCompound){var e=this._children[t];if(e)return l.keys(e)}else{if(t===f)return this.nodes();if(this.hasNode(t))return[]}},r.prototype.predecessors=function(t){var e=this._preds[t];if(e)return l.keys(e)},r.prototype.successors=function(t){var e=this._sucs[t];if(e)return l.keys(e)},r.prototype.neighbors=function(t){var e=this.predecessors(t);if(e)return l.union(e,this.successors(t))},r.prototype.filterNodes=function(t){function e(t){var o=r.parent(t);return void 0===o||n.hasNode(o)?(i[t]=o,o):o in i?i[o]:e(o)}var n=new this.constructor({directed:this._isDirected,multigraph:this._isMultigraph,compound:this._isCompound});n.setGraph(this.graph()),l.each(this._nodes,function(e,r){t(r)&&n.setNode(r,e)},this),l.each(this._edgeObjs,function(t){n.hasNode(t.v)&&n.hasNode(t.w)&&n.setEdge(t,this.edge(t))},this);var r=this,i={};return this._isCompound&&l.each(n.nodes(),function(t){n.setParent(t,e(t))}),n},r.prototype.setDefaultEdgeLabel=function(t){return l.isFunction(t)||(t=l.constant(t)),this._defaultEdgeLabelFn=t,this},r.prototype.edgeCount=function(){return this._edgeCount},r.prototype.edges=function(){return l.values(this._edgeObjs)},r.prototype.setPath=function(t,e){var n=this,r=arguments;return l.reduce(t,function(t,i){return r.length>1?n.setEdge(t,i,e):n.setEdge(t,i),i}),this},r.prototype.setEdge=function(){var t,e,n,r,o=!1,u=arguments[0];"object"==typeof u&&null!==u&&"v"in u?(t=u.v,e=u.w,n=u.name,2===arguments.length&&(r=arguments[1],o=!0)):(t=u,e=arguments[1],n=arguments[3],arguments.length>2&&(r=arguments[2],o=!0)),t=""+t,e=""+e,l.isUndefined(n)||(n=""+n);var c=a(this._isDirected,t,e,n);if(l.has(this._edgeLabels,c))return o&&(this._edgeLabels[c]=r),this;if(!l.isUndefined(n)&&!this._isMultigraph)throw new Error("Cannot set a named edge when isMultigraph = false");this.setNode(t),this.setNode(e),this._edgeLabels[c]=o?r:this._defaultEdgeLabelFn(t,e,n);var f=s(this._isDirected,t,e,n);return t=f.v,e=f.w,Object.freeze(f),this._edgeObjs[c]=f,i(this._preds[e],t),i(this._sucs[t],e),this._in[e][c]=f,this._out[t][c]=f,this._edgeCount++,this},r.prototype.edge=function(t,e,n){var r=1===arguments.length?u(this._isDirected,arguments[0]):a(this._isDirected,t,e,n);return this._edgeLabels[r]},r.prototype.hasEdge=function(t,e,n){var r=1===arguments.length?u(this._isDirected,arguments[0]):a(this._isDirected,t,e,n);return l.has(this._edgeLabels,r)},r.prototype.removeEdge=function(t,e,n){var r=1===arguments.length?u(this._isDirected,arguments[0]):a(this._isDirected,t,e,n),i=this._edgeObjs[r];return i&&(t=i.v,e=i.w,delete this._edgeLabels[r],delete this._edgeObjs[r],o(this._preds[e],t),o(this._sucs[t],e),delete this._in[e][r],delete this._out[t][r],this._edgeCount--),this},r.prototype.inEdges=function(t,e){var n=this._in[t];if(n){var r=l.values(n);return e?l.filter(r,function(t){return t.v===e}):r}},r.prototype.outEdges=function(t,e){var n=this._out[t];if(n){var r=l.values(n);return e?l.filter(r,function(t){return t.w===e}):r}},r.prototype.nodeEdges=function(t,e){var n=this.inEdges(t,e);if(n)return n.concat(this.outEdges(t,e))}},{"./lodash":20}],18:[function(t,e,n){e.exports={Graph:t("./graph"),version:t("./version")}},{"./graph":17,"./version":21}],19:[function(t,e,n){function r(t){var e={options:{directed:t.isDirected(),multigraph:t.isMultigraph(),compound:t.isCompound()},nodes:i(t),edges:o(t)};return s.isUndefined(t.graph())||(e.value=s.clone(t.graph())),e}function i(t){return s.map(t.nodes(),function(e){var n=t.node(e),r=t.parent(e),i={v:e};return s.isUndefined(n)||(i.value=n),s.isUndefined(r)||(i.parent=r),i})}function o(t){return s.map(t.edges(),function(e){var n=t.edge(e),r={v:e.v,w:e.w};return s.isUndefined(e.name)||(r.name=e.name),s.isUndefined(n)||(r.value=n),r})}function a(t){var e=new u(t.options).setGraph(t.value);return s.each(t.nodes,function(t){e.setNode(t.v,t.value),t.parent&&e.setParent(t.v,t.parent)}),s.each(t.edges,function(t){e.setEdge({v:t.v,w:t.w,name:t.name},t.value)}),e}var s=t("./lodash"),u=t("./graph");e.exports={write:r,read:a}},{"./graph":17,"./lodash":20}],20:[function(t,e,n){var r;if("function"==typeof t)try{r=t("lodash")}catch(i){}r||(r=window._),e.exports=r},{lodash:void 0}],21:[function(t,e,n){e.exports="1.0.7"},{}]},{},[1]),function(t,e){"use strict";"function"==typeof define&&define.amd?define(["ev-emitter/ev-emitter"],function(n){return e(t,n)}):"object"==typeof module&&module.exports?module.exports=e(t,require("ev-emitter")):t.imagesLoaded=e(t,t.EvEmitter)}(window,function(t,e){"use strict";function n(t,e){for(var n in e)t[n]=e[n];return t}function r(t){var e=[];if(Array.isArray(t))e=t;else if("number"==typeof t.length)for(var n=0;n<t.length;n++)e.push(t[n]);else e.push(t);return e}function i(t,e,o){return this instanceof i?("string"==typeof t&&(t=document.querySelectorAll(t)),this.elements=r(t),this.options=n({},this.options),"function"==typeof e?o=e:n(this.options,e),o&&this.on("always",o),this.getImages(),s&&(this.jqDeferred=new s.Deferred),void setTimeout(function(){this.check()}.bind(this))):new i(t,e,o)}function o(t){this.img=t}function a(t,e){this.url=t,this.element=e,this.img=new Image}var s=t.jQuery,u=t.console;i.prototype=Object.create(e.prototype),i.prototype.options={},i.prototype.getImages=function(){this.images=[],this.elements.forEach(this.addElementImages,this)},i.prototype.addElementImages=function(t){"IMG"==t.nodeName&&this.addImage(t),this.options.background===!0&&this.addElementBackgroundImages(t);var e=t.nodeType;if(e&&l[e]){for(var n=t.querySelectorAll("img"),r=0;r<n.length;r++){var i=n[r];this.addImage(i)}if("string"==typeof this.options.background){var o=t.querySelectorAll(this.options.background);for(r=0;r<o.length;r++){var a=o[r];this.addElementBackgroundImages(a)}}}};var l={1:!0,9:!0,11:!0};return i.prototype.addElementBackgroundImages=function(t){var e=getComputedStyle(t);if(e)for(var n=/url\((['"])?(.*?)\1\)/gi,r=n.exec(e.backgroundImage);null!==r;){var i=r&&r[2];i&&this.addBackground(i,t),r=n.exec(e.backgroundImage)}},i.prototype.addImage=function(t){var e=new o(t);this.images.push(e)},i.prototype.addBackground=function(t,e){var n=new a(t,e);this.images.push(n)},i.prototype.check=function(){function t(t,n,r){setTimeout(function(){e.progress(t,n,r)})}var e=this;return this.progressedCount=0,this.hasAnyBroken=!1,this.images.length?void this.images.forEach(function(e){e.once("progress",t),e.check()}):void this.complete()},i.prototype.progress=function(t,e,n){this.progressedCount++,this.hasAnyBroken=this.hasAnyBroken||!t.isLoaded,this.emitEvent("progress",[this,t,e]),this.jqDeferred&&this.jqDeferred.notify&&this.jqDeferred.notify(this,t),this.progressedCount==this.images.length&&this.complete(),this.options.debug&&u&&u.log("progress: "+n,t,e)},i.prototype.complete=function(){var t=this.hasAnyBroken?"fail":"done";if(this.isComplete=!0,this.emitEvent(t,[this]),this.emitEvent("always",[this]),this.jqDeferred){var e=this.hasAnyBroken?"reject":"resolve";this.jqDeferred[e](this)}},o.prototype=Object.create(e.prototype),o.prototype.check=function(){var t=this.getIsImageComplete();return t?void this.confirm(0!==this.img.naturalWidth,"naturalWidth"):(this.proxyImage=new Image,this.proxyImage.addEventListener("load",this),this.proxyImage.addEventListener("error",this),this.img.addEventListener("load",this),this.img.addEventListener("error",this),void(this.proxyImage.src=this.img.src))},o.prototype.getIsImageComplete=function(){return this.img.complete&&void 0!==this.img.naturalWidth},o.prototype.confirm=function(t,e){this.isLoaded=t,this.emitEvent("progress",[this,this.img,e])},o.prototype.handleEvent=function(t){var e="on"+t.type;this[e]&&this[e](t)},o.prototype.onload=function(){this.confirm(!0,"onload"),this.unbindEvents()},o.prototype.onerror=function(){this.confirm(!1,"onerror"),this.unbindEvents()},o.prototype.unbindEvents=function(){this.proxyImage.removeEventListener("load",this),this.proxyImage.removeEventListener("error",this),this.img.removeEventListener("load",this),this.img.removeEventListener("error",this)},a.prototype=Object.create(o.prototype),a.prototype.check=function(){this.img.addEventListener("load",this),this.img.addEventListener("error",this),this.img.src=this.url;var t=this.getIsImageComplete();t&&(this.confirm(0!==this.img.naturalWidth,"naturalWidth"),this.unbindEvents())},a.prototype.unbindEvents=function(){this.img.removeEventListener("load",this),this.img.removeEventListener("error",this)},a.prototype.confirm=function(t,e){this.isLoaded=t,this.emitEvent("progress",[this,this.element,e])},i.makeJQueryPlugin=function(e){e=e||t.jQuery,e&&(s=e,s.fn.imagesLoaded=function(t,e){var n=new i(this,t,e);return n.jqDeferred.promise(s(this))})},i.makeJQueryPlugin(),i}),!function(t){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{var e;"undefined"!=typeof window?e=window:"undefined"!=typeof global?e=global:"undefined"!=typeof self&&(e=self),e.dagre=t()}}(function(){return function t(e,n,r){function i(a,s){if(!n[a]){if(!e[a]){var u="function"==typeof require&&require;if(!s&&u)return u(a,!0);if(o)return o(a,!0);var l=new Error("Cannot find module '"+a+"'");throw l.code="MODULE_NOT_FOUND",l}var c=n[a]={exports:{}};e[a][0].call(c.exports,function(t){var n=e[a][1][t];return i(n?n:t)},c,c.exports,t,e,n,r)}return n[a].exports}for(var o="function"==typeof require&&require,a=0;a<r.length;a++)i(r[a]);return i}({1:[function(t,e,n){e.exports={graphlib:t("./lib/graphlib"),layout:t("./lib/layout"),debug:t("./lib/debug"),util:{time:t("./lib/util").time,notime:t("./lib/util").notime},version:t("./lib/version")}},{"./lib/debug":6,"./lib/graphlib":7,"./lib/layout":9,"./lib/util":29,"./lib/version":30}],2:[function(t,e,n){"use strict";function r(t){function e(t){return function(e){return t.edge(e).weight}}var n="greedy"===t.graph().acyclicer?s(t,e(t)):i(t);a.each(n,function(e){var n=t.edge(e);t.removeEdge(e),n.forwardName=e.name,n.reversed=!0,t.setEdge(e.w,e.v,n,a.uniqueId("rev"))})}function i(t){function e(o){a.has(i,o)||(i[o]=!0,r[o]=!0,a.each(t.outEdges(o),function(t){a.has(r,t.w)?n.push(t):e(t.w)}),delete r[o])}var n=[],r={},i={};return a.each(t.nodes(),e),n}function o(t){a.each(t.edges(),function(e){var n=t.edge(e);if(n.reversed){t.removeEdge(e);var r=n.forwardName;delete n.reversed,delete n.forwardName,t.setEdge(e.w,e.v,n,r)}})}var a=t("./lodash"),s=t("./greedy-fas");e.exports={run:r,undo:o}},{"./greedy-fas":8,"./lodash":10}],3:[function(t,e,n){function r(t){function e(n){var r=t.children(n),a=t.node(n);if(r.length&&o.each(r,e),o.has(a,"minRank")){a.borderLeft=[],a.borderRight=[];for(var s=a.minRank,u=a.maxRank+1;s<u;++s)i(t,"borderLeft","_bl",n,a,s),i(t,"borderRight","_br",n,a,s)}}o.each(t.children(),e)}function i(t,e,n,r,i,o){var s={width:0,height:0,rank:o,borderType:e},u=i[e][o-1],l=a.addDummyNode(t,"border",s,n);i[e][o]=l,t.setParent(l,r),u&&t.setEdge(u,l,{weight:1})}var o=t("./lodash"),a=t("./util");e.exports=r},{"./lodash":10,"./util":29}],4:[function(t,e,n){"use strict";function r(t){var e=t.graph().rankdir.toLowerCase();"lr"!==e&&"rl"!==e||o(t)}function i(t){var e=t.graph().rankdir.toLowerCase();"bt"!==e&&"rl"!==e||s(t),"lr"!==e&&"rl"!==e||(l(t),o(t))}function o(t){f.each(t.nodes(),function(e){a(t.node(e))}),f.each(t.edges(),function(e){a(t.edge(e))})}function a(t){var e=t.width;t.width=t.height,t.height=e}function s(t){f.each(t.nodes(),function(e){u(t.node(e))}),f.each(t.edges(),function(e){var n=t.edge(e);f.each(n.points,u),f.has(n,"y")&&u(n)})}function u(t){t.y=-t.y}function l(t){f.each(t.nodes(),function(e){c(t.node(e))}),f.each(t.edges(),function(e){var n=t.edge(e);f.each(n.points,c),f.has(n,"x")&&c(n)})}function c(t){var e=t.x;t.x=t.y,t.y=e}var f=t("./lodash");e.exports={adjust:r,undo:i}},{"./lodash":10}],5:[function(t,e,n){function r(){var t={};t._next=t._prev=t,this._sentinel=t}function i(t){t._prev._next=t._next,t._next._prev=t._prev,delete t._next,delete t._prev}function o(t,e){if("_next"!==t&&"_prev"!==t)return e}e.exports=r,r.prototype.dequeue=function(){var t=this._sentinel,e=t._prev;if(e!==t)return i(e),e},r.prototype.enqueue=function(t){var e=this._sentinel;t._prev&&t._next&&i(t),t._next=e._next,e._next._prev=t,e._next=t,t._prev=e},r.prototype.toString=function(){for(var t=[],e=this._sentinel,n=e._prev;n!==e;)t.push(JSON.stringify(n,o)),n=n._prev;return"["+t.join(", ")+"]"}},{}],6:[function(t,e,n){function r(t){var e=o.buildLayerMatrix(t),n=new a({compound:!0,multigraph:!0}).setGraph({});return i.each(t.nodes(),function(e){n.setNode(e,{label:e}),n.setParent(e,"layer"+t.node(e).rank)}),i.each(t.edges(),function(t){n.setEdge(t.v,t.w,{},t.name)}),i.each(e,function(t,e){var r="layer"+e;n.setNode(r,{rank:"same"}),i.reduce(t,function(t,e){return n.setEdge(t,e,{style:"invis"}),e})}),n}var i=t("./lodash"),o=t("./util"),a=t("./graphlib").Graph;e.exports={debugOrdering:r}},{"./graphlib":7,"./lodash":10,"./util":29}],7:[function(t,e,n){var r;if("function"==typeof t)try{r=t("graphlib")}catch(i){}r||(r=window.graphlib),e.exports=r},{graphlib:void 0}],8:[function(t,e,n){function r(t,e){if(t.nodeCount()<=1)return[];var n=a(t,e||f),r=i(n.graph,n.buckets,n.zeroIdx);return u.flatten(u.map(r,function(e){return t.outEdges(e.v,e.w)}),!0)}function i(t,e,n){for(var r,i=[],a=e[e.length-1],s=e[0];t.nodeCount();){for(;r=s.dequeue();)o(t,e,n,r);for(;r=a.dequeue();)o(t,e,n,r);if(t.nodeCount())for(var u=e.length-2;u>0;--u)if(r=e[u].dequeue()){i=i.concat(o(t,e,n,r,!0));break}}return i}function o(t,e,n,r,i){var o=i?[]:void 0;return u.each(t.inEdges(r.v),function(r){var a=t.edge(r),u=t.node(r.v);i&&o.push({v:r.v,w:r.w}),u.out-=a,s(e,n,u)}),u.each(t.outEdges(r.v),function(r){var i=t.edge(r),o=r.w,a=t.node(o);a["in"]-=i,s(e,n,a)}),t.removeNode(r.v),o}function a(t,e){var n=new l,r=0,i=0;u.each(t.nodes(),function(t){n.setNode(t,{v:t,"in":0,out:0})}),u.each(t.edges(),function(t){var o=n.edge(t.v,t.w)||0,a=e(t),s=o+a;n.setEdge(t.v,t.w,s),i=Math.max(i,n.node(t.v).out+=a),r=Math.max(r,n.node(t.w)["in"]+=a)});var o=u.range(i+r+3).map(function(){return new c}),a=r+1;return u.each(n.nodes(),function(t){s(o,a,n.node(t))}),{graph:n,buckets:o,zeroIdx:a}}function s(t,e,n){n.out?n["in"]?t[n.out-n["in"]+e].enqueue(n):t[t.length-1].enqueue(n):t[0].enqueue(n)}var u=t("./lodash"),l=t("./graphlib").Graph,c=t("./data/list");e.exports=r;var f=u.constant(1)},{"./data/list":5,"./graphlib":7,"./lodash":10}],9:[function(t,e,n){"use strict";function r(t,e){var n=e&&e.debugTiming?O.time:O.notime;n("layout",function(){var e=n("  buildLayoutGraph",function(){return a(t)});n("  runLayout",function(){i(e,n)}),n("  updateInputGraph",function(){o(t,e)})})}function i(t,e){e("    makeSpaceForEdgeLabels",function(){s(t)}),e("    removeSelfEdges",function(){v(t)}),e("    acyclic",function(){$.run(t)}),e("    nestingGraph.run",function(){E.run(t)}),e("    rank",function(){_(O.asNonCompoundGraph(t))}),e("    injectEdgeLabelProxies",function(){u(t)}),e("    removeEmptyRanks",function(){S(t)}),e("    nestingGraph.cleanup",function(){E.cleanup(t)}),e("    normalizeRanks",function(){M(t)}),e("    assignRankMinMax",function(){l(t)}),e("    removeEdgeLabelProxies",function(){c(t)}),e("    normalize.run",function(){k.run(t)}),e("    parentDummyChains",function(){C(t)}),e("    addBorderSegments",function(){A(t)}),e("    order",function(){N(t)}),e("    insertSelfEdges",function(){
-m(t)}),e("    adjustCoordinateSystem",function(){T.adjust(t)}),e("    position",function(){D(t)}),e("    positionSelfEdges",function(){y(t)}),e("    removeBorderNodes",function(){g(t)}),e("    normalize.undo",function(){k.undo(t)}),e("    fixupEdgeLabelCoords",function(){h(t)}),e("    undoCoordinateSystem",function(){T.undo(t)}),e("    translateGraph",function(){f(t)}),e("    assignNodeIntersects",function(){d(t)}),e("    reversePoints",function(){p(t)}),e("    acyclic.undo",function(){$.undo(t)})}function o(t,e){w.each(t.nodes(),function(n){var r=t.node(n),i=e.node(n);r&&(r.x=i.x,r.y=i.y,e.children(n).length&&(r.width=i.width,r.height=i.height))}),w.each(t.edges(),function(n){var r=t.edge(n),i=e.edge(n);r.points=i.points,w.has(i,"x")&&(r.x=i.x,r.y=i.y)}),t.graph().width=e.graph().width,t.graph().height=e.graph().height}function a(t){var e=new j({multigraph:!0,compound:!0}),n=x(t.graph());return e.setGraph(w.merge({},L,b(n,I),w.pick(n,P))),w.each(t.nodes(),function(n){var r=x(t.node(n));e.setNode(n,w.defaults(b(r,F),q)),e.setParent(n,t.parent(n))}),w.each(t.edges(),function(n){var r=x(t.edge(n));e.setEdge(n,w.merge({},W,b(r,z),w.pick(r,R)))}),e}function s(t){var e=t.graph();e.ranksep/=2,w.each(t.edges(),function(n){var r=t.edge(n);r.minlen*=2,"c"!==r.labelpos.toLowerCase()&&("TB"===e.rankdir||"BT"===e.rankdir?r.width+=r.labeloffset:r.height+=r.labeloffset)})}function u(t){w.each(t.edges(),function(e){var n=t.edge(e);if(n.width&&n.height){var r=t.node(e.v),i=t.node(e.w),o={rank:(i.rank-r.rank)/2+r.rank,e:e};O.addDummyNode(t,"edge-proxy",o,"_ep")}})}function l(t){var e=0;w.each(t.nodes(),function(n){var r=t.node(n);r.borderTop&&(r.minRank=t.node(r.borderTop).rank,r.maxRank=t.node(r.borderBottom).rank,e=w.max(e,r.maxRank))}),t.graph().maxRank=e}function c(t){w.each(t.nodes(),function(e){var n=t.node(e);"edge-proxy"===n.dummy&&(t.edge(n.e).labelRank=n.rank,t.removeNode(e))})}function f(t){function e(t){var e=t.x,a=t.y,s=t.width,u=t.height;n=Math.min(n,e-s/2),r=Math.max(r,e+s/2),i=Math.min(i,a-u/2),o=Math.max(o,a+u/2)}var n=Number.POSITIVE_INFINITY,r=0,i=Number.POSITIVE_INFINITY,o=0,a=t.graph(),s=a.marginx||0,u=a.marginy||0;w.each(t.nodes(),function(n){e(t.node(n))}),w.each(t.edges(),function(n){var r=t.edge(n);w.has(r,"x")&&e(r)}),n-=s,i-=u,w.each(t.nodes(),function(e){var r=t.node(e);r.x-=n,r.y-=i}),w.each(t.edges(),function(e){var r=t.edge(e);w.each(r.points,function(t){t.x-=n,t.y-=i}),w.has(r,"x")&&(r.x-=n),w.has(r,"y")&&(r.y-=i)}),a.width=r-n+s,a.height=o-i+u}function d(t){w.each(t.edges(),function(e){var n,r,i=t.edge(e),o=t.node(e.v),a=t.node(e.w);i.points?(n=i.points[0],r=i.points[i.points.length-1]):(i.points=[],n=a,r=o),i.points.unshift(O.intersectRect(o,n)),i.points.push(O.intersectRect(a,r))})}function h(t){w.each(t.edges(),function(e){var n=t.edge(e);if(w.has(n,"x"))switch("l"!==n.labelpos&&"r"!==n.labelpos||(n.width-=n.labeloffset),n.labelpos){case"l":n.x-=n.width/2+n.labeloffset;break;case"r":n.x+=n.width/2+n.labeloffset}})}function p(t){w.each(t.edges(),function(e){var n=t.edge(e);n.reversed&&n.points.reverse()})}function g(t){w.each(t.nodes(),function(e){if(t.children(e).length){var n=t.node(e),r=t.node(n.borderTop),i=t.node(n.borderBottom),o=t.node(w.last(n.borderLeft)),a=t.node(w.last(n.borderRight));n.width=Math.abs(a.x-o.x),n.height=Math.abs(i.y-r.y),n.x=o.x+n.width/2,n.y=r.y+n.height/2}}),w.each(t.nodes(),function(e){"border"===t.node(e).dummy&&t.removeNode(e)})}function v(t){w.each(t.edges(),function(e){if(e.v===e.w){var n=t.node(e.v);n.selfEdges||(n.selfEdges=[]),n.selfEdges.push({e:e,label:t.edge(e)}),t.removeEdge(e)}})}function m(t){var e=O.buildLayerMatrix(t);w.each(e,function(e){var n=0;w.each(e,function(e,r){var i=t.node(e);i.order=r+n,w.each(i.selfEdges,function(e){O.addDummyNode(t,"selfedge",{width:e.label.width,height:e.label.height,rank:i.rank,order:r+ ++n,e:e.e,label:e.label},"_se")}),delete i.selfEdges})})}function y(t){w.each(t.nodes(),function(e){var n=t.node(e);if("selfedge"===n.dummy){var r=t.node(n.e.v),i=r.x+r.width/2,o=r.y,a=n.x-i,s=r.height/2;t.setEdge(n.e,n.label),t.removeNode(e),n.label.points=[{x:i+2*a/3,y:o-s},{x:i+5*a/6,y:o-s},{x:i+a,y:o},{x:i+5*a/6,y:o+s},{x:i+2*a/3,y:o+s}],n.label.x=n.x,n.label.y=n.y}})}function b(t,e){return w.mapValues(w.pick(t,e),Number)}function x(t){var e={};return w.each(t,function(t,n){e[n.toLowerCase()]=t}),e}var w=t("./lodash"),$=t("./acyclic"),k=t("./normalize"),_=t("./rank"),M=t("./util").normalizeRanks,C=t("./parent-dummy-chains"),S=t("./util").removeEmptyRanks,E=t("./nesting-graph"),A=t("./add-border-segments"),T=t("./coordinate-system"),N=t("./order"),D=t("./position"),O=t("./util"),j=t("./graphlib").Graph;e.exports=r;var I=["nodesep","edgesep","ranksep","marginx","marginy"],L={ranksep:50,edgesep:20,nodesep:50,rankdir:"tb"},P=["acyclicer","ranker","rankdir","align"],F=["width","height"],q={width:0,height:0},z=["minlen","weight","width","height","labeloffset"],W={minlen:1,weight:1,width:0,height:0,labeloffset:10,labelpos:"r"},R=["labelpos"]},{"./acyclic":2,"./add-border-segments":3,"./coordinate-system":4,"./graphlib":7,"./lodash":10,"./nesting-graph":11,"./normalize":12,"./order":17,"./parent-dummy-chains":22,"./position":24,"./rank":26,"./util":29}],10:[function(t,e,n){var r;if("function"==typeof t)try{r=t("lodash")}catch(i){}r||(r=window._),e.exports=r},{lodash:void 0}],11:[function(t,e,n){function r(t){var e=l.addDummyNode(t,"root",{},"_root"),n=o(t),r=u.max(n)-1,s=2*r+1;t.graph().nestingRoot=e,u.each(t.edges(),function(e){t.edge(e).minlen*=s});var c=a(t)+1;u.each(t.children(),function(o){i(t,e,s,c,r,n,o)}),t.graph().nodeRankFactor=s}function i(t,e,n,r,o,a,s){var c=t.children(s);if(!c.length)return void(s!==e&&t.setEdge(e,s,{weight:0,minlen:n}));var f=l.addBorderNode(t,"_bt"),d=l.addBorderNode(t,"_bb"),h=t.node(s);t.setParent(f,s),h.borderTop=f,t.setParent(d,s),h.borderBottom=d,u.each(c,function(u){i(t,e,n,r,o,a,u);var l=t.node(u),c=l.borderTop?l.borderTop:u,h=l.borderBottom?l.borderBottom:u,p=l.borderTop?r:2*r,g=c!==h?1:o-a[s]+1;t.setEdge(f,c,{weight:p,minlen:g,nestingEdge:!0}),t.setEdge(h,d,{weight:p,minlen:g,nestingEdge:!0})}),t.parent(s)||t.setEdge(e,f,{weight:0,minlen:o+a[s]})}function o(t){function e(r,i){var o=t.children(r);o&&o.length&&u.each(o,function(t){e(t,i+1)}),n[r]=i}var n={};return u.each(t.children(),function(t){e(t,1)}),n}function a(t){return u.reduce(t.edges(),function(e,n){return e+t.edge(n).weight},0)}function s(t){var e=t.graph();t.removeNode(e.nestingRoot),delete e.nestingRoot,u.each(t.edges(),function(e){var n=t.edge(e);n.nestingEdge&&t.removeEdge(e)})}var u=t("./lodash"),l=t("./util");e.exports={run:r,cleanup:s}},{"./lodash":10,"./util":29}],12:[function(t,e,n){"use strict";function r(t){t.graph().dummyChains=[],a.each(t.edges(),function(e){i(t,e)})}function i(t,e){var n=e.v,r=t.node(n).rank,i=e.w,o=t.node(i).rank,a=e.name,u=t.edge(e),l=u.labelRank;if(o!==r+1){t.removeEdge(e);var c,f,d;for(d=0,++r;r<o;++d,++r)u.points=[],f={width:0,height:0,edgeLabel:u,edgeObj:e,rank:r},c=s.addDummyNode(t,"edge",f,"_d"),r===l&&(f.width=u.width,f.height=u.height,f.dummy="edge-label",f.labelpos=u.labelpos),t.setEdge(n,c,{weight:u.weight},a),0===d&&t.graph().dummyChains.push(c),n=c;t.setEdge(n,i,{weight:u.weight},a)}}function o(t){a.each(t.graph().dummyChains,function(e){var n,r=t.node(e),i=r.edgeLabel;for(t.setEdge(r.edgeObj,i);r.dummy;)n=t.successors(e)[0],t.removeNode(e),i.points.push({x:r.x,y:r.y}),"edge-label"===r.dummy&&(i.x=r.x,i.y=r.y,i.width=r.width,i.height=r.height),e=n,r=t.node(e)})}var a=t("./lodash"),s=t("./util");e.exports={run:r,undo:o}},{"./lodash":10,"./util":29}],13:[function(t,e,n){function r(t,e,n){var r,o={};i.each(n,function(n){for(var i,a,s=t.parent(n);s;){if(i=t.parent(s),i?(a=o[i],o[i]=s):(a=r,r=s),a&&a!==s)return void e.setEdge(a,s);s=i}})}var i=t("../lodash");e.exports=r},{"../lodash":10}],14:[function(t,e,n){function r(t,e){return i.map(e,function(e){var n=t.inEdges(e);if(n.length){var r=i.reduce(n,function(e,n){var r=t.edge(n),i=t.node(n.v);return{sum:e.sum+r.weight*i.order,weight:e.weight+r.weight}},{sum:0,weight:0});return{v:e,barycenter:r.sum/r.weight,weight:r.weight}}return{v:e}})}var i=t("../lodash");e.exports=r},{"../lodash":10}],15:[function(t,e,n){function r(t,e,n){var r=i(t),s=new a({compound:!0}).setGraph({root:r}).setDefaultNodeLabel(function(e){return t.node(e)});return o.each(t.nodes(),function(i){var a=t.node(i),u=t.parent(i);(a.rank===e||a.minRank<=e&&e<=a.maxRank)&&(s.setNode(i),s.setParent(i,u||r),o.each(t[n](i),function(e){var n=e.v===i?e.w:e.v,r=s.edge(n,i),a=o.isUndefined(r)?0:r.weight;s.setEdge(n,i,{weight:t.edge(e).weight+a})}),o.has(a,"minRank")&&s.setNode(i,{borderLeft:a.borderLeft[e],borderRight:a.borderRight[e]}))}),s}function i(t){for(var e;t.hasNode(e=o.uniqueId("_root")););return e}var o=t("../lodash"),a=t("../graphlib").Graph;e.exports=r},{"../graphlib":7,"../lodash":10}],16:[function(t,e,n){"use strict";function r(t,e){for(var n=0,r=1;r<e.length;++r)n+=i(t,e[r-1],e[r]);return n}function i(t,e,n){for(var r=o.zipObject(n,o.map(n,function(t,e){return e})),i=o.flatten(o.map(e,function(e){return o.chain(t.outEdges(e)).map(function(e){return{pos:r[e.w],weight:t.edge(e).weight}}).sortBy("pos").value()}),!0),a=1;a<n.length;)a<<=1;var s=2*a-1;a-=1;var u=o.map(new Array(s),function(){return 0}),l=0;return o.each(i.forEach(function(t){var e=t.pos+a;u[e]+=t.weight;for(var n=0;e>0;)e%2&&(n+=u[e+1]),e=e-1>>1,u[e]+=t.weight;l+=t.weight*n})),l}var o=t("../lodash");e.exports=r},{"../lodash":10}],17:[function(t,e,n){"use strict";function r(t){var e=p.maxRank(t),n=i(t,s.range(1,e+1),"inEdges"),r=i(t,s.range(e-1,-1,-1),"outEdges"),c=u(t);a(t,c);for(var f,d=Number.POSITIVE_INFINITY,h=0,g=0;g<4;++h,++g){o(h%2?n:r,h%4>=2),c=p.buildLayerMatrix(t);var v=l(t,c);v<d&&(g=0,f=s.cloneDeep(c),d=v)}a(t,f)}function i(t,e,n){return s.map(e,function(e){return f(t,e,n)})}function o(t,e){var n=new h;s.each(t,function(t){var r=t.graph().root,i=c(t,r,n,e);s.each(i.vs,function(e,n){t.node(e).order=n}),d(t,n,i.vs)})}function a(t,e){s.each(e,function(e){s.each(e,function(e,n){t.node(e).order=n})})}var s=t("../lodash"),u=t("./init-order"),l=t("./cross-count"),c=t("./sort-subgraph"),f=t("./build-layer-graph"),d=t("./add-subgraph-constraints"),h=t("../graphlib").Graph,p=t("../util");e.exports=r},{"../graphlib":7,"../lodash":10,"../util":29,"./add-subgraph-constraints":13,"./build-layer-graph":15,"./cross-count":16,"./init-order":18,"./sort-subgraph":20}],18:[function(t,e,n){"use strict";function r(t){function e(r){if(!i.has(n,r)){n[r]=!0;var o=t.node(r);a[o.rank].push(r),i.each(t.successors(r),e)}}var n={},r=i.filter(t.nodes(),function(e){return!t.children(e).length}),o=i.max(i.map(r,function(e){return t.node(e).rank})),a=i.map(i.range(o+1),function(){return[]}),s=i.sortBy(r,function(e){return t.node(e).rank});return i.each(s,e),a}var i=t("../lodash");e.exports=r},{"../lodash":10}],19:[function(t,e,n){"use strict";function r(t,e){var n={};a.each(t,function(t,e){var r=n[t.v]={indegree:0,"in":[],out:[],vs:[t.v],i:e};a.isUndefined(t.barycenter)||(r.barycenter=t.barycenter,r.weight=t.weight)}),a.each(e.edges(),function(t){var e=n[t.v],r=n[t.w];a.isUndefined(e)||a.isUndefined(r)||(r.indegree++,e.out.push(n[t.w]))});var r=a.filter(n,function(t){return!t.indegree});return i(r)}function i(t){function e(t){return function(e){e.merged||(a.isUndefined(e.barycenter)||a.isUndefined(t.barycenter)||e.barycenter>=t.barycenter)&&o(t,e)}}function n(e){return function(n){n["in"].push(e),0===--n.indegree&&t.push(n)}}for(var r=[];t.length;){var i=t.pop();r.push(i),a.each(i["in"].reverse(),e(i)),a.each(i.out,n(i))}return a.chain(r).filter(function(t){return!t.merged}).map(function(t){return a.pick(t,["vs","i","barycenter","weight"])}).value()}function o(t,e){var n=0,r=0;t.weight&&(n+=t.barycenter*t.weight,r+=t.weight),e.weight&&(n+=e.barycenter*e.weight,r+=e.weight),t.vs=e.vs.concat(t.vs),t.barycenter=n/r,t.weight=r,t.i=Math.min(e.i,t.i),e.merged=!0}var a=t("../lodash");e.exports=r},{"../lodash":10}],20:[function(t,e,n){function r(t,e,n,c){var f=t.children(e),d=t.node(e),h=d?d.borderLeft:void 0,p=d?d.borderRight:void 0,g={};h&&(f=a.filter(f,function(t){return t!==h&&t!==p}));var v=s(t,f);a.each(v,function(e){if(t.children(e.v).length){var i=r(t,e.v,n,c);g[e.v]=i,a.has(i,"barycenter")&&o(e,i)}});var m=u(v,n);i(m,g);var y=l(m,c);if(h&&(y.vs=a.flatten([h,y.vs,p],!0),t.predecessors(h).length)){var b=t.node(t.predecessors(h)[0]),x=t.node(t.predecessors(p)[0]);a.has(y,"barycenter")||(y.barycenter=0,y.weight=0),y.barycenter=(y.barycenter*y.weight+b.order+x.order)/(y.weight+2),y.weight+=2}return y}function i(t,e){a.each(t,function(t){t.vs=a.flatten(t.vs.map(function(t){return e[t]?e[t].vs:t}),!0)})}function o(t,e){a.isUndefined(t.barycenter)?(t.barycenter=e.barycenter,t.weight=e.weight):(t.barycenter=(t.barycenter*t.weight+e.barycenter*e.weight)/(t.weight+e.weight),t.weight+=e.weight)}var a=t("../lodash"),s=t("./barycenter"),u=t("./resolve-conflicts"),l=t("./sort");e.exports=r},{"../lodash":10,"./barycenter":14,"./resolve-conflicts":19,"./sort":21}],21:[function(t,e,n){function r(t,e){var n=s.partition(t,function(t){return a.has(t,"barycenter")}),r=n.lhs,u=a.sortBy(n.rhs,function(t){return-t.i}),l=[],c=0,f=0,d=0;r.sort(o(!!e)),d=i(l,u,d),a.each(r,function(t){d+=t.vs.length,l.push(t.vs),c+=t.barycenter*t.weight,f+=t.weight,d=i(l,u,d)});var h={vs:a.flatten(l,!0)};return f&&(h.barycenter=c/f,h.weight=f),h}function i(t,e,n){for(var r;e.length&&(r=a.last(e)).i<=n;)e.pop(),t.push(r.vs),n++;return n}function o(t){return function(e,n){return e.barycenter<n.barycenter?-1:e.barycenter>n.barycenter?1:t?n.i-e.i:e.i-n.i}}var a=t("../lodash"),s=t("../util");e.exports=r},{"../lodash":10,"../util":29}],22:[function(t,e,n){function r(t){var e=o(t);a.each(t.graph().dummyChains,function(n){for(var r=t.node(n),o=r.edgeObj,a=i(t,e,o.v,o.w),s=a.path,u=a.lca,l=0,c=s[l],f=!0;n!==o.w;){if(r=t.node(n),f){for(;(c=s[l])!==u&&t.node(c).maxRank<r.rank;)l++;c===u&&(f=!1)}if(!f){for(;l<s.length-1&&t.node(c=s[l+1]).minRank<=r.rank;)l++;c=s[l]}t.setParent(n,c),n=t.successors(n)[0]}})}function i(t,e,n,r){var i,o,a=[],s=[],u=Math.min(e[n].low,e[r].low),l=Math.max(e[n].lim,e[r].lim);i=n;do i=t.parent(i),a.push(i);while(i&&(e[i].low>u||l>e[i].lim));for(o=i,i=r;(i=t.parent(i))!==o;)s.push(i);return{path:a.concat(s.reverse()),lca:o}}function o(t){function e(i){var o=r;a.each(t.children(i),e),n[i]={low:o,lim:r++}}var n={},r=0;return a.each(t.children(),e),n}var a=t("./lodash");e.exports=r},{"./lodash":10}],23:[function(t,e,n){"use strict";function r(t,e){function n(e,n){var i=0,s=0,u=e.length,l=m.last(n);return m.each(n,function(e,c){var f=o(t,e),d=f?t.node(f).order:u;(f||e===l)&&(m.each(n.slice(s,c+1),function(e){m.each(t.predecessors(e),function(n){var o=t.node(n),s=o.order;!(s<i||d<s)||o.dummy&&t.node(e).dummy||a(r,n,e)})}),s=c+1,i=d)}),n}var r={};return m.reduce(e,n),r}function i(t,e){function n(e,n,r,o,s){var u;m.each(m.range(n,r),function(n){u=e[n],t.node(u).dummy&&m.each(t.predecessors(u),function(e){var n=t.node(e);n.dummy&&(n.order<o||n.order>s)&&a(i,e,u)})})}function r(e,r){var i,o=-1,a=0;return m.each(r,function(s,u){if("border"===t.node(s).dummy){var l=t.predecessors(s);l.length&&(i=t.node(l[0]).order,n(r,a,u,o,i),a=u,o=i)}n(r,a,r.length,i,e.length)}),r}var i={};return m.reduce(e,r),i}function o(t,e){if(t.node(e).dummy)return m.find(t.predecessors(e),function(e){return t.node(e).dummy})}function a(t,e,n){if(e>n){var r=e;e=n,n=r}var i=t[e];i||(t[e]=i={}),i[n]=!0}function s(t,e,n){if(e>n){var r=e;e=n,n=r}return m.has(t[e],n)}function u(t,e,n,r){var i={},o={},a={};return m.each(e,function(t){m.each(t,function(t,e){i[t]=t,o[t]=t,a[t]=e})}),m.each(e,function(t){var e=-1;m.each(t,function(t){var u=r(t);if(u.length){u=m.sortBy(u,function(t){return a[t]});for(var l=(u.length-1)/2,c=Math.floor(l),f=Math.ceil(l);c<=f;++c){var d=u[c];o[t]===t&&e<a[d]&&!s(n,t,d)&&(o[d]=t,o[t]=i[t]=i[d],e=a[d])}}})}),{root:i,align:o}}function l(t,e,n,r,i){function o(t){m.has(l,t)||(l[t]=!0,s[t]=m.reduce(u.inEdges(t),function(t,e){return o(e.v),Math.max(t,s[e.v]+u.edge(e))},0))}function a(e){if(2!==l[e]){l[e]++;var n=t.node(e),r=m.reduce(u.outEdges(e),function(t,e){return a(e.w),Math.min(t,s[e.w]-u.edge(e))},Number.POSITIVE_INFINITY);r!==Number.POSITIVE_INFINITY&&n.borderType!==f&&(s[e]=Math.max(s[e],r))}}var s={},u=c(t,e,n,i),l={};m.each(u.nodes(),o);var f=i?"borderLeft":"borderRight";return m.each(u.nodes(),a),m.each(r,function(t){s[t]=s[n[t]]}),s}function c(t,e,n,r){var i=new y,o=t.graph(),a=g(o.nodesep,o.edgesep,r);return m.each(e,function(e){var r;m.each(e,function(e){var o=n[e];if(i.setNode(o),r){var s=n[r],u=i.edge(s,o);i.setEdge(s,o,Math.max(a(t,e,r),u||0))}r=e})}),i}function f(t,e){return m.min(e,function(e){var n=m.min(e,function(e,n){return e-v(t,n)/2}),r=m.max(e,function(e,n){return e+v(t,n)/2});return r-n})}function d(t,e){var n=m.min(e),r=m.max(e);m.each(["u","d"],function(i){m.each(["l","r"],function(o){var a,s=i+o,u=t[s];u!==e&&(a="l"===o?n-m.min(u):r-m.max(u),a&&(t[s]=m.mapValues(u,function(t){return t+a})))})})}function h(t,e){return m.mapValues(t.ul,function(n,r){if(e)return t[e.toLowerCase()][r];var i=m.sortBy(m.pluck(t,r));return(i[1]+i[2])/2})}function p(t){var e,n=b.buildLayerMatrix(t),o=m.merge(r(t,n),i(t,n)),a={};m.each(["u","d"],function(r){e="u"===r?n:m.values(n).reverse(),m.each(["l","r"],function(n){"r"===n&&(e=m.map(e,function(t){return m.values(t).reverse()}));var i=m.bind("u"===r?t.predecessors:t.successors,t),s=u(t,e,o,i),c=l(t,e,s.root,s.align,"r"===n);"r"===n&&(c=m.mapValues(c,function(t){return-t})),a[r+n]=c})});var s=f(t,a);return d(a,s),h(a,t.graph().align)}function g(t,e,n){return function(r,i,o){var a,s=r.node(i),u=r.node(o),l=0;if(l+=s.width/2,m.has(s,"labelpos"))switch(s.labelpos.toLowerCase()){case"l":a=-s.width/2;break;case"r":a=s.width/2}if(a&&(l+=n?a:-a),a=0,l+=(s.dummy?e:t)/2,l+=(u.dummy?e:t)/2,l+=u.width/2,m.has(u,"labelpos"))switch(u.labelpos.toLowerCase()){case"l":a=u.width/2;break;case"r":a=-u.width/2}return a&&(l+=n?a:-a),a=0,l}}function v(t,e){return t.node(e).width}var m=t("../lodash"),y=t("../graphlib").Graph,b=t("../util");e.exports={positionX:p,findType1Conflicts:r,findType2Conflicts:i,addConflict:a,hasConflict:s,verticalAlignment:u,horizontalCompaction:l,alignCoordinates:d,findSmallestWidthAlignment:f,balance:h}},{"../graphlib":7,"../lodash":10,"../util":29}],24:[function(t,e,n){"use strict";function r(t){t=a.asNonCompoundGraph(t),i(t),o.each(s(t),function(e,n){t.node(n).x=e})}function i(t){var e=a.buildLayerMatrix(t),n=t.graph().ranksep,r=0;o.each(e,function(e){var i=o.max(o.map(e,function(e){return t.node(e).height}));o.each(e,function(e){t.node(e).y=r+i/2}),r+=i+n})}var o=t("../lodash"),a=t("../util"),s=t("./bk").positionX;e.exports=r},{"../lodash":10,"../util":29,"./bk":23}],25:[function(t,e,n){"use strict";function r(t){var e=new u({directed:!1}),n=t.nodes()[0],r=t.nodeCount();e.setNode(n,{});for(var s,c;i(e,t)<r;)s=o(e,t),c=e.hasNode(s.v)?l(t,s):-l(t,s),a(e,t,c);return e}function i(t,e){function n(r){s.each(e.nodeEdges(r),function(i){var o=i.v,a=r===o?i.w:o;t.hasNode(a)||l(e,i)||(t.setNode(a,{}),t.setEdge(r,a,{}),n(a))})}return s.each(t.nodes(),n),t.nodeCount()}function o(t,e){return s.min(e.edges(),function(n){if(t.hasNode(n.v)!==t.hasNode(n.w))return l(e,n)})}function a(t,e,n){s.each(t.nodes(),function(t){e.node(t).rank+=n})}var s=t("../lodash"),u=t("../graphlib").Graph,l=t("./util").slack;e.exports=r},{"../graphlib":7,"../lodash":10,"./util":28}],26:[function(t,e,n){"use strict";function r(t){switch(t.graph().ranker){case"network-simplex":o(t);break;case"tight-tree":i(t);break;case"longest-path":c(t);break;default:o(t)}}function i(t){s(t),u(t)}function o(t){l(t)}var a=t("./util"),s=a.longestPath,u=t("./feasible-tree"),l=t("./network-simplex");e.exports=r;var c=s},{"./feasible-tree":25,"./network-simplex":27,"./util":28}],27:[function(t,e,n){"use strict";function r(t){t=w(t),y(t);var e=v(t);s(e),i(e,t);for(var n,r;n=l(e);)r=c(e,t,n),f(e,t,n,r)}function i(t,e){var n=x(t,t.nodes());n=n.slice(0,n.length-1),g.each(n,function(n){o(t,e,n)})}function o(t,e,n){var r=t.node(n),i=r.parent;t.edge(n,i).cutvalue=a(t,e,n)}function a(t,e,n){var r=t.node(n),i=r.parent,o=!0,a=e.edge(n,i),s=0;return a||(o=!1,a=e.edge(i,n)),s=a.weight,g.each(e.nodeEdges(n),function(r){var a=r.v===n,u=a?r.w:r.v;if(u!==i){var l=a===o,c=e.edge(r).weight;if(s+=l?c:-c,h(t,n,u)){var f=t.edge(n,u).cutvalue;s+=l?-f:f}}}),s}function s(t,e){arguments.length<2&&(e=t.nodes()[0]),u(t,{},1,e)}function u(t,e,n,r,i){var o=n,a=t.node(r);return e[r]=!0,g.each(t.neighbors(r),function(i){g.has(e,i)||(n=u(t,e,n,i,r))}),a.low=o,a.lim=n++,i?a.parent=i:delete a.parent,n}function l(t){return g.find(t.edges(),function(e){return t.edge(e).cutvalue<0})}function c(t,e,n){var r=n.v,i=n.w;e.hasEdge(r,i)||(r=n.w,i=n.v);var o=t.node(r),a=t.node(i),s=o,u=!1;o.lim>a.lim&&(s=a,u=!0);var l=g.filter(e.edges(),function(e){return u===p(t,t.node(e.v),s)&&u!==p(t,t.node(e.w),s)});return g.min(l,function(t){return m(e,t)})}function f(t,e,n,r){var o=n.v,a=n.w;t.removeEdge(o,a),t.setEdge(r.v,r.w,{}),s(t),i(t,e),d(t,e)}function d(t,e){var n=g.find(t.nodes(),function(t){return!e.node(t).parent}),r=b(t,n);r=r.slice(1),g.each(r,function(n){var r=t.node(n).parent,i=e.edge(n,r),o=!1;i||(i=e.edge(r,n),o=!0),e.node(n).rank=e.node(r).rank+(o?i.minlen:-i.minlen)})}function h(t,e,n){return t.hasEdge(e,n)}function p(t,e,n){return n.low<=e.lim&&e.lim<=n.lim}var g=t("../lodash"),v=t("./feasible-tree"),m=t("./util").slack,y=t("./util").longestPath,b=t("../graphlib").alg.preorder,x=t("../graphlib").alg.postorder,w=t("../util").simplify;e.exports=r,r.initLowLimValues=s,r.initCutValues=i,r.calcCutValue=a,r.leaveEdge=l,r.enterEdge=c,r.exchangeEdges=f},{"../graphlib":7,"../lodash":10,"../util":29,"./feasible-tree":25,"./util":28}],28:[function(t,e,n){"use strict";function r(t){function e(r){var i=t.node(r);if(o.has(n,r))return i.rank;n[r]=!0;var a=o.min(o.map(t.outEdges(r),function(n){return e(n.w)-t.edge(n).minlen}));return a===Number.POSITIVE_INFINITY&&(a=0),i.rank=a}var n={};o.each(t.sources(),e)}function i(t,e){return t.node(e.w).rank-t.node(e.v).rank-t.edge(e).minlen}var o=t("../lodash");e.exports={longestPath:r,slack:i}},{"../lodash":10}],29:[function(t,e,n){"use strict";function r(t,e,n,r){var i;do i=m.uniqueId(r);while(t.hasNode(i));return n.dummy=e,t.setNode(i,n),i}function i(t){var e=(new y).setGraph(t.graph());return m.each(t.nodes(),function(n){e.setNode(n,t.node(n))}),m.each(t.edges(),function(n){var r=e.edge(n.v,n.w)||{weight:0,minlen:1},i=t.edge(n);e.setEdge(n.v,n.w,{weight:r.weight+i.weight,minlen:Math.max(r.minlen,i.minlen)})}),e}function o(t){var e=new y({multigraph:t.isMultigraph()}).setGraph(t.graph());return m.each(t.nodes(),function(n){t.children(n).length||e.setNode(n,t.node(n))}),m.each(t.edges(),function(n){e.setEdge(n,t.edge(n))}),e}function a(t){var e=m.map(t.nodes(),function(e){var n={};return m.each(t.outEdges(e),function(e){n[e.w]=(n[e.w]||0)+t.edge(e).weight}),n});return m.zipObject(t.nodes(),e)}function s(t){var e=m.map(t.nodes(),function(e){var n={};return m.each(t.inEdges(e),function(e){n[e.v]=(n[e.v]||0)+t.edge(e).weight}),n});return m.zipObject(t.nodes(),e)}function u(t,e){var n=t.x,r=t.y,i=e.x-n,o=e.y-r,a=t.width/2,s=t.height/2;if(!i&&!o)throw new Error("Not possible to find intersection inside of the rectangle");var u,l;return Math.abs(o)*a>Math.abs(i)*s?(o<0&&(s=-s),u=s*i/o,l=s):(i<0&&(a=-a),u=a,l=a*o/i),{x:n+u,y:r+l}}function l(t){var e=m.map(m.range(h(t)+1),function(){return[]});return m.each(t.nodes(),function(n){var r=t.node(n),i=r.rank;m.isUndefined(i)||(e[i][r.order]=n)}),e}function c(t){var e=m.min(m.map(t.nodes(),function(e){return t.node(e).rank}));m.each(t.nodes(),function(n){var r=t.node(n);m.has(r,"rank")&&(r.rank-=e)})}function f(t){var e=m.min(m.map(t.nodes(),function(e){return t.node(e).rank})),n=[];m.each(t.nodes(),function(r){var i=t.node(r).rank-e;n[i]||(n[i]=[]),n[i].push(r)});var r=0,i=t.graph().nodeRankFactor;m.each(n,function(e,n){m.isUndefined(e)&&n%i!==0?--r:r&&m.each(e,function(e){t.node(e).rank+=r})})}function d(t,e,n,i){var o={width:0,height:0};return arguments.length>=4&&(o.rank=n,o.order=i),r(t,"border",o,e)}function h(t){return m.max(m.map(t.nodes(),function(e){var n=t.node(e).rank;if(!m.isUndefined(n))return n}))}function p(t,e){var n={lhs:[],rhs:[]};return m.each(t,function(t){e(t)?n.lhs.push(t):n.rhs.push(t)}),n}function g(t,e){var n=m.now();try{return e()}finally{console.log(t+" time: "+(m.now()-n)+"ms")}}function v(t,e){return e()}var m=t("./lodash"),y=t("./graphlib").Graph;e.exports={addDummyNode:r,simplify:i,asNonCompoundGraph:o,successorWeights:a,predecessorWeights:s,intersectRect:u,buildLayerMatrix:l,normalizeRanks:c,removeEmptyRanks:f,addBorderNode:d,maxRank:h,partition:p,time:g,notime:v}},{"./graphlib":7,"./lodash":10}],30:[function(t,e,n){e.exports="0.7.4"},{}]},{},[1])(1)}),function(t,e,n){!function(t){"use strict";"function"==typeof define&&define.amd?define(["jquery"],t):jQuery&&!jQuery.fn.qtip&&t(jQuery)}(function(r){"use strict";function i(t,e,n,i){this.id=n,this.target=t,this.tooltip=N,this.elements={target:t},this._id=V+"-"+n,this.timers={img:{}},this.options=e,this.plugins={},this.cache={event:{},target:r(),disabled:T,attr:i,onTooltip:T,lastClass:""},this.rendered=this.destroyed=this.disabled=this.waiting=this.hiddenDuringWait=this.positioning=this.triggering=T}function o(t){return t===N||"object"!==r.type(t)}function a(t){return!(r.isFunction(t)||t&&t.attr||t.length||"object"===r.type(t)&&(t.jquery||t.then))}function s(t){var e,n,i,s;return o(t)?T:(o(t.metadata)&&(t.metadata={type:t.metadata}),"content"in t&&(e=t.content,o(e)||e.jquery||e.done?e=t.content={text:n=a(e)?T:e}:n=e.text,"ajax"in e&&(i=e.ajax,s=i&&i.once!==T,delete e.ajax,e.text=function(t,e){var o=n||r(this).attr(e.options.content.attr)||"Loading...",a=r.ajax(r.extend({},i,{context:e})).then(i.success,N,i.error).then(function(t){return t&&s&&e.set("content.text",t),t},function(t,n,r){e.destroyed||0===t.status||e.set("content.text",n+": "+r)});return s?o:(e.set("content.text",o),a)}),"title"in e&&(r.isPlainObject(e.title)&&(e.button=e.title.button,e.title=e.title.text),a(e.title||T)&&(e.title=T))),"position"in t&&o(t.position)&&(t.position={my:t.position,at:t.position}),"show"in t&&o(t.show)&&(t.show=t.show.jquery?{target:t.show}:t.show===A?{ready:A}:{event:t.show}),"hide"in t&&o(t.hide)&&(t.hide=t.hide.jquery?{target:t.hide}:{event:t.hide}),"style"in t&&o(t.style)&&(t.style={classes:t.style}),r.each(B,function(){this.sanitize&&this.sanitize(t)}),t)}function u(t,e){for(var n,r=0,i=t,o=e.split(".");i=i[o[r++]];)r<o.length&&(n=i);return[n||t,o.pop()]}function l(t,e){var n,r,i;for(n in this.checks)for(r in this.checks[n])(i=new RegExp(r,"i").exec(t))&&(e.push(i),("builtin"===n||this.plugins[n])&&this.checks[n][r].apply(this.plugins[n]||this,e))}function c(t){return Y.concat("").join(t?"-"+t+" ":" ")}function f(t,e){return e>0?setTimeout(r.proxy(t,this),e):void t.call(this)}function d(t){this.tooltip.hasClass(tt)||(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this.timers.show=f.call(this,function(){this.toggle(A,t)},this.options.show.delay))}function h(t){if(!this.tooltip.hasClass(tt)&&!this.destroyed){var e=r(t.relatedTarget),n=e.closest(G)[0]===this.tooltip[0],i=e[0]===this.options.show.target[0];if(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this!==e[0]&&"mouse"===this.options.position.target&&n||this.options.hide.fixed&&/mouse(out|leave|move)/.test(t.type)&&(n||i))try{t.preventDefault(),t.stopImmediatePropagation()}catch(o){}else this.timers.hide=f.call(this,function(){this.toggle(T,t)},this.options.hide.delay,this)}}function p(t){!this.tooltip.hasClass(tt)&&this.options.hide.inactive&&(clearTimeout(this.timers.inactive),this.timers.inactive=f.call(this,function(){this.hide(t)},this.options.hide.inactive))}function g(t){this.rendered&&this.tooltip[0].offsetWidth>0&&this.reposition(t)}function v(t,n,i){r(e.body).delegate(t,(n.split?n:n.join("."+V+" "))+"."+V,function(){var t=_.api[r.attr(this,U)];t&&!t.disabled&&i.apply(t,arguments)})}function m(t,n,o){var a,u,l,c,f,d=r(e.body),h=t[0]===e?d:t,p=t.metadata?t.metadata(o.metadata):N,g="html5"===o.metadata.type&&p?p[o.metadata.name]:N,v=t.data(o.metadata.name||"qtipopts");try{v="string"==typeof v?r.parseJSON(v):v}catch(m){}if(c=r.extend(A,{},_.defaults,o,"object"==typeof v?s(v):N,s(g||p)),u=c.position,c.id=n,"boolean"==typeof c.content.text){if(l=t.attr(c.content.attr),c.content.attr===T||!l)return T;c.content.text=l}if(u.container.length||(u.container=d),u.target===T&&(u.target=h),c.show.target===T&&(c.show.target=h),c.show.solo===A&&(c.show.solo=u.container.closest("body")),c.hide.target===T&&(c.hide.target=h),c.position.viewport===A&&(c.position.viewport=u.container),u.container=u.container.eq(0),u.at=new C(u.at,A),u.my=new C(u.my),t.data(V))if(c.overwrite)t.qtip("destroy",!0);else if(c.overwrite===T)return T;return t.attr(H,n),c.suppress&&(f=t.attr("title"))&&t.removeAttr("title").attr(nt,f).attr("title",""),a=new i(t,c,n,(!!l)),t.data(V,a),a}function y(t){return t.charAt(0).toUpperCase()+t.slice(1)}function b(t,e){var r,i,o=e.charAt(0).toUpperCase()+e.slice(1),a=(e+" "+mt.join(o+" ")+o).split(" "),s=0;if(vt[e])return t.css(vt[e]);for(;r=a[s++];)if((i=t.css(r))!==n)return vt[e]=r,i}function x(t,e){return Math.ceil(parseFloat(b(t,e)))}function w(t,e){this._ns="tip",this.options=e,this.offset=e.offset,this.size=[e.width,e.height],this.init(this.qtip=t)}function $(t,e){this.options=e,this._ns="-modal",this.init(this.qtip=t)}function k(t,e){this._ns="ie6",this.init(this.qtip=t)}var _,M,C,S,E,A=!0,T=!1,N=null,D="x",O="y",j="width",I="height",L="top",P="left",F="bottom",q="right",z="center",W="flipinvert",R="shift",B={},V="qtip",H="data-hasqtip",U="data-qtip-id",Y=["ui-widget","ui-tooltip"],G="."+V,X="click dblclick mousedown mouseup mousemove mouseleave mouseenter".split(" "),Z=V+"-fixed",Q=V+"-default",K=V+"-focus",J=V+"-hover",tt=V+"-disabled",et="_replacedByqTip",nt="oldtitle",rt={ie:function(){for(var t=4,n=e.createElement("div");(n.innerHTML="<!--[if gt IE "+t+"]><i></i><![endif]-->")&&n.getElementsByTagName("i")[0];t+=1);return t>4?t:NaN}(),iOS:parseFloat((""+(/CPU.*OS ([0-9_]{1,5})|(CPU like).*AppleWebKit.*Mobile/i.exec(navigator.userAgent)||[0,""])[1]).replace("undefined","3_2").replace("_",".").replace("_",""))||T};M=i.prototype,M._when=function(t){return r.when.apply(r,t)},M.render=function(t){if(this.rendered||this.destroyed)return this;var e,n=this,i=this.options,o=this.cache,a=this.elements,s=i.content.text,u=i.content.title,l=i.content.button,c=i.position,f=("."+this._id+" ",[]);return r.attr(this.target[0],"aria-describedby",this._id),o.posClass=this._createPosClass((this.position={my:c.my,at:c.at}).my),this.tooltip=a.tooltip=e=r("<div/>",{id:this._id,"class":[V,Q,i.style.classes,o.posClass].join(" "),width:i.style.width||"",height:i.style.height||"",tracking:"mouse"===c.target&&c.adjust.mouse,role:"alert","aria-live":"polite","aria-atomic":T,"aria-describedby":this._id+"-content","aria-hidden":A}).toggleClass(tt,this.disabled).attr(U,this.id).data(V,this).appendTo(c.container).append(a.content=r("<div />",{"class":V+"-content",id:this._id+"-content","aria-atomic":A})),this.rendered=-1,this.positioning=A,u&&(this._createTitle(),r.isFunction(u)||f.push(this._updateTitle(u,T))),l&&this._createButton(),r.isFunction(s)||f.push(this._updateContent(s,T)),this.rendered=A,this._setWidget(),r.each(B,function(t){var e;"render"===this.initialize&&(e=this(n))&&(n.plugins[t]=e)}),this._unassignEvents(),this._assignEvents(),this._when(f).then(function(){n._trigger("render"),n.positioning=T,n.hiddenDuringWait||!i.show.ready&&!t||n.toggle(A,o.event,T),n.hiddenDuringWait=T}),_.api[this.id]=this,this},M.destroy=function(t){function e(){if(!this.destroyed){this.destroyed=A;var t,e=this.target,n=e.attr(nt);this.rendered&&this.tooltip.stop(1,0).find("*").remove().end().remove(),r.each(this.plugins,function(t){
-this.destroy&&this.destroy()});for(t in this.timers)clearTimeout(this.timers[t]);e.removeData(V).removeAttr(U).removeAttr(H).removeAttr("aria-describedby"),this.options.suppress&&n&&e.attr("title",n).removeAttr(nt),this._unassignEvents(),this.options=this.elements=this.cache=this.timers=this.plugins=this.mouse=N,delete _.api[this.id]}}return this.destroyed?this.target:(t===A&&"hide"!==this.triggering||!this.rendered?e.call(this):(this.tooltip.one("tooltiphidden",r.proxy(e,this)),!this.triggering&&this.hide()),this.target)},S=M.checks={builtin:{"^id$":function(t,e,n,i){var o=n===A?_.nextid:n,a=V+"-"+o;o!==T&&o.length>0&&!r("#"+a).length?(this._id=a,this.rendered&&(this.tooltip[0].id=this._id,this.elements.content[0].id=this._id+"-content",this.elements.title[0].id=this._id+"-title")):t[e]=i},"^prerender":function(t,e,n){n&&!this.rendered&&this.render(this.options.show.ready)},"^content.text$":function(t,e,n){this._updateContent(n)},"^content.attr$":function(t,e,n,r){this.options.content.text===this.target.attr(r)&&this._updateContent(this.target.attr(n))},"^content.title$":function(t,e,n){return n?(n&&!this.elements.title&&this._createTitle(),void this._updateTitle(n)):this._removeTitle()},"^content.button$":function(t,e,n){this._updateButton(n)},"^content.title.(text|button)$":function(t,e,n){this.set("content."+e,n)},"^position.(my|at)$":function(t,e,n){"string"==typeof n&&(this.position[e]=t[e]=new C(n,"at"===e))},"^position.container$":function(t,e,n){this.rendered&&this.tooltip.appendTo(n)},"^show.ready$":function(t,e,n){n&&(!this.rendered&&this.render(A)||this.toggle(A))},"^style.classes$":function(t,e,n,r){this.rendered&&this.tooltip.removeClass(r).addClass(n)},"^style.(width|height)":function(t,e,n){this.rendered&&this.tooltip.css(e,n)},"^style.widget|content.title":function(){this.rendered&&this._setWidget()},"^style.def":function(t,e,n){this.rendered&&this.tooltip.toggleClass(Q,!!n)},"^events.(render|show|move|hide|focus|blur)$":function(t,e,n){this.rendered&&this.tooltip[(r.isFunction(n)?"":"un")+"bind"]("tooltip"+e,n)},"^(show|hide|position).(event|target|fixed|inactive|leave|distance|viewport|adjust)":function(){if(this.rendered){var t=this.options.position;this.tooltip.attr("tracking","mouse"===t.target&&t.adjust.mouse),this._unassignEvents(),this._assignEvents()}}}},M.get=function(t){if(this.destroyed)return this;var e=u(this.options,t.toLowerCase()),n=e[0][e[1]];return n.precedance?n.string():n};var it=/^position\.(my|at|adjust|target|container|viewport)|style|content|show\.ready/i,ot=/^prerender|show\.ready/i;M.set=function(t,e){if(this.destroyed)return this;var n,i=this.rendered,o=T,a=this.options;this.checks;return"string"==typeof t?(n=t,t={},t[n]=e):t=r.extend({},t),r.each(t,function(e,n){if(i&&ot.test(e))return void delete t[e];var s,l=u(a,e.toLowerCase());s=l[0][l[1]],l[0][l[1]]=n&&n.nodeType?r(n):n,o=it.test(e)||o,t[e]=[l[0],l[1],n,s]}),s(a),this.positioning=A,r.each(t,r.proxy(l,this)),this.positioning=T,this.rendered&&this.tooltip[0].offsetWidth>0&&o&&this.reposition("mouse"===a.position.target?N:this.cache.event),this},M._update=function(t,e,n){var i=this,o=this.cache;return this.rendered&&t?(r.isFunction(t)&&(t=t.call(this.elements.target,o.event,this)||""),r.isFunction(t.then)?(o.waiting=A,t.then(function(t){return o.waiting=T,i._update(t,e)},N,function(t){return i._update(t,e)})):t===T||!t&&""!==t?T:(t.jquery&&t.length>0?e.empty().append(t.css({display:"block",visibility:"visible"})):e.html(t),this._waitForContent(e).then(function(t){i.rendered&&i.tooltip[0].offsetWidth>0&&i.reposition(o.event,!t.length)}))):T},M._waitForContent=function(t){var e=this.cache;return e.waiting=A,(r.fn.imagesLoaded?t.imagesLoaded():r.Deferred().resolve([])).done(function(){e.waiting=T}).promise()},M._updateContent=function(t,e){this._update(t,this.elements.content,e)},M._updateTitle=function(t,e){this._update(t,this.elements.title,e)===T&&this._removeTitle(T)},M._createTitle=function(){var t=this.elements,e=this._id+"-title";t.titlebar&&this._removeTitle(),t.titlebar=r("<div />",{"class":V+"-titlebar "+(this.options.style.widget?c("header"):"")}).append(t.title=r("<div />",{id:e,"class":V+"-title","aria-atomic":A})).insertBefore(t.content).delegate(".qtip-close","mousedown keydown mouseup keyup mouseout",function(t){r(this).toggleClass("ui-state-active ui-state-focus","down"===t.type.substr(-4))}).delegate(".qtip-close","mouseover mouseout",function(t){r(this).toggleClass("ui-state-hover","mouseover"===t.type)}),this.options.content.button&&this._createButton()},M._removeTitle=function(t){var e=this.elements;e.title&&(e.titlebar.remove(),e.titlebar=e.title=e.button=N,t!==T&&this.reposition())},M._createPosClass=function(t){return V+"-pos-"+(t||this.options.position.my).abbrev()},M.reposition=function(n,i){if(!this.rendered||this.positioning||this.destroyed)return this;this.positioning=A;var o,a,s,u,l=this.cache,c=this.tooltip,f=this.options.position,d=f.target,h=f.my,p=f.at,g=f.viewport,v=f.container,m=f.adjust,y=m.method.split(" "),b=c.outerWidth(T),x=c.outerHeight(T),w=0,$=0,k=c.css("position"),_={left:0,top:0},M=c[0].offsetWidth>0,C=n&&"scroll"===n.type,S=r(t),E=v[0].ownerDocument,N=this.mouse;if(r.isArray(d)&&2===d.length)p={x:P,y:L},_={left:d[0],top:d[1]};else if("mouse"===d)p={x:P,y:L},(!m.mouse||this.options.hide.distance)&&l.origin&&l.origin.pageX?n=l.origin:!n||n&&("resize"===n.type||"scroll"===n.type)?n=l.event:N&&N.pageX&&(n=N),"static"!==k&&(_=v.offset()),E.body.offsetWidth!==(t.innerWidth||E.documentElement.clientWidth)&&(a=r(e.body).offset()),_={left:n.pageX-_.left+(a&&a.left||0),top:n.pageY-_.top+(a&&a.top||0)},m.mouse&&C&&N&&(_.left-=(N.scrollX||0)-S.scrollLeft(),_.top-=(N.scrollY||0)-S.scrollTop());else{if("event"===d?n&&n.target&&"scroll"!==n.type&&"resize"!==n.type?l.target=r(n.target):n.target||(l.target=this.elements.target):"event"!==d&&(l.target=r(d.jquery?d:this.elements.target)),d=l.target,d=r(d).eq(0),0===d.length)return this;d[0]===e||d[0]===t?(w=rt.iOS?t.innerWidth:d.width(),$=rt.iOS?t.innerHeight:d.height(),d[0]===t&&(_={top:(g||d).scrollTop(),left:(g||d).scrollLeft()})):B.imagemap&&d.is("area")?o=B.imagemap(this,d,p,B.viewport?y:T):B.svg&&d&&d[0].ownerSVGElement?o=B.svg(this,d,p,B.viewport?y:T):(w=d.outerWidth(T),$=d.outerHeight(T),_=d.offset()),o&&(w=o.width,$=o.height,a=o.offset,_=o.position),_=this.reposition.offset(d,_,v),(rt.iOS>3.1&&rt.iOS<4.1||rt.iOS>=4.3&&rt.iOS<4.33||!rt.iOS&&"fixed"===k)&&(_.left-=S.scrollLeft(),_.top-=S.scrollTop()),(!o||o&&o.adjustable!==T)&&(_.left+=p.x===q?w:p.x===z?w/2:0,_.top+=p.y===F?$:p.y===z?$/2:0)}return _.left+=m.x+(h.x===q?-b:h.x===z?-b/2:0),_.top+=m.y+(h.y===F?-x:h.y===z?-x/2:0),B.viewport?(s=_.adjusted=B.viewport(this,_,f,w,$,b,x),a&&s.left&&(_.left+=a.left),a&&s.top&&(_.top+=a.top),s.my&&(this.position.my=s.my)):_.adjusted={left:0,top:0},l.posClass!==(u=this._createPosClass(this.position.my))&&c.removeClass(l.posClass).addClass(l.posClass=u),this._trigger("move",[_,g.elem||g],n)?(delete _.adjusted,i===T||!M||isNaN(_.left)||isNaN(_.top)||"mouse"===d||!r.isFunction(f.effect)?c.css(_):r.isFunction(f.effect)&&(f.effect.call(c,this,r.extend({},_)),c.queue(function(t){r(this).css({opacity:"",height:""}),rt.ie&&this.style.removeAttribute("filter"),t()})),this.positioning=T,this):this},M.reposition.offset=function(t,n,i){function o(t,e){n.left+=e*t.scrollLeft(),n.top+=e*t.scrollTop()}if(!i[0])return n;var a,s,u,l,c=r(t[0].ownerDocument),f=!!rt.ie&&"CSS1Compat"!==e.compatMode,d=i[0];do"static"!==(s=r.css(d,"position"))&&("fixed"===s?(u=d.getBoundingClientRect(),o(c,-1)):(u=r(d).position(),u.left+=parseFloat(r.css(d,"borderLeftWidth"))||0,u.top+=parseFloat(r.css(d,"borderTopWidth"))||0),n.left-=u.left+(parseFloat(r.css(d,"marginLeft"))||0),n.top-=u.top+(parseFloat(r.css(d,"marginTop"))||0),a||"hidden"===(l=r.css(d,"overflow"))||"visible"===l||(a=r(d)));while(d=d.offsetParent);return a&&(a[0]!==c[0]||f)&&o(a,1),n};var at=(C=M.reposition.Corner=function(t,e){t=(""+t).replace(/([A-Z])/," $1").replace(/middle/gi,z).toLowerCase(),this.x=(t.match(/left|right/i)||t.match(/center/)||["inherit"])[0].toLowerCase(),this.y=(t.match(/top|bottom|center/i)||["inherit"])[0].toLowerCase(),this.forceY=!!e;var n=t.charAt(0);this.precedance="t"===n||"b"===n?O:D}).prototype;at.invert=function(t,e){this[t]=this[t]===P?q:this[t]===q?P:e||this[t]},at.string=function(t){var e=this.x,n=this.y,r=e!==n?"center"===e||"center"!==n&&(this.precedance===O||this.forceY)?[n,e]:[e,n]:[e];return t!==!1?r.join(" "):r},at.abbrev=function(){var t=this.string(!1);return t[0].charAt(0)+(t[1]&&t[1].charAt(0)||"")},at.clone=function(){return new C(this.string(),this.forceY)},M.toggle=function(t,n){var i=this.cache,o=this.options,a=this.tooltip;if(n){if(/over|enter/.test(n.type)&&i.event&&/out|leave/.test(i.event.type)&&o.show.target.add(n.target).length===o.show.target.length&&a.has(n.relatedTarget).length)return this;i.event=r.event.fix(n)}if(this.waiting&&!t&&(this.hiddenDuringWait=A),!this.rendered)return t?this.render(1):this;if(this.destroyed||this.disabled)return this;var s,u,l,c=t?"show":"hide",f=this.options[c],d=(this.options[t?"hide":"show"],this.options.position),h=this.options.content,p=this.tooltip.css("width"),g=this.tooltip.is(":visible"),v=t||1===f.target.length,m=!n||f.target.length<2||i.target[0]===n.target;return(typeof t).search("boolean|number")&&(t=!g),s=!a.is(":animated")&&g===t&&m,u=s?N:!!this._trigger(c,[90]),this.destroyed?this:(u!==T&&t&&this.focus(n),!u||s?this:(r.attr(a[0],"aria-hidden",!t),t?(this.mouse&&(i.origin=r.event.fix(this.mouse)),r.isFunction(h.text)&&this._updateContent(h.text,T),r.isFunction(h.title)&&this._updateTitle(h.title,T),!E&&"mouse"===d.target&&d.adjust.mouse&&(r(e).bind("mousemove."+V,this._storeMouse),E=A),p||a.css("width",a.outerWidth(T)),this.reposition(n,arguments[2]),p||a.css("width",""),f.solo&&("string"==typeof f.solo?r(f.solo):r(G,f.solo)).not(a).not(f.target).qtip("hide",r.Event("tooltipsolo"))):(clearTimeout(this.timers.show),delete i.origin,E&&!r(G+'[tracking="true"]:visible',f.solo).not(a).length&&(r(e).unbind("mousemove."+V),E=T),this.blur(n)),l=r.proxy(function(){t?(rt.ie&&a[0].style.removeAttribute("filter"),a.css("overflow",""),"string"==typeof f.autofocus&&r(this.options.show.autofocus,a).focus(),this.options.show.target.trigger("qtip-"+this.id+"-inactive")):a.css({display:"",visibility:"",opacity:"",left:"",top:""}),this._trigger(t?"visible":"hidden")},this),f.effect===T||v===T?(a[c](),l()):r.isFunction(f.effect)?(a.stop(1,1),f.effect.call(a,this),a.queue("fx",function(t){l(),t()})):a.fadeTo(90,t?1:0,l),t&&f.target.trigger("qtip-"+this.id+"-inactive"),this))},M.show=function(t){return this.toggle(A,t)},M.hide=function(t){return this.toggle(T,t)},M.focus=function(t){if(!this.rendered||this.destroyed)return this;var e=r(G),n=this.tooltip,i=parseInt(n[0].style.zIndex,10),o=_.zindex+e.length;return n.hasClass(K)||this._trigger("focus",[o],t)&&(i!==o&&(e.each(function(){this.style.zIndex>i&&(this.style.zIndex=this.style.zIndex-1)}),e.filter("."+K).qtip("blur",t)),n.addClass(K)[0].style.zIndex=o),this},M.blur=function(t){return!this.rendered||this.destroyed?this:(this.tooltip.removeClass(K),this._trigger("blur",[this.tooltip.css("zIndex")],t),this)},M.disable=function(t){return this.destroyed?this:("toggle"===t?t=!(this.rendered?this.tooltip.hasClass(tt):this.disabled):"boolean"!=typeof t&&(t=A),this.rendered&&this.tooltip.toggleClass(tt,t).attr("aria-disabled",t),this.disabled=!!t,this)},M.enable=function(){return this.disable(T)},M._createButton=function(){var t=this,e=this.elements,n=e.tooltip,i=this.options.content.button,o="string"==typeof i,a=o?i:"Close tooltip";e.button&&e.button.remove(),i.jquery?e.button=i:e.button=r("<a />",{"class":"qtip-close "+(this.options.style.widget?"":V+"-icon"),title:a,"aria-label":a}).prepend(r("<span />",{"class":"ui-icon ui-icon-close",html:"&times;"})),e.button.appendTo(e.titlebar||n).attr("role","button").click(function(e){return n.hasClass(tt)||t.hide(e),T})},M._updateButton=function(t){if(!this.rendered)return T;var e=this.elements.button;t?this._createButton():e.remove()},M._setWidget=function(){var t=this.options.style.widget,e=this.elements,n=e.tooltip,r=n.hasClass(tt);n.removeClass(tt),tt=t?"ui-state-disabled":"qtip-disabled",n.toggleClass(tt,r),n.toggleClass("ui-helper-reset "+c(),t).toggleClass(Q,this.options.style.def&&!t),e.content&&e.content.toggleClass(c("content"),t),e.titlebar&&e.titlebar.toggleClass(c("header"),t),e.button&&e.button.toggleClass(V+"-icon",!t)},M._storeMouse=function(t){return(this.mouse=r.event.fix(t)).type="mousemove",this},M._bind=function(t,e,n,i,o){if(t&&n&&e.length){var a="."+this._id+(i?"-"+i:"");return r(t).bind((e.split?e:e.join(a+" "))+a,r.proxy(n,o||this)),this}},M._unbind=function(t,e){return t&&r(t).unbind("."+this._id+(e?"-"+e:"")),this},M._trigger=function(t,e,n){var i=r.Event("tooltip"+t);return i.originalEvent=n&&r.extend({},n)||this.cache.event||N,this.triggering=t,this.tooltip.trigger(i,[this].concat(e||[])),this.triggering=T,!i.isDefaultPrevented()},M._bindEvents=function(t,e,n,i,o,a){var s=n.filter(i).add(i.filter(n)),u=[];s.length&&(r.each(e,function(e,n){var i=r.inArray(n,t);i>-1&&u.push(t.splice(i,1)[0])}),u.length&&(this._bind(s,u,function(t){var e=!!this.rendered&&this.tooltip[0].offsetWidth>0;(e?a:o).call(this,t)}),n=n.not(s),i=i.not(s))),this._bind(n,t,o),this._bind(i,e,a)},M._assignInitialEvents=function(t){function e(t){return this.disabled||this.destroyed?T:(this.cache.event=t&&r.event.fix(t),this.cache.target=t&&r(t.target),clearTimeout(this.timers.show),void(this.timers.show=f.call(this,function(){this.render("object"==typeof t||n.show.ready)},n.prerender?0:n.show.delay)))}var n=this.options,i=n.show.target,o=n.hide.target,a=n.show.event?r.trim(""+n.show.event).split(" "):[],s=n.hide.event?r.trim(""+n.hide.event).split(" "):[];this._bind(this.elements.target,["remove","removeqtip"],function(t){this.destroy(!0)},"destroy"),/mouse(over|enter)/i.test(n.show.event)&&!/mouse(out|leave)/i.test(n.hide.event)&&s.push("mouseleave"),this._bind(i,"mousemove",function(t){this._storeMouse(t),this.cache.onTarget=A}),this._bindEvents(a,s,i,o,e,function(){return this.timers?void clearTimeout(this.timers.show):T}),(n.show.ready||n.prerender)&&e.call(this,t)},M._assignEvents=function(){var n=this,i=this.options,o=i.position,a=this.tooltip,s=i.show.target,u=i.hide.target,l=o.container,c=o.viewport,f=r(e),v=(r(e.body),r(t)),m=i.show.event?r.trim(""+i.show.event).split(" "):[],y=i.hide.event?r.trim(""+i.hide.event).split(" "):[];r.each(i.events,function(t,e){n._bind(a,"toggle"===t?["tooltipshow","tooltiphide"]:["tooltip"+t],e,null,a)}),/mouse(out|leave)/i.test(i.hide.event)&&"window"===i.hide.leave&&this._bind(f,["mouseout","blur"],function(t){/select|option/.test(t.target.nodeName)||t.relatedTarget||this.hide(t)}),i.hide.fixed?u=u.add(a.addClass(Z)):/mouse(over|enter)/i.test(i.show.event)&&this._bind(u,"mouseleave",function(){clearTimeout(this.timers.show)}),(""+i.hide.event).indexOf("unfocus")>-1&&this._bind(l.closest("html"),["mousedown","touchstart"],function(t){var e=r(t.target),n=this.rendered&&!this.tooltip.hasClass(tt)&&this.tooltip[0].offsetWidth>0,i=e.parents(G).filter(this.tooltip[0]).length>0;e[0]===this.target[0]||e[0]===this.tooltip[0]||i||this.target.has(e[0]).length||!n||this.hide(t)}),"number"==typeof i.hide.inactive&&(this._bind(s,"qtip-"+this.id+"-inactive",p,"inactive"),this._bind(u.add(a),_.inactiveEvents,p)),this._bindEvents(m,y,s,u,d,h),this._bind(s.add(a),"mousemove",function(t){if("number"==typeof i.hide.distance){var e=this.cache.origin||{},n=this.options.hide.distance,r=Math.abs;(r(t.pageX-e.pageX)>=n||r(t.pageY-e.pageY)>=n)&&this.hide(t)}this._storeMouse(t)}),"mouse"===o.target&&o.adjust.mouse&&(i.hide.event&&this._bind(s,["mouseenter","mouseleave"],function(t){return this.cache?void(this.cache.onTarget="mouseenter"===t.type):T}),this._bind(f,"mousemove",function(t){this.rendered&&this.cache.onTarget&&!this.tooltip.hasClass(tt)&&this.tooltip[0].offsetWidth>0&&this.reposition(t)})),(o.adjust.resize||c.length)&&this._bind(r.event.special.resize?c:v,"resize",g),o.adjust.scroll&&this._bind(v.add(o.container),"scroll",g)},M._unassignEvents=function(){var n=this.options,i=n.show.target,o=n.hide.target,a=r.grep([this.elements.target[0],this.rendered&&this.tooltip[0],n.position.container[0],n.position.viewport[0],n.position.container.closest("html")[0],t,e],function(t){return"object"==typeof t});i&&i.toArray&&(a=a.concat(i.toArray())),o&&o.toArray&&(a=a.concat(o.toArray())),this._unbind(a)._unbind(a,"destroy")._unbind(a,"inactive")},r(function(){v(G,["mouseenter","mouseleave"],function(t){var e="mouseenter"===t.type,n=r(t.currentTarget),i=r(t.relatedTarget||t.target),o=this.options;e?(this.focus(t),n.hasClass(Z)&&!n.hasClass(tt)&&clearTimeout(this.timers.hide)):"mouse"===o.position.target&&o.position.adjust.mouse&&o.hide.event&&o.show.target&&!i.closest(o.show.target[0]).length&&this.hide(t),n.toggleClass(J,e)}),v("["+U+"]",X,p)}),_=r.fn.qtip=function(t,e,i){var o=(""+t).toLowerCase(),a=N,u=r.makeArray(arguments).slice(1),l=u[u.length-1],c=this[0]?r.data(this[0],V):N;return!arguments.length&&c||"api"===o?c:"string"==typeof t?(this.each(function(){var t=r.data(this,V);if(!t)return A;if(l&&l.timeStamp&&(t.cache.event=l),!e||"option"!==o&&"options"!==o)t[o]&&t[o].apply(t,u);else{if(i===n&&!r.isPlainObject(e))return a=t.get(e),T;t.set(e,i)}}),a!==N?a:this):"object"!=typeof t&&arguments.length?void 0:(c=s(r.extend(A,{},t)),this.each(function(t){var e,n;return n=r.isArray(c.id)?c.id[t]:c.id,n=!n||n===T||n.length<1||_.api[n]?_.nextid++:n,e=m(r(this),n,c),e===T?A:(_.api[n]=e,r.each(B,function(){"initialize"===this.initialize&&this(e)}),void e._assignInitialEvents(l))}))},r.qtip=i,_.api={},r.each({attr:function(t,e){if(this.length){var n=this[0],i="title",o=r.data(n,"qtip");if(t===i&&o&&"object"==typeof o&&o.options.suppress)return arguments.length<2?r.attr(n,nt):(o&&o.options.content.attr===i&&o.cache.attr&&o.set("content.text",e),this.attr(nt,e))}return r.fn["attr"+et].apply(this,arguments)},clone:function(t){var e=(r([]),r.fn["clone"+et].apply(this,arguments));return t||e.filter("["+nt+"]").attr("title",function(){return r.attr(this,nt)}).removeAttr(nt),e}},function(t,e){if(!e||r.fn[t+et])return A;var n=r.fn[t+et]=r.fn[t];r.fn[t]=function(){return e.apply(this,arguments)||n.apply(this,arguments)}}),r.ui||(r["cleanData"+et]=r.cleanData,r.cleanData=function(t){for(var e,n=0;(e=r(t[n])).length;n++)if(e.attr(H))try{e.triggerHandler("removeqtip")}catch(i){}r["cleanData"+et].apply(this,arguments)}),_.version="2.2.1",_.nextid=0,_.inactiveEvents=X,_.zindex=15e3,_.defaults={prerender:T,id:T,overwrite:A,suppress:A,content:{text:A,attr:"title",title:T,button:T},position:{my:"top left",at:"bottom right",target:T,container:T,viewport:T,adjust:{x:0,y:0,mouse:A,scroll:A,resize:A,method:"flipinvert flipinvert"},effect:function(t,e,n){r(this).animate(e,{duration:200,queue:T})}},show:{target:T,event:"mouseenter",effect:A,delay:90,solo:T,ready:T,autofocus:T},hide:{target:T,event:"mouseleave",effect:A,delay:0,fixed:T,inactive:T,leave:"window",distance:T},style:{classes:"",widget:T,width:T,height:T,def:A},events:{render:N,move:N,show:N,hide:N,toggle:N,visible:N,hidden:N,focus:N,blur:N}};var st,ut="margin",lt="border",ct="color",ft="background-color",dt="transparent",ht=" !important",pt=!!e.createElement("canvas").getContext,gt=/rgba?\(0, 0, 0(, 0)?\)|transparent|#123456/i,vt={},mt=["Webkit","O","Moz","ms"];if(pt)var yt=t.devicePixelRatio||1,bt=function(){var t=e.createElement("canvas").getContext("2d");return t.backingStorePixelRatio||t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||1}(),xt=yt/bt;else var wt=function(t,e,n){return"<qtipvml:"+t+' xmlns="urn:schemas-microsoft.com:vml" class="qtip-vml" '+(e||"")+' style="behavior: url(#default#VML); '+(n||"")+'" />'};r.extend(w.prototype,{init:function(t){var e,n;n=this.element=t.elements.tip=r("<div />",{"class":V+"-tip"}).prependTo(t.tooltip),pt?(e=r("<canvas />").appendTo(this.element)[0].getContext("2d"),e.lineJoin="miter",e.miterLimit=1e5,e.save()):(e=wt("shape",'coordorigin="0,0"',"position:absolute;"),this.element.html(e+e),t._bind(r("*",n).add(n),["click","mousedown"],function(t){t.stopPropagation()},this._ns)),t._bind(t.tooltip,"tooltipmove",this.reposition,this._ns,this),this.create()},_swapDimensions:function(){this.size[0]=this.options.height,this.size[1]=this.options.width},_resetDimensions:function(){this.size[0]=this.options.width,this.size[1]=this.options.height},_useTitle:function(t){var e=this.qtip.elements.titlebar;return e&&(t.y===L||t.y===z&&this.element.position().top+this.size[1]/2+this.options.offset<e.outerHeight(A))},_parseCorner:function(t){var e=this.qtip.options.position.my;return t===T||e===T?t=T:t===A?t=new C(e.string()):t.string||(t=new C(t),t.fixed=A),t},_parseWidth:function(t,e,n){var r=this.qtip.elements,i=lt+y(e)+"Width";return(n?x(n,i):x(r.content,i)||x(this._useTitle(t)&&r.titlebar||r.content,i)||x(r.tooltip,i))||0},_parseRadius:function(t){var e=this.qtip.elements,n=lt+y(t.y)+y(t.x)+"Radius";return rt.ie<9?0:x(this._useTitle(t)&&e.titlebar||e.content,n)||x(e.tooltip,n)||0},_invalidColour:function(t,e,n){var r=t.css(e);return!r||n&&r===t.css(n)||gt.test(r)?T:r},_parseColours:function(t){var e=this.qtip.elements,n=this.element.css("cssText",""),i=lt+y(t[t.precedance])+y(ct),o=this._useTitle(t)&&e.titlebar||e.content,a=this._invalidColour,s=[];return s[0]=a(n,ft)||a(o,ft)||a(e.content,ft)||a(e.tooltip,ft)||n.css(ft),s[1]=a(n,i,ct)||a(o,i,ct)||a(e.content,i,ct)||a(e.tooltip,i,ct)||e.tooltip.css(i),r("*",n).add(n).css("cssText",ft+":"+dt+ht+";"+lt+":0"+ht+";"),s},_calculateSize:function(t){var e,n,r,i=t.precedance===O,o=this.options.width,a=this.options.height,s="c"===t.abbrev(),u=(i?o:a)*(s?.5:1),l=Math.pow,c=Math.round,f=Math.sqrt(l(u,2)+l(a,2)),d=[this.border/u*f,this.border/a*f];return d[2]=Math.sqrt(l(d[0],2)-l(this.border,2)),d[3]=Math.sqrt(l(d[1],2)-l(this.border,2)),e=f+d[2]+d[3]+(s?0:d[0]),n=e/f,r=[c(n*o),c(n*a)],i?r:r.reverse()},_calculateTip:function(t,e,n){n=n||1,e=e||this.size;var r=e[0]*n,i=e[1]*n,o=Math.ceil(r/2),a=Math.ceil(i/2),s={br:[0,0,r,i,r,0],bl:[0,0,r,0,0,i],tr:[0,i,r,0,r,i],tl:[0,0,0,i,r,i],tc:[0,i,o,0,r,i],bc:[0,0,r,0,o,i],rc:[0,0,r,a,0,i],lc:[r,0,r,i,0,a]};return s.lt=s.br,s.rt=s.bl,s.lb=s.tr,s.rb=s.tl,s[t.abbrev()]},_drawCoords:function(t,e){t.beginPath(),t.moveTo(e[0],e[1]),t.lineTo(e[2],e[3]),t.lineTo(e[4],e[5]),t.closePath()},create:function(){var t=this.corner=(pt||rt.ie)&&this._parseCorner(this.options.corner);return(this.enabled=!!this.corner&&"c"!==this.corner.abbrev())&&(this.qtip.cache.corner=t.clone(),this.update()),this.element.toggle(this.enabled),this.corner},update:function(e,n){if(!this.enabled)return this;var i,o,a,s,u,l,c,f,d=this.qtip.elements,h=this.element,p=h.children(),g=this.options,v=this.size,m=g.mimic,y=Math.round;e||(e=this.qtip.cache.corner||this.corner),m===T?m=e:(m=new C(m),m.precedance=e.precedance,"inherit"===m.x?m.x=e.x:"inherit"===m.y?m.y=e.y:m.x===m.y&&(m[e.precedance]=e[e.precedance])),o=m.precedance,e.precedance===D?this._swapDimensions():this._resetDimensions(),i=this.color=this._parseColours(e),i[1]!==dt?(f=this.border=this._parseWidth(e,e[e.precedance]),g.border&&f<1&&!gt.test(i[1])&&(i[0]=i[1]),this.border=f=g.border!==A?g.border:f):this.border=f=0,c=this.size=this._calculateSize(e),h.css({width:c[0],height:c[1],lineHeight:c[1]+"px"}),l=e.precedance===O?[y(m.x===P?f:m.x===q?c[0]-v[0]-f:(c[0]-v[0])/2),y(m.y===L?c[1]-v[1]:0)]:[y(m.x===P?c[0]-v[0]:0),y(m.y===L?f:m.y===F?c[1]-v[1]-f:(c[1]-v[1])/2)],pt?(a=p[0].getContext("2d"),a.restore(),a.save(),a.clearRect(0,0,6e3,6e3),s=this._calculateTip(m,v,xt),u=this._calculateTip(m,this.size,xt),p.attr(j,c[0]*xt).attr(I,c[1]*xt),p.css(j,c[0]).css(I,c[1]),this._drawCoords(a,u),a.fillStyle=i[1],a.fill(),a.translate(l[0]*xt,l[1]*xt),this._drawCoords(a,s),a.fillStyle=i[0],a.fill()):(s=this._calculateTip(m),s="m"+s[0]+","+s[1]+" l"+s[2]+","+s[3]+" "+s[4]+","+s[5]+" xe",l[2]=f&&/^(r|b)/i.test(e.string())?8===rt.ie?2:1:0,p.css({coordsize:c[0]+f+" "+(c[1]+f),antialias:""+(m.string().indexOf(z)>-1),left:l[0]-l[2]*Number(o===D),top:l[1]-l[2]*Number(o===O),width:c[0]+f,height:c[1]+f}).each(function(t){var e=r(this);e[e.prop?"prop":"attr"]({coordsize:c[0]+f+" "+(c[1]+f),path:s,fillcolor:i[0],filled:!!t,stroked:!t}).toggle(!(!f&&!t)),!t&&e.html(wt("stroke",'weight="'+2*f+'px" color="'+i[1]+'" miterlimit="1000" joinstyle="miter"'))})),t.opera&&setTimeout(function(){d.tip.css({display:"inline-block",visibility:"visible"})},1),n!==T&&this.calculate(e,c)},calculate:function(t,e){if(!this.enabled)return T;var n,i,o=this,a=this.qtip.elements,s=this.element,u=this.options.offset,l=(a.tooltip.hasClass("ui-widget"),{});return t=t||this.corner,n=t.precedance,e=e||this._calculateSize(t),i=[t.x,t.y],n===D&&i.reverse(),r.each(i,function(r,i){var s,c,f;i===z?(s=n===O?P:L,l[s]="50%",l[ut+"-"+s]=-Math.round(e[n===O?0:1]/2)+u):(s=o._parseWidth(t,i,a.tooltip),c=o._parseWidth(t,i,a.content),f=o._parseRadius(t),l[i]=Math.max(-o.border,r?c:u+(f>s?f:-s)))}),l[t[n]]-=e[n===D?0:1],s.css({margin:"",top:"",bottom:"",left:"",right:""}).css(l),l},reposition:function(t,e,r,i){function o(t,e,n,r,i){t===R&&c.precedance===e&&f[r]&&c[n]!==z?c.precedance=c.precedance===D?O:D:t!==R&&f[r]&&(c[e]=c[e]===z?f[r]>0?r:i:c[e]===r?i:r)}function a(t,e,i){c[t]===z?v[ut+"-"+e]=g[t]=s[ut+"-"+e]-f[e]:(u=s[i]!==n?[f[e],-s[e]]:[-f[e],s[e]],(g[t]=Math.max(u[0],u[1]))>u[0]&&(r[e]-=f[e],g[e]=T),v[s[i]!==n?i:e]=g[t])}if(this.enabled){var s,u,l=e.cache,c=this.corner.clone(),f=r.adjusted,d=e.options.position.adjust.method.split(" "),h=d[0],p=d[1]||d[0],g={left:T,top:T,x:0,y:0},v={};this.corner.fixed!==A&&(o(h,D,O,P,q),o(p,O,D,L,F),c.string()===l.corner.string()&&l.cornerTop===f.top&&l.cornerLeft===f.left||this.update(c,T)),s=this.calculate(c),s.right!==n&&(s.left=-s.right),s.bottom!==n&&(s.top=-s.bottom),s.user=this.offset,(g.left=h===R&&!!f.left)&&a(D,P,q),(g.top=p===R&&!!f.top)&&a(O,L,F),this.element.css(v).toggle(!(g.x&&g.y||c.x===z&&g.y||c.y===z&&g.x)),r.left-=s.left.charAt?s.user:h!==R||g.top||!g.left&&!g.top?s.left+this.border:0,r.top-=s.top.charAt?s.user:p!==R||g.left||!g.left&&!g.top?s.top+this.border:0,l.cornerLeft=f.left,l.cornerTop=f.top,l.corner=c.clone()}},destroy:function(){this.qtip._unbind(this.qtip.tooltip,this._ns),this.qtip.elements.tip&&this.qtip.elements.tip.find("*").remove().end().remove()}}),st=B.tip=function(t){return new w(t,t.options.style.tip)},st.initialize="render",st.sanitize=function(t){if(t.style&&"tip"in t.style){var e=t.style.tip;"object"!=typeof e&&(e=t.style.tip={corner:e}),/string|boolean/i.test(typeof e.corner)||(e.corner=A)}},S.tip={"^position.my|style.tip.(corner|mimic|border)$":function(){this.create(),this.qtip.reposition()},"^style.tip.(height|width)$":function(t){this.size=[t.width,t.height],this.update(),this.qtip.reposition()},"^content.title|style.(classes|widget)$":function(){this.update()}},r.extend(A,_.defaults,{style:{tip:{corner:A,mimic:T,width:6,height:6,border:A,offset:0}}});var $t,kt,_t="qtip-modal",Mt="."+_t;kt=function(){function t(t){if(r.expr[":"].focusable)return r.expr[":"].focusable;var e,n,i,o=!isNaN(r.attr(t,"tabindex")),a=t.nodeName&&t.nodeName.toLowerCase();return"area"===a?(e=t.parentNode,n=e.name,!(!t.href||!n||"map"!==e.nodeName.toLowerCase())&&(i=r("img[usemap=#"+n+"]")[0],!!i&&i.is(":visible"))):/input|select|textarea|button|object/.test(a)?!t.disabled:"a"===a?t.href||o:o}function n(t){c.length<1&&t.length?t.not("body").blur():c.first().focus()}function i(t){if(u.is(":visible")){var e,i=r(t.target),s=o.tooltip,l=i.closest(G);e=l.length<1?T:parseInt(l[0].style.zIndex,10)>parseInt(s[0].style.zIndex,10),e||i.closest(G)[0]===s[0]||n(i),a=t.target===c[c.length-1]}}var o,a,s,u,l=this,c={};r.extend(l,{init:function(){return u=l.elem=r("<div />",{id:"qtip-overlay",html:"<div></div>",mousedown:function(){return T}}).hide(),r(e.body).bind("focusin"+Mt,i),r(e).bind("keydown"+Mt,function(t){o&&o.options.show.modal.escape&&27===t.keyCode&&o.hide(t)}),u.bind("click"+Mt,function(t){o&&o.options.show.modal.blur&&o.hide(t)}),l},update:function(e){o=e,c=e.options.show.modal.stealfocus!==T?e.tooltip.find("*").filter(function(){return t(this)}):[]},toggle:function(t,i,a){var c=(r(e.body),t.tooltip),f=t.options.show.modal,d=f.effect,h=i?"show":"hide",p=u.is(":visible"),g=r(Mt).filter(":visible:not(:animated)").not(c);return l.update(t),i&&f.stealfocus!==T&&n(r(":focus")),u.toggleClass("blurs",f.blur),i&&u.appendTo(e.body),u.is(":animated")&&p===i&&s!==T||!i&&g.length?l:(u.stop(A,T),r.isFunction(d)?d.call(u,i):d===T?u[h]():u.fadeTo(parseInt(a,10)||90,i?1:0,function(){i||u.hide()}),i||u.queue(function(t){u.css({left:"",top:""}),r(Mt).length||u.detach(),t()}),s=i,o.destroyed&&(o=N),l)}}),l.init()},kt=new kt,r.extend($.prototype,{init:function(t){var e=t.tooltip;return this.options.on?(t.elements.overlay=kt.elem,e.addClass(_t).css("z-index",_.modal_zindex+r(Mt).length),t._bind(e,["tooltipshow","tooltiphide"],function(t,n,i){var o=t.originalEvent;if(t.target===e[0])if(o&&"tooltiphide"===t.type&&/mouse(leave|enter)/.test(o.type)&&r(o.relatedTarget).closest(kt.elem[0]).length)try{t.preventDefault()}catch(a){}else(!o||o&&"tooltipsolo"!==o.type)&&this.toggle(t,"tooltipshow"===t.type,i)},this._ns,this),t._bind(e,"tooltipfocus",function(t,n){if(!t.isDefaultPrevented()&&t.target===e[0]){var i=r(Mt),o=_.modal_zindex+i.length,a=parseInt(e[0].style.zIndex,10);kt.elem[0].style.zIndex=o-1,i.each(function(){this.style.zIndex>a&&(this.style.zIndex-=1)}),i.filter("."+K).qtip("blur",t.originalEvent),e.addClass(K)[0].style.zIndex=o,kt.update(n);try{t.preventDefault()}catch(s){}}},this._ns,this),void t._bind(e,"tooltiphide",function(t){t.target===e[0]&&r(Mt).filter(":visible").not(e).last().qtip("focus",t)},this._ns,this)):this},toggle:function(t,e,n){return t&&t.isDefaultPrevented()?this:void kt.toggle(this.qtip,!!e,n)},destroy:function(){this.qtip.tooltip.removeClass(_t),this.qtip._unbind(this.qtip.tooltip,this._ns),kt.toggle(this.qtip,T),delete this.qtip.elements.overlay}}),$t=B.modal=function(t){return new $(t,t.options.show.modal)},$t.sanitize=function(t){t.show&&("object"!=typeof t.show.modal?t.show.modal={on:!!t.show.modal}:"undefined"==typeof t.show.modal.on&&(t.show.modal.on=A))},_.modal_zindex=_.zindex-200,$t.initialize="render",S.modal={"^show.modal.(on|blur)$":function(){this.destroy(),this.init(),this.qtip.elems.overlay.toggle(this.qtip.tooltip[0].offsetWidth>0)}},r.extend(A,_.defaults,{show:{modal:{on:T,effect:A,blur:A,stealfocus:A,escape:A}}}),B.viewport=function(n,r,i,o,a,s,u){function l(t,e,n,i,o,a,s,u,l){var c=r[o],y=x[t],b=w[t],$=n===R,k=y===o?l:y===a?-l:-l/2,_=b===o?u:b===a?-u:-u/2,M=v[o]+m[o]-(h?0:d[o]),C=M-c,S=c+l-(s===j?p:g)-M,E=k-(x.precedance===t||y===x[e]?_:0)-(b===z?u/2:0);return $?(E=(y===o?1:-1)*k,r[o]+=C>0?C:S>0?-S:0,r[o]=Math.max(-d[o]+m[o],c-E,Math.min(Math.max(-d[o]+m[o]+(s===j?p:g),c+E),r[o],"center"===y?c-k:1e9))):(i*=n===W?2:0,C>0&&(y!==o||S>0)?(r[o]-=E+i,f.invert(t,o)):S>0&&(y!==a||C>0)&&(r[o]-=(y===z?-E:E)+i,f.invert(t,a)),r[o]<v&&-r[o]>S&&(r[o]=c,f=x.clone())),r[o]-c}var c,f,d,h,p,g,v,m,y=i.target,b=n.elements.tooltip,x=i.my,w=i.at,$=i.adjust,k=$.method.split(" "),_=k[0],M=k[1]||k[0],C=i.viewport,S=i.container,E=(n.cache,{left:0,top:0});return C.jquery&&y[0]!==t&&y[0]!==e.body&&"none"!==$.method?(d=S.offset()||E,h="static"===S.css("position"),c="fixed"===b.css("position"),p=C[0]===t?C.width():C.outerWidth(T),g=C[0]===t?C.height():C.outerHeight(T),v={left:c?0:C.scrollLeft(),top:c?0:C.scrollTop()},m=C.offset()||E,"shift"===_&&"shift"===M||(f=x.clone()),E={left:"none"!==_?l(D,O,_,$.x,P,q,j,o,s):0,top:"none"!==M?l(O,D,M,$.y,L,F,I,a,u):0,my:f}):E},B.polys={polygon:function(t,e){var n,r,i,o={width:0,height:0,position:{top:1e10,right:0,bottom:0,left:1e10},adjustable:T},a=0,s=[],u=1,l=1,c=0,f=0;
-for(a=t.length;a--;)n=[parseInt(t[--a],10),parseInt(t[a+1],10)],n[0]>o.position.right&&(o.position.right=n[0]),n[0]<o.position.left&&(o.position.left=n[0]),n[1]>o.position.bottom&&(o.position.bottom=n[1]),n[1]<o.position.top&&(o.position.top=n[1]),s.push(n);if(r=o.width=Math.abs(o.position.right-o.position.left),i=o.height=Math.abs(o.position.bottom-o.position.top),"c"===e.abbrev())o.position={left:o.position.left+o.width/2,top:o.position.top+o.height/2};else{for(;r>0&&i>0&&u>0&&l>0;)for(r=Math.floor(r/2),i=Math.floor(i/2),e.x===P?u=r:e.x===q?u=o.width-r:u+=Math.floor(r/2),e.y===L?l=i:e.y===F?l=o.height-i:l+=Math.floor(i/2),a=s.length;a--&&!(s.length<2);)c=s[a][0]-o.position.left,f=s[a][1]-o.position.top,(e.x===P&&c>=u||e.x===q&&c<=u||e.x===z&&(c<u||c>o.width-u)||e.y===L&&f>=l||e.y===F&&f<=l||e.y===z&&(f<l||f>o.height-l))&&s.splice(a,1);o.position={left:s[0][0],top:s[0][1]}}return o},rect:function(t,e,n,r){return{width:Math.abs(n-t),height:Math.abs(r-e),position:{left:Math.min(t,n),top:Math.min(e,r)}}},_angles:{tc:1.5,tr:7/4,tl:5/4,bc:.5,br:.25,bl:.75,rc:2,lc:1,c:0},ellipse:function(t,e,n,r,i){var o=B.polys._angles[i.abbrev()],a=0===o?0:n*Math.cos(o*Math.PI),s=r*Math.sin(o*Math.PI);return{width:2*n-Math.abs(a),height:2*r-Math.abs(s),position:{left:t+a,top:e+s},adjustable:T}},circle:function(t,e,n,r){return B.polys.ellipse(t,e,n,n,r)}},B.svg=function(t,n,i){for(var o,a,s,u,l,c,f,d,h,p=(r(e),n[0]),g=r(p.ownerSVGElement),v=p.ownerDocument,m=(parseInt(n.css("stroke-width"),10)||0)/2;!p.getBBox;)p=p.parentNode;if(!p.getBBox||!p.parentNode)return T;switch(p.nodeName){case"ellipse":case"circle":d=B.polys.ellipse(p.cx.baseVal.value,p.cy.baseVal.value,(p.rx||p.r).baseVal.value+m,(p.ry||p.r).baseVal.value+m,i);break;case"line":case"polygon":case"polyline":for(f=p.points||[{x:p.x1.baseVal.value,y:p.y1.baseVal.value},{x:p.x2.baseVal.value,y:p.y2.baseVal.value}],d=[],c=-1,u=f.numberOfItems||f.length;++c<u;)l=f.getItem?f.getItem(c):f[c],d.push.apply(d,[l.x,l.y]);d=B.polys.polygon(d,i);break;default:d=p.getBBox(),d={width:d.width,height:d.height,position:{left:d.x,top:d.y}}}return h=d.position,g=g[0],g.createSVGPoint&&(a=p.getScreenCTM(),f=g.createSVGPoint(),f.x=h.left,f.y=h.top,s=f.matrixTransform(a),h.left=s.x,h.top=s.y),v!==e&&"mouse"!==t.position.target&&(o=r((v.defaultView||v.parentWindow).frameElement).offset(),o&&(h.left+=o.left,h.top+=o.top)),v=r(v),h.left+=v.scrollLeft(),h.top+=v.scrollTop(),d},B.imagemap=function(t,e,n,i){e.jquery||(e=r(e));var o,a,s,u,l,c=(e.attr("shape")||"rect").toLowerCase().replace("poly","polygon"),f=r('img[usemap="#'+e.parent("map").attr("name")+'"]'),d=r.trim(e.attr("coords")),h=d.replace(/,$/,"").split(",");if(!f.length)return T;if("polygon"===c)u=B.polys.polygon(h,n);else{if(!B.polys[c])return T;for(s=-1,l=h.length,a=[];++s<l;)a.push(parseInt(h[s],10));u=B.polys[c].apply(this,a.concat(n))}return o=f.offset(),o.left+=Math.ceil((f.outerWidth(T)-f.width())/2),o.top+=Math.ceil((f.outerHeight(T)-f.height())/2),u.position.left+=o.left,u.position.top+=o.top,u};var Ct,St='<iframe class="qtip-bgiframe" frameborder="0" tabindex="-1" src="javascript:\'\';"  style="display:block; position:absolute; z-index:-1; filter:alpha(opacity=0); -ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";"></iframe>';r.extend(k.prototype,{_scroll:function(){var e=this.qtip.elements.overlay;e&&(e[0].style.top=r(t).scrollTop()+"px")},init:function(n){var i=n.tooltip;r("select, object").length<1&&(this.bgiframe=n.elements.bgiframe=r(St).appendTo(i),n._bind(i,"tooltipmove",this.adjustBGIFrame,this._ns,this)),this.redrawContainer=r("<div/>",{id:V+"-rcontainer"}).appendTo(e.body),n.elements.overlay&&n.elements.overlay.addClass("qtipmodal-ie6fix")&&(n._bind(t,["scroll","resize"],this._scroll,this._ns,this),n._bind(i,["tooltipshow"],this._scroll,this._ns,this)),this.redraw()},adjustBGIFrame:function(){var t,e,n=this.qtip.tooltip,r={height:n.outerHeight(T),width:n.outerWidth(T)},i=this.qtip.plugins.tip,o=this.qtip.elements.tip;e=parseInt(n.css("borderLeftWidth"),10)||0,e={left:-e,top:-e},i&&o&&(t="x"===i.corner.precedance?[j,P]:[I,L],e[t[1]]-=o[t[0]]()),this.bgiframe.css(e).css(r)},redraw:function(){if(this.qtip.rendered<1||this.drawing)return this;var t,e,n,r,i=this.qtip.tooltip,o=this.qtip.options.style,a=this.qtip.options.position.container;return this.qtip.drawing=1,o.height&&i.css(I,o.height),o.width?i.css(j,o.width):(i.css(j,"").appendTo(this.redrawContainer),e=i.width(),e%2<1&&(e+=1),n=i.css("maxWidth")||"",r=i.css("minWidth")||"",t=(n+r).indexOf("%")>-1?a.width()/100:0,n=(n.indexOf("%")>-1?t:1)*parseInt(n,10)||e,r=(r.indexOf("%")>-1?t:1)*parseInt(r,10)||0,e=n+r?Math.min(Math.max(e,r),n):e,i.css(j,Math.round(e)).appendTo(a)),this.drawing=0,this},destroy:function(){this.bgiframe&&this.bgiframe.remove(),this.qtip._unbind([t,this.qtip.tooltip],this._ns)}}),Ct=B.ie6=function(t){return 6===rt.ie?new k(t):T},Ct.initialize="render",S.ie6={"^content|style$":function(){this.redraw()}}})}(window,document),function(t,e,n){!function(t){"use strict";"function"==typeof define&&define.amd?define(["jquery"],t):jQuery&&!jQuery.fn.qtip&&t(jQuery)}(function(r){"use strict";function i(t,e,n,i){this.id=n,this.target=t,this.tooltip=M,this.elements={target:t},this._id=j+"-"+n,this.timers={img:{}},this.options=e,this.plugins={},this.cache={event:{},target:r(),disabled:_,attr:i,onTooltip:_,lastClass:""},this.rendered=this.destroyed=this.disabled=this.waiting=this.hiddenDuringWait=this.positioning=this.triggering=_}function o(t){return t===M||"object"!==r.type(t)}function a(t){return!(r.isFunction(t)||t&&t.attr||t.length||"object"===r.type(t)&&(t.jquery||t.then))}function s(t){var e,n,i,s;return o(t)?_:(o(t.metadata)&&(t.metadata={type:t.metadata}),"content"in t&&(e=t.content,o(e)||e.jquery||e.done?e=t.content={text:n=a(e)?_:e}:n=e.text,"ajax"in e&&(i=e.ajax,s=i&&i.once!==_,delete e.ajax,e.text=function(t,e){var o=n||r(this).attr(e.options.content.attr)||"Loading...",a=r.ajax(r.extend({},i,{context:e})).then(i.success,M,i.error).then(function(t){return t&&s&&e.set("content.text",t),t},function(t,n,r){e.destroyed||0===t.status||e.set("content.text",n+": "+r)});return s?o:(e.set("content.text",o),a)}),"title"in e&&(r.isPlainObject(e.title)&&(e.button=e.title.button,e.title=e.title.text),a(e.title||_)&&(e.title=_))),"position"in t&&o(t.position)&&(t.position={my:t.position,at:t.position}),"show"in t&&o(t.show)&&(t.show=t.show.jquery?{target:t.show}:t.show===k?{ready:k}:{event:t.show}),"hide"in t&&o(t.hide)&&(t.hide=t.hide.jquery?{target:t.hide}:{event:t.hide}),"style"in t&&o(t.style)&&(t.style={classes:t.style}),r.each(O,function(){this.sanitize&&this.sanitize(t)}),t)}function u(t,e){for(var n,r=0,i=t,o=e.split(".");i=i[o[r++]];)r<o.length&&(n=i);return[n||t,o.pop()]}function l(t,e){var n,r,i;for(n in this.checks)for(r in this.checks[n])(i=new RegExp(r,"i").exec(t))&&(e.push(i),("builtin"===n||this.plugins[n])&&this.checks[n][r].apply(this.plugins[n]||this,e))}function c(t){return P.concat("").join(t?"-"+t+" ":" ")}function f(t,e){return e>0?setTimeout(r.proxy(t,this),e):void t.call(this)}function d(t){this.tooltip.hasClass(V)||(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this.timers.show=f.call(this,function(){this.toggle(k,t)},this.options.show.delay))}function h(t){if(!this.tooltip.hasClass(V)&&!this.destroyed){var e=r(t.relatedTarget),n=e.closest(F)[0]===this.tooltip[0],i=e[0]===this.options.show.target[0];if(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this!==e[0]&&"mouse"===this.options.position.target&&n||this.options.hide.fixed&&/mouse(out|leave|move)/.test(t.type)&&(n||i))try{t.preventDefault(),t.stopImmediatePropagation()}catch(o){}else this.timers.hide=f.call(this,function(){this.toggle(_,t)},this.options.hide.delay,this)}}function p(t){!this.tooltip.hasClass(V)&&this.options.hide.inactive&&(clearTimeout(this.timers.inactive),this.timers.inactive=f.call(this,function(){this.hide(t)},this.options.hide.inactive))}function g(t){this.rendered&&this.tooltip[0].offsetWidth>0&&this.reposition(t)}function v(t,n,i){r(e.body).delegate(t,(n.split?n:n.join("."+j+" "))+"."+j,function(){var t=y.api[r.attr(this,L)];t&&!t.disabled&&i.apply(t,arguments)})}function m(t,n,o){var a,u,l,c,f,d=r(e.body),h=t[0]===e?d:t,p=t.metadata?t.metadata(o.metadata):M,g="html5"===o.metadata.type&&p?p[o.metadata.name]:M,v=t.data(o.metadata.name||"qtipopts");try{v="string"==typeof v?r.parseJSON(v):v}catch(m){}if(c=r.extend(k,{},y.defaults,o,"object"==typeof v?s(v):M,s(g||p)),u=c.position,c.id=n,"boolean"==typeof c.content.text){if(l=t.attr(c.content.attr),c.content.attr===_||!l)return _;c.content.text=l}if(u.container.length||(u.container=d),u.target===_&&(u.target=h),c.show.target===_&&(c.show.target=h),c.show.solo===k&&(c.show.solo=u.container.closest("body")),c.hide.target===_&&(c.hide.target=h),c.position.viewport===k&&(c.position.viewport=u.container),u.container=u.container.eq(0),u.at=new x(u.at,k),u.my=new x(u.my),t.data(j))if(c.overwrite)t.qtip("destroy",!0);else if(c.overwrite===_)return _;return t.attr(I,n),c.suppress&&(f=t.attr("title"))&&t.removeAttr("title").attr(U,f).attr("title",""),a=new i(t,c,n,(!!l)),t.data(j,a),a}var y,b,x,w,$,k=!0,_=!1,M=null,C="x",S="y",E="top",A="left",T="bottom",N="right",D="center",O={},j="qtip",I="data-hasqtip",L="data-qtip-id",P=["ui-widget","ui-tooltip"],F="."+j,q="click dblclick mousedown mouseup mousemove mouseleave mouseenter".split(" "),z=j+"-fixed",W=j+"-default",R=j+"-focus",B=j+"-hover",V=j+"-disabled",H="_replacedByqTip",U="oldtitle",Y={ie:function(){for(var t=4,n=e.createElement("div");(n.innerHTML="<!--[if gt IE "+t+"]><i></i><![endif]-->")&&n.getElementsByTagName("i")[0];t+=1);return t>4?t:NaN}(),iOS:parseFloat((""+(/CPU.*OS ([0-9_]{1,5})|(CPU like).*AppleWebKit.*Mobile/i.exec(navigator.userAgent)||[0,""])[1]).replace("undefined","3_2").replace("_",".").replace("_",""))||_};b=i.prototype,b._when=function(t){return r.when.apply(r,t)},b.render=function(t){if(this.rendered||this.destroyed)return this;var e,n=this,i=this.options,o=this.cache,a=this.elements,s=i.content.text,u=i.content.title,l=i.content.button,c=i.position,f=("."+this._id+" ",[]);return r.attr(this.target[0],"aria-describedby",this._id),o.posClass=this._createPosClass((this.position={my:c.my,at:c.at}).my),this.tooltip=a.tooltip=e=r("<div/>",{id:this._id,"class":[j,W,i.style.classes,o.posClass].join(" "),width:i.style.width||"",height:i.style.height||"",tracking:"mouse"===c.target&&c.adjust.mouse,role:"alert","aria-live":"polite","aria-atomic":_,"aria-describedby":this._id+"-content","aria-hidden":k}).toggleClass(V,this.disabled).attr(L,this.id).data(j,this).appendTo(c.container).append(a.content=r("<div />",{"class":j+"-content",id:this._id+"-content","aria-atomic":k})),this.rendered=-1,this.positioning=k,u&&(this._createTitle(),r.isFunction(u)||f.push(this._updateTitle(u,_))),l&&this._createButton(),r.isFunction(s)||f.push(this._updateContent(s,_)),this.rendered=k,this._setWidget(),r.each(O,function(t){var e;"render"===this.initialize&&(e=this(n))&&(n.plugins[t]=e)}),this._unassignEvents(),this._assignEvents(),this._when(f).then(function(){n._trigger("render"),n.positioning=_,n.hiddenDuringWait||!i.show.ready&&!t||n.toggle(k,o.event,_),n.hiddenDuringWait=_}),y.api[this.id]=this,this},b.destroy=function(t){function e(){if(!this.destroyed){this.destroyed=k;var t,e=this.target,n=e.attr(U);this.rendered&&this.tooltip.stop(1,0).find("*").remove().end().remove(),r.each(this.plugins,function(t){this.destroy&&this.destroy()});for(t in this.timers)clearTimeout(this.timers[t]);e.removeData(j).removeAttr(L).removeAttr(I).removeAttr("aria-describedby"),this.options.suppress&&n&&e.attr("title",n).removeAttr(U),this._unassignEvents(),this.options=this.elements=this.cache=this.timers=this.plugins=this.mouse=M,delete y.api[this.id]}}return this.destroyed?this.target:(t===k&&"hide"!==this.triggering||!this.rendered?e.call(this):(this.tooltip.one("tooltiphidden",r.proxy(e,this)),!this.triggering&&this.hide()),this.target)},w=b.checks={builtin:{"^id$":function(t,e,n,i){var o=n===k?y.nextid:n,a=j+"-"+o;o!==_&&o.length>0&&!r("#"+a).length?(this._id=a,this.rendered&&(this.tooltip[0].id=this._id,this.elements.content[0].id=this._id+"-content",this.elements.title[0].id=this._id+"-title")):t[e]=i},"^prerender":function(t,e,n){n&&!this.rendered&&this.render(this.options.show.ready)},"^content.text$":function(t,e,n){this._updateContent(n)},"^content.attr$":function(t,e,n,r){this.options.content.text===this.target.attr(r)&&this._updateContent(this.target.attr(n))},"^content.title$":function(t,e,n){return n?(n&&!this.elements.title&&this._createTitle(),void this._updateTitle(n)):this._removeTitle()},"^content.button$":function(t,e,n){this._updateButton(n)},"^content.title.(text|button)$":function(t,e,n){this.set("content."+e,n)},"^position.(my|at)$":function(t,e,n){"string"==typeof n&&(this.position[e]=t[e]=new x(n,"at"===e))},"^position.container$":function(t,e,n){this.rendered&&this.tooltip.appendTo(n)},"^show.ready$":function(t,e,n){n&&(!this.rendered&&this.render(k)||this.toggle(k))},"^style.classes$":function(t,e,n,r){this.rendered&&this.tooltip.removeClass(r).addClass(n)},"^style.(width|height)":function(t,e,n){this.rendered&&this.tooltip.css(e,n)},"^style.widget|content.title":function(){this.rendered&&this._setWidget()},"^style.def":function(t,e,n){this.rendered&&this.tooltip.toggleClass(W,!!n)},"^events.(render|show|move|hide|focus|blur)$":function(t,e,n){this.rendered&&this.tooltip[(r.isFunction(n)?"":"un")+"bind"]("tooltip"+e,n)},"^(show|hide|position).(event|target|fixed|inactive|leave|distance|viewport|adjust)":function(){if(this.rendered){var t=this.options.position;this.tooltip.attr("tracking","mouse"===t.target&&t.adjust.mouse),this._unassignEvents(),this._assignEvents()}}}},b.get=function(t){if(this.destroyed)return this;var e=u(this.options,t.toLowerCase()),n=e[0][e[1]];return n.precedance?n.string():n};var G=/^position\.(my|at|adjust|target|container|viewport)|style|content|show\.ready/i,X=/^prerender|show\.ready/i;b.set=function(t,e){if(this.destroyed)return this;var n,i=this.rendered,o=_,a=this.options;this.checks;return"string"==typeof t?(n=t,t={},t[n]=e):t=r.extend({},t),r.each(t,function(e,n){if(i&&X.test(e))return void delete t[e];var s,l=u(a,e.toLowerCase());s=l[0][l[1]],l[0][l[1]]=n&&n.nodeType?r(n):n,o=G.test(e)||o,t[e]=[l[0],l[1],n,s]}),s(a),this.positioning=k,r.each(t,r.proxy(l,this)),this.positioning=_,this.rendered&&this.tooltip[0].offsetWidth>0&&o&&this.reposition("mouse"===a.position.target?M:this.cache.event),this},b._update=function(t,e,n){var i=this,o=this.cache;return this.rendered&&t?(r.isFunction(t)&&(t=t.call(this.elements.target,o.event,this)||""),r.isFunction(t.then)?(o.waiting=k,t.then(function(t){return o.waiting=_,i._update(t,e)},M,function(t){return i._update(t,e)})):t===_||!t&&""!==t?_:(t.jquery&&t.length>0?e.empty().append(t.css({display:"block",visibility:"visible"})):e.html(t),this._waitForContent(e).then(function(t){i.rendered&&i.tooltip[0].offsetWidth>0&&i.reposition(o.event,!t.length)}))):_},b._waitForContent=function(t){var e=this.cache;return e.waiting=k,(r.fn.imagesLoaded?t.imagesLoaded():r.Deferred().resolve([])).done(function(){e.waiting=_}).promise()},b._updateContent=function(t,e){this._update(t,this.elements.content,e)},b._updateTitle=function(t,e){this._update(t,this.elements.title,e)===_&&this._removeTitle(_)},b._createTitle=function(){var t=this.elements,e=this._id+"-title";t.titlebar&&this._removeTitle(),t.titlebar=r("<div />",{"class":j+"-titlebar "+(this.options.style.widget?c("header"):"")}).append(t.title=r("<div />",{id:e,"class":j+"-title","aria-atomic":k})).insertBefore(t.content).delegate(".qtip-close","mousedown keydown mouseup keyup mouseout",function(t){r(this).toggleClass("ui-state-active ui-state-focus","down"===t.type.substr(-4))}).delegate(".qtip-close","mouseover mouseout",function(t){r(this).toggleClass("ui-state-hover","mouseover"===t.type)}),this.options.content.button&&this._createButton()},b._removeTitle=function(t){var e=this.elements;e.title&&(e.titlebar.remove(),e.titlebar=e.title=e.button=M,t!==_&&this.reposition())},b._createPosClass=function(t){return j+"-pos-"+(t||this.options.position.my).abbrev()},b.reposition=function(n,i){if(!this.rendered||this.positioning||this.destroyed)return this;this.positioning=k;var o,a,s,u,l=this.cache,c=this.tooltip,f=this.options.position,d=f.target,h=f.my,p=f.at,g=f.viewport,v=f.container,m=f.adjust,y=m.method.split(" "),b=c.outerWidth(_),x=c.outerHeight(_),w=0,$=0,M=c.css("position"),C={left:0,top:0},S=c[0].offsetWidth>0,j=n&&"scroll"===n.type,I=r(t),L=v[0].ownerDocument,P=this.mouse;if(r.isArray(d)&&2===d.length)p={x:A,y:E},C={left:d[0],top:d[1]};else if("mouse"===d)p={x:A,y:E},(!m.mouse||this.options.hide.distance)&&l.origin&&l.origin.pageX?n=l.origin:!n||n&&("resize"===n.type||"scroll"===n.type)?n=l.event:P&&P.pageX&&(n=P),"static"!==M&&(C=v.offset()),L.body.offsetWidth!==(t.innerWidth||L.documentElement.clientWidth)&&(a=r(e.body).offset()),C={left:n.pageX-C.left+(a&&a.left||0),top:n.pageY-C.top+(a&&a.top||0)},m.mouse&&j&&P&&(C.left-=(P.scrollX||0)-I.scrollLeft(),C.top-=(P.scrollY||0)-I.scrollTop());else{if("event"===d?n&&n.target&&"scroll"!==n.type&&"resize"!==n.type?l.target=r(n.target):n.target||(l.target=this.elements.target):"event"!==d&&(l.target=r(d.jquery?d:this.elements.target)),d=l.target,d=r(d).eq(0),0===d.length)return this;d[0]===e||d[0]===t?(w=Y.iOS?t.innerWidth:d.width(),$=Y.iOS?t.innerHeight:d.height(),d[0]===t&&(C={top:(g||d).scrollTop(),left:(g||d).scrollLeft()})):O.imagemap&&d.is("area")?o=O.imagemap(this,d,p,O.viewport?y:_):O.svg&&d&&d[0].ownerSVGElement?o=O.svg(this,d,p,O.viewport?y:_):(w=d.outerWidth(_),$=d.outerHeight(_),C=d.offset()),o&&(w=o.width,$=o.height,a=o.offset,C=o.position),C=this.reposition.offset(d,C,v),(Y.iOS>3.1&&Y.iOS<4.1||Y.iOS>=4.3&&Y.iOS<4.33||!Y.iOS&&"fixed"===M)&&(C.left-=I.scrollLeft(),C.top-=I.scrollTop()),(!o||o&&o.adjustable!==_)&&(C.left+=p.x===N?w:p.x===D?w/2:0,C.top+=p.y===T?$:p.y===D?$/2:0)}return C.left+=m.x+(h.x===N?-b:h.x===D?-b/2:0),C.top+=m.y+(h.y===T?-x:h.y===D?-x/2:0),O.viewport?(s=C.adjusted=O.viewport(this,C,f,w,$,b,x),a&&s.left&&(C.left+=a.left),a&&s.top&&(C.top+=a.top),s.my&&(this.position.my=s.my)):C.adjusted={left:0,top:0},l.posClass!==(u=this._createPosClass(this.position.my))&&c.removeClass(l.posClass).addClass(l.posClass=u),this._trigger("move",[C,g.elem||g],n)?(delete C.adjusted,i===_||!S||isNaN(C.left)||isNaN(C.top)||"mouse"===d||!r.isFunction(f.effect)?c.css(C):r.isFunction(f.effect)&&(f.effect.call(c,this,r.extend({},C)),c.queue(function(t){r(this).css({opacity:"",height:""}),Y.ie&&this.style.removeAttribute("filter"),t()})),this.positioning=_,this):this},b.reposition.offset=function(t,n,i){function o(t,e){n.left+=e*t.scrollLeft(),n.top+=e*t.scrollTop()}if(!i[0])return n;var a,s,u,l,c=r(t[0].ownerDocument),f=!!Y.ie&&"CSS1Compat"!==e.compatMode,d=i[0];do"static"!==(s=r.css(d,"position"))&&("fixed"===s?(u=d.getBoundingClientRect(),o(c,-1)):(u=r(d).position(),u.left+=parseFloat(r.css(d,"borderLeftWidth"))||0,u.top+=parseFloat(r.css(d,"borderTopWidth"))||0),n.left-=u.left+(parseFloat(r.css(d,"marginLeft"))||0),n.top-=u.top+(parseFloat(r.css(d,"marginTop"))||0),a||"hidden"===(l=r.css(d,"overflow"))||"visible"===l||(a=r(d)));while(d=d.offsetParent);return a&&(a[0]!==c[0]||f)&&o(a,1),n};var Z=(x=b.reposition.Corner=function(t,e){t=(""+t).replace(/([A-Z])/," $1").replace(/middle/gi,D).toLowerCase(),this.x=(t.match(/left|right/i)||t.match(/center/)||["inherit"])[0].toLowerCase(),this.y=(t.match(/top|bottom|center/i)||["inherit"])[0].toLowerCase(),this.forceY=!!e;var n=t.charAt(0);this.precedance="t"===n||"b"===n?S:C}).prototype;Z.invert=function(t,e){this[t]=this[t]===A?N:this[t]===N?A:e||this[t]},Z.string=function(t){var e=this.x,n=this.y,r=e!==n?"center"===e||"center"!==n&&(this.precedance===S||this.forceY)?[n,e]:[e,n]:[e];return t!==!1?r.join(" "):r},Z.abbrev=function(){var t=this.string(!1);return t[0].charAt(0)+(t[1]&&t[1].charAt(0)||"")},Z.clone=function(){return new x(this.string(),this.forceY)},b.toggle=function(t,n){var i=this.cache,o=this.options,a=this.tooltip;if(n){if(/over|enter/.test(n.type)&&i.event&&/out|leave/.test(i.event.type)&&o.show.target.add(n.target).length===o.show.target.length&&a.has(n.relatedTarget).length)return this;i.event=r.event.fix(n)}if(this.waiting&&!t&&(this.hiddenDuringWait=k),!this.rendered)return t?this.render(1):this;if(this.destroyed||this.disabled)return this;var s,u,l,c=t?"show":"hide",f=this.options[c],d=(this.options[t?"hide":"show"],this.options.position),h=this.options.content,p=this.tooltip.css("width"),g=this.tooltip.is(":visible"),v=t||1===f.target.length,m=!n||f.target.length<2||i.target[0]===n.target;return(typeof t).search("boolean|number")&&(t=!g),s=!a.is(":animated")&&g===t&&m,u=s?M:!!this._trigger(c,[90]),this.destroyed?this:(u!==_&&t&&this.focus(n),!u||s?this:(r.attr(a[0],"aria-hidden",!t),t?(this.mouse&&(i.origin=r.event.fix(this.mouse)),r.isFunction(h.text)&&this._updateContent(h.text,_),r.isFunction(h.title)&&this._updateTitle(h.title,_),!$&&"mouse"===d.target&&d.adjust.mouse&&(r(e).bind("mousemove."+j,this._storeMouse),$=k),p||a.css("width",a.outerWidth(_)),this.reposition(n,arguments[2]),p||a.css("width",""),f.solo&&("string"==typeof f.solo?r(f.solo):r(F,f.solo)).not(a).not(f.target).qtip("hide",r.Event("tooltipsolo"))):(clearTimeout(this.timers.show),delete i.origin,$&&!r(F+'[tracking="true"]:visible',f.solo).not(a).length&&(r(e).unbind("mousemove."+j),$=_),this.blur(n)),l=r.proxy(function(){t?(Y.ie&&a[0].style.removeAttribute("filter"),a.css("overflow",""),"string"==typeof f.autofocus&&r(this.options.show.autofocus,a).focus(),this.options.show.target.trigger("qtip-"+this.id+"-inactive")):a.css({display:"",visibility:"",opacity:"",left:"",top:""}),this._trigger(t?"visible":"hidden")},this),f.effect===_||v===_?(a[c](),l()):r.isFunction(f.effect)?(a.stop(1,1),f.effect.call(a,this),a.queue("fx",function(t){l(),t()})):a.fadeTo(90,t?1:0,l),t&&f.target.trigger("qtip-"+this.id+"-inactive"),this))},b.show=function(t){return this.toggle(k,t)},b.hide=function(t){return this.toggle(_,t)},b.focus=function(t){if(!this.rendered||this.destroyed)return this;var e=r(F),n=this.tooltip,i=parseInt(n[0].style.zIndex,10),o=y.zindex+e.length;return n.hasClass(R)||this._trigger("focus",[o],t)&&(i!==o&&(e.each(function(){this.style.zIndex>i&&(this.style.zIndex=this.style.zIndex-1)}),e.filter("."+R).qtip("blur",t)),n.addClass(R)[0].style.zIndex=o),this},b.blur=function(t){return!this.rendered||this.destroyed?this:(this.tooltip.removeClass(R),this._trigger("blur",[this.tooltip.css("zIndex")],t),this)},b.disable=function(t){return this.destroyed?this:("toggle"===t?t=!(this.rendered?this.tooltip.hasClass(V):this.disabled):"boolean"!=typeof t&&(t=k),this.rendered&&this.tooltip.toggleClass(V,t).attr("aria-disabled",t),this.disabled=!!t,this)},b.enable=function(){return this.disable(_)},b._createButton=function(){var t=this,e=this.elements,n=e.tooltip,i=this.options.content.button,o="string"==typeof i,a=o?i:"Close tooltip";e.button&&e.button.remove(),i.jquery?e.button=i:e.button=r("<a />",{"class":"qtip-close "+(this.options.style.widget?"":j+"-icon"),title:a,"aria-label":a}).prepend(r("<span />",{"class":"ui-icon ui-icon-close",html:"&times;"})),e.button.appendTo(e.titlebar||n).attr("role","button").click(function(e){return n.hasClass(V)||t.hide(e),_})},b._updateButton=function(t){if(!this.rendered)return _;var e=this.elements.button;t?this._createButton():e.remove()},b._setWidget=function(){var t=this.options.style.widget,e=this.elements,n=e.tooltip,r=n.hasClass(V);n.removeClass(V),V=t?"ui-state-disabled":"qtip-disabled",n.toggleClass(V,r),n.toggleClass("ui-helper-reset "+c(),t).toggleClass(W,this.options.style.def&&!t),e.content&&e.content.toggleClass(c("content"),t),e.titlebar&&e.titlebar.toggleClass(c("header"),t),e.button&&e.button.toggleClass(j+"-icon",!t)},b._storeMouse=function(t){return(this.mouse=r.event.fix(t)).type="mousemove",this},b._bind=function(t,e,n,i,o){if(t&&n&&e.length){var a="."+this._id+(i?"-"+i:"");return r(t).bind((e.split?e:e.join(a+" "))+a,r.proxy(n,o||this)),this}},b._unbind=function(t,e){return t&&r(t).unbind("."+this._id+(e?"-"+e:"")),this},b._trigger=function(t,e,n){var i=r.Event("tooltip"+t);return i.originalEvent=n&&r.extend({},n)||this.cache.event||M,this.triggering=t,this.tooltip.trigger(i,[this].concat(e||[])),this.triggering=_,!i.isDefaultPrevented()},b._bindEvents=function(t,e,n,i,o,a){var s=n.filter(i).add(i.filter(n)),u=[];s.length&&(r.each(e,function(e,n){var i=r.inArray(n,t);i>-1&&u.push(t.splice(i,1)[0])}),u.length&&(this._bind(s,u,function(t){var e=!!this.rendered&&this.tooltip[0].offsetWidth>0;(e?a:o).call(this,t)}),n=n.not(s),i=i.not(s))),this._bind(n,t,o),this._bind(i,e,a)},b._assignInitialEvents=function(t){function e(t){return this.disabled||this.destroyed?_:(this.cache.event=t&&r.event.fix(t),this.cache.target=t&&r(t.target),clearTimeout(this.timers.show),void(this.timers.show=f.call(this,function(){this.render("object"==typeof t||n.show.ready)},n.prerender?0:n.show.delay)))}var n=this.options,i=n.show.target,o=n.hide.target,a=n.show.event?r.trim(""+n.show.event).split(" "):[],s=n.hide.event?r.trim(""+n.hide.event).split(" "):[];this._bind(this.elements.target,["remove","removeqtip"],function(t){this.destroy(!0)},"destroy"),/mouse(over|enter)/i.test(n.show.event)&&!/mouse(out|leave)/i.test(n.hide.event)&&s.push("mouseleave"),this._bind(i,"mousemove",function(t){this._storeMouse(t),this.cache.onTarget=k}),this._bindEvents(a,s,i,o,e,function(){return this.timers?void clearTimeout(this.timers.show):_}),(n.show.ready||n.prerender)&&e.call(this,t)},b._assignEvents=function(){var n=this,i=this.options,o=i.position,a=this.tooltip,s=i.show.target,u=i.hide.target,l=o.container,c=o.viewport,f=r(e),v=(r(e.body),r(t)),m=i.show.event?r.trim(""+i.show.event).split(" "):[],b=i.hide.event?r.trim(""+i.hide.event).split(" "):[];r.each(i.events,function(t,e){n._bind(a,"toggle"===t?["tooltipshow","tooltiphide"]:["tooltip"+t],e,null,a)}),/mouse(out|leave)/i.test(i.hide.event)&&"window"===i.hide.leave&&this._bind(f,["mouseout","blur"],function(t){/select|option/.test(t.target.nodeName)||t.relatedTarget||this.hide(t)}),i.hide.fixed?u=u.add(a.addClass(z)):/mouse(over|enter)/i.test(i.show.event)&&this._bind(u,"mouseleave",function(){clearTimeout(this.timers.show)}),(""+i.hide.event).indexOf("unfocus")>-1&&this._bind(l.closest("html"),["mousedown","touchstart"],function(t){var e=r(t.target),n=this.rendered&&!this.tooltip.hasClass(V)&&this.tooltip[0].offsetWidth>0,i=e.parents(F).filter(this.tooltip[0]).length>0;e[0]===this.target[0]||e[0]===this.tooltip[0]||i||this.target.has(e[0]).length||!n||this.hide(t)}),"number"==typeof i.hide.inactive&&(this._bind(s,"qtip-"+this.id+"-inactive",p,"inactive"),this._bind(u.add(a),y.inactiveEvents,p)),this._bindEvents(m,b,s,u,d,h),this._bind(s.add(a),"mousemove",function(t){if("number"==typeof i.hide.distance){var e=this.cache.origin||{},n=this.options.hide.distance,r=Math.abs;(r(t.pageX-e.pageX)>=n||r(t.pageY-e.pageY)>=n)&&this.hide(t)}this._storeMouse(t)}),"mouse"===o.target&&o.adjust.mouse&&(i.hide.event&&this._bind(s,["mouseenter","mouseleave"],function(t){return this.cache?void(this.cache.onTarget="mouseenter"===t.type):_}),this._bind(f,"mousemove",function(t){this.rendered&&this.cache.onTarget&&!this.tooltip.hasClass(V)&&this.tooltip[0].offsetWidth>0&&this.reposition(t)})),(o.adjust.resize||c.length)&&this._bind(r.event.special.resize?c:v,"resize",g),o.adjust.scroll&&this._bind(v.add(o.container),"scroll",g)},b._unassignEvents=function(){var n=this.options,i=n.show.target,o=n.hide.target,a=r.grep([this.elements.target[0],this.rendered&&this.tooltip[0],n.position.container[0],n.position.viewport[0],n.position.container.closest("html")[0],t,e],function(t){return"object"==typeof t});i&&i.toArray&&(a=a.concat(i.toArray())),o&&o.toArray&&(a=a.concat(o.toArray())),this._unbind(a)._unbind(a,"destroy")._unbind(a,"inactive")},r(function(){v(F,["mouseenter","mouseleave"],function(t){var e="mouseenter"===t.type,n=r(t.currentTarget),i=r(t.relatedTarget||t.target),o=this.options;e?(this.focus(t),n.hasClass(z)&&!n.hasClass(V)&&clearTimeout(this.timers.hide)):"mouse"===o.position.target&&o.position.adjust.mouse&&o.hide.event&&o.show.target&&!i.closest(o.show.target[0]).length&&this.hide(t),n.toggleClass(B,e)}),v("["+L+"]",q,p)}),y=r.fn.qtip=function(t,e,i){var o=(""+t).toLowerCase(),a=M,u=r.makeArray(arguments).slice(1),l=u[u.length-1],c=this[0]?r.data(this[0],j):M;return!arguments.length&&c||"api"===o?c:"string"==typeof t?(this.each(function(){var t=r.data(this,j);if(!t)return k;if(l&&l.timeStamp&&(t.cache.event=l),!e||"option"!==o&&"options"!==o)t[o]&&t[o].apply(t,u);else{if(i===n&&!r.isPlainObject(e))return a=t.get(e),_;t.set(e,i)}}),a!==M?a:this):"object"!=typeof t&&arguments.length?void 0:(c=s(r.extend(k,{},t)),this.each(function(t){var e,n;return n=r.isArray(c.id)?c.id[t]:c.id,n=!n||n===_||n.length<1||y.api[n]?y.nextid++:n,e=m(r(this),n,c),e===_?k:(y.api[n]=e,r.each(O,function(){"initialize"===this.initialize&&this(e)}),void e._assignInitialEvents(l))}))},r.qtip=i,y.api={},r.each({attr:function(t,e){if(this.length){var n=this[0],i="title",o=r.data(n,"qtip");if(t===i&&o&&"object"==typeof o&&o.options.suppress)return arguments.length<2?r.attr(n,U):(o&&o.options.content.attr===i&&o.cache.attr&&o.set("content.text",e),this.attr(U,e))}return r.fn["attr"+H].apply(this,arguments)},clone:function(t){var e=(r([]),r.fn["clone"+H].apply(this,arguments));return t||e.filter("["+U+"]").attr("title",function(){return r.attr(this,U)}).removeAttr(U),e}},function(t,e){if(!e||r.fn[t+H])return k;var n=r.fn[t+H]=r.fn[t];r.fn[t]=function(){return e.apply(this,arguments)||n.apply(this,arguments)}}),r.ui||(r["cleanData"+H]=r.cleanData,r.cleanData=function(t){for(var e,n=0;(e=r(t[n])).length;n++)if(e.attr(I))try{e.triggerHandler("removeqtip")}catch(i){}r["cleanData"+H].apply(this,arguments)}),y.version="2.2.1",y.nextid=0,y.inactiveEvents=q,y.zindex=15e3,y.defaults={prerender:_,id:_,overwrite:k,suppress:k,content:{text:k,attr:"title",title:_,button:_},position:{my:"top left",at:"bottom right",target:_,container:_,viewport:_,adjust:{x:0,y:0,mouse:k,scroll:k,resize:k,method:"flipinvert flipinvert"},effect:function(t,e,n){r(this).animate(e,{duration:200,queue:_})}},show:{target:_,event:"mouseenter",effect:k,delay:90,solo:_,ready:_,autofocus:_},hide:{target:_,event:"mouseleave",effect:k,delay:0,fixed:_,inactive:_,leave:"window",distance:_},style:{classes:"",widget:_,width:_,height:_,def:k},events:{render:M,move:M,show:M,hide:M,toggle:M,visible:M,hidden:M,focus:M,blur:M}}})}(window,document),function(t){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{var e;e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,e.dagreD3=t()}}(function(){return function t(e,n,r){function i(a,s){if(!n[a]){if(!e[a]){var u="function"==typeof require&&require;if(!s&&u)return u(a,!0);if(o)return o(a,!0);var l=new Error("Cannot find module '"+a+"'");throw l.code="MODULE_NOT_FOUND",l}var c=n[a]={exports:{}};e[a][0].call(c.exports,function(t){var n=e[a][1][t];return i(n?n:t)},c,c.exports,t,e,n,r)}return n[a].exports}for(var o="function"==typeof require&&require,a=0;a<r.length;a++)i(r[a]);return i}({1:[function(t,e,n){e.exports={graphlib:t("./lib/graphlib"),dagre:t("./lib/dagre"),intersect:t("./lib/intersect"),render:t("./lib/render"),util:t("./lib/util"),version:t("./lib/version")}},{"./lib/dagre":8,"./lib/graphlib":9,"./lib/intersect":10,"./lib/render":25,"./lib/util":27,"./lib/version":28}],2:[function(t,e,n){function r(t,e,n,r){var i=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",9).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",8).attr("markerHeight",6).attr("orient","auto"),o=i.append("path").attr("d","M 0 0 L 10 5 L 0 10 z").style("stroke-width",1).style("stroke-dasharray","1,0");
+var b=l.selectAll("g.nv-wrap.nv-sparkline").data([e]),x=b.enter().append("g").attr("class","nvd3 nv-wrap nv-sparkline");x.append("g"),b.select("g");b.attr("transform","translate("+a.left+","+a.top+")");var w=b.selectAll("path").data(function(t){return[t]});w.enter().append("path"),w.exit().remove(),w.style("stroke",function(t,e){return t.color||g(t,e)}).attr("d",d3.svg.line().x(function(t,e){return f(h(t,e))}).y(function(t,e){return d(p(t,e))}));var $=b.selectAll("circle.nv-point").data(function(t){function e(e){if(e!=-1){var n=t[e];return n.pointIndex=e,n}return null}var n=t.map(function(t,e){return p(t,e)}),r=e(n.lastIndexOf(d.domain()[1])),i=e(n.indexOf(d.domain()[0])),o=e(n.length-1);return[v?i:null,v?r:null,m?o:null].filter(function(t){return null!=t})});$.enter().append("circle"),$.exit().remove(),$.attr("cx",function(t,e){return f(h(t,t.pointIndex))}).attr("cy",function(t,e){return d(p(t,t.pointIndex))}).attr("r",2).attr("class",function(t,e){return h(t,t.pointIndex)==f.domain()[1]?"nv-point nv-currentValue":p(t,t.pointIndex)==d.domain()[0]?"nv-point nv-minValue":"nv-point nv-maxValue"})}),b.renderEnd("sparkline immediate"),e}var n,r,i,o,a={top:2,right:0,bottom:2,left:0},s=400,u=32,l=null,c=!0,f=d3.scale.linear(),d=d3.scale.linear(),h=function(t){return t.x},p=function(t){return t.y},g=t.utils.getColor(["#000"]),v=!0,m=!0,y=d3.dispatch("renderEnd"),b=t.utils.renderWatch(y);return e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return s},set:function(t){s=t}},height:{get:function(){return u},set:function(t){u=t}},xDomain:{get:function(){return n},set:function(t){n=t}},yDomain:{get:function(){return r},set:function(t){r=t}},xRange:{get:function(){return i},set:function(t){i=t}},yRange:{get:function(){return o},set:function(t){o=t}},xScale:{get:function(){return f},set:function(t){f=t}},yScale:{get:function(){return d},set:function(t){d=t}},animate:{get:function(){return c},set:function(t){c=t}},showMinMaxPoints:{get:function(){return v},set:function(t){v=t}},showCurrentPoint:{get:function(){return m},set:function(t){m=t}},x:{get:function(){return h},set:function(t){h=d3.functor(t)}},y:{get:function(){return p},set:function(t){p=d3.functor(t)}},margin:{get:function(){return a},set:function(t){a.top=void 0!==t.top?t.top:a.top,a.right=void 0!==t.right?t.right:a.right,a.bottom=void 0!==t.bottom?t.bottom:a.bottom,a.left=void 0!==t.left?t.left:a.left}},color:{get:function(){return g},set:function(e){g=t.utils.getColor(e)}}}),e.dispatch=y,t.utils.initOptions(e),e},t.models.sparklinePlus=function(){"use strict";function e(g){return m.reset(),m.models(i),g.each(function(g){function v(){if(!l){var t=M.selectAll(".nv-hoverValue").data(u),e=t.enter().append("g").attr("class","nv-hoverValue").style("stroke-opacity",0).style("fill-opacity",0);t.exit().transition().duration(250).style("stroke-opacity",0).style("fill-opacity",0).remove(),t.attr("transform",function(t){return"translate("+n(i.x()(g[t],t))+",0)"}).transition().duration(250).style("stroke-opacity",1).style("fill-opacity",1),u.length&&(e.append("line").attr("x1",0).attr("y1",-o.top).attr("x2",0).attr("y2",x),e.append("text").attr("class","nv-xValue").attr("x",-6).attr("y",-o.top).attr("text-anchor","end").attr("dy",".9em"),M.select(".nv-hoverValue .nv-xValue").text(c(i.x()(g[u[0]],u[0]))),e.append("text").attr("class","nv-yValue").attr("x",6).attr("y",-o.top).attr("text-anchor","start").attr("dy",".9em"),M.select(".nv-hoverValue .nv-yValue").text(f(i.y()(g[u[0]],u[0]))))}}function m(){function t(t,e){for(var n=Math.abs(i.x()(t[0],0)-e),r=0,o=0;o<t.length;o++)Math.abs(i.x()(t[o],o)-e)<n&&(n=Math.abs(i.x()(t[o],o)-e),r=o);return r}if(!l){var e=d3.mouse(this)[0]-o.left;u=[t(g,Math.round(n.invert(e)))],v()}}var y=d3.select(this);t.utils.initSVG(y);var b=t.utils.availableWidth(a,y,o),x=t.utils.availableHeight(s,y,o);if(e.update=function(){y.call(e)},e.container=this,!g||!g.length)return t.utils.noData(e,y),e;y.selectAll(".nv-noData").remove();var w=i.y()(g[g.length-1],g.length-1);n=i.xScale(),r=i.yScale();var $=y.selectAll("g.nv-wrap.nv-sparklineplus").data([g]),k=$.enter().append("g").attr("class","nvd3 nv-wrap nv-sparklineplus"),_=k.append("g"),M=$.select("g");_.append("g").attr("class","nv-sparklineWrap"),_.append("g").attr("class","nv-valueWrap"),_.append("g").attr("class","nv-hoverArea"),$.attr("transform","translate("+o.left+","+o.top+")");var C=M.select(".nv-sparklineWrap");if(i.width(b).height(x),C.call(i),d){var S=M.select(".nv-valueWrap"),E=S.selectAll(".nv-currentValue").data([w]);E.enter().append("text").attr("class","nv-currentValue").attr("dx",p?-8:8).attr("dy",".9em").style("text-anchor",p?"end":"start"),E.attr("x",b+(p?o.right:0)).attr("y",h?function(t){return r(t)}:0).style("fill",i.color()(g[g.length-1],g.length-1)).text(f(w))}_.select(".nv-hoverArea").append("rect").on("mousemove",m).on("click",function(){l=!l}).on("mouseout",function(){u=[],v()}),M.select(".nv-hoverArea rect").attr("transform",function(t){return"translate("+-o.left+","+-o.top+")"}).attr("width",b+o.left+o.right).attr("height",x+o.top)}),m.renderEnd("sparklinePlus immediate"),e}var n,r,i=t.models.sparkline(),o={top:15,right:100,bottom:10,left:50},a=null,s=null,u=[],l=!1,c=d3.format(",r"),f=d3.format(",.2f"),d=!0,h=!0,p=!1,g=null,v=d3.dispatch("renderEnd"),m=t.utils.renderWatch(v);return e.dispatch=v,e.sparkline=i,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return a},set:function(t){a=t}},height:{get:function(){return s},set:function(t){s=t}},xTickFormat:{get:function(){return c},set:function(t){c=t}},yTickFormat:{get:function(){return f},set:function(t){f=t}},showLastValue:{get:function(){return d},set:function(t){d=t}},alignValue:{get:function(){return h},set:function(t){h=t}},rightAlignValue:{get:function(){return p},set:function(t){p=t}},noData:{get:function(){return g},set:function(t){g=t}},margin:{get:function(){return o},set:function(t){o.top=void 0!==t.top?t.top:o.top,o.right=void 0!==t.right?t.right:o.right,o.bottom=void 0!==t.bottom?t.bottom:o.bottom,o.left=void 0!==t.left?t.left:o.left}}}),t.utils.inheritOptions(e,i),t.utils.initOptions(e),e},t.models.stackedArea=function(){"use strict";function e(h){return w.reset(),w.models(y),h.each(function(h){var b=o-i.left-i.right,$=a-i.top-i.bottom;l=d3.select(this),t.utils.initSVG(l),n=y.xScale(),r=y.yScale();var k=h;h.forEach(function(t,e){t.seriesIndex=e,t.values=t.values.map(function(t,n){return t.index=n,t.seriesIndex=e,t})});var _=h.filter(function(t){return!t.disabled});h=d3.layout.stack().order(g).offset(p).values(function(t){return t.values}).x(c).y(f).out(function(t,e,n){t.display={y:n,y0:e}})(_);var M=l.selectAll("g.nv-wrap.nv-stackedarea").data([h]),C=M.enter().append("g").attr("class","nvd3 nv-wrap nv-stackedarea"),S=C.append("defs"),E=C.append("g"),A=M.select("g");E.append("g").attr("class","nv-areaWrap"),E.append("g").attr("class","nv-scatterWrap"),M.attr("transform","translate("+i.left+","+i.top+")"),0==y.forceY().length&&y.forceY().push(0),y.width(b).height($).x(c).y(function(t){if(void 0!==t.display)return t.display.y+t.display.y0}).color(h.map(function(t,e){return t.color=t.color||s(t,t.seriesIndex),t.color}));var T=A.select(".nv-scatterWrap").datum(h);T.call(y),S.append("clipPath").attr("id","nv-edge-clip-"+u).append("rect"),M.select("#nv-edge-clip-"+u+" rect").attr("width",b).attr("height",$),A.attr("clip-path",m?"url(#nv-edge-clip-"+u+")":"");var N=d3.svg.area().defined(d).x(function(t,e){return n(c(t,e))}).y0(function(t){return r(t.display.y0)}).y1(function(t){return r(t.display.y+t.display.y0)}).interpolate(v),D=d3.svg.area().defined(d).x(function(t,e){return n(c(t,e))}).y0(function(t){return r(t.display.y0)}).y1(function(t){return r(t.display.y0)}),O=A.select(".nv-areaWrap").selectAll("path.nv-area").data(function(t){return t});O.enter().append("path").attr("class",function(t,e){return"nv-area nv-area-"+e}).attr("d",function(t,e){return D(t.values,t.seriesIndex)}).on("mouseover",function(t,e){d3.select(this).classed("hover",!0),x.areaMouseover({point:t,series:t.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:t.seriesIndex})}).on("mouseout",function(t,e){d3.select(this).classed("hover",!1),x.areaMouseout({point:t,series:t.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:t.seriesIndex})}).on("click",function(t,e){d3.select(this).classed("hover",!1),x.areaClick({point:t,series:t.key,pos:[d3.event.pageX,d3.event.pageY],seriesIndex:t.seriesIndex})}),O.exit().remove(),O.style("fill",function(t,e){return t.color||s(t,t.seriesIndex)}).style("stroke",function(t,e){return t.color||s(t,t.seriesIndex)}),O.watchTransition(w,"stackedArea path").attr("d",function(t,e){return N(t.values,e)}),y.dispatch.on("elementMouseover.area",function(t){A.select(".nv-chart-"+u+" .nv-area-"+t.seriesIndex).classed("hover",!0)}),y.dispatch.on("elementMouseout.area",function(t){A.select(".nv-chart-"+u+" .nv-area-"+t.seriesIndex).classed("hover",!1)}),e.d3_stackedOffset_stackPercent=function(t){var e,n,r,i=t.length,o=t[0].length,a=[];for(n=0;n<o;++n){for(e=0,r=0;e<k.length;e++)r+=f(k[e].values[n]);if(r)for(e=0;e<i;e++)t[e][n][1]/=r;else for(e=0;e<i;e++)t[e][n][1]=0}for(n=0;n<o;++n)a[n]=0;return a}}),w.renderEnd("stackedArea immediate"),e}var n,r,i={top:0,right:0,bottom:0,left:0},o=960,a=500,s=t.utils.defaultColor(),u=Math.floor(1e5*Math.random()),l=null,c=function(t){return t.x},f=function(t){return t.y},d=function(t,e){return!isNaN(f(t,e))&&null!==f(t,e)},h="stack",p="zero",g="default",v="linear",m=!1,y=t.models.scatter(),b=250,x=d3.dispatch("areaClick","areaMouseover","areaMouseout","renderEnd","elementClick","elementMouseover","elementMouseout");y.pointSize(2.2).pointDomain([2.2,2.2]);var w=t.utils.renderWatch(x,b);return e.dispatch=x,e.scatter=y,y.dispatch.on("elementClick",function(){x.elementClick.apply(this,arguments)}),y.dispatch.on("elementMouseover",function(){x.elementMouseover.apply(this,arguments)}),y.dispatch.on("elementMouseout",function(){x.elementMouseout.apply(this,arguments)}),e.interpolate=function(t){return arguments.length?(v=t,e):v},e.duration=function(t){return arguments.length?(b=t,w.reset(b),y.duration(b),e):b},e.dispatch=x,e.scatter=y,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return o},set:function(t){o=t}},height:{get:function(){return a},set:function(t){a=t}},defined:{get:function(){return d},set:function(t){d=t}},clipEdge:{get:function(){return m},set:function(t){m=t}},offset:{get:function(){return p},set:function(t){p=t}},order:{get:function(){return g},set:function(t){g=t}},interpolate:{get:function(){return v},set:function(t){v=t}},x:{get:function(){return c},set:function(t){c=d3.functor(t)}},y:{get:function(){return f},set:function(t){f=d3.functor(t)}},margin:{get:function(){return i},set:function(t){i.top=void 0!==t.top?t.top:i.top,i.right=void 0!==t.right?t.right:i.right,i.bottom=void 0!==t.bottom?t.bottom:i.bottom,i.left=void 0!==t.left?t.left:i.left}},color:{get:function(){return s},set:function(e){s=t.utils.getColor(e)}},style:{get:function(){return h},set:function(t){switch(h=t){case"stack":e.offset("zero"),e.order("default");break;case"stream":e.offset("wiggle"),e.order("inside-out");break;case"stream-center":e.offset("silhouette"),e.order("inside-out");break;case"expand":e.offset("expand"),e.order("default");break;case"stack_percent":e.offset(e.d3_stackedOffset_stackPercent),e.order("default")}}},duration:{get:function(){return b},set:function(t){b=t,w.reset(b),y.duration(b)}}}),t.utils.inheritOptions(e,y),t.utils.initOptions(e),e},t.models.stackedAreaChart=function(){"use strict";function e(c){return I.reset(),I.models(i),y&&I.models(o),b&&I.models(a),c.each(function(c){function S(){y&&Y.select(".nv-focus .nv-x.nv-axis").attr("transform","translate(0,"+B+")").transition().duration(D).call(o)}function I(){if(b){if("expand"===i.style()||"stack_percent"===i.style()){var t=a.tickFormat();O&&t===q||(O=t),a.tickFormat(q)}else O&&(a.tickFormat(O),O=null);Y.select(".nv-focus .nv-y.nv-axis").transition().duration(0).call(a)}}function z(t){var e=Y.select(".nv-focus .nv-stackedWrap").datum(c.filter(function(t){return!t.disabled}).map(function(e,n){return{key:e.key,area:e.area,classed:e.classed,values:e.values.filter(function(e,n){return i.x()(e,n)>=t[0]&&i.x()(e,n)<=t[1]}),disableTooltip:e.disableTooltip}}));e.transition().duration(D).call(i),S(),I()}var W=d3.select(this);t.utils.initSVG(W);var R=t.utils.availableWidth(h,W,d),B=t.utils.availableHeight(p,W,d)-(w?f.height():0);if(e.update=function(){W.transition().duration(D).call(e)},e.container=this,M.setter(F(c),e.update).getter(P(c)).update(),M.disabled=c.map(function(t){return!!t.disabled}),!C){var V;C={};for(V in M)M[V]instanceof Array?C[V]=M[V].slice(0):C[V]=M[V]}if(!(c&&c.length&&c.filter(function(t){return t.values.length}).length))return t.utils.noData(e,W),e;W.selectAll(".nv-noData").remove(),n=i.xScale(),r=i.yScale();var H=W.selectAll("g.nv-wrap.nv-stackedAreaChart").data([c]),U=H.enter().append("g").attr("class","nvd3 nv-wrap nv-stackedAreaChart").append("g"),Y=H.select("g");U.append("g").attr("class","nv-legendWrap"),U.append("g").attr("class","nv-controlsWrap");var G=U.append("g").attr("class","nv-focus");G.append("g").attr("class","nv-background").append("rect"),G.append("g").attr("class","nv-x nv-axis"),G.append("g").attr("class","nv-y nv-axis"),G.append("g").attr("class","nv-stackedWrap"),G.append("g").attr("class","nv-interactive");U.append("g").attr("class","nv-focusWrap");if(m){var X=v?R-A:R;s.width(X),Y.select(".nv-legendWrap").datum(c).call(s),s.height()>d.top&&(d.top=s.height(),B=t.utils.availableHeight(p,W,d)-(w?f.height():0)),Y.select(".nv-legendWrap").attr("transform","translate("+(R-X)+","+-d.top+")")}else Y.select(".nv-legendWrap").selectAll("*").remove();if(v){var Z=[{key:N.stacked||"Stacked",metaKey:"Stacked",disabled:"stack"!=i.style(),style:"stack"},{key:N.stream||"Stream",metaKey:"Stream",disabled:"stream"!=i.style(),style:"stream"},{key:N.expanded||"Expanded",metaKey:"Expanded",disabled:"expand"!=i.style(),style:"expand"},{key:N.stack_percent||"Stack %",metaKey:"Stack_Percent",disabled:"stack_percent"!=i.style(),style:"stack_percent"}];A=T.length/3*260,Z=Z.filter(function(t){return T.indexOf(t.metaKey)!==-1}),u.width(A).color(["#444","#444","#444"]),Y.select(".nv-controlsWrap").datum(Z).call(u),Math.max(u.height(),s.height())>d.top&&(d.top=Math.max(u.height(),s.height()),B=t.utils.availableHeight(p,W,d)),Y.select(".nv-controlsWrap").attr("transform","translate(0,"+-d.top+")")}else Y.select(".nv-controlsWrap").selectAll("*").remove();H.attr("transform","translate("+d.left+","+d.top+")"),x&&Y.select(".nv-y.nv-axis").attr("transform","translate("+R+",0)"),$&&(l.width(R).height(B).margin({left:d.left,top:d.top}).svgContainer(W).xScale(n),H.select(".nv-interactive").call(l)),Y.select(".nv-focus .nv-background rect").attr("width",R).attr("height",B),i.width(R).height(B).color(c.map(function(t,e){return t.color||g(t,e)}).filter(function(t,e){return!c[e].disabled}));var Q=Y.select(".nv-focus .nv-stackedWrap").datum(c.filter(function(t){return!t.disabled}));if(y&&o.scale(n)._ticks(t.utils.calcTicksX(R/100,c)).tickSize(-B,0),b){var K;K="wiggle"===i.offset()?0:t.utils.calcTicksY(B/36,c),a.scale(r)._ticks(K).tickSize(-R,0)}if(w){f.width(R),Y.select(".nv-focusWrap").attr("transform","translate(0,"+(B+d.bottom+f.margin().top)+")").datum(c.filter(function(t){return!t.disabled})).call(f);var J=f.brush.empty()?f.xDomain():f.brush.extent();null!==J&&z(J)}else Q.transition().call(i),S(),I();i.dispatch.on("areaClick.toggle",function(t){1===c.filter(function(t){return!t.disabled}).length?c.forEach(function(t){t.disabled=!1}):c.forEach(function(e,n){e.disabled=n!=t.seriesIndex}),M.disabled=c.map(function(t){return!!t.disabled}),E.stateChange(M),e.update()}),s.dispatch.on("stateChange",function(t){for(var n in t)M[n]=t[n];E.stateChange(M),e.update()}),u.dispatch.on("legendClick",function(t,n){t.disabled&&(Z=Z.map(function(t){return t.disabled=!0,t}),t.disabled=!1,i.style(t.style),M.style=i.style(),E.stateChange(M),e.update())}),l.dispatch.on("elementMousemove",function(n){i.clearHighlights();var r,o,a,s=[],u=0,f=!0;if(c.filter(function(t,e){return t.seriesIndex=e,!t.disabled}).forEach(function(l,c){o=t.interactiveBisect(l.values,n.pointXValue,e.x());var d=l.values[o],h=e.y()(d,o);if(null!=h&&i.highlightPoint(c,o,!0),"undefined"!=typeof d){"undefined"==typeof r&&(r=d),"undefined"==typeof a&&(a=e.xScale()(e.x()(d,o)));var p="expand"==i.style()?d.display.y:e.y()(d,o);s.push({key:l.key,value:p,color:g(l,l.seriesIndex),point:d}),k&&"expand"!=i.style()&&null!=p&&(u+=p,f=!1)}}),s.reverse(),s.length>2){var d=e.yScale().invert(n.mouseY),h=null;s.forEach(function(t,e){d=Math.abs(d);var n=Math.abs(t.point.display.y0),r=Math.abs(t.point.display.y);if(d>=n&&d<=r+n)return void(h=e)}),null!=h&&(s[h].highlight=!0)}k&&"expand"!=i.style()&&s.length>=2&&!f&&s.push({key:_,value:u,total:!0});var p=e.x()(r,o),v=l.tooltip.valueFormatter();"expand"===i.style()||"stack_percent"===i.style()?(j||(j=v),v=d3.format(".1%")):j&&(v=j,j=null),l.tooltip.valueFormatter(v).data({value:p,series:s})(),l.renderGuideLine(a)}),l.dispatch.on("elementMouseout",function(t){i.clearHighlights()}),f.dispatch.on("onBrush",function(t){z(t)}),E.on("changeState",function(t){"undefined"!=typeof t.disabled&&c.length===t.disabled.length&&(c.forEach(function(e,n){e.disabled=t.disabled[n]}),M.disabled=t.disabled),"undefined"!=typeof t.style&&(i.style(t.style),L=t.style),e.update()})}),I.renderEnd("stacked Area chart immediate"),e}var n,r,i=t.models.stackedArea(),o=t.models.axis(),a=t.models.axis(),s=t.models.legend(),u=t.models.legend(),l=t.interactiveGuideline(),c=t.models.tooltip(),f=t.models.focus(t.models.stackedArea()),d={top:30,right:25,bottom:50,left:60},h=null,p=null,g=t.utils.defaultColor(),v=!0,m=!0,y=!0,b=!0,x=!1,w=!1,$=!1,k=!0,_="TOTAL",M=t.utils.state(),C=null,S=null,E=d3.dispatch("stateChange","changeState","renderEnd"),A=250,T=["Stacked","Stream","Expanded"],N={},D=250;M.style=i.style(),o.orient("bottom").tickPadding(7),a.orient(x?"right":"left"),c.headerFormatter(function(t,e){return o.tickFormat()(t,e)}).valueFormatter(function(t,e){return a.tickFormat()(t,e)}),l.tooltip.headerFormatter(function(t,e){return o.tickFormat()(t,e)}).valueFormatter(function(t,e){return null==t?"N/A":a.tickFormat()(t,e)});var O=null,j=null;u.updateState(!1);var I=t.utils.renderWatch(E),L=i.style(),P=function(t){return function(){return{active:t.map(function(t){return!t.disabled}),style:i.style()}}},F=function(t){return function(e){void 0!==e.style&&(L=e.style),void 0!==e.active&&t.forEach(function(t,n){t.disabled=!e.active[n]})}},q=d3.format("%");return i.dispatch.on("elementMouseover.tooltip",function(t){t.point.x=i.x()(t.point),t.point.y=i.y()(t.point),c.data(t).hidden(!1)}),i.dispatch.on("elementMouseout.tooltip",function(t){c.hidden(!0)}),e.dispatch=E,e.stacked=i,e.legend=s,e.controls=u,e.xAxis=o,e.x2Axis=f.xAxis,e.yAxis=a,e.y2Axis=f.yAxis,e.interactiveLayer=l,e.tooltip=c,e.focus=f,e.dispatch=E,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{width:{get:function(){return h},set:function(t){h=t}},height:{get:function(){return p},set:function(t){p=t}},showLegend:{get:function(){return m},set:function(t){m=t}},showXAxis:{get:function(){return y},set:function(t){y=t}},showYAxis:{get:function(){return b},set:function(t){b=t}},defaultState:{get:function(){return C},set:function(t){C=t}},noData:{get:function(){return S},set:function(t){S=t}},showControls:{get:function(){return v},set:function(t){v=t}},controlLabels:{get:function(){return N},set:function(t){N=t}},controlOptions:{get:function(){return T},set:function(t){T=t}},showTotalInTooltip:{get:function(){return k},set:function(t){k=t}},totalLabel:{get:function(){return _},set:function(t){_=t}},focusEnable:{get:function(){return w},set:function(t){w=t}},focusHeight:{get:function(){return f.height()},set:function(t){f.height(t)}},brushExtent:{get:function(){return f.brushExtent()},set:function(t){f.brushExtent(t)}},margin:{get:function(){return d},set:function(t){d.top=void 0!==t.top?t.top:d.top,d.right=void 0!==t.right?t.right:d.right,d.bottom=void 0!==t.bottom?t.bottom:d.bottom,d.left=void 0!==t.left?t.left:d.left}},focusMargin:{get:function(){return f.margin},set:function(t){f.margin.top=void 0!==t.top?t.top:f.margin.top,f.margin.right=void 0!==t.right?t.right:f.margin.right,f.margin.bottom=void 0!==t.bottom?t.bottom:f.margin.bottom,f.margin.left=void 0!==t.left?t.left:f.margin.left}},duration:{get:function(){return D},set:function(t){D=t,I.reset(D),i.duration(D),o.duration(D),a.duration(D)}},color:{get:function(){return g},set:function(e){g=t.utils.getColor(e),s.color(g),i.color(g),f.color(g)}},x:{get:function(){return i.x()},set:function(t){i.x(t),f.x(t)}},y:{get:function(){return i.y()},set:function(t){i.y(t),f.y(t)}},rightAlignYAxis:{get:function(){return x},set:function(t){x=t,a.orient(x?"right":"left")}},useInteractiveGuideline:{get:function(){return $},set:function(t){$=!!t,e.interactive(!t),e.useVoronoi(!t),i.scatter.interactive(!t)}}}),t.utils.inheritOptions(e,i),t.utils.initOptions(e),e},t.models.stackedAreaWithFocusChart=function(){return t.models.stackedAreaChart().margin({bottom:30}).focusEnable(!0)},t.models.sunburst=function(){"use strict";function e(t){var e=n(t);return e>90?180:0}function n(t){var e=Math.max(0,Math.min(2*Math.PI,N(t.x))),n=Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx))),r=(e+n)/2*(180/Math.PI)-90;return r}function r(t){var e=Math.max(0,Math.min(2*Math.PI,N(t.x))),n=Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx)));return(n-e)/(2*Math.PI)}function i(t){var e=Math.max(0,Math.min(2*Math.PI,N(t.x))),n=Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx))),r=n-e;return r>M}function o(t,e){var n=d3.interpolate(N.domain(),[f.x,f.x+f.dx]),r=d3.interpolate(D.domain(),[f.y,1]),i=d3.interpolate(D.range(),[f.y?20:0,p]);return 0===e?function(){return I(t)}:function(e){return N.domain(n(e)),D.domain(r(e)).range(i(e)),I(t)}}function a(t){var e=d3.interpolate({x:t.x0,dx:t.dx0,y:t.y0,dy:t.dy0},t);return function(n){var r=e(n);return t.x0=r.x,t.dx0=r.dx,t.y0=r.y,t.dy0=r.dy,I(r)}}function s(t){var e=S(t);j[e]||(j[e]={});var n=j[e];n.dx=t.dx,n.x=t.x,n.dy=t.dy,n.y=t.y}function u(t){t.forEach(function(t){var e=S(t),n=j[e];n?(t.dx0=n.dx,t.x0=n.x,t.dy0=n.dy,t.y0=n.y):(t.dx0=t.dx,t.x0=t.x,t.dy0=t.dy,t.y0=t.y),s(t)})}function l(t){var r=w.selectAll("text"),a=w.selectAll("path");r.transition().attr("opacity",0),f=t,a.transition().duration(A).attrTween("d",o).each("end",function(r){if(r.x>=t.x&&r.x<t.x+t.dx&&r.depth>=t.depth){var o=d3.select(this.parentNode),a=o.select("text");a.transition().duration(A).text(function(t){return _(t)}).attr("opacity",function(t){return i(t)?1:0}).attr("transform",function(){var i=this.getBBox().width;if(0===r.depth)return"translate("+i/2*-1+",0)";if(r.depth===t.depth)return"translate("+(D(r.y)+5)+",0)";var o=n(r),a=e(r);return 0===a?"rotate("+o+")translate("+(D(r.y)+5)+",0)":"rotate("+o+")translate("+(D(r.y)+i+5)+",0)rotate("+a+")"})}})}function c(o){return L.reset(),o.each(function(o){w=d3.select(this),d=t.utils.availableWidth(v,w,g),h=t.utils.availableHeight(m,w,g),p=Math.min(d,h)/2,D.range([0,p]);var s=w.select("g.nvd3.nv-wrap.nv-sunburst");s[0][0]?s.attr("transform","translate("+(d/2+g.left+g.right)+","+(h/2+g.top+g.bottom)+")"):s=w.append("g").attr("class","nvd3 nv-wrap nv-sunburst nv-chart-"+x).attr("transform","translate("+(d/2+g.left+g.right)+","+(h/2+g.top+g.bottom)+")"),w.on("click",function(t,e){T.chartClick({data:t,index:e,pos:d3.event,id:x})}),O.value(b[y]||b.count);var c=O.nodes(o[0]).reverse();u(c);var f=s.selectAll(".arc-container").data(c,S),M=f.enter().append("g").attr("class","arc-container");M.append("path").attr("d",I).style("fill",function(t){return t.color?t.color:$(E?(t.children?t:t.parent).name:t.name)}).style("stroke","#FFF").on("click",l).on("mouseover",function(t,e){d3.select(this).classed("hover",!0).style("opacity",.8),T.elementMouseover({data:t,color:d3.select(this).style("fill"),percent:r(t)})}).on("mouseout",function(t,e){d3.select(this).classed("hover",!1).style("opacity",1),T.elementMouseout({data:t})}).on("mousemove",function(t,e){T.elementMousemove({data:t})}),f.each(function(t){d3.select(this).select("path").transition().duration(A).attrTween("d",a)}),k&&(f.selectAll("text").remove(),f.append("text").text(function(t){return _(t)}).transition().duration(A).attr("opacity",function(t){return i(t)?1:0}).attr("transform",function(t){var r=this.getBBox().width;if(0===t.depth)return"rotate(0)translate("+r/2*-1+",0)";var i=n(t),o=e(t);return 0===o?"rotate("+i+")translate("+(D(t.y)+5)+",0)":"rotate("+i+")translate("+(D(t.y)+r+5)+",0)rotate("+o+")"})),l(c[c.length-1]),f.exit().transition().duration(A).attr("opacity",0).each("end",function(t){var e=S(t);j[e]=void 0}).remove()}),L.renderEnd("sunburst immediate"),c}var f,d,h,p,g={top:0,right:0,bottom:0,left:0},v=600,m=600,y="count",b={count:function(t){return 1},value:function(t){return t.value||t.size},size:function(t){return t.value||t.size}},x=Math.floor(1e4*Math.random()),w=null,$=t.utils.defaultColor(),k=!1,_=function(t){return"count"===y?t.name+" #"+t.value:t.name+" "+(t.value||t.size)},M=.02,C=function(t,e){return t.name>e.name},S=function(t,e){return t.name},E=!0,A=500,T=d3.dispatch("chartClick","elementClick","elementDblClick","elementMousemove","elementMouseover","elementMouseout","renderEnd"),N=d3.scale.linear().range([0,2*Math.PI]),D=d3.scale.sqrt(),O=d3.layout.partition().sort(C),j={},I=d3.svg.arc().startAngle(function(t){return Math.max(0,Math.min(2*Math.PI,N(t.x)))}).endAngle(function(t){return Math.max(0,Math.min(2*Math.PI,N(t.x+t.dx)))}).innerRadius(function(t){return Math.max(0,D(t.y))}).outerRadius(function(t){return Math.max(0,D(t.y+t.dy))}),L=t.utils.renderWatch(T);return c.dispatch=T,c.options=t.utils.optionsFunc.bind(c),c._options=Object.create({},{width:{get:function(){return v},set:function(t){v=t}},height:{get:function(){return m},set:function(t){m=t}},mode:{get:function(){return y},set:function(t){y=t}},id:{get:function(){return x},set:function(t){x=t}},duration:{get:function(){return A},set:function(t){A=t}},groupColorByParent:{get:function(){return E},set:function(t){E=!!t}},showLabels:{get:function(){return k},set:function(t){k=!!t}},labelFormat:{get:function(){return _},set:function(t){_=t}},labelThreshold:{get:function(){return M},set:function(t){M=t}},sort:{get:function(){return C},set:function(t){C=t}},key:{get:function(){return S},set:function(t){S=t}},margin:{get:function(){return g},set:function(t){g.top=void 0!=t.top?t.top:g.top,g.right=void 0!=t.right?t.right:g.right,g.bottom=void 0!=t.bottom?t.bottom:g.bottom,g.left=void 0!=t.left?t.left:g.left}},color:{get:function(){return $},set:function(e){$=t.utils.getColor(e)}}}),t.utils.initOptions(c),c},t.models.sunburstChart=function(){"use strict";function e(r){return h.reset(),h.models(n),r.each(function(r){var s=d3.select(this);t.utils.initSVG(s);var u=t.utils.availableWidth(o,s,i),l=t.utils.availableHeight(a,s,i);return e.update=function(){0===f?s.call(e):s.transition().duration(f).call(e)},e.container=s,r&&r.length?(s.selectAll(".nv-noData").remove(),n.width(u).height(l).margin(i),void s.call(n)):(t.utils.noData(e,s),e)}),h.renderEnd("sunburstChart immediate"),e}var n=t.models.sunburst(),r=t.models.tooltip(),i={top:30,right:20,bottom:20,left:20},o=null,a=null,s=t.utils.defaultColor(),u=!1,l=(Math.round(1e5*Math.random()),null),c=null,f=250,d=d3.dispatch("stateChange","changeState","renderEnd"),h=t.utils.renderWatch(d);return r.duration(0).headerEnabled(!1).valueFormatter(function(t){return t}),n.dispatch.on("elementMouseover.tooltip",function(t){t.series={key:t.data.name,value:t.data.value||t.data.size,color:t.color,percent:t.percent},u||(delete t.percent,delete t.series.percent),r.data(t).hidden(!1)}),n.dispatch.on("elementMouseout.tooltip",function(t){r.hidden(!0)}),n.dispatch.on("elementMousemove.tooltip",function(t){r()}),e.dispatch=d,e.sunburst=n,e.tooltip=r,e.options=t.utils.optionsFunc.bind(e),e._options=Object.create({},{noData:{get:function(){return c},set:function(t){c=t}},defaultState:{get:function(){return l},set:function(t){l=t}},showTooltipPercent:{get:function(){return u},set:function(t){u=t}},color:{get:function(){return s},set:function(t){s=t,n.color(s)}},duration:{get:function(){return f},set:function(t){f=t,h.reset(f),n.duration(f)}},margin:{get:function(){return i},set:function(t){i.top=void 0!==t.top?t.top:i.top,i.right=void 0!==t.right?t.right:i.right,i.bottom=void 0!==t.bottom?t.bottom:i.bottom,i.left=void 0!==t.left?t.left:i.left,n.margin(i)}}}),t.utils.inheritOptions(e,n),t.utils.initOptions(e),e},t.version="1.8.4"}(),function(){var t=this,e="addEventListener",n="removeEventListener",r="getBoundingClientRect",i=t.attachEvent&&!t[e],o=t.document,a=function(){for(var t,e=["","-webkit-","-moz-","-o-"],n=0;n<e.length;n++)if(t=o.createElement("div"),t.style.cssText="width:"+e[n]+"calc(9px)",t.style.length)return e[n]+"calc"}(),s=function(t){return"string"==typeof t||t instanceof String?o.querySelector(t):t},u=function(u,l){var c,f,d,h,p,g,v,m,y=[];l="undefined"!=typeof l?l:{},l.gutterSize||(l.gutterSize=10),l.minSize||(l.minSize=100),l.snapOffset||(l.snapOffset=30),l.direction||(l.direction="horizontal"),"horizontal"==l.direction?(c="width",d="clientWidth",h="clientX",p="left",g="gutter gutter-horizontal",v="paddingLeft",m="paddingRight",l.cursor||(l.cursor="ew-resize")):"vertical"==l.direction&&(c="height",d="clientHeight",h="clientY",p="top",g="gutter gutter-vertical",v="paddingTop",m="paddingBottom",l.cursor||(l.cursor="ns-resize"));var b=function(n){var r=this,i=r.a,o=r.b;!r.dragging&&l.onDragStart&&l.onDragStart(),n.preventDefault(),r.dragging=!0,r.move=w.bind(r),r.stop=x.bind(r),t[e]("mouseup",r.stop),t[e]("touchend",r.stop),t[e]("touchcancel",r.stop),r.parent[e]("mousemove",r.move),r.parent[e]("touchmove",r.move),i[e]("selectstart",S),i[e]("dragstart",S),o[e]("selectstart",S),o[e]("dragstart",S),i.style.userSelect="none",i.style.webkitUserSelect="none",i.style.MozUserSelect="none",i.style.pointerEvents="none",o.style.userSelect="none",o.style.webkitUserSelect="none",o.style.MozUserSelect="none",o.style.pointerEvents="none",r.gutter.style.cursor=l.cursor,r.parent.style.cursor=l.cursor,$.call(r)},x=function(){var e=this,r=e.a,i=e.b;e.dragging&&l.onDragEnd&&l.onDragEnd(),e.dragging=!1,t[n]("mouseup",e.stop),t[n]("touchend",e.stop),t[n]("touchcancel",e.stop),e.parent[n]("mousemove",e.move),e.parent[n]("touchmove",e.move),delete e.stop,delete e.move,r[n]("selectstart",S),r[n]("dragstart",S),i[n]("selectstart",S),i[n]("dragstart",S),r.style.userSelect="",r.style.webkitUserSelect="",r.style.MozUserSelect="",r.style.pointerEvents="",i.style.userSelect="",i.style.webkitUserSelect="",i.style.MozUserSelect="",i.style.pointerEvents="",e.gutter.style.cursor="",e.parent.style.cursor=""},w=function(t){var e;this.dragging&&(e="touches"in t?t.touches[0][h]-this.start:t[h]-this.start,e<=this.aMin+l.snapOffset?e=this.aMin:e>=this.size-this.bMin-l.snapOffset&&(e=this.size-this.bMin),k.call(this,e),l.onDrag&&l.onDrag())},$=function(){var e=t.getComputedStyle(this.parent),n=this.parent[d]-parseFloat(e[v])-parseFloat(e[m]);this.size=this.a[r]()[c]+this.b[r]()[c]+this.aGutterSize+this.bGutterSize,this.percentage=Math.min(this.size/n*100,100),this.start=this.a[r]()[p]},k=function(t){this.a.style[c]=a+"("+t/this.size*this.percentage+"% - "+this.aGutterSize+"px)",this.b.style[c]=a+"("+(this.percentage-t/this.size*this.percentage)+"% - "+this.bGutterSize+"px)"},_=function(){var t=this,e=t.a,n=t.b;e[r]()[c]<t.aMin?(e.style[c]=t.aMin-t.aGutterSize+"px",n.style[c]=t.size-t.aMin-t.aGutterSize+"px"):n[r]()[c]<t.bMin&&(e.style[c]=t.size-t.bMin-t.bGutterSize+"px",n.style[c]=t.bMin-t.bGutterSize+"px")},M=function(){var t=this,e=t.a,n=t.b;n[r]()[c]<t.bMin?(e.style[c]=t.size-t.bMin-t.bGutterSize+"px",n.style[c]=t.bMin-t.bGutterSize+"px"):e[r]()[c]<t.aMin&&(e.style[c]=t.aMin-t.aGutterSize+"px",
+n.style[c]=t.size-t.aMin-t.aGutterSize+"px")},C=function(t){for(var e=0;e<t.length;e++)$.call(t[e]),_.call(t[e]);for(e=t.length-1;e>=0;e--)$.call(t[e]),M.call(t[e])},S=function(){return!1},E=s(u[0]).parentNode;if(!l.sizes){var A=100/u.length;for(l.sizes=[],f=0;f<u.length;f++)l.sizes.push(A)}if(!Array.isArray(l.minSize)){var T=[];for(f=0;f<u.length;f++)T.push(l.minSize);l.minSize=T}for(f=0;f<u.length;f++){var N,D,O=s(u[f]),j=1==f,I=f==u.length-1,L=l.gutterSize;if(f>0&&(D={a:s(u[f-1]),b:O,aMin:l.minSize[f-1],bMin:l.minSize[f],dragging:!1,parent:E,isFirst:j,isLast:I,direction:l.direction},D.aGutterSize=l.gutterSize,D.bGutterSize=l.gutterSize,j&&(D.aGutterSize=l.gutterSize/2),I&&(D.bGutterSize=l.gutterSize/2)),i)N="string"==typeof l.sizes[f]||l.sizes[f]instanceof String?l.sizes[f]:l.sizes[f]+"%";else{if(f>0){var P=o.createElement("div");P.className=g,P.style[c]=l.gutterSize+"px",P[e]("mousedown",b.bind(D)),P[e]("touchstart",b.bind(D)),E.insertBefore(P,O),D.gutter=P}0!==f&&f!=u.length-1||(L=l.gutterSize/2),N="string"==typeof l.sizes[f]||l.sizes[f]instanceof String?l.sizes[f]:a+"("+l.sizes[f]+"% - "+L+"px)"}O.style[c]=N,f>0&&y.push(D)}C(y)};"undefined"!=typeof exports?("undefined"!=typeof module&&module.exports&&(exports=module.exports=u),exports.Split=u):t.Split=u}.call(window),function(){"use strict";function t(t,e){return t.module("angularMoment",[]).constant("angularMomentConfig",{preprocess:null,timezone:"",format:null,statefulFilters:!0}).constant("moment",e).constant("amTimeAgoConfig",{withoutSuffix:!1,serverTime:null,titleFormat:null,fullDateThreshold:null,fullDateFormat:null}).directive("amTimeAgo",["$window","moment","amMoment","amTimeAgoConfig","angularMomentConfig",function(e,n,r,i,o){return function(a,s,u){function l(){var t;if(g)t=g;else if(i.serverTime){var e=(new Date).getTime(),r=e-$+i.serverTime;t=n(r)}else t=n();return t}function c(){v&&(e.clearTimeout(v),v=null)}function f(t){var n=l().diff(t,"day"),r=x&&n>=x;if(r?s.text(t.format(w)):s.text(t.from(l(),y)),b&&!s.attr("title")&&s.attr("title",t.local().format(b)),!r){var i=Math.abs(l().diff(t,"minute")),o=3600;i<1?o=1:i<60?o=30:i<180&&(o=300),v=e.setTimeout(function(){f(t)},1e3*o)}}function d(t){M&&s.attr("datetime",t)}function h(){if(c(),p){var t=r.preprocessDate(p,k,m);f(t),d(t.toISOString())}}var p,g,v=null,m=o.format,y=i.withoutSuffix,b=i.titleFormat,x=i.fullDateThreshold,w=i.fullDateFormat,$=(new Date).getTime(),k=o.preprocess,_=u.amTimeAgo,M="TIME"===s[0].nodeName.toUpperCase();a.$watch(_,function(t){return"undefined"==typeof t||null===t||""===t?(c(),void(p&&(s.text(""),d(""),p=null))):(p=t,void h())}),t.isDefined(u.amFrom)&&a.$watch(u.amFrom,function(t){g="undefined"==typeof t||null===t||""===t?null:n(t),h()}),t.isDefined(u.amWithoutSuffix)&&a.$watch(u.amWithoutSuffix,function(t){"boolean"==typeof t?(y=t,h()):y=i.withoutSuffix}),u.$observe("amFormat",function(t){"undefined"!=typeof t&&(m=t,h())}),u.$observe("amPreprocess",function(t){k=t,h()}),u.$observe("amFullDateThreshold",function(t){x=t,h()}),u.$observe("amFullDateFormat",function(t){w=t,h()}),a.$on("$destroy",function(){c()}),a.$on("amMoment:localeChanged",function(){h()})}}]).service("amMoment",["moment","$rootScope","$log","angularMomentConfig",function(e,n,r,i){this.preprocessors={utc:e.utc,unix:e.unix},this.changeLocale=function(r,i){var o=e.locale(r,i);return t.isDefined(r)&&n.$broadcast("amMoment:localeChanged"),o},this.changeTimezone=function(t){i.timezone=t,n.$broadcast("amMoment:timezoneChanged")},this.preprocessDate=function(n,o,a){return t.isUndefined(o)&&(o=i.preprocess),this.preprocessors[o]?this.preprocessors[o](n,a):(o&&r.warn("angular-moment: Ignoring unsupported value for preprocess: "+o),!isNaN(parseFloat(n))&&isFinite(n)?e(parseInt(n,10)):e(n,a))},this.applyTimezone=function(t,e){return(e=e||i.timezone)?(e.match(/^Z|[+-]\d\d:?\d\d$/i)?t=t.utcOffset(e):t.tz?t=t.tz(e):r.warn("angular-moment: named timezone specified but moment.tz() is undefined. Did you forget to include moment-timezone.js?"),t):t}}]).filter("amCalendar",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(n,r,i){if("undefined"==typeof n||null===n)return"";n=e.preprocessDate(n,r);var o=t(n);return o.isValid()?e.applyTimezone(o,i).calendar():""}return r.$stateful=n.statefulFilters,r}]).filter("amDifference",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(n,r,i,o,a,s){if("undefined"==typeof n||null===n)return"";n=e.preprocessDate(n,a);var u=t(n);if(!u.isValid())return"";var l;if("undefined"==typeof r||null===r)l=t();else if(r=e.preprocessDate(r,s),l=t(r),!l.isValid())return"";return e.applyTimezone(u).diff(e.applyTimezone(l),i,o)}return r.$stateful=n.statefulFilters,r}]).filter("amDateFormat",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(r,i,o,a,s){var u=s||n.format;if("undefined"==typeof r||null===r)return"";r=e.preprocessDate(r,o,u);var l=t(r);return l.isValid()?e.applyTimezone(l,a).format(i):""}return r.$stateful=n.statefulFilters,r}]).filter("amDurationFormat",["moment","angularMomentConfig",function(t,e){function n(e,n,r){return"undefined"==typeof e||null===e?"":t.duration(e,n).humanize(r)}return n.$stateful=e.statefulFilters,n}]).filter("amTimeAgo",["moment","amMoment","angularMomentConfig",function(t,e,n){function r(n,r,i,o){var a,s;return"undefined"==typeof n||null===n?"":(n=e.preprocessDate(n,r),a=t(n),a.isValid()?(s=t(o),"undefined"!=typeof o&&s.isValid()?e.applyTimezone(a).from(s,i):e.applyTimezone(a).fromNow(i)):"")}return r.$stateful=n.statefulFilters,r}]).filter("amSubtract",["moment","angularMomentConfig",function(t,e){function n(e,n,r){return"undefined"==typeof e||null===e?"":t(e).subtract(parseInt(n,10),r)}return n.$stateful=e.statefulFilters,n}]).filter("amAdd",["moment","angularMomentConfig",function(t,e){function n(e,n,r){return"undefined"==typeof e||null===e?"":t(e).add(parseInt(n,10),r)}return n.$stateful=e.statefulFilters,n}])}"function"==typeof define&&define.amd?define(["angular","moment"],t):"undefined"!=typeof module&&module&&module.exports?(t(angular,require("moment")),module.exports="angularMoment"):t(angular,("undefined"!=typeof global?global:window).moment)}(),function t(e,n,r){function i(a,s){if(!n[a]){if(!e[a]){var u="function"==typeof require&&require;if(!s&&u)return u(a,!0);if(o)return o(a,!0);var l=new Error("Cannot find module '"+a+"'");throw l.code="MODULE_NOT_FOUND",l}var c=n[a]={exports:{}};e[a][0].call(c.exports,function(t){var n=e[a][1][t];return i(n?n:t)},c,c.exports,t,e,n,r)}return n[a].exports}for(var o="function"==typeof require&&require,a=0;a<r.length;a++)i(r[a]);return i}({1:[function(t,e,n){(function(e){e.graphlib=t("./index")}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./index":2}],2:[function(t,e,n){var r=t("./lib");e.exports={Graph:r.Graph,json:t("./lib/json"),alg:t("./lib/alg"),version:r.version}},{"./lib":18,"./lib/alg":9,"./lib/json":19}],3:[function(t,e,n){function r(t){function e(o){i.has(r,o)||(r[o]=!0,n.push(o),i.each(t.successors(o),e),i.each(t.predecessors(o),e))}var n,r={},o=[];return i.each(t.nodes(),function(t){n=[],e(t),n.length&&o.push(n)}),o}var i=t("../lodash");e.exports=r},{"../lodash":20}],4:[function(t,e,n){function r(t,e,n){o.isArray(e)||(e=[e]);var r=[],a={};return o.each(e,function(e){if(!t.hasNode(e))throw new Error("Graph does not have node: "+e);i(t,e,"post"===n,a,r)}),r}function i(t,e,n,r,a){o.has(r,e)||(r[e]=!0,n||a.push(e),o.each(t.neighbors(e),function(e){i(t,e,n,r,a)}),n&&a.push(e))}var o=t("../lodash");e.exports=r},{"../lodash":20}],5:[function(t,e,n){function r(t,e,n){return o.transform(t.nodes(),function(r,o){r[o]=i(t,o,e,n)},{})}var i=t("./dijkstra"),o=t("../lodash");e.exports=r},{"../lodash":20,"./dijkstra":6}],6:[function(t,e,n){function r(t,e,n,r){return i(t,String(e),n||s,r||function(e){return t.outEdges(e)})}function i(t,e,n,r){var i,o,s={},u=new a,l=function(t){var e=t.v!==i?t.v:t.w,r=s[e],a=n(t),l=o.distance+a;if(a<0)throw new Error("dijkstra does not allow negative edge weights. Bad edge: "+t+" Weight: "+a);l<r.distance&&(r.distance=l,r.predecessor=i,u.decrease(e,l))};for(t.nodes().forEach(function(t){var n=t===e?0:Number.POSITIVE_INFINITY;s[t]={distance:n},u.add(t,n)});u.size()>0&&(i=u.removeMin(),o=s[i],o.distance!==Number.POSITIVE_INFINITY);)r(i).forEach(l);return s}var o=t("../lodash"),a=t("../data/priority-queue");e.exports=r;var s=o.constant(1)},{"../data/priority-queue":16,"../lodash":20}],7:[function(t,e,n){function r(t){return i.filter(o(t),function(e){return e.length>1||1===e.length&&t.hasEdge(e[0],e[0])})}var i=t("../lodash"),o=t("./tarjan");e.exports=r},{"../lodash":20,"./tarjan":14}],8:[function(t,e,n){function r(t,e,n){return i(t,e||a,n||function(e){return t.outEdges(e)})}function i(t,e,n){var r={},i=t.nodes();return i.forEach(function(t){r[t]={},r[t][t]={distance:0},i.forEach(function(e){t!==e&&(r[t][e]={distance:Number.POSITIVE_INFINITY})}),n(t).forEach(function(n){var i=n.v===t?n.w:n.v,o=e(n);r[t][i]={distance:o,predecessor:t}})}),i.forEach(function(t){var e=r[t];i.forEach(function(n){var o=r[n];i.forEach(function(n){var r=o[t],i=e[n],a=o[n],s=r.distance+i.distance;s<a.distance&&(a.distance=s,a.predecessor=i.predecessor)})})}),r}var o=t("../lodash");e.exports=r;var a=o.constant(1)},{"../lodash":20}],9:[function(t,e,n){e.exports={components:t("./components"),dijkstra:t("./dijkstra"),dijkstraAll:t("./dijkstra-all"),findCycles:t("./find-cycles"),floydWarshall:t("./floyd-warshall"),isAcyclic:t("./is-acyclic"),postorder:t("./postorder"),preorder:t("./preorder"),prim:t("./prim"),tarjan:t("./tarjan"),topsort:t("./topsort")}},{"./components":3,"./dijkstra":6,"./dijkstra-all":5,"./find-cycles":7,"./floyd-warshall":8,"./is-acyclic":10,"./postorder":11,"./preorder":12,"./prim":13,"./tarjan":14,"./topsort":15}],10:[function(t,e,n){function r(t){try{i(t)}catch(e){if(e instanceof i.CycleException)return!1;throw e}return!0}var i=t("./topsort");e.exports=r},{"./topsort":15}],11:[function(t,e,n){function r(t,e){return i(t,e,"post")}var i=t("./dfs");e.exports=r},{"./dfs":4}],12:[function(t,e,n){function r(t,e){return i(t,e,"pre")}var i=t("./dfs");e.exports=r},{"./dfs":4}],13:[function(t,e,n){function r(t,e){function n(t){var n=t.v===r?t.w:t.v,i=l.priority(n);if(void 0!==i){var o=e(t);o<i&&(u[n]=r,l.decrease(n,o))}}var r,s=new o,u={},l=new a;if(0===t.nodeCount())return s;i.each(t.nodes(),function(t){l.add(t,Number.POSITIVE_INFINITY),s.setNode(t)}),l.decrease(t.nodes()[0],0);for(var c=!1;l.size()>0;){if(r=l.removeMin(),i.has(u,r))s.setEdge(r,u[r]);else{if(c)throw new Error("Input graph is not connected: "+t);c=!0}t.nodeEdges(r).forEach(n)}return s}var i=t("../lodash"),o=t("../graph"),a=t("../data/priority-queue");e.exports=r},{"../data/priority-queue":16,"../graph":17,"../lodash":20}],14:[function(t,e,n){function r(t){function e(s){var u=o[s]={onStack:!0,lowlink:n,index:n++};if(r.push(s),t.successors(s).forEach(function(t){i.has(o,t)?o[t].onStack&&(u.lowlink=Math.min(u.lowlink,o[t].index)):(e(t),u.lowlink=Math.min(u.lowlink,o[t].lowlink))}),u.lowlink===u.index){var l,c=[];do l=r.pop(),o[l].onStack=!1,c.push(l);while(s!==l);a.push(c)}}var n=0,r=[],o={},a=[];return t.nodes().forEach(function(t){i.has(o,t)||e(t)}),a}var i=t("../lodash");e.exports=r},{"../lodash":20}],15:[function(t,e,n){function r(t){function e(s){if(o.has(r,s))throw new i;o.has(n,s)||(r[s]=!0,n[s]=!0,o.each(t.predecessors(s),e),delete r[s],a.push(s))}var n={},r={},a=[];if(o.each(t.sinks(),e),o.size(n)!==t.nodeCount())throw new i;return a}function i(){}var o=t("../lodash");e.exports=r,r.CycleException=i},{"../lodash":20}],16:[function(t,e,n){function r(){this._arr=[],this._keyIndices={}}var i=t("../lodash");e.exports=r,r.prototype.size=function(){return this._arr.length},r.prototype.keys=function(){return this._arr.map(function(t){return t.key})},r.prototype.has=function(t){return i.has(this._keyIndices,t)},r.prototype.priority=function(t){var e=this._keyIndices[t];if(void 0!==e)return this._arr[e].priority},r.prototype.min=function(){if(0===this.size())throw new Error("Queue underflow");return this._arr[0].key},r.prototype.add=function(t,e){var n=this._keyIndices;if(t=String(t),!i.has(n,t)){var r=this._arr,o=r.length;return n[t]=o,r.push({key:t,priority:e}),this._decrease(o),!0}return!1},r.prototype.removeMin=function(){this._swap(0,this._arr.length-1);var t=this._arr.pop();return delete this._keyIndices[t.key],this._heapify(0),t.key},r.prototype.decrease=function(t,e){var n=this._keyIndices[t];if(e>this._arr[n].priority)throw new Error("New priority is greater than current priority. Key: "+t+" Old: "+this._arr[n].priority+" New: "+e);this._arr[n].priority=e,this._decrease(n)},r.prototype._heapify=function(t){var e=this._arr,n=2*t,r=n+1,i=t;n<e.length&&(i=e[n].priority<e[i].priority?n:i,r<e.length&&(i=e[r].priority<e[i].priority?r:i),i!==t&&(this._swap(t,i),this._heapify(i)))},r.prototype._decrease=function(t){for(var e,n=this._arr,r=n[t].priority;0!==t&&(e=t>>1,!(n[e].priority<r));)this._swap(t,e),t=e},r.prototype._swap=function(t,e){var n=this._arr,r=this._keyIndices,i=n[t],o=n[e];n[t]=o,n[e]=i,r[o.key]=t,r[i.key]=e}},{"../lodash":20}],17:[function(t,e,n){"use strict";function r(t){this._isDirected=!l.has(t,"directed")||t.directed,this._isMultigraph=!!l.has(t,"multigraph")&&t.multigraph,this._isCompound=!!l.has(t,"compound")&&t.compound,this._label=void 0,this._defaultNodeLabelFn=l.constant(void 0),this._defaultEdgeLabelFn=l.constant(void 0),this._nodes={},this._isCompound&&(this._parent={},this._children={},this._children[f]={}),this._in={},this._preds={},this._out={},this._sucs={},this._edgeObjs={},this._edgeLabels={}}function i(t,e){t[e]?t[e]++:t[e]=1}function o(t,e){--t[e]||delete t[e]}function a(t,e,n,r){var i=""+e,o=""+n;if(!t&&i>o){var a=i;i=o,o=a}return i+d+o+d+(l.isUndefined(r)?c:r)}function s(t,e,n,r){var i=""+e,o=""+n;if(!t&&i>o){var a=i;i=o,o=a}var s={v:i,w:o};return r&&(s.name=r),s}function u(t,e){return a(t,e.v,e.w,e.name)}var l=t("./lodash");e.exports=r;var c="\0",f="\0",d="
";r.prototype._nodeCount=0,r.prototype._edgeCount=0,r.prototype.isDirected=function(){return this._isDirected},r.prototype.isMultigraph=function(){return this._isMultigraph},r.prototype.isCompound=function(){return this._isCompound},r.prototype.setGraph=function(t){return this._label=t,this},r.prototype.graph=function(){return this._label},r.prototype.setDefaultNodeLabel=function(t){return l.isFunction(t)||(t=l.constant(t)),this._defaultNodeLabelFn=t,this},r.prototype.nodeCount=function(){return this._nodeCount},r.prototype.nodes=function(){return l.keys(this._nodes)},r.prototype.sources=function(){return l.filter(this.nodes(),function(t){return l.isEmpty(this._in[t])},this)},r.prototype.sinks=function(){return l.filter(this.nodes(),function(t){return l.isEmpty(this._out[t])},this)},r.prototype.setNodes=function(t,e){var n=arguments;return l.each(t,function(t){n.length>1?this.setNode(t,e):this.setNode(t)},this),this},r.prototype.setNode=function(t,e){return l.has(this._nodes,t)?(arguments.length>1&&(this._nodes[t]=e),this):(this._nodes[t]=arguments.length>1?e:this._defaultNodeLabelFn(t),this._isCompound&&(this._parent[t]=f,this._children[t]={},this._children[f][t]=!0),this._in[t]={},this._preds[t]={},this._out[t]={},this._sucs[t]={},++this._nodeCount,this)},r.prototype.node=function(t){return this._nodes[t]},r.prototype.hasNode=function(t){return l.has(this._nodes,t)},r.prototype.removeNode=function(t){var e=this;if(l.has(this._nodes,t)){var n=function(t){e.removeEdge(e._edgeObjs[t])};delete this._nodes[t],this._isCompound&&(this._removeFromParentsChildList(t),delete this._parent[t],l.each(this.children(t),function(t){this.setParent(t)},this),delete this._children[t]),l.each(l.keys(this._in[t]),n),delete this._in[t],delete this._preds[t],l.each(l.keys(this._out[t]),n),delete this._out[t],delete this._sucs[t],--this._nodeCount}return this},r.prototype.setParent=function(t,e){if(!this._isCompound)throw new Error("Cannot set parent in a non-compound graph");if(l.isUndefined(e))e=f;else{e+="";for(var n=e;!l.isUndefined(n);n=this.parent(n))if(n===t)throw new Error("Setting "+e+" as parent of "+t+" would create create a cycle");this.setNode(e)}return this.setNode(t),this._removeFromParentsChildList(t),this._parent[t]=e,this._children[e][t]=!0,this},r.prototype._removeFromParentsChildList=function(t){delete this._children[this._parent[t]][t]},r.prototype.parent=function(t){if(this._isCompound){var e=this._parent[t];if(e!==f)return e}},r.prototype.children=function(t){if(l.isUndefined(t)&&(t=f),this._isCompound){var e=this._children[t];if(e)return l.keys(e)}else{if(t===f)return this.nodes();if(this.hasNode(t))return[]}},r.prototype.predecessors=function(t){var e=this._preds[t];if(e)return l.keys(e)},r.prototype.successors=function(t){var e=this._sucs[t];if(e)return l.keys(e)},r.prototype.neighbors=function(t){var e=this.predecessors(t);if(e)return l.union(e,this.successors(t))},r.prototype.filterNodes=function(t){function e(t){var o=r.parent(t);return void 0===o||n.hasNode(o)?(i[t]=o,o):o in i?i[o]:e(o)}var n=new this.constructor({directed:this._isDirected,multigraph:this._isMultigraph,compound:this._isCompound});n.setGraph(this.graph()),l.each(this._nodes,function(e,r){t(r)&&n.setNode(r,e)},this),l.each(this._edgeObjs,function(t){n.hasNode(t.v)&&n.hasNode(t.w)&&n.setEdge(t,this.edge(t))},this);var r=this,i={};return this._isCompound&&l.each(n.nodes(),function(t){n.setParent(t,e(t))}),n},r.prototype.setDefaultEdgeLabel=function(t){return l.isFunction(t)||(t=l.constant(t)),this._defaultEdgeLabelFn=t,this},r.prototype.edgeCount=function(){return this._edgeCount},r.prototype.edges=function(){return l.values(this._edgeObjs)},r.prototype.setPath=function(t,e){var n=this,r=arguments;return l.reduce(t,function(t,i){return r.length>1?n.setEdge(t,i,e):n.setEdge(t,i),i}),this},r.prototype.setEdge=function(){var t,e,n,r,o=!1,u=arguments[0];"object"==typeof u&&null!==u&&"v"in u?(t=u.v,e=u.w,n=u.name,2===arguments.length&&(r=arguments[1],o=!0)):(t=u,e=arguments[1],n=arguments[3],arguments.length>2&&(r=arguments[2],o=!0)),t=""+t,e=""+e,l.isUndefined(n)||(n=""+n);var c=a(this._isDirected,t,e,n);if(l.has(this._edgeLabels,c))return o&&(this._edgeLabels[c]=r),this;if(!l.isUndefined(n)&&!this._isMultigraph)throw new Error("Cannot set a named edge when isMultigraph = false");this.setNode(t),this.setNode(e),this._edgeLabels[c]=o?r:this._defaultEdgeLabelFn(t,e,n);var f=s(this._isDirected,t,e,n);return t=f.v,e=f.w,Object.freeze(f),this._edgeObjs[c]=f,i(this._preds[e],t),i(this._sucs[t],e),this._in[e][c]=f,this._out[t][c]=f,this._edgeCount++,this},r.prototype.edge=function(t,e,n){var r=1===arguments.length?u(this._isDirected,arguments[0]):a(this._isDirected,t,e,n);return this._edgeLabels[r]},r.prototype.hasEdge=function(t,e,n){var r=1===arguments.length?u(this._isDirected,arguments[0]):a(this._isDirected,t,e,n);return l.has(this._edgeLabels,r)},r.prototype.removeEdge=function(t,e,n){var r=1===arguments.length?u(this._isDirected,arguments[0]):a(this._isDirected,t,e,n),i=this._edgeObjs[r];return i&&(t=i.v,e=i.w,delete this._edgeLabels[r],delete this._edgeObjs[r],o(this._preds[e],t),o(this._sucs[t],e),delete this._in[e][r],delete this._out[t][r],this._edgeCount--),this},r.prototype.inEdges=function(t,e){var n=this._in[t];if(n){var r=l.values(n);return e?l.filter(r,function(t){return t.v===e}):r}},r.prototype.outEdges=function(t,e){var n=this._out[t];if(n){var r=l.values(n);return e?l.filter(r,function(t){return t.w===e}):r}},r.prototype.nodeEdges=function(t,e){var n=this.inEdges(t,e);if(n)return n.concat(this.outEdges(t,e))}},{"./lodash":20}],18:[function(t,e,n){e.exports={Graph:t("./graph"),version:t("./version")}},{"./graph":17,"./version":21}],19:[function(t,e,n){function r(t){var e={options:{directed:t.isDirected(),multigraph:t.isMultigraph(),compound:t.isCompound()},nodes:i(t),edges:o(t)};return s.isUndefined(t.graph())||(e.value=s.clone(t.graph())),e}function i(t){return s.map(t.nodes(),function(e){var n=t.node(e),r=t.parent(e),i={v:e};return s.isUndefined(n)||(i.value=n),s.isUndefined(r)||(i.parent=r),i})}function o(t){return s.map(t.edges(),function(e){var n=t.edge(e),r={v:e.v,w:e.w};return s.isUndefined(e.name)||(r.name=e.name),s.isUndefined(n)||(r.value=n),r})}function a(t){var e=new u(t.options).setGraph(t.value);return s.each(t.nodes,function(t){e.setNode(t.v,t.value),t.parent&&e.setParent(t.v,t.parent)}),s.each(t.edges,function(t){e.setEdge({v:t.v,w:t.w,name:t.name},t.value)}),e}var s=t("./lodash"),u=t("./graph");e.exports={write:r,read:a}},{"./graph":17,"./lodash":20}],20:[function(t,e,n){var r;if("function"==typeof t)try{r=t("lodash")}catch(i){}r||(r=window._),e.exports=r},{lodash:void 0}],21:[function(t,e,n){e.exports="1.0.7"},{}]},{},[1]),function(t,e){"use strict";"function"==typeof define&&define.amd?define(["ev-emitter/ev-emitter"],function(n){return e(t,n)}):"object"==typeof module&&module.exports?module.exports=e(t,require("ev-emitter")):t.imagesLoaded=e(t,t.EvEmitter)}("undefined"!=typeof window?window:this,function(t,e){"use strict";function n(t,e){for(var n in e)t[n]=e[n];return t}function r(t){var e=[];if(Array.isArray(t))e=t;else if("number"==typeof t.length)for(var n=0;n<t.length;n++)e.push(t[n]);else e.push(t);return e}function i(t,e,o){return this instanceof i?("string"==typeof t&&(t=document.querySelectorAll(t)),this.elements=r(t),this.options=n({},this.options),"function"==typeof e?o=e:n(this.options,e),o&&this.on("always",o),this.getImages(),s&&(this.jqDeferred=new s.Deferred),void setTimeout(function(){this.check()}.bind(this))):new i(t,e,o)}function o(t){this.img=t}function a(t,e){this.url=t,this.element=e,this.img=new Image}var s=t.jQuery,u=t.console;i.prototype=Object.create(e.prototype),i.prototype.options={},i.prototype.getImages=function(){this.images=[],this.elements.forEach(this.addElementImages,this)},i.prototype.addElementImages=function(t){"IMG"==t.nodeName&&this.addImage(t),this.options.background===!0&&this.addElementBackgroundImages(t);var e=t.nodeType;if(e&&l[e]){for(var n=t.querySelectorAll("img"),r=0;r<n.length;r++){var i=n[r];this.addImage(i)}if("string"==typeof this.options.background){var o=t.querySelectorAll(this.options.background);for(r=0;r<o.length;r++){var a=o[r];this.addElementBackgroundImages(a)}}}};var l={1:!0,9:!0,11:!0};return i.prototype.addElementBackgroundImages=function(t){var e=getComputedStyle(t);if(e)for(var n=/url\((['"])?(.*?)\1\)/gi,r=n.exec(e.backgroundImage);null!==r;){var i=r&&r[2];i&&this.addBackground(i,t),r=n.exec(e.backgroundImage)}},i.prototype.addImage=function(t){var e=new o(t);this.images.push(e)},i.prototype.addBackground=function(t,e){var n=new a(t,e);this.images.push(n)},i.prototype.check=function(){function t(t,n,r){setTimeout(function(){e.progress(t,n,r)})}var e=this;return this.progressedCount=0,this.hasAnyBroken=!1,this.images.length?void this.images.forEach(function(e){e.once("progress",t),e.check()}):void this.complete()},i.prototype.progress=function(t,e,n){this.progressedCount++,this.hasAnyBroken=this.hasAnyBroken||!t.isLoaded,this.emitEvent("progress",[this,t,e]),this.jqDeferred&&this.jqDeferred.notify&&this.jqDeferred.notify(this,t),this.progressedCount==this.images.length&&this.complete(),this.options.debug&&u&&u.log("progress: "+n,t,e)},i.prototype.complete=function(){var t=this.hasAnyBroken?"fail":"done";if(this.isComplete=!0,this.emitEvent(t,[this]),this.emitEvent("always",[this]),this.jqDeferred){var e=this.hasAnyBroken?"reject":"resolve";this.jqDeferred[e](this)}},o.prototype=Object.create(e.prototype),o.prototype.check=function(){var t=this.getIsImageComplete();return t?void this.confirm(0!==this.img.naturalWidth,"naturalWidth"):(this.proxyImage=new Image,this.proxyImage.addEventListener("load",this),this.proxyImage.addEventListener("error",this),this.img.addEventListener("load",this),this.img.addEventListener("error",this),void(this.proxyImage.src=this.img.src))},o.prototype.getIsImageComplete=function(){return this.img.complete&&void 0!==this.img.naturalWidth},o.prototype.confirm=function(t,e){this.isLoaded=t,this.emitEvent("progress",[this,this.img,e])},o.prototype.handleEvent=function(t){var e="on"+t.type;this[e]&&this[e](t)},o.prototype.onload=function(){this.confirm(!0,"onload"),this.unbindEvents()},o.prototype.onerror=function(){this.confirm(!1,"onerror"),this.unbindEvents()},o.prototype.unbindEvents=function(){this.proxyImage.removeEventListener("load",this),this.proxyImage.removeEventListener("error",this),this.img.removeEventListener("load",this),this.img.removeEventListener("error",this)},a.prototype=Object.create(o.prototype),a.prototype.check=function(){this.img.addEventListener("load",this),this.img.addEventListener("error",this),this.img.src=this.url;var t=this.getIsImageComplete();t&&(this.confirm(0!==this.img.naturalWidth,"naturalWidth"),this.unbindEvents())},a.prototype.unbindEvents=function(){this.img.removeEventListener("load",this),this.img.removeEventListener("error",this)},a.prototype.confirm=function(t,e){this.isLoaded=t,this.emitEvent("progress",[this,this.element,e])},i.makeJQueryPlugin=function(e){e=e||t.jQuery,e&&(s=e,s.fn.imagesLoaded=function(t,e){var n=new i(this,t,e);return n.jqDeferred.promise(s(this))})},i.makeJQueryPlugin(),i}),!function(t){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{var e;"undefined"!=typeof window?e=window:"undefined"!=typeof global?e=global:"undefined"!=typeof self&&(e=self),e.dagre=t()}}(function(){return function t(e,n,r){function i(a,s){if(!n[a]){if(!e[a]){var u="function"==typeof require&&require;if(!s&&u)return u(a,!0);if(o)return o(a,!0);var l=new Error("Cannot find module '"+a+"'");throw l.code="MODULE_NOT_FOUND",l}var c=n[a]={exports:{}};e[a][0].call(c.exports,function(t){var n=e[a][1][t];return i(n?n:t)},c,c.exports,t,e,n,r)}return n[a].exports}for(var o="function"==typeof require&&require,a=0;a<r.length;a++)i(r[a]);return i}({1:[function(t,e,n){e.exports={graphlib:t("./lib/graphlib"),layout:t("./lib/layout"),debug:t("./lib/debug"),util:{time:t("./lib/util").time,notime:t("./lib/util").notime},version:t("./lib/version")}},{"./lib/debug":6,"./lib/graphlib":7,"./lib/layout":9,"./lib/util":29,"./lib/version":30}],2:[function(t,e,n){"use strict";function r(t){function e(t){return function(e){return t.edge(e).weight}}var n="greedy"===t.graph().acyclicer?s(t,e(t)):i(t);a.each(n,function(e){var n=t.edge(e);t.removeEdge(e),n.forwardName=e.name,n.reversed=!0,t.setEdge(e.w,e.v,n,a.uniqueId("rev"))})}function i(t){function e(o){a.has(i,o)||(i[o]=!0,r[o]=!0,a.each(t.outEdges(o),function(t){a.has(r,t.w)?n.push(t):e(t.w)}),delete r[o])}var n=[],r={},i={};return a.each(t.nodes(),e),n}function o(t){a.each(t.edges(),function(e){var n=t.edge(e);if(n.reversed){t.removeEdge(e);var r=n.forwardName;delete n.reversed,delete n.forwardName,t.setEdge(e.w,e.v,n,r)}})}var a=t("./lodash"),s=t("./greedy-fas");e.exports={run:r,undo:o}},{"./greedy-fas":8,"./lodash":10}],3:[function(t,e,n){function r(t){function e(n){var r=t.children(n),a=t.node(n);if(r.length&&o.each(r,e),o.has(a,"minRank")){a.borderLeft=[],a.borderRight=[];for(var s=a.minRank,u=a.maxRank+1;s<u;++s)i(t,"borderLeft","_bl",n,a,s),i(t,"borderRight","_br",n,a,s)}}o.each(t.children(),e)}function i(t,e,n,r,i,o){var s={width:0,height:0,rank:o,borderType:e},u=i[e][o-1],l=a.addDummyNode(t,"border",s,n);i[e][o]=l,t.setParent(l,r),u&&t.setEdge(u,l,{weight:1})}var o=t("./lodash"),a=t("./util");e.exports=r},{"./lodash":10,"./util":29}],4:[function(t,e,n){"use strict";function r(t){var e=t.graph().rankdir.toLowerCase();"lr"!==e&&"rl"!==e||o(t)}function i(t){var e=t.graph().rankdir.toLowerCase();"bt"!==e&&"rl"!==e||s(t),"lr"!==e&&"rl"!==e||(l(t),o(t))}function o(t){f.each(t.nodes(),function(e){a(t.node(e))}),f.each(t.edges(),function(e){a(t.edge(e))})}function a(t){var e=t.width;t.width=t.height,t.height=e}function s(t){f.each(t.nodes(),function(e){u(t.node(e))}),f.each(t.edges(),function(e){var n=t.edge(e);f.each(n.points,u),f.has(n,"y")&&u(n)})}function u(t){t.y=-t.y}function l(t){f.each(t.nodes(),function(e){c(t.node(e))}),f.each(t.edges(),function(e){var n=t.edge(e);f.each(n.points,c),f.has(n,"x")&&c(n)})}function c(t){var e=t.x;t.x=t.y,t.y=e}var f=t("./lodash");e.exports={adjust:r,undo:i}},{"./lodash":10}],5:[function(t,e,n){function r(){var t={};t._next=t._prev=t,this._sentinel=t}function i(t){t._prev._next=t._next,t._next._prev=t._prev,delete t._next,delete t._prev}function o(t,e){if("_next"!==t&&"_prev"!==t)return e}e.exports=r,r.prototype.dequeue=function(){var t=this._sentinel,e=t._prev;if(e!==t)return i(e),e},r.prototype.enqueue=function(t){var e=this._sentinel;t._prev&&t._next&&i(t),t._next=e._next,e._next._prev=t,e._next=t,t._prev=e},r.prototype.toString=function(){for(var t=[],e=this._sentinel,n=e._prev;n!==e;)t.push(JSON.stringify(n,o)),n=n._prev;return"["+t.join(", ")+"]"}},{}],6:[function(t,e,n){function r(t){var e=o.buildLayerMatrix(t),n=new a({compound:!0,multigraph:!0}).setGraph({});return i.each(t.nodes(),function(e){n.setNode(e,{label:e}),n.setParent(e,"layer"+t.node(e).rank)}),i.each(t.edges(),function(t){n.setEdge(t.v,t.w,{},t.name)}),i.each(e,function(t,e){var r="layer"+e;n.setNode(r,{rank:"same"}),i.reduce(t,function(t,e){return n.setEdge(t,e,{style:"invis"}),e})}),n}var i=t("./lodash"),o=t("./util"),a=t("./graphlib").Graph;e.exports={debugOrdering:r}},{"./graphlib":7,"./lodash":10,"./util":29}],7:[function(t,e,n){var r;if("function"==typeof t)try{r=t("graphlib")}catch(i){}r||(r=window.graphlib),e.exports=r},{graphlib:void 0}],8:[function(t,e,n){function r(t,e){if(t.nodeCount()<=1)return[];var n=a(t,e||f),r=i(n.graph,n.buckets,n.zeroIdx);return u.flatten(u.map(r,function(e){return t.outEdges(e.v,e.w)}),!0)}function i(t,e,n){for(var r,i=[],a=e[e.length-1],s=e[0];t.nodeCount();){for(;r=s.dequeue();)o(t,e,n,r);for(;r=a.dequeue();)o(t,e,n,r);if(t.nodeCount())for(var u=e.length-2;u>0;--u)if(r=e[u].dequeue()){i=i.concat(o(t,e,n,r,!0));break}}return i}function o(t,e,n,r,i){var o=i?[]:void 0;return u.each(t.inEdges(r.v),function(r){var a=t.edge(r),u=t.node(r.v);i&&o.push({v:r.v,w:r.w}),u.out-=a,s(e,n,u)}),u.each(t.outEdges(r.v),function(r){var i=t.edge(r),o=r.w,a=t.node(o);a["in"]-=i,s(e,n,a)}),t.removeNode(r.v),o}function a(t,e){var n=new l,r=0,i=0;u.each(t.nodes(),function(t){n.setNode(t,{v:t,"in":0,out:0})}),u.each(t.edges(),function(t){var o=n.edge(t.v,t.w)||0,a=e(t),s=o+a;n.setEdge(t.v,t.w,s),i=Math.max(i,n.node(t.v).out+=a),r=Math.max(r,n.node(t.w)["in"]+=a)});var o=u.range(i+r+3).map(function(){return new c}),a=r+1;return u.each(n.nodes(),function(t){s(o,a,n.node(t))}),{graph:n,buckets:o,zeroIdx:a}}function s(t,e,n){n.out?n["in"]?t[n.out-n["in"]+e].enqueue(n):t[t.length-1].enqueue(n):t[0].enqueue(n)}var u=t("./lodash"),l=t("./graphlib").Graph,c=t("./data/list");e.exports=r;var f=u.constant(1)},{"./data/list":5,"./graphlib":7,"./lodash":10}],9:[function(t,e,n){"use strict";function r(t,e){var n=e&&e.debugTiming?O.time:O.notime;n("layout",function(){var e=n("  buildLayoutGraph",function(){return a(t)});n("  runLayout",function(){i(e,n)}),n("  updateInputGraph",function(){o(t,e)})})}function i(t,e){e("    makeSpaceForEdgeLabels",function(){s(t)}),e("    removeSelfEdges",function(){v(t)}),e("    acyclic",function(){$.run(t)}),e("    nestingGraph.run",function(){E.run(t)}),e("    rank",function(){_(O.asNonCompoundGraph(t))}),e("    injectEdgeLabelProxies",function(){u(t)}),e("    removeEmptyRanks",function(){S(t)}),e("    nestingGraph.cleanup",function(){E.cleanup(t)}),e("    normalizeRanks",function(){M(t)}),e("    assignRankMinMax",function(){l(t)}),e("    removeEdgeLabelProxies",function(){c(t)}),e("    normalize.run",function(){k.run(t)}),e("    parentDummyChains",function(){C(t)}),e("    addBorderSegments",function(){A(t)}),e("    order",function(){N(t)}),e("    insertSelfEdges",function(){m(t)}),e("    adjustCoordinateSystem",function(){
+T.adjust(t)}),e("    position",function(){D(t)}),e("    positionSelfEdges",function(){y(t)}),e("    removeBorderNodes",function(){g(t)}),e("    normalize.undo",function(){k.undo(t)}),e("    fixupEdgeLabelCoords",function(){h(t)}),e("    undoCoordinateSystem",function(){T.undo(t)}),e("    translateGraph",function(){f(t)}),e("    assignNodeIntersects",function(){d(t)}),e("    reversePoints",function(){p(t)}),e("    acyclic.undo",function(){$.undo(t)})}function o(t,e){w.each(t.nodes(),function(n){var r=t.node(n),i=e.node(n);r&&(r.x=i.x,r.y=i.y,e.children(n).length&&(r.width=i.width,r.height=i.height))}),w.each(t.edges(),function(n){var r=t.edge(n),i=e.edge(n);r.points=i.points,w.has(i,"x")&&(r.x=i.x,r.y=i.y)}),t.graph().width=e.graph().width,t.graph().height=e.graph().height}function a(t){var e=new j({multigraph:!0,compound:!0}),n=x(t.graph());return e.setGraph(w.merge({},L,b(n,I),w.pick(n,P))),w.each(t.nodes(),function(n){var r=x(t.node(n));e.setNode(n,w.defaults(b(r,F),q)),e.setParent(n,t.parent(n))}),w.each(t.edges(),function(n){var r=x(t.edge(n));e.setEdge(n,w.merge({},W,b(r,z),w.pick(r,R)))}),e}function s(t){var e=t.graph();e.ranksep/=2,w.each(t.edges(),function(n){var r=t.edge(n);r.minlen*=2,"c"!==r.labelpos.toLowerCase()&&("TB"===e.rankdir||"BT"===e.rankdir?r.width+=r.labeloffset:r.height+=r.labeloffset)})}function u(t){w.each(t.edges(),function(e){var n=t.edge(e);if(n.width&&n.height){var r=t.node(e.v),i=t.node(e.w),o={rank:(i.rank-r.rank)/2+r.rank,e:e};O.addDummyNode(t,"edge-proxy",o,"_ep")}})}function l(t){var e=0;w.each(t.nodes(),function(n){var r=t.node(n);r.borderTop&&(r.minRank=t.node(r.borderTop).rank,r.maxRank=t.node(r.borderBottom).rank,e=w.max(e,r.maxRank))}),t.graph().maxRank=e}function c(t){w.each(t.nodes(),function(e){var n=t.node(e);"edge-proxy"===n.dummy&&(t.edge(n.e).labelRank=n.rank,t.removeNode(e))})}function f(t){function e(t){var e=t.x,a=t.y,s=t.width,u=t.height;n=Math.min(n,e-s/2),r=Math.max(r,e+s/2),i=Math.min(i,a-u/2),o=Math.max(o,a+u/2)}var n=Number.POSITIVE_INFINITY,r=0,i=Number.POSITIVE_INFINITY,o=0,a=t.graph(),s=a.marginx||0,u=a.marginy||0;w.each(t.nodes(),function(n){e(t.node(n))}),w.each(t.edges(),function(n){var r=t.edge(n);w.has(r,"x")&&e(r)}),n-=s,i-=u,w.each(t.nodes(),function(e){var r=t.node(e);r.x-=n,r.y-=i}),w.each(t.edges(),function(e){var r=t.edge(e);w.each(r.points,function(t){t.x-=n,t.y-=i}),w.has(r,"x")&&(r.x-=n),w.has(r,"y")&&(r.y-=i)}),a.width=r-n+s,a.height=o-i+u}function d(t){w.each(t.edges(),function(e){var n,r,i=t.edge(e),o=t.node(e.v),a=t.node(e.w);i.points?(n=i.points[0],r=i.points[i.points.length-1]):(i.points=[],n=a,r=o),i.points.unshift(O.intersectRect(o,n)),i.points.push(O.intersectRect(a,r))})}function h(t){w.each(t.edges(),function(e){var n=t.edge(e);if(w.has(n,"x"))switch("l"!==n.labelpos&&"r"!==n.labelpos||(n.width-=n.labeloffset),n.labelpos){case"l":n.x-=n.width/2+n.labeloffset;break;case"r":n.x+=n.width/2+n.labeloffset}})}function p(t){w.each(t.edges(),function(e){var n=t.edge(e);n.reversed&&n.points.reverse()})}function g(t){w.each(t.nodes(),function(e){if(t.children(e).length){var n=t.node(e),r=t.node(n.borderTop),i=t.node(n.borderBottom),o=t.node(w.last(n.borderLeft)),a=t.node(w.last(n.borderRight));n.width=Math.abs(a.x-o.x),n.height=Math.abs(i.y-r.y),n.x=o.x+n.width/2,n.y=r.y+n.height/2}}),w.each(t.nodes(),function(e){"border"===t.node(e).dummy&&t.removeNode(e)})}function v(t){w.each(t.edges(),function(e){if(e.v===e.w){var n=t.node(e.v);n.selfEdges||(n.selfEdges=[]),n.selfEdges.push({e:e,label:t.edge(e)}),t.removeEdge(e)}})}function m(t){var e=O.buildLayerMatrix(t);w.each(e,function(e){var n=0;w.each(e,function(e,r){var i=t.node(e);i.order=r+n,w.each(i.selfEdges,function(e){O.addDummyNode(t,"selfedge",{width:e.label.width,height:e.label.height,rank:i.rank,order:r+ ++n,e:e.e,label:e.label},"_se")}),delete i.selfEdges})})}function y(t){w.each(t.nodes(),function(e){var n=t.node(e);if("selfedge"===n.dummy){var r=t.node(n.e.v),i=r.x+r.width/2,o=r.y,a=n.x-i,s=r.height/2;t.setEdge(n.e,n.label),t.removeNode(e),n.label.points=[{x:i+2*a/3,y:o-s},{x:i+5*a/6,y:o-s},{x:i+a,y:o},{x:i+5*a/6,y:o+s},{x:i+2*a/3,y:o+s}],n.label.x=n.x,n.label.y=n.y}})}function b(t,e){return w.mapValues(w.pick(t,e),Number)}function x(t){var e={};return w.each(t,function(t,n){e[n.toLowerCase()]=t}),e}var w=t("./lodash"),$=t("./acyclic"),k=t("./normalize"),_=t("./rank"),M=t("./util").normalizeRanks,C=t("./parent-dummy-chains"),S=t("./util").removeEmptyRanks,E=t("./nesting-graph"),A=t("./add-border-segments"),T=t("./coordinate-system"),N=t("./order"),D=t("./position"),O=t("./util"),j=t("./graphlib").Graph;e.exports=r;var I=["nodesep","edgesep","ranksep","marginx","marginy"],L={ranksep:50,edgesep:20,nodesep:50,rankdir:"tb"},P=["acyclicer","ranker","rankdir","align"],F=["width","height"],q={width:0,height:0},z=["minlen","weight","width","height","labeloffset"],W={minlen:1,weight:1,width:0,height:0,labeloffset:10,labelpos:"r"},R=["labelpos"]},{"./acyclic":2,"./add-border-segments":3,"./coordinate-system":4,"./graphlib":7,"./lodash":10,"./nesting-graph":11,"./normalize":12,"./order":17,"./parent-dummy-chains":22,"./position":24,"./rank":26,"./util":29}],10:[function(t,e,n){var r;if("function"==typeof t)try{r=t("lodash")}catch(i){}r||(r=window._),e.exports=r},{lodash:void 0}],11:[function(t,e,n){function r(t){var e=l.addDummyNode(t,"root",{},"_root"),n=o(t),r=u.max(n)-1,s=2*r+1;t.graph().nestingRoot=e,u.each(t.edges(),function(e){t.edge(e).minlen*=s});var c=a(t)+1;u.each(t.children(),function(o){i(t,e,s,c,r,n,o)}),t.graph().nodeRankFactor=s}function i(t,e,n,r,o,a,s){var c=t.children(s);if(!c.length)return void(s!==e&&t.setEdge(e,s,{weight:0,minlen:n}));var f=l.addBorderNode(t,"_bt"),d=l.addBorderNode(t,"_bb"),h=t.node(s);t.setParent(f,s),h.borderTop=f,t.setParent(d,s),h.borderBottom=d,u.each(c,function(u){i(t,e,n,r,o,a,u);var l=t.node(u),c=l.borderTop?l.borderTop:u,h=l.borderBottom?l.borderBottom:u,p=l.borderTop?r:2*r,g=c!==h?1:o-a[s]+1;t.setEdge(f,c,{weight:p,minlen:g,nestingEdge:!0}),t.setEdge(h,d,{weight:p,minlen:g,nestingEdge:!0})}),t.parent(s)||t.setEdge(e,f,{weight:0,minlen:o+a[s]})}function o(t){function e(r,i){var o=t.children(r);o&&o.length&&u.each(o,function(t){e(t,i+1)}),n[r]=i}var n={};return u.each(t.children(),function(t){e(t,1)}),n}function a(t){return u.reduce(t.edges(),function(e,n){return e+t.edge(n).weight},0)}function s(t){var e=t.graph();t.removeNode(e.nestingRoot),delete e.nestingRoot,u.each(t.edges(),function(e){var n=t.edge(e);n.nestingEdge&&t.removeEdge(e)})}var u=t("./lodash"),l=t("./util");e.exports={run:r,cleanup:s}},{"./lodash":10,"./util":29}],12:[function(t,e,n){"use strict";function r(t){t.graph().dummyChains=[],a.each(t.edges(),function(e){i(t,e)})}function i(t,e){var n=e.v,r=t.node(n).rank,i=e.w,o=t.node(i).rank,a=e.name,u=t.edge(e),l=u.labelRank;if(o!==r+1){t.removeEdge(e);var c,f,d;for(d=0,++r;r<o;++d,++r)u.points=[],f={width:0,height:0,edgeLabel:u,edgeObj:e,rank:r},c=s.addDummyNode(t,"edge",f,"_d"),r===l&&(f.width=u.width,f.height=u.height,f.dummy="edge-label",f.labelpos=u.labelpos),t.setEdge(n,c,{weight:u.weight},a),0===d&&t.graph().dummyChains.push(c),n=c;t.setEdge(n,i,{weight:u.weight},a)}}function o(t){a.each(t.graph().dummyChains,function(e){var n,r=t.node(e),i=r.edgeLabel;for(t.setEdge(r.edgeObj,i);r.dummy;)n=t.successors(e)[0],t.removeNode(e),i.points.push({x:r.x,y:r.y}),"edge-label"===r.dummy&&(i.x=r.x,i.y=r.y,i.width=r.width,i.height=r.height),e=n,r=t.node(e)})}var a=t("./lodash"),s=t("./util");e.exports={run:r,undo:o}},{"./lodash":10,"./util":29}],13:[function(t,e,n){function r(t,e,n){var r,o={};i.each(n,function(n){for(var i,a,s=t.parent(n);s;){if(i=t.parent(s),i?(a=o[i],o[i]=s):(a=r,r=s),a&&a!==s)return void e.setEdge(a,s);s=i}})}var i=t("../lodash");e.exports=r},{"../lodash":10}],14:[function(t,e,n){function r(t,e){return i.map(e,function(e){var n=t.inEdges(e);if(n.length){var r=i.reduce(n,function(e,n){var r=t.edge(n),i=t.node(n.v);return{sum:e.sum+r.weight*i.order,weight:e.weight+r.weight}},{sum:0,weight:0});return{v:e,barycenter:r.sum/r.weight,weight:r.weight}}return{v:e}})}var i=t("../lodash");e.exports=r},{"../lodash":10}],15:[function(t,e,n){function r(t,e,n){var r=i(t),s=new a({compound:!0}).setGraph({root:r}).setDefaultNodeLabel(function(e){return t.node(e)});return o.each(t.nodes(),function(i){var a=t.node(i),u=t.parent(i);(a.rank===e||a.minRank<=e&&e<=a.maxRank)&&(s.setNode(i),s.setParent(i,u||r),o.each(t[n](i),function(e){var n=e.v===i?e.w:e.v,r=s.edge(n,i),a=o.isUndefined(r)?0:r.weight;s.setEdge(n,i,{weight:t.edge(e).weight+a})}),o.has(a,"minRank")&&s.setNode(i,{borderLeft:a.borderLeft[e],borderRight:a.borderRight[e]}))}),s}function i(t){for(var e;t.hasNode(e=o.uniqueId("_root")););return e}var o=t("../lodash"),a=t("../graphlib").Graph;e.exports=r},{"../graphlib":7,"../lodash":10}],16:[function(t,e,n){"use strict";function r(t,e){for(var n=0,r=1;r<e.length;++r)n+=i(t,e[r-1],e[r]);return n}function i(t,e,n){for(var r=o.zipObject(n,o.map(n,function(t,e){return e})),i=o.flatten(o.map(e,function(e){return o.chain(t.outEdges(e)).map(function(e){return{pos:r[e.w],weight:t.edge(e).weight}}).sortBy("pos").value()}),!0),a=1;a<n.length;)a<<=1;var s=2*a-1;a-=1;var u=o.map(new Array(s),function(){return 0}),l=0;return o.each(i.forEach(function(t){var e=t.pos+a;u[e]+=t.weight;for(var n=0;e>0;)e%2&&(n+=u[e+1]),e=e-1>>1,u[e]+=t.weight;l+=t.weight*n})),l}var o=t("../lodash");e.exports=r},{"../lodash":10}],17:[function(t,e,n){"use strict";function r(t){var e=p.maxRank(t),n=i(t,s.range(1,e+1),"inEdges"),r=i(t,s.range(e-1,-1,-1),"outEdges"),c=u(t);a(t,c);for(var f,d=Number.POSITIVE_INFINITY,h=0,g=0;g<4;++h,++g){o(h%2?n:r,h%4>=2),c=p.buildLayerMatrix(t);var v=l(t,c);v<d&&(g=0,f=s.cloneDeep(c),d=v)}a(t,f)}function i(t,e,n){return s.map(e,function(e){return f(t,e,n)})}function o(t,e){var n=new h;s.each(t,function(t){var r=t.graph().root,i=c(t,r,n,e);s.each(i.vs,function(e,n){t.node(e).order=n}),d(t,n,i.vs)})}function a(t,e){s.each(e,function(e){s.each(e,function(e,n){t.node(e).order=n})})}var s=t("../lodash"),u=t("./init-order"),l=t("./cross-count"),c=t("./sort-subgraph"),f=t("./build-layer-graph"),d=t("./add-subgraph-constraints"),h=t("../graphlib").Graph,p=t("../util");e.exports=r},{"../graphlib":7,"../lodash":10,"../util":29,"./add-subgraph-constraints":13,"./build-layer-graph":15,"./cross-count":16,"./init-order":18,"./sort-subgraph":20}],18:[function(t,e,n){"use strict";function r(t){function e(r){if(!i.has(n,r)){n[r]=!0;var o=t.node(r);a[o.rank].push(r),i.each(t.successors(r),e)}}var n={},r=i.filter(t.nodes(),function(e){return!t.children(e).length}),o=i.max(i.map(r,function(e){return t.node(e).rank})),a=i.map(i.range(o+1),function(){return[]}),s=i.sortBy(r,function(e){return t.node(e).rank});return i.each(s,e),a}var i=t("../lodash");e.exports=r},{"../lodash":10}],19:[function(t,e,n){"use strict";function r(t,e){var n={};a.each(t,function(t,e){var r=n[t.v]={indegree:0,"in":[],out:[],vs:[t.v],i:e};a.isUndefined(t.barycenter)||(r.barycenter=t.barycenter,r.weight=t.weight)}),a.each(e.edges(),function(t){var e=n[t.v],r=n[t.w];a.isUndefined(e)||a.isUndefined(r)||(r.indegree++,e.out.push(n[t.w]))});var r=a.filter(n,function(t){return!t.indegree});return i(r)}function i(t){function e(t){return function(e){e.merged||(a.isUndefined(e.barycenter)||a.isUndefined(t.barycenter)||e.barycenter>=t.barycenter)&&o(t,e)}}function n(e){return function(n){n["in"].push(e),0===--n.indegree&&t.push(n)}}for(var r=[];t.length;){var i=t.pop();r.push(i),a.each(i["in"].reverse(),e(i)),a.each(i.out,n(i))}return a.chain(r).filter(function(t){return!t.merged}).map(function(t){return a.pick(t,["vs","i","barycenter","weight"])}).value()}function o(t,e){var n=0,r=0;t.weight&&(n+=t.barycenter*t.weight,r+=t.weight),e.weight&&(n+=e.barycenter*e.weight,r+=e.weight),t.vs=e.vs.concat(t.vs),t.barycenter=n/r,t.weight=r,t.i=Math.min(e.i,t.i),e.merged=!0}var a=t("../lodash");e.exports=r},{"../lodash":10}],20:[function(t,e,n){function r(t,e,n,c){var f=t.children(e),d=t.node(e),h=d?d.borderLeft:void 0,p=d?d.borderRight:void 0,g={};h&&(f=a.filter(f,function(t){return t!==h&&t!==p}));var v=s(t,f);a.each(v,function(e){if(t.children(e.v).length){var i=r(t,e.v,n,c);g[e.v]=i,a.has(i,"barycenter")&&o(e,i)}});var m=u(v,n);i(m,g);var y=l(m,c);if(h&&(y.vs=a.flatten([h,y.vs,p],!0),t.predecessors(h).length)){var b=t.node(t.predecessors(h)[0]),x=t.node(t.predecessors(p)[0]);a.has(y,"barycenter")||(y.barycenter=0,y.weight=0),y.barycenter=(y.barycenter*y.weight+b.order+x.order)/(y.weight+2),y.weight+=2}return y}function i(t,e){a.each(t,function(t){t.vs=a.flatten(t.vs.map(function(t){return e[t]?e[t].vs:t}),!0)})}function o(t,e){a.isUndefined(t.barycenter)?(t.barycenter=e.barycenter,t.weight=e.weight):(t.barycenter=(t.barycenter*t.weight+e.barycenter*e.weight)/(t.weight+e.weight),t.weight+=e.weight)}var a=t("../lodash"),s=t("./barycenter"),u=t("./resolve-conflicts"),l=t("./sort");e.exports=r},{"../lodash":10,"./barycenter":14,"./resolve-conflicts":19,"./sort":21}],21:[function(t,e,n){function r(t,e){var n=s.partition(t,function(t){return a.has(t,"barycenter")}),r=n.lhs,u=a.sortBy(n.rhs,function(t){return-t.i}),l=[],c=0,f=0,d=0;r.sort(o(!!e)),d=i(l,u,d),a.each(r,function(t){d+=t.vs.length,l.push(t.vs),c+=t.barycenter*t.weight,f+=t.weight,d=i(l,u,d)});var h={vs:a.flatten(l,!0)};return f&&(h.barycenter=c/f,h.weight=f),h}function i(t,e,n){for(var r;e.length&&(r=a.last(e)).i<=n;)e.pop(),t.push(r.vs),n++;return n}function o(t){return function(e,n){return e.barycenter<n.barycenter?-1:e.barycenter>n.barycenter?1:t?n.i-e.i:e.i-n.i}}var a=t("../lodash"),s=t("../util");e.exports=r},{"../lodash":10,"../util":29}],22:[function(t,e,n){function r(t){var e=o(t);a.each(t.graph().dummyChains,function(n){for(var r=t.node(n),o=r.edgeObj,a=i(t,e,o.v,o.w),s=a.path,u=a.lca,l=0,c=s[l],f=!0;n!==o.w;){if(r=t.node(n),f){for(;(c=s[l])!==u&&t.node(c).maxRank<r.rank;)l++;c===u&&(f=!1)}if(!f){for(;l<s.length-1&&t.node(c=s[l+1]).minRank<=r.rank;)l++;c=s[l]}t.setParent(n,c),n=t.successors(n)[0]}})}function i(t,e,n,r){var i,o,a=[],s=[],u=Math.min(e[n].low,e[r].low),l=Math.max(e[n].lim,e[r].lim);i=n;do i=t.parent(i),a.push(i);while(i&&(e[i].low>u||l>e[i].lim));for(o=i,i=r;(i=t.parent(i))!==o;)s.push(i);return{path:a.concat(s.reverse()),lca:o}}function o(t){function e(i){var o=r;a.each(t.children(i),e),n[i]={low:o,lim:r++}}var n={},r=0;return a.each(t.children(),e),n}var a=t("./lodash");e.exports=r},{"./lodash":10}],23:[function(t,e,n){"use strict";function r(t,e){function n(e,n){var i=0,s=0,u=e.length,l=m.last(n);return m.each(n,function(e,c){var f=o(t,e),d=f?t.node(f).order:u;(f||e===l)&&(m.each(n.slice(s,c+1),function(e){m.each(t.predecessors(e),function(n){var o=t.node(n),s=o.order;!(s<i||d<s)||o.dummy&&t.node(e).dummy||a(r,n,e)})}),s=c+1,i=d)}),n}var r={};return m.reduce(e,n),r}function i(t,e){function n(e,n,r,o,s){var u;m.each(m.range(n,r),function(n){u=e[n],t.node(u).dummy&&m.each(t.predecessors(u),function(e){var n=t.node(e);n.dummy&&(n.order<o||n.order>s)&&a(i,e,u)})})}function r(e,r){var i,o=-1,a=0;return m.each(r,function(s,u){if("border"===t.node(s).dummy){var l=t.predecessors(s);l.length&&(i=t.node(l[0]).order,n(r,a,u,o,i),a=u,o=i)}n(r,a,r.length,i,e.length)}),r}var i={};return m.reduce(e,r),i}function o(t,e){if(t.node(e).dummy)return m.find(t.predecessors(e),function(e){return t.node(e).dummy})}function a(t,e,n){if(e>n){var r=e;e=n,n=r}var i=t[e];i||(t[e]=i={}),i[n]=!0}function s(t,e,n){if(e>n){var r=e;e=n,n=r}return m.has(t[e],n)}function u(t,e,n,r){var i={},o={},a={};return m.each(e,function(t){m.each(t,function(t,e){i[t]=t,o[t]=t,a[t]=e})}),m.each(e,function(t){var e=-1;m.each(t,function(t){var u=r(t);if(u.length){u=m.sortBy(u,function(t){return a[t]});for(var l=(u.length-1)/2,c=Math.floor(l),f=Math.ceil(l);c<=f;++c){var d=u[c];o[t]===t&&e<a[d]&&!s(n,t,d)&&(o[d]=t,o[t]=i[t]=i[d],e=a[d])}}})}),{root:i,align:o}}function l(t,e,n,r,i){function o(t){m.has(l,t)||(l[t]=!0,s[t]=m.reduce(u.inEdges(t),function(t,e){return o(e.v),Math.max(t,s[e.v]+u.edge(e))},0))}function a(e){if(2!==l[e]){l[e]++;var n=t.node(e),r=m.reduce(u.outEdges(e),function(t,e){return a(e.w),Math.min(t,s[e.w]-u.edge(e))},Number.POSITIVE_INFINITY);r!==Number.POSITIVE_INFINITY&&n.borderType!==f&&(s[e]=Math.max(s[e],r))}}var s={},u=c(t,e,n,i),l={};m.each(u.nodes(),o);var f=i?"borderLeft":"borderRight";return m.each(u.nodes(),a),m.each(r,function(t){s[t]=s[n[t]]}),s}function c(t,e,n,r){var i=new y,o=t.graph(),a=g(o.nodesep,o.edgesep,r);return m.each(e,function(e){var r;m.each(e,function(e){var o=n[e];if(i.setNode(o),r){var s=n[r],u=i.edge(s,o);i.setEdge(s,o,Math.max(a(t,e,r),u||0))}r=e})}),i}function f(t,e){return m.min(e,function(e){var n=m.min(e,function(e,n){return e-v(t,n)/2}),r=m.max(e,function(e,n){return e+v(t,n)/2});return r-n})}function d(t,e){var n=m.min(e),r=m.max(e);m.each(["u","d"],function(i){m.each(["l","r"],function(o){var a,s=i+o,u=t[s];u!==e&&(a="l"===o?n-m.min(u):r-m.max(u),a&&(t[s]=m.mapValues(u,function(t){return t+a})))})})}function h(t,e){return m.mapValues(t.ul,function(n,r){if(e)return t[e.toLowerCase()][r];var i=m.sortBy(m.pluck(t,r));return(i[1]+i[2])/2})}function p(t){var e,n=b.buildLayerMatrix(t),o=m.merge(r(t,n),i(t,n)),a={};m.each(["u","d"],function(r){e="u"===r?n:m.values(n).reverse(),m.each(["l","r"],function(n){"r"===n&&(e=m.map(e,function(t){return m.values(t).reverse()}));var i=m.bind("u"===r?t.predecessors:t.successors,t),s=u(t,e,o,i),c=l(t,e,s.root,s.align,"r"===n);"r"===n&&(c=m.mapValues(c,function(t){return-t})),a[r+n]=c})});var s=f(t,a);return d(a,s),h(a,t.graph().align)}function g(t,e,n){return function(r,i,o){var a,s=r.node(i),u=r.node(o),l=0;if(l+=s.width/2,m.has(s,"labelpos"))switch(s.labelpos.toLowerCase()){case"l":a=-s.width/2;break;case"r":a=s.width/2}if(a&&(l+=n?a:-a),a=0,l+=(s.dummy?e:t)/2,l+=(u.dummy?e:t)/2,l+=u.width/2,m.has(u,"labelpos"))switch(u.labelpos.toLowerCase()){case"l":a=u.width/2;break;case"r":a=-u.width/2}return a&&(l+=n?a:-a),a=0,l}}function v(t,e){return t.node(e).width}var m=t("../lodash"),y=t("../graphlib").Graph,b=t("../util");e.exports={positionX:p,findType1Conflicts:r,findType2Conflicts:i,addConflict:a,hasConflict:s,verticalAlignment:u,horizontalCompaction:l,alignCoordinates:d,findSmallestWidthAlignment:f,balance:h}},{"../graphlib":7,"../lodash":10,"../util":29}],24:[function(t,e,n){"use strict";function r(t){t=a.asNonCompoundGraph(t),i(t),o.each(s(t),function(e,n){t.node(n).x=e})}function i(t){var e=a.buildLayerMatrix(t),n=t.graph().ranksep,r=0;o.each(e,function(e){var i=o.max(o.map(e,function(e){return t.node(e).height}));o.each(e,function(e){t.node(e).y=r+i/2}),r+=i+n})}var o=t("../lodash"),a=t("../util"),s=t("./bk").positionX;e.exports=r},{"../lodash":10,"../util":29,"./bk":23}],25:[function(t,e,n){"use strict";function r(t){var e=new u({directed:!1}),n=t.nodes()[0],r=t.nodeCount();e.setNode(n,{});for(var s,c;i(e,t)<r;)s=o(e,t),c=e.hasNode(s.v)?l(t,s):-l(t,s),a(e,t,c);return e}function i(t,e){function n(r){s.each(e.nodeEdges(r),function(i){var o=i.v,a=r===o?i.w:o;t.hasNode(a)||l(e,i)||(t.setNode(a,{}),t.setEdge(r,a,{}),n(a))})}return s.each(t.nodes(),n),t.nodeCount()}function o(t,e){return s.min(e.edges(),function(n){if(t.hasNode(n.v)!==t.hasNode(n.w))return l(e,n)})}function a(t,e,n){s.each(t.nodes(),function(t){e.node(t).rank+=n})}var s=t("../lodash"),u=t("../graphlib").Graph,l=t("./util").slack;e.exports=r},{"../graphlib":7,"../lodash":10,"./util":28}],26:[function(t,e,n){"use strict";function r(t){switch(t.graph().ranker){case"network-simplex":o(t);break;case"tight-tree":i(t);break;case"longest-path":c(t);break;default:o(t)}}function i(t){s(t),u(t)}function o(t){l(t)}var a=t("./util"),s=a.longestPath,u=t("./feasible-tree"),l=t("./network-simplex");e.exports=r;var c=s},{"./feasible-tree":25,"./network-simplex":27,"./util":28}],27:[function(t,e,n){"use strict";function r(t){t=w(t),y(t);var e=v(t);s(e),i(e,t);for(var n,r;n=l(e);)r=c(e,t,n),f(e,t,n,r)}function i(t,e){var n=x(t,t.nodes());n=n.slice(0,n.length-1),g.each(n,function(n){o(t,e,n)})}function o(t,e,n){var r=t.node(n),i=r.parent;t.edge(n,i).cutvalue=a(t,e,n)}function a(t,e,n){var r=t.node(n),i=r.parent,o=!0,a=e.edge(n,i),s=0;return a||(o=!1,a=e.edge(i,n)),s=a.weight,g.each(e.nodeEdges(n),function(r){var a=r.v===n,u=a?r.w:r.v;if(u!==i){var l=a===o,c=e.edge(r).weight;if(s+=l?c:-c,h(t,n,u)){var f=t.edge(n,u).cutvalue;s+=l?-f:f}}}),s}function s(t,e){arguments.length<2&&(e=t.nodes()[0]),u(t,{},1,e)}function u(t,e,n,r,i){var o=n,a=t.node(r);return e[r]=!0,g.each(t.neighbors(r),function(i){g.has(e,i)||(n=u(t,e,n,i,r))}),a.low=o,a.lim=n++,i?a.parent=i:delete a.parent,n}function l(t){return g.find(t.edges(),function(e){return t.edge(e).cutvalue<0})}function c(t,e,n){var r=n.v,i=n.w;e.hasEdge(r,i)||(r=n.w,i=n.v);var o=t.node(r),a=t.node(i),s=o,u=!1;o.lim>a.lim&&(s=a,u=!0);var l=g.filter(e.edges(),function(e){return u===p(t,t.node(e.v),s)&&u!==p(t,t.node(e.w),s)});return g.min(l,function(t){return m(e,t)})}function f(t,e,n,r){var o=n.v,a=n.w;t.removeEdge(o,a),t.setEdge(r.v,r.w,{}),s(t),i(t,e),d(t,e)}function d(t,e){var n=g.find(t.nodes(),function(t){return!e.node(t).parent}),r=b(t,n);r=r.slice(1),g.each(r,function(n){var r=t.node(n).parent,i=e.edge(n,r),o=!1;i||(i=e.edge(r,n),o=!0),e.node(n).rank=e.node(r).rank+(o?i.minlen:-i.minlen)})}function h(t,e,n){return t.hasEdge(e,n)}function p(t,e,n){return n.low<=e.lim&&e.lim<=n.lim}var g=t("../lodash"),v=t("./feasible-tree"),m=t("./util").slack,y=t("./util").longestPath,b=t("../graphlib").alg.preorder,x=t("../graphlib").alg.postorder,w=t("../util").simplify;e.exports=r,r.initLowLimValues=s,r.initCutValues=i,r.calcCutValue=a,r.leaveEdge=l,r.enterEdge=c,r.exchangeEdges=f},{"../graphlib":7,"../lodash":10,"../util":29,"./feasible-tree":25,"./util":28}],28:[function(t,e,n){"use strict";function r(t){function e(r){var i=t.node(r);if(o.has(n,r))return i.rank;n[r]=!0;var a=o.min(o.map(t.outEdges(r),function(n){return e(n.w)-t.edge(n).minlen}));return a===Number.POSITIVE_INFINITY&&(a=0),i.rank=a}var n={};o.each(t.sources(),e)}function i(t,e){return t.node(e.w).rank-t.node(e.v).rank-t.edge(e).minlen}var o=t("../lodash");e.exports={longestPath:r,slack:i}},{"../lodash":10}],29:[function(t,e,n){"use strict";function r(t,e,n,r){var i;do i=m.uniqueId(r);while(t.hasNode(i));return n.dummy=e,t.setNode(i,n),i}function i(t){var e=(new y).setGraph(t.graph());return m.each(t.nodes(),function(n){e.setNode(n,t.node(n))}),m.each(t.edges(),function(n){var r=e.edge(n.v,n.w)||{weight:0,minlen:1},i=t.edge(n);e.setEdge(n.v,n.w,{weight:r.weight+i.weight,minlen:Math.max(r.minlen,i.minlen)})}),e}function o(t){var e=new y({multigraph:t.isMultigraph()}).setGraph(t.graph());return m.each(t.nodes(),function(n){t.children(n).length||e.setNode(n,t.node(n))}),m.each(t.edges(),function(n){e.setEdge(n,t.edge(n))}),e}function a(t){var e=m.map(t.nodes(),function(e){var n={};return m.each(t.outEdges(e),function(e){n[e.w]=(n[e.w]||0)+t.edge(e).weight}),n});return m.zipObject(t.nodes(),e)}function s(t){var e=m.map(t.nodes(),function(e){var n={};return m.each(t.inEdges(e),function(e){n[e.v]=(n[e.v]||0)+t.edge(e).weight}),n});return m.zipObject(t.nodes(),e)}function u(t,e){var n=t.x,r=t.y,i=e.x-n,o=e.y-r,a=t.width/2,s=t.height/2;if(!i&&!o)throw new Error("Not possible to find intersection inside of the rectangle");var u,l;return Math.abs(o)*a>Math.abs(i)*s?(o<0&&(s=-s),u=s*i/o,l=s):(i<0&&(a=-a),u=a,l=a*o/i),{x:n+u,y:r+l}}function l(t){var e=m.map(m.range(h(t)+1),function(){return[]});return m.each(t.nodes(),function(n){var r=t.node(n),i=r.rank;m.isUndefined(i)||(e[i][r.order]=n)}),e}function c(t){var e=m.min(m.map(t.nodes(),function(e){return t.node(e).rank}));m.each(t.nodes(),function(n){var r=t.node(n);m.has(r,"rank")&&(r.rank-=e)})}function f(t){var e=m.min(m.map(t.nodes(),function(e){return t.node(e).rank})),n=[];m.each(t.nodes(),function(r){var i=t.node(r).rank-e;n[i]||(n[i]=[]),n[i].push(r)});var r=0,i=t.graph().nodeRankFactor;m.each(n,function(e,n){m.isUndefined(e)&&n%i!==0?--r:r&&m.each(e,function(e){t.node(e).rank+=r})})}function d(t,e,n,i){var o={width:0,height:0};return arguments.length>=4&&(o.rank=n,o.order=i),r(t,"border",o,e)}function h(t){return m.max(m.map(t.nodes(),function(e){var n=t.node(e).rank;if(!m.isUndefined(n))return n}))}function p(t,e){var n={lhs:[],rhs:[]};return m.each(t,function(t){e(t)?n.lhs.push(t):n.rhs.push(t)}),n}function g(t,e){var n=m.now();try{return e()}finally{console.log(t+" time: "+(m.now()-n)+"ms")}}function v(t,e){return e()}var m=t("./lodash"),y=t("./graphlib").Graph;e.exports={addDummyNode:r,simplify:i,asNonCompoundGraph:o,successorWeights:a,predecessorWeights:s,intersectRect:u,buildLayerMatrix:l,normalizeRanks:c,removeEmptyRanks:f,addBorderNode:d,maxRank:h,partition:p,time:g,notime:v}},{"./graphlib":7,"./lodash":10}],30:[function(t,e,n){e.exports="0.7.4"},{}]},{},[1])(1)}),function(t,e,n){!function(t){"use strict";"function"==typeof define&&define.amd?define(["jquery"],t):jQuery&&!jQuery.fn.qtip&&t(jQuery)}(function(r){"use strict";function i(t,e,n,i){this.id=n,this.target=t,this.tooltip=N,this.elements={target:t},this._id=V+"-"+n,this.timers={img:{}},this.options=e,this.plugins={},this.cache={event:{},target:r(),disabled:T,attr:i,onTooltip:T,lastClass:""},this.rendered=this.destroyed=this.disabled=this.waiting=this.hiddenDuringWait=this.positioning=this.triggering=T}function o(t){return t===N||"object"!==r.type(t)}function a(t){return!(r.isFunction(t)||t&&t.attr||t.length||"object"===r.type(t)&&(t.jquery||t.then))}function s(t){var e,n,i,s;return o(t)?T:(o(t.metadata)&&(t.metadata={type:t.metadata}),"content"in t&&(e=t.content,o(e)||e.jquery||e.done?e=t.content={text:n=a(e)?T:e}:n=e.text,"ajax"in e&&(i=e.ajax,s=i&&i.once!==T,delete e.ajax,e.text=function(t,e){var o=n||r(this).attr(e.options.content.attr)||"Loading...",a=r.ajax(r.extend({},i,{context:e})).then(i.success,N,i.error).then(function(t){return t&&s&&e.set("content.text",t),t},function(t,n,r){e.destroyed||0===t.status||e.set("content.text",n+": "+r)});return s?o:(e.set("content.text",o),a)}),"title"in e&&(r.isPlainObject(e.title)&&(e.button=e.title.button,e.title=e.title.text),a(e.title||T)&&(e.title=T))),"position"in t&&o(t.position)&&(t.position={my:t.position,at:t.position}),"show"in t&&o(t.show)&&(t.show=t.show.jquery?{target:t.show}:t.show===A?{ready:A}:{event:t.show}),"hide"in t&&o(t.hide)&&(t.hide=t.hide.jquery?{target:t.hide}:{event:t.hide}),"style"in t&&o(t.style)&&(t.style={classes:t.style}),r.each(B,function(){this.sanitize&&this.sanitize(t)}),t)}function u(t,e){for(var n,r=0,i=t,o=e.split(".");i=i[o[r++]];)r<o.length&&(n=i);return[n||t,o.pop()]}function l(t,e){var n,r,i;for(n in this.checks)for(r in this.checks[n])(i=new RegExp(r,"i").exec(t))&&(e.push(i),("builtin"===n||this.plugins[n])&&this.checks[n][r].apply(this.plugins[n]||this,e))}function c(t){return Y.concat("").join(t?"-"+t+" ":" ")}function f(t,e){return e>0?setTimeout(r.proxy(t,this),e):void t.call(this)}function d(t){this.tooltip.hasClass(tt)||(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this.timers.show=f.call(this,function(){this.toggle(A,t)},this.options.show.delay))}function h(t){if(!this.tooltip.hasClass(tt)&&!this.destroyed){var e=r(t.relatedTarget),n=e.closest(G)[0]===this.tooltip[0],i=e[0]===this.options.show.target[0];if(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this!==e[0]&&"mouse"===this.options.position.target&&n||this.options.hide.fixed&&/mouse(out|leave|move)/.test(t.type)&&(n||i))try{t.preventDefault(),t.stopImmediatePropagation()}catch(o){}else this.timers.hide=f.call(this,function(){this.toggle(T,t)},this.options.hide.delay,this)}}function p(t){!this.tooltip.hasClass(tt)&&this.options.hide.inactive&&(clearTimeout(this.timers.inactive),this.timers.inactive=f.call(this,function(){this.hide(t)},this.options.hide.inactive))}function g(t){this.rendered&&this.tooltip[0].offsetWidth>0&&this.reposition(t)}function v(t,n,i){r(e.body).delegate(t,(n.split?n:n.join("."+V+" "))+"."+V,function(){var t=_.api[r.attr(this,U)];t&&!t.disabled&&i.apply(t,arguments)})}function m(t,n,o){var a,u,l,c,f,d=r(e.body),h=t[0]===e?d:t,p=t.metadata?t.metadata(o.metadata):N,g="html5"===o.metadata.type&&p?p[o.metadata.name]:N,v=t.data(o.metadata.name||"qtipopts");try{v="string"==typeof v?r.parseJSON(v):v}catch(m){}if(c=r.extend(A,{},_.defaults,o,"object"==typeof v?s(v):N,s(g||p)),u=c.position,c.id=n,"boolean"==typeof c.content.text){if(l=t.attr(c.content.attr),c.content.attr===T||!l)return T;c.content.text=l}if(u.container.length||(u.container=d),u.target===T&&(u.target=h),c.show.target===T&&(c.show.target=h),c.show.solo===A&&(c.show.solo=u.container.closest("body")),c.hide.target===T&&(c.hide.target=h),c.position.viewport===A&&(c.position.viewport=u.container),u.container=u.container.eq(0),u.at=new C(u.at,A),u.my=new C(u.my),t.data(V))if(c.overwrite)t.qtip("destroy",!0);else if(c.overwrite===T)return T;return t.attr(H,n),c.suppress&&(f=t.attr("title"))&&t.removeAttr("title").attr(nt,f).attr("title",""),a=new i(t,c,n,(!!l)),t.data(V,a),a}function y(t){return t.charAt(0).toUpperCase()+t.slice(1)}function b(t,e){var r,i,o=e.charAt(0).toUpperCase()+e.slice(1),a=(e+" "+mt.join(o+" ")+o).split(" "),s=0;if(vt[e])return t.css(vt[e]);for(;r=a[s++];)if((i=t.css(r))!==n)return vt[e]=r,i}function x(t,e){return Math.ceil(parseFloat(b(t,e)))}function w(t,e){this._ns="tip",this.options=e,this.offset=e.offset,this.size=[e.width,e.height],this.init(this.qtip=t)}function $(t,e){this.options=e,this._ns="-modal",this.init(this.qtip=t)}function k(t,e){this._ns="ie6",this.init(this.qtip=t)}var _,M,C,S,E,A=!0,T=!1,N=null,D="x",O="y",j="width",I="height",L="top",P="left",F="bottom",q="right",z="center",W="flipinvert",R="shift",B={},V="qtip",H="data-hasqtip",U="data-qtip-id",Y=["ui-widget","ui-tooltip"],G="."+V,X="click dblclick mousedown mouseup mousemove mouseleave mouseenter".split(" "),Z=V+"-fixed",Q=V+"-default",K=V+"-focus",J=V+"-hover",tt=V+"-disabled",et="_replacedByqTip",nt="oldtitle",rt={ie:function(){for(var t=4,n=e.createElement("div");(n.innerHTML="<!--[if gt IE "+t+"]><i></i><![endif]-->")&&n.getElementsByTagName("i")[0];t+=1);return t>4?t:NaN}(),iOS:parseFloat((""+(/CPU.*OS ([0-9_]{1,5})|(CPU like).*AppleWebKit.*Mobile/i.exec(navigator.userAgent)||[0,""])[1]).replace("undefined","3_2").replace("_",".").replace("_",""))||T};M=i.prototype,M._when=function(t){return r.when.apply(r,t)},M.render=function(t){if(this.rendered||this.destroyed)return this;var e,n=this,i=this.options,o=this.cache,a=this.elements,s=i.content.text,u=i.content.title,l=i.content.button,c=i.position,f=("."+this._id+" ",[]);return r.attr(this.target[0],"aria-describedby",this._id),o.posClass=this._createPosClass((this.position={my:c.my,at:c.at}).my),this.tooltip=a.tooltip=e=r("<div/>",{id:this._id,"class":[V,Q,i.style.classes,o.posClass].join(" "),width:i.style.width||"",height:i.style.height||"",tracking:"mouse"===c.target&&c.adjust.mouse,role:"alert","aria-live":"polite","aria-atomic":T,"aria-describedby":this._id+"-content","aria-hidden":A}).toggleClass(tt,this.disabled).attr(U,this.id).data(V,this).appendTo(c.container).append(a.content=r("<div />",{"class":V+"-content",id:this._id+"-content","aria-atomic":A})),this.rendered=-1,this.positioning=A,u&&(this._createTitle(),r.isFunction(u)||f.push(this._updateTitle(u,T))),l&&this._createButton(),r.isFunction(s)||f.push(this._updateContent(s,T)),this.rendered=A,this._setWidget(),r.each(B,function(t){var e;"render"===this.initialize&&(e=this(n))&&(n.plugins[t]=e)}),this._unassignEvents(),this._assignEvents(),this._when(f).then(function(){n._trigger("render"),n.positioning=T,n.hiddenDuringWait||!i.show.ready&&!t||n.toggle(A,o.event,T),n.hiddenDuringWait=T}),_.api[this.id]=this,this},M.destroy=function(t){function e(){if(!this.destroyed){this.destroyed=A;var t,e=this.target,n=e.attr(nt);this.rendered&&this.tooltip.stop(1,0).find("*").remove().end().remove(),r.each(this.plugins,function(t){this.destroy&&this.destroy();
+});for(t in this.timers)clearTimeout(this.timers[t]);e.removeData(V).removeAttr(U).removeAttr(H).removeAttr("aria-describedby"),this.options.suppress&&n&&e.attr("title",n).removeAttr(nt),this._unassignEvents(),this.options=this.elements=this.cache=this.timers=this.plugins=this.mouse=N,delete _.api[this.id]}}return this.destroyed?this.target:(t===A&&"hide"!==this.triggering||!this.rendered?e.call(this):(this.tooltip.one("tooltiphidden",r.proxy(e,this)),!this.triggering&&this.hide()),this.target)},S=M.checks={builtin:{"^id$":function(t,e,n,i){var o=n===A?_.nextid:n,a=V+"-"+o;o!==T&&o.length>0&&!r("#"+a).length?(this._id=a,this.rendered&&(this.tooltip[0].id=this._id,this.elements.content[0].id=this._id+"-content",this.elements.title[0].id=this._id+"-title")):t[e]=i},"^prerender":function(t,e,n){n&&!this.rendered&&this.render(this.options.show.ready)},"^content.text$":function(t,e,n){this._updateContent(n)},"^content.attr$":function(t,e,n,r){this.options.content.text===this.target.attr(r)&&this._updateContent(this.target.attr(n))},"^content.title$":function(t,e,n){return n?(n&&!this.elements.title&&this._createTitle(),void this._updateTitle(n)):this._removeTitle()},"^content.button$":function(t,e,n){this._updateButton(n)},"^content.title.(text|button)$":function(t,e,n){this.set("content."+e,n)},"^position.(my|at)$":function(t,e,n){"string"==typeof n&&(this.position[e]=t[e]=new C(n,"at"===e))},"^position.container$":function(t,e,n){this.rendered&&this.tooltip.appendTo(n)},"^show.ready$":function(t,e,n){n&&(!this.rendered&&this.render(A)||this.toggle(A))},"^style.classes$":function(t,e,n,r){this.rendered&&this.tooltip.removeClass(r).addClass(n)},"^style.(width|height)":function(t,e,n){this.rendered&&this.tooltip.css(e,n)},"^style.widget|content.title":function(){this.rendered&&this._setWidget()},"^style.def":function(t,e,n){this.rendered&&this.tooltip.toggleClass(Q,!!n)},"^events.(render|show|move|hide|focus|blur)$":function(t,e,n){this.rendered&&this.tooltip[(r.isFunction(n)?"":"un")+"bind"]("tooltip"+e,n)},"^(show|hide|position).(event|target|fixed|inactive|leave|distance|viewport|adjust)":function(){if(this.rendered){var t=this.options.position;this.tooltip.attr("tracking","mouse"===t.target&&t.adjust.mouse),this._unassignEvents(),this._assignEvents()}}}},M.get=function(t){if(this.destroyed)return this;var e=u(this.options,t.toLowerCase()),n=e[0][e[1]];return n.precedance?n.string():n};var it=/^position\.(my|at|adjust|target|container|viewport)|style|content|show\.ready/i,ot=/^prerender|show\.ready/i;M.set=function(t,e){if(this.destroyed)return this;var n,i=this.rendered,o=T,a=this.options;this.checks;return"string"==typeof t?(n=t,t={},t[n]=e):t=r.extend({},t),r.each(t,function(e,n){if(i&&ot.test(e))return void delete t[e];var s,l=u(a,e.toLowerCase());s=l[0][l[1]],l[0][l[1]]=n&&n.nodeType?r(n):n,o=it.test(e)||o,t[e]=[l[0],l[1],n,s]}),s(a),this.positioning=A,r.each(t,r.proxy(l,this)),this.positioning=T,this.rendered&&this.tooltip[0].offsetWidth>0&&o&&this.reposition("mouse"===a.position.target?N:this.cache.event),this},M._update=function(t,e,n){var i=this,o=this.cache;return this.rendered&&t?(r.isFunction(t)&&(t=t.call(this.elements.target,o.event,this)||""),r.isFunction(t.then)?(o.waiting=A,t.then(function(t){return o.waiting=T,i._update(t,e)},N,function(t){return i._update(t,e)})):t===T||!t&&""!==t?T:(t.jquery&&t.length>0?e.empty().append(t.css({display:"block",visibility:"visible"})):e.html(t),this._waitForContent(e).then(function(t){i.rendered&&i.tooltip[0].offsetWidth>0&&i.reposition(o.event,!t.length)}))):T},M._waitForContent=function(t){var e=this.cache;return e.waiting=A,(r.fn.imagesLoaded?t.imagesLoaded():r.Deferred().resolve([])).done(function(){e.waiting=T}).promise()},M._updateContent=function(t,e){this._update(t,this.elements.content,e)},M._updateTitle=function(t,e){this._update(t,this.elements.title,e)===T&&this._removeTitle(T)},M._createTitle=function(){var t=this.elements,e=this._id+"-title";t.titlebar&&this._removeTitle(),t.titlebar=r("<div />",{"class":V+"-titlebar "+(this.options.style.widget?c("header"):"")}).append(t.title=r("<div />",{id:e,"class":V+"-title","aria-atomic":A})).insertBefore(t.content).delegate(".qtip-close","mousedown keydown mouseup keyup mouseout",function(t){r(this).toggleClass("ui-state-active ui-state-focus","down"===t.type.substr(-4))}).delegate(".qtip-close","mouseover mouseout",function(t){r(this).toggleClass("ui-state-hover","mouseover"===t.type)}),this.options.content.button&&this._createButton()},M._removeTitle=function(t){var e=this.elements;e.title&&(e.titlebar.remove(),e.titlebar=e.title=e.button=N,t!==T&&this.reposition())},M._createPosClass=function(t){return V+"-pos-"+(t||this.options.position.my).abbrev()},M.reposition=function(n,i){if(!this.rendered||this.positioning||this.destroyed)return this;this.positioning=A;var o,a,s,u,l=this.cache,c=this.tooltip,f=this.options.position,d=f.target,h=f.my,p=f.at,g=f.viewport,v=f.container,m=f.adjust,y=m.method.split(" "),b=c.outerWidth(T),x=c.outerHeight(T),w=0,$=0,k=c.css("position"),_={left:0,top:0},M=c[0].offsetWidth>0,C=n&&"scroll"===n.type,S=r(t),E=v[0].ownerDocument,N=this.mouse;if(r.isArray(d)&&2===d.length)p={x:P,y:L},_={left:d[0],top:d[1]};else if("mouse"===d)p={x:P,y:L},(!m.mouse||this.options.hide.distance)&&l.origin&&l.origin.pageX?n=l.origin:!n||n&&("resize"===n.type||"scroll"===n.type)?n=l.event:N&&N.pageX&&(n=N),"static"!==k&&(_=v.offset()),E.body.offsetWidth!==(t.innerWidth||E.documentElement.clientWidth)&&(a=r(e.body).offset()),_={left:n.pageX-_.left+(a&&a.left||0),top:n.pageY-_.top+(a&&a.top||0)},m.mouse&&C&&N&&(_.left-=(N.scrollX||0)-S.scrollLeft(),_.top-=(N.scrollY||0)-S.scrollTop());else{if("event"===d?n&&n.target&&"scroll"!==n.type&&"resize"!==n.type?l.target=r(n.target):n.target||(l.target=this.elements.target):"event"!==d&&(l.target=r(d.jquery?d:this.elements.target)),d=l.target,d=r(d).eq(0),0===d.length)return this;d[0]===e||d[0]===t?(w=rt.iOS?t.innerWidth:d.width(),$=rt.iOS?t.innerHeight:d.height(),d[0]===t&&(_={top:(g||d).scrollTop(),left:(g||d).scrollLeft()})):B.imagemap&&d.is("area")?o=B.imagemap(this,d,p,B.viewport?y:T):B.svg&&d&&d[0].ownerSVGElement?o=B.svg(this,d,p,B.viewport?y:T):(w=d.outerWidth(T),$=d.outerHeight(T),_=d.offset()),o&&(w=o.width,$=o.height,a=o.offset,_=o.position),_=this.reposition.offset(d,_,v),(rt.iOS>3.1&&rt.iOS<4.1||rt.iOS>=4.3&&rt.iOS<4.33||!rt.iOS&&"fixed"===k)&&(_.left-=S.scrollLeft(),_.top-=S.scrollTop()),(!o||o&&o.adjustable!==T)&&(_.left+=p.x===q?w:p.x===z?w/2:0,_.top+=p.y===F?$:p.y===z?$/2:0)}return _.left+=m.x+(h.x===q?-b:h.x===z?-b/2:0),_.top+=m.y+(h.y===F?-x:h.y===z?-x/2:0),B.viewport?(s=_.adjusted=B.viewport(this,_,f,w,$,b,x),a&&s.left&&(_.left+=a.left),a&&s.top&&(_.top+=a.top),s.my&&(this.position.my=s.my)):_.adjusted={left:0,top:0},l.posClass!==(u=this._createPosClass(this.position.my))&&c.removeClass(l.posClass).addClass(l.posClass=u),this._trigger("move",[_,g.elem||g],n)?(delete _.adjusted,i===T||!M||isNaN(_.left)||isNaN(_.top)||"mouse"===d||!r.isFunction(f.effect)?c.css(_):r.isFunction(f.effect)&&(f.effect.call(c,this,r.extend({},_)),c.queue(function(t){r(this).css({opacity:"",height:""}),rt.ie&&this.style.removeAttribute("filter"),t()})),this.positioning=T,this):this},M.reposition.offset=function(t,n,i){function o(t,e){n.left+=e*t.scrollLeft(),n.top+=e*t.scrollTop()}if(!i[0])return n;var a,s,u,l,c=r(t[0].ownerDocument),f=!!rt.ie&&"CSS1Compat"!==e.compatMode,d=i[0];do"static"!==(s=r.css(d,"position"))&&("fixed"===s?(u=d.getBoundingClientRect(),o(c,-1)):(u=r(d).position(),u.left+=parseFloat(r.css(d,"borderLeftWidth"))||0,u.top+=parseFloat(r.css(d,"borderTopWidth"))||0),n.left-=u.left+(parseFloat(r.css(d,"marginLeft"))||0),n.top-=u.top+(parseFloat(r.css(d,"marginTop"))||0),a||"hidden"===(l=r.css(d,"overflow"))||"visible"===l||(a=r(d)));while(d=d.offsetParent);return a&&(a[0]!==c[0]||f)&&o(a,1),n};var at=(C=M.reposition.Corner=function(t,e){t=(""+t).replace(/([A-Z])/," $1").replace(/middle/gi,z).toLowerCase(),this.x=(t.match(/left|right/i)||t.match(/center/)||["inherit"])[0].toLowerCase(),this.y=(t.match(/top|bottom|center/i)||["inherit"])[0].toLowerCase(),this.forceY=!!e;var n=t.charAt(0);this.precedance="t"===n||"b"===n?O:D}).prototype;at.invert=function(t,e){this[t]=this[t]===P?q:this[t]===q?P:e||this[t]},at.string=function(t){var e=this.x,n=this.y,r=e!==n?"center"===e||"center"!==n&&(this.precedance===O||this.forceY)?[n,e]:[e,n]:[e];return t!==!1?r.join(" "):r},at.abbrev=function(){var t=this.string(!1);return t[0].charAt(0)+(t[1]&&t[1].charAt(0)||"")},at.clone=function(){return new C(this.string(),this.forceY)},M.toggle=function(t,n){var i=this.cache,o=this.options,a=this.tooltip;if(n){if(/over|enter/.test(n.type)&&i.event&&/out|leave/.test(i.event.type)&&o.show.target.add(n.target).length===o.show.target.length&&a.has(n.relatedTarget).length)return this;i.event=r.event.fix(n)}if(this.waiting&&!t&&(this.hiddenDuringWait=A),!this.rendered)return t?this.render(1):this;if(this.destroyed||this.disabled)return this;var s,u,l,c=t?"show":"hide",f=this.options[c],d=(this.options[t?"hide":"show"],this.options.position),h=this.options.content,p=this.tooltip.css("width"),g=this.tooltip.is(":visible"),v=t||1===f.target.length,m=!n||f.target.length<2||i.target[0]===n.target;return(typeof t).search("boolean|number")&&(t=!g),s=!a.is(":animated")&&g===t&&m,u=s?N:!!this._trigger(c,[90]),this.destroyed?this:(u!==T&&t&&this.focus(n),!u||s?this:(r.attr(a[0],"aria-hidden",!t),t?(this.mouse&&(i.origin=r.event.fix(this.mouse)),r.isFunction(h.text)&&this._updateContent(h.text,T),r.isFunction(h.title)&&this._updateTitle(h.title,T),!E&&"mouse"===d.target&&d.adjust.mouse&&(r(e).bind("mousemove."+V,this._storeMouse),E=A),p||a.css("width",a.outerWidth(T)),this.reposition(n,arguments[2]),p||a.css("width",""),f.solo&&("string"==typeof f.solo?r(f.solo):r(G,f.solo)).not(a).not(f.target).qtip("hide",r.Event("tooltipsolo"))):(clearTimeout(this.timers.show),delete i.origin,E&&!r(G+'[tracking="true"]:visible',f.solo).not(a).length&&(r(e).unbind("mousemove."+V),E=T),this.blur(n)),l=r.proxy(function(){t?(rt.ie&&a[0].style.removeAttribute("filter"),a.css("overflow",""),"string"==typeof f.autofocus&&r(this.options.show.autofocus,a).focus(),this.options.show.target.trigger("qtip-"+this.id+"-inactive")):a.css({display:"",visibility:"",opacity:"",left:"",top:""}),this._trigger(t?"visible":"hidden")},this),f.effect===T||v===T?(a[c](),l()):r.isFunction(f.effect)?(a.stop(1,1),f.effect.call(a,this),a.queue("fx",function(t){l(),t()})):a.fadeTo(90,t?1:0,l),t&&f.target.trigger("qtip-"+this.id+"-inactive"),this))},M.show=function(t){return this.toggle(A,t)},M.hide=function(t){return this.toggle(T,t)},M.focus=function(t){if(!this.rendered||this.destroyed)return this;var e=r(G),n=this.tooltip,i=parseInt(n[0].style.zIndex,10),o=_.zindex+e.length;return n.hasClass(K)||this._trigger("focus",[o],t)&&(i!==o&&(e.each(function(){this.style.zIndex>i&&(this.style.zIndex=this.style.zIndex-1)}),e.filter("."+K).qtip("blur",t)),n.addClass(K)[0].style.zIndex=o),this},M.blur=function(t){return!this.rendered||this.destroyed?this:(this.tooltip.removeClass(K),this._trigger("blur",[this.tooltip.css("zIndex")],t),this)},M.disable=function(t){return this.destroyed?this:("toggle"===t?t=!(this.rendered?this.tooltip.hasClass(tt):this.disabled):"boolean"!=typeof t&&(t=A),this.rendered&&this.tooltip.toggleClass(tt,t).attr("aria-disabled",t),this.disabled=!!t,this)},M.enable=function(){return this.disable(T)},M._createButton=function(){var t=this,e=this.elements,n=e.tooltip,i=this.options.content.button,o="string"==typeof i,a=o?i:"Close tooltip";e.button&&e.button.remove(),i.jquery?e.button=i:e.button=r("<a />",{"class":"qtip-close "+(this.options.style.widget?"":V+"-icon"),title:a,"aria-label":a}).prepend(r("<span />",{"class":"ui-icon ui-icon-close",html:"&times;"})),e.button.appendTo(e.titlebar||n).attr("role","button").click(function(e){return n.hasClass(tt)||t.hide(e),T})},M._updateButton=function(t){if(!this.rendered)return T;var e=this.elements.button;t?this._createButton():e.remove()},M._setWidget=function(){var t=this.options.style.widget,e=this.elements,n=e.tooltip,r=n.hasClass(tt);n.removeClass(tt),tt=t?"ui-state-disabled":"qtip-disabled",n.toggleClass(tt,r),n.toggleClass("ui-helper-reset "+c(),t).toggleClass(Q,this.options.style.def&&!t),e.content&&e.content.toggleClass(c("content"),t),e.titlebar&&e.titlebar.toggleClass(c("header"),t),e.button&&e.button.toggleClass(V+"-icon",!t)},M._storeMouse=function(t){return(this.mouse=r.event.fix(t)).type="mousemove",this},M._bind=function(t,e,n,i,o){if(t&&n&&e.length){var a="."+this._id+(i?"-"+i:"");return r(t).bind((e.split?e:e.join(a+" "))+a,r.proxy(n,o||this)),this}},M._unbind=function(t,e){return t&&r(t).unbind("."+this._id+(e?"-"+e:"")),this},M._trigger=function(t,e,n){var i=r.Event("tooltip"+t);return i.originalEvent=n&&r.extend({},n)||this.cache.event||N,this.triggering=t,this.tooltip.trigger(i,[this].concat(e||[])),this.triggering=T,!i.isDefaultPrevented()},M._bindEvents=function(t,e,n,i,o,a){var s=n.filter(i).add(i.filter(n)),u=[];s.length&&(r.each(e,function(e,n){var i=r.inArray(n,t);i>-1&&u.push(t.splice(i,1)[0])}),u.length&&(this._bind(s,u,function(t){var e=!!this.rendered&&this.tooltip[0].offsetWidth>0;(e?a:o).call(this,t)}),n=n.not(s),i=i.not(s))),this._bind(n,t,o),this._bind(i,e,a)},M._assignInitialEvents=function(t){function e(t){return this.disabled||this.destroyed?T:(this.cache.event=t&&r.event.fix(t),this.cache.target=t&&r(t.target),clearTimeout(this.timers.show),void(this.timers.show=f.call(this,function(){this.render("object"==typeof t||n.show.ready)},n.prerender?0:n.show.delay)))}var n=this.options,i=n.show.target,o=n.hide.target,a=n.show.event?r.trim(""+n.show.event).split(" "):[],s=n.hide.event?r.trim(""+n.hide.event).split(" "):[];this._bind(this.elements.target,["remove","removeqtip"],function(t){this.destroy(!0)},"destroy"),/mouse(over|enter)/i.test(n.show.event)&&!/mouse(out|leave)/i.test(n.hide.event)&&s.push("mouseleave"),this._bind(i,"mousemove",function(t){this._storeMouse(t),this.cache.onTarget=A}),this._bindEvents(a,s,i,o,e,function(){return this.timers?void clearTimeout(this.timers.show):T}),(n.show.ready||n.prerender)&&e.call(this,t)},M._assignEvents=function(){var n=this,i=this.options,o=i.position,a=this.tooltip,s=i.show.target,u=i.hide.target,l=o.container,c=o.viewport,f=r(e),v=(r(e.body),r(t)),m=i.show.event?r.trim(""+i.show.event).split(" "):[],y=i.hide.event?r.trim(""+i.hide.event).split(" "):[];r.each(i.events,function(t,e){n._bind(a,"toggle"===t?["tooltipshow","tooltiphide"]:["tooltip"+t],e,null,a)}),/mouse(out|leave)/i.test(i.hide.event)&&"window"===i.hide.leave&&this._bind(f,["mouseout","blur"],function(t){/select|option/.test(t.target.nodeName)||t.relatedTarget||this.hide(t)}),i.hide.fixed?u=u.add(a.addClass(Z)):/mouse(over|enter)/i.test(i.show.event)&&this._bind(u,"mouseleave",function(){clearTimeout(this.timers.show)}),(""+i.hide.event).indexOf("unfocus")>-1&&this._bind(l.closest("html"),["mousedown","touchstart"],function(t){var e=r(t.target),n=this.rendered&&!this.tooltip.hasClass(tt)&&this.tooltip[0].offsetWidth>0,i=e.parents(G).filter(this.tooltip[0]).length>0;e[0]===this.target[0]||e[0]===this.tooltip[0]||i||this.target.has(e[0]).length||!n||this.hide(t)}),"number"==typeof i.hide.inactive&&(this._bind(s,"qtip-"+this.id+"-inactive",p,"inactive"),this._bind(u.add(a),_.inactiveEvents,p)),this._bindEvents(m,y,s,u,d,h),this._bind(s.add(a),"mousemove",function(t){if("number"==typeof i.hide.distance){var e=this.cache.origin||{},n=this.options.hide.distance,r=Math.abs;(r(t.pageX-e.pageX)>=n||r(t.pageY-e.pageY)>=n)&&this.hide(t)}this._storeMouse(t)}),"mouse"===o.target&&o.adjust.mouse&&(i.hide.event&&this._bind(s,["mouseenter","mouseleave"],function(t){return this.cache?void(this.cache.onTarget="mouseenter"===t.type):T}),this._bind(f,"mousemove",function(t){this.rendered&&this.cache.onTarget&&!this.tooltip.hasClass(tt)&&this.tooltip[0].offsetWidth>0&&this.reposition(t)})),(o.adjust.resize||c.length)&&this._bind(r.event.special.resize?c:v,"resize",g),o.adjust.scroll&&this._bind(v.add(o.container),"scroll",g)},M._unassignEvents=function(){var n=this.options,i=n.show.target,o=n.hide.target,a=r.grep([this.elements.target[0],this.rendered&&this.tooltip[0],n.position.container[0],n.position.viewport[0],n.position.container.closest("html")[0],t,e],function(t){return"object"==typeof t});i&&i.toArray&&(a=a.concat(i.toArray())),o&&o.toArray&&(a=a.concat(o.toArray())),this._unbind(a)._unbind(a,"destroy")._unbind(a,"inactive")},r(function(){v(G,["mouseenter","mouseleave"],function(t){var e="mouseenter"===t.type,n=r(t.currentTarget),i=r(t.relatedTarget||t.target),o=this.options;e?(this.focus(t),n.hasClass(Z)&&!n.hasClass(tt)&&clearTimeout(this.timers.hide)):"mouse"===o.position.target&&o.position.adjust.mouse&&o.hide.event&&o.show.target&&!i.closest(o.show.target[0]).length&&this.hide(t),n.toggleClass(J,e)}),v("["+U+"]",X,p)}),_=r.fn.qtip=function(t,e,i){var o=(""+t).toLowerCase(),a=N,u=r.makeArray(arguments).slice(1),l=u[u.length-1],c=this[0]?r.data(this[0],V):N;return!arguments.length&&c||"api"===o?c:"string"==typeof t?(this.each(function(){var t=r.data(this,V);if(!t)return A;if(l&&l.timeStamp&&(t.cache.event=l),!e||"option"!==o&&"options"!==o)t[o]&&t[o].apply(t,u);else{if(i===n&&!r.isPlainObject(e))return a=t.get(e),T;t.set(e,i)}}),a!==N?a:this):"object"!=typeof t&&arguments.length?void 0:(c=s(r.extend(A,{},t)),this.each(function(t){var e,n;return n=r.isArray(c.id)?c.id[t]:c.id,n=!n||n===T||n.length<1||_.api[n]?_.nextid++:n,e=m(r(this),n,c),e===T?A:(_.api[n]=e,r.each(B,function(){"initialize"===this.initialize&&this(e)}),void e._assignInitialEvents(l))}))},r.qtip=i,_.api={},r.each({attr:function(t,e){if(this.length){var n=this[0],i="title",o=r.data(n,"qtip");if(t===i&&o&&"object"==typeof o&&o.options.suppress)return arguments.length<2?r.attr(n,nt):(o&&o.options.content.attr===i&&o.cache.attr&&o.set("content.text",e),this.attr(nt,e))}return r.fn["attr"+et].apply(this,arguments)},clone:function(t){var e=(r([]),r.fn["clone"+et].apply(this,arguments));return t||e.filter("["+nt+"]").attr("title",function(){return r.attr(this,nt)}).removeAttr(nt),e}},function(t,e){if(!e||r.fn[t+et])return A;var n=r.fn[t+et]=r.fn[t];r.fn[t]=function(){return e.apply(this,arguments)||n.apply(this,arguments)}}),r.ui||(r["cleanData"+et]=r.cleanData,r.cleanData=function(t){for(var e,n=0;(e=r(t[n])).length;n++)if(e.attr(H))try{e.triggerHandler("removeqtip")}catch(i){}r["cleanData"+et].apply(this,arguments)}),_.version="2.2.1",_.nextid=0,_.inactiveEvents=X,_.zindex=15e3,_.defaults={prerender:T,id:T,overwrite:A,suppress:A,content:{text:A,attr:"title",title:T,button:T},position:{my:"top left",at:"bottom right",target:T,container:T,viewport:T,adjust:{x:0,y:0,mouse:A,scroll:A,resize:A,method:"flipinvert flipinvert"},effect:function(t,e,n){r(this).animate(e,{duration:200,queue:T})}},show:{target:T,event:"mouseenter",effect:A,delay:90,solo:T,ready:T,autofocus:T},hide:{target:T,event:"mouseleave",effect:A,delay:0,fixed:T,inactive:T,leave:"window",distance:T},style:{classes:"",widget:T,width:T,height:T,def:A},events:{render:N,move:N,show:N,hide:N,toggle:N,visible:N,hidden:N,focus:N,blur:N}};var st,ut="margin",lt="border",ct="color",ft="background-color",dt="transparent",ht=" !important",pt=!!e.createElement("canvas").getContext,gt=/rgba?\(0, 0, 0(, 0)?\)|transparent|#123456/i,vt={},mt=["Webkit","O","Moz","ms"];if(pt)var yt=t.devicePixelRatio||1,bt=function(){var t=e.createElement("canvas").getContext("2d");return t.backingStorePixelRatio||t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||1}(),xt=yt/bt;else var wt=function(t,e,n){return"<qtipvml:"+t+' xmlns="urn:schemas-microsoft.com:vml" class="qtip-vml" '+(e||"")+' style="behavior: url(#default#VML); '+(n||"")+'" />'};r.extend(w.prototype,{init:function(t){var e,n;n=this.element=t.elements.tip=r("<div />",{"class":V+"-tip"}).prependTo(t.tooltip),pt?(e=r("<canvas />").appendTo(this.element)[0].getContext("2d"),e.lineJoin="miter",e.miterLimit=1e5,e.save()):(e=wt("shape",'coordorigin="0,0"',"position:absolute;"),this.element.html(e+e),t._bind(r("*",n).add(n),["click","mousedown"],function(t){t.stopPropagation()},this._ns)),t._bind(t.tooltip,"tooltipmove",this.reposition,this._ns,this),this.create()},_swapDimensions:function(){this.size[0]=this.options.height,this.size[1]=this.options.width},_resetDimensions:function(){this.size[0]=this.options.width,this.size[1]=this.options.height},_useTitle:function(t){var e=this.qtip.elements.titlebar;return e&&(t.y===L||t.y===z&&this.element.position().top+this.size[1]/2+this.options.offset<e.outerHeight(A))},_parseCorner:function(t){var e=this.qtip.options.position.my;return t===T||e===T?t=T:t===A?t=new C(e.string()):t.string||(t=new C(t),t.fixed=A),t},_parseWidth:function(t,e,n){var r=this.qtip.elements,i=lt+y(e)+"Width";return(n?x(n,i):x(r.content,i)||x(this._useTitle(t)&&r.titlebar||r.content,i)||x(r.tooltip,i))||0},_parseRadius:function(t){var e=this.qtip.elements,n=lt+y(t.y)+y(t.x)+"Radius";return rt.ie<9?0:x(this._useTitle(t)&&e.titlebar||e.content,n)||x(e.tooltip,n)||0},_invalidColour:function(t,e,n){var r=t.css(e);return!r||n&&r===t.css(n)||gt.test(r)?T:r},_parseColours:function(t){var e=this.qtip.elements,n=this.element.css("cssText",""),i=lt+y(t[t.precedance])+y(ct),o=this._useTitle(t)&&e.titlebar||e.content,a=this._invalidColour,s=[];return s[0]=a(n,ft)||a(o,ft)||a(e.content,ft)||a(e.tooltip,ft)||n.css(ft),s[1]=a(n,i,ct)||a(o,i,ct)||a(e.content,i,ct)||a(e.tooltip,i,ct)||e.tooltip.css(i),r("*",n).add(n).css("cssText",ft+":"+dt+ht+";"+lt+":0"+ht+";"),s},_calculateSize:function(t){var e,n,r,i=t.precedance===O,o=this.options.width,a=this.options.height,s="c"===t.abbrev(),u=(i?o:a)*(s?.5:1),l=Math.pow,c=Math.round,f=Math.sqrt(l(u,2)+l(a,2)),d=[this.border/u*f,this.border/a*f];return d[2]=Math.sqrt(l(d[0],2)-l(this.border,2)),d[3]=Math.sqrt(l(d[1],2)-l(this.border,2)),e=f+d[2]+d[3]+(s?0:d[0]),n=e/f,r=[c(n*o),c(n*a)],i?r:r.reverse()},_calculateTip:function(t,e,n){n=n||1,e=e||this.size;var r=e[0]*n,i=e[1]*n,o=Math.ceil(r/2),a=Math.ceil(i/2),s={br:[0,0,r,i,r,0],bl:[0,0,r,0,0,i],tr:[0,i,r,0,r,i],tl:[0,0,0,i,r,i],tc:[0,i,o,0,r,i],bc:[0,0,r,0,o,i],rc:[0,0,r,a,0,i],lc:[r,0,r,i,0,a]};return s.lt=s.br,s.rt=s.bl,s.lb=s.tr,s.rb=s.tl,s[t.abbrev()]},_drawCoords:function(t,e){t.beginPath(),t.moveTo(e[0],e[1]),t.lineTo(e[2],e[3]),t.lineTo(e[4],e[5]),t.closePath()},create:function(){var t=this.corner=(pt||rt.ie)&&this._parseCorner(this.options.corner);return(this.enabled=!!this.corner&&"c"!==this.corner.abbrev())&&(this.qtip.cache.corner=t.clone(),this.update()),this.element.toggle(this.enabled),this.corner},update:function(e,n){if(!this.enabled)return this;var i,o,a,s,u,l,c,f,d=this.qtip.elements,h=this.element,p=h.children(),g=this.options,v=this.size,m=g.mimic,y=Math.round;e||(e=this.qtip.cache.corner||this.corner),m===T?m=e:(m=new C(m),m.precedance=e.precedance,"inherit"===m.x?m.x=e.x:"inherit"===m.y?m.y=e.y:m.x===m.y&&(m[e.precedance]=e[e.precedance])),o=m.precedance,e.precedance===D?this._swapDimensions():this._resetDimensions(),i=this.color=this._parseColours(e),i[1]!==dt?(f=this.border=this._parseWidth(e,e[e.precedance]),g.border&&f<1&&!gt.test(i[1])&&(i[0]=i[1]),this.border=f=g.border!==A?g.border:f):this.border=f=0,c=this.size=this._calculateSize(e),h.css({width:c[0],height:c[1],lineHeight:c[1]+"px"}),l=e.precedance===O?[y(m.x===P?f:m.x===q?c[0]-v[0]-f:(c[0]-v[0])/2),y(m.y===L?c[1]-v[1]:0)]:[y(m.x===P?c[0]-v[0]:0),y(m.y===L?f:m.y===F?c[1]-v[1]-f:(c[1]-v[1])/2)],pt?(a=p[0].getContext("2d"),a.restore(),a.save(),a.clearRect(0,0,6e3,6e3),s=this._calculateTip(m,v,xt),u=this._calculateTip(m,this.size,xt),p.attr(j,c[0]*xt).attr(I,c[1]*xt),p.css(j,c[0]).css(I,c[1]),this._drawCoords(a,u),a.fillStyle=i[1],a.fill(),a.translate(l[0]*xt,l[1]*xt),this._drawCoords(a,s),a.fillStyle=i[0],a.fill()):(s=this._calculateTip(m),s="m"+s[0]+","+s[1]+" l"+s[2]+","+s[3]+" "+s[4]+","+s[5]+" xe",l[2]=f&&/^(r|b)/i.test(e.string())?8===rt.ie?2:1:0,p.css({coordsize:c[0]+f+" "+(c[1]+f),antialias:""+(m.string().indexOf(z)>-1),left:l[0]-l[2]*Number(o===D),top:l[1]-l[2]*Number(o===O),width:c[0]+f,height:c[1]+f}).each(function(t){var e=r(this);e[e.prop?"prop":"attr"]({coordsize:c[0]+f+" "+(c[1]+f),path:s,fillcolor:i[0],filled:!!t,stroked:!t}).toggle(!(!f&&!t)),!t&&e.html(wt("stroke",'weight="'+2*f+'px" color="'+i[1]+'" miterlimit="1000" joinstyle="miter"'))})),t.opera&&setTimeout(function(){d.tip.css({display:"inline-block",visibility:"visible"})},1),n!==T&&this.calculate(e,c)},calculate:function(t,e){if(!this.enabled)return T;var n,i,o=this,a=this.qtip.elements,s=this.element,u=this.options.offset,l=(a.tooltip.hasClass("ui-widget"),{});return t=t||this.corner,n=t.precedance,e=e||this._calculateSize(t),i=[t.x,t.y],n===D&&i.reverse(),r.each(i,function(r,i){var s,c,f;i===z?(s=n===O?P:L,l[s]="50%",l[ut+"-"+s]=-Math.round(e[n===O?0:1]/2)+u):(s=o._parseWidth(t,i,a.tooltip),c=o._parseWidth(t,i,a.content),f=o._parseRadius(t),l[i]=Math.max(-o.border,r?c:u+(f>s?f:-s)))}),l[t[n]]-=e[n===D?0:1],s.css({margin:"",top:"",bottom:"",left:"",right:""}).css(l),l},reposition:function(t,e,r,i){function o(t,e,n,r,i){t===R&&c.precedance===e&&f[r]&&c[n]!==z?c.precedance=c.precedance===D?O:D:t!==R&&f[r]&&(c[e]=c[e]===z?f[r]>0?r:i:c[e]===r?i:r)}function a(t,e,i){c[t]===z?v[ut+"-"+e]=g[t]=s[ut+"-"+e]-f[e]:(u=s[i]!==n?[f[e],-s[e]]:[-f[e],s[e]],(g[t]=Math.max(u[0],u[1]))>u[0]&&(r[e]-=f[e],g[e]=T),v[s[i]!==n?i:e]=g[t])}if(this.enabled){var s,u,l=e.cache,c=this.corner.clone(),f=r.adjusted,d=e.options.position.adjust.method.split(" "),h=d[0],p=d[1]||d[0],g={left:T,top:T,x:0,y:0},v={};this.corner.fixed!==A&&(o(h,D,O,P,q),o(p,O,D,L,F),c.string()===l.corner.string()&&l.cornerTop===f.top&&l.cornerLeft===f.left||this.update(c,T)),s=this.calculate(c),s.right!==n&&(s.left=-s.right),s.bottom!==n&&(s.top=-s.bottom),s.user=this.offset,(g.left=h===R&&!!f.left)&&a(D,P,q),(g.top=p===R&&!!f.top)&&a(O,L,F),this.element.css(v).toggle(!(g.x&&g.y||c.x===z&&g.y||c.y===z&&g.x)),r.left-=s.left.charAt?s.user:h!==R||g.top||!g.left&&!g.top?s.left+this.border:0,r.top-=s.top.charAt?s.user:p!==R||g.left||!g.left&&!g.top?s.top+this.border:0,l.cornerLeft=f.left,l.cornerTop=f.top,l.corner=c.clone()}},destroy:function(){this.qtip._unbind(this.qtip.tooltip,this._ns),this.qtip.elements.tip&&this.qtip.elements.tip.find("*").remove().end().remove()}}),st=B.tip=function(t){return new w(t,t.options.style.tip)},st.initialize="render",st.sanitize=function(t){if(t.style&&"tip"in t.style){var e=t.style.tip;"object"!=typeof e&&(e=t.style.tip={corner:e}),/string|boolean/i.test(typeof e.corner)||(e.corner=A)}},S.tip={"^position.my|style.tip.(corner|mimic|border)$":function(){this.create(),this.qtip.reposition()},"^style.tip.(height|width)$":function(t){this.size=[t.width,t.height],this.update(),this.qtip.reposition()},"^content.title|style.(classes|widget)$":function(){this.update()}},r.extend(A,_.defaults,{style:{tip:{corner:A,mimic:T,width:6,height:6,border:A,offset:0}}});var $t,kt,_t="qtip-modal",Mt="."+_t;kt=function(){function t(t){if(r.expr[":"].focusable)return r.expr[":"].focusable;var e,n,i,o=!isNaN(r.attr(t,"tabindex")),a=t.nodeName&&t.nodeName.toLowerCase();return"area"===a?(e=t.parentNode,n=e.name,!(!t.href||!n||"map"!==e.nodeName.toLowerCase())&&(i=r("img[usemap=#"+n+"]")[0],!!i&&i.is(":visible"))):/input|select|textarea|button|object/.test(a)?!t.disabled:"a"===a?t.href||o:o}function n(t){c.length<1&&t.length?t.not("body").blur():c.first().focus()}function i(t){if(u.is(":visible")){var e,i=r(t.target),s=o.tooltip,l=i.closest(G);e=l.length<1?T:parseInt(l[0].style.zIndex,10)>parseInt(s[0].style.zIndex,10),e||i.closest(G)[0]===s[0]||n(i),a=t.target===c[c.length-1]}}var o,a,s,u,l=this,c={};r.extend(l,{init:function(){return u=l.elem=r("<div />",{id:"qtip-overlay",html:"<div></div>",mousedown:function(){return T}}).hide(),r(e.body).bind("focusin"+Mt,i),r(e).bind("keydown"+Mt,function(t){o&&o.options.show.modal.escape&&27===t.keyCode&&o.hide(t)}),u.bind("click"+Mt,function(t){o&&o.options.show.modal.blur&&o.hide(t)}),l},update:function(e){o=e,c=e.options.show.modal.stealfocus!==T?e.tooltip.find("*").filter(function(){return t(this)}):[]},toggle:function(t,i,a){var c=(r(e.body),t.tooltip),f=t.options.show.modal,d=f.effect,h=i?"show":"hide",p=u.is(":visible"),g=r(Mt).filter(":visible:not(:animated)").not(c);return l.update(t),i&&f.stealfocus!==T&&n(r(":focus")),u.toggleClass("blurs",f.blur),i&&u.appendTo(e.body),u.is(":animated")&&p===i&&s!==T||!i&&g.length?l:(u.stop(A,T),r.isFunction(d)?d.call(u,i):d===T?u[h]():u.fadeTo(parseInt(a,10)||90,i?1:0,function(){i||u.hide()}),i||u.queue(function(t){u.css({left:"",top:""}),r(Mt).length||u.detach(),t()}),s=i,o.destroyed&&(o=N),l)}}),l.init()},kt=new kt,r.extend($.prototype,{init:function(t){var e=t.tooltip;return this.options.on?(t.elements.overlay=kt.elem,e.addClass(_t).css("z-index",_.modal_zindex+r(Mt).length),t._bind(e,["tooltipshow","tooltiphide"],function(t,n,i){var o=t.originalEvent;if(t.target===e[0])if(o&&"tooltiphide"===t.type&&/mouse(leave|enter)/.test(o.type)&&r(o.relatedTarget).closest(kt.elem[0]).length)try{t.preventDefault()}catch(a){}else(!o||o&&"tooltipsolo"!==o.type)&&this.toggle(t,"tooltipshow"===t.type,i)},this._ns,this),t._bind(e,"tooltipfocus",function(t,n){if(!t.isDefaultPrevented()&&t.target===e[0]){var i=r(Mt),o=_.modal_zindex+i.length,a=parseInt(e[0].style.zIndex,10);kt.elem[0].style.zIndex=o-1,i.each(function(){this.style.zIndex>a&&(this.style.zIndex-=1)}),i.filter("."+K).qtip("blur",t.originalEvent),e.addClass(K)[0].style.zIndex=o,kt.update(n);try{t.preventDefault()}catch(s){}}},this._ns,this),void t._bind(e,"tooltiphide",function(t){t.target===e[0]&&r(Mt).filter(":visible").not(e).last().qtip("focus",t)},this._ns,this)):this},toggle:function(t,e,n){return t&&t.isDefaultPrevented()?this:void kt.toggle(this.qtip,!!e,n)},destroy:function(){this.qtip.tooltip.removeClass(_t),this.qtip._unbind(this.qtip.tooltip,this._ns),kt.toggle(this.qtip,T),delete this.qtip.elements.overlay}}),$t=B.modal=function(t){return new $(t,t.options.show.modal)},$t.sanitize=function(t){t.show&&("object"!=typeof t.show.modal?t.show.modal={on:!!t.show.modal}:"undefined"==typeof t.show.modal.on&&(t.show.modal.on=A))},_.modal_zindex=_.zindex-200,$t.initialize="render",S.modal={"^show.modal.(on|blur)$":function(){this.destroy(),this.init(),this.qtip.elems.overlay.toggle(this.qtip.tooltip[0].offsetWidth>0)}},r.extend(A,_.defaults,{show:{modal:{on:T,effect:A,blur:A,stealfocus:A,escape:A}}}),B.viewport=function(n,r,i,o,a,s,u){function l(t,e,n,i,o,a,s,u,l){var c=r[o],y=x[t],b=w[t],$=n===R,k=y===o?l:y===a?-l:-l/2,_=b===o?u:b===a?-u:-u/2,M=v[o]+m[o]-(h?0:d[o]),C=M-c,S=c+l-(s===j?p:g)-M,E=k-(x.precedance===t||y===x[e]?_:0)-(b===z?u/2:0);return $?(E=(y===o?1:-1)*k,r[o]+=C>0?C:S>0?-S:0,r[o]=Math.max(-d[o]+m[o],c-E,Math.min(Math.max(-d[o]+m[o]+(s===j?p:g),c+E),r[o],"center"===y?c-k:1e9))):(i*=n===W?2:0,C>0&&(y!==o||S>0)?(r[o]-=E+i,f.invert(t,o)):S>0&&(y!==a||C>0)&&(r[o]-=(y===z?-E:E)+i,f.invert(t,a)),r[o]<v&&-r[o]>S&&(r[o]=c,f=x.clone())),r[o]-c}var c,f,d,h,p,g,v,m,y=i.target,b=n.elements.tooltip,x=i.my,w=i.at,$=i.adjust,k=$.method.split(" "),_=k[0],M=k[1]||k[0],C=i.viewport,S=i.container,E=(n.cache,{left:0,top:0});return C.jquery&&y[0]!==t&&y[0]!==e.body&&"none"!==$.method?(d=S.offset()||E,h="static"===S.css("position"),c="fixed"===b.css("position"),p=C[0]===t?C.width():C.outerWidth(T),g=C[0]===t?C.height():C.outerHeight(T),v={left:c?0:C.scrollLeft(),top:c?0:C.scrollTop()},m=C.offset()||E,"shift"===_&&"shift"===M||(f=x.clone()),E={left:"none"!==_?l(D,O,_,$.x,P,q,j,o,s):0,top:"none"!==M?l(O,D,M,$.y,L,F,I,a,u):0,my:f}):E},B.polys={polygon:function(t,e){var n,r,i,o={width:0,height:0,position:{top:1e10,right:0,bottom:0,left:1e10},adjustable:T},a=0,s=[],u=1,l=1,c=0,f=0;for(a=t.length;a--;)n=[parseInt(t[--a],10),parseInt(t[a+1],10)],
+n[0]>o.position.right&&(o.position.right=n[0]),n[0]<o.position.left&&(o.position.left=n[0]),n[1]>o.position.bottom&&(o.position.bottom=n[1]),n[1]<o.position.top&&(o.position.top=n[1]),s.push(n);if(r=o.width=Math.abs(o.position.right-o.position.left),i=o.height=Math.abs(o.position.bottom-o.position.top),"c"===e.abbrev())o.position={left:o.position.left+o.width/2,top:o.position.top+o.height/2};else{for(;r>0&&i>0&&u>0&&l>0;)for(r=Math.floor(r/2),i=Math.floor(i/2),e.x===P?u=r:e.x===q?u=o.width-r:u+=Math.floor(r/2),e.y===L?l=i:e.y===F?l=o.height-i:l+=Math.floor(i/2),a=s.length;a--&&!(s.length<2);)c=s[a][0]-o.position.left,f=s[a][1]-o.position.top,(e.x===P&&c>=u||e.x===q&&c<=u||e.x===z&&(c<u||c>o.width-u)||e.y===L&&f>=l||e.y===F&&f<=l||e.y===z&&(f<l||f>o.height-l))&&s.splice(a,1);o.position={left:s[0][0],top:s[0][1]}}return o},rect:function(t,e,n,r){return{width:Math.abs(n-t),height:Math.abs(r-e),position:{left:Math.min(t,n),top:Math.min(e,r)}}},_angles:{tc:1.5,tr:7/4,tl:5/4,bc:.5,br:.25,bl:.75,rc:2,lc:1,c:0},ellipse:function(t,e,n,r,i){var o=B.polys._angles[i.abbrev()],a=0===o?0:n*Math.cos(o*Math.PI),s=r*Math.sin(o*Math.PI);return{width:2*n-Math.abs(a),height:2*r-Math.abs(s),position:{left:t+a,top:e+s},adjustable:T}},circle:function(t,e,n,r){return B.polys.ellipse(t,e,n,n,r)}},B.svg=function(t,n,i){for(var o,a,s,u,l,c,f,d,h,p=(r(e),n[0]),g=r(p.ownerSVGElement),v=p.ownerDocument,m=(parseInt(n.css("stroke-width"),10)||0)/2;!p.getBBox;)p=p.parentNode;if(!p.getBBox||!p.parentNode)return T;switch(p.nodeName){case"ellipse":case"circle":d=B.polys.ellipse(p.cx.baseVal.value,p.cy.baseVal.value,(p.rx||p.r).baseVal.value+m,(p.ry||p.r).baseVal.value+m,i);break;case"line":case"polygon":case"polyline":for(f=p.points||[{x:p.x1.baseVal.value,y:p.y1.baseVal.value},{x:p.x2.baseVal.value,y:p.y2.baseVal.value}],d=[],c=-1,u=f.numberOfItems||f.length;++c<u;)l=f.getItem?f.getItem(c):f[c],d.push.apply(d,[l.x,l.y]);d=B.polys.polygon(d,i);break;default:d=p.getBBox(),d={width:d.width,height:d.height,position:{left:d.x,top:d.y}}}return h=d.position,g=g[0],g.createSVGPoint&&(a=p.getScreenCTM(),f=g.createSVGPoint(),f.x=h.left,f.y=h.top,s=f.matrixTransform(a),h.left=s.x,h.top=s.y),v!==e&&"mouse"!==t.position.target&&(o=r((v.defaultView||v.parentWindow).frameElement).offset(),o&&(h.left+=o.left,h.top+=o.top)),v=r(v),h.left+=v.scrollLeft(),h.top+=v.scrollTop(),d},B.imagemap=function(t,e,n,i){e.jquery||(e=r(e));var o,a,s,u,l,c=(e.attr("shape")||"rect").toLowerCase().replace("poly","polygon"),f=r('img[usemap="#'+e.parent("map").attr("name")+'"]'),d=r.trim(e.attr("coords")),h=d.replace(/,$/,"").split(",");if(!f.length)return T;if("polygon"===c)u=B.polys.polygon(h,n);else{if(!B.polys[c])return T;for(s=-1,l=h.length,a=[];++s<l;)a.push(parseInt(h[s],10));u=B.polys[c].apply(this,a.concat(n))}return o=f.offset(),o.left+=Math.ceil((f.outerWidth(T)-f.width())/2),o.top+=Math.ceil((f.outerHeight(T)-f.height())/2),u.position.left+=o.left,u.position.top+=o.top,u};var Ct,St='<iframe class="qtip-bgiframe" frameborder="0" tabindex="-1" src="javascript:\'\';"  style="display:block; position:absolute; z-index:-1; filter:alpha(opacity=0); -ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";"></iframe>';r.extend(k.prototype,{_scroll:function(){var e=this.qtip.elements.overlay;e&&(e[0].style.top=r(t).scrollTop()+"px")},init:function(n){var i=n.tooltip;r("select, object").length<1&&(this.bgiframe=n.elements.bgiframe=r(St).appendTo(i),n._bind(i,"tooltipmove",this.adjustBGIFrame,this._ns,this)),this.redrawContainer=r("<div/>",{id:V+"-rcontainer"}).appendTo(e.body),n.elements.overlay&&n.elements.overlay.addClass("qtipmodal-ie6fix")&&(n._bind(t,["scroll","resize"],this._scroll,this._ns,this),n._bind(i,["tooltipshow"],this._scroll,this._ns,this)),this.redraw()},adjustBGIFrame:function(){var t,e,n=this.qtip.tooltip,r={height:n.outerHeight(T),width:n.outerWidth(T)},i=this.qtip.plugins.tip,o=this.qtip.elements.tip;e=parseInt(n.css("borderLeftWidth"),10)||0,e={left:-e,top:-e},i&&o&&(t="x"===i.corner.precedance?[j,P]:[I,L],e[t[1]]-=o[t[0]]()),this.bgiframe.css(e).css(r)},redraw:function(){if(this.qtip.rendered<1||this.drawing)return this;var t,e,n,r,i=this.qtip.tooltip,o=this.qtip.options.style,a=this.qtip.options.position.container;return this.qtip.drawing=1,o.height&&i.css(I,o.height),o.width?i.css(j,o.width):(i.css(j,"").appendTo(this.redrawContainer),e=i.width(),e%2<1&&(e+=1),n=i.css("maxWidth")||"",r=i.css("minWidth")||"",t=(n+r).indexOf("%")>-1?a.width()/100:0,n=(n.indexOf("%")>-1?t:1)*parseInt(n,10)||e,r=(r.indexOf("%")>-1?t:1)*parseInt(r,10)||0,e=n+r?Math.min(Math.max(e,r),n):e,i.css(j,Math.round(e)).appendTo(a)),this.drawing=0,this},destroy:function(){this.bgiframe&&this.bgiframe.remove(),this.qtip._unbind([t,this.qtip.tooltip],this._ns)}}),Ct=B.ie6=function(t){return 6===rt.ie?new k(t):T},Ct.initialize="render",S.ie6={"^content|style$":function(){this.redraw()}}})}(window,document),function(t,e,n){!function(t){"use strict";"function"==typeof define&&define.amd?define(["jquery"],t):jQuery&&!jQuery.fn.qtip&&t(jQuery)}(function(r){"use strict";function i(t,e,n,i){this.id=n,this.target=t,this.tooltip=M,this.elements={target:t},this._id=j+"-"+n,this.timers={img:{}},this.options=e,this.plugins={},this.cache={event:{},target:r(),disabled:_,attr:i,onTooltip:_,lastClass:""},this.rendered=this.destroyed=this.disabled=this.waiting=this.hiddenDuringWait=this.positioning=this.triggering=_}function o(t){return t===M||"object"!==r.type(t)}function a(t){return!(r.isFunction(t)||t&&t.attr||t.length||"object"===r.type(t)&&(t.jquery||t.then))}function s(t){var e,n,i,s;return o(t)?_:(o(t.metadata)&&(t.metadata={type:t.metadata}),"content"in t&&(e=t.content,o(e)||e.jquery||e.done?e=t.content={text:n=a(e)?_:e}:n=e.text,"ajax"in e&&(i=e.ajax,s=i&&i.once!==_,delete e.ajax,e.text=function(t,e){var o=n||r(this).attr(e.options.content.attr)||"Loading...",a=r.ajax(r.extend({},i,{context:e})).then(i.success,M,i.error).then(function(t){return t&&s&&e.set("content.text",t),t},function(t,n,r){e.destroyed||0===t.status||e.set("content.text",n+": "+r)});return s?o:(e.set("content.text",o),a)}),"title"in e&&(r.isPlainObject(e.title)&&(e.button=e.title.button,e.title=e.title.text),a(e.title||_)&&(e.title=_))),"position"in t&&o(t.position)&&(t.position={my:t.position,at:t.position}),"show"in t&&o(t.show)&&(t.show=t.show.jquery?{target:t.show}:t.show===k?{ready:k}:{event:t.show}),"hide"in t&&o(t.hide)&&(t.hide=t.hide.jquery?{target:t.hide}:{event:t.hide}),"style"in t&&o(t.style)&&(t.style={classes:t.style}),r.each(O,function(){this.sanitize&&this.sanitize(t)}),t)}function u(t,e){for(var n,r=0,i=t,o=e.split(".");i=i[o[r++]];)r<o.length&&(n=i);return[n||t,o.pop()]}function l(t,e){var n,r,i;for(n in this.checks)for(r in this.checks[n])(i=new RegExp(r,"i").exec(t))&&(e.push(i),("builtin"===n||this.plugins[n])&&this.checks[n][r].apply(this.plugins[n]||this,e))}function c(t){return P.concat("").join(t?"-"+t+" ":" ")}function f(t,e){return e>0?setTimeout(r.proxy(t,this),e):void t.call(this)}function d(t){this.tooltip.hasClass(V)||(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this.timers.show=f.call(this,function(){this.toggle(k,t)},this.options.show.delay))}function h(t){if(!this.tooltip.hasClass(V)&&!this.destroyed){var e=r(t.relatedTarget),n=e.closest(F)[0]===this.tooltip[0],i=e[0]===this.options.show.target[0];if(clearTimeout(this.timers.show),clearTimeout(this.timers.hide),this!==e[0]&&"mouse"===this.options.position.target&&n||this.options.hide.fixed&&/mouse(out|leave|move)/.test(t.type)&&(n||i))try{t.preventDefault(),t.stopImmediatePropagation()}catch(o){}else this.timers.hide=f.call(this,function(){this.toggle(_,t)},this.options.hide.delay,this)}}function p(t){!this.tooltip.hasClass(V)&&this.options.hide.inactive&&(clearTimeout(this.timers.inactive),this.timers.inactive=f.call(this,function(){this.hide(t)},this.options.hide.inactive))}function g(t){this.rendered&&this.tooltip[0].offsetWidth>0&&this.reposition(t)}function v(t,n,i){r(e.body).delegate(t,(n.split?n:n.join("."+j+" "))+"."+j,function(){var t=y.api[r.attr(this,L)];t&&!t.disabled&&i.apply(t,arguments)})}function m(t,n,o){var a,u,l,c,f,d=r(e.body),h=t[0]===e?d:t,p=t.metadata?t.metadata(o.metadata):M,g="html5"===o.metadata.type&&p?p[o.metadata.name]:M,v=t.data(o.metadata.name||"qtipopts");try{v="string"==typeof v?r.parseJSON(v):v}catch(m){}if(c=r.extend(k,{},y.defaults,o,"object"==typeof v?s(v):M,s(g||p)),u=c.position,c.id=n,"boolean"==typeof c.content.text){if(l=t.attr(c.content.attr),c.content.attr===_||!l)return _;c.content.text=l}if(u.container.length||(u.container=d),u.target===_&&(u.target=h),c.show.target===_&&(c.show.target=h),c.show.solo===k&&(c.show.solo=u.container.closest("body")),c.hide.target===_&&(c.hide.target=h),c.position.viewport===k&&(c.position.viewport=u.container),u.container=u.container.eq(0),u.at=new x(u.at,k),u.my=new x(u.my),t.data(j))if(c.overwrite)t.qtip("destroy",!0);else if(c.overwrite===_)return _;return t.attr(I,n),c.suppress&&(f=t.attr("title"))&&t.removeAttr("title").attr(U,f).attr("title",""),a=new i(t,c,n,(!!l)),t.data(j,a),a}var y,b,x,w,$,k=!0,_=!1,M=null,C="x",S="y",E="top",A="left",T="bottom",N="right",D="center",O={},j="qtip",I="data-hasqtip",L="data-qtip-id",P=["ui-widget","ui-tooltip"],F="."+j,q="click dblclick mousedown mouseup mousemove mouseleave mouseenter".split(" "),z=j+"-fixed",W=j+"-default",R=j+"-focus",B=j+"-hover",V=j+"-disabled",H="_replacedByqTip",U="oldtitle",Y={ie:function(){for(var t=4,n=e.createElement("div");(n.innerHTML="<!--[if gt IE "+t+"]><i></i><![endif]-->")&&n.getElementsByTagName("i")[0];t+=1);return t>4?t:NaN}(),iOS:parseFloat((""+(/CPU.*OS ([0-9_]{1,5})|(CPU like).*AppleWebKit.*Mobile/i.exec(navigator.userAgent)||[0,""])[1]).replace("undefined","3_2").replace("_",".").replace("_",""))||_};b=i.prototype,b._when=function(t){return r.when.apply(r,t)},b.render=function(t){if(this.rendered||this.destroyed)return this;var e,n=this,i=this.options,o=this.cache,a=this.elements,s=i.content.text,u=i.content.title,l=i.content.button,c=i.position,f=("."+this._id+" ",[]);return r.attr(this.target[0],"aria-describedby",this._id),o.posClass=this._createPosClass((this.position={my:c.my,at:c.at}).my),this.tooltip=a.tooltip=e=r("<div/>",{id:this._id,"class":[j,W,i.style.classes,o.posClass].join(" "),width:i.style.width||"",height:i.style.height||"",tracking:"mouse"===c.target&&c.adjust.mouse,role:"alert","aria-live":"polite","aria-atomic":_,"aria-describedby":this._id+"-content","aria-hidden":k}).toggleClass(V,this.disabled).attr(L,this.id).data(j,this).appendTo(c.container).append(a.content=r("<div />",{"class":j+"-content",id:this._id+"-content","aria-atomic":k})),this.rendered=-1,this.positioning=k,u&&(this._createTitle(),r.isFunction(u)||f.push(this._updateTitle(u,_))),l&&this._createButton(),r.isFunction(s)||f.push(this._updateContent(s,_)),this.rendered=k,this._setWidget(),r.each(O,function(t){var e;"render"===this.initialize&&(e=this(n))&&(n.plugins[t]=e)}),this._unassignEvents(),this._assignEvents(),this._when(f).then(function(){n._trigger("render"),n.positioning=_,n.hiddenDuringWait||!i.show.ready&&!t||n.toggle(k,o.event,_),n.hiddenDuringWait=_}),y.api[this.id]=this,this},b.destroy=function(t){function e(){if(!this.destroyed){this.destroyed=k;var t,e=this.target,n=e.attr(U);this.rendered&&this.tooltip.stop(1,0).find("*").remove().end().remove(),r.each(this.plugins,function(t){this.destroy&&this.destroy()});for(t in this.timers)clearTimeout(this.timers[t]);e.removeData(j).removeAttr(L).removeAttr(I).removeAttr("aria-describedby"),this.options.suppress&&n&&e.attr("title",n).removeAttr(U),this._unassignEvents(),this.options=this.elements=this.cache=this.timers=this.plugins=this.mouse=M,delete y.api[this.id]}}return this.destroyed?this.target:(t===k&&"hide"!==this.triggering||!this.rendered?e.call(this):(this.tooltip.one("tooltiphidden",r.proxy(e,this)),!this.triggering&&this.hide()),this.target)},w=b.checks={builtin:{"^id$":function(t,e,n,i){var o=n===k?y.nextid:n,a=j+"-"+o;o!==_&&o.length>0&&!r("#"+a).length?(this._id=a,this.rendered&&(this.tooltip[0].id=this._id,this.elements.content[0].id=this._id+"-content",this.elements.title[0].id=this._id+"-title")):t[e]=i},"^prerender":function(t,e,n){n&&!this.rendered&&this.render(this.options.show.ready)},"^content.text$":function(t,e,n){this._updateContent(n)},"^content.attr$":function(t,e,n,r){this.options.content.text===this.target.attr(r)&&this._updateContent(this.target.attr(n))},"^content.title$":function(t,e,n){return n?(n&&!this.elements.title&&this._createTitle(),void this._updateTitle(n)):this._removeTitle()},"^content.button$":function(t,e,n){this._updateButton(n)},"^content.title.(text|button)$":function(t,e,n){this.set("content."+e,n)},"^position.(my|at)$":function(t,e,n){"string"==typeof n&&(this.position[e]=t[e]=new x(n,"at"===e))},"^position.container$":function(t,e,n){this.rendered&&this.tooltip.appendTo(n)},"^show.ready$":function(t,e,n){n&&(!this.rendered&&this.render(k)||this.toggle(k))},"^style.classes$":function(t,e,n,r){this.rendered&&this.tooltip.removeClass(r).addClass(n)},"^style.(width|height)":function(t,e,n){this.rendered&&this.tooltip.css(e,n)},"^style.widget|content.title":function(){this.rendered&&this._setWidget()},"^style.def":function(t,e,n){this.rendered&&this.tooltip.toggleClass(W,!!n)},"^events.(render|show|move|hide|focus|blur)$":function(t,e,n){this.rendered&&this.tooltip[(r.isFunction(n)?"":"un")+"bind"]("tooltip"+e,n)},"^(show|hide|position).(event|target|fixed|inactive|leave|distance|viewport|adjust)":function(){if(this.rendered){var t=this.options.position;this.tooltip.attr("tracking","mouse"===t.target&&t.adjust.mouse),this._unassignEvents(),this._assignEvents()}}}},b.get=function(t){if(this.destroyed)return this;var e=u(this.options,t.toLowerCase()),n=e[0][e[1]];return n.precedance?n.string():n};var G=/^position\.(my|at|adjust|target|container|viewport)|style|content|show\.ready/i,X=/^prerender|show\.ready/i;b.set=function(t,e){if(this.destroyed)return this;var n,i=this.rendered,o=_,a=this.options;this.checks;return"string"==typeof t?(n=t,t={},t[n]=e):t=r.extend({},t),r.each(t,function(e,n){if(i&&X.test(e))return void delete t[e];var s,l=u(a,e.toLowerCase());s=l[0][l[1]],l[0][l[1]]=n&&n.nodeType?r(n):n,o=G.test(e)||o,t[e]=[l[0],l[1],n,s]}),s(a),this.positioning=k,r.each(t,r.proxy(l,this)),this.positioning=_,this.rendered&&this.tooltip[0].offsetWidth>0&&o&&this.reposition("mouse"===a.position.target?M:this.cache.event),this},b._update=function(t,e,n){var i=this,o=this.cache;return this.rendered&&t?(r.isFunction(t)&&(t=t.call(this.elements.target,o.event,this)||""),r.isFunction(t.then)?(o.waiting=k,t.then(function(t){return o.waiting=_,i._update(t,e)},M,function(t){return i._update(t,e)})):t===_||!t&&""!==t?_:(t.jquery&&t.length>0?e.empty().append(t.css({display:"block",visibility:"visible"})):e.html(t),this._waitForContent(e).then(function(t){i.rendered&&i.tooltip[0].offsetWidth>0&&i.reposition(o.event,!t.length)}))):_},b._waitForContent=function(t){var e=this.cache;return e.waiting=k,(r.fn.imagesLoaded?t.imagesLoaded():r.Deferred().resolve([])).done(function(){e.waiting=_}).promise()},b._updateContent=function(t,e){this._update(t,this.elements.content,e)},b._updateTitle=function(t,e){this._update(t,this.elements.title,e)===_&&this._removeTitle(_)},b._createTitle=function(){var t=this.elements,e=this._id+"-title";t.titlebar&&this._removeTitle(),t.titlebar=r("<div />",{"class":j+"-titlebar "+(this.options.style.widget?c("header"):"")}).append(t.title=r("<div />",{id:e,"class":j+"-title","aria-atomic":k})).insertBefore(t.content).delegate(".qtip-close","mousedown keydown mouseup keyup mouseout",function(t){r(this).toggleClass("ui-state-active ui-state-focus","down"===t.type.substr(-4))}).delegate(".qtip-close","mouseover mouseout",function(t){r(this).toggleClass("ui-state-hover","mouseover"===t.type)}),this.options.content.button&&this._createButton()},b._removeTitle=function(t){var e=this.elements;e.title&&(e.titlebar.remove(),e.titlebar=e.title=e.button=M,t!==_&&this.reposition())},b._createPosClass=function(t){return j+"-pos-"+(t||this.options.position.my).abbrev()},b.reposition=function(n,i){if(!this.rendered||this.positioning||this.destroyed)return this;this.positioning=k;var o,a,s,u,l=this.cache,c=this.tooltip,f=this.options.position,d=f.target,h=f.my,p=f.at,g=f.viewport,v=f.container,m=f.adjust,y=m.method.split(" "),b=c.outerWidth(_),x=c.outerHeight(_),w=0,$=0,M=c.css("position"),C={left:0,top:0},S=c[0].offsetWidth>0,j=n&&"scroll"===n.type,I=r(t),L=v[0].ownerDocument,P=this.mouse;if(r.isArray(d)&&2===d.length)p={x:A,y:E},C={left:d[0],top:d[1]};else if("mouse"===d)p={x:A,y:E},(!m.mouse||this.options.hide.distance)&&l.origin&&l.origin.pageX?n=l.origin:!n||n&&("resize"===n.type||"scroll"===n.type)?n=l.event:P&&P.pageX&&(n=P),"static"!==M&&(C=v.offset()),L.body.offsetWidth!==(t.innerWidth||L.documentElement.clientWidth)&&(a=r(e.body).offset()),C={left:n.pageX-C.left+(a&&a.left||0),top:n.pageY-C.top+(a&&a.top||0)},m.mouse&&j&&P&&(C.left-=(P.scrollX||0)-I.scrollLeft(),C.top-=(P.scrollY||0)-I.scrollTop());else{if("event"===d?n&&n.target&&"scroll"!==n.type&&"resize"!==n.type?l.target=r(n.target):n.target||(l.target=this.elements.target):"event"!==d&&(l.target=r(d.jquery?d:this.elements.target)),d=l.target,d=r(d).eq(0),0===d.length)return this;d[0]===e||d[0]===t?(w=Y.iOS?t.innerWidth:d.width(),$=Y.iOS?t.innerHeight:d.height(),d[0]===t&&(C={top:(g||d).scrollTop(),left:(g||d).scrollLeft()})):O.imagemap&&d.is("area")?o=O.imagemap(this,d,p,O.viewport?y:_):O.svg&&d&&d[0].ownerSVGElement?o=O.svg(this,d,p,O.viewport?y:_):(w=d.outerWidth(_),$=d.outerHeight(_),C=d.offset()),o&&(w=o.width,$=o.height,a=o.offset,C=o.position),C=this.reposition.offset(d,C,v),(Y.iOS>3.1&&Y.iOS<4.1||Y.iOS>=4.3&&Y.iOS<4.33||!Y.iOS&&"fixed"===M)&&(C.left-=I.scrollLeft(),C.top-=I.scrollTop()),(!o||o&&o.adjustable!==_)&&(C.left+=p.x===N?w:p.x===D?w/2:0,C.top+=p.y===T?$:p.y===D?$/2:0)}return C.left+=m.x+(h.x===N?-b:h.x===D?-b/2:0),C.top+=m.y+(h.y===T?-x:h.y===D?-x/2:0),O.viewport?(s=C.adjusted=O.viewport(this,C,f,w,$,b,x),a&&s.left&&(C.left+=a.left),a&&s.top&&(C.top+=a.top),s.my&&(this.position.my=s.my)):C.adjusted={left:0,top:0},l.posClass!==(u=this._createPosClass(this.position.my))&&c.removeClass(l.posClass).addClass(l.posClass=u),this._trigger("move",[C,g.elem||g],n)?(delete C.adjusted,i===_||!S||isNaN(C.left)||isNaN(C.top)||"mouse"===d||!r.isFunction(f.effect)?c.css(C):r.isFunction(f.effect)&&(f.effect.call(c,this,r.extend({},C)),c.queue(function(t){r(this).css({opacity:"",height:""}),Y.ie&&this.style.removeAttribute("filter"),t()})),this.positioning=_,this):this},b.reposition.offset=function(t,n,i){function o(t,e){n.left+=e*t.scrollLeft(),n.top+=e*t.scrollTop()}if(!i[0])return n;var a,s,u,l,c=r(t[0].ownerDocument),f=!!Y.ie&&"CSS1Compat"!==e.compatMode,d=i[0];do"static"!==(s=r.css(d,"position"))&&("fixed"===s?(u=d.getBoundingClientRect(),o(c,-1)):(u=r(d).position(),u.left+=parseFloat(r.css(d,"borderLeftWidth"))||0,u.top+=parseFloat(r.css(d,"borderTopWidth"))||0),n.left-=u.left+(parseFloat(r.css(d,"marginLeft"))||0),n.top-=u.top+(parseFloat(r.css(d,"marginTop"))||0),a||"hidden"===(l=r.css(d,"overflow"))||"visible"===l||(a=r(d)));while(d=d.offsetParent);return a&&(a[0]!==c[0]||f)&&o(a,1),n};var Z=(x=b.reposition.Corner=function(t,e){t=(""+t).replace(/([A-Z])/," $1").replace(/middle/gi,D).toLowerCase(),this.x=(t.match(/left|right/i)||t.match(/center/)||["inherit"])[0].toLowerCase(),this.y=(t.match(/top|bottom|center/i)||["inherit"])[0].toLowerCase(),this.forceY=!!e;var n=t.charAt(0);this.precedance="t"===n||"b"===n?S:C}).prototype;Z.invert=function(t,e){this[t]=this[t]===A?N:this[t]===N?A:e||this[t]},Z.string=function(t){var e=this.x,n=this.y,r=e!==n?"center"===e||"center"!==n&&(this.precedance===S||this.forceY)?[n,e]:[e,n]:[e];return t!==!1?r.join(" "):r},Z.abbrev=function(){var t=this.string(!1);return t[0].charAt(0)+(t[1]&&t[1].charAt(0)||"")},Z.clone=function(){return new x(this.string(),this.forceY)},b.toggle=function(t,n){var i=this.cache,o=this.options,a=this.tooltip;if(n){if(/over|enter/.test(n.type)&&i.event&&/out|leave/.test(i.event.type)&&o.show.target.add(n.target).length===o.show.target.length&&a.has(n.relatedTarget).length)return this;i.event=r.event.fix(n)}if(this.waiting&&!t&&(this.hiddenDuringWait=k),!this.rendered)return t?this.render(1):this;if(this.destroyed||this.disabled)return this;var s,u,l,c=t?"show":"hide",f=this.options[c],d=(this.options[t?"hide":"show"],this.options.position),h=this.options.content,p=this.tooltip.css("width"),g=this.tooltip.is(":visible"),v=t||1===f.target.length,m=!n||f.target.length<2||i.target[0]===n.target;return(typeof t).search("boolean|number")&&(t=!g),s=!a.is(":animated")&&g===t&&m,u=s?M:!!this._trigger(c,[90]),this.destroyed?this:(u!==_&&t&&this.focus(n),!u||s?this:(r.attr(a[0],"aria-hidden",!t),t?(this.mouse&&(i.origin=r.event.fix(this.mouse)),r.isFunction(h.text)&&this._updateContent(h.text,_),r.isFunction(h.title)&&this._updateTitle(h.title,_),!$&&"mouse"===d.target&&d.adjust.mouse&&(r(e).bind("mousemove."+j,this._storeMouse),$=k),p||a.css("width",a.outerWidth(_)),this.reposition(n,arguments[2]),p||a.css("width",""),f.solo&&("string"==typeof f.solo?r(f.solo):r(F,f.solo)).not(a).not(f.target).qtip("hide",r.Event("tooltipsolo"))):(clearTimeout(this.timers.show),delete i.origin,$&&!r(F+'[tracking="true"]:visible',f.solo).not(a).length&&(r(e).unbind("mousemove."+j),$=_),this.blur(n)),l=r.proxy(function(){t?(Y.ie&&a[0].style.removeAttribute("filter"),a.css("overflow",""),"string"==typeof f.autofocus&&r(this.options.show.autofocus,a).focus(),this.options.show.target.trigger("qtip-"+this.id+"-inactive")):a.css({display:"",visibility:"",opacity:"",left:"",top:""}),this._trigger(t?"visible":"hidden")},this),f.effect===_||v===_?(a[c](),l()):r.isFunction(f.effect)?(a.stop(1,1),f.effect.call(a,this),a.queue("fx",function(t){l(),t()})):a.fadeTo(90,t?1:0,l),t&&f.target.trigger("qtip-"+this.id+"-inactive"),this))},b.show=function(t){return this.toggle(k,t)},b.hide=function(t){return this.toggle(_,t)},b.focus=function(t){if(!this.rendered||this.destroyed)return this;var e=r(F),n=this.tooltip,i=parseInt(n[0].style.zIndex,10),o=y.zindex+e.length;return n.hasClass(R)||this._trigger("focus",[o],t)&&(i!==o&&(e.each(function(){this.style.zIndex>i&&(this.style.zIndex=this.style.zIndex-1)}),e.filter("."+R).qtip("blur",t)),n.addClass(R)[0].style.zIndex=o),this},b.blur=function(t){return!this.rendered||this.destroyed?this:(this.tooltip.removeClass(R),this._trigger("blur",[this.tooltip.css("zIndex")],t),this)},b.disable=function(t){return this.destroyed?this:("toggle"===t?t=!(this.rendered?this.tooltip.hasClass(V):this.disabled):"boolean"!=typeof t&&(t=k),this.rendered&&this.tooltip.toggleClass(V,t).attr("aria-disabled",t),this.disabled=!!t,this)},b.enable=function(){return this.disable(_)},b._createButton=function(){var t=this,e=this.elements,n=e.tooltip,i=this.options.content.button,o="string"==typeof i,a=o?i:"Close tooltip";e.button&&e.button.remove(),i.jquery?e.button=i:e.button=r("<a />",{"class":"qtip-close "+(this.options.style.widget?"":j+"-icon"),title:a,"aria-label":a}).prepend(r("<span />",{"class":"ui-icon ui-icon-close",html:"&times;"})),e.button.appendTo(e.titlebar||n).attr("role","button").click(function(e){return n.hasClass(V)||t.hide(e),_})},b._updateButton=function(t){if(!this.rendered)return _;var e=this.elements.button;t?this._createButton():e.remove()},b._setWidget=function(){var t=this.options.style.widget,e=this.elements,n=e.tooltip,r=n.hasClass(V);n.removeClass(V),V=t?"ui-state-disabled":"qtip-disabled",n.toggleClass(V,r),n.toggleClass("ui-helper-reset "+c(),t).toggleClass(W,this.options.style.def&&!t),e.content&&e.content.toggleClass(c("content"),t),e.titlebar&&e.titlebar.toggleClass(c("header"),t),e.button&&e.button.toggleClass(j+"-icon",!t)},b._storeMouse=function(t){return(this.mouse=r.event.fix(t)).type="mousemove",this},b._bind=function(t,e,n,i,o){if(t&&n&&e.length){var a="."+this._id+(i?"-"+i:"");return r(t).bind((e.split?e:e.join(a+" "))+a,r.proxy(n,o||this)),this}},b._unbind=function(t,e){return t&&r(t).unbind("."+this._id+(e?"-"+e:"")),this},b._trigger=function(t,e,n){var i=r.Event("tooltip"+t);return i.originalEvent=n&&r.extend({},n)||this.cache.event||M,this.triggering=t,this.tooltip.trigger(i,[this].concat(e||[])),this.triggering=_,!i.isDefaultPrevented()},b._bindEvents=function(t,e,n,i,o,a){var s=n.filter(i).add(i.filter(n)),u=[];s.length&&(r.each(e,function(e,n){var i=r.inArray(n,t);i>-1&&u.push(t.splice(i,1)[0])}),u.length&&(this._bind(s,u,function(t){var e=!!this.rendered&&this.tooltip[0].offsetWidth>0;(e?a:o).call(this,t)}),n=n.not(s),i=i.not(s))),this._bind(n,t,o),this._bind(i,e,a)},b._assignInitialEvents=function(t){function e(t){return this.disabled||this.destroyed?_:(this.cache.event=t&&r.event.fix(t),this.cache.target=t&&r(t.target),clearTimeout(this.timers.show),void(this.timers.show=f.call(this,function(){this.render("object"==typeof t||n.show.ready)},n.prerender?0:n.show.delay)))}var n=this.options,i=n.show.target,o=n.hide.target,a=n.show.event?r.trim(""+n.show.event).split(" "):[],s=n.hide.event?r.trim(""+n.hide.event).split(" "):[];this._bind(this.elements.target,["remove","removeqtip"],function(t){this.destroy(!0)},"destroy"),/mouse(over|enter)/i.test(n.show.event)&&!/mouse(out|leave)/i.test(n.hide.event)&&s.push("mouseleave"),this._bind(i,"mousemove",function(t){this._storeMouse(t),this.cache.onTarget=k}),this._bindEvents(a,s,i,o,e,function(){return this.timers?void clearTimeout(this.timers.show):_}),(n.show.ready||n.prerender)&&e.call(this,t)},b._assignEvents=function(){var n=this,i=this.options,o=i.position,a=this.tooltip,s=i.show.target,u=i.hide.target,l=o.container,c=o.viewport,f=r(e),v=(r(e.body),r(t)),m=i.show.event?r.trim(""+i.show.event).split(" "):[],b=i.hide.event?r.trim(""+i.hide.event).split(" "):[];r.each(i.events,function(t,e){n._bind(a,"toggle"===t?["tooltipshow","tooltiphide"]:["tooltip"+t],e,null,a)}),/mouse(out|leave)/i.test(i.hide.event)&&"window"===i.hide.leave&&this._bind(f,["mouseout","blur"],function(t){/select|option/.test(t.target.nodeName)||t.relatedTarget||this.hide(t)}),i.hide.fixed?u=u.add(a.addClass(z)):/mouse(over|enter)/i.test(i.show.event)&&this._bind(u,"mouseleave",function(){clearTimeout(this.timers.show)}),(""+i.hide.event).indexOf("unfocus")>-1&&this._bind(l.closest("html"),["mousedown","touchstart"],function(t){var e=r(t.target),n=this.rendered&&!this.tooltip.hasClass(V)&&this.tooltip[0].offsetWidth>0,i=e.parents(F).filter(this.tooltip[0]).length>0;e[0]===this.target[0]||e[0]===this.tooltip[0]||i||this.target.has(e[0]).length||!n||this.hide(t)}),"number"==typeof i.hide.inactive&&(this._bind(s,"qtip-"+this.id+"-inactive",p,"inactive"),this._bind(u.add(a),y.inactiveEvents,p)),this._bindEvents(m,b,s,u,d,h),this._bind(s.add(a),"mousemove",function(t){if("number"==typeof i.hide.distance){var e=this.cache.origin||{},n=this.options.hide.distance,r=Math.abs;(r(t.pageX-e.pageX)>=n||r(t.pageY-e.pageY)>=n)&&this.hide(t)}this._storeMouse(t)}),"mouse"===o.target&&o.adjust.mouse&&(i.hide.event&&this._bind(s,["mouseenter","mouseleave"],function(t){return this.cache?void(this.cache.onTarget="mouseenter"===t.type):_}),this._bind(f,"mousemove",function(t){this.rendered&&this.cache.onTarget&&!this.tooltip.hasClass(V)&&this.tooltip[0].offsetWidth>0&&this.reposition(t)})),(o.adjust.resize||c.length)&&this._bind(r.event.special.resize?c:v,"resize",g),o.adjust.scroll&&this._bind(v.add(o.container),"scroll",g)},b._unassignEvents=function(){var n=this.options,i=n.show.target,o=n.hide.target,a=r.grep([this.elements.target[0],this.rendered&&this.tooltip[0],n.position.container[0],n.position.viewport[0],n.position.container.closest("html")[0],t,e],function(t){return"object"==typeof t});i&&i.toArray&&(a=a.concat(i.toArray())),o&&o.toArray&&(a=a.concat(o.toArray())),this._unbind(a)._unbind(a,"destroy")._unbind(a,"inactive")},r(function(){v(F,["mouseenter","mouseleave"],function(t){var e="mouseenter"===t.type,n=r(t.currentTarget),i=r(t.relatedTarget||t.target),o=this.options;e?(this.focus(t),n.hasClass(z)&&!n.hasClass(V)&&clearTimeout(this.timers.hide)):"mouse"===o.position.target&&o.position.adjust.mouse&&o.hide.event&&o.show.target&&!i.closest(o.show.target[0]).length&&this.hide(t),n.toggleClass(B,e)}),v("["+L+"]",q,p)}),y=r.fn.qtip=function(t,e,i){var o=(""+t).toLowerCase(),a=M,u=r.makeArray(arguments).slice(1),l=u[u.length-1],c=this[0]?r.data(this[0],j):M;return!arguments.length&&c||"api"===o?c:"string"==typeof t?(this.each(function(){var t=r.data(this,j);if(!t)return k;if(l&&l.timeStamp&&(t.cache.event=l),!e||"option"!==o&&"options"!==o)t[o]&&t[o].apply(t,u);else{if(i===n&&!r.isPlainObject(e))return a=t.get(e),_;t.set(e,i)}}),a!==M?a:this):"object"!=typeof t&&arguments.length?void 0:(c=s(r.extend(k,{},t)),this.each(function(t){var e,n;return n=r.isArray(c.id)?c.id[t]:c.id,n=!n||n===_||n.length<1||y.api[n]?y.nextid++:n,e=m(r(this),n,c),e===_?k:(y.api[n]=e,r.each(O,function(){"initialize"===this.initialize&&this(e)}),void e._assignInitialEvents(l))}))},r.qtip=i,y.api={},r.each({attr:function(t,e){if(this.length){var n=this[0],i="title",o=r.data(n,"qtip");if(t===i&&o&&"object"==typeof o&&o.options.suppress)return arguments.length<2?r.attr(n,U):(o&&o.options.content.attr===i&&o.cache.attr&&o.set("content.text",e),this.attr(U,e))}return r.fn["attr"+H].apply(this,arguments)},clone:function(t){var e=(r([]),r.fn["clone"+H].apply(this,arguments));return t||e.filter("["+U+"]").attr("title",function(){return r.attr(this,U)}).removeAttr(U),e}},function(t,e){if(!e||r.fn[t+H])return k;var n=r.fn[t+H]=r.fn[t];r.fn[t]=function(){return e.apply(this,arguments)||n.apply(this,arguments)}}),r.ui||(r["cleanData"+H]=r.cleanData,r.cleanData=function(t){for(var e,n=0;(e=r(t[n])).length;n++)if(e.attr(I))try{e.triggerHandler("removeqtip")}catch(i){}r["cleanData"+H].apply(this,arguments)}),y.version="2.2.1",y.nextid=0,y.inactiveEvents=q,y.zindex=15e3,y.defaults={prerender:_,id:_,overwrite:k,suppress:k,content:{text:k,attr:"title",title:_,button:_},position:{my:"top left",at:"bottom right",target:_,container:_,viewport:_,adjust:{x:0,y:0,mouse:k,scroll:k,resize:k,method:"flipinvert flipinvert"},effect:function(t,e,n){r(this).animate(e,{duration:200,queue:_})}},show:{target:_,event:"mouseenter",effect:k,delay:90,solo:_,ready:_,autofocus:_},hide:{target:_,event:"mouseleave",effect:k,delay:0,fixed:_,inactive:_,leave:"window",distance:_},style:{classes:"",widget:_,width:_,height:_,def:k},events:{render:M,move:M,show:M,hide:M,toggle:M,visible:M,hidden:M,focus:M,blur:M}}})}(window,document),function(t){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{var e;e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,e.dagreD3=t()}}(function(){return function t(e,n,r){function i(a,s){if(!n[a]){if(!e[a]){var u="function"==typeof require&&require;if(!s&&u)return u(a,!0);if(o)return o(a,!0);var l=new Error("Cannot find module '"+a+"'");throw l.code="MODULE_NOT_FOUND",l}var c=n[a]={exports:{}};e[a][0].call(c.exports,function(t){var n=e[a][1][t];return i(n?n:t)},c,c.exports,t,e,n,r)}return n[a].exports}for(var o="function"==typeof require&&require,a=0;a<r.length;a++)i(r[a]);return i}({1:[function(t,e,n){e.exports={graphlib:t("./lib/graphlib"),dagre:t("./lib/dagre"),intersect:t("./lib/intersect"),render:t("./lib/render"),util:t("./lib/util"),version:t("./lib/version")}},{"./lib/dagre":8,"./lib/graphlib":9,"./lib/intersect":10,"./lib/render":25,"./lib/util":27,"./lib/version":28}],2:[function(t,e,n){function r(t,e,n,r){var i=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",9).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",8).attr("markerHeight",6).attr("orient","auto"),o=i.append("path").attr("d","M 0 0 L 10 5 L 0 10 z").style("stroke-width",1).style("stroke-dasharray","1,0");
 a.applyStyle(o,n[r+"Style"]),n[r+"Class"]&&o.attr("class",n[r+"Class"])}function i(t,e,n,r){var i=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",9).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",8).attr("markerHeight",6).attr("orient","auto"),o=i.append("path").attr("d","M 0 0 L 10 5 L 0 10 L 4 5 z").style("stroke-width",1).style("stroke-dasharray","1,0");a.applyStyle(o,n[r+"Style"]),n[r+"Class"]&&o.attr("class",n[r+"Class"])}function o(t,e,n,r){var i=t.append("marker").attr("id",e).attr("viewBox","0 0 10 10").attr("refX",9).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",8).attr("markerHeight",6).attr("orient","auto"),o=i.append("path").attr("d","M 0 5 L 10 5").style("stroke-width",1).style("stroke-dasharray","1,0");a.applyStyle(o,n[r+"Style"]),n[r+"Class"]&&o.attr("class",n[r+"Class"])}var a=t("./util");e.exports={"default":r,normal:r,vee:i,undirected:o}},{"./util":27}],3:[function(t,e,n){function r(t,e){var n=e.nodes().filter(function(t){return i.isSubgraph(e,t)}),r=t.selectAll("g.cluster").data(n,function(t){return t});return r.selectAll("*").remove(),r.enter().append("g").attr("class","cluster").attr("id",function(t){var n=e.node(t);return n.id}).style("opacity",0),i.applyTransition(r,e).style("opacity",1),r.each(function(t){var n=e.node(t),r=d3.select(this);d3.select(this).append("rect");var i=r.append("g").attr("class","label");o(i,n,n.clusterLabelPos)}),r.selectAll("rect").each(function(t){var n=e.node(t),r=d3.select(this);i.applyStyle(r,n.style)}),i.applyTransition(r.exit(),e).style("opacity",0).remove(),r}var i=t("./util"),o=t("./label/add-label");e.exports=r},{"./label/add-label":18,"./util":27}],4:[function(t,e,n){"use strict";function r(t,e){var n=t.selectAll("g.edgeLabel").data(e.edges(),function(t){return a.edgeToId(t)}).classed("update",!0);return n.selectAll("*").remove(),n.enter().append("g").classed("edgeLabel",!0).style("opacity",0),n.each(function(t){var n=e.edge(t),r=o(s.select(this),e.edge(t),0,0).classed("label",!0),a=r.node().getBBox();n.labelId&&r.attr("id",n.labelId),i.has(n,"width")||(n.width=a.width),i.has(n,"height")||(n.height=a.height)}),a.applyTransition(n.exit(),e).style("opacity",0).remove(),n}var i=t("./lodash"),o=t("./label/add-label"),a=t("./util"),s=t("./d3");e.exports=r},{"./d3":7,"./label/add-label":18,"./lodash":21,"./util":27}],5:[function(t,e,n){"use strict";function r(t,e,n){var r=t.selectAll("g.edgePath").data(e.edges(),function(t){return d.edgeToId(t)}).classed("update",!0);return u(r,e),l(r,e),d.applyTransition(r,e).style("opacity",1),r.each(function(t){var n=h.select(this),r=e.edge(t);r.elem=this,r.id&&n.attr("id",r.id),d.applyClass(n,r["class"],(n.classed("update")?"update ":"")+"edgePath")}),r.selectAll("path.path").each(function(t){var n=e.edge(t);n.arrowheadId=c.uniqueId("arrowhead");var r=h.select(this).attr("marker-end",function(){return"url("+i(location.href,n.arrowheadId)+")"}).style("fill","none");d.applyTransition(r,e).attr("d",function(t){return o(e,t)}),d.applyStyle(r,n.style)}),r.selectAll("defs *").remove(),r.selectAll("defs").each(function(t){var r=e.edge(t),i=n[r.arrowhead];i(h.select(this),r.arrowheadId,r,"arrowhead")}),r}function i(t,e){var n=t.split("#")[0];return n+"#"+e}function o(t,e){var n=t.edge(e),r=t.node(e.v),i=t.node(e.w),o=n.points.slice(1,n.points.length-1);return o.unshift(f(r,o[0])),o.push(f(i,o[o.length-1])),a(n,o)}function a(t,e){var n=h.svg.line().x(function(t){return t.x}).y(function(t){return t.y});return c.has(t,"lineInterpolate")&&n.interpolate(t.lineInterpolate),c.has(t,"lineTension")&&n.tension(Number(t.lineTension)),n(e)}function s(t){var e=t.getBBox(),n=t.ownerSVGElement.getScreenCTM().inverse().multiply(t.getScreenCTM()).translate(e.width/2,e.height/2);return{x:n.e,y:n.f}}function u(t,e){var n=t.enter().append("g").attr("class","edgePath").style("opacity",0);n.append("path").attr("class","path").attr("d",function(t){var n=e.edge(t),r=e.node(t.v).elem,i=c.range(n.points.length).map(function(){return s(r)});return a(n,i)}),n.append("defs")}function l(t,e){var n=t.exit();d.applyTransition(n,e).style("opacity",0).remove(),d.applyTransition(n.select("path.path"),e).attr("d",function(t){var n=e.node(t.v);if(n){var r=c.range(this.getTotalLength()).map(function(){return n});return a({},r)}return h.select(this).attr("d")})}var c=t("./lodash"),f=t("./intersect/intersect-node"),d=t("./util"),h=t("./d3");e.exports=r},{"./d3":7,"./intersect/intersect-node":14,"./lodash":21,"./util":27}],6:[function(t,e,n){"use strict";function r(t,e,n){var r=e.nodes().filter(function(t){return!a.isSubgraph(e,t)}),u=t.selectAll("g.node").data(r,function(t){return t}).classed("update",!0);return u.selectAll("*").remove(),u.enter().append("g").attr("class","node").style("opacity",0),u.each(function(t){var r=e.node(t),u=s.select(this),l=u.append("g").attr("class","label"),c=o(l,r),f=n[r.shape],d=i.pick(c.node().getBBox(),"width","height");r.elem=this,r.id&&u.attr("id",r.id),r.labelId&&l.attr("id",r.labelId),a.applyClass(u,r["class"],(u.classed("update")?"update ":"")+"node"),i.has(r,"width")&&(d.width=r.width),i.has(r,"height")&&(d.height=r.height),d.width+=r.paddingLeft+r.paddingRight,d.height+=r.paddingTop+r.paddingBottom,l.attr("transform","translate("+(r.paddingLeft-r.paddingRight)/2+","+(r.paddingTop-r.paddingBottom)/2+")");var h=f(s.select(this),d,r);a.applyStyle(h,r.style);var p=h.node().getBBox();r.width=p.width,r.height=p.height}),a.applyTransition(u.exit(),e).style("opacity",0).remove(),u}var i=t("./lodash"),o=t("./label/add-label"),a=t("./util"),s=t("./d3");e.exports=r},{"./d3":7,"./label/add-label":18,"./lodash":21,"./util":27}],7:[function(t,e,n){e.exports=window.d3},{}],8:[function(t,e,n){var r;if(t)try{r=t("dagre")}catch(i){}r||(r=window.dagre),e.exports=r},{dagre:void 0}],9:[function(t,e,n){var r;if(t)try{r=t("graphlib")}catch(i){}r||(r=window.graphlib),e.exports=r},{graphlib:void 0}],10:[function(t,e,n){e.exports={node:t("./intersect-node"),circle:t("./intersect-circle"),ellipse:t("./intersect-ellipse"),polygon:t("./intersect-polygon"),rect:t("./intersect-rect")}},{"./intersect-circle":11,"./intersect-ellipse":12,"./intersect-node":14,"./intersect-polygon":15,"./intersect-rect":16}],11:[function(t,e,n){function r(t,e,n){return i(t,e,e,n)}var i=t("./intersect-ellipse");e.exports=r},{"./intersect-ellipse":12}],12:[function(t,e,n){function r(t,e,n,r){var i=t.x,o=t.y,a=i-r.x,s=o-r.y,u=Math.sqrt(e*e*s*s+n*n*a*a),l=Math.abs(e*n*a/u);r.x<i&&(l=-l);var c=Math.abs(e*n*s/u);return r.y<o&&(c=-c),{x:i+l,y:o+c}}e.exports=r},{}],13:[function(t,e,n){function r(t,e,n,r){var o,a,s,u,l,c,f,d,h,p,g,v,m,y,b;if(o=e.y-t.y,s=t.x-e.x,l=e.x*t.y-t.x*e.y,h=o*n.x+s*n.y+l,p=o*r.x+s*r.y+l,!(0!==h&&0!==p&&i(h,p)||(a=r.y-n.y,u=n.x-r.x,c=r.x*n.y-n.x*r.y,f=a*t.x+u*t.y+c,d=a*e.x+u*e.y+c,0!==f&&0!==d&&i(f,d)||(g=o*u-a*s,0===g))))return v=Math.abs(g/2),m=s*c-u*l,y=m<0?(m-v)/g:(m+v)/g,m=a*l-o*c,b=m<0?(m-v)/g:(m+v)/g,{x:y,y:b}}function i(t,e){return t*e>0}e.exports=r},{}],14:[function(t,e,n){function r(t,e){return t.intersect(e)}e.exports=r},{}],15:[function(t,e,n){function r(t,e,n){var r=t.x,o=t.y,a=[],s=Number.POSITIVE_INFINITY,u=Number.POSITIVE_INFINITY;e.forEach(function(t){s=Math.min(s,t.x),u=Math.min(u,t.y)});for(var l=r-t.width/2-s,c=o-t.height/2-u,f=0;f<e.length;f++){var d=e[f],h=e[f<e.length-1?f+1:0],p=i(t,n,{x:l+d.x,y:c+d.y},{x:l+h.x,y:c+h.y});p&&a.push(p)}return a.length?(a.length>1&&a.sort(function(t,e){var r=t.x-n.x,i=t.y-n.y,o=Math.sqrt(r*r+i*i),a=e.x-n.x,s=e.y-n.y,u=Math.sqrt(a*a+s*s);return o<u?-1:o===u?0:1}),a[0]):(console.log("NO INTERSECTION FOUND, RETURN NODE CENTER",t),t)}var i=t("./intersect-line");e.exports=r},{"./intersect-line":13}],16:[function(t,e,n){function r(t,e){var n,r,i=t.x,o=t.y,a=e.x-i,s=e.y-o,u=t.width/2,l=t.height/2;return Math.abs(s)*u>Math.abs(a)*l?(s<0&&(l=-l),n=0===s?0:l*a/s,r=l):(a<0&&(u=-u),n=u,r=0===a?0:u*s/a),{x:i+n,y:o+r}}e.exports=r},{}],17:[function(t,e,n){function r(t,e){var n=t.append("foreignObject").attr("width","100000"),r=n.append("xhtml:div");r.attr("xmlns","http://www.w3.org/1999/xhtml");var o=e.label;switch(typeof o){case"function":r.insert(o);break;case"object":r.insert(function(){return o});break;default:r.html(o)}i.applyStyle(r,e.labelStyle),r.style("display","inline-block"),r.style("white-space","nowrap");var a=r[0][0].getBoundingClientRect();return n.attr("width",a.width).attr("height",a.height),n}var i=t("../util");e.exports=r},{"../util":27}],18:[function(t,e,n){function r(t,e,n){var r=e.label,s=t.append("g");"svg"===e.labelType?a(s,e):"string"!=typeof r||"html"===e.labelType?o(s,e):i(s,e);var u,l=s.node().getBBox();switch(n){case"top":u=-e.height/2;break;case"bottom":u=e.height/2-l.height;break;default:u=-l.height/2}return s.attr("transform","translate("+-l.width/2+","+u+")"),s}var i=t("./add-text-label"),o=t("./add-html-label"),a=t("./add-svg-label");e.exports=r},{"./add-html-label":17,"./add-svg-label":19,"./add-text-label":20}],19:[function(t,e,n){function r(t,e){var n=t;return n.node().appendChild(e.label),i.applyStyle(n,e.labelStyle),n}var i=t("../util");e.exports=r},{"../util":27}],20:[function(t,e,n){function r(t,e){for(var n=t.append("text"),r=i(e.label).split("\n"),a=0;a<r.length;a++)n.append("tspan").attr("xml:space","preserve").attr("dy","1em").attr("x","1").text(r[a]);return o.applyStyle(n,e.labelStyle),n}function i(t){for(var e,n="",r=!1,i=0;i<t.length;++i)if(e=t[i],r){switch(e){case"n":n+="\n";break;default:n+=e}r=!1}else"\\"===e?r=!0:n+=e;return n}var o=t("../util");e.exports=r},{"../util":27}],21:[function(t,e,n){var r;if(t)try{r=t("lodash")}catch(i){}r||(r=window._),e.exports=r},{lodash:void 0}],22:[function(t,e,n){"use strict";function r(t,e){function n(t){var n=e.node(t);return"translate("+n.x+","+n.y+")"}var r=t.filter(function(){return!o.select(this).classed("update")});r.attr("transform",n),i.applyTransition(t,e).style("opacity",1).attr("transform",n),i.applyTransition(r.selectAll("rect"),e).attr("width",function(t){return e.node(t).width}).attr("height",function(t){return e.node(t).height}).attr("x",function(t){var n=e.node(t);return-n.width/2}).attr("y",function(t){var n=e.node(t);return-n.height/2})}var i=t("./util"),o=t("./d3");e.exports=r},{"./d3":7,"./util":27}],23:[function(t,e,n){"use strict";function r(t,e){function n(t){var n=e.edge(t);return a.has(n,"x")?"translate("+n.x+","+n.y+")":""}var r=t.filter(function(){return!o.select(this).classed("update")});r.attr("transform",n),i.applyTransition(t,e).style("opacity",1).attr("transform",n)}var i=t("./util"),o=t("./d3"),a=t("./lodash");e.exports=r},{"./d3":7,"./lodash":21,"./util":27}],24:[function(t,e,n){"use strict";function r(t,e){function n(t){var n=e.node(t);return"translate("+n.x+","+n.y+")"}var r=t.filter(function(){return!o.select(this).classed("update")});r.attr("transform",n),i.applyTransition(t,e).style("opacity",1).attr("transform",n)}var i=t("./util"),o=t("./d3");e.exports=r},{"./d3":7,"./util":27}],25:[function(t,e,n){function r(){var e=t("./create-nodes"),n=t("./create-clusters"),r=t("./create-edge-labels"),s=t("./create-edge-paths"),l=t("./position-nodes"),c=t("./position-edge-labels"),f=t("./position-clusters"),d=t("./shapes"),h=t("./arrows"),p=function(t,p){i(p);var g=a(t,"output"),v=a(g,"clusters"),m=a(g,"edgePaths"),y=r(a(g,"edgeLabels"),p),b=e(a(g,"nodes"),p,d);u(p),l(b,p),c(y,p),s(m,p,h);var x=n(v,p);f(x,p),o(p)};return p.createNodes=function(t){return arguments.length?(e=t,p):e},p.createClusters=function(t){return arguments.length?(n=t,p):n},p.createEdgeLabels=function(t){return arguments.length?(r=t,p):r},p.createEdgePaths=function(t){return arguments.length?(s=t,p):s},p.shapes=function(t){return arguments.length?(d=t,p):d},p.arrows=function(t){return arguments.length?(h=t,p):h},p}function i(t){t.nodes().forEach(function(e){var n=t.node(e);s.has(n,"label")||t.children(e).length||(n.label=e),s.has(n,"paddingX")&&s.defaults(n,{paddingLeft:n.paddingX,paddingRight:n.paddingX}),s.has(n,"paddingY")&&s.defaults(n,{paddingTop:n.paddingY,paddingBottom:n.paddingY}),s.has(n,"padding")&&s.defaults(n,{paddingLeft:n.padding,paddingRight:n.padding,paddingTop:n.padding,paddingBottom:n.padding}),s.defaults(n,l),s.each(["paddingLeft","paddingRight","paddingTop","paddingBottom"],function(t){n[t]=Number(n[t])}),s.has(n,"width")&&(n._prevWidth=n.width),s.has(n,"height")&&(n._prevHeight=n.height)}),t.edges().forEach(function(e){var n=t.edge(e);s.has(n,"label")||(n.label=""),s.defaults(n,c)})}function o(t){s.each(t.nodes(),function(e){var n=t.node(e);s.has(n,"_prevWidth")?n.width=n._prevWidth:delete n.width,s.has(n,"_prevHeight")?n.height=n._prevHeight:delete n.height,delete n._prevWidth,delete n._prevHeight})}function a(t,e){var n=t.select("g."+e);return n.empty()&&(n=t.append("g").attr("class",e)),n}var s=t("./lodash"),u=t("./dagre").layout;e.exports=r;var l={paddingLeft:10,paddingRight:10,paddingTop:10,paddingBottom:10,rx:0,ry:0,shape:"rect"},c={arrowhead:"normal",lineInterpolate:"linear"}},{"./arrows":2,"./create-clusters":3,"./create-edge-labels":4,"./create-edge-paths":5,"./create-nodes":6,"./dagre":8,"./lodash":21,"./position-clusters":22,"./position-edge-labels":23,"./position-nodes":24,"./shapes":26}],26:[function(t,e,n){"use strict";function r(t,e,n){var r=t.insert("rect",":first-child").attr("rx",n.rx).attr("ry",n.ry).attr("x",-e.width/2).attr("y",-e.height/2).attr("width",e.width).attr("height",e.height);return n.intersect=function(t){return s(n,t)},r}function i(t,e,n){var r=e.width/2,i=e.height/2,o=t.insert("ellipse",":first-child").attr("x",-e.width/2).attr("y",-e.height/2).attr("rx",r).attr("ry",i);return n.intersect=function(t){return u(n,r,i,t)},o}function o(t,e,n){var r=Math.max(e.width,e.height)/2,i=t.insert("circle",":first-child").attr("x",-e.width/2).attr("y",-e.height/2).attr("r",r);return n.intersect=function(t){return l(n,r,t)},i}function a(t,e,n){var r=e.width*Math.SQRT2/2,i=e.height*Math.SQRT2/2,o=[{x:0,y:-i},{x:-r,y:0},{x:0,y:i},{x:r,y:0}],a=t.insert("polygon",":first-child").attr("points",o.map(function(t){return t.x+","+t.y}).join(" "));return n.intersect=function(t){return c(n,o,t)},a}var s=t("./intersect/intersect-rect"),u=t("./intersect/intersect-ellipse"),l=t("./intersect/intersect-circle"),c=t("./intersect/intersect-polygon");e.exports={rect:r,ellipse:i,circle:o,diamond:a}},{"./intersect/intersect-circle":11,"./intersect/intersect-ellipse":12,"./intersect/intersect-polygon":15,"./intersect/intersect-rect":16}],27:[function(t,e,n){function r(t,e){return!!t.children(e).length}function i(t){return o(t.v)+":"+o(t.w)+":"+o(t.name)}function o(t){return t?String(t).replace(c,"\\:"):""}function a(t,e){e&&t.attr("style",e)}function s(t,e,n){e&&t.attr("class",e).attr("class",n+" "+t.attr("class"))}function u(t,e){var n=e.graph();if(l.isPlainObject(n)){var r=n.transition;if(l.isFunction(r))return r(t)}return t}var l=t("./lodash");e.exports={isSubgraph:r,edgeToId:i,applyStyle:a,applyClass:s,applyTransition:u};var c=/:/g},{"./lodash":21}],28:[function(t,e,n){e.exports="0.4.17"},{}]},{},[1])(1)}),function(){d3.timeline=function(){function t(t){function e(t,e){return U(t.starting_time)}function a(t,e){return U(t.ending_time)}function O(t,e){return U(t.starting_time)}function j(t,e){return U(t.starting_time)+5}function I(t,e){return U(t.ending_time)-U(t.starting_time)}function L(t,e){var n=U(t.ending_time)-U(t.starting_time);return n>5?n-5:n}function P(){if(c||z.attr("height"))c?z.attr("height",c):c=z.attr("height");else{if(!_)throw"height of the timeline is not set";c=K.height+K.top-q.top,d3.select(t[0][0]).attr("height",c)}}function F(){if(l||q.width){if(!l||!q.width)try{l=z.attr("width")}catch(t){console.log(t)}}else try{if(l=z.attr("width"),!l)throw"width of the timeline is not set. As of Firefox 27, timeline().with(x) needs to be explicitly set in order to render"}catch(t){console.log(t)}}var q=t[0][0].getBoundingClientRect(),z=d3.select(t[0][0]),W={},R=1,B=0,V=0;F();var H=(t.append("svg:clipPath").attr("id",D+"-gclip").append("svg:rect").attr("clipPathUnits","objectBoundingBox").attr("x",b.left).attr("y",b.top).attr("width",l-b.left-b.right).attr("height",1e3),t.append("g").attr("clip-path","url(#"+D+"-gclip)"));k&&H.each(function(t,e){t.forEach(function(t,e){t.times.forEach(function(t,n){0===e&&0===n?(originTime=t.starting_time,t.starting_time=0,t.ending_time=t.ending_time-originTime):(t.starting_time=t.starting_time-originTime,t.ending_time=t.ending_time-originTime)})})}),(x||0===y||0===m)&&(H.each(function(t,e){t.forEach(function(t,e){x&&Object.keys(W).indexOf(e)==-1&&(W[e]=R,R++),t.times.forEach(function(t,e){0===m&&(t.starting_time<B||0===B&&k===!1)&&(B=t.starting_time),0===y&&t.ending_time>V&&(V=t.ending_time)})})}),0===y&&(y=V),0===m&&(m=B));var U=d3.time.scale().domain([m,y]).range([b.left,l-b.right]),Y=d3.svg.axis().scale(U).orient(u).tickFormat(h.format).ticks(h.numTicks||h.tickTime,h.tickInterval).tickSize(h.tickSize);if(T)var G=d3.svg.axis().scale(U).orient(u).tickFormat(d3.time.format("%X")).ticks(h.numTicks||h.tickTime,h.tickInterval).tickSize(0);if(C){var X=b.top+(_+M)*R;H.append("g").attr("class","axis").attr("transform","translate(0,"+X+")").call(Y),T&&H.append("g").attr("class","axis-hour").attr("transform","translate(0,"+(X+20)+")").call(G)}S&&H.append("g").attr("class","axis axis-tick").attr("transform","translate(0,"+(b.top+(_+M)*R)+")").attr(E.stroke,E.spacing).call(Y.tickFormat("").tickSize(-(b.top+(_+M)*(R-1)+3),0,0)),H.each(function(a,u){a.forEach(function(a,c){function h(t,e){return x?b.top+(_+M)*W[c]:b.top}function m(t,e){return x?b.top+(_+M)*W[c]+.65*_:b.top+.65*_}function y(t,e){return x?b.top+(_+M)*W[c]+_-3:b.top+_-3}var w=a.times,k="undefined"!=typeof a.label,C=function(t){return null==s?t:s(t)};if("undefined"!=typeof a.id&&console.warn("d3Timeline Warning: Ids per dataset is deprecated in favor of a 'class' key. Ids are now per data element."),d){var S=(_+M)*W[c];H.selectAll("svg").data(w).enter().insert("rect").attr("class","row-green-bar").attr("x",0+b.left).attr("width",l-b.right-b.left).attr("y",S).attr("height",_).attr("fill",d)}var E=H.selectAll("svg").data(w).enter().append("g").attr("class",function(t,e){return"bar-container bar-type-"+t.type}).attr("width",I);"scheduled"!=w[0].type&&E.append("svg:clipPath").attr("id",D+"-timeline-textclip-"+u+"-"+c).attr("class","timeline-clip").append("svg:rect").attr("clipPathUnits","objectBoundingBox").attr("x",O).attr("y",h).attr("width",L).attr("height",_);var T=E.append(function(t,e){return document.createElementNS(d3.ns.prefix.svg,"display"in t?t.display:v)}).attr("x",O).attr("y",h).attr("rx",5).attr("ry",5).attr("width",I).attr("cy",function(t,e){return h(t,e)+_/2}).attr("cx",O).attr("r",_/2).attr("height",_).style("stroke",function(t,e){return t.borderColor}).style("stroke-width",1).style("fill",function(t,e){var n;return t.color?t.color:g?(n=t[g],p(n?n:a[g])):p(c)}).on("mousemove",function(t,e){n(t,c,a)}).on("mouseover",function(t,e){r(t,e,a)}).on("mouseout",function(t,e){i(t,e,a)}).on("click",function(t,e){o(t,c,a)}).attr("class",function(t,e){return a["class"]?"timeline-series timelineSeries_"+a["class"]:"timeline-series timelineSeries_"+c}).attr("id",function(t,e){return a.id&&!t.id?"timelineItem_"+a.id:t.id?t.id:"timelineItem_"+c+"_"+e}),P=E.append("text").attr("class","timeline-insidelabel").attr("x",j).attr("y",m).attr("height",_).attr("clip-path","url(#"+D+"-timeline-textclip-"+u+"-"+c+")").text(function(t){return t.label}).on("click",function(t,e){o(t,c,a)});if("scheduled"==w[0].type&&T.attr("width",P.node().getComputedTextLength()+10),H.selectAll("svg .bar-container").each(function(t,e){$(this).qtip({content:{text:t.label},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}),f){var F=_+M/2+b.top+(_+M)*W[c];t.append("svg:line").attr("class","row-seperator").attr("x1",0+b.left).attr("x2",l-b.right).attr("y1",F).attr("y2",F).attr("stroke-width",1).attr("stroke",f)}A&&"scheduled"==w[0].type&&H.selectAll("svg").data(w).enter().append("svg:line").attr("class","line-start").attr("x1",e).attr("y1",y).attr("x2",e).attr("y2",b.top+(_+M)*R).style("stroke",function(t,e){return t.color}).style("stroke-width",N.width),k&&t.append("text").attr("class","timeline-label").attr("transform","translate(0,"+(.75*_+b.top+(_+M)*W[c])+")").text(k?C(a.label):a.id).on("click",function(t,e){o(t,c,a)}),"undefined"!=typeof a.icon&&t.append("image").attr("class","timeline-label").attr("transform","translate(0,"+(b.top+(_+M)*W[c])+")").attr("xlink:href",a.icon).attr("width",b.left).attr("height",_)})});var Z=function(){$(".qtip.qtip-timeline-bar").qtip("hide"),H.selectAll(".bar-type-scheduled .timeline-series").attr("x",O),H.selectAll(".bar-type-regular .timeline-series").attr("x",O).attr("width",I),H.selectAll(".timeline-insidelabel").attr("x",j),H.selectAll(".bar-type-scheduled .timeline-clip").select("rect").attr("x",O),H.selectAll(".bar-type-regular .timeline-clip").select("rect").attr("x",O).attr("width",L),H.selectAll("g.axis").call(Y),T&&H.selectAll("g.axis-hour").call(G),A&&(H.selectAll("line.line-start").attr("x1",e).attr("x2",e),H.selectAll("line.line-end").attr("x1",a).attr("x2",a))},Q=d3.behavior.zoom().x(U).on("zoom",Z);t.call(Q),w&&H.selectAll(".tick text").attr("transform",function(t){return"rotate("+w+")translate("+(this.getBBox().width/2+10)+","+this.getBBox().height/2+")"});var K=H[0][0].getBoundingClientRect();P(),bbox=H[0][0].getBBox(),t.attr("height",bbox.height+40)}var e=["circle","rect"],n=function(){},r=function(){},i=function(){},o=function(){},a=function(){},s=function(){},u="bottom",l=null,c=null,f=null,d=null,h={format:d3.time.format("%I %p"),tickTime:d3.time.hours,tickInterval:1,tickSize:6},p=d3.scale.category20(),g=null,v="rect",m=0,y=0,b={left:30,right:30,top:30,bottom:30},x=!1,w=!1,k=!1,_=20,M=5,C=!0,S=!1,E={stroke:"stroke-dasharray",spacing:"4 10"},A=!1,T=!1,N={marginTop:25,marginBottom:0,width:1,color:p},D="timeline";return t.margin=function(e){return arguments.length?(b=e,t):b},t.orient=function(e){return arguments.length?(u=e,t):u},t.itemHeight=function(e){return arguments.length?(_=e,t):_},t.itemMargin=function(e){return arguments.length?(M=e,t):M},t.height=function(e){return arguments.length?(c=e,t):c},t.width=function(e){return arguments.length?(l=e,t):l},t.display=function(n){return arguments.length&&e.indexOf(n)!=-1?(v=n,t):v},t.labelFormat=function(e){return arguments.length?(s=e,t):null},t.tickFormat=function(e){return arguments.length?(h=e,t):h},t.prefix=function(e){return arguments.length?(D=e,t):D},t.hover=function(e){return arguments.length?(n=e,t):n},t.mouseover=function(e){return arguments.length?(r=e,t):e},t.mouseout=function(e){return arguments.length?(i=e,t):e},t.click=function(e){return arguments.length?(o=e,t):o},t.scroll=function(e){return arguments.length?(a=e,t):a},t.colors=function(e){return arguments.length?(p=e,t):p},t.beginning=function(e){return arguments.length?(m=e,t):m},t.ending=function(e){return arguments.length?(y=e,t):y},t.rotateTicks=function(e){return w=e,t},t.stack=function(){return x=!x,t},t.relativeTime=function(){return k=!k,t},t.showBorderLine=function(){return A=!A,t},t.showHourTimeline=function(){return T=!T,t},t.showBorderFormat=function(e){return arguments.length?(N=e,t):N},t.colorProperty=function(e){return arguments.length?(g=e,t):g},t.rowSeperators=function(e){return arguments.length?(f=e,t):f},t.background=function(e){return arguments.length?(d=e,t):d},t.showTimeAxis=function(){return C=!C,t},t.showTimeAxisTick=function(){return S=!S,t},t.showTimeAxisTickFormat=function(e){return arguments.length?(E=e,t):E},t}}();
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html b/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html
index 26a1e65de3759..daaf7c4269120 100644
--- a/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html
+++ b/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html
@@ -39,7 +39,7 @@
 </div>
 <ul ng-if="nodeid &amp;&amp; metrics.length &gt; 0" dnd-list="metrics" dnd-drop="dropped(event, index, item, external, type, external)" class="metric-row">
   <li ng-repeat="metric in metrics track by metric.id" dnd-draggable="metric" dnd-dragstart="dragStart()" dnd-dragend="dragEnd()" dnd-canceled="dragEnd()" ng-class="{big: metric.size != 'small'}" class="metric-col">
-    <metrics-graph metric="metric" window="window" get-values="getValues(metric.id)" remove-metric="removeMetric(metric)" set-metric-size="setMetricSize"></metrics-graph>
+    <metrics-graph metric="metric" window="window" get-values="getValues(metric.id)" remove-metric="removeMetric(metric)" set-metric-size="setMetricSize" set-metric-view="setMetricView"></metrics-graph>
   </li>
 </ul>
 <div class="clearfix"></div>
\ No newline at end of file

From 186871dad1889903459a1f1c97edd8afcd4a5844 Mon Sep 17 00:00:00 2001
From: paul <nellboy@gmail.com>
Date: Mon, 20 Feb 2017 10:28:14 +0100
Subject: [PATCH 209/253] [FLINK-5831] [webui] order, search and filter metrics

This closes #3369
---
 .../jobs/job.plan.node-list.metrics.jade       |  7 ++++++-
 .../app/scripts/common/filters.coffee          |  6 ++++++
 .../app/scripts/modules/jobs/jobs.ctrl.coffee  | 12 +++++++++++-
 .../web-dashboard/app/styles/metric.styl       | 18 ++++++++++++++++--
 .../web-dashboard/web/css/index.css            |  2 +-
 .../web-dashboard/web/js/hs/index.js           |  4 ++--
 .../web-dashboard/web/js/index.js              |  4 ++--
 .../jobs/job.plan.node-list.metrics.html       |  6 +++++-
 8 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade
index f3ec8dc021733..a1d22e3701994 100644
--- a/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade
+++ b/flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node-list.metrics.jade
@@ -28,7 +28,12 @@ nav.navbar.navbar-default.navbar-secondary-additional.navbar-secondary-additiona
       | &nbsp;
       span.caret
     ul.dropdown-menu.dropdown-menu-right.metric-menu
-      li(ng-repeat="metric in availableMetrics track by $index")
+      section(class="search")
+        label(for="search-input")
+          i(class="fa fa-search" aria-hidden="true")
+            span(class="sr-only") Search icons
+        input(type="text" ng-model="metricsFilterQuery" class="metrics-filter" placeholder="Search available metrics...")
+      li(ng-repeat="metric in availableMetrics | searchMetrics:metricsFilterQuery track by $index")
         a(ng-click="addMetric(metric)") {{ metric.id | limit }}
 
   .dropdown.add-metrics(ng-if="!availableMetrics.length")
diff --git a/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee b/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee
index e2b83391e8975..211139f400640 100644
--- a/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee
+++ b/flink-runtime-web/web-dashboard/app/scripts/common/filters.coffee
@@ -132,3 +132,9 @@ angular.module('flinkApp')
         return_val = value
     return return_val
 ]
+
+.filter "searchMetrics", ->
+  (availableMetrics, query)->
+    queryRegex = new RegExp(query, "gi")
+    return (metric for metric in availableMetrics when metric.id.match(queryRegex))
+
diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee
index d4315ea2499b4..ebe8e98c4c91a 100644
--- a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee
+++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee
@@ -353,13 +353,23 @@ angular.module('flinkApp')
       $scope.vertex = data
 
     MetricsService.getAvailableMetrics($scope.jobid, $scope.nodeid).then (data) ->
-      $scope.availableMetrics = data
+      $scope.availableMetrics = data.sort(alphabeticalSortById)
       $scope.metrics = MetricsService.getMetricsSetup($scope.jobid, $scope.nodeid).names
 
       MetricsService.registerObserver($scope.jobid, $scope.nodeid, (data) ->
         $scope.$broadcast "metrics:data:update", data.timestamp, data.values
       )
 
+  alphabeticalSortById = (a, b) ->
+    A = a.id.toLowerCase()
+    B = b.id.toLowerCase()
+    if A < B
+      return -1
+    else if A > B
+      return 1
+    else
+      return 0
+
   $scope.dropped = (event, index, item, external, type) ->
 
     MetricsService.orderMetrics($scope.jobid, $scope.nodeid, item, index)
diff --git a/flink-runtime-web/web-dashboard/app/styles/metric.styl b/flink-runtime-web/web-dashboard/app/styles/metric.styl
index 94c494838a9bf..ec0afaf047045 100644
--- a/flink-runtime-web/web-dashboard/app/styles/metric.styl
+++ b/flink-runtime-web/web-dashboard/app/styles/metric.styl
@@ -38,9 +38,23 @@
 
       .metric-menu
         max-height: 300px
-        max-width: 900px
+        width: 300px
         overflow-y: scroll
-        text-align: right
+        text-align: left
+
+  .search
+    label
+      position: absolute;
+      top: 15px
+      left: 22px
+    input.metrics-filter
+      height: 30px
+      padding: 5px 5px 5px 25px
+      margin: 5px 5px 5px 15px
+      width: 90%
+      border-radius: 5px
+      border: 1px solid #999
+      box-shadow: none
 
 $metric-row-height = 180px + 85px
 
diff --git a/flink-runtime-web/web-dashboard/web/css/index.css b/flink-runtime-web/web-dashboard/web/css/index.css
index db615b1fb0081..64297368deb15 100644
--- a/flink-runtime-web/web-dashboard/web/css/index.css
+++ b/flink-runtime-web/web-dashboard/web/css/index.css
@@ -1 +1 @@
-#main,#sidebar,body,html{height:100%}#content,#sidebar{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s}.gutter{background-color:transparent;background-repeat:no-repeat;background-position:50%}.gutter-vertical{cursor:row-resize;background-image:url(../images/grips/horizontal.png)}#sidebar{overflow:hidden;position:fixed;left:-250px;top:0;bottom:0;width:250px;background:#151515;transition:.4s;-webkit-box-shadow:inset -10px 0 10px rgba(0,0,0,.2);box-shadow:inset -10px 0 10px rgba(0,0,0,.2)}#sidebar.sidebar-visible{left:0}#sidebar .logo{width:auto;height:22px}#sidebar .logo img{display:inline-block}#sidebar .navbar-static-top{overflow:hidden;height:51px}#sidebar .navbar-static-top .navbar-header{width:100%}#sidebar .navbar-brand.navbar-brand-text{font-size:14px;font-weight:700;color:#fff;padding-left:0}#sidebar .nav>li>a{color:#aaa;margin-bottom:1px}#sidebar .nav>li>a:focus,#sidebar .nav>li>a:hover{background-color:rgba(40,40,40,.5)}#sidebar .nav>li>a.active{background-color:rgba(100,100,100,.5)}#content{background-color:#fff;margin-left:0;padding-top:70px;height:100%;transition:.4s}.table .table,.table.table-inner{background-color:transparent}#content .navbar-main,#content .navbar-main-additional{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s;transition:.4s}#content .navbar-main-additional{margin-top:51px;border-bottom:none;padding:0 20px}#content .navbar-main-additional .nav-tabs{margin:0 -20px;padding:0 20px}#content .navbar-secondary-additional{border:none;padding:0 20px;margin-bottom:0}#content .navbar-secondary-additional .nav-tabs{margin:0 -20px}#content.sidebar-visible{margin-left:250px}#content.sidebar-visible .navbar-main,#content.sidebar-visible .navbar-main-additional{left:250px}#content #fold-button{display:inline-block;margin-left:20px}#content #content-inner{padding:0 20px 20px}#content #content-inner.has-navbar-main-additional{padding-top:42px}.table#add-file-table span.btn,.table#job-submit-table td>span.btn{padding:2px 4px;font-size:14px}.page-header{margin:0 0 20px}.nav>li>a,.nav>li>a:focus,.nav>li>a:hover{color:#aaa;background-color:transparent;border-bottom:2px solid transparent}.nav>li.active>a,.nav>li.active>a:focus,.nav>li.active>a:hover{color:#000;border-bottom:2px solid #000}.nav.nav-tabs{margin-bottom:20px}.table th{font-weight:400;color:#999}.table td.td-long{width:20%;white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.table.table-clickable tr{cursor:pointer}.table.table-properties{table-layout:fixed;white-space:nowrap}.table.table-properties td{width:50%;white-space:nowrap;overflow:hidden;-o-text-overflow:ellipsis;text-overflow:ellipsis}.table.table-body-hover>tbody{border-top:none;border-left:2px solid transparent}.table.table-body-hover>tbody.active{border-left:2px solid #000}.table.table-body-hover>tbody.active td.tab-column li.active,.table.table-body-hover>tbody.active td:not(.tab-column),.table.table-body-hover>tbody:hover td.tab-column li.active,.table.table-body-hover>tbody:hover td:not(.tab-column){background-color:#f0f0f0}.table.table-activable td.tab-column,.table.table-activable th.tab-column{border-top:none;width:47px}.table.table-activable td.tab-column{border-right:1px solid #ddd}.table.table-activable td{position:relative}.table.table-no-border td,.table.table-no-border th{border-top:none!important}.table#job-submit-table{table-layout:fixed;white-space:nowrap}.table#job-submit-table td.td-large{width:40%}.table#job-submit-table td{width:15%}.table#job-submit-table td>input{height:28px;font-size:14px}.table#add-file-table{table-layout:fixed}.table#add-file-table span.btn{position:relative;overflow:hidden;border-radius:2px;margin-top:-3px}.table#add-file-table td#add-file-button{width:100px}.table#add-file-table td#add-file-button input[type=file]{position:absolute;top:0;right:0;min-width:100%;min-height:100%;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);outline:0;cursor:inherit;display:block}.timeline-canvas .timeline-insidelabel,.timeline-canvas .timeline-series,svg.graph .node{cursor:pointer}.table#add-file-table td#add-file-name{width:250px;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;white-space:nowrap}.table#add-file-table td#add-file-status{width:100%}.table#add-file-table td#add-file-status span.btn-progress-bar{padding:0!important;width:100%;background-color:#f5f5f5;text-align:left}.table#add-file-table td#add-file-status span.btn-progress{padding:2px;font-size:10px}.table span.error-area{color:red}.table span.row-button{padding:1px 2px;margin:0;border:none!important}.table .small-label{text-transform:uppercase;font-size:13px;color:#999}span.icon-wrapper{width:1.2em;display:inline-block}.panel.panel-dashboard .huge{font-size:28px}.panel.panel-lg{font-size:16px}.panel.panel-lg .badge{font-size:14px}.navbar-secondary{overflow:auto}.navbar-main .navbar-title,.navbar-main .navbar-title-job,.navbar-main .panel-title,.navbar-main-additional .navbar-title,.navbar-main-additional .navbar-title-job,.navbar-main-additional .panel-title,.navbar-secondary .navbar-title,.navbar-secondary .navbar-title-job,.navbar-secondary .panel-title,.navbar-secondary-additional .navbar-title,.navbar-secondary-additional .navbar-title-job,.navbar-secondary-additional .panel-title,.panel.panel-multi .navbar-title,.panel.panel-multi .navbar-title-job,.panel.panel-multi .panel-title{float:left;font-size:18px;padding:12px 20px 13px 10px;color:#333;display:inline-block}.navbar-main .navbar-info,.navbar-main .panel-info,.navbar-main-additional .navbar-info,.navbar-main-additional .panel-info,.navbar-secondary .navbar-info,.navbar-secondary .panel-info,.navbar-secondary-additional .navbar-info,.navbar-secondary-additional .panel-info,.panel.panel-multi .navbar-info,.panel.panel-multi .panel-info{float:left;font-size:14px;padding:15px;color:#999;display:inline-block;border-right:1px solid #e7e7e7;overflow:hidden}.navbar-main .navbar-info .overflow,.navbar-main .panel-info .overflow,.navbar-main-additional .navbar-info .overflow,.navbar-main-additional .panel-info .overflow,.navbar-secondary .navbar-info .overflow,.navbar-secondary .panel-info .overflow,.navbar-secondary-additional .navbar-info .overflow,.navbar-secondary-additional .panel-info .overflow,.panel.panel-multi .navbar-info .overflow,.panel.panel-multi .panel-info .overflow{position:absolute;display:block;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;height:22px;line-height:22px;vertical-align:middle}.navbar-main .navbar-info.first,.navbar-main .panel-info.first,.navbar-main-additional .navbar-info.first,.navbar-main-additional .panel-info.first,.navbar-secondary .navbar-info.first,.navbar-secondary .panel-info.first,.navbar-secondary-additional .navbar-info.first,.navbar-secondary-additional .panel-info.first,.panel.panel-multi .navbar-info.first,.panel.panel-multi .panel-info.first{border-left:1px solid #e7e7e7}.navbar-main .navbar-info.last,.navbar-main .panel-info.last,.navbar-main-additional .navbar-info.last,.navbar-main-additional .panel-info.last,.navbar-secondary .navbar-info.last,.navbar-secondary .panel-info.last,.navbar-secondary-additional .navbar-info.last,.navbar-secondary-additional .panel-info.last,.panel.panel-multi .navbar-info.last,.panel.panel-multi .panel-info.last{border-right:none}.panel.panel-multi .panel-heading{padding:0}.panel.panel-multi .panel-heading .panel-info.thin{padding:8px 10px}.panel.panel-multi .panel-body{padding:10px;background-color:#fdfdfd;color:#999;font-size:13px}.panel.panel-multi .panel-body.clean{color:inherit;font-size:inherit}.navbar-main-additional,.navbar-secondary-additional{min-height:40px;background-color:#fdfdfd}.navbar-main-additional .navbar-info,.navbar-secondary-additional .navbar-info{font-size:13px;padding:10px 15px}.nav-top-affix.affix{width:100%;top:50px;margin-left:-20px;padding-left:20px;margin-right:-20px;padding-right:20px;background-color:#fff;z-index:1}.badge-default[href]:focus,.badge-default[href]:hover{background-color:grey}.badge-primary{background-color:#428bca}.badge-primary[href]:focus,.badge-primary[href]:hover{background-color:#3071a9}.badge-success{background-color:#5cb85c}.badge-success[href]:focus,.badge-success[href]:hover{background-color:#449d44}.badge-info{background-color:#5bc0de}.badge-info[href]:focus,.badge-info[href]:hover{background-color:#31b0d5}.badge-warning{background-color:#f0ad4e}.badge-warning[href]:focus,.badge-warning[href]:hover{background-color:#ec971f}.badge-danger{background-color:#d9534f}.badge-danger[href]:focus,.badge-danger[href]:hover{background-color:#c9302c}.indicator{display:inline-block;margin-right:15px}.indicator.indicator-primary{color:#428bca}.indicator.indicator-success{color:#5cb85c}.indicator.indicator-info{color:#5bc0de}.indicator.indicator-warning{color:#f0ad4e}.indicator.indicator-danger{color:#d9534f}pre.exception{border:none;background-color:transparent;padding:0;margin:0}pre{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.nav-tabs.tabs-vertical{position:absolute;left:0;top:0;border-bottom:none;z-index:100}.nav-tabs.tabs-vertical li{float:none;margin-bottom:0;margin-right:-1px}.nav-tabs.tabs-vertical li>a{margin-right:0;border-radius:0;border-bottom:none;border-left:2px solid transparent}.nav-tabs.tabs-vertical li.active>a,.nav-tabs.tabs-vertical li>a:focus,.nav-tabs.tabs-vertical li>a:hover{border-bottom:none;border-left:2px solid #000}.navbar-main .navbar-title,.navbar-main-additional .navbar-title,.navbar-secondary .navbar-title,.navbar-secondary-additional .navbar-title{padding:12px 20px 13px}.navbar-main .navbar-title-job{padding:8px 20px}.navbar-main .navbar-title-job .indicator-primary{padding:8px 0 0}.navbar-main .navbar-title-job .no-padding{padding:0}.navbar-main .navbar-title-job .no-margin{margin:0}.navbar-main .navbar-title-job .job-name{font-size:14px}.navbar-main .navbar-title-job .job-id{color:#999;font-size:11px}.checkpoint-overview a,svg.graph .node h4{color:#000}livechart{width:30%;height:30%;text-align:center}.canvas-wrapper{border:1px solid #ddd;position:relative;margin-bottom:20px;height:100%}.canvas-wrapper .main-canvas{height:100%;overflow:hidden}.canvas-wrapper .main-canvas .zoom-buttons{position:absolute;top:10px;right:10px}.label-group .label{display:inline-block;padding-left:.4em;padding-right:.4em;margin:0;border-right:1px solid #fff;border-radius:0}.label-group .label.label-black{background-color:#000}.navbar-info-button{padding:3px 4px;font-size:12px;font-family:inherit;margin-top:-2px}svg.graph .edge-label,svg.graph text{font-size:14px}.checkpoints-view{padding-top:1em}.subtask-details .blank{height:2em}.checkpoint-overview td span{padding-left:2em}svg.graph{overflow:hidden;height:100%;width:100%!important}svg.graph g.type-TK>rect{fill:#00ffd0}svg.graph text{font-weight:300}svg.graph .node>rect{stroke:#999;stroke-width:5px;fill:#fff;margin:0;padding:0}svg.graph .node[active]>rect{fill:#eee}svg.graph .node.node-mirror>rect{stroke:#a8a8a8}svg.graph .node.node-iteration>rect{stroke:#cd3333}svg.graph .node.node-source>rect{stroke:#4ce199}svg.graph .node.node-sink>rect{stroke:#e6ec8b}svg.graph .node.node-normal>rect{stroke:#3fb6d8}svg.graph .node h5{color:#999}svg.graph .edgeLabel rect{fill:#fff}svg.graph .edgePath path{stroke:#333;stroke-width:2px;fill:#333}svg.graph .label{color:#777;margin:0}svg.graph .node-label{display:block;margin:0;text-decoration:none}.timeline{overflow:hidden}.timeline-canvas{overflow:hidden;padding:10px}.timeline-canvas .bar-container{overflow:hidden}.timeline-canvas.secondary .timeline-insidelabel,.timeline-canvas.secondary .timeline-series{cursor:auto}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics a,.show-pointer{cursor:pointer}.qtip-timeline-bar{font-size:14px;line-height:1.4}#content .navbar-secondary-additional.navbar-secondary-additional-2{margin:-10px -10px 10px;padding:0;border-bottom:1px solid #e4e4e4}#content .navbar-secondary-additional.navbar-secondary-additional-2 .navbar-info{padding-top:12px;padding-bottom:12px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics{margin-right:15px;float:right}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics .btn{margin-top:5px;margin-bottom:5px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .metric-menu{max-height:300px;max-width:900px;overflow-y:scroll;text-align:right}.metric-row{margin:0;min-height:275px;padding:0;list-style-type:none}.metric-row .metric-col{background-color:transparent;width:33.33%;float:left}.metric-row .metric-col.big{width:100%}.metric-row .metric-col .panel{margin-left:5px;margin-right:5px;min-height:265px;margin-bottom:10px}.metric-row .metric-col .panel .panel-body{background-color:transparent;height:265px;position:relative}.metric-row .metric-col .panel .panel-body .metric-numeric{text-align:center;margin-top:75px;font-size:40px;font-weight:700}.metric-row .metric-col .panel .panel-heading{padding:0 10px;background-color:transparent;height:41px;line-height:41px;position:relative;overflow:hidden;cursor:pointer}.metric-row .metric-col .panel .panel-heading .metric-title{padding:10px 0}.metric-row .metric-col .panel .panel-heading .buttons{position:absolute;top:0;right:0;padding:0 10px;background-color:#fff}.metric-row .metric-col.dndDraggingSource{display:none}.metric-row .dndPlaceholder{position:relative;background-color:#f0f0f0;min-height:305px;display:block;width:33.33%;float:left;margin-bottom:10px;border-radius:5px}.p-info{padding-left:5px;padding-right:5px}@media (min-width:1024px) and (max-width:1279px){#content #fold-button,#sidebar .navbar-static-top .navbar-brand-text{display:none}#sidebar{left:0;width:160px}#content{margin-left:160px}#content .navbar-main,#content .navbar-main-additional{left:160px}.table td.td-long{width:20%}}@media (min-width:1280px){#sidebar{left:0}#content{margin-left:250px}#content #fold-button{display:none}#content .navbar-main,#content .navbar-main-additional{left:250px}.table td.td-long{width:30%}}.legend-box{font-size:10px;width:2em}#total-mem{background-color:#7cb5ec}#heap-mem{background-color:#434348}#non-heap-mem{background-color:#90ed7d}#fetch-plan,#job-submit{width:100px}#content-inner,#details,#node-details{height:100%}#job-panel{overflow-y:auto}
\ No newline at end of file
+#main,#sidebar,body,html{height:100%}#content,#sidebar{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s}.gutter{background-color:transparent;background-repeat:no-repeat;background-position:50%}.gutter-vertical{cursor:row-resize;background-image:url(../images/grips/horizontal.png)}#sidebar{overflow:hidden;position:fixed;left:-250px;top:0;bottom:0;width:250px;background:#151515;transition:.4s;-webkit-box-shadow:inset -10px 0 10px rgba(0,0,0,.2);box-shadow:inset -10px 0 10px rgba(0,0,0,.2)}#sidebar.sidebar-visible{left:0}#sidebar .logo{width:auto;height:22px}#sidebar .logo img{display:inline-block}#sidebar .navbar-static-top{overflow:hidden;height:51px}#sidebar .navbar-static-top .navbar-header{width:100%}#sidebar .navbar-brand.navbar-brand-text{font-size:14px;font-weight:700;color:#fff;padding-left:0}#sidebar .nav>li>a{color:#aaa;margin-bottom:1px}#sidebar .nav>li>a:focus,#sidebar .nav>li>a:hover{background-color:rgba(40,40,40,.5)}#sidebar .nav>li>a.active{background-color:rgba(100,100,100,.5)}#content{background-color:#fff;margin-left:0;padding-top:70px;height:100%;transition:.4s}.table .table,.table.table-inner{background-color:transparent}#content .navbar-main,#content .navbar-main-additional{-webkit-transition:.4s;-moz-transition:.4s;-o-transition:.4s;-ms-transition:.4s;transition:.4s}#content .navbar-main-additional{margin-top:51px;border-bottom:none;padding:0 20px}#content .navbar-main-additional .nav-tabs{margin:0 -20px;padding:0 20px}#content .navbar-secondary-additional{border:none;padding:0 20px;margin-bottom:0}#content .navbar-secondary-additional .nav-tabs{margin:0 -20px}#content.sidebar-visible{margin-left:250px}#content.sidebar-visible .navbar-main,#content.sidebar-visible .navbar-main-additional{left:250px}#content #fold-button{display:inline-block;margin-left:20px}#content #content-inner{padding:0 20px 20px}#content #content-inner.has-navbar-main-additional{padding-top:42px}.table#add-file-table span.btn,.table#job-submit-table td>span.btn{padding:2px 4px;font-size:14px}.page-header{margin:0 0 20px}.nav>li>a,.nav>li>a:focus,.nav>li>a:hover{color:#aaa;background-color:transparent;border-bottom:2px solid transparent}.nav>li.active>a,.nav>li.active>a:focus,.nav>li.active>a:hover{color:#000;border-bottom:2px solid #000}.nav.nav-tabs{margin-bottom:20px}.table th{font-weight:400;color:#999}.table td.td-long{width:20%;white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.table.table-clickable tr{cursor:pointer}.table.table-properties{table-layout:fixed;white-space:nowrap}.table.table-properties td{width:50%;white-space:nowrap;overflow:hidden;-o-text-overflow:ellipsis;text-overflow:ellipsis}.table.table-body-hover>tbody{border-top:none;border-left:2px solid transparent}.table.table-body-hover>tbody.active{border-left:2px solid #000}.table.table-body-hover>tbody.active td.tab-column li.active,.table.table-body-hover>tbody.active td:not(.tab-column),.table.table-body-hover>tbody:hover td.tab-column li.active,.table.table-body-hover>tbody:hover td:not(.tab-column){background-color:#f0f0f0}.table.table-activable td.tab-column,.table.table-activable th.tab-column{border-top:none;width:47px}.table.table-activable td.tab-column{border-right:1px solid #ddd}.table.table-activable td{position:relative}.table.table-no-border td,.table.table-no-border th{border-top:none!important}.table#job-submit-table{table-layout:fixed;white-space:nowrap}.table#job-submit-table td.td-large{width:40%}.table#job-submit-table td{width:15%}.table#job-submit-table td>input{height:28px;font-size:14px}.table#add-file-table{table-layout:fixed}.table#add-file-table span.btn{position:relative;overflow:hidden;border-radius:2px;margin-top:-3px}.table#add-file-table td#add-file-button{width:100px}.table#add-file-table td#add-file-button input[type=file]{position:absolute;top:0;right:0;min-width:100%;min-height:100%;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);outline:0;cursor:inherit;display:block}.timeline-canvas .timeline-insidelabel,.timeline-canvas .timeline-series,svg.graph .node{cursor:pointer}.table#add-file-table td#add-file-name{width:250px;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;white-space:nowrap}.table#add-file-table td#add-file-status{width:100%}.table#add-file-table td#add-file-status span.btn-progress-bar{padding:0!important;width:100%;background-color:#f5f5f5;text-align:left}.table#add-file-table td#add-file-status span.btn-progress{padding:2px;font-size:10px}.table span.error-area{color:red}.table span.row-button{padding:1px 2px;margin:0;border:none!important}.table .small-label{text-transform:uppercase;font-size:13px;color:#999}span.icon-wrapper{width:1.2em;display:inline-block}.panel.panel-dashboard .huge{font-size:28px}.panel.panel-lg{font-size:16px}.panel.panel-lg .badge{font-size:14px}.navbar-secondary{overflow:auto}.navbar-main .navbar-title,.navbar-main .navbar-title-job,.navbar-main .panel-title,.navbar-main-additional .navbar-title,.navbar-main-additional .navbar-title-job,.navbar-main-additional .panel-title,.navbar-secondary .navbar-title,.navbar-secondary .navbar-title-job,.navbar-secondary .panel-title,.navbar-secondary-additional .navbar-title,.navbar-secondary-additional .navbar-title-job,.navbar-secondary-additional .panel-title,.panel.panel-multi .navbar-title,.panel.panel-multi .navbar-title-job,.panel.panel-multi .panel-title{float:left;font-size:18px;padding:12px 20px 13px 10px;color:#333;display:inline-block}.navbar-main .navbar-info,.navbar-main .panel-info,.navbar-main-additional .navbar-info,.navbar-main-additional .panel-info,.navbar-secondary .navbar-info,.navbar-secondary .panel-info,.navbar-secondary-additional .navbar-info,.navbar-secondary-additional .panel-info,.panel.panel-multi .navbar-info,.panel.panel-multi .panel-info{float:left;font-size:14px;padding:15px;color:#999;display:inline-block;border-right:1px solid #e7e7e7;overflow:hidden}.navbar-main .navbar-info .overflow,.navbar-main .panel-info .overflow,.navbar-main-additional .navbar-info .overflow,.navbar-main-additional .panel-info .overflow,.navbar-secondary .navbar-info .overflow,.navbar-secondary .panel-info .overflow,.navbar-secondary-additional .navbar-info .overflow,.navbar-secondary-additional .panel-info .overflow,.panel.panel-multi .navbar-info .overflow,.panel.panel-multi .panel-info .overflow{position:absolute;display:block;-o-text-overflow:ellipsis;text-overflow:ellipsis;overflow:hidden;height:22px;line-height:22px;vertical-align:middle}.navbar-main .navbar-info.first,.navbar-main .panel-info.first,.navbar-main-additional .navbar-info.first,.navbar-main-additional .panel-info.first,.navbar-secondary .navbar-info.first,.navbar-secondary .panel-info.first,.navbar-secondary-additional .navbar-info.first,.navbar-secondary-additional .panel-info.first,.panel.panel-multi .navbar-info.first,.panel.panel-multi .panel-info.first{border-left:1px solid #e7e7e7}.navbar-main .navbar-info.last,.navbar-main .panel-info.last,.navbar-main-additional .navbar-info.last,.navbar-main-additional .panel-info.last,.navbar-secondary .navbar-info.last,.navbar-secondary .panel-info.last,.navbar-secondary-additional .navbar-info.last,.navbar-secondary-additional .panel-info.last,.panel.panel-multi .navbar-info.last,.panel.panel-multi .panel-info.last{border-right:none}.panel.panel-multi .panel-heading{padding:0}.panel.panel-multi .panel-heading .panel-info.thin{padding:8px 10px}.panel.panel-multi .panel-body{padding:10px;background-color:#fdfdfd;color:#999;font-size:13px}.panel.panel-multi .panel-body.clean{color:inherit;font-size:inherit}.navbar-main-additional,.navbar-secondary-additional{min-height:40px;background-color:#fdfdfd}.navbar-main-additional .navbar-info,.navbar-secondary-additional .navbar-info{font-size:13px;padding:10px 15px}.nav-top-affix.affix{width:100%;top:50px;margin-left:-20px;padding-left:20px;margin-right:-20px;padding-right:20px;background-color:#fff;z-index:1}.badge-default[href]:focus,.badge-default[href]:hover{background-color:grey}.badge-primary{background-color:#428bca}.badge-primary[href]:focus,.badge-primary[href]:hover{background-color:#3071a9}.badge-success{background-color:#5cb85c}.badge-success[href]:focus,.badge-success[href]:hover{background-color:#449d44}.badge-info{background-color:#5bc0de}.badge-info[href]:focus,.badge-info[href]:hover{background-color:#31b0d5}.badge-warning{background-color:#f0ad4e}.badge-warning[href]:focus,.badge-warning[href]:hover{background-color:#ec971f}.badge-danger{background-color:#d9534f}.badge-danger[href]:focus,.badge-danger[href]:hover{background-color:#c9302c}.indicator{display:inline-block;margin-right:15px}.indicator.indicator-primary{color:#428bca}.indicator.indicator-success{color:#5cb85c}.indicator.indicator-info{color:#5bc0de}.indicator.indicator-warning{color:#f0ad4e}.indicator.indicator-danger{color:#d9534f}pre.exception{border:none;background-color:transparent;padding:0;margin:0}pre{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.nav-tabs.tabs-vertical{position:absolute;left:0;top:0;border-bottom:none;z-index:100}.nav-tabs.tabs-vertical li{float:none;margin-bottom:0;margin-right:-1px}.nav-tabs.tabs-vertical li>a{margin-right:0;border-radius:0;border-bottom:none;border-left:2px solid transparent}.nav-tabs.tabs-vertical li.active>a,.nav-tabs.tabs-vertical li>a:focus,.nav-tabs.tabs-vertical li>a:hover{border-bottom:none;border-left:2px solid #000}.navbar-main .navbar-title,.navbar-main-additional .navbar-title,.navbar-secondary .navbar-title,.navbar-secondary-additional .navbar-title{padding:12px 20px 13px}.navbar-main .navbar-title-job{padding:8px 20px}.navbar-main .navbar-title-job .indicator-primary{padding:8px 0 0}.navbar-main .navbar-title-job .no-padding{padding:0}.navbar-main .navbar-title-job .no-margin{margin:0}.navbar-main .navbar-title-job .job-name{font-size:14px}.navbar-main .navbar-title-job .job-id{color:#999;font-size:11px}.checkpoint-overview a,svg.graph .node h4{color:#000}livechart{width:30%;height:30%;text-align:center}.canvas-wrapper{border:1px solid #ddd;position:relative;margin-bottom:20px;height:100%}.canvas-wrapper .main-canvas{height:100%;overflow:hidden}.canvas-wrapper .main-canvas .zoom-buttons{position:absolute;top:10px;right:10px}.label-group .label{display:inline-block;padding-left:.4em;padding-right:.4em;margin:0;border-right:1px solid #fff;border-radius:0}.label-group .label.label-black{background-color:#000}.navbar-info-button{padding:3px 4px;font-size:12px;font-family:inherit;margin-top:-2px}svg.graph .edge-label,svg.graph text{font-size:14px}.checkpoints-view{padding-top:1em}.subtask-details .blank{height:2em}.checkpoint-overview td span{padding-left:2em}svg.graph{overflow:hidden;height:100%;width:100%!important}svg.graph g.type-TK>rect{fill:#00ffd0}svg.graph text{font-weight:300}svg.graph .node>rect{stroke:#999;stroke-width:5px;fill:#fff;margin:0;padding:0}svg.graph .node[active]>rect{fill:#eee}svg.graph .node.node-mirror>rect{stroke:#a8a8a8}svg.graph .node.node-iteration>rect{stroke:#cd3333}svg.graph .node.node-source>rect{stroke:#4ce199}svg.graph .node.node-sink>rect{stroke:#e6ec8b}svg.graph .node.node-normal>rect{stroke:#3fb6d8}svg.graph .node h5{color:#999}svg.graph .edgeLabel rect{fill:#fff}svg.graph .edgePath path{stroke:#333;stroke-width:2px;fill:#333}svg.graph .label{color:#777;margin:0}svg.graph .node-label{display:block;margin:0;text-decoration:none}.timeline{overflow:hidden}.timeline-canvas{overflow:hidden;padding:10px}.timeline-canvas .bar-container{overflow:hidden}.timeline-canvas.secondary .timeline-insidelabel,.timeline-canvas.secondary .timeline-series{cursor:auto}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics a,.show-pointer{cursor:pointer}.qtip-timeline-bar{font-size:14px;line-height:1.4}#content .navbar-secondary-additional.navbar-secondary-additional-2{margin:-10px -10px 10px;padding:0;border-bottom:1px solid #e4e4e4}#content .navbar-secondary-additional.navbar-secondary-additional-2 .navbar-info{padding-top:12px;padding-bottom:12px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics{margin-right:15px;float:right}#content .navbar-secondary-additional.navbar-secondary-additional-2 .add-metrics .btn{margin-top:5px;margin-bottom:5px}#content .navbar-secondary-additional.navbar-secondary-additional-2 .metric-menu{max-height:300px;width:300px;overflow-y:scroll;text-align:left}#content .search label{position:absolute;top:15px;left:22px}#content .search input.metrics-filter{height:30px;padding:5px 5px 5px 25px;margin:5px 5px 5px 15px;width:90%;border-radius:5px;border:1px solid #999;-webkit-box-shadow:none;box-shadow:none}.metric-row{margin:0;min-height:275px;padding:0;list-style-type:none}.metric-row .metric-col{background-color:transparent;width:33.33%;float:left}.metric-row .metric-col.big{width:100%}.metric-row .metric-col .panel{margin-left:5px;margin-right:5px;min-height:265px;margin-bottom:10px}.metric-row .metric-col .panel .panel-body{background-color:transparent;height:265px;position:relative}.metric-row .metric-col .panel .panel-body .metric-numeric{text-align:center;margin-top:75px;font-size:40px;font-weight:700}.metric-row .metric-col .panel .panel-heading{padding:0 10px;background-color:transparent;height:41px;line-height:41px;position:relative;overflow:hidden;cursor:pointer}.metric-row .metric-col .panel .panel-heading .metric-title{padding:10px 0}.metric-row .metric-col .panel .panel-heading .buttons{position:absolute;top:0;right:0;padding:0 10px;background-color:#fff}.metric-row .metric-col.dndDraggingSource{display:none}.metric-row .dndPlaceholder{position:relative;background-color:#f0f0f0;min-height:305px;display:block;width:33.33%;float:left;margin-bottom:10px;border-radius:5px}.p-info{padding-left:5px;padding-right:5px}@media (min-width:1024px) and (max-width:1279px){#content #fold-button,#sidebar .navbar-static-top .navbar-brand-text{display:none}#sidebar{left:0;width:160px}#content{margin-left:160px}#content .navbar-main,#content .navbar-main-additional{left:160px}.table td.td-long{width:20%}}@media (min-width:1280px){#sidebar{left:0}#content{margin-left:250px}#content #fold-button{display:none}#content .navbar-main,#content .navbar-main-additional{left:250px}.table td.td-long{width:30%}}.legend-box{font-size:10px;width:2em}#total-mem{background-color:#7cb5ec}#heap-mem{background-color:#434348}#non-heap-mem{background-color:#90ed7d}#fetch-plan,#job-submit{width:100px}#content-inner,#details,#node-details{height:100%}#job-panel{overflow-y:auto}
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/web/js/hs/index.js b/flink-runtime-web/web-dashboard/web/js/hs/index.js
index 766120610c899..6354dd0a20939 100644
--- a/flink-runtime-web/web-dashboard/web/js/hs/index.js
+++ b/flink-runtime-web/web-dashboard/web/js/hs/index.js
@@ -1,2 +1,2 @@
-angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}),t.otherwise("/completed-jobs")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,s,a;return"undefined"==typeof e||null===e?"":(o=e%1e3,a=Math.floor(e/1e3),s=a%60,a=Math.floor(a/60),i=a%60,a=Math.floor(a/60),n=a%24,a=Math.floor(a/24),r=a,0===r?0===n?0===i?0===s?o+"ms":s+"s ":i+"m "+s+"s":t?n+"h "+i+"m":n+"h "+i+"m "+s+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+s+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,s,a,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=a(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},s["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,a.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,s;return o=function(t){return function(t){var r,n,o,s;return r=l.defer(),o=e.job.jid,s=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,s).then(function(e){var t,n,i,o,s,a,l;i=NaN,l={},o=e.values;for(t in o)a=o[t],s=t.replace(".currentLowWatermark",""),l[s]=a,(isNaN(i)||a<i)&&(i=a);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),s={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(s[i]=e,t>=n-1)return r.resolve(s)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),n=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t,e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},e.dropped=function(t,i,o,s,a){return r.orderMetrics(e.jobid,e.nodeid,o,i),e.$broadcast("metrics:refresh",o),n(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),n()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),n()},e.setMetricSize=function(t,i){return r.setMetricSize(e.jobid,e.nodeid,t,i),n()},e.setMetricView=function(t,i){return r.setMetricView(e.jobid,e.nodeid,t,i),n()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return n()}),e.nodeid)return n()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,s,a;s=r.children()[0],o=r.width(),angular.element(s).attr("width",o),a=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(s).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:a(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:a(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(s).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,s,a,l,u,c,d,f,p,m,h,g,b,v,k,j,S,w,C,$,y,J,M;p=null,C=d3.behavior.zoom(),M=[],g=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},h=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":h(t)?"node-iteration":"node-normal"},s=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+g+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=J(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(h(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+J(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},J=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},a=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:s(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:s(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:s(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:s(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:s(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:s(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:s(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,s,l,u,d,f,p,m,h,g,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),s=0,u=v.length;s<u;s++)if(r=v[s],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),M[r.id]=b,k(b,r),h=new dagreD3.render,c.select("g").call(h,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),a(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(g=r.inputs,l=0,d=g.length;l<d;l++)m=g[l],i(e,t,r,n,m);return e},y=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,s;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in M)i=M[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),s=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,s]),u.attr("transform","translate("+o+", "+s+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var s,a,l,u,c,d;return s=null,a=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,i,o){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return s=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,a,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),s=i,l.job.resolve(s)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,s;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(s=r(e,o.step_function)),s)return s}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,s.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=s.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var a;return a=i.seekVertex(r),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return a.subtasks=e.subtasks,n.resolve(a)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(s.jid),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/exceptions").success(function(e){return s.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;
-return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,s){if(o.id===r.id&&(i.metrics[e][t].splice(s,1),s<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var s;return n.setupLSFor(i,o),s=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var s;if(s=n.metrics[i][o].indexOf(e.id),s===-1&&(s=_.findIndex(n.metrics[i][o],{id:e.id})),s===-1)return t.push(e)}),s.resolve(t)}),s.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var s,a;return s=t.defer(),a=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+a).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),s.resolve(r)}}(this)),s.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var s;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),s=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(s)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e.taskmanagers)}),i.promise},this.loadLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
+angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}),t.otherwise("/completed-jobs")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,s,a;return"undefined"==typeof e||null===e?"":(o=e%1e3,a=Math.floor(e/1e3),s=a%60,a=Math.floor(a/60),i=a%60,a=Math.floor(a/60),n=a%24,a=Math.floor(a/24),r=a,0===r?0===n?0===i?0===s?o+"ms":s+"s ":i+"m "+s+"s":t?n+"h "+i+"m":n+"h "+i+"m "+s+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+s+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]).filter("searchMetrics",function(){return function(e,t){var r,n;return n=new RegExp(t,"gi"),function(){var t,i,o;for(o=[],t=0,i=e.length;t<i;t++)r=e[t],r.id.match(n)&&o.push(r);return o}()}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,s,a,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=a(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},s["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,a.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,s;return o=function(t){return function(t){var r,n,o,s;return r=l.defer(),o=e.job.jid,s=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,s).then(function(e){var t,n,i,o,s,a,l;i=NaN,l={},o=e.values;for(t in o)a=o[t],s=t.replace(".currentLowWatermark",""),l[s]=a,(isNaN(i)||a<i)&&(i=a);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),s={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(s[i]=e,t>=n-1)return r.resolve(s)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n,i;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),i=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t.sort(n),e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},n=function(e,t){var r,n;return r=e.id.toLowerCase(),n=t.id.toLowerCase(),r<n?-1:r>n?1:0},e.dropped=function(t,n,o,s,a){return r.orderMetrics(e.jobid,e.nodeid,o,n),e.$broadcast("metrics:refresh",o),i(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),i()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),i()},e.setMetricSize=function(t,n){return r.setMetricSize(e.jobid,e.nodeid,t,n),i()},e.setMetricView=function(t,n){return r.setMetricView(e.jobid,e.nodeid,t,n),i()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return i()}),e.nodeid)return i()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,s,a;s=r.children()[0],o=r.width(),angular.element(s).attr("width",o),a=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(s).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:a(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:a(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(s).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,s,a,l,u,c,d,f,p,m,h,g,b,v,k,j,S,w,C,$,y,J,M;p=null,C=d3.behavior.zoom(),M=[],g=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},h=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":h(t)?"node-iteration":"node-normal"},s=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+g+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=J(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(h(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+J(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},J=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},a=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:s(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:s(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:s(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:s(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:s(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:s(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:s(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,s,l,u,d,f,p,m,h,g,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),s=0,u=v.length;s<u;s++)if(r=v[s],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),M[r.id]=b,k(b,r),h=new dagreD3.render,c.select("g").call(h,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),a(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(g=r.inputs,l=0,d=g.length;l<d;l++)m=g[l],i(e,t,r,n,m);return e},y=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,s;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in M)i=M[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),s=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,s]),u.attr("transform","translate("+o+", "+s+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var s,a,l,u,c,d;return s=null,a=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,i,o){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return s=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,a,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),s=i,l.job.resolve(s)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,s;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(s=r(e,o.step_function)),s)return s}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,s.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=s.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var a;return a=i.seekVertex(r),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return a.subtasks=e.subtasks,n.resolve(a)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(s.jid),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/exceptions").success(function(e){return s.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],
+e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,s){if(o.id===r.id&&(i.metrics[e][t].splice(s,1),s<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var s;return n.setupLSFor(i,o),s=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var s;if(s=n.metrics[i][o].indexOf(e.id),s===-1&&(s=_.findIndex(n.metrics[i][o],{id:e.id})),s===-1)return t.push(e)}),s.resolve(t)}),s.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var s,a;return s=t.defer(),a=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+a).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),s.resolve(r)}}(this)),s.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var s;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),s=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(s)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e.taskmanagers)}),i.promise},this.loadLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/web/js/index.js b/flink-runtime-web/web-dashboard/web/js/index.js
index 9dfb735ac6e87..117cbad6bf1ad 100644
--- a/flink-runtime-web/web-dashboard/web/js/index.js
+++ b/flink-runtime-web/web-dashboard/web/js/index.js
@@ -1,2 +1,2 @@
-angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("overview",{url:"/overview",views:{main:{templateUrl:"partials/overview.html",controller:"OverviewController"}}}).state("running-jobs",{url:"/running-jobs",views:{main:{templateUrl:"partials/jobs/running-jobs.html",controller:"RunningJobsController"}}}).state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}).state("all-manager",{url:"/taskmanagers",views:{main:{templateUrl:"partials/taskmanager/index.html",controller:"AllTaskManagersController"}}}).state("single-manager",{url:"/taskmanager/{taskmanagerid}","abstract":!0,views:{main:{templateUrl:"partials/taskmanager/taskmanager.html",controller:"SingleTaskManagerController"}}}).state("single-manager.metrics",{url:"/metrics",views:{details:{templateUrl:"partials/taskmanager/taskmanager.metrics.html"}}}).state("single-manager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/taskmanager/taskmanager.stdout.html",controller:"SingleTaskManagerStdoutController"}}}).state("single-manager.log",{url:"/log",views:{details:{templateUrl:"partials/taskmanager/taskmanager.log.html",controller:"SingleTaskManagerLogsController"}}}).state("jobmanager",{url:"/jobmanager",views:{main:{templateUrl:"partials/jobmanager/index.html"}}}).state("jobmanager.config",{url:"/config",views:{details:{templateUrl:"partials/jobmanager/config.html",controller:"JobManagerConfigController"}}}).state("jobmanager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/jobmanager/stdout.html",controller:"JobManagerStdoutController"}}}).state("jobmanager.log",{url:"/log",views:{details:{templateUrl:"partials/jobmanager/log.html",controller:"JobManagerLogsController"}}}).state("submit",{url:"/submit",views:{main:{templateUrl:"partials/submit.html",controller:"JobSubmitController"}}}),t.otherwise("/overview")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,a,s;return"undefined"==typeof e||null===e?"":(o=e%1e3,s=Math.floor(e/1e3),a=s%60,s=Math.floor(s/60),i=s%60,s=Math.floor(s/60),n=s%24,s=Math.floor(s/24),r=s,0===r?0===n?0===i?0===a?o+"ms":a+"s ":i+"m "+a+"s":t?n+"h "+i+"m":n+"h "+i+"m "+a+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+a+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,a,s,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=s(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},a["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,s.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,a;return o=function(t){return function(t){var r,n,o,a;return r=l.defer(),o=e.job.jid,a=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,a).then(function(e){var t,n,i,o,a,s,l;i=NaN,l={},o=e.values;for(t in o)s=o[t],a=t.replace(".currentLowWatermark",""),l[a]=s,(isNaN(i)||s<i)&&(i=s);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),a={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(a[i]=e,t>=n-1)return r.resolve(a)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),n=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t,e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},e.dropped=function(t,i,o,a,s){return r.orderMetrics(e.jobid,e.nodeid,o,i),e.$broadcast("metrics:refresh",o),n(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),n()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),n()},e.setMetricSize=function(t,i){return r.setMetricSize(e.jobid,e.nodeid,t,i),n()},e.setMetricView=function(t,i){return r.setMetricView(e.jobid,e.nodeid,t,i),n()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return n()}),e.nodeid)return n()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,a,s;a=r.children()[0],o=r.width(),angular.element(a).attr("width",o),s=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(a).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:s(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:s(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(a).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,a,s,l,u,c,d,f,p,m,g,h,b,v,k,j,S,w,C,$,J,M,y;p=null,C=d3.behavior.zoom(),y=[],h=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},g=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":g(t)?"node-iteration":"node-normal"},a=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+h+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=M(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(g(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+M(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},M=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},s=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:a(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:a(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:a(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:a(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:a(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:a(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:a(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,a,l,u,d,f,p,m,g,h,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),a=0,u=v.length;a<u;a++)if(r=v[a],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),y[r.id]=b,k(b,r),g=new dagreD3.render,c.select("g").call(g,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),s(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(h=r.inputs,l=0,d=h.length;l<d;l++)m=h[l],i(e,t,r,n,m);return e},J=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,a;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in y)i=y[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),a=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,a]),u.attr("transform","translate("+o+", "+a+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var a,s,l,u,c,d;return a=null,s=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,i,o){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return a=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,s,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),a=i,l.job.resolve(a)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,a;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(a=r(e,o.step_function)),a)return a}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,a.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=a.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var s;return s=i.seekVertex(r),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return s.subtasks=e.subtasks,n.resolve(s)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(a.jid),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/exceptions").success(function(e){return a.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){
-return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,a){if(o.id===r.id&&(i.metrics[e][t].splice(a,1),a<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var a;return n.setupLSFor(i,o),a=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var a;if(a=n.metrics[i][o].indexOf(e.id),a===-1&&(a=_.findIndex(n.metrics[i][o],{id:e.id})),a===-1)return t.push(e)}),a.resolve(t)}),a.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var a,s;return a=t.defer(),s=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+s).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),a.resolve(r)}}(this)),a.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var a;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),a=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(a)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e.taskmanagers)}),i.promise},this.loadLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
+angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("overview",{url:"/overview",views:{main:{templateUrl:"partials/overview.html",controller:"OverviewController"}}}).state("running-jobs",{url:"/running-jobs",views:{main:{templateUrl:"partials/jobs/running-jobs.html",controller:"RunningJobsController"}}}).state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}).state("all-manager",{url:"/taskmanagers",views:{main:{templateUrl:"partials/taskmanager/index.html",controller:"AllTaskManagersController"}}}).state("single-manager",{url:"/taskmanager/{taskmanagerid}","abstract":!0,views:{main:{templateUrl:"partials/taskmanager/taskmanager.html",controller:"SingleTaskManagerController"}}}).state("single-manager.metrics",{url:"/metrics",views:{details:{templateUrl:"partials/taskmanager/taskmanager.metrics.html"}}}).state("single-manager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/taskmanager/taskmanager.stdout.html",controller:"SingleTaskManagerStdoutController"}}}).state("single-manager.log",{url:"/log",views:{details:{templateUrl:"partials/taskmanager/taskmanager.log.html",controller:"SingleTaskManagerLogsController"}}}).state("jobmanager",{url:"/jobmanager",views:{main:{templateUrl:"partials/jobmanager/index.html"}}}).state("jobmanager.config",{url:"/config",views:{details:{templateUrl:"partials/jobmanager/config.html",controller:"JobManagerConfigController"}}}).state("jobmanager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/jobmanager/stdout.html",controller:"JobManagerStdoutController"}}}).state("jobmanager.log",{url:"/log",views:{details:{templateUrl:"partials/jobmanager/log.html",controller:"JobManagerLogsController"}}}).state("submit",{url:"/submit",views:{main:{templateUrl:"partials/submit.html",controller:"JobSubmitController"}}}),t.otherwise("/overview")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,a,s;return"undefined"==typeof e||null===e?"":(o=e%1e3,s=Math.floor(e/1e3),a=s%60,s=Math.floor(s/60),i=s%60,s=Math.floor(s/60),n=s%24,s=Math.floor(s/24),r=s,0===r?0===n?0===i?0===a?o+"ms":a+"s ":i+"m "+a+"s":t?n+"h "+i+"m":n+"h "+i+"m "+a+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+a+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]).filter("searchMetrics",function(){return function(e,t){var r,n;return n=new RegExp(t,"gi"),function(){var t,i,o;for(o=[],t=0,i=e.length;t<i;t++)r=e[t],r.id.match(n)&&o.push(r);return o}()}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,a,s,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=s(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},a["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,s.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,a;return o=function(t){return function(t){var r,n,o,a;return r=l.defer(),o=e.job.jid,a=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,a).then(function(e){var t,n,i,o,a,s,l;i=NaN,l={},o=e.values;for(t in o)s=o[t],a=t.replace(".currentLowWatermark",""),l[a]=s,(isNaN(i)||s<i)&&(i=s);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),a={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(a[i]=e,t>=n-1)return r.resolve(a)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanTaskManagersController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n,i;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),i=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t.sort(n),e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},n=function(e,t){var r,n;return r=e.id.toLowerCase(),n=t.id.toLowerCase(),r<n?-1:r>n?1:0},e.dropped=function(t,n,o,a,s){return r.orderMetrics(e.jobid,e.nodeid,o,n),e.$broadcast("metrics:refresh",o),i(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),i()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),i()},e.setMetricSize=function(t,n){return r.setMetricSize(e.jobid,e.nodeid,t,n),i()},e.setMetricView=function(t,n){return r.setMetricView(e.jobid,e.nodeid,t,n),i()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return i()}),e.nodeid)return i()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,a,s;a=r.children()[0],o=r.width(),angular.element(a).attr("width",o),s=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(a).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:s(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:s(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(a).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,a,s,l,u,c,d,f,p,m,g,h,b,v,k,j,S,w,C,$,J,M,y;p=null,C=d3.behavior.zoom(),y=[],h=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},g=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":g(t)?"node-iteration":"node-normal"},a=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+h+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=M(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(g(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+M(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},M=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},s=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:a(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:a(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:a(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:a(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:a(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:a(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:a(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,a,l,u,d,f,p,m,g,h,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),a=0,u=v.length;a<u;a++)if(r=v[a],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),y[r.id]=b,k(b,r),g=new dagreD3.render,c.select("g").call(g,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),s(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(h=r.inputs,l=0,d=h.length;l<d;l++)m=h[l],i(e,t,r,n,m);return e},J=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,a;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in y)i=y[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),a=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,a]),u.attr("transform","translate("+o+", "+a+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var a,s,l,u,c,d;return a=null,s=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"joboverview").success(function(e){return function(t,n,i,o){return angular.forEach(t,function(t,r){switch(r){case"running":return c.running=e.setEndTimes(t);case"finished":return c.finished=e.setEndTimes(t);case"cancelled":return c.cancelled=e.setEndTimes(t);case"failed":return c.failed=e.setEndTimes(t)}}),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return a=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,s,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),a=i,l.job.resolve(a)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,a;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(a=r(e,o.step_function)),a)return a}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,a.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=a.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var s;return s=i.seekVertex(r),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return s.subtasks=e.subtasks,n.resolve(s)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(a.jid),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),
+l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/exceptions").success(function(e){return a.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,a){if(o.id===r.id&&(i.metrics[e][t].splice(a,1),a<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var a;return n.setupLSFor(i,o),a=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var a;if(a=n.metrics[i][o].indexOf(e.id),a===-1&&(a=_.findIndex(n.metrics[i][o],{id:e.id})),a===-1)return t.push(e)}),a.resolve(t)}),a.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var a,s;return a=t.defer(),s=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+s).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),a.resolve(r)}}(this)),a.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var a;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),a=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(a)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t[0]})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e.taskmanagers)}),i.promise},this.loadLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
diff --git a/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html b/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html
index daaf7c4269120..e3af4ec33f014 100644
--- a/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html
+++ b/flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node-list.metrics.html
@@ -27,7 +27,11 @@
       &nbsp;<span class="caret"></span>
     </button>
     <ul class="dropdown-menu dropdown-menu-right metric-menu">
-      <li ng-repeat="metric in availableMetrics track by $index"><a ng-click="addMetric(metric)">{{ metric.id | limit }}</a></li>
+      <section class="search">
+        <label for="search-input"><i aria-hidden="true" class="fa fa-search"><span class="sr-only">Search icons</span></i></label>
+        <input type="text" ng-model="metricsFilterQuery" placeholder="Search available metrics..." class="metrics-filter"/>
+      </section>
+      <li ng-repeat="metric in availableMetrics | searchMetrics:metricsFilterQuery track by $index"><a ng-click="addMetric(metric)">{{ metric.id | limit }}</a></li>
     </ul>
   </div>
   <div ng-if="!availableMetrics.length" class="dropdown add-metrics">

From 62b41a7fd6268fce443ec45d0966cfc7713f6d7a Mon Sep 17 00:00:00 2001
From: Patrick Lucas <me@patricklucas.com>
Date: Fri, 21 Apr 2017 15:00:53 +0200
Subject: [PATCH 210/253] [FLINK-6330] [docs] Add basic Docker, K8s docs

This closes #3751
---
 docs/docker/run.sh       |   4 +-
 docs/setup/docker.md     |  99 ++++++++++++++++++++++++
 docs/setup/kubernetes.md | 157 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 259 insertions(+), 1 deletion(-)
 create mode 100644 docs/setup/docker.md
 create mode 100644 docs/setup/kubernetes.md

diff --git a/docs/docker/run.sh b/docs/docker/run.sh
index 3c8878a9d2eba..5598c0a5338e7 100755
--- a/docs/docker/run.sh
+++ b/docs/docker/run.sh
@@ -31,10 +31,12 @@ if [ "$(uname -s)" == "Linux" ]; then
   USER_NAME=${SUDO_USER:=$USER}
   USER_ID=$(id -u "${USER_NAME}")
   GROUP_ID=$(id -g "${USER_NAME}")
+  LOCAL_HOME="/home/${USER_NAME}"
 else # boot2docker uid and gid
   USER_NAME=$USER
   USER_ID=1000
   GROUP_ID=50
+  LOCAL_HOME="/Users/${USER_NAME}"
 fi
 
 docker build -t "${IMAGE_NAME}-${USER_NAME}" - <<UserSpecificDocker
@@ -65,7 +67,7 @@ docker run -i -t \
   -w ${FLINK_DOC_ROOT} \
   -u "${USER}" \
   -v "${FLINK_DOC_ROOT}:${FLINK_DOC_ROOT}" \
-  -v "/home/${USER_NAME}:/home/${USER_NAME}" \
+  -v "${LOCAL_HOME}:/home/${USER_NAME}" \
   -p 4000:4000 \
   ${IMAGE_NAME}-${USER_NAME} \
   bash -c "${CMD}"
diff --git a/docs/setup/docker.md b/docs/setup/docker.md
new file mode 100644
index 0000000000000..29e696fcb4d6f
--- /dev/null
+++ b/docs/setup/docker.md
@@ -0,0 +1,99 @@
+---
+title:  "Docker Setup"
+nav-title: Docker
+nav-parent_id: deployment
+nav-pos: 4
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+[Docker](https://www.docker.com) is a popular container runtime. There are
+official Flink Docker images available on Docker Hub which can be used directly
+or extended to better integrate into a production environment.
+
+* This will be replaced by the TOC
+{:toc}
+
+## Official Flink Docker Images
+
+The [official Flink Docker repository](https://hub.docker.com/_/flink/) is
+hosted on Docker Hub and serves images of Flink version 1.2.1 and later.
+
+Images for each supported combination of Hadoop and Scala are available, and
+tag aliases are provided for convenience.
+
+For example, the following aliases can be used: *(`1.2.y` indicates the latest
+release of Flink 1.2)*
+
+* `flink:latest` →
+`flink:<latest-flink>-hadoop<latest-hadoop>-scala_<latest-scala>`
+* `flink:1.2` → `flink:1.2.y-hadoop27-scala_2.11`
+* `flink:1.2.1-scala_2.10` → `flink:1.2.1-hadoop27-scala_2.10`
+* `flink:1.2-hadoop26` → `flink:1.2.y-hadoop26-scala_2.11`
+
+<!-- NOTE: uncomment when docker-flink/docker-flink/issues/14 is resolved. -->
+<!--
+Additionally, images based on Alpine Linux are available. Reference them by
+appending `-alpine` to the tag. For the Alpine version of `flink:latest`, use
+`flink:alpine`.
+
+For example:
+
+* `flink:alpine`
+* `flink:1.2.1-alpine`
+* `flink:1.2-scala_2.10-alpine`
+-->
+
+## Flink with Docker Compose
+
+[Docker Compose](https://docs.docker.com/compose/) is a convenient way to run a
+group of Docker containers locally.
+
+An [example config file](https://github.com/docker-flink/examples) is available
+on GitHub.
+
+### Usage
+
+* Launch a cluster in the foreground
+
+        docker-compose up
+
+* Launch a cluster in the background
+
+        docker-compose up -d
+
+* Scale the cluster up or down to *N* TaskManagers
+
+        docker-compose scale taskmanager=<N>
+
+When the cluster is running, you can visit the web UI at [http://localhost:8081
+](http://localhost:8081) and submit a job.
+
+To submit a job via the command line, you must copy the JAR to the Jobmanager
+container and submit the job from there.
+
+For example:
+
+{% raw %}
+    $ JOBMANAGER_CONTAINER=$(docker ps --filter name=jobmanager --format={{.ID}})
+    $ docker cp path/to/jar "$JOBMANAGER_CONTAINER":/job.jar
+    $ docker exec -t -i "$JOBMANAGER_CONTAINER" flink run /job.jar
+{% endraw %}
+
+{% top %}
diff --git a/docs/setup/kubernetes.md b/docs/setup/kubernetes.md
new file mode 100644
index 0000000000000..0790a056918f2
--- /dev/null
+++ b/docs/setup/kubernetes.md
@@ -0,0 +1,157 @@
+---
+title:  "Kubernetes Setup"
+nav-title: Kubernetes
+nav-parent_id: deployment
+nav-pos: 4
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+[Kubernetes](https://kubernetes.io) is a container orchestration system.
+
+* This will be replaced by the TOC
+{:toc}
+
+## Simple Kubernetes Flink Cluster
+
+A basic Flink cluster deployment in Kubernetes has three components:
+
+* a Deployment for a single Jobmanager
+* a Deployment for a pool of Taskmanagers
+* a Service exposing the Jobmanager's RPC and UI ports
+
+### Launching the cluster
+
+Using the [resource definitions found below](#simple-kubernetes-flink-cluster-
+resources), launch the cluster with the `kubectl` command:
+
+    kubectl create -f jobmanager-deployment.yaml
+    kubectl create -f taskmanager-deployment.yaml
+    kubectl create -f jobmanager-service.yaml
+
+You can then access the Flink UI via `kubectl proxy`:
+
+1. Run `kubectl proxy` in a terminal
+2. Navigate to [http://localhost:8001/api/v1/proxy/namespaces/default/services/flink-jobmanager:8081
+](http://localhost:8001/api/v1/proxy/namespaces/default/services/flink-
+jobmanager:8081) in your browser
+
+### Deleting the cluster
+
+Again, use `kubectl` to delete the cluster:
+
+    kubectl delete -f jobmanager-deployment.yaml
+    kubectl delete -f taskmanager-deployment.yaml
+    kubectl delete -f jobmanager-service.yaml
+
+## Advanced Cluster Deployment
+
+An early version of a [Flink Helm chart](https://github.com/docker-flink/
+examples) is available on GitHub.
+
+## Appendix
+
+### Simple Kubernetes Flink cluster resources
+
+`jobmanager-deployment.yaml`
+{% highlight yaml %}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: flink-jobmanager
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: flink
+        component: jobmanager
+    spec:
+      containers:
+      - name: jobmanager
+        image: flink:latest
+        args:
+        - jobmanager
+        ports:
+        - containerPort: 6123
+          name: rpc
+        - containerPort: 6124
+          name: blob
+        - containerPort: 6125
+          name: query
+        - containerPort: 8081
+          name: ui
+        env:
+        - name: JOB_MANAGER_RPC_ADDRESS
+          value: flink-jobmanager
+{% endhighlight %}
+
+`taskmanager-deployment.yaml`
+{% highlight yaml %}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: flink-taskmanager
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: flink
+        component: taskmanager
+    spec:
+      containers:
+      - name: taskmanager
+        image: flink:latest
+        args:
+        - taskmanager
+        ports:
+        - containerPort: 6121
+          name: data
+        - containerPort: 6122
+          name: rpc
+        - containerPort: 6125
+          name: query
+        env:
+        - name: JOB_MANAGER_RPC_ADDRESS
+          value: flink-jobmanager
+{% endhighlight %}
+
+`jobmanager-service.yaml`
+{% highlight yaml %}
+apiVersion: v1
+kind: Service
+metadata:
+  name: flink-jobmanager
+spec:
+  ports:
+  - name: rpc
+    port: 6123
+  - name: blob
+    port: 6124
+  - name: query
+    port: 6125
+  - name: ui
+    port: 8081
+  selector:
+    app: flink
+    component: jobmanager
+{% endhighlight %}
+
+{% top %}

From cd70a50162e711c1be27020813fde7284c815715 Mon Sep 17 00:00:00 2001
From: David Anderson <david@alpinegizmo.com>
Date: Tue, 9 May 2017 17:23:46 +0200
Subject: [PATCH 211/253] [FLINK-6512] [docs] improved code formatting in some
 examples

This closes #3857
---
 docs/dev/best_practices.md |  30 ++--
 docs/dev/migration.md      | 300 +++++++++++++++++++------------------
 2 files changed, 171 insertions(+), 159 deletions(-)

diff --git a/docs/dev/best_practices.md b/docs/dev/best_practices.md
index b2111c45b2a4a..4dfd7fd7ce492 100644
--- a/docs/dev/best_practices.md
+++ b/docs/dev/best_practices.md
@@ -59,8 +59,8 @@ ParameterTool parameter = ParameterTool.fromPropertiesFile(propertiesFile);
 This allows getting arguments like `--input hdfs:///mydata --elements 42` from the command line.
 {% highlight java %}
 public static void main(String[] args) {
-	ParameterTool parameter = ParameterTool.fromArgs(args);
-	// .. regular code ..
+    ParameterTool parameter = ParameterTool.fromArgs(args);
+    // .. regular code ..
 {% endhighlight %}
 
 
@@ -114,17 +114,18 @@ The example below shows how to pass the parameters as a `Configuration` object t
 
 {% highlight java %}
 ParameterTool parameters = ParameterTool.fromArgs(args);
-DataSet<Tuple2<String, Integer>> counts = text.flatMap(new Tokenizer()).withParameters(parameters.getConfiguration())
+DataSet<Tuple2<String, Integer>> counts = text
+        .flatMap(new Tokenizer()).withParameters(parameters.getConfiguration())
 {% endhighlight %}
 
 In the `Tokenizer`, the object is now accessible in the `open(Configuration conf)` method:
 
 {% highlight java %}
 public static final class Tokenizer extends RichFlatMapFunction<String, Tuple2<String, Integer>> {
-	@Override
-	public void open(Configuration parameters) throws Exception {
-		parameters.getInteger("myInt", -1);
-		// .. do
+    @Override
+    public void open(Configuration parameters) throws Exception {
+	parameters.getInteger("myInt", -1);
+	// .. do
 {% endhighlight %}
 
 
@@ -147,11 +148,12 @@ Access them in any rich user function:
 {% highlight java %}
 public static final class Tokenizer extends RichFlatMapFunction<String, Tuple2<String, Integer>> {
 
-	@Override
-	public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
-		ParameterTool parameters = (ParameterTool) getRuntimeContext().getExecutionConfig().getGlobalJobParameters();
-		parameters.getRequired("input");
-		// .. do more ..
+    @Override
+    public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
+	ParameterTool parameters = (ParameterTool)
+	    getRuntimeContext().getExecutionConfig().getGlobalJobParameters();
+	parameters.getRequired("input");
+	// .. do more ..
 {% endhighlight %}
 
 
@@ -198,8 +200,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class MyClass implements MapFunction {
-	private static final Logger LOG = LoggerFactory.getLogger(MyClass.class);
-	// ...
+    private static final Logger LOG = LoggerFactory.getLogger(MyClass.class);
+    // ...
 {% endhighlight %}
 
 
diff --git a/docs/dev/migration.md b/docs/dev/migration.md
index a5910a8162a86..11eb42c3bb4d7 100644
--- a/docs/dev/migration.md
+++ b/docs/dev/migration.md
@@ -51,69 +51,70 @@ As running examples for the remainder of this document we will use the `CountMap
 functions. The first is an example of a function with **keyed** state, while
 the second has **non-keyed** state. The code for the aforementioned two functions in Flink 1.1 is presented below:
 
-    public class CountMapper extends RichFlatMapFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
+{% highlight java %}
+public class CountMapper extends RichFlatMapFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
 
-        private transient ValueState<Integer> counter;
+    private transient ValueState<Integer> counter;
 
-        private final int numberElements;
+    private final int numberElements;
 
-        public CountMapper(int numberElements) {
-            this.numberElements = numberElements;
-        }
+    public CountMapper(int numberElements) {
+        this.numberElements = numberElements;
+    }
 
-        @Override
-        public void open(Configuration parameters) throws Exception {
-            counter = getRuntimeContext().getState(
-      	        new ValueStateDescriptor<>("counter", Integer.class, 0));
-        }
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        counter = getRuntimeContext().getState(
+            new ValueStateDescriptor<>("counter", Integer.class, 0));
+    }
 
-        @Override
-        public void flatMap(Tuple2<String, Integer> value, Collector<Tuple2<String, Integer>> out) throws Exception {
-            int count = counter.value() + 1;
-      	    counter.update(count);
+    @Override
+    public void flatMap(Tuple2<String, Integer> value, Collector<Tuple2<String, Integer>> out) throws Exception {
+        int count = counter.value() + 1;
+        counter.update(count);
 
-      	    if (count % numberElements == 0) {
-     		    out.collect(Tuple2.of(value.f0, count));
-     		    counter.update(0); // reset to 0
-     	    }
+        if (count % numberElements == 0) {
+            out.collect(Tuple2.of(value.f0, count));
+            counter.update(0); // reset to 0
         }
     }
+}
 
+public class BufferingSink implements SinkFunction<Tuple2<String, Integer>>,
+    Checkpointed<ArrayList<Tuple2<String, Integer>>> {
 
-    public class BufferingSink implements SinkFunction<Tuple2<String, Integer>>,
-            Checkpointed<ArrayList<Tuple2<String, Integer>>> {
-
-	    private final int threshold;
+    private final int threshold;
 
-	    private ArrayList<Tuple2<String, Integer>> bufferedElements;
+    private ArrayList<Tuple2<String, Integer>> bufferedElements;
 
-	    BufferingSink(int threshold) {
-		    this.threshold = threshold;
-		    this.bufferedElements = new ArrayList<>();
-	    }
+    BufferingSink(int threshold) {
+        this.threshold = threshold;
+        this.bufferedElements = new ArrayList<>();
+    }
 
-    	@Override
-	    public void invoke(Tuple2<String, Integer> value) throws Exception {
-		    bufferedElements.add(value);
-		    if (bufferedElements.size() == threshold) {
-			    for (Tuple2<String, Integer> element: bufferedElements) {
-				    // send it to the sink
-			    }
-			    bufferedElements.clear();
-		    }
+    @Override
+    public void invoke(Tuple2<String, Integer> value) throws Exception {
+        bufferedElements.add(value);
+        if (bufferedElements.size() == threshold) {
+            for (Tuple2<String, Integer> element: bufferedElements) {
+	        // send it to the sink
 	    }
+	    bufferedElements.clear();
+	}
+    }
 
-	    @Override
-	    public ArrayList<Tuple2<String, Integer>> snapshotState(
-	            long checkpointId, long checkpointTimestamp) throws Exception {
-		    return bufferedElements;
-	    }
+    @Override
+    public ArrayList<Tuple2<String, Integer>> snapshotState(
+        long checkpointId, long checkpointTimestamp) throws Exception {
+	    return bufferedElements;
+    }
 
-	    @Override
-	    public void restoreState(ArrayList<Tuple2<String, Integer>> state) throws Exception {
-	    	bufferedElements.addAll(state);
-        }
+    @Override
+    public void restoreState(ArrayList<Tuple2<String, Integer>> state) throws Exception {
+        bufferedElements.addAll(state);
     }
+}
+{% endhighlight %}
 
 
 The `CountMapper` is a `RichFlatMapFuction` which assumes a grouped-by-key input stream of the form
@@ -160,9 +161,11 @@ the [State documentation]({{ site.baseurl }}/dev/stream/state.html).
 
 The `ListCheckpointed` interface requires the implementation of two methods:
 
-    List<T> snapshotState(long checkpointId, long timestamp) throws Exception;
+{% highlight java %}
+List<T> snapshotState(long checkpointId, long timestamp) throws Exception;
 
-    void restoreState(List<T> state) throws Exception;
+void restoreState(List<T> state) throws Exception;
+{% endhighlight %}
 
 Their semantics are the same as their counterparts in the old `Checkpointed` interface. The only difference
 is that now `snapshotState()` should return a list of objects to checkpoint, as stated earlier, and
@@ -170,53 +173,55 @@ is that now `snapshotState()` should return a list of objects to checkpoint, as
 return a `Collections.singletonList(MY_STATE)` in the `snapshotState()`. The updated code for `BufferingSink`
 is included below:
 
-    public class BufferingSinkListCheckpointed implements
-            SinkFunction<Tuple2<String, Integer>>,
-            ListCheckpointed<Tuple2<String, Integer>>,
-            CheckpointedRestoring<ArrayList<Tuple2<String, Integer>>> {
+{% highlight java %}
+public class BufferingSinkListCheckpointed implements
+        SinkFunction<Tuple2<String, Integer>>,
+        ListCheckpointed<Tuple2<String, Integer>>,
+        CheckpointedRestoring<ArrayList<Tuple2<String, Integer>>> {
 
-        private final int threshold;
+    private final int threshold;
 
-        private transient ListState<Tuple2<String, Integer>> checkpointedState;
+    private transient ListState<Tuple2<String, Integer>> checkpointedState;
 
-        private List<Tuple2<String, Integer>> bufferedElements;
+    private List<Tuple2<String, Integer>> bufferedElements;
 
-        public BufferingSinkListCheckpointed(int threshold) {
-            this.threshold = threshold;
-            this.bufferedElements = new ArrayList<>();
-        }
+    public BufferingSinkListCheckpointed(int threshold) {
+        this.threshold = threshold;
+        this.bufferedElements = new ArrayList<>();
+    }
 
-        @Override
-        public void invoke(Tuple2<String, Integer> value) throws Exception {
-            this.bufferedElements.add(value);
-            if (bufferedElements.size() == threshold) {
-                for (Tuple2<String, Integer> element: bufferedElements) {
-                    // send it to the sink
-                }
-                bufferedElements.clear();
+    @Override
+    public void invoke(Tuple2<String, Integer> value) throws Exception {
+        this.bufferedElements.add(value);
+        if (bufferedElements.size() == threshold) {
+            for (Tuple2<String, Integer> element: bufferedElements) {
+                // send it to the sink
             }
+            bufferedElements.clear();
         }
+    }
 
-        @Override
-        public List<Tuple2<String, Integer>> snapshotState(
-                long checkpointId, long timestamp) throws Exception {
-            return this.bufferedElements;
-        }
-
-        @Override
-        public void restoreState(List<Tuple2<String, Integer>> state) throws Exception {
-            if (!state.isEmpty()) {
-                this.bufferedElements.addAll(state);
-            }
-        }
+    @Override
+    public List<Tuple2<String, Integer>> snapshotState(
+            long checkpointId, long timestamp) throws Exception {
+        return this.bufferedElements;
+    }
 
-        @Override
-        public void restoreState(ArrayList<Tuple2<String, Integer>> state) throws Exception {
-            // this is from the CheckpointedRestoring interface.
+    @Override
+    public void restoreState(List<Tuple2<String, Integer>> state) throws Exception {
+        if (!state.isEmpty()) {
             this.bufferedElements.addAll(state);
         }
     }
 
+    @Override
+    public void restoreState(ArrayList<Tuple2<String, Integer>> state) throws Exception {
+        // this is from the CheckpointedRestoring interface.
+        this.bufferedElements.addAll(state);
+    }
+}
+{% endhighlight %}
+
 As shown in the code, the updated function also implements the `CheckpointedRestoring` interface. This is for backwards
 compatibility reasons and more details will be explained at the end of this section.
 
@@ -224,9 +229,11 @@ compatibility reasons and more details will be explained at the end of this sect
 
 The `CheckpointedFunction` interface requires again the implementation of two methods:
 
-    void snapshotState(FunctionSnapshotContext context) throws Exception;
+{% highlight java %}
+void snapshotState(FunctionSnapshotContext context) throws Exception;
 
-    void initializeState(FunctionInitializationContext context) throws Exception;
+void initializeState(FunctionInitializationContext context) throws Exception;
+{% endhighlight %}
 
 As in Flink 1.1, `snapshotState()` is called whenever a checkpoint is performed, but now `initializeState()` (which is
 the counterpart of the `restoreState()`) is called every time the user-defined function is initialized, rather than only
@@ -234,57 +241,59 @@ in the case that we are recovering from a failure. Given this, `initializeState(
 types of state are initialized, but also where state recovery logic is included. An implementation of the
 `CheckpointedFunction` interface for `BufferingSink` is presented below.
 
-    public class BufferingSink implements SinkFunction<Tuple2<String, Integer>>,
-            CheckpointedFunction, CheckpointedRestoring<ArrayList<Tuple2<String, Integer>>> {
+{% highlight java %}
+public class BufferingSink implements SinkFunction<Tuple2<String, Integer>>,
+        CheckpointedFunction, CheckpointedRestoring<ArrayList<Tuple2<String, Integer>>> {
 
-        private final int threshold;
+    private final int threshold;
 
-        private transient ListState<Tuple2<String, Integer>> checkpointedState;
+    private transient ListState<Tuple2<String, Integer>> checkpointedState;
 
-        private List<Tuple2<String, Integer>> bufferedElements;
+    private List<Tuple2<String, Integer>> bufferedElements;
 
-        public BufferingSink(int threshold) {
-            this.threshold = threshold;
-            this.bufferedElements = new ArrayList<>();
-        }
+    public BufferingSink(int threshold) {
+        this.threshold = threshold;
+        this.bufferedElements = new ArrayList<>();
+    }
 
-        @Override
-        public void invoke(Tuple2<String, Integer> value) throws Exception {
-            bufferedElements.add(value);
-            if (bufferedElements.size() == threshold) {
-                for (Tuple2<String, Integer> element: bufferedElements) {
-                    // send it to the sink
-                }
-                bufferedElements.clear();
+    @Override
+    public void invoke(Tuple2<String, Integer> value) throws Exception {
+        bufferedElements.add(value);
+        if (bufferedElements.size() == threshold) {
+            for (Tuple2<String, Integer> element: bufferedElements) {
+                // send it to the sink
             }
+            bufferedElements.clear();
         }
+    }
 
-        @Override
-        public void snapshotState(FunctionSnapshotContext context) throws Exception {
-            checkpointedState.clear();
-            for (Tuple2<String, Integer> element : bufferedElements) {
-                checkpointedState.add(element);
-            }
+    @Override
+    public void snapshotState(FunctionSnapshotContext context) throws Exception {
+        checkpointedState.clear();
+        for (Tuple2<String, Integer> element : bufferedElements) {
+            checkpointedState.add(element);
         }
+    }
 
-        @Override
-        public void initializeState(FunctionInitializationContext context) throws Exception {
-            checkpointedState = context.getOperatorStateStore().
-                getSerializableListState("buffered-elements");
+    @Override
+    public void initializeState(FunctionInitializationContext context) throws Exception {
+        checkpointedState = context.getOperatorStateStore().
+            getSerializableListState("buffered-elements");
 
-            if (context.isRestored()) {
-                for (Tuple2<String, Integer> element : checkpointedState.get()) {
-                    bufferedElements.add(element);
-                }
+        if (context.isRestored()) {
+            for (Tuple2<String, Integer> element : checkpointedState.get()) {
+                bufferedElements.add(element);
             }
         }
+    }
 
-        @Override
-        public void restoreState(ArrayList<Tuple2<String, Integer>> state) throws Exception {
-            // this is from the CheckpointedRestoring interface.
-            this.bufferedElements.addAll(state);
-        }
+    @Override
+    public void restoreState(ArrayList<Tuple2<String, Integer>> state) throws Exception {
+        // this is from the CheckpointedRestoring interface.
+        this.bufferedElements.addAll(state);
     }
+}
+{% endhighlight %}
 
 The `initializeState` takes as argument a `FunctionInitializationContext`. This is used to initialize
 the non-keyed state "container". This is a container of type `ListState` where the non-keyed state objects
@@ -305,40 +314,41 @@ for Flink 1.1. If the `CheckpointedFunction` interface was to be used in the `Co
 the old `open()` method could be removed and the new `snapshotState()` and `initializeState()` methods
 would look like this:
 
-    public class CountMapper extends RichFlatMapFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>
-            implements CheckpointedFunction {
+{% highlight java %}
+public class CountMapper extends RichFlatMapFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>
+        implements CheckpointedFunction {
 
-        private transient ValueState<Integer> counter;
+    private transient ValueState<Integer> counter;
 
-        private final int numberElements;
+    private final int numberElements;
 
-        public CountMapper(int numberElements) {
-            this.numberElements = numberElements;
-        }
+    public CountMapper(int numberElements) {
+        this.numberElements = numberElements;
+    }
 
-        @Override
-        public void flatMap(Tuple2<String, Integer> value, Collector<Tuple2<String, Integer>> out) throws Exception {
-            int count = counter.value() + 1;
-            counter.update(count);
+    @Override
+    public void flatMap(Tuple2<String, Integer> value, Collector<Tuple2<String, Integer>> out) throws Exception {
+        int count = counter.value() + 1;
+        counter.update(count);
 
-            if (count % numberElements == 0) {
-                out.collect(Tuple2.of(value.f0, count));
-             	counter.update(0); // reset to 0
-             	}
-            }
+        if (count % numberElements == 0) {
+            out.collect(Tuple2.of(value.f0, count));
+            counter.update(0); // reset to 0
         }
+    }
 
-        @Override
-        public void snapshotState(FunctionSnapshotContext context) throws Exception {
-            //all managed, nothing to do.
-        }
+    @Override
+    public void snapshotState(FunctionSnapshotContext context) throws Exception {
+        // all managed, nothing to do.
+    }
 
-        @Override
-        public void initializeState(FunctionInitializationContext context) throws Exception {
-            counter = context.getKeyedStateStore().getState(
-                new ValueStateDescriptor<>("counter", Integer.class, 0));
-        }
+    @Override
+    public void initializeState(FunctionInitializationContext context) throws Exception {
+        counter = context.getKeyedStateStore().getState(
+            new ValueStateDescriptor<>("counter", Integer.class, 0));
     }
+}
+{% endhighlight %}
 
 Notice that the `snapshotState()` method is empty as Flink itself takes care of snapshotting managed keyed state
 upon checkpointing.

From e25fc7b2fb7ed3c21468006428c352546d200f48 Mon Sep 17 00:00:00 2001
From: David Anderson <david@alpinegizmo.com>
Date: Tue, 9 May 2017 16:50:53 +0200
Subject: [PATCH 212/253] [FLINK-6513] [docs] cleaned up some typos and
 grammatical flaws

This closes #3858
---
 docs/dev/best_practices.md             | 34 ++++++++++++-------------
 docs/dev/stream/checkpointing.md       | 11 ++++----
 docs/dev/stream/process_function.md    |  4 +--
 docs/dev/stream/side_output.md         |  4 +--
 docs/internals/stream_checkpointing.md | 35 +++++++++++++++-----------
 5 files changed, 46 insertions(+), 42 deletions(-)

diff --git a/docs/dev/best_practices.md b/docs/dev/best_practices.md
index 4dfd7fd7ce492..6328d22e18253 100644
--- a/docs/dev/best_practices.md
+++ b/docs/dev/best_practices.md
@@ -30,14 +30,12 @@ This page contains a collection of best practices for Flink programmers on how t
 
 ## Parsing command line arguments and passing them around in your Flink application
 
+Almost all Flink applications, both batch and streaming, rely on external configuration parameters.
+They are used to specify input and output sources (like paths or addresses), system parameters (parallelism, runtime configuration), and application specific parameters (typically used within user functions).
 
-Almost all Flink applications, both batch and streaming rely on external configuration parameters.
-For example for specifying input and output sources (like paths or addresses), also system parameters (parallelism, runtime configuration) and application specific parameters (often used within the user functions).
-
-Since version 0.9 we are providing a simple utility called `ParameterTool` to provide at least some basic tooling for solving these problems.
-
-Please note that you don't have to use the `ParameterTool` explained here. Other frameworks such as [Commons CLI](https://commons.apache.org/proper/commons-cli/),
-[argparse4j](http://argparse4j.sourceforge.net/) and others work well with Flink as well.
+Flink provides a simple utility called `ParameterTool` to provide some basic tooling for solving these problems.
+Please note that you don't have to use the `ParameterTool` described here. Other frameworks such as [Commons CLI](https://commons.apache.org/proper/commons-cli/) and
+[argparse4j](http://argparse4j.sourceforge.net/) also work well with Flink.
 
 
 ### Getting your configuration values into the `ParameterTool`
@@ -89,8 +87,8 @@ parameter.getNumberOfParameters()
 // .. there are more methods available.
 {% endhighlight %}
 
-You can use the return values of these methods directly in the main() method (=the client submitting the application).
-For example you could set the parallelism of a operator like this:
+You can use the return values of these methods directly in the `main()` method of the client submitting the application.
+For example, you could set the parallelism of a operator like this:
 
 {% highlight java %}
 ParameterTool parameters = ParameterTool.fromArgs(args);
@@ -105,10 +103,10 @@ ParameterTool parameters = ParameterTool.fromArgs(args);
 DataSet<Tuple2<String, Integer>> counts = text.flatMap(new Tokenizer(parameters));
 {% endhighlight %}
 
-and then use them inside the function for getting values from the command line.
+and then use it inside the function for getting values from the command line.
 
 
-#### Passing it as a `Configuration` object to single functions
+#### Passing parameters as a `Configuration` object to single functions
 
 The example below shows how to pass the parameters as a `Configuration` object to a user defined function.
 
@@ -131,9 +129,9 @@ public static final class Tokenizer extends RichFlatMapFunction<String, Tuple2<S
 
 #### Register the parameters globally
 
-Parameters registered as a global job parameter at the `ExecutionConfig` allow you to access the configuration values from the JobManager web interface and all functions defined by the user.
+Parameters registered as global job parameters in the `ExecutionConfig` can be accessed as configuration values from the JobManager web interface and in all functions defined by the user.
 
-**Register the parameters globally**
+Register the parameters globally:
 
 {% highlight java %}
 ParameterTool parameters = ParameterTool.fromArgs(args);
@@ -286,14 +284,14 @@ Change your projects `pom.xml` file like this:
 
 The following changes were done in the `<dependencies>` section:
 
- * Exclude all `log4j` dependencies from all Flink dependencies: This causes Maven to ignore Flink's transitive dependencies to log4j.
- * Exclude the `slf4j-log4j12` artifact from Flink's dependencies: Since we are going to use the slf4j to logback binding, we have to remove the slf4j to log4j binding.
+ * Exclude all `log4j` dependencies from all Flink dependencies: this causes Maven to ignore Flink's transitive dependencies to log4j.
+ * Exclude the `slf4j-log4j12` artifact from Flink's dependencies: since we are going to use the slf4j to logback binding, we have to remove the slf4j to log4j binding.
  * Add the Logback dependencies: `logback-core` and `logback-classic`
  * Add dependencies for `log4j-over-slf4j`. `log4j-over-slf4j` is a tool which allows legacy applications which are directly using the Log4j APIs to use the Slf4j interface. Flink depends on Hadoop which is directly using Log4j for logging. Therefore, we need to redirect all logger calls from Log4j to Slf4j which is in turn logging to Logback.
 
 Please note that you need to manually add the exclusions to all new Flink dependencies you are adding to the pom file.
 
-You may also need to check if other dependencies (non Flink) are pulling in log4j bindings. You can analyze the dependencies of your project with `mvn dependency:tree`.
+You may also need to check if other (non-Flink) dependencies are pulling in log4j bindings. You can analyze the dependencies of your project with `mvn dependency:tree`.
 
 
 
@@ -301,7 +299,7 @@ You may also need to check if other dependencies (non Flink) are pulling in log4
 
 This tutorial is applicable when running Flink on YARN or as a standalone cluster.
 
-In order to use Logback instead of Log4j with Flink, you need to remove the `log4j-1.2.xx.jar` and `sfl4j-log4j12-xxx.jar` from the `lib/` directory.
+In order to use Logback instead of Log4j with Flink, you need to remove `log4j-1.2.xx.jar` and `sfl4j-log4j12-xxx.jar` from the `lib/` directory.
 
 Next, you need to put the following jar files into the `lib/` folder:
 
@@ -309,7 +307,7 @@ Next, you need to put the following jar files into the `lib/` folder:
  * `logback-core.jar`
  * `log4j-over-slf4j.jar`: This bridge needs to be present in the classpath for redirecting logging calls from Hadoop (which is using Log4j) to Slf4j.
 
-Note that you need to explicitly set the `lib/` directory when using a per job YARN cluster.
+Note that you need to explicitly set the `lib/` directory when using a per-job YARN cluster.
 
 The command to submit Flink on YARN with a custom logger is: `./bin/flink run -yt $FLINK_HOME/lib <... remaining arguments ...>`
 
diff --git a/docs/dev/stream/checkpointing.md b/docs/dev/stream/checkpointing.md
index 774d9efd207e8..3a0a1ae44a0b5 100644
--- a/docs/dev/stream/checkpointing.md
+++ b/docs/dev/stream/checkpointing.md
@@ -32,7 +32,7 @@ any type of more elaborate operation.
 In order to make state fault tolerant, Flink needs to **checkpoint** the state. Checkpoints allow Flink to recover state and positions
 in the streams to give the application the same semantics as a failure-free execution.
 
-The [documentation on streaming fault tolerance](../../internals/stream_checkpointing.html) describe in detail the technique behind Flink's streaming fault tolerance mechanism.
+The [documentation on streaming fault tolerance](../../internals/stream_checkpointing.html) describes in detail the technique behind Flink's streaming fault tolerance mechanism.
 
 
 ## Prerequisites
@@ -124,12 +124,13 @@ env.getCheckpointConfig.setMaxConcurrentCheckpoints(1)
 
 ## Selecting a State Backend
 
-The checkpointing mechanism stores the progress in the data sources and data sinks, the state of windows, as well as the [user-defined state](state.html) consistently to
-provide *exactly once* processing semantics. Where the checkpoints are stored (e.g., JobManager memory, file system, database) depends on the configured
+Flink's [checkpointing mechanism]({{ site.baseurl }}/internals/stream_checkpointing.html) stores consistent snapshots
+of all the state in timers and stateful operators, including connectors, windows, and any [user-defined state](state.html).
+Where the checkpoints are stored (e.g., JobManager memory, file system, database) depends on the configured
 **State Backend**. 
 
-By default state will be kept in memory, and checkpoints will be stored in-memory at the master node (the JobManager). For proper persistence of large state,
-Flink supports various forms of storing and checkpointing state in so called **State Backends**, which can be set via `StreamExecutionEnvironment.setStateBackend(…)`.
+By default, state is kept in memory in the TaskManagers and checkpoints are stored in memory in the JobManager. For proper persistence of large state,
+Flink supports various approaches for storing and checkpointing state in other state backends. The choice of state backend can be configured via `StreamExecutionEnvironment.setStateBackend(…)`.
 
 See [state backends](../../ops/state_backends.html) for more details on the available state backends and options for job-wide and cluster-wide configuration.
 
diff --git a/docs/dev/stream/process_function.md b/docs/dev/stream/process_function.md
index d682a89766953..fb5f39db61f4f 100644
--- a/docs/dev/stream/process_function.md
+++ b/docs/dev/stream/process_function.md
@@ -42,11 +42,11 @@ For fault-tolerant state, the `ProcessFunction` gives access to Flink's [keyed s
 `RuntimeContext`, similar to the way other stateful functions can access keyed state.
 
 The timers allow applications to react to changes in processing time and in [event time](../event_time.html).
-Every call to the function `processElement(...)` gets a `Context` object with gives access to the element's
+Every call to the function `processElement(...)` gets a `Context` object which gives access to the element's
 event time timestamp, and to the *TimerService*. The `TimerService` can be used to register callbacks for future
 event-/processing-time instants. When a timer's particular time is reached, the `onTimer(...)` method is
 called. During that call, all states are again scoped to the key with which the timer was created, allowing
-timers to perform keyed state manipulation as well.
+timers to manipulate keyed state.
 
 <span class="label label-info">Note</span> If you want to access keyed state and timers you have
 to apply the `ProcessFunction` on a keyed stream:
diff --git a/docs/dev/stream/side_output.md b/docs/dev/stream/side_output.md
index e4c4c1997b258..63b71721a57a3 100644
--- a/docs/dev/stream/side_output.md
+++ b/docs/dev/stream/side_output.md
@@ -55,8 +55,8 @@ val outputTag = OutputTag[String]("side-output")
 Notice how the `OutputTag` is typed according to the type of elements that the side output stream
 contains.
 
-Emitting data to a side output it only possible when using a
-[ProcessFunction]({{ site.baseurl }}/dev/stream/process_function.html). In the function, you can use the `Context` parameter
+Emitting data to a side output is only possible from within a
+[ProcessFunction]({{ site.baseurl }}/dev/stream/process_function.html). You can use the `Context` parameter
 to emit data to a side output identified by an `OutputTag`:
 
 <div class="codetabs" markdown="1">
diff --git a/docs/internals/stream_checkpointing.md b/docs/internals/stream_checkpointing.md
index edc7967d4422a..d701c5e13c38c 100644
--- a/docs/internals/stream_checkpointing.md
+++ b/docs/internals/stream_checkpointing.md
@@ -37,17 +37,20 @@ record from the data stream **exactly once**. Note that there is a switch to *do
 (described below).
 
 The fault tolerance mechanism continuously draws snapshots of the distributed streaming data flow. For streaming applications
-with small state, these snapshots are very light-weight and can be drawn frequently without impacting the performance much.
+with small state, these snapshots are very light-weight and can be drawn frequently without much impact on performance.
 The state of the streaming applications is stored at a configurable place (such as the master node, or HDFS).
 
 In case of a program failure (due to machine-, network-, or software failure), Flink stops the distributed streaming dataflow.
 The system then restarts the operators and resets them to the latest successful checkpoint. The input streams are reset to the
 point of the state snapshot. Any records that are processed as part of the restarted parallel dataflow are guaranteed to not
-have been part of the checkpointed state before.
+have been part of the previously checkpointed state.
+
+*Note:* By default, checkpointing is disabled. See [Checkpointing]({{ site.baseurl }}/dev/stream/checkpointing.html) for details on how to enable and configure checkpointing.
 
 *Note:* For this mechanism to realize its full guarantees, the data stream source (such as message queue or broker) needs to be able
 to rewind the stream to a defined recent point. [Apache Kafka](http://kafka.apache.org) has this ability and Flink's connector to
-Kafka exploits this ability.
+Kafka exploits this ability. See [Fault Tolerance Guarantees of Data Sources and Sinks]({{ site.baseurl }}/dev/connectors/guarantees.html) for
+more information about the guarantees provided by Flink's connectors.
 
 *Note:* Because Flink's checkpoints are realized through distributed snapshots, we use the words *snapshot* and *checkpoint* interchangeably.
 
@@ -79,12 +82,12 @@ Stream barriers are injected into the parallel data flow at the stream sources.
 (let's call it <i>S<sub>n</sub></i>) is the position in the source stream up to which the snapshot covers the data. For example, in Apache Kafka, this
 position would be the last record's offset in the partition. This position <i>S<sub>n</sub></i> is reported to the *checkpoint coordinator* (Flink's JobManager).
 
-The barriers then flow downstream. When an intermediate operator has received a barrier for snapshot *n* from all of its input streams, it emits itself a barrier
+The barriers then flow downstream. When an intermediate operator has received a barrier for snapshot *n* from all of its input streams, it emits a barrier
 for snapshot *n* into all of its outgoing streams. Once a sink operator (the end of a streaming DAG) has received the barrier *n* from all of its
-input streams, it acknowledges that snapshot *n* to the checkpoint coordinator. After all sinks acknowledged a snapshot, it is considered completed.
+input streams, it acknowledges that snapshot *n* to the checkpoint coordinator. After all sinks have acknowledged a snapshot, it is considered completed.
 
-When snapshot *n* is completed, it is certain that no records from before <i>S<sub>n</sub></i> will be needed any more from the source, because these records (and
-their descendant records) have passed through the entire data flow topology.
+Once snapshot *n* has been completed, the job will never again ask the source for records from before <i>S<sub>n</sub></i>, since at that point these records (and
+their descendant records) will have passed through the entire data flow topology.
 
 <div style="text-align: center">
   <img src="{{ site.baseurl }}/fig/stream_aligning.svg" alt="Aligning data streams at operators with multiple inputs" style="width:100%; padding-top:10px; padding-bottom:10px;" />
@@ -92,8 +95,8 @@ their descendant records) have passed through the entire data flow topology.
 
 Operators that receive more than one input stream need to *align* the input streams on the snapshot barriers. The figure above illustrates this:
 
-  - As soon as the operator received snapshot barrier *n* from an incoming stream, it cannot process any further records from that stream until it has received
-the barrier *n* from the other inputs as well. Otherwise, it would have mixed records that belong to snapshot *n* and with records that belong to snapshot *n+1*.
+  - As soon as the operator receives snapshot barrier *n* from an incoming stream, it cannot process any further records from that stream until it has received
+the barrier *n* from the other inputs as well. Otherwise, it would mix records that belong to snapshot *n* and with records that belong to snapshot *n+1*.
   - Streams that report barrier *n* are temporarily set aside. Records that are received from these streams are not processed, but put into an input buffer.
   - Once the last stream has received barrier *n*, the operator emits all pending outgoing records, and then emits snapshot *n* barriers itself.
   - After that, it resumes processing records from all input streams, processing records from the input buffers before processing the records from the streams.
@@ -103,10 +106,10 @@ the barrier *n* from the other inputs as well. Otherwise, it would have mixed re
 
 When operators contain any form of *state*, this state must be part of the snapshots as well. Operator state comes in different forms:
 
-  - *User-defined state*: This is state that is created and modified directly by the transformation functions (like `map()` or `filter()`). User-defined state can either be a simple variable in the function's java object, or the associated key/value state of a function (see [State in Streaming Applications]({{ site.baseurl }}/dev/stream/state.html) for details).
+  - *User-defined state*: This is state that is created and modified directly by the transformation functions (like `map()` or `filter()`). See [State in Streaming Applications]({{ site.baseurl }}/dev/stream/state.html) for details.
   - *System state*: This state refers to data buffers that are part of the operator's computation. A typical example for this state are the *window buffers*, inside which the system collects (and aggregates) records for windows until the window is evaluated and evicted.
 
-Operators snapshot their state at the point in time when they received all snapshot barriers from their input streams, before emitting the barriers to their output streams. At that point, all updates to the state from records before the barriers will have been made, and no updates that depend on records from after the barriers have been applied. Because the state of a snapshot may be potentially large, it is stored in a configurable *state backend*. By default, this is the JobManager's memory, but for serious setups, a distributed reliable storage should be configured (such as HDFS). After the state has been stored, the operator acknowledges the checkpoint, emits the snapshot barrier into the output streams, and proceeds.
+Operators snapshot their state at the point in time when they have received all snapshot barriers from their input streams, and before emitting the barriers to their output streams. At that point, all updates to the state from records before the barriers will have been made, and no updates that depend on records from after the barriers have been applied. Because the state of a snapshot may be large, it is stored in a configurable *[state backend]({{ site.baseurl }}/ops/state_backends.html)*. By default, this is the JobManager's memory, but for production use a distributed reliable storage should be configured (such as HDFS). After the state has been stored, the operator acknowledges the checkpoint, emits the snapshot barrier into the output streams, and proceeds.
 
 The resulting snapshot now contains:
 
@@ -120,7 +123,7 @@ The resulting snapshot now contains:
 
 ### Exactly Once vs. At Least Once
 
-The alignment step may add latency to the streaming program. Usually, this extra latency is in the order of a few milliseconds, but we have seen cases where the latency
+The alignment step may add latency to the streaming program. Usually, this extra latency is on the order of a few milliseconds, but we have seen cases where the latency
 of some outliers increased noticeably. For applications that require consistently super low latencies (few milliseconds) for all records, Flink has a switch to skip the
 stream alignment during a checkpoint. Checkpoint snapshots are still drawn as soon as an operator has seen the checkpoint barrier from each input.
 
@@ -138,9 +141,9 @@ in *at least once* mode.
 
 Note that the above described mechanism implies that operators stop processing input records while they are storing a snapshot of their state in the *state backend*. This *synchronous* state snapshot introduces a delay every time a snapshot is taken.
 
-It is possible to let an operator continue processing while it stores its state snapshot, effectively letting the state snapshots happen *asynchronously* in the background. To do that, the operator must be able to produce a state object that should be stored in a way such that further modifications to the operator state do not affect that state object. An example for that are *copy-on-write* style data structures, such as used for example in RocksDB.
+It is possible to let an operator continue processing while it stores its state snapshot, effectively letting the state snapshots happen *asynchronously* in the background. To do that, the operator must be able to produce a state object that should be stored in a way such that further modifications to the operator state do not affect that state object. For example, *copy-on-write* data structures, such as are used in RocksDB, have this behavior.
 
-After receiving the checkpoint barriers on its inputs, the operator starts the asynchronous snapshot copying of its state. It immediately emits the barrier to its outputs and continues with the regular stream processing. Once the background copy process has completed, it acknowledges the checkpoint to the checkpoint coordinator (the JobManager). The checkpoint is now only complete after all sinks received the barriers and all stateful operators acknowledged their completed backup (which may be later than the barriers reaching the sinks).
+After receiving the checkpoint barriers on its inputs, the operator starts the asynchronous snapshot copying of its state. It immediately emits the barrier to its outputs and continues with the regular stream processing. Once the background copy process has completed, it acknowledges the checkpoint to the checkpoint coordinator (the JobManager). The checkpoint is now only complete after all sinks have received the barriers and all stateful operators have acknowledged their completed backup (which may be after the barriers reach the sinks).
 
 See [State Backends]({{ site.baseurl }}/ops/state_backends.html) for details on the state snapshots.
 
@@ -153,6 +156,8 @@ stream from position <i>S<sub>k</sub></i>. For example in Apache Kafka, that mea
 
 If state was snapshotted incrementally, the operators start with the state of the latest full snapshot and then apply a series of incremental snapshot updates to that state.
 
+See [Restart Strategies]({{ site.baseurl }}/dev/restart_strategies.html) for more information.
+
 ## Operator Snapshot Implementation
 
 When operator snapshots are taken, there are two parts: the **synchronous** and the **asynchronous** parts.
@@ -163,4 +168,4 @@ is completed and the *asynchronous* part is pending. The asynchronous part is th
 Operators that checkpoint purely synchronously return an already completed `FutureTask`.
 If an asynchronous operation needs to be performed, it is executed in the `run()` method of that `FutureTask`.
 
-The tasks are cancelable, in order to release streams and other resource consuming handles.
+The tasks are cancelable, so that streams and other resource consuming handles can be released.

From 8a99e814c59e15960d9f17b800c1aa2992d1a162 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <trohrmann@apache.org>
Date: Thu, 11 May 2017 11:22:09 +0200
Subject: [PATCH 213/253] [hotfix] Use Flink's FileUtils in BlobUtils to have
 proper exception handling

---
 .../src/main/java/org/apache/flink/runtime/blob/BlobUtils.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
index b0cab78241adc..3c14f2fe7b08f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
@@ -19,13 +19,13 @@
 package org.apache.flink.runtime.blob;
 
 import com.google.common.io.BaseEncoding;
-import org.apache.commons.io.FileUtils;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.runtime.highavailability.zookeeper.ZooKeeperHaServices;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
+import org.apache.flink.util.FileUtils;
 import org.apache.flink.util.StringUtils;
 import org.slf4j.Logger;
 

From f13cb4882389ecff6ec0ef6ae8e248a44f54e71b Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Mon, 8 May 2017 20:58:01 +0200
Subject: [PATCH 214/253] [hotfix] [build] Move JSON dependency in flink-storm
 into proper section

---
 flink-contrib/flink-storm/pom.xml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml
index ec34ba991585c..5189b3b535332 100644
--- a/flink-contrib/flink-storm/pom.xml
+++ b/flink-contrib/flink-storm/pom.xml
@@ -164,6 +164,12 @@ under the License.
 			</exclusions>
 		</dependency>
 
+		<dependency>
+			<groupId>com.googlecode.json-simple</groupId>
+			<artifactId>json-simple</artifactId>
+			<version>1.1</version>
+		</dependency>
+
 		<!-- test dependencies -->
 
 		<dependency>
@@ -173,11 +179,6 @@ under the License.
 			<type>test-jar</type>
 			<scope>test</scope>
 		</dependency>
-		<dependency>
-			<groupId>com.googlecode.json-simple</groupId>
-			<artifactId>json-simple</artifactId>
-			<version>1.1</version>
-		</dependency>
 
 	</dependencies>
 

From 4ef8779c46d2a4688b92a01a68b72002e3bee406 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Mon, 8 May 2017 20:58:26 +0200
Subject: [PATCH 215/253] [FLINK-6501] [build] Add NOTICE transformers to
 shading

This makes sure that transitive NOTICE files are added to the shaded JAR files.
---
 pom.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/pom.xml b/pom.xml
index 10677a4b3c585..4e0bf525773b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1173,6 +1173,14 @@ under the License.
 									<shadedPattern>org.apache.flink.shaded.org.objectweb.asm</shadedPattern>
 								</relocation>
 							</relocations>
+							<transformers>
+								<!-- The service transformer is needed to merge META-INF/services files -->
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+								<!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+								<!-- The ApacheLicenseResourceTransformer prevents duplicate Apache Licenses -->
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+							</transformers>
 						</configuration>
 					</execution>
 				</executions>

From f7d41dac8db94a70e4f00a9b1866000310b6875f Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 10 May 2017 11:20:07 +0200
Subject: [PATCH 216/253] [FLINK-6515] [runtime] Fix classloading of
 JavaSerializer

---
 flink-runtime/pom.xml                         |  10 +-
 .../flink/runtime/state/JavaSerializer.java   |  17 ++-
 .../runtime/state/JavaSerializerTest.java     | 116 ++++++++++++++++
 .../flink/core/testutils/CommonTestUtils.java | 124 ++++++++++++++++++
 4 files changed, 257 insertions(+), 10 deletions(-)
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java

diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index b7ccace69e216..8abdc3c74d4da 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -162,7 +162,7 @@ under the License.
 			<version>${project.version}</version>
 		</dependency>
 
-		<!-- core dependencies -->
+		<!-- test dependencies -->
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
@@ -171,6 +171,14 @@ under the License.
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-core</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+			<type>test-jar</type>
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.curator</groupId>
 			<artifactId>curator-test</artifactId>
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
index d49b1d22112d6..5252b3de67adf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
@@ -24,12 +24,12 @@
 import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.FlinkRuntimeException;
 import org.apache.flink.util.InstantiationUtil;
 
 import java.io.IOException;
 import java.io.Serializable;
 
-@SuppressWarnings("serial")
 @Internal
 final class JavaSerializer<T extends Serializable> extends TypeSerializerSingleton<T> {
 
@@ -47,11 +47,10 @@ public T createInstance() {
 
 	@Override
 	public T copy(T from) {
-
 		try {
-			return InstantiationUtil.clone(from);
+			return InstantiationUtil.clone(from, Thread.currentThread().getContextClassLoader());
 		} catch (IOException | ClassNotFoundException e) {
-			throw new RuntimeException("Could not copy instance of " + from + '.', e);
+			throw new FlinkRuntimeException("Could not copy element via serialization: " + from, e);
 		}
 	}
 
@@ -62,7 +61,7 @@ public T copy(T from, T reuse) {
 
 	@Override
 	public int getLength() {
-		return 0;
+		return -1;
 	}
 
 	@Override
@@ -74,7 +73,8 @@ public void serialize(T record, DataOutputView target) throws IOException {
 	public T deserialize(DataInputView source) throws IOException {
 		try {
 			return InstantiationUtil.deserializeObject(
-					new DataInputViewStream(source), Thread.currentThread().getContextClassLoader());
+					new DataInputViewStream(source),
+					Thread.currentThread().getContextClassLoader());
 		} catch (ClassNotFoundException e) {
 			throw new IOException("Could not deserialize object.", e);
 		}
@@ -87,9 +87,8 @@ public T deserialize(T reuse, DataInputView source) throws IOException {
 
 	@Override
 	public void copy(DataInputView source, DataOutputView target) throws IOException {
-		int size = source.readInt();
-		target.writeInt(size);
-		target.write(source, size);
+		T tmp = deserialize(source);
+		serialize(tmp, target);
 	}
 
 	@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java
new file mode 100644
index 0000000000000..de6fbce15b8a8
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.typeutils.SerializerTestBase;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.Serializable;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+import static org.junit.Assert.*;
+
+/**
+ * A test that verifies that the {@link JavaSerializer} properly handles class loading. 
+ */
+public class JavaSerializerTest extends SerializerTestBase<Serializable> {
+
+	/** Class loader for the object that is not in the test class path */
+	private static final ClassLoader CLASS_LOADER = 
+			new URLClassLoader(new URL[0], JavaSerializerTest.class.getClassLoader());
+
+	/** An object that is not in the test class path */
+	private static final Serializable OBJECT_OUT_OF_CLASSPATH = 
+			CommonTestUtils.createObjectForClassNotInClassPath(CLASS_LOADER);
+
+	// ------------------------------------------------------------------------
+
+	private ClassLoader originalClassLoader;
+
+	@Before
+	public void setupClassLoader() {
+		originalClassLoader = Thread.currentThread().getContextClassLoader();
+		Thread.currentThread().setContextClassLoader(CLASS_LOADER);
+	}
+
+	@After
+	public void restoreOriginalClassLoader() {
+		Thread.currentThread().setContextClassLoader(originalClassLoader);
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Test
+	public void guardTest() {
+		// make sure that this test's assumptions hold
+		try {
+			Class.forName(OBJECT_OUT_OF_CLASSPATH.getClass().getName());
+			fail("Test ineffective: The test class that should not be on the classpath is actually on the classpath.");
+		} catch (ClassNotFoundException e) {
+			// expected
+		}
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	protected TypeSerializer<Serializable> createSerializer() {
+		Thread.currentThread().setContextClassLoader(CLASS_LOADER);
+		return new JavaSerializer<>();
+	}
+
+	@Override
+	protected int getLength() {
+		return -1;
+	}
+
+	@Override
+	protected Class<Serializable> getTypeClass() {
+		return Serializable.class;
+	}
+
+	@Override
+	protected Serializable[] getTestData() {
+		return new Serializable[] {
+				new Integer(42),
+				new File("/some/path/that/I/made/up"),
+
+				// an object that is not in the classpath
+				OBJECT_OUT_OF_CLASSPATH,
+
+				// an object that is in the classpath with a nested object not in the classpath
+				new Tuple1<>(OBJECT_OUT_OF_CLASSPATH)
+		};
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public void testInstantiate() {
+		// this serializer does not support instantiation
+	}
+}
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
index 639b0655f6cbc..cf2bb7f20b9a1 100644
--- a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
@@ -29,7 +29,12 @@
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
+import java.io.Serializable;
 import java.lang.reflect.Field;
+import java.security.CodeSource;
+import java.security.Permissions;
+import java.security.ProtectionDomain;
+import java.security.cert.Certificate;
 import java.util.Map;
 
 import static org.junit.Assert.fail;
@@ -178,4 +183,123 @@ public static void setEnv(Map<String, String> newenv, boolean clearExisting) {
 			throw new RuntimeException(e1);
 		}
 	}
+
+	// ------------------------------------------------------------------------
+	//  Testing of objects not in the application class loader
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Creates a new class that is not part of the classpath that the current JVM uses, and
+	 * instantiates it.
+	 *
+	 * <p>This method uses {@link #createClassNotInClassPath(ClassLoader)} to define the new class.
+	 *
+	 * @param targetClassLoader The class loader to attach the class to
+	 * @return The object instantiated from the newly defined class.
+	 */
+	public static Serializable createObjectForClassNotInClassPath(ClassLoader targetClassLoader) {
+		try {
+			Class<? extends Serializable> clazz = createClassNotInClassPath(targetClassLoader);
+			return clazz.newInstance();
+		}
+		catch (Exception e) {
+			throw new AssertionError("test setup broken", e);
+		}
+	}
+
+	/**
+	 * Creates a new class that is not part of the classpath that the current JVM uses.
+	 * The class is ad-hoc defined and attached to the given ClassLoader.
+	 *
+	 * @param targetClassLoader The class loader to attach the class to
+	 * @return The newly defined class
+	 */
+	public static Class<? extends Serializable> createClassNotInClassPath(ClassLoader targetClassLoader) {
+		final byte[] classData = {-54, -2, -70, -66, 0, 0, 0, 51, 0, 65, 10, 0, 15, 0, 43, 7, 0, 44,
+				10, 0, 2, 0, 43, 10, 0, 2, 0, 45, 9, 0, 7, 0, 46, 10, 0, 15, 0, 47, 7, 0, 48, 7, 0,
+				49, 10, 0, 8, 0, 43, 8, 0, 50, 10, 0, 8, 0, 51, 10, 0, 8, 0, 52, 10, 0, 8, 0, 53, 10,
+				0, 8, 0, 54, 7, 0, 55, 7, 0, 56, 1, 0, 16, 115, 101, 114, 105, 97, 108, 86, 101, 114,
+				115, 105, 111, 110, 85, 73, 68, 1, 0, 1, 74, 1, 0, 13, 67, 111, 110, 115, 116, 97, 110,
+				116, 86, 97, 108, 117, 101, 5, -1, -1, -1, -1, -1, -1, -1, -3, 1, 0, 6, 114, 97, 110,
+				100, 111, 109, 1, 0, 6, 60, 105, 110, 105, 116, 62, 1, 0, 3, 40, 41, 86, 1, 0, 4, 67,
+				111, 100, 101, 1, 0, 15, 76, 105, 110, 101, 78, 117, 109, 98, 101, 114, 84, 97, 98, 108,
+				101, 1, 0, 18, 76, 111, 99, 97, 108, 86, 97, 114, 105, 97, 98, 108, 101, 84, 97, 98,
+				108, 101, 1, 0, 4, 116, 104, 105, 115, 1, 0, 35, 76, 111, 114, 103, 47, 97, 112, 97, 99,
+				104, 101, 47, 102, 108, 105, 110, 107, 47, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108,
+				105, 122, 97, 98, 108, 101, 59, 1, 0, 6, 101, 113, 117, 97, 108, 115, 1, 0, 21, 40, 76,
+				106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 79, 98, 106, 101, 99, 116, 59, 41, 90, 1, 0,
+				1, 111, 1, 0, 18, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 79, 98, 106, 101, 99,
+				116, 59, 1, 0, 4, 116, 104, 97, 116, 1, 0, 13, 83, 116, 97, 99, 107, 77, 97, 112, 84, 97,
+				98, 108, 101, 7, 0, 48, 1, 0, 8, 104, 97, 115, 104, 67, 111, 100, 101, 1, 0, 3, 40, 41,
+				73, 1, 0, 8, 116, 111, 83, 116, 114, 105, 110, 103, 1, 0, 20, 40, 41, 76, 106, 97, 118, 97,
+				47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110, 103, 59, 1, 0, 10, 83, 111, 117, 114,
+				99, 101, 70, 105, 108, 101, 1, 0, 21, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108, 105,
+				122, 97, 98, 108, 101, 46, 106, 97, 118, 97, 12, 0, 23, 0, 24, 1, 0, 16, 106, 97, 118, 97,
+				47, 117, 116, 105, 108, 47, 82, 97, 110, 100, 111, 109, 12, 0, 57, 0, 58, 12, 0, 22, 0, 18,
+				12, 0, 59, 0, 60, 1, 0, 33, 111, 114, 103, 47, 97, 112, 97, 99, 104, 101, 47, 102, 108, 105,
+				110, 107, 47, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108, 105, 122, 97, 98, 108, 101, 1,
+				0, 23, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110, 103, 66, 117,
+				105, 108, 100, 101, 114, 1, 0, 24, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108, 105, 122,
+				97, 98, 108, 101, 123, 114, 97, 110, 100, 111, 109, 61, 12, 0, 61, 0, 62, 12, 0, 61, 0, 63,
+				12, 0, 61, 0, 64, 12, 0, 39, 0, 40, 1, 0, 16, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47,
+				79, 98, 106, 101, 99, 116, 1, 0, 20, 106, 97, 118, 97, 47, 105, 111, 47, 83, 101, 114, 105,
+				97, 108, 105, 122, 97, 98, 108, 101, 1, 0, 8, 110, 101, 120, 116, 76, 111, 110, 103, 1, 0,
+				3, 40, 41, 74, 1, 0, 8, 103, 101, 116, 67, 108, 97, 115, 115, 1, 0, 19, 40, 41, 76, 106, 97,
+				118, 97, 47, 108, 97, 110, 103, 47, 67, 108, 97, 115, 115, 59, 1, 0, 6, 97, 112, 112, 101,
+				110, 100, 1, 0, 45, 40, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105,
+				110, 103, 59, 41, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110,
+				103, 66, 117, 105, 108, 100, 101, 114, 59, 1, 0, 28, 40, 74, 41, 76, 106, 97, 118, 97, 47,
+				108, 97, 110, 103, 47, 83, 116, 114, 105, 110, 103, 66, 117, 105, 108, 100, 101, 114, 59, 1,
+				0, 28, 40, 67, 41, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110,
+				103, 66, 117, 105, 108, 100, 101, 114, 59, 0, 33, 0, 7, 0, 15, 0, 1, 0, 16, 0, 2, 0, 26, 0,
+				17, 0, 18, 0, 1, 0, 19, 0, 0, 0, 2, 0, 20, 0, 18, 0, 22, 0, 18, 0, 0, 0, 4, 0, 1, 0, 23, 0,
+				24, 0, 1, 0, 25, 0, 0, 0, 69, 0, 3, 0, 1, 0, 0, 0, 19, 42, -73, 0, 1, 42, -69, 0, 2, 89, -73,
+				0, 3, -74, 0, 4, -75, 0, 5, -79, 0, 0, 0, 2, 0, 26, 0, 0, 0, 14, 0, 3, 0, 0, 0, 30, 0, 4, 0,
+				31, 0, 18, 0, 32, 0, 27, 0, 0, 0, 12, 0, 1, 0, 0, 0, 19, 0, 28, 0, 29, 0, 0, 0, 1, 0, 30, 0,
+				31, 0, 1, 0, 25, 0, 0, 0, -116, 0, 4, 0, 3, 0, 0, 0, 47, 42, 43, -90, 0, 5, 4, -84, 43, -58,
+				0, 14, 42, -74, 0, 6, 43, -74, 0, 6, -91, 0, 5, 3, -84, 43, -64, 0, 7, 77, 42, -76, 0, 5, 44,
+				-76, 0, 5, -108, -102, 0, 7, 4, -89, 0, 4, 3, -84, 0, 0, 0, 3, 0, 26, 0, 0, 0, 18, 0, 4, 0, 0,
+				0, 36, 0, 7, 0, 37, 0, 24, 0, 39, 0, 29, 0, 40, 0, 27, 0, 0, 0, 32, 0, 3, 0, 0, 0, 47, 0, 28,
+				0, 29, 0, 0, 0, 0, 0, 47, 0, 32, 0, 33, 0, 1, 0, 29, 0, 18, 0, 34, 0, 29, 0, 2, 0, 35, 0, 0,
+				0, 13, 0, 5, 7, 14, 1, -4, 0, 20, 7, 0, 36, 64, 1, 0, 1, 0, 37, 0, 38, 0, 1, 0, 25, 0, 0, 0,
+				56, 0, 5, 0, 1, 0, 0, 0, 14, 42, -76, 0, 5, 42, -76, 0, 5, 16, 32, 125, -125, -120, -84, 0, 0,
+				0, 2, 0, 26, 0, 0, 0, 6, 0, 1, 0, 0, 0, 46, 0, 27, 0, 0, 0, 12, 0, 1, 0, 0, 0, 14, 0, 28, 0,
+				29, 0, 0, 0, 1, 0, 39, 0, 40, 0, 1, 0, 25, 0, 0, 0, 70, 0, 3, 0, 1, 0, 0, 0, 28, -69, 0, 8,
+				89, -73, 0, 9, 18, 10, -74, 0, 11, 42, -76, 0, 5, -74, 0, 12, 16, 125, -74, 0, 13, -74, 0, 14,
+				-80, 0, 0, 0, 2, 0, 26, 0, 0, 0, 6, 0, 1, 0, 0, 0, 51, 0, 27, 0, 0, 0, 12, 0, 1, 0, 0, 0, 28,
+				0, 28, 0, 29, 0, 0, 0, 1, 0, 41, 0, 0, 0, 2, 0, 42,};
+
+		try {
+			// define a class into the classloader
+			Class<?> clazz = getUnsafe().defineClass(
+					"org.apache.flink.TestSerializable",
+					classData, 0, classData.length,
+					targetClassLoader,
+					new ProtectionDomain(new CodeSource(null, (Certificate[]) null), new Permissions()));
+
+			return clazz.asSubclass(Serializable.class);
+		}
+		catch (Exception e) {
+			throw new AssertionError("test setup broken", e);
+		}
+	}
+
+	@SuppressWarnings("restriction")
+	private static sun.misc.Unsafe getUnsafe() {
+		try {
+			Field unsafeField = sun.misc.Unsafe.class.getDeclaredField("theUnsafe");
+			unsafeField.setAccessible(true);
+			return (sun.misc.Unsafe) unsafeField.get(null);
+		} catch (SecurityException e) {
+			throw new RuntimeException("Could not access the sun.misc.Unsafe handle, permission denied by security manager.", e);
+		} catch (NoSuchFieldException e) {
+			throw new RuntimeException("The static handle field in sun.misc.Unsafe was not found.");
+		} catch (IllegalArgumentException e) {
+			throw new RuntimeException("Bug: Illegal argument reflection access for static field.", e);
+		} catch (IllegalAccessException e) {
+			throw new RuntimeException("Access to sun.misc.Unsafe is forbidden by the runtime.", e);
+		} catch (Throwable t) {
+			throw new RuntimeException("Unclassified error while trying to access the sun.misc.Unsafe handle.", t);
+		}
+	}
 }

From 09dfbe5547326d69a9a66215acbed8325ac770f0 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 10 May 2017 11:28:55 +0200
Subject: [PATCH 217/253] [hotfix] [core] Minor code cleanups in JavaSerializer
 and SerializerTestBase

---
 .../typeutils/runtime/kryo/JavaSerializer.java   |  4 ++--
 .../api/common/typeutils/SerializerTestBase.java | 16 ++++++++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java
index a51647c11c738..711c8140721ef 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java
@@ -45,7 +45,7 @@ public class JavaSerializer<T> extends Serializer<T> {
 
 	public JavaSerializer() {}
 
-	@SuppressWarnings("unchecked")
+	@SuppressWarnings({"unchecked", "rawtypes"})
 	@Override
 	public void write(Kryo kryo, Output output, T o) {
 		try {
@@ -62,7 +62,7 @@ public void write(Kryo kryo, Output output, T o) {
 		}
 	}
 
-	@SuppressWarnings("unchecked")
+	@SuppressWarnings({"unchecked", "rawtypes"})
 	@Override
 	public T read(Kryo kryo, Input input, Class aClass) {
 		try {
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
index a84670349cb38..f2879ac7f783e 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
@@ -55,7 +55,13 @@
 public abstract class SerializerTestBase<T> extends TestLogger {
 	
 	protected abstract TypeSerializer<T> createSerializer();
-	
+
+	/**
+	 * Gets the expected length for the serializer's {@link TypeSerializer#getLength()} method.
+	 * 
+	 * <p>The expected length should be positive, for fix-length data types, or {@code -1} for
+	 * variable-length types.
+	 */
 	protected abstract int getLength();
 	
 	protected abstract Class<T> getTypeClass();
@@ -124,9 +130,15 @@ public void testSnapshotConfigurationAndReconfigure() throws Exception {
 	
 	@Test
 	public void testGetLength() {
+		final int len = getLength();
+
+		if (len == 0) {
+			fail("Broken serializer test base - zero length cannot be the expected length");
+		}
+
 		try {
 			TypeSerializer<T> serializer = getSerializer();
-			assertEquals(getLength(), serializer.getLength());
+			assertEquals(len, serializer.getLength());
 		}
 		catch (Exception e) {
 			System.err.println(e.getMessage());

From d36eb407ccc6f66a4ad817f0604e8bf7aa0039fa Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 10 May 2017 12:37:28 +0200
Subject: [PATCH 218/253] [hotfix] [tests] Add a lightweight test for
 classloading in the Kryo Serializer

---
 .../kryo/KryoSerializerClassLoadingTest.java  | 113 ++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java

diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java
new file mode 100644
index 0000000000000..9823e114f7bfa
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.SerializerTestBase;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.core.testutils.CommonTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+import static org.junit.Assert.fail;
+
+/**
+ * This test validates that the Kryo-based serializer handles classes with custom
+ * class loaders correctly.
+ */
+public class KryoSerializerClassLoadingTest extends SerializerTestBase<Object> {
+
+	/** Class loader for the object that is not in the test class path */
+	private static final ClassLoader CLASS_LOADER =
+			new URLClassLoader(new URL[0], KryoSerializerClassLoadingTest.class.getClassLoader());
+
+	/** An object that is not in the test class path */
+	private static final Serializable OBJECT_OUT_OF_CLASSPATH =
+			CommonTestUtils.createObjectForClassNotInClassPath(CLASS_LOADER);
+
+	// ------------------------------------------------------------------------
+
+	private ClassLoader originalClassLoader;
+
+	@Before
+	public void setupClassLoader() {
+		originalClassLoader = Thread.currentThread().getContextClassLoader();
+		Thread.currentThread().setContextClassLoader(CLASS_LOADER);
+	}
+
+	@After
+	public void restoreOriginalClassLoader() {
+		Thread.currentThread().setContextClassLoader(originalClassLoader);
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Test
+	public void guardTestAssumptions() {
+		try {
+			Class.forName(OBJECT_OUT_OF_CLASSPATH.getClass().getName());
+			fail("This test's assumptions are broken");
+		}
+		catch (ClassNotFoundException ignored) {
+			// expected
+		}
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	protected TypeSerializer<Object> createSerializer() {
+		return new KryoSerializer<>(Object.class, new ExecutionConfig());
+	}
+
+	@Override
+	protected int getLength() {
+		return -1;
+	}
+
+	@Override
+	protected Class<Object> getTypeClass() {
+		return Object.class;
+	}
+
+	@Override
+	protected Object[] getTestData() {
+		return new Object[] {
+				new Integer(7),
+
+				// an object whose class is not on the classpath
+				OBJECT_OUT_OF_CLASSPATH,
+
+				// an object whose class IS on the classpath with a nested object whose class
+				// is NOT on the classpath
+				new Tuple1<>(OBJECT_OUT_OF_CLASSPATH)
+		};
+	}
+
+	@Override
+	public void testInstantiate() {
+		// this serializer does not support instantiation
+	}
+}

From 77ce850444e928915b0510f76aff61ce692cdfaa Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 10 May 2017 16:39:53 +0200
Subject: [PATCH 219/253] [hotfix] [gelly] Support log output when running
 examples in the IDE

This adds the dependencies / config files necessary for log output
---
 flink-libraries/flink-gelly-examples/pom.xml  | 15 ++++++++++
 .../src/main/resources/log4j.properties       | 23 +++++++++++++++
 .../src/main/resources/logback.xml            | 29 +++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties
 create mode 100644 flink-libraries/flink-gelly-examples/src/main/resources/logback.xml

diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index 6fb0979d65744..009781f11271e 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -75,6 +75,21 @@
 			<scope>compile</scope>
 		</dependency>
 
+		<!-- Add a logging Framework, to make the examples produce -->
+		<!--             logs when executing in the IDE            -->
+
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<scope>compile</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<scope>compile</scope>
+		</dependency>
+
 		<!-- test dependencies -->
 
 		<dependency>
diff --git a/flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties b/flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties
new file mode 100644
index 0000000000000..da32ea0f44d37
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties
@@ -0,0 +1,23 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=INFO, console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
diff --git a/flink-libraries/flink-gelly-examples/src/main/resources/logback.xml b/flink-libraries/flink-gelly-examples/src/main/resources/logback.xml
new file mode 100644
index 0000000000000..95f2d0419393b
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/resources/logback.xml
@@ -0,0 +1,29 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<configuration>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+    </root>
+</configuration>
\ No newline at end of file

From b08dbe907586decf75c612a196278f03de710702 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 10 May 2017 16:42:15 +0200
Subject: [PATCH 220/253] [FLINK-6508] [build] Include licenses of
 bundled/shaded dependencies where required

---
 flink-core/packaged_licenses/LICENSE.asm.txt  | 31 ++++++++++++++++++
 flink-java/packaged_licenses/LICENSE.asm.txt  | 31 ++++++++++++++++++
 .../packaged_licenses/LICENSE.janino.txt      | 31 ++++++++++++++++++
 .../packaged_licenses/LICENSE.reflections.txt | 14 ++++++++
 .../packaged_licenses/LICENSE.asm.txt         | 31 ++++++++++++++++++
 flink-scala/packaged_licenses/LICENSE.asm.txt | 31 ++++++++++++++++++
 pom.xml                                       | 32 ++++++++++++++++++-
 7 files changed, 200 insertions(+), 1 deletion(-)
 create mode 100644 flink-core/packaged_licenses/LICENSE.asm.txt
 create mode 100644 flink-java/packaged_licenses/LICENSE.asm.txt
 create mode 100644 flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt
 create mode 100644 flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt
 create mode 100644 flink-runtime/packaged_licenses/LICENSE.asm.txt
 create mode 100644 flink-scala/packaged_licenses/LICENSE.asm.txt

diff --git a/flink-core/packaged_licenses/LICENSE.asm.txt b/flink-core/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000000000..62ffbccb6837a
--- /dev/null
+++ b/flink-core/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/flink-java/packaged_licenses/LICENSE.asm.txt b/flink-java/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000000000..62ffbccb6837a
--- /dev/null
+++ b/flink-java/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt b/flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt
new file mode 100644
index 0000000000000..ef871e2426218
--- /dev/null
+++ b/flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt
@@ -0,0 +1,31 @@
+Janino - An embedded Java[TM] compiler
+
+Copyright (c) 2001-2016, Arno Unkrig
+Copyright (c) 2015-2016  TIBCO Software Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+   2. Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials
+      provided with the distribution.
+   3. Neither the name of JANINO nor the names of its contributors
+      may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt b/flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt
new file mode 100644
index 0000000000000..236d111dfbccf
--- /dev/null
+++ b/flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt
@@ -0,0 +1,14 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.
+  
\ No newline at end of file
diff --git a/flink-runtime/packaged_licenses/LICENSE.asm.txt b/flink-runtime/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000000000..62ffbccb6837a
--- /dev/null
+++ b/flink-runtime/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/flink-scala/packaged_licenses/LICENSE.asm.txt b/flink-scala/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000000000..62ffbccb6837a
--- /dev/null
+++ b/flink-scala/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 4e0bf525773b2..4cc38956385dc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -912,6 +912,9 @@ under the License.
 						<exclude>**/resources/**/bootstrap*</exclude>
 						<exclude>flink-clients/src/main/resources/web-docs/js/*d3.js</exclude>
 
+						<!-- the licenses that are re-bundled -->
+						<exclude>**/packaged_licenses/LICENSE.*.txt</exclude>
+
 						<!-- web dashboard config JSON files -->
 						<exclude>flink-runtime-web/web-dashboard/package.json</exclude>
 						<exclude>flink-runtime-web/web-dashboard/bower.json</exclude>
@@ -1115,6 +1118,34 @@ under the License.
 				</executions>
 			</plugin>
 
+			<!-- make sure all licenses that need to be repackaged are in the target folder -->
+			<plugin>
+				<artifactId>maven-resources-plugin</artifactId>
+				<configuration>
+					<encoding>UTF-8</encoding>
+				</configuration>
+				<executions>
+					<execution>
+						<id>copy-packaged-licenses</id>
+						<phase>prepare-package</phase>
+						<goals>
+							<goal>copy-resources</goal>
+						</goals>
+						<configuration>
+							<outputDirectory>${basedir}/target/classes/META-INF/license/</outputDirectory>
+							<resources>
+								<resource>
+									<directory>${basedir}/packaged_licenses</directory>
+									<includes>
+										<include>LICENSE.*.txt</include>
+									</includes>
+								</resource>
+							</resources>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+
 			<!-- We use shading in all packages for relocating some classes, such as
 				Guava and ASM.
 				By doing so, users adding Flink as a dependency won't run into conflicts.
@@ -1185,7 +1216,6 @@ under the License.
 					</execution>
 				</executions>
 			</plugin>
-
 		</plugins>
 
 		<!-- Plugin configurations for plugins activated in sub-projects --> 

From b2dc961185c05d72e102975f57306662b81d9d4e Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Thu, 27 Apr 2017 14:43:18 -0400
Subject: [PATCH 221/253] [FLINK-6414] [build] Use scala.binary.version in
 place of change-scala-version.sh

Use scala.binary.version as defined in the parent POM and remove the
script to swap scala version identifiers.

This closes #3800
---
 flink-clients/pom.xml                         |   6 +-
 flink-connectors/flink-avro/pom.xml           |   6 +-
 .../flink-connector-cassandra/pom.xml         |  12 +-
 .../pom.xml                                   |  10 +-
 .../flink-connector-elasticsearch/pom.xml     |  12 +-
 .../flink-connector-elasticsearch2/pom.xml    |  12 +-
 .../flink-connector-elasticsearch5/pom.xml    |  12 +-
 .../flink-connector-filesystem/pom.xml        |  14 +--
 .../flink-connector-kafka-0.10/pom.xml        |  20 +--
 .../flink-connector-kafka-0.8/pom.xml         |  18 +--
 .../flink-connector-kafka-0.9/pom.xml         |  18 +--
 .../flink-connector-kafka-base/pom.xml        |  16 +--
 .../flink-connector-kinesis/pom.xml           |  12 +-
 flink-connectors/flink-connector-nifi/pom.xml |  10 +-
 .../flink-connector-rabbitmq/pom.xml          |   8 +-
 .../flink-connector-twitter/pom.xml           |   4 +-
 .../flink-hadoop-compatibility/pom.xml        |   6 +-
 flink-connectors/flink-hbase/pom.xml          |  18 +--
 flink-connectors/flink-hcatalog/pom.xml       |   2 +-
 flink-connectors/flink-jdbc/pom.xml           |   4 +-
 .../flink-connector-wikiedits/pom.xml         |   4 +-
 .../flink-statebackend-rocksdb/pom.xml        |  10 +-
 flink-contrib/flink-storm-examples/pom.xml    |  24 ++--
 flink-contrib/flink-storm/pom.xml             |   6 +-
 flink-contrib/flink-streaming-contrib/pom.xml |  10 +-
 flink-contrib/flink-tweet-inputformat/pom.xml |   6 +-
 flink-dist/pom.xml                            |  42 +++----
 flink-dist/src/main/assemblies/bin.xml        |   4 +-
 flink-dist/src/main/assemblies/opt.xml        |  24 ++--
 flink-examples/flink-examples-batch/pom.xml   |  22 ++--
 .../flink-examples-streaming/pom.xml          |  36 +++---
 flink-examples/flink-examples-table/pom.xml   |  10 +-
 flink-examples/pom.xml                        |   4 +-
 flink-fs-tests/pom.xml                        |  12 +-
 flink-java8/pom.xml                           |  12 +-
 flink-libraries/flink-cep-scala/pom.xml       |  14 +--
 flink-libraries/flink-cep/pom.xml             |  12 +-
 flink-libraries/flink-gelly-examples/pom.xml  |  14 +--
 flink-libraries/flink-gelly-scala/pom.xml     |  12 +-
 flink-libraries/flink-gelly/pom.xml           |   8 +-
 flink-libraries/flink-ml/pom.xml              |   8 +-
 flink-libraries/flink-python/pom.xml          |   6 +-
 flink-libraries/flink-table/pom.xml           |  14 +--
 flink-mesos/pom.xml                           |  10 +-
 flink-metrics/flink-metrics-datadog/pom.xml   |   2 +-
 .../flink-metrics-dropwizard/pom.xml          |   2 +-
 flink-metrics/flink-metrics-jmx/pom.xml       |   4 +-
 flink-metrics/flink-metrics-statsd/pom.xml    |   2 +-
 flink-optimizer/pom.xml                       |   4 +-
 .../resources/archetype-resources/pom.xml     |  30 ++---
 .../resources/archetype-resources/pom.xml     |  34 ++---
 flink-runtime-web/pom.xml                     |   8 +-
 flink-runtime/pom.xml                         |   2 +-
 flink-scala-shell/pom.xml                     |  14 +--
 flink-scala/pom.xml                           |   8 +-
 flink-streaming-java/pom.xml                  |   8 +-
 flink-streaming-scala/pom.xml                 |  14 +--
 .../flink-test-utils/pom.xml                  |   8 +-
 flink-tests/pom.xml                           |  36 +++---
 flink-yarn-tests/pom.xml                      |  12 +-
 flink-yarn/pom.xml                            |  10 +-
 tools/change-scala-version.sh                 | 117 ------------------
 tools/create_release_files.sh                 |  10 +-
 tools/deploy_to_maven.sh                      |   8 +-
 64 files changed, 376 insertions(+), 501 deletions(-)
 delete mode 100755 tools/change-scala-version.sh

diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 8dde2e9f67e28..deac7fcd55f1e 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-clients_2.10</artifactId>
+	<artifactId>flink-clients_${scala.binary.version}</artifactId>
 	<name>flink-clients</name>
 
 	<packaging>jar</packaging>
@@ -47,13 +47,13 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-optimizer_2.10</artifactId>
+			<artifactId>flink-optimizer_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
diff --git a/flink-connectors/flink-avro/pom.xml b/flink-connectors/flink-avro/pom.xml
index 170b3445987d1..ed99eac12a47a 100644
--- a/flink-connectors/flink-avro/pom.xml
+++ b/flink-connectors/flink-avro/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-avro_2.10</artifactId>
+	<artifactId>flink-avro_${scala.binary.version}</artifactId>
 	<name>flink-avro</name>
 
 	<packaging>jar</packaging>
@@ -62,14 +62,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-connectors/flink-connector-cassandra/pom.xml b/flink-connectors/flink-connector-cassandra/pom.xml
index d01c76984ebcf..2722c30f38bb7 100644
--- a/flink-connectors/flink-connector-cassandra/pom.xml
+++ b/flink-connectors/flink-connector-cassandra/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-cassandra_2.10</artifactId>
+	<artifactId>flink-connector-cassandra_${scala.binary.version}</artifactId>
 	<name>flink-connector-cassandra</name>
 
 	<packaging>jar</packaging>
@@ -94,7 +94,7 @@ under the License.
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -136,27 +136,27 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-connectors/flink-connector-elasticsearch-base/pom.xml b/flink-connectors/flink-connector-elasticsearch-base/pom.xml
index e183354568941..79b4f50906b1e 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch-base/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-elasticsearch-base_2.10</artifactId>
+	<artifactId>flink-connector-elasticsearch-base_${scala.binary.version}</artifactId>
 	<name>flink-connector-elasticsearch-base</name>
 
 	<packaging>jar</packaging>
@@ -46,7 +46,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -61,14 +61,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -76,7 +76,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-connectors/flink-connector-elasticsearch/pom.xml b/flink-connectors/flink-connector-elasticsearch/pom.xml
index 07028dfde03d4..93e4eb6f1716d 100644
--- a/flink-connectors/flink-connector-elasticsearch/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-elasticsearch_2.10</artifactId>
+	<artifactId>flink-connector-elasticsearch_${scala.binary.version}</artifactId>
 	<name>flink-connector-elasticsearch</name>
 
 	<packaging>jar</packaging>
@@ -46,14 +46,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-elasticsearch-base_2.10</artifactId>
+			<artifactId>flink-connector-elasticsearch-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -61,14 +61,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -76,7 +76,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-elasticsearch-base_2.10</artifactId>
+			<artifactId>flink-connector-elasticsearch-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml
index ec1b7da32c3d9..7e21b8fbc0423 100644
--- a/flink-connectors/flink-connector-elasticsearch2/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-elasticsearch2_2.10</artifactId>
+	<artifactId>flink-connector-elasticsearch2_${scala.binary.version}</artifactId>
 	<name>flink-connector-elasticsearch2</name>
 
 	<packaging>jar</packaging>
@@ -46,14 +46,14 @@ under the License.
  
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-elasticsearch-base_2.10</artifactId>
+			<artifactId>flink-connector-elasticsearch-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -68,14 +68,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -83,7 +83,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-elasticsearch-base_2.10</artifactId>
+			<artifactId>flink-connector-elasticsearch-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml
index 93d7bbe37c7fc..72a0d18ef9fcf 100644
--- a/flink-connectors/flink-connector-elasticsearch5/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-elasticsearch5_2.10</artifactId>
+	<artifactId>flink-connector-elasticsearch5_${scala.binary.version}</artifactId>
 	<name>flink-connector-elasticsearch5</name>
 
 	<packaging>jar</packaging>
@@ -46,14 +46,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-elasticsearch-base_2.10</artifactId>
+			<artifactId>flink-connector-elasticsearch-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<!-- Elasticsearch Java Client has been moved to a different module in 5.x -->
@@ -88,14 +88,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -103,7 +103,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-elasticsearch-base_2.10</artifactId>
+			<artifactId>flink-connector-elasticsearch-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<exclusion>
diff --git a/flink-connectors/flink-connector-filesystem/pom.xml b/flink-connectors/flink-connector-filesystem/pom.xml
index 28d996288c8f9..07b0ae11b798f 100644
--- a/flink-connectors/flink-connector-filesystem/pom.xml
+++ b/flink-connectors/flink-connector-filesystem/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-filesystem_2.10</artifactId>
+	<artifactId>flink-connector-filesystem_${scala.binary.version}</artifactId>
 	<name>flink-connector-filesystem</name>
 
 	<packaging>jar</packaging>
@@ -45,7 +45,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -68,21 +68,21 @@ under the License.
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-hadoop-compatibility_2.10</artifactId>
+			<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -90,7 +90,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -98,7 +98,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml
index c17386d31821d..231b22e6cb875 100644
--- a/flink-connectors/flink-connector-kafka-0.10/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-kafka-0.10_2.10</artifactId>
+	<artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
 	<name>flink-connector-kafka-0.10</name>
 
 	<packaging>jar</packaging>
@@ -46,7 +46,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-0.9_2.10</artifactId>
+			<artifactId>flink-connector-kafka-0.9_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<exclusion>
@@ -60,7 +60,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -75,7 +75,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 			<!-- Projects depending on this project,
@@ -87,7 +87,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -95,7 +95,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-0.9_2.10</artifactId>
+			<artifactId>flink-connector-kafka-0.9_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<!-- exclude Kafka dependencies -->
@@ -110,7 +110,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-base_2.10</artifactId>
+			<artifactId>flink-connector-kafka-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<!-- exclude Kafka dependencies -->
@@ -133,7 +133,7 @@ under the License.
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -141,14 +141,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-connectors/flink-connector-kafka-0.8/pom.xml b/flink-connectors/flink-connector-kafka-0.8/pom.xml
index ceccb3e400b73..5e2ed2d4338d2 100644
--- a/flink-connectors/flink-connector-kafka-0.8/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.8/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-kafka-0.8_2.10</artifactId>
+	<artifactId>flink-connector-kafka-0.8_${scala.binary.version}</artifactId>
 	<name>flink-connector-kafka-0.8</name>
 
 	<packaging>jar</packaging>
@@ -46,7 +46,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -60,13 +60,13 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-base_2.10</artifactId>
+			<artifactId>flink-connector-kafka-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 			<!-- Projects depending on this project,
@@ -122,7 +122,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -144,7 +144,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-base_2.10</artifactId>
+			<artifactId>flink-connector-kafka-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -152,7 +152,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -160,14 +160,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml
index de4dedbc3a7ec..0140353778eec 100644
--- a/flink-connectors/flink-connector-kafka-0.9/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-kafka-0.9_2.10</artifactId>
+	<artifactId>flink-connector-kafka-0.9_${scala.binary.version}</artifactId>
 	<name>flink-connector-kafka-0.9</name>
 
 	<packaging>jar</packaging>
@@ -46,14 +46,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-base_2.10</artifactId>
+			<artifactId>flink-connector-kafka-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<exclusion>
@@ -65,7 +65,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 			<!-- Projects depending on this project,
@@ -83,7 +83,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -91,7 +91,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-base_2.10</artifactId>
+			<artifactId>flink-connector-kafka-base_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<!-- exclude 0.8 dependencies -->
@@ -121,7 +121,7 @@ under the License.
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -129,14 +129,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml
index a6b23499482d4..2cc94b0b766b6 100644
--- a/flink-connectors/flink-connector-kafka-base/pom.xml
+++ b/flink-connectors/flink-connector-kafka-base/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-kafka-base_2.10</artifactId>
+	<artifactId>flink-connector-kafka-base_${scala.binary.version}</artifactId>
 	<name>flink-connector-kafka-base</name>
 
 	<packaging>jar</packaging>
@@ -46,14 +46,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 			<!-- Projects depending on this project,
@@ -132,7 +132,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -140,14 +140,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -155,7 +155,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -163,7 +163,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-avro_2.10</artifactId>
+			<artifactId>flink-avro_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-connectors/flink-connector-kinesis/pom.xml b/flink-connectors/flink-connector-kinesis/pom.xml
index 7515f3160a985..462893784dcb5 100644
--- a/flink-connectors/flink-connector-kinesis/pom.xml
+++ b/flink-connectors/flink-connector-kinesis/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-kinesis_2.10</artifactId>
+	<artifactId>flink-connector-kinesis_${scala.binary.version}</artifactId>
 	<name>flink-connector-kinesis</name>
 	<properties>
 		<aws.sdk.version>1.10.71</aws.sdk.version>
@@ -44,7 +44,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -57,7 +57,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -65,7 +65,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -73,7 +73,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -81,7 +81,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-connectors/flink-connector-nifi/pom.xml b/flink-connectors/flink-connector-nifi/pom.xml
index 67bfb29928040..efa7ae7727905 100644
--- a/flink-connectors/flink-connector-nifi/pom.xml
+++ b/flink-connectors/flink-connector-nifi/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-nifi_2.10</artifactId>
+	<artifactId>flink-connector-nifi_${scala.binary.version}</artifactId>
 	<name>flink-connector-nifi</name>
 
 	<packaging>jar</packaging>
@@ -48,26 +48,26 @@ under the License.
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-java_2.10</artifactId>
+            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
 			<scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-java_2.10</artifactId>
+            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
             <type>test-jar</type>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-tests_2.10</artifactId>
+            <artifactId>flink-tests_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-test-utils_2.10</artifactId>
+            <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
 			<type>test-jar</type>
             <scope>test</scope>
diff --git a/flink-connectors/flink-connector-rabbitmq/pom.xml b/flink-connectors/flink-connector-rabbitmq/pom.xml
index f26e57c99ca93..38337f46389b8 100644
--- a/flink-connectors/flink-connector-rabbitmq/pom.xml
+++ b/flink-connectors/flink-connector-rabbitmq/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-rabbitmq_2.10</artifactId>
+	<artifactId>flink-connector-rabbitmq_${scala.binary.version}</artifactId>
 	<name>flink-connector-rabbitmq</name>
 
 	<packaging>jar</packaging>
@@ -44,7 +44,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -57,7 +57,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -65,7 +65,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-connectors/flink-connector-twitter/pom.xml b/flink-connectors/flink-connector-twitter/pom.xml
index c683da13820a5..38bfb30e0321d 100644
--- a/flink-connectors/flink-connector-twitter/pom.xml
+++ b/flink-connectors/flink-connector-twitter/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-twitter_2.10</artifactId>
+	<artifactId>flink-connector-twitter_${scala.binary.version}</artifactId>
 	<name>flink-connector-twitter</name>
 
 	<packaging>jar</packaging>
@@ -43,7 +43,7 @@ under the License.
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
index 003225b3057e3..2dee17d4e2643 100644
--- a/flink-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-hadoop-compatibility_2.10</artifactId>
+	<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
 	<name>flink-hadoop-compatibility</name>
 
 	<packaging>jar</packaging>
@@ -47,7 +47,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -70,7 +70,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml
index 6a295a8ef345a..949e27a020a2d 100644
--- a/flink-connectors/flink-hbase/pom.xml
+++ b/flink-connectors/flink-hbase/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-hbase_2.10</artifactId>
+	<artifactId>flink-hbase_${scala.binary.version}</artifactId>
 	<name>flink-hbase</name>
 	<packaging>jar</packaging>
 
@@ -89,7 +89,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -98,7 +98,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -107,7 +107,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 			<!-- Projects depending on this project,
@@ -124,7 +124,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 
@@ -206,7 +206,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<exclusions>
@@ -219,13 +219,13 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-hadoop-compatibility_2.10</artifactId>
+			<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<exclusions>
 				<exclusion>
 					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-shaded-include-yarn_2.10</artifactId>
+					<artifactId>flink-shaded-include-yarn_${scala.binary.version}</artifactId>
 				</exclusion>
 			</exclusions>
 		</dependency>
@@ -271,7 +271,7 @@ under the License.
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml
index a975e53776128..a9fbceb924edf 100644
--- a/flink-connectors/flink-hcatalog/pom.xml
+++ b/flink-connectors/flink-hcatalog/pom.xml
@@ -44,7 +44,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-hadoop-compatibility_2.10</artifactId>
+			<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
diff --git a/flink-connectors/flink-jdbc/pom.xml b/flink-connectors/flink-jdbc/pom.xml
index 38a0163559c9b..a2bbaf4680c8d 100644
--- a/flink-connectors/flink-jdbc/pom.xml
+++ b/flink-connectors/flink-jdbc/pom.xml
@@ -39,7 +39,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 			<!-- Projects depending on this project,
@@ -56,7 +56,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-contrib/flink-connector-wikiedits/pom.xml b/flink-contrib/flink-connector-wikiedits/pom.xml
index fed8656ef3fee..20b6db498dff2 100644
--- a/flink-contrib/flink-connector-wikiedits/pom.xml
+++ b/flink-contrib/flink-connector-wikiedits/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-connector-wikiedits_2.10</artifactId>
+	<artifactId>flink-connector-wikiedits_${scala.binary.version}</artifactId>
 	<name>flink-connector-wikiedits</name>
 
 	<packaging>jar</packaging>
@@ -37,7 +37,7 @@ under the License.
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
diff --git a/flink-contrib/flink-statebackend-rocksdb/pom.xml b/flink-contrib/flink-statebackend-rocksdb/pom.xml
index 2148706807f65..527ca185c2b0c 100644
--- a/flink-contrib/flink-statebackend-rocksdb/pom.xml
+++ b/flink-contrib/flink-statebackend-rocksdb/pom.xml
@@ -31,7 +31,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-statebackend-rocksdb_2.10</artifactId>
+	<artifactId>flink-statebackend-rocksdb_${scala.binary.version}</artifactId>
 	<name>flink-statebackend-rocksdb</name>
 
 	<packaging>jar</packaging>
@@ -42,14 +42,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -78,7 +78,7 @@ under the License.
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -86,7 +86,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml
index 4d9d76cae5eea..6ef0f7b653bae 100644
--- a/flink-contrib/flink-storm-examples/pom.xml
+++ b/flink-contrib/flink-storm-examples/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-storm-examples_2.10</artifactId>
+	<artifactId>flink-storm-examples_${scala.binary.version}</artifactId>
 	<name>flink-storm-examples</name>
 
 	<packaging>jar</packaging>
@@ -56,19 +56,19 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-storm_2.10</artifactId>
+			<artifactId>flink-storm_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-examples-batch_2.10</artifactId>
+			<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -95,14 +95,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -128,7 +128,7 @@ under the License.
 							<artifactItems>
 								<artifactItem>
 									<groupId>org.apache.flink</groupId>
-									<artifactId>flink-examples-batch_2.10</artifactId>
+									<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 									<version>${project.version}</version>
 									<type>jar</type>
 									<overWrite>false</overWrite>
@@ -138,7 +138,7 @@ under the License.
 								</artifactItem>
 								<artifactItem>
 									<groupId>org.apache.flink</groupId>
-									<artifactId>flink-storm_2.10</artifactId>
+									<artifactId>flink-storm_${scala.binary.version}</artifactId>
 									<version>${project.version}</version>
 									<type>jar</type>
 									<overWrite>false</overWrite>
@@ -326,8 +326,8 @@ under the License.
 									<!-- Storm's recursive dependencies -->
 									<include>org.yaml:snakeyaml</include>
 									<include>com.googlecode.json-simple:json-simple</include>
-									<include>org.apache.flink:flink-storm_2.10</include>
-									<include>org.apache.flink:flink-storm-examples_2.10</include>
+									<include>org.apache.flink:flink-storm_${scala.binary.version}</include>
+									<include>org.apache.flink:flink-storm-examples_${scala.binary.version}</include>
 								</includes>
 							</artifactSet>
 							<filters>
@@ -354,7 +354,7 @@ under the License.
 									</includes>
 								</filter>
 								<filter>
-									<artifact>org.apache.flink:flink-storm-examples_2.10</artifact>
+									<artifact>org.apache.flink:flink-storm-examples_${scala.binary.version}</artifact>
 									<includes>
 										<include>org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.class
 										</include>
@@ -366,7 +366,7 @@ under the License.
 									</includes>
 								</filter>
 								<filter>
-									<artifact>org.apache.flink:flink-storm_2.10</artifact>
+									<artifact>org.apache.flink:flink-storm_${scala.binary.version}</artifact>
 									<includes>
 										<include>org/apache/flink/storm/api/*.class</include>
 										<include>org/apache/flink/storm/util/*.class</include>
diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml
index 5189b3b535332..a10ff68bc3d84 100644
--- a/flink-contrib/flink-storm/pom.xml
+++ b/flink-contrib/flink-storm/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-storm_2.10</artifactId>
+	<artifactId>flink-storm_${scala.binary.version}</artifactId>
 	<name>flink-storm</name>
 
 	<packaging>jar</packaging>
@@ -65,7 +65,7 @@ under the License.
 		<!-- Core streaming API -->
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -174,7 +174,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-contrib/flink-streaming-contrib/pom.xml b/flink-contrib/flink-streaming-contrib/pom.xml
index 2df0231a2501c..2381d707ea653 100644
--- a/flink-contrib/flink-streaming-contrib/pom.xml
+++ b/flink-contrib/flink-streaming-contrib/pom.xml
@@ -31,7 +31,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-streaming-contrib_2.10</artifactId>
+	<artifactId>flink-streaming-contrib_${scala.binary.version}</artifactId>
 	<name>flink-streaming-contrib</name>
 
 	<packaging>jar</packaging>
@@ -42,19 +42,19 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -62,7 +62,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-contrib/flink-tweet-inputformat/pom.xml b/flink-contrib/flink-tweet-inputformat/pom.xml
index 5e0d79ce68e17..9328bb612bbfd 100644
--- a/flink-contrib/flink-tweet-inputformat/pom.xml
+++ b/flink-contrib/flink-tweet-inputformat/pom.xml
@@ -31,7 +31,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-tweet-inputformat_2.10</artifactId>
+	<artifactId>flink-tweet-inputformat_${scala.binary.version}</artifactId>
 	<name>flink-tweet-inputformat</name>
 
 	<packaging>jar</packaging>
@@ -44,12 +44,12 @@ under the License.
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 6d8debfdba217..0493b9829c254 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-dist_2.10</artifactId>
+	<artifactId>flink-dist_${scala.binary.version}</artifactId>
 	<name>flink-dist</name>
 	<packaging>jar</packaging>
 
@@ -51,61 +51,61 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime-web_2.10</artifactId>
+			<artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-optimizer_2.10</artifactId>
+			<artifactId>flink-optimizer_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-avro_2.10</artifactId>
+			<artifactId>flink-avro_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-python_2.10</artifactId>
+			<artifactId>flink-python_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala-shell_2.10</artifactId>
+			<artifactId>flink-scala-shell_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -123,19 +123,19 @@ under the License.
         
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-mesos_2.10</artifactId>
+			<artifactId>flink-mesos_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-statebackend-rocksdb_2.10</artifactId>
+			<artifactId>flink-statebackend-rocksdb_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-yarn_2.10</artifactId>
+			<artifactId>flink-yarn_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -163,7 +163,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-gelly-examples_2.10</artifactId>
+			<artifactId>flink-gelly-examples_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -214,42 +214,42 @@ under the License.
 		<!-- start optional Flink libraries -->
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-cep_2.10</artifactId>
+			<artifactId>flink-cep_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-cep-scala_2.10</artifactId>
+			<artifactId>flink-cep-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-gelly_2.10</artifactId>
+			<artifactId>flink-gelly_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-gelly-scala_2.10</artifactId>
+			<artifactId>flink-gelly-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-ml_2.10</artifactId>
+			<artifactId>flink-ml_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index c3385e17d0e89..909beb43d1b55 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -39,7 +39,7 @@ under the License.
 			<useTransitiveFiltering>true</useTransitiveFiltering>
 
 			<includes>
-				<include>org.apache.flink:flink-python_2.10</include>
+				<include>org.apache.flink:flink-python_${scala.binary.version}</include>
 				<include>org.slf4j:slf4j-log4j12</include>
 				<include>log4j:log4j</include>
 			</includes>
@@ -49,7 +49,7 @@ under the License.
 	<files>
 		<!-- copy fat jar -->
 		<file>
-			<source>target/flink-dist_2.10-${project.version}.jar</source>
+			<source>target/flink-dist_${scala.binary.version}-${project.version}.jar</source>
 			<outputDirectory>lib/</outputDirectory>
 			<fileMode>0644</fileMode>
 		</file>
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 0386b92807c8b..23846be45150d 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -31,47 +31,47 @@
 	<files>
 		<!-- CEP -->
 		<file>
-			<source>../flink-libraries/flink-cep/target/flink-cep_2.10-${project.version}.jar</source>
+			<source>../flink-libraries/flink-cep/target/flink-cep_${scala.binary.version}-${project.version}.jar</source>
 			<outputDirectory>opt/</outputDirectory>
-			<destName>flink-cep_2.10-${project.version}.jar</destName>
+			<destName>flink-cep_${scala.binary.version}-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
 
 		<file>
-			<source>../flink-libraries/flink-cep-scala/target/flink-cep-scala_2.10-${project.version}.jar</source>
+			<source>../flink-libraries/flink-cep-scala/target/flink-cep-scala_${scala.binary.version}-${project.version}.jar</source>
 			<outputDirectory>opt/</outputDirectory>
-			<destName>flink-cep-scala_2.10-${project.version}.jar</destName>
+			<destName>flink-cep-scala_${scala.binary.version}-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
 
 		<!-- Gelly -->
 		<file>
-			<source>../flink-libraries/flink-gelly/target/flink-gelly_2.10-${project.version}.jar</source>
+			<source>../flink-libraries/flink-gelly/target/flink-gelly_${scala.binary.version}-${project.version}.jar</source>
 			<outputDirectory>opt/</outputDirectory>
-			<destName>flink-gelly_2.10-${project.version}.jar</destName>
+			<destName>flink-gelly_${scala.binary.version}-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
 
 		<file>
-			<source>../flink-libraries/flink-gelly-scala/target/flink-gelly-scala_2.10-${project.version}-jar-with-dependencies.jar</source>
+			<source>../flink-libraries/flink-gelly-scala/target/flink-gelly-scala_${scala.binary.version}-${project.version}-jar-with-dependencies.jar</source>
 			<outputDirectory>opt/</outputDirectory>
-			<destName>flink-gelly-scala_2.10-${project.version}.jar</destName>
+			<destName>flink-gelly-scala_${scala.binary.version}-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
 
 		<!-- TableAPI-->
 		<file>
-			<source>../flink-libraries/flink-table/target/flink-table_2.10-${project.version}.jar</source>
+			<source>../flink-libraries/flink-table/target/flink-table_${scala.binary.version}-${project.version}.jar</source>
 			<outputDirectory>opt/</outputDirectory>
-			<destName>flink-table_2.10-${project.version}.jar</destName>
+			<destName>flink-table_${scala.binary.version}-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
 
 		<!-- ML -->
 		<file>
-			<source>../flink-libraries/flink-ml/target/flink-ml_2.10-${project.version}-jar-with-dependencies.jar</source>
+			<source>../flink-libraries/flink-ml/target/flink-ml_${scala.binary.version}-${project.version}-jar-with-dependencies.jar</source>
 			<outputDirectory>opt/</outputDirectory>
-			<destName>flink-ml_2.10-${project.version}.jar</destName>
+			<destName>flink-ml_${scala.binary.version}-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
 
diff --git a/flink-examples/flink-examples-batch/pom.xml b/flink-examples/flink-examples-batch/pom.xml
index 269f5a60bdbd8..cbff7e260ce47 100644
--- a/flink-examples/flink-examples-batch/pom.xml
+++ b/flink-examples/flink-examples-batch/pom.xml
@@ -23,12 +23,12 @@ under the License.
 
 	<parent>
 		<groupId>org.apache.flink</groupId>
-		<artifactId>flink-examples_2.10</artifactId>
+		<artifactId>flink-examples_${scala.binary.version}</artifactId>
 		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-examples-batch_2.10</artifactId>
+	<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 	<name>flink-examples-batch</name>
 	<packaging>jar</packaging>
 
@@ -41,7 +41,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 	</dependencies>
@@ -363,14 +363,14 @@ under the License.
 						</goals>
 						<configuration> 
 							<target>
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-KMeans.jar" tofile="${project.basedir}/target/KMeans.jar" />
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-ConnectedComponents.jar" tofile="${project.basedir}/target/ConnectedComponents.jar" />
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-EnumTriangles.jar" tofile="${project.basedir}/target/EnumTriangles.jar" />
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-PageRank.jar" tofile="${project.basedir}/target/PageRank.jar" />
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-TransitiveClosure.jar" tofile="${project.basedir}/target/TransitiveClosure.jar" />
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-WebLogAnalysis.jar" tofile="${project.basedir}/target/WebLogAnalysis.jar" />
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-WordCount.jar" tofile="${project.basedir}/target/WordCount.jar" />
-								<copy file="${project.basedir}/target/flink-examples-batch_2.10-${project.version}-DistCp.jar" tofile="${project.basedir}/target/DistCp.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-KMeans.jar" tofile="${project.basedir}/target/KMeans.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-ConnectedComponents.jar" tofile="${project.basedir}/target/ConnectedComponents.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-EnumTriangles.jar" tofile="${project.basedir}/target/EnumTriangles.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-PageRank.jar" tofile="${project.basedir}/target/PageRank.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-TransitiveClosure.jar" tofile="${project.basedir}/target/TransitiveClosure.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-WebLogAnalysis.jar" tofile="${project.basedir}/target/WebLogAnalysis.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-WordCount.jar" tofile="${project.basedir}/target/WordCount.jar" />
+								<copy file="${project.basedir}/target/flink-examples-batch_${scala.binary.version}-${project.version}-DistCp.jar" tofile="${project.basedir}/target/DistCp.jar" />
 							</target>
 						</configuration>
 					</execution>
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index 2bacf252e107d..b52116b9ab13c 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -24,12 +24,12 @@ under the License.
 
 	<parent>
 		<groupId>org.apache.flink</groupId>
-		<artifactId>flink-examples_2.10</artifactId>
+		<artifactId>flink-examples_${scala.binary.version}</artifactId>
 		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-examples-streaming_2.10</artifactId>
+	<artifactId>flink-examples-streaming_${scala.binary.version}</artifactId>
 	<name>flink-examples-streaming</name>
 
 	<packaging>jar</packaging>
@@ -40,31 +40,31 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-examples-batch_2.10</artifactId>
+			<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-twitter_2.10</artifactId>
+			<artifactId>flink-connector-twitter_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-0.8_2.10</artifactId>
+			<artifactId>flink-connector-kafka-0.8_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -72,14 +72,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -116,7 +116,7 @@ under the License.
 								<!-- For WordCount example data -->
 								<artifactItem>
 									<groupId>org.apache.flink</groupId>
-									<artifactId>flink-examples-batch_2.10</artifactId>
+									<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 									<version>${project.version}</version>
 									<type>jar</type>
 									<overWrite>false</overWrite>
@@ -126,7 +126,7 @@ under the License.
 								<!-- For JSON utilities -->
 								<artifactItem>
 									<groupId>org.apache.flink</groupId>
-									<artifactId>flink-connector-twitter_2.10</artifactId>
+									<artifactId>flink-connector-twitter_${scala.binary.version}</artifactId>
 									<version>${project.version}</version>
 									<type>jar</type>
 									<overWrite>false</overWrite>
@@ -475,13 +475,13 @@ under the License.
 						</goals>
 						<configuration>
 							<target>
-								<copy file="${project.basedir}/target/flink-examples-streaming_2.10-${project.version}-IncrementalLearning.jar" tofile="${project.basedir}/target/IncrementalLearning.jar" />
-								<copy file="${project.basedir}/target/flink-examples-streaming_2.10-${project.version}-Iteration.jar" tofile="${project.basedir}/target/Iteration.jar" />
-								<copy file="${project.basedir}/target/flink-examples-streaming_2.10-${project.version}-SessionWindowing.jar" tofile="${project.basedir}/target/SessionWindowing.jar" />
-								<copy file="${project.basedir}/target/flink-examples-streaming_2.10-${project.version}-TopSpeedWindowing.jar" tofile="${project.basedir}/target/TopSpeedWindowing.jar" />
-								<copy file="${project.basedir}/target/flink-examples-streaming_2.10-${project.version}-WindowJoin.jar" tofile="${project.basedir}/target/WindowJoin.jar" />
-								<copy file="${project.basedir}/target/flink-examples-streaming_2.10-${project.version}-WordCount.jar" tofile="${project.basedir}/target/WordCount.jar" />
-								<copy file="${project.basedir}/target/flink-examples-streaming_2.10-${project.version}-SocketWindowWordCount.jar" tofile="${project.basedir}/target/SocketWindowWordCount.jar" />
+								<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-IncrementalLearning.jar" tofile="${project.basedir}/target/IncrementalLearning.jar" />
+								<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-Iteration.jar" tofile="${project.basedir}/target/Iteration.jar" />
+								<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-SessionWindowing.jar" tofile="${project.basedir}/target/SessionWindowing.jar" />
+								<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-TopSpeedWindowing.jar" tofile="${project.basedir}/target/TopSpeedWindowing.jar" />
+								<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-WindowJoin.jar" tofile="${project.basedir}/target/WindowJoin.jar" />
+								<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-WordCount.jar" tofile="${project.basedir}/target/WordCount.jar" />
+								<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-SocketWindowWordCount.jar" tofile="${project.basedir}/target/SocketWindowWordCount.jar" />
 							</target>
 						</configuration>
 					</execution>
diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml
index ce1c701da3e88..e59d8c68f69c3 100644
--- a/flink-examples/flink-examples-table/pom.xml
+++ b/flink-examples/flink-examples-table/pom.xml
@@ -24,27 +24,27 @@ under the License.
 
 	<parent>
 		<groupId>org.apache.flink</groupId>
-		<artifactId>flink-examples_2.10</artifactId>
+		<artifactId>flink-examples_${scala.binary.version}</artifactId>
 		<version>1.4-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
 	<name>flink-examples-table</name>
-	<artifactId>flink-examples-table_2.10</artifactId>
+	<artifactId>flink-examples-table_${scala.binary.version}</artifactId>
 	<packaging>jar</packaging>
 
 	<dependencies>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-table_2.10</artifactId>
+			<artifactId>flink-table_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -78,7 +78,7 @@ under the License.
 				<artifactId>maven-shade-plugin</artifactId>
 				<executions>
 					<execution>
-						<id>flink-table-examples_2.10</id>
+						<id>flink-table-examples_${scala.binary.version}</id>
 						<phase>package</phase>
 						<goals>
 							<goal>shade</goal>
diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml
index 409c8f1bda87b..644d9eb35f9ef 100644
--- a/flink-examples/pom.xml
+++ b/flink-examples/pom.xml
@@ -28,7 +28,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-examples_2.10</artifactId>
+	<artifactId>flink-examples_${scala.binary.version}</artifactId>
 	<name>flink-examples</name>
 	<packaging>pom</packaging>
 
@@ -44,7 +44,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml
index c354c801a0557..446d167eb8473 100644
--- a/flink-fs-tests/pom.xml
+++ b/flink-fs-tests/pom.xml
@@ -27,7 +27,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-fs-tests_2.10</artifactId>
+	<artifactId>flink-fs-tests_${scala.binary.version}</artifactId>
 	<name>flink-fs-tests</name>
 
 	<packaging>jar</packaging>
@@ -45,7 +45,7 @@ under the License.
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -53,28 +53,28 @@ under the License.
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-examples-batch_2.10</artifactId>
+			<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-avro_2.10</artifactId>
+			<artifactId>flink-avro_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-java8/pom.xml b/flink-java8/pom.xml
index 8de3b92ccac4b..fb7da026583f4 100644
--- a/flink-java8/pom.xml
+++ b/flink-java8/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-java8_2.10</artifactId>
+	<artifactId>flink-java8_${scala.binary.version}</artifactId>
 	<name>flink-java8</name>
 
 	<packaging>jar</packaging>
@@ -51,13 +51,13 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-examples-batch_2.10</artifactId>
+			<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -65,14 +65,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-cep_2.10</artifactId>
+			<artifactId>flink-cep_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
@@ -116,7 +116,7 @@ under the License.
 							<artifactItems>
 								<artifactItem>
 									<groupId>org.apache.flink</groupId>
-									<artifactId>flink-examples-batch_2.10</artifactId>
+									<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 									<version>${project.version}</version>
 									<type>jar</type>
 									<overWrite>false</overWrite>
diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml
index ca5bbced5c749..c9b4a46686afb 100644
--- a/flink-libraries/flink-cep-scala/pom.xml
+++ b/flink-libraries/flink-cep-scala/pom.xml
@@ -30,7 +30,7 @@ under the License.
         <relativePath>..</relativePath>
     </parent>
     
-    <artifactId>flink-cep-scala_2.10</artifactId>
+    <artifactId>flink-cep-scala_${scala.binary.version}</artifactId>
     <name>flink-cep-scala</name>
     <packaging>jar</packaging>
 
@@ -40,13 +40,13 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cep_2.10</artifactId>
+            <artifactId>flink-cep_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_2.10</artifactId>
+            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -83,14 +83,14 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-test-utils_2.10</artifactId>
+            <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-tests_2.10</artifactId>
+            <artifactId>flink-tests_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
             <type>test-jar</type>
@@ -98,7 +98,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_2.10</artifactId>
+            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
             <type>test-jar</type>
@@ -106,7 +106,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cep_2.10</artifactId>
+            <artifactId>flink-cep_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
             <type>test-jar</type>
diff --git a/flink-libraries/flink-cep/pom.xml b/flink-libraries/flink-cep/pom.xml
index 7f89c24539734..35045c03d9c19 100644
--- a/flink-libraries/flink-cep/pom.xml
+++ b/flink-libraries/flink-cep/pom.xml
@@ -30,7 +30,7 @@ under the License.
         <relativePath>..</relativePath>
     </parent>
 
-    <artifactId>flink-cep_2.10</artifactId>
+    <artifactId>flink-cep_${scala.binary.version}</artifactId>
     <name>flink-cep</name>
     <packaging>jar</packaging>
 
@@ -47,7 +47,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-java_2.10</artifactId>
+            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -62,14 +62,14 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-test-utils_2.10</artifactId>
+            <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
         
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-java_2.10</artifactId>
+            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <type>test-jar</type>
             <scope>test</scope>
@@ -77,7 +77,7 @@ under the License.
 
        <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-runtime_2.10</artifactId>
+            <artifactId>flink-runtime_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <type>test-jar</type>
             <scope>test</scope>
@@ -85,7 +85,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-statebackend-rocksdb_2.10</artifactId>
+            <artifactId>flink-statebackend-rocksdb_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index 009781f11271e..68ad0506d3444 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -27,7 +27,7 @@
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-gelly-examples_2.10</artifactId>
+	<artifactId>flink-gelly-examples_${scala.binary.version}</artifactId>
 	<name>flink-gelly-examples</name>
 	<packaging>jar</packaging>
 
@@ -42,22 +42,22 @@
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-gelly_2.10</artifactId>
+			<artifactId>flink-gelly_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-gelly-scala_2.10</artifactId>
+			<artifactId>flink-gelly-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -94,14 +94,14 @@
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
index 256fc53cd33ae..35b2188b89114 100644
--- a/flink-libraries/flink-gelly-scala/pom.xml
+++ b/flink-libraries/flink-gelly-scala/pom.xml
@@ -28,7 +28,7 @@ under the License.
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>flink-gelly-scala_2.10</artifactId>
+    <artifactId>flink-gelly-scala_${scala.binary.version}</artifactId>
     <name>flink-gelly-scala</name>
 
     <packaging>jar</packaging>
@@ -38,21 +38,21 @@ under the License.
         <!-- core dependencies -->
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-scala_2.10</artifactId>
+            <artifactId>flink-scala_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-clients_2.10</artifactId>
+            <artifactId>flink-clients_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-gelly_2.10</artifactId>
+            <artifactId>flink-gelly_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
 
@@ -87,7 +87,7 @@ under the License.
         
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-tests_2.10</artifactId>
+            <artifactId>flink-tests_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
             <type>test-jar</type>
@@ -95,7 +95,7 @@ under the License.
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-test-utils_2.10</artifactId>
+            <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
diff --git a/flink-libraries/flink-gelly/pom.xml b/flink-libraries/flink-gelly/pom.xml
index d656bb36131dc..d620a66a56fe7 100644
--- a/flink-libraries/flink-gelly/pom.xml
+++ b/flink-libraries/flink-gelly/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-gelly_2.10</artifactId>
+	<artifactId>flink-gelly_${scala.binary.version}</artifactId>
 	<name>flink-gelly</name>
 
 	<packaging>jar</packaging>
@@ -47,7 +47,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -62,7 +62,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
@@ -77,7 +77,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-optimizer_2.10</artifactId>
+			<artifactId>flink-optimizer_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-libraries/flink-ml/pom.xml b/flink-libraries/flink-ml/pom.xml
index 8aa84d22478c7..0f203b92d48e8 100644
--- a/flink-libraries/flink-ml/pom.xml
+++ b/flink-libraries/flink-ml/pom.xml
@@ -28,7 +28,7 @@
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-ml_2.10</artifactId>
+	<artifactId>flink-ml_${scala.binary.version}</artifactId>
 	<name>flink-ml</name>
 
 	<packaging>jar</packaging>
@@ -39,7 +39,7 @@
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -86,14 +86,14 @@
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-libraries/flink-python/pom.xml b/flink-libraries/flink-python/pom.xml
index 9d2f0c3c8b20a..5b6405e42cc15 100644
--- a/flink-libraries/flink-python/pom.xml
+++ b/flink-libraries/flink-python/pom.xml
@@ -27,7 +27,7 @@ under the License.
         <relativePath>..</relativePath>
     </parent>
 
-    <artifactId>flink-python_2.10</artifactId>
+    <artifactId>flink-python_${scala.binary.version}</artifactId>
     <name>flink-python</name>
     <packaging>jar</packaging>
 
@@ -69,7 +69,7 @@ under the License.
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-runtime_2.10</artifactId>
+            <artifactId>flink-runtime_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
 			<scope>provided</scope>
         </dependency>
@@ -78,7 +78,7 @@ under the License.
         
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-test-utils_2.10</artifactId>
+            <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml
index 8fa2ed27178d1..a34fa1b4992ea 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -27,7 +27,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-table_2.10</artifactId>
+	<artifactId>flink-table_${scala.binary.version}</artifactId>
 	<name>flink-table</name>
 
 	<packaging>jar</packaging>
@@ -38,7 +38,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -114,14 +114,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -137,20 +137,20 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-statebackend-rocksdb_2.10</artifactId>
+			<artifactId>flink-statebackend-rocksdb_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-mesos/pom.xml b/flink-mesos/pom.xml
index d03144fe8b1b4..a1ef9006d47bc 100644
--- a/flink-mesos/pom.xml
+++ b/flink-mesos/pom.xml
@@ -27,7 +27,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 	
-	<artifactId>flink-mesos_2.10</artifactId>
+	<artifactId>flink-mesos_${scala.binary.version}</artifactId>
 	<name>flink-mesos</name>
 	<packaging>jar</packaging>
 
@@ -38,13 +38,13 @@ under the License.
     <dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -130,7 +130,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -138,7 +138,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-metrics/flink-metrics-datadog/pom.xml b/flink-metrics/flink-metrics-datadog/pom.xml
index 0d473fc0743ce..656f308a923f0 100644
--- a/flink-metrics/flink-metrics-datadog/pom.xml
+++ b/flink-metrics/flink-metrics-datadog/pom.xml
@@ -63,7 +63,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-metrics/flink-metrics-dropwizard/pom.xml b/flink-metrics/flink-metrics-dropwizard/pom.xml
index 2ba12e65aa2a3..631d68e312243 100644
--- a/flink-metrics/flink-metrics-dropwizard/pom.xml
+++ b/flink-metrics/flink-metrics-dropwizard/pom.xml
@@ -58,7 +58,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-metrics/flink-metrics-jmx/pom.xml b/flink-metrics/flink-metrics-jmx/pom.xml
index d738fc4133d7d..1e8ecc8d42f99 100644
--- a/flink-metrics/flink-metrics-jmx/pom.xml
+++ b/flink-metrics/flink-metrics-jmx/pom.xml
@@ -49,7 +49,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -65,7 +65,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-metrics/flink-metrics-statsd/pom.xml b/flink-metrics/flink-metrics-statsd/pom.xml
index 80cfa324664ac..5b77f8de2a581 100644
--- a/flink-metrics/flink-metrics-statsd/pom.xml
+++ b/flink-metrics/flink-metrics-statsd/pom.xml
@@ -51,7 +51,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-optimizer/pom.xml b/flink-optimizer/pom.xml
index c347082777b0a..499487b4488a1 100644
--- a/flink-optimizer/pom.xml
+++ b/flink-optimizer/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-optimizer_2.10</artifactId>
+	<artifactId>flink-optimizer_${scala.binary.version}</artifactId>
 	<name>flink-optimizer</name>
 
 	<packaging>jar</packaging>
@@ -46,7 +46,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 2fa1835e88dd6..9991d2c75efe2 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -82,12 +82,12 @@ under the License.
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${flink.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${flink.version}</version>
 		</dependency>
 
@@ -124,13 +124,13 @@ under the License.
 				</dependency>
 				<dependency>
 					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-streaming-java_2.10</artifactId>
+					<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 					<version>${flink.version}</version>
 					<scope>provided</scope>
 				</dependency>
 				<dependency>
 					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-clients_2.10</artifactId>
+					<artifactId>flink-clients_${scala.binary.version}</artifactId>
 					<version>${flink.version}</version>
 					<scope>provided</scope>
 				</dependency>
@@ -201,20 +201,20 @@ under the License.
 									<exclude>org.apache.flink:flink-shaded-curator-recipes</exclude>
 									<exclude>org.apache.flink:flink-core</exclude>
 									<exclude>org.apache.flink:flink-java</exclude>
-									<exclude>org.apache.flink:flink-scala_2.10</exclude>
-									<exclude>org.apache.flink:flink-runtime_2.10</exclude>
-									<exclude>org.apache.flink:flink-optimizer_2.10</exclude>
-									<exclude>org.apache.flink:flink-clients_2.10</exclude>
-									<exclude>org.apache.flink:flink-avro_2.10</exclude>
-									<exclude>org.apache.flink:flink-examples-batch_2.10</exclude>
-									<exclude>org.apache.flink:flink-examples-streaming_2.10</exclude>
-									<exclude>org.apache.flink:flink-streaming-java_2.10</exclude>
-									<exclude>org.apache.flink:flink-streaming-scala_2.10</exclude>
-									<exclude>org.apache.flink:flink-scala-shell_2.10</exclude>
+									<exclude>org.apache.flink:flink-scala_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-runtime_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-optimizer_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-clients_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-avro_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-examples-batch_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-examples-streaming_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-streaming-java_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-streaming-scala_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-scala-shell_${scala.binary.version}</exclude>
 									<exclude>org.apache.flink:flink-python</exclude>
 									<exclude>org.apache.flink:flink-metrics-core</exclude>
 									<exclude>org.apache.flink:flink-metrics-jmx</exclude>
-									<exclude>org.apache.flink:flink-statebackend-rocksdb_2.10</exclude>
+									<exclude>org.apache.flink:flink-statebackend-rocksdb_${scala.binary.version}</exclude>
 
 									<!-- Also exclude very big transitive dependencies of Flink
 
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 189bbceea4f91..2139c6bc336d1 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -78,17 +78,17 @@ under the License.
 		<!-- Apache Flink dependencies -->
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${flink.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${flink.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${flink.version}</version>
 		</dependency>
 		
@@ -116,19 +116,19 @@ under the License.
 			<dependencies>
 				<dependency>
 					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-scala_2.10</artifactId>
+					<artifactId>flink-scala_${scala.binary.version}</artifactId>
 					<version>${flink.version}</version>
 					<scope>provided</scope>
 				</dependency>
 				<dependency>
 					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-streaming-scala_2.10</artifactId>
+					<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 					<version>${flink.version}</version>
 					<scope>provided</scope>
 				</dependency>
 				<dependency>
 					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-clients_2.10</artifactId>
+					<artifactId>flink-clients_${scala.binary.version}</artifactId>
 					<version>${flink.version}</version>
 					<scope>provided</scope>
 				</dependency>
@@ -202,20 +202,20 @@ under the License.
 									<exclude>org.apache.flink:flink-shaded-curator-recipes</exclude>
 									<exclude>org.apache.flink:flink-core</exclude>
 									<exclude>org.apache.flink:flink-java</exclude>
-									<exclude>org.apache.flink:flink-scala_2.10</exclude>
-									<exclude>org.apache.flink:flink-runtime_2.10</exclude>
-									<exclude>org.apache.flink:flink-optimizer_2.10</exclude>
-									<exclude>org.apache.flink:flink-clients_2.10</exclude>
-									<exclude>org.apache.flink:flink-avro_2.10</exclude>
-									<exclude>org.apache.flink:flink-examples-batch_2.10</exclude>
-									<exclude>org.apache.flink:flink-examples-streaming_2.10</exclude>
-									<exclude>org.apache.flink:flink-streaming-java_2.10</exclude>
-									<exclude>org.apache.flink:flink-streaming-scala_2.10</exclude>
-									<exclude>org.apache.flink:flink-scala-shell_2.10</exclude>
+									<exclude>org.apache.flink:flink-scala_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-runtime_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-optimizer_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-clients_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-avro_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-examples-batch_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-examples-streaming_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-streaming-java_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-streaming-scala_${scala.binary.version}</exclude>
+									<exclude>org.apache.flink:flink-scala-shell_${scala.binary.version}</exclude>
 									<exclude>org.apache.flink:flink-python</exclude>
 									<exclude>org.apache.flink:flink-metrics-core</exclude>
 									<exclude>org.apache.flink:flink-metrics-jmx</exclude>
-									<exclude>org.apache.flink:flink-statebackend-rocksdb_2.10</exclude>
+									<exclude>org.apache.flink:flink-statebackend-rocksdb_${scala.binary.version}</exclude>
 
 									<!-- Also exclude very big transitive dependencies of Flink
 
diff --git a/flink-runtime-web/pom.xml b/flink-runtime-web/pom.xml
index e7870f393f188..8a4671bc7b136 100644
--- a/flink-runtime-web/pom.xml
+++ b/flink-runtime-web/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-runtime-web_2.10</artifactId>
+	<artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
 	<name>flink-runtime-web</name>
 
 	<packaging>jar</packaging>
@@ -42,12 +42,12 @@ under the License.
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -99,7 +99,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 8abdc3c74d4da..a13a9851d1d89 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-runtime_2.10</artifactId>
+	<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 	<name>flink-runtime</name>
 
 	<packaging>jar</packaging>
diff --git a/flink-scala-shell/pom.xml b/flink-scala-shell/pom.xml
index 0aadfc6a664e4..c9acbec63936d 100644
--- a/flink-scala-shell/pom.xml
+++ b/flink-scala-shell/pom.xml
@@ -27,7 +27,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-scala-shell_2.10</artifactId>
+	<artifactId>flink-scala-shell_${scala.binary.version}</artifactId>
 	<name>flink-scala-shell</name>
 
 	<packaging>jar</packaging>
@@ -44,19 +44,19 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-scala_2.10</artifactId>
+			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -82,7 +82,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
@@ -214,14 +214,14 @@ under the License.
 			<id>scala-2.10</id>
 			<activation>
 				<property>
-					<!-- this is the default scala profile -->
+					<!-- only required for Scala 2.10 -->
 					<name>!scala-2.11</name>
 				</property>
 			</activation>
 			<dependencies>
 				<dependency>
 					<groupId>org.scalamacros</groupId>
-					<artifactId>quasiquotes_${scala.binary.version}</artifactId>
+					<artifactId>quasiquotes_2.10</artifactId>
 					<version>${scala.macros.version}</version>
 				</dependency>
 
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 7be1e292327bc..7fad4879d1a64 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -28,7 +28,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-scala_2.10</artifactId>
+	<artifactId>flink-scala_${scala.binary.version}</artifactId>
 	<name>flink-scala</name>
 	<packaging>jar</packaging>
 
@@ -82,7 +82,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -241,14 +241,14 @@ under the License.
 			<id>scala-2.10</id>
 			<activation>
 				<property>
-					<!-- this is the default scala profile -->
+					<!-- only required for Scala 2.10 -->
 					<name>!scala-2.11</name>
 				</property>
 			</activation>
 			<dependencies>
 				<dependency>
 					<groupId>org.scalamacros</groupId>
-					<artifactId>quasiquotes_${scala.binary.version}</artifactId>
+					<artifactId>quasiquotes_2.10</artifactId>
 					<version>${scala.macros.version}</version>
 				</dependency>
 			</dependencies>
diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml
index ea987af0a9894..46142decbcee1 100644
--- a/flink-streaming-java/pom.xml
+++ b/flink-streaming-java/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-streaming-java_2.10</artifactId>
+	<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 	<name>flink-streaming-java</name>
 
 	<packaging>jar</packaging>
@@ -46,13 +46,13 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -85,7 +85,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml
index 4ec12bdb56971..586dca760a947 100644
--- a/flink-streaming-scala/pom.xml
+++ b/flink-streaming-scala/pom.xml
@@ -28,7 +28,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-streaming-scala_2.10</artifactId>
+	<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 	<name>flink-streaming-scala</name>
 	<packaging>jar</packaging>
 
@@ -38,13 +38,13 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -85,14 +85,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -100,7 +100,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-tests_2.10</artifactId>
+			<artifactId>flink-tests_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -108,7 +108,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml
index 36bdcebc597e6..4f537948b6ff6 100644
--- a/flink-test-utils-parent/flink-test-utils/pom.xml
+++ b/flink-test-utils-parent/flink-test-utils/pom.xml
@@ -29,7 +29,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-test-utils_2.10</artifactId>
+	<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 	<name>flink-test-utils</name>
 
 	<packaging>jar</packaging>
@@ -45,21 +45,21 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>compile</scope>
 		</dependency>
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index b67edbbe9e75d..3c0b184ff97ec 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -30,7 +30,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-tests_2.10</artifactId>
+	<artifactId>flink-tests_${scala.binary.version}</artifactId>
 	<name>flink-tests</name>
 
 	<packaging>jar</packaging>
@@ -54,28 +54,28 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-optimizer_2.10</artifactId>
+			<artifactId>flink-optimizer_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime-web_2.10</artifactId>
+			<artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime-web_2.10</artifactId>
+			<artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -83,7 +83,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
@@ -97,14 +97,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-scala_2.10</artifactId>
+			<artifactId>flink-scala_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -112,21 +112,21 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-examples-batch_2.10</artifactId>
+			<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
@@ -141,14 +141,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-hadoop-compatibility_2.10</artifactId>
+			<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-hadoop-compatibility_2.10</artifactId>
+			<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -156,7 +156,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-avro_2.10</artifactId>
+			<artifactId>flink-avro_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -164,14 +164,14 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-avro_2.10</artifactId>
+			<artifactId>flink-avro_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-optimizer_2.10</artifactId>
+			<artifactId>flink-optimizer_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -179,7 +179,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -224,7 +224,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-statebackend-rocksdb_2.10</artifactId>
+			<artifactId>flink-statebackend-rocksdb_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index f296e8d76237f..626d886540649 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -33,7 +33,7 @@ under the License.
 	We need the YARN fat jar build by flink-dist for the tests.
 	-->
 	
-	<artifactId>flink-yarn-tests_2.10</artifactId>
+	<artifactId>flink-yarn-tests_${scala.binary.version}</artifactId>
 	<name>flink-yarn-tests</name>
 	<packaging>jar</packaging>
 
@@ -43,14 +43,14 @@ under the License.
 		
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 			<type>test-jar</type>
@@ -59,21 +59,21 @@ under the License.
 		<!-- Needed for the streaming wordcount example -->
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-java_2.10</artifactId>
+			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-yarn_2.10</artifactId>
+			<artifactId>flink-yarn_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-yarn_2.10</artifactId>
+			<artifactId>flink-yarn_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/flink-yarn/pom.xml b/flink-yarn/pom.xml
index 5946094dbd651..e97fee55246a0 100644
--- a/flink-yarn/pom.xml
+++ b/flink-yarn/pom.xml
@@ -27,7 +27,7 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 	
-	<artifactId>flink-yarn_2.10</artifactId>
+	<artifactId>flink-yarn_${scala.binary.version}</artifactId>
 	<name>flink-yarn</name>
 	<packaging>jar</packaging>
 
@@ -37,7 +37,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<exclusion>
@@ -49,7 +49,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients_2.10</artifactId>
+			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -61,7 +61,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-test-utils_2.10</artifactId>
+			<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
@@ -98,7 +98,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime_2.10</artifactId>
+			<artifactId>flink-runtime_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 			<type>test-jar</type>
 			<scope>test</scope>
diff --git a/tools/change-scala-version.sh b/tools/change-scala-version.sh
deleted file mode 100755
index 56c48c68c66b7..0000000000000
--- a/tools/change-scala-version.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This shell script is from Apache Spark with some modification.
-
-set -e
-
-VALID_VERSIONS=( 2.10 2.11 )
-
-usage() {
-  echo "Usage: $(basename $0) [-h|--help] <scala version to be used>
-where :
-  -h| --help Display this help text
-  valid scala version values : ${VALID_VERSIONS[*]}
-" 1>&2
-  exit 1
-}
-
-if [[ ($# -ne 1) || ( $1 == "--help") ||  $1 == "-h" ]]; then
-  usage
-fi
-
-TO_VERSION=$1
-
-check_scala_version() {
-  for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done
-  echo "Invalid Scala version: $1. Valid versions: ${VALID_VERSIONS[*]}" 1>&2
-  exit 1
-}
-
-check_scala_version "$TO_VERSION"
-
-if [ $TO_VERSION = "2.11" ]; then
-  FROM_SUFFIX="_2\.10"
-  TO_SUFFIX="_2\.11"
-else
-  FROM_SUFFIX="_2\.11"
-  TO_SUFFIX="_2\.10"
-fi
-
-sed_i() {
-  sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2"
-}
-
-export -f sed_i
-
-echo "sed_i 's/\(artifactId>flink.*'$FROM_SUFFIX'\)<\/artifactId>/\1'$TO_SUFFIX'<\/artifactId>/g' {}";
-
-BASEDIR=$(dirname $0)/..
-find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(artifactId>flink.*\)'$FROM_SUFFIX'<\/artifactId>/\1'$TO_SUFFIX'<\/artifactId>/g' {}" \;
-
-# fix for examples
-find "$BASEDIR/flink-examples/flink-examples-batch" -name 'pom.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(<copy file=\".*flink-examples-batch\)'$FROM_SUFFIX'/\1'$TO_SUFFIX'/g' {}" \;
-
-find "$BASEDIR/flink-examples/flink-examples-streaming" -name 'pom.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(<copy file=\".*flink-examples-streaming\)'$FROM_SUFFIX'/\1'$TO_SUFFIX'/g' {}" \;
-
-# fix for quickstart
-find "$BASEDIR/flink-quickstart" -name 'pom.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(<exclude>org\.apache\.flink:flink-.*\)'$FROM_SUFFIX'<\/exclude>/\1'$TO_SUFFIX'<\/exclude>/g' {}" \;
-
-# fix for flink-dist (bin.xml)
-find "$BASEDIR/flink-dist" -name 'bin.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(<source>.*flink-dist\)'$FROM_SUFFIX'/\1'$TO_SUFFIX'/g' {}" \;
-find "$BASEDIR/flink-dist" -name 'bin.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(<include>org\.apache\.flink:flink-.*\)'$FROM_SUFFIX'<\/include>/\1'$TO_SUFFIX'<\/include>/g' {}" \;
-
-# fix for flink-dist (opt.xml)
-find "$BASEDIR/flink-dist" -name 'opt.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(<source>.*flink-.*\)'$FROM_SUFFIX'/\1'$TO_SUFFIX'/g' {}" \;
-find "$BASEDIR/flink-dist" -name 'opt.xml' -not -path '*target*' -print \
-  -exec bash -c "sed_i 's/\(<destName>flink-.*\)'$FROM_SUFFIX'/\1'$TO_SUFFIX'/g' {}" \;
-
-# fix for shading curator with Scala 2.11
-find "$BASEDIR/flink-runtime" -name 'pom.xml' -not -path '*target*' -print \
-     -exec bash -c "sed_i 's/\(<include>org\.apache\.flink:flink-shaded-curator.*\)'$FROM_SUFFIX'<\/include>/\1'$TO_SUFFIX'<\/include>/g' {}" \;
-
-if [ "$TO_VERSION" == "2.11" ]; then
-  # set the profile activation to !scala-2.11 in parent pom, so that it activates by default
-  bash -c "sed_i 's/<name>scala-2.11<\/name>/<name>!scala-2.11<\/name>/g' $BASEDIR/pom.xml" \;
-  # set the profile activation in all sub modules to scala-2.11 (so that they are disabled by default)
-  find $BASEDIR/flink-* -name 'pom.xml' -not -path '*target*' -print \
-    -exec bash -c "sed_i 's/<name>!scala-2.11<\/name>/<name>scala-2.11<\/name>/g' {}" \;
-
-  # set the name of the shading artifact properly
-  bash -c "sed_i 's/\(shading-artifact.name>flink-shaded[a-z0-9\-]*\)'$FROM_SUFFIX'<\/shading-artifact.name>/\1'$TO_SUFFIX'<\/shading-artifact.name>/g' $BASEDIR/pom.xml" \;
-fi
-
-if [ "$TO_VERSION" == "2.10" ]; then
-  # do the opposite as above
-  bash -c "sed_i 's/<name>!scala-2.11<\/name>/<name>scala-2.11<\/name>/g' $BASEDIR/pom.xml" \;
-  # also for the other files
-  find $BASEDIR/flink-* -name 'pom.xml' -not -path '*target*' -print \
-    -exec bash -c "sed_i 's/<name>scala-2.11<\/name>/<name>!scala-2.11<\/name>/g' {}" \;
-
-  # unset shading artifact name
-  bash -c "sed_i 's/\(shading-artifact.name>flink-shaded[a-z0-9\-]*\)'$FROM_SUFFIX'<\/shading-artifact.name>/\1'$TO_SUFFIX'<\/shading-artifact.name>/g' $BASEDIR/pom.xml" \;
-fi
-
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
index f4c46735efab8..40d038c71e9c8 100755
--- a/tools/create_release_files.sh
+++ b/tools/create_release_files.sh
@@ -216,10 +216,9 @@ make_binary_release() {
 
   # make distribution
   cd "${dir_name}"
-  ./tools/change-scala-version.sh ${SCALA_VERSION}
 
   # enable release profile here (to check for the maven version)
-  $MVN clean package $FLAGS -DskipTests -Prelease -Dgpg.skip
+  $MVN clean package $FLAGS -DskipTests -Prelease,scala-${SCALA_VERSION} -Dgpg.skip
 
   cd flink-dist/target/flink-*-bin/
   tar czf "${dir_name}.tgz" flink-*
@@ -243,15 +242,12 @@ deploy_to_maven() {
   cp ../../deploysettings.xml .
   
   echo "Deploying Scala 2.11 version"
-  cd tools && ./change-scala-version.sh 2.11 && cd ..
-
-  $MVN clean deploy -Prelease,docs-and-source --settings deploysettings.xml -DskipTests -Dgpg.executable=$GPG -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10
+  $MVN clean deploy -Prelease,docs-and-source,scala-2.11 --settings deploysettings.xml -DskipTests -Dgpg.executable=$GPG -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10
   
   # It is important to first deploy scala 2.11 and then scala 2.10 so that the quickstarts (which are independent of the scala version)
   # are depending on scala 2.10.
   echo "Deploying Scala 2.10 version"
-  cd tools && ./change-scala-version.sh 2.10 && cd ..
-  $MVN clean deploy -Dgpg.executable=$GPG -Prelease,docs-and-source --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10
+  $MVN clean deploy -Prelease,docs-and-source,scala-2.10 --settings deploysettings.xml -DskipTests -Dgpg.executable=$GPG -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10
 }
 
 copy_data() {
diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh
index 16d529e7f76f4..32acc5a1c673f 100755
--- a/tools/deploy_to_maven.sh
+++ b/tools/deploy_to_maven.sh
@@ -89,18 +89,14 @@ if [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
 
     # hadoop2 scala 2.10
     echo "deploy standard version (hadoop2) for scala 2.10"
-    mvn ${MVN_SNAPSHOT_OPTS}
+    mvn ${MVN_SNAPSHOT_OPTS} -Pscala-2.10
     deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2"
 
     # hadoop2 scala 2.11
     echo "deploy hadoop2 version (standard) for scala 2.11"
-    ./tools/change-scala-version.sh 2.11
-    mvn ${MVN_SNAPSHOT_OPTS}
+    mvn ${MVN_SNAPSHOT_OPTS} -Pscala-2.11
     deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2_2.11"
 
-    echo "Changing back to scala 2.10"
-    ./tools/change-scala-version.sh 2.10
-
     exit 0
 else
     exit 1

From c2e9372629ce70784c54c168491b68a2d7708d39 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 10 May 2017 20:57:08 +0200
Subject: [PATCH 222/253] [hotfix] [tests] Share proper test mini cluster for
 tests in DistributedCacheTest

---
 .../DistributedCacheTest.java                 | 56 +++++++++++++++----
 1 file changed, 46 insertions(+), 10 deletions(-)

diff --git a/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java b/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
index 19bcf76d22164..21aa40a28f659 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
@@ -22,19 +22,30 @@
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.tuple.Tuple1;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
+import org.apache.flink.test.util.TestEnvironment;
 import org.apache.flink.util.Collector;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
-import static org.junit.Assert.assertTrue;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
-import java.io.*;
+import static org.junit.Assert.assertTrue;
 
-import java.util.*;
 
+public class DistributedCacheTest extends AbstractTestBase {
 
-public class DistributedCacheTest extends StreamingMultipleProgramsTestBase {
 	public static final String data
 			= "machen\n"
 			+ "zeit\n"
@@ -42,6 +53,31 @@ public class DistributedCacheTest extends StreamingMultipleProgramsTestBase {
 			+ "keiner\n"
 			+ "meine\n";
 
+	private static final int PARALLELISM = 4;
+
+	private static LocalFlinkMiniCluster cluster;
+
+	@BeforeClass
+	public static void setup() throws Exception {
+		cluster = TestBaseUtils.startCluster(1, PARALLELISM, false, false, true);
+		TestStreamEnvironment.setAsContext(cluster, PARALLELISM);
+		TestEnvironment.setAsContext(cluster, PARALLELISM);
+	}
+
+	@AfterClass
+	public static void teardown() throws Exception {
+		TestStreamEnvironment.unsetAsContext();
+		TestEnvironment.unsetAsContext();
+		TestBaseUtils.stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
+	}
+
+	// ------------------------------------------------------------------------
+
+	public DistributedCacheTest() {
+		super(new Configuration());
+	}
+	
+	// ------------------------------------------------------------------------
 
 	@Test
 	public void testStreamingDistributedCache() throws Exception {
@@ -68,12 +104,12 @@ public static class WordChecker extends RichFlatMapFunction<String, Tuple1<Strin
 		@Override
 		public void open(Configuration conf) throws IOException {
 			File file = getRuntimeContext().getDistributedCache().getFile("cache_test");
-			BufferedReader reader = new BufferedReader(new FileReader(file));
-			String tempString;
-			while ((tempString = reader.readLine()) != null) {
-				wordList.add(tempString);
+			try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
+				String tempString;
+				while ((tempString= reader.readLine()) != null) {
+					wordList.add(tempString);
+				}
 			}
-			reader.close();
 		}
 
 		@Override

From 873e7d5cf1df0e4ea9d130959d389633d46114aa Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 11 May 2017 10:55:22 +0200
Subject: [PATCH 223/253] [FLINK-5679] [tests] Refactor
 PartitionedStateCheckpointingITCase

  - Massively speeds up the test by using fewer test elements and better coordination
    of source throttling and checkpointing.

  - Makes the test compatible with Windows by using proper URI encoding

  - Drops use of deprecated ValueState constructors
---
 .../KeyedStateCheckpointingITCase.java        | 401 ++++++++++++++++++
 .../PartitionedStateCheckpointingITCase.java  | 308 --------------
 2 files changed, 401 insertions(+), 308 deletions(-)
 create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java
 delete mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java

diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java
new file mode 100644
index 0000000000000..147d38529931b
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.checkpointing;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
+import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.runtime.state.AbstractStateBackend;
+import org.apache.flink.runtime.state.CheckpointListener;
+import org.apache.flink.runtime.state.filesystem.FsStateBackend;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * A simple test that runs a streaming topology with checkpointing enabled.
+ * 
+ * The test triggers a failure after a while and verifies that, after
+ * completion, the state reflects the "exactly once" semantics.
+ * 
+ * It is designed to check partitioned states.
+ */
+@SuppressWarnings("serial")
+public class KeyedStateCheckpointingITCase extends TestLogger {
+
+	protected static final int MAX_MEM_STATE_SIZE = 10 * 1024 * 1024;
+
+	protected static final int NUM_STRINGS = 10_000;
+	protected static final int NUM_KEYS = 40;
+
+	protected static final int NUM_TASK_MANAGERS = 2;
+	protected static final int NUM_TASK_SLOTS = 2;
+	protected static final int PARALLELISM = NUM_TASK_MANAGERS * NUM_TASK_SLOTS;
+
+	// ------------------------------------------------------------------------
+
+	private static LocalFlinkMiniCluster cluster;
+
+	@BeforeClass
+	public static void startCluster() throws Exception {
+		Configuration config = new Configuration();
+		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS);
+		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS);
+		config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L);
+
+		cluster = new LocalFlinkMiniCluster(config, false);
+		cluster.start();
+	}
+
+	@AfterClass
+	public static void stopCluster() throws Exception{
+		if (cluster != null) {
+			cluster.stop();
+		}
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Rule
+	public final TemporaryFolder tmpFolder = new TemporaryFolder();
+
+	@Test
+	public void testWithMemoryBackendSync() throws Exception {
+		MemoryStateBackend syncMemBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE, false);
+		testProgramWithBackend(syncMemBackend);
+	}
+
+	@Test
+	public void testWithMemoryBackendAsync() throws Exception {
+		MemoryStateBackend asyncMemBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE, true);
+		testProgramWithBackend(asyncMemBackend);
+	}
+
+	@Test
+	public void testWithFsBackendSync() throws Exception {
+		FsStateBackend syncFsBackend = new FsStateBackend(tmpFolder.newFolder().toURI().toString(), false);
+		testProgramWithBackend(syncFsBackend);
+	}
+
+	@Test
+	public void testWithFsBackendAsync() throws Exception {
+		FsStateBackend asyncFsBackend = new FsStateBackend(tmpFolder.newFolder().toURI().toString(), true);
+		testProgramWithBackend(asyncFsBackend);
+	}
+
+	@Test
+	public void testWithRocksDbBackendFull() throws Exception {
+		RocksDBStateBackend fullRocksDbBackend = new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), false);
+		fullRocksDbBackend.setDbStoragePath(tmpFolder.newFolder().getAbsolutePath());
+
+		testProgramWithBackend(fullRocksDbBackend);
+	}
+
+	@Test
+	public void testWithRocksDbBackendIncremental() throws Exception {
+		RocksDBStateBackend incRocksDbBackend = new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), true);
+		incRocksDbBackend.setDbStoragePath(tmpFolder.newFolder().getAbsolutePath());
+
+		testProgramWithBackend(incRocksDbBackend);
+	}
+
+	// ------------------------------------------------------------------------
+
+	protected void testProgramWithBackend(AbstractStateBackend stateBackend) throws Exception {
+		assertEquals("Broken test setup", 0, (NUM_STRINGS / 2) % NUM_KEYS);
+
+		final StreamExecutionEnvironment env = new TestStreamEnvironment(cluster, PARALLELISM);
+		env.setParallelism(PARALLELISM);
+		env.enableCheckpointing(500);
+		env.getConfig().disableSysoutLogging();
+		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));
+
+		env.setStateBackend(stateBackend);
+
+		// compute when (randomly) the failure should happen
+		final int failurePosMin = (int) (0.6 * NUM_STRINGS / PARALLELISM);
+		final int failurePosMax = (int) (0.8 * NUM_STRINGS / PARALLELISM);
+		final int failurePos = (new Random().nextInt(failurePosMax - failurePosMin) + failurePosMin);
+
+		final DataStream<Integer> stream1 = env.addSource(
+				new IntGeneratingSourceFunction(NUM_STRINGS / 2, NUM_STRINGS / 4));
+
+		final DataStream<Integer> stream2 = env.addSource(
+				new IntGeneratingSourceFunction(NUM_STRINGS / 2, NUM_STRINGS / 4));
+
+		stream1.union(stream2)
+				.keyBy(new IdentityKeySelector<Integer>())
+				.map(new OnceFailingPartitionedSum(failurePos))
+				.keyBy(0)
+				.addSink(new CounterSink());
+
+		env.execute();
+
+		// verify that we counted exactly right
+		assertEquals(NUM_KEYS, CounterSink.ALL_COUNTS.size());
+		assertEquals(NUM_KEYS, OnceFailingPartitionedSum.ALL_SUMS.size());
+
+		for (Entry<Integer, Long> sum : OnceFailingPartitionedSum.ALL_SUMS.entrySet()) {
+			assertEquals((long) sum.getKey() * NUM_STRINGS / NUM_KEYS, sum.getValue().longValue());
+		}
+		for (long count : CounterSink.ALL_COUNTS.values()) {
+			assertEquals(NUM_STRINGS / NUM_KEYS, count);
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Custom Functions
+	// --------------------------------------------------------------------------------------------
+
+	/**
+	 * A source that generates a sequence of integers and throttles down until a checkpoint
+	 * has happened.
+	 */
+	private static class IntGeneratingSourceFunction extends RichParallelSourceFunction<Integer> 
+		implements ListCheckpointed<Integer>, CheckpointListener {
+
+		private final int numElements;
+		private final int checkpointLatestAt;
+
+		private int lastEmitted = -1;
+
+		private boolean checkpointHappened;
+
+		private volatile boolean isRunning = true;
+
+		IntGeneratingSourceFunction(int numElements, int checkpointLatestAt) {
+			this.numElements = numElements;
+			this.checkpointLatestAt = checkpointLatestAt;
+		}
+
+		@Override
+		public void run(SourceContext<Integer> ctx) throws Exception {
+			final Object lockingObject = ctx.getCheckpointLock();
+			final int step = getRuntimeContext().getNumberOfParallelSubtasks();
+
+			int nextElement = lastEmitted >= 0 ? lastEmitted + step :
+				getRuntimeContext().getIndexOfThisSubtask();
+
+			while (isRunning && nextElement < numElements) {
+
+				// throttle / block if we are still waiting for the checkpoint
+				if (!checkpointHappened) {
+					if (nextElement < checkpointLatestAt) {
+						// only throttle
+						Thread.sleep(1);
+					} else {
+						// hard block
+						synchronized (this) {
+							while (!checkpointHappened) {
+								this.wait();
+							}
+						}
+					}
+				}
+
+				//noinspection SynchronizationOnLocalVariableOrMethodParameter
+				synchronized (lockingObject) {
+					ctx.collect(nextElement % NUM_KEYS);
+					lastEmitted = nextElement;
+				}
+
+				nextElement += step;
+			}
+		}
+
+		@Override
+		public void cancel() {
+			isRunning = false;
+		}
+
+		@Override
+		public List<Integer> snapshotState(long checkpointId, long timestamp) throws Exception {
+			return Collections.singletonList(lastEmitted);
+		}
+
+		@Override
+		public void restoreState(List<Integer> state) throws Exception {
+			assertEquals("Test failed due to unexpected recovered state size", 1, state.size());
+			lastEmitted = state.get(0);
+			checkpointHappened = true;
+		}
+
+		@Override
+		public void notifyCheckpointComplete(long checkpointId) throws Exception {
+			synchronized (this) {
+				checkpointHappened = true;
+				this.notifyAll();
+			}
+		}
+	}
+
+	private static class OnceFailingPartitionedSum
+			extends RichMapFunction<Integer, Tuple2<Integer, Long>>
+			implements ListCheckpointed<Integer> {
+
+		private static final Map<Integer, Long> ALL_SUMS = new ConcurrentHashMap<>();
+
+		private final int failurePos;
+		private int count;
+
+		private boolean shouldFail = true;
+
+		private transient ValueState<Long> sum;
+
+		OnceFailingPartitionedSum(int failurePos) {
+			this.failurePos = failurePos;
+		}
+
+		@Override
+		public void open(Configuration parameters) throws IOException {
+			sum = getRuntimeContext().getState(new ValueStateDescriptor<>("my_state", Long.class));
+		}
+
+		@Override
+		public Tuple2<Integer, Long> map(Integer value) throws Exception {
+			if (shouldFail && count++ >= failurePos) {
+				shouldFail = false;
+				throw new Exception("Test Failure");
+			}
+
+			Long oldSum = sum.value();
+			long currentSum = (oldSum == null ? 0L : oldSum) + value;
+
+			sum.update(currentSum);
+			ALL_SUMS.put(value, currentSum);
+			return new Tuple2<>(value, currentSum);
+		}
+
+		@Override
+		public List<Integer> snapshotState(long checkpointId, long timestamp) throws Exception {
+			return Collections.singletonList(count);
+		}
+
+		@Override
+		public void restoreState(List<Integer> state) throws Exception {
+			assertEquals("Test failed due to unexpected recovered state size", 1, state.size());
+			count = state.get(0);
+			shouldFail = false;
+		}
+
+		@Override
+		public void close() throws Exception {
+			if (shouldFail) {
+				fail("Test ineffective: Function cleanly finished without ever failing.");
+			}
+		}
+	}
+
+	private static class CounterSink extends RichSinkFunction<Tuple2<Integer, Long>> {
+
+		private static final Map<Integer, Long> ALL_COUNTS = new ConcurrentHashMap<>();
+
+		private transient ValueState<NonSerializableLong> aCounts;
+		private transient ValueState<Long> bCounts;
+
+		@Override
+		public void open(Configuration parameters) throws IOException {
+			aCounts = getRuntimeContext().getState(new ValueStateDescriptor<>("a", NonSerializableLong.class));
+			bCounts = getRuntimeContext().getState(new ValueStateDescriptor<>("b", Long.class));
+		}
+
+		@Override
+		public void invoke(Tuple2<Integer, Long> value) throws Exception {
+			final NonSerializableLong acRaw = aCounts.value();
+			final Long bcRaw = bCounts.value();
+
+			final long ac = acRaw == null ? 0L : acRaw.value;
+			final long bc = bcRaw == null ? 0L : bcRaw;
+
+			assertEquals(ac, bc);
+
+			long currentCount = ac + 1;
+			aCounts.update(NonSerializableLong.of(currentCount));
+			bCounts.update(currentCount);
+
+			ALL_COUNTS.put(value.f0, currentCount);
+		}
+	}
+
+	public static class IdentityKeySelector<T> implements KeySelector<T, T> {
+
+		@Override
+		public T getKey(T value) throws Exception {
+			return value;
+		}
+	}
+
+	// ------------------------------------------------------------------------
+	//  data types
+	// ------------------------------------------------------------------------
+
+	public static class NonSerializableLong {
+
+		public long value;
+
+		private NonSerializableLong(long value) {
+			this.value = value;
+		}
+
+		public static NonSerializableLong of(long value) {
+			return new NonSerializableLong(value);
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return this == obj || 
+					obj != null && obj.getClass() == getClass() && ((NonSerializableLong) obj).value == this.value;
+		}
+
+		@Override
+		public int hashCode() {
+			return (int) (value ^ (value >>> 32));
+		}
+	}
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java
deleted file mode 100644
index 517c82b404146..0000000000000
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/PartitionedStateCheckpointingITCase.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.checkpointing;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.state.ValueState;
-import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
-import org.apache.flink.runtime.state.AbstractStateBackend;
-import org.apache.flink.runtime.state.filesystem.FsStateBackend;
-import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-/**
- * A simple test that runs a streaming topology with checkpointing enabled.
- * 
- * The test triggers a failure after a while and verifies that, after
- * completion, the state reflects the "exactly once" semantics.
- * 
- * It is designed to check partitioned states.
- */
-@SuppressWarnings("serial")
-@RunWith(Parameterized.class)
-public class PartitionedStateCheckpointingITCase extends StreamFaultToleranceTestBase {
-
-	private static final int MAX_MEM_STATE_SIZE = 10 * 1024 * 1024;
-
-	final long NUM_STRINGS = 10_000_000L;
-	final static int NUM_KEYS = 40;
-
-	@Parameterized.Parameters
-	public static Collection<AbstractStateBackend> parameters() throws IOException {
-		TemporaryFolder tempFolder = new TemporaryFolder();
-		tempFolder.create();
-
-		MemoryStateBackend syncMemBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE, false);
-		MemoryStateBackend asyncMemBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE, true);
-
-		FsStateBackend syncFsBackend = new FsStateBackend("file://" + tempFolder.newFolder().getAbsolutePath(), false);
-		FsStateBackend asyncFsBackend = new FsStateBackend("file://" + tempFolder.newFolder().getAbsolutePath(), true);
-
-		RocksDBStateBackend fullRocksDbBackend = new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), false);
-		fullRocksDbBackend.setDbStoragePath(tempFolder.newFolder().getAbsolutePath());
-
-		RocksDBStateBackend incRocksDbBackend = new RocksDBStateBackend(new MemoryStateBackend(MAX_MEM_STATE_SIZE), true);
-		incRocksDbBackend.setDbStoragePath(tempFolder.newFolder().getAbsolutePath());
-
-		return Arrays.asList(
-			syncMemBackend,
-			asyncMemBackend,
-			syncFsBackend,
-			asyncFsBackend,
-			fullRocksDbBackend,
-			incRocksDbBackend);
-	}
-
-	@Parameterized.Parameter
-	public AbstractStateBackend stateBackend;
-
-	@Override
-	public void testProgram(StreamExecutionEnvironment env) {
-		assertTrue("Broken test setup", (NUM_STRINGS/2) % NUM_KEYS == 0);
-
-		env.setStateBackend(stateBackend);
-
-		DataStream<Integer> stream1 = env.addSource(new IntGeneratingSourceFunction(NUM_STRINGS / 2));
-		DataStream<Integer> stream2 = env.addSource(new IntGeneratingSourceFunction(NUM_STRINGS / 2));
-
-		stream1.union(stream2)
-				.keyBy(new IdentityKeySelector<Integer>())
-				.map(new OnceFailingPartitionedSum(NUM_STRINGS))
-				.keyBy(0)
-				.addSink(new CounterSink());
-	}
-
-	@Override
-	public void postSubmit() {
-		// verify that we counted exactly right
-		for (Entry<Integer, Long> sum : OnceFailingPartitionedSum.allSums.entrySet()) {
-			assertEquals(new Long(sum.getKey() * NUM_STRINGS / NUM_KEYS), sum.getValue());
-		}
-		for (Long count : CounterSink.allCounts.values()) {
-			assertEquals(new Long(NUM_STRINGS / NUM_KEYS), count);
-		}
-
-		assertEquals(NUM_KEYS, CounterSink.allCounts.size());
-		assertEquals(NUM_KEYS, OnceFailingPartitionedSum.allSums.size());
-	}
-
-	// --------------------------------------------------------------------------------------------
-	// Custom Functions
-	// --------------------------------------------------------------------------------------------
-
-	private static class IntGeneratingSourceFunction extends RichParallelSourceFunction<Integer> 
-		implements ListCheckpointed<Integer> {
-
-		private final long numElements;
-
-		private int index;
-		private int step;
-
-		private volatile boolean isRunning = true;
-
-		static final long[] counts = new long[PARALLELISM];
-
-		@Override
-		public void close() throws IOException {
-			counts[getRuntimeContext().getIndexOfThisSubtask()] = index;
-		}
-
-		IntGeneratingSourceFunction(long numElements) {
-			this.numElements = numElements;
-		}
-
-		@Override
-		public void open(Configuration parameters) throws IOException {
-			step = getRuntimeContext().getNumberOfParallelSubtasks();
-			if (index == 0) {
-				index = getRuntimeContext().getIndexOfThisSubtask();
-			}
-		}
-
-		@Override
-		public void run(SourceContext<Integer> ctx) throws Exception {
-			final Object lockingObject = ctx.getCheckpointLock();
-
-			while (isRunning && index < numElements) {
-
-				synchronized (lockingObject) {
-					index += step;
-					ctx.collect(index % NUM_KEYS);
-				}
-			}
-		}
-
-		@Override
-		public void cancel() {
-			isRunning = false;
-		}
-
-		@Override
-		public List<Integer> snapshotState(long checkpointId, long timestamp) throws Exception {
-			return Collections.singletonList(this.index);
-		}
-
-		@Override
-		public void restoreState(List<Integer> state) throws Exception {
-			if (state.isEmpty() || state.size() > 1) {
-				throw new RuntimeException("Test failed due to unexpected recovered state size " + state.size());
-			}
-			this.index = state.get(0);
-		}
-	}
-
-	private static class OnceFailingPartitionedSum extends RichMapFunction<Integer, Tuple2<Integer, Long>> {
-
-		private static Map<Integer, Long> allSums = new ConcurrentHashMap<Integer, Long>();
-		
-		private static volatile boolean hasFailed = false;
-
-		private final long numElements;
-
-		private long failurePos;
-		private long count;
-
-		private ValueState<Long> sum;
-
-		OnceFailingPartitionedSum(long numElements) {
-			this.numElements = numElements;
-			this.hasFailed = false;
-		}
-
-		@Override
-		public void open(Configuration parameters) throws IOException {
-			long failurePosMin = (long) (0.6 * numElements / getRuntimeContext()
-					.getNumberOfParallelSubtasks());
-			long failurePosMax = (long) (0.8 * numElements / getRuntimeContext()
-					.getNumberOfParallelSubtasks());
-
-			failurePos = (new Random().nextLong() % (failurePosMax - failurePosMin)) + failurePosMin;
-			count = 0;
-			sum = getRuntimeContext().getState(
-					new ValueStateDescriptor<>("my_state", Long.class, 0L));
-		}
-
-		@Override
-		public Tuple2<Integer, Long> map(Integer value) throws Exception {
-			count++;
-
-			if (!hasFailed && count >= failurePos) {
-				hasFailed = true;
-				throw new Exception("Test Failure");
-			}
-
-			long currentSum = sum.value() + value;
-			sum.update(currentSum);
-			allSums.put(value, currentSum);
-			return new Tuple2<Integer, Long>(value, currentSum);
-		}
-	}
-
-	private static class CounterSink extends RichSinkFunction<Tuple2<Integer, Long>> {
-
-		private static Map<Integer, Long> allCounts = new ConcurrentHashMap<Integer, Long>();
-		
-		private ValueState<NonSerializableLong> aCounts;
-		private ValueState<Long> bCounts;
-
-		@Override
-		public void open(Configuration parameters) throws IOException {
-			
-			aCounts = getRuntimeContext().getState(
-					new ValueStateDescriptor<>("a", NonSerializableLong.class, NonSerializableLong.of(0L)));
-			
-			bCounts = getRuntimeContext().getState(
-					new ValueStateDescriptor<>("b", Long.class, 0L));
-		}
-
-		@Override
-		public void invoke(Tuple2<Integer, Long> value) throws Exception {
-			long ac = aCounts.value().value;
-			long bc = bCounts.value();
-			assertEquals(ac, bc);
-			
-			long currentCount = ac + 1;
-			aCounts.update(NonSerializableLong.of(currentCount));
-			bCounts.update(currentCount);
-			
-			allCounts.put(value.f0, currentCount);
-		}
-	}
-	
-	public static class NonSerializableLong {
-		public Long value;
-
-		private NonSerializableLong(long value) {
-			this.value = value;
-		}
-
-		public static NonSerializableLong of(long value) {
-			return new NonSerializableLong(value);
-		}
-
-		@Override
-		public boolean equals(Object o) {
-			if (this == o) return true;
-			if (o == null || getClass() != o.getClass()) return false;
-
-			NonSerializableLong that = (NonSerializableLong) o;
-
-			return value.equals(that.value);
-
-		}
-
-		@Override
-		public int hashCode() {
-			return value.hashCode();
-		}
-	}
-	
-	public static class IdentityKeySelector<T> implements KeySelector<T, T> {
-
-		@Override
-		public T getKey(T value) throws Exception {
-			return value;
-		}
-
-	}
-}

From 2bbcac4db1a48d1df62de26eed1bf638999043ac Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Wed, 10 May 2017 22:03:49 +0200
Subject: [PATCH 224/253] [FLINK-6531] [checkpoints] Ensure proper classloading
 for user-defined checkpoint hooks

This closes #3868
---
 .../executiongraph/ExecutionGraphBuilder.java |  13 +-
 .../tasks/JobCheckpointingSettings.java       |  12 +-
 .../CheckpointSettingsSerializableTest.java   | 122 ++++++++++++++++++
 .../api/graph/StreamingJobGraphGenerator.java |  20 ++-
 .../WithMasterCheckpointHookConfigTest.java   |  10 +-
 5 files changed, 167 insertions(+), 10 deletions(-)
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
index aa28fbcae24d3..0e76cfb8e8bd8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
@@ -51,6 +51,7 @@
 import org.apache.flink.runtime.state.AbstractStateBackend;
 import org.apache.flink.runtime.state.StateBackend;
 import org.apache.flink.util.DynamicCodeLoadingException;
+import org.apache.flink.util.SerializedValue;
 import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
@@ -240,13 +241,21 @@ public static ExecutionGraph buildGraph(
 
 			// instantiate the user-defined checkpoint hooks
 
-			final MasterTriggerRestoreHook.Factory[] hookFactories = snapshotSettings.getMasterHooks();
+			final SerializedValue<MasterTriggerRestoreHook.Factory[]> serializedHooks = snapshotSettings.getMasterHooks();
 			final List<MasterTriggerRestoreHook<?>> hooks;
 
-			if (hookFactories == null || hookFactories.length == 0) {
+			if (serializedHooks == null) {
 				hooks = Collections.emptyList();
 			}
 			else {
+				final MasterTriggerRestoreHook.Factory[] hookFactories;
+				try {
+					hookFactories = serializedHooks.deserializeValue(classLoader);
+				}
+				catch (IOException | ClassNotFoundException e) {
+					throw new JobExecutionException(jobId, "Could not instantiate user-defined checkpoint hooks", e);
+				}
+
 				hooks = new ArrayList<>(hookFactories.length);
 				for (MasterTriggerRestoreHook.Factory factory : hookFactories) {
 					hooks.add(factory.create());
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
index 3dd037ed71505..a30a2baf4f949 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobCheckpointingSettings.java
@@ -21,6 +21,7 @@
 import org.apache.flink.runtime.checkpoint.MasterTriggerRestoreHook;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.state.StateBackend;
+import org.apache.flink.util.SerializedValue;
 
 import javax.annotation.Nullable;
 import java.util.List;
@@ -58,7 +59,8 @@ public class JobCheckpointingSettings implements java.io.Serializable {
 	private final StateBackend defaultStateBackend;
 
 	/** (Factories for) hooks that are executed on the checkpoint coordinator */
-	private final MasterTriggerRestoreHook.Factory[] masterHooks;
+	@Nullable
+	private final SerializedValue<MasterTriggerRestoreHook.Factory[]> masterHooks;
 
 	/**
 	 * Flag indicating whether exactly once checkpoint mode has been configured.
@@ -96,7 +98,7 @@ public JobCheckpointingSettings(
 			int maxConcurrentCheckpoints,
 			ExternalizedCheckpointSettings externalizedCheckpointSettings,
 			@Nullable StateBackend defaultStateBackend,
-			@Nullable MasterTriggerRestoreHook.Factory[] masterHooks,
+			@Nullable SerializedValue<MasterTriggerRestoreHook.Factory[]> masterHooks,
 			boolean isExactlyOnce) {
 
 		// sanity checks
@@ -115,8 +117,7 @@ public JobCheckpointingSettings(
 		this.externalizedCheckpointSettings = requireNonNull(externalizedCheckpointSettings);
 		this.defaultStateBackend = defaultStateBackend;
 		this.isExactlyOnce = isExactlyOnce;
-
-		this.masterHooks = masterHooks != null ? masterHooks : new MasterTriggerRestoreHook.Factory[0];
+		this.masterHooks = masterHooks;
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -158,7 +159,8 @@ public StateBackend getDefaultStateBackend() {
 		return defaultStateBackend;
 	}
 
-	public MasterTriggerRestoreHook.Factory[] getMasterHooks() {
+	@Nullable
+	public SerializedValue<MasterTriggerRestoreHook.Factory[]> getMasterHooks() {
 		return masterHooks;
 	}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
new file mode 100644
index 0000000000000..02461803299e3
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder;
+import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * This test validates that the checkpoint settings serialize correctly
+ * in the presence of user-defined objects.
+ */
+public class CheckpointSettingsSerializableTest extends TestLogger {
+
+	@Test
+	public void testClassLoaderForCheckpointHooks() throws Exception {
+		final ClassLoader classLoader = new URLClassLoader(new URL[0], getClass().getClassLoader());
+		final Serializable outOfClassPath = CommonTestUtils.createObjectForClassNotInClassPath(classLoader);
+
+		final MasterTriggerRestoreHook.Factory[] hooks = {
+				new TestFactory(outOfClassPath) };
+		final SerializedValue<MasterTriggerRestoreHook.Factory[]> serHooks = new SerializedValue<>(hooks);
+
+		final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(
+				Collections.<JobVertexID>emptyList(),
+				Collections.<JobVertexID>emptyList(),
+				Collections.<JobVertexID>emptyList(),
+				1000L,
+				10000L,
+				0L,
+				1,
+				ExternalizedCheckpointSettings.none(),
+				null,
+				serHooks,
+				true);
+
+		final JobGraph jobGraph = new JobGraph(new JobID(), "test job");
+		jobGraph.setSnapshotSettings(checkpointingSettings);
+
+		// to serialize/deserialize the job graph to see if the behavior is correct under
+		// distributed execution
+		final JobGraph copy = CommonTestUtils.createCopySerializable(jobGraph);
+
+		final ExecutionGraph eg = ExecutionGraphBuilder.buildGraph(
+				null,
+				copy,
+				new Configuration(),
+				TestingUtils.defaultExecutor(),
+				TestingUtils.defaultExecutor(),
+				mock(SlotProvider.class),
+				classLoader,
+				new StandaloneCheckpointRecoveryFactory(),
+				Time.seconds(10),
+				new NoRestartStrategy(),
+				new UnregisteredMetricsGroup(),
+				10,
+				log);
+
+		assertEquals(1, eg.getCheckpointCoordinator().getNumberOfRegisteredMasterHooks());
+	}
+
+	// ------------------------------------------------------------------------
+
+	private static final class TestFactory implements MasterTriggerRestoreHook.Factory {
+
+		private static final long serialVersionUID = -612969579110202607L;
+		
+		private final Serializable payload;
+
+		TestFactory(Serializable payload) {
+			this.payload = payload;
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		public <V> MasterTriggerRestoreHook<V> create() {
+			MasterTriggerRestoreHook<V> hook = mock(MasterTriggerRestoreHook.class);
+			when(hook.getIdentifier()).thenReturn("id");
+			return hook;
+		}
+	}
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index b3a6cf8260136..6d1af7258c7cb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -66,6 +66,8 @@
 import org.apache.flink.streaming.runtime.tasks.StreamIterationHead;
 import org.apache.flink.streaming.runtime.tasks.StreamIterationTail;
 
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.SerializedValue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -642,6 +644,22 @@ private void configureCheckpointing() {
 			}
 		}
 
+		// because the hooks can have user-defined code, they need to be stored as
+		// eagerly serialized values
+		final SerializedValue<MasterTriggerRestoreHook.Factory[]> serializedHooks;
+		if (hooks.isEmpty()) {
+			serializedHooks = null;
+		} else {
+			try {
+				MasterTriggerRestoreHook.Factory[] asArray =
+						hooks.toArray(new MasterTriggerRestoreHook.Factory[hooks.size()]);
+				serializedHooks = new SerializedValue<>(asArray);
+			}
+			catch (IOException e) {
+				throw new FlinkRuntimeException("Trigger/restore hook is not serializable", e);
+			}
+		}
+
 		//  --- done, put it all together ---
 
 		JobCheckpointingSettings settings = new JobCheckpointingSettings(
@@ -650,7 +668,7 @@ private void configureCheckpointing() {
 				cfg.getMaxConcurrentCheckpoints(),
 				externalizedCheckpointSettings,
 				streamGraph.getStateBackend(),
-				hooks.toArray(new MasterTriggerRestoreHook.Factory[hooks.size()]),
+				serializedHooks,
 				isExactlyOnce);
 
 		jobGraph.setSnapshotSettings(settings);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java
index b5a95eb3c0c18..8065cf15540be 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/WithMasterCheckpointHookConfigTest.java
@@ -29,6 +29,7 @@
 import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
 
+import org.apache.flink.util.SerializedValue;
 import org.junit.Test;
 
 import javax.annotation.Nullable;
@@ -80,10 +81,15 @@ public void testHookConfiguration() throws Exception {
 			.addSink(new DiscardingSink<String>());
 
 		final JobGraph jg = env.getStreamGraph().getJobGraph();
-		assertEquals(hooks.size(), jg.getCheckpointingSettings().getMasterHooks().length);
+
+		SerializedValue<Factory[]> serializedConfiguredHooks = jg.getCheckpointingSettings().getMasterHooks();
+		assertNotNull(serializedConfiguredHooks);
+
+		Factory[] configuredHooks = serializedConfiguredHooks.deserializeValue(getClass().getClassLoader());
+		assertEquals(hooks.size(), configuredHooks.length);
 
 		// check that all hooks are contained and exist exactly once
-		for (Factory f : jg.getCheckpointingSettings().getMasterHooks()) {
+		for (Factory f : configuredHooks) {
 			MasterTriggerRestoreHook<?> hook = f.create();
 			assertTrue(hooks.remove(hook));
 		}

From 27e98fc95b05c8336dd1fe6b433b4105a83afb6f Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 11 May 2017 15:12:10 +0200
Subject: [PATCH 225/253] [hotfix] [build] Drop transitive
 jersey/jettison/servlet dependencies pulled via Hadoop

---
 .../flink-shaded-hadoop2/pom.xml              | 61 +++++++++++++++----
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
index 9ac033e64cbb4..a1159d672f8a3 100644
--- a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
+++ b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
@@ -68,6 +68,10 @@ under the License.
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jetty</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.mortbay.jetty</groupId>
+					<artifactId>jetty-util</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jsp-api-2.1</artifactId>
@@ -76,14 +80,13 @@ under the License.
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jsp-2.1</artifactId>
 				</exclusion>
-
 				<exclusion>
 					<groupId>org.eclipse.jdt</groupId>
 					<artifactId>core</artifactId>
 				</exclusion>
 				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jetty</artifactId>
+					<groupId>com.sun.jersey</groupId>
+					<artifactId>jersey-core</artifactId>
 				</exclusion>
 				<exclusion>
 					<groupId>com.sun.jersey</groupId>
@@ -105,6 +108,10 @@ under the License.
 					<groupId>tomcat</groupId>
 					<artifactId>jasper-runtime</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>javax.servlet</groupId>
+					<artifactId>servlet-api</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>javax.servlet.jsp</groupId>
 					<artifactId>jsp-api</artifactId>
@@ -189,6 +196,10 @@ under the License.
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jetty</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.mortbay.jetty</groupId>
+					<artifactId>jetty-util</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jsp-api-2.1</artifactId>
@@ -202,8 +213,8 @@ under the License.
 					<artifactId>core</artifactId>
 				</exclusion>
 				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jetty</artifactId>
+					<groupId>com.sun.jersey</groupId>
+					<artifactId>jersey-core</artifactId>
 				</exclusion>
 				<exclusion>
 					<groupId>com.sun.jersey</groupId>
@@ -225,6 +236,10 @@ under the License.
 					<groupId>tomcat</groupId>
 					<artifactId>jasper-runtime</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>javax.servlet</groupId>
+					<artifactId>servlet-api</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>javax.servlet.jsp</groupId>
 					<artifactId>jsp-api</artifactId>
@@ -297,6 +312,10 @@ under the License.
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jetty</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.mortbay.jetty</groupId>
+					<artifactId>jetty-util</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jsp-api-2.1</artifactId>
@@ -322,8 +341,8 @@ under the License.
 					<artifactId>netty</artifactId>
 				</exclusion>
 				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jetty</artifactId>
+					<groupId>com.sun.jersey</groupId>
+					<artifactId>jersey-core</artifactId>
 				</exclusion>
 				<exclusion>
 					<groupId>com.sun.jersey</groupId>
@@ -345,6 +364,10 @@ under the License.
 					<groupId>tomcat</groupId>
 					<artifactId>jasper-runtime</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>javax.servlet</groupId>
+					<artifactId>servlet-api</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>javax.servlet.jsp</groupId>
 					<artifactId>jsp-api</artifactId>
@@ -417,6 +440,10 @@ under the License.
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jetty</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.mortbay.jetty</groupId>
+					<artifactId>jetty-util</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jsp-api-2.1</artifactId>
@@ -442,8 +469,8 @@ under the License.
 					<artifactId>netty</artifactId>
 				</exclusion>
 				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jetty</artifactId>
+					<groupId>com.sun.jersey</groupId>
+					<artifactId>jersey-core</artifactId>
 				</exclusion>
 				<exclusion>
 					<groupId>com.sun.jersey</groupId>
@@ -465,6 +492,10 @@ under the License.
 					<groupId>tomcat</groupId>
 					<artifactId>jasper-runtime</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>javax.servlet</groupId>
+					<artifactId>servlet-api</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>javax.servlet.jsp</groupId>
 					<artifactId>jsp-api</artifactId>
@@ -537,6 +568,10 @@ under the License.
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jetty</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.mortbay.jetty</groupId>
+					<artifactId>jetty-util</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>org.mortbay.jetty</groupId>
 					<artifactId>jsp-api-2.1</artifactId>
@@ -562,8 +597,8 @@ under the License.
 					<artifactId>netty</artifactId>
 				</exclusion>
 				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jetty</artifactId>
+					<groupId>com.sun.jersey</groupId>
+					<artifactId>jersey-core</artifactId>
 				</exclusion>
 				<exclusion>
 					<groupId>com.sun.jersey</groupId>
@@ -585,6 +620,10 @@ under the License.
 					<groupId>tomcat</groupId>
 					<artifactId>jasper-runtime</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>javax.servlet</groupId>
+					<artifactId>servlet-api</artifactId>
+				</exclusion>
 				<exclusion>
 					<groupId>javax.servlet.jsp</groupId>
 					<artifactId>jsp-api</artifactId>

From b306e8866a930d979090ee4f0bc77a6c154dd07e Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Thu, 4 May 2017 13:37:15 -0400
Subject: [PATCH 226/253] [hotfix] [docs] Update file path of Gelly examples

---
 docs/dev/libs/gelly/index.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/dev/libs/gelly/index.md b/docs/dev/libs/gelly/index.md
index 193ba866fcec0..7ae796824a0a3 100644
--- a/docs/dev/libs/gelly/index.md
+++ b/docs/dev/libs/gelly/index.md
@@ -86,7 +86,7 @@ After configuring and starting the cluster, list the available algorithm classes
 
 ~~~bash
 ./bin/start-cluster.sh
-./bin/flink run examples/flink-gelly-examples_*.jar
+./bin/flink run examples/gelly/flink-gelly-examples_*.jar
 ~~~
 
 The Gelly drivers can generate graph data or read the edge list from a CSV file (each node in a cluster must have access
@@ -94,13 +94,13 @@ to the input file). The algorithm description, available inputs and outputs, and
 algorithm is selected. Print usage for [JaccardIndex](./library_methods.html#jaccard-index):
 
 ~~~bash
-./bin/flink run examples/flink-gelly-examples_*.jar --algorithm JaccardIndex
+./bin/flink run examples/gelly/flink-gelly-examples_*.jar --algorithm JaccardIndex
 ~~~
 
 Display [graph metrics](./library_methods.html#metric) for a million vertex graph:
 
 ~~~bash
-./bin/flink run examples/flink-gelly-examples_*.jar \
+./bin/flink run examples/gelly/flink-gelly-examples_*.jar \
     --algorithm GraphMetrics --order directed \
     --input RMatGraph --type integer --scale 20 --simplify directed \
     --output print
@@ -117,17 +117,17 @@ Run a few algorithms and monitor the job progress in Flink's Web UI:
 ~~~bash
 wget -O - http://snap.stanford.edu/data/bigdata/communities/com-lj.ungraph.txt.gz | gunzip -c > com-lj.ungraph.txt
 
-./bin/flink run -q examples/flink-gelly-examples_*.jar \
+./bin/flink run -q examples/gelly/flink-gelly-examples_*.jar \
     --algorithm GraphMetrics --order undirected \
     --input CSV --type integer --simplify undirected --input_filename com-lj.ungraph.txt --input_field_delimiter $'\t' \
     --output print
 
-./bin/flink run -q examples/flink-gelly-examples_*.jar \
+./bin/flink run -q examples/gelly/flink-gelly-examples_*.jar \
     --algorithm ClusteringCoefficient --order undirected \
     --input CSV --type integer --simplify undirected --input_filename com-lj.ungraph.txt --input_field_delimiter $'\t' \
     --output hash
 
-./bin/flink run -q examples/flink-gelly-examples_*.jar \
+./bin/flink run -q examples/gelly/flink-gelly-examples_*.jar \
     --algorithm JaccardIndex \
     --input CSV --type integer --simplify undirected --input_filename com-lj.ungraph.txt --input_field_delimiter $'\t' \
     --output hash

From 5137ca52cc579119d7da0cc76af3d63184f69dda Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Mon, 8 May 2017 12:45:19 -0400
Subject: [PATCH 227/253] [hotfix] [gelly] Improve graph generator
 documentation

---
 docs/dev/libs/gelly/graph_generators.md        | 18 +++++++++++-------
 .../flink/graph/generator/CompleteGraph.java   |  2 +-
 .../flink/graph/generator/CycleGraph.java      |  6 ++++--
 .../flink/graph/generator/EmptyGraph.java      |  2 +-
 .../flink/graph/generator/GridGraph.java       |  5 +++--
 .../flink/graph/generator/HypercubeGraph.java  |  6 ++++--
 .../flink/graph/generator/PathGraph.java       |  9 +++++++--
 .../flink/graph/generator/RMatGraph.java       |  6 +++---
 .../graph/generator/SingletonEdgeGraph.java    |  7 ++++---
 .../flink/graph/generator/StarGraph.java       |  6 ++++--
 10 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/docs/dev/libs/gelly/graph_generators.md b/docs/dev/libs/gelly/graph_generators.md
index 5598d83d9f778..2532ee49a94f8 100644
--- a/docs/dev/libs/gelly/graph_generators.md
+++ b/docs/dev/libs/gelly/graph_generators.md
@@ -48,7 +48,7 @@ boolean wrapEndpoints = false;
 
 int parallelism = 4;
 
-Graph<LongValue,NullValue,NullValue> graph = new GridGraph(env)
+Graph<LongValue, NullValue, NullValue> graph = new GridGraph(env)
     .addDimension(2, wrapEndpoints)
     .addDimension(4, wrapEndpoints)
     .setParallelism(parallelism)
@@ -138,7 +138,8 @@ val graph = new CompleteGraph(env.getJavaEnv, vertexCount).generate()
 
 ## Cycle Graph
 
-An undirected graph where all edges form a single cycle.
+An undirected graph where the set of edges form a single cycle by connecting
+each vertex to two adjacent vertices in a chained loop.
 
 <div class="codetabs" markdown="1">
 <div data-lang="java" markdown="1">
@@ -194,7 +195,7 @@ val graph = new CycleGraph(env.getJavaEnv, vertexCount).generate()
 
 ## Empty Graph
 
-The graph containing no edges.
+A graph containing no edges.
 
 <div class="codetabs" markdown="1">
 <div data-lang="java" markdown="1">
@@ -316,7 +317,7 @@ val graph = new GridGraph(env.getJavaEnv).addDimension(2, wrapEndpoints).addDime
 
 ## Hypercube Graph
 
-An undirected graph where edges form an n-dimensional hypercube. Each vertex
+An undirected graph where edges form an `n`-dimensional hypercube. Each vertex
 in a hypercube connects to one other vertex in each dimension.
 
 <div class="codetabs" markdown="1">
@@ -391,7 +392,9 @@ val graph = new HypercubeGraph(env.getJavaEnv, dimensions).generate()
 
 ## Path Graph
 
-An undirected Graph where all edges form a single path.
+An undirected graph where the set of edges form a single path by connecting
+two `endpoint` vertices with degree `1` and all midpoint vertices with degree
+`2`. A path graph can be formed by removing a single edge from a cycle graph.
 
 <div class="codetabs" markdown="1">
 <div data-lang="java" markdown="1">
@@ -443,7 +446,7 @@ val graph = new PathGraph(env.getJavaEnv, vertexCount).generate()
 
 ## RMat Graph
 
-A directed or undirected power-law graph generated using the
+A directed power-law multigraph generated using the
 [Recursive Matrix (R-Mat)](http://www.cs.cmu.edu/~christos/PUBLICATIONS/siam04.pdf) model.
 
 RMat is a stochastic generator configured with a source of randomness implementing the
@@ -528,7 +531,8 @@ val graph = new RMatGraph(env.getJavaEnv, rnd, vertexCount, edgeCount).setConsta
 
 ## Singleton Edge Graph
 
-An undirected graph containing isolated two-paths.
+An undirected graph containing isolated two-paths where every vertex has degree
+`1`.
 
 <div class="codetabs" markdown="1">
 <div data-lang="java" markdown="1">
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
index 84cb791957dac..11c0bb0bcfc47 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
@@ -31,7 +31,7 @@
 import org.apache.flink.util.LongValueSequenceIterator;
 import org.apache.flink.util.Preconditions;
 
-/*
+/**
  * @see <a href="http://mathworld.wolfram.com/CompleteGraph.html">Complete Graph at Wolfram MathWorld</a>
  */
 public class CompleteGraph
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CycleGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CycleGraph.java
index b04d78c299633..2386fe8cf6bbc 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CycleGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CycleGraph.java
@@ -24,7 +24,7 @@
 import org.apache.flink.types.NullValue;
 import org.apache.flink.util.Preconditions;
 
-/*
+/**
  * @see <a href="http://mathworld.wolfram.com/CycleGraph.html">Cycle Graph at Wolfram MathWorld</a>
  */
 public class CycleGraph
@@ -39,7 +39,9 @@ public class CycleGraph
 	private long vertexCount;
 
 	/**
-	 * An undirected {@link Graph} where all edges form a single cycle.
+	 * An undirected {@link Graph} with {@code n} vertices where each vertex
+	 * v<sub>i</sub> is connected to adjacent vertices v<sub>(i+1)%n</sub> and
+	 * v<sub>(i-1)%n</sub>.
 	 *
 	 * @param env the Flink execution environment
 	 * @param vertexCount number of vertices
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EmptyGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EmptyGraph.java
index 25584ea60bbfe..23e3a9c603148 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EmptyGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EmptyGraph.java
@@ -33,7 +33,7 @@
 
 import java.util.Collections;
 
-/*
+/**
  * @see <a href="http://mathworld.wolfram.com/EmptyGraph.html">Empty Graph at Wolfram MathWorld</a>
  */
 public class EmptyGraph
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
index 16a9ab94ead02..23a6f98bafb9b 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
@@ -35,7 +35,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-/*
+/**
  * @see <a href="http://mathworld.wolfram.com/GridGraph.html">Grid Graph at Wolfram MathWorld</a>
  */
 public class GridGraph
@@ -50,7 +50,8 @@ public class GridGraph
 	private long vertexCount = 1;
 
 	/**
-	 * An undirected {@link Graph} connecting vertices in a regular tiling in one or more dimensions.
+	 * An undirected {@code Graph} connecting vertices in a regular tiling in
+	 * one or more dimensions and where the endpoints are optionally connected.
 	 *
 	 * @param env the Flink execution environment
 	 */
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/HypercubeGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/HypercubeGraph.java
index 37590ffefb928..e4eac699e11d5 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/HypercubeGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/HypercubeGraph.java
@@ -24,7 +24,7 @@
 import org.apache.flink.types.NullValue;
 import org.apache.flink.util.Preconditions;
 
-/*
+/**
  * @see <a href="http://mathworld.wolfram.com/HypercubeGraph.html">Hypercube Graph at Wolfram MathWorld</a>
  */
 public class HypercubeGraph
@@ -39,7 +39,9 @@ public class HypercubeGraph
 	private long dimensions;
 
 	/**
-	 * An undirected {@link Graph} where edges form an n-dimensional hypercube.
+	 * An undirected {@code Graph} where edges form an n-dimensional hypercube.
+	 * Each vertex in a hypercube connects to one other vertex in each
+	 * dimension.
 	 *
 	 * @param env the Flink execution environment
 	 * @param dimensions number of dimensions
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/PathGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/PathGraph.java
index dcc4c984fa157..5c4343b4bcd46 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/PathGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/PathGraph.java
@@ -24,7 +24,7 @@
 import org.apache.flink.types.NullValue;
 import org.apache.flink.util.Preconditions;
 
-/*
+/**
  * @see <a href="http://mathworld.wolfram.com/PathGraph.html">Path Graph at Wolfram MathWorld</a>
  */
 public class PathGraph
@@ -39,7 +39,12 @@ public class PathGraph
 	private long vertexCount;
 
 	/**
-	 * An undirected {@link Graph} where all edges form a single path.
+	 * An undirected {@link Graph} with {@code n} vertices where each vertex
+	 * v<sub>i</sub> connects to adjacent vertices v<sub>i+1</sub> when
+	 * {@code i < n-1} and v<sub>i-1</sub> when {@code i > 0}.
+	 * <p>
+	 * A {@code PathGraph} is distinguished from a {@code CycleGraph} in that
+	 * the first and last vertex are not connected, breaking the cycle.
 	 *
 	 * @param env the Flink execution environment
 	 * @param vertexCount number of vertices
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java
index 0fa4127d22efe..071b415bb63a7 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/RMatGraph.java
@@ -34,7 +34,7 @@
 
 import java.util.List;
 
-/*
+/**
  * @see <a href="http://www.cs.cmu.edu/~christos/PUBLICATIONS/siam04.pdf">R-MAT: A Recursive Model for Graph Mining</a>
  */
 public class RMatGraph<T extends RandomGenerator>
@@ -75,8 +75,8 @@ public class RMatGraph<T extends RandomGenerator>
 	public float noise = DEFAULT_NOISE;
 
 	/**
-	 * Generate a directed or undirected power-law {@link Graph} using the
-	 * Recursive Matrix (R-Mat) model.
+	 * A directed power-law multi{@link Graph graph} generated using the
+	 * stochastic Recursive Matrix (R-Mat) model.
 	 *
 	 * @param env the Flink execution environment
 	 * @param randomGeneratorFactory source of randomness
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/SingletonEdgeGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/SingletonEdgeGraph.java
index f3c087e3d2653..125501cd726d8 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/SingletonEdgeGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/SingletonEdgeGraph.java
@@ -31,8 +31,7 @@
 import org.apache.flink.util.Preconditions;
 
 /**
- * A singleton-edge {@link Graph} contains one or more isolated two-paths. The in- and out-degree
- * of every vertex is 1. For {@code n} vertices there are {@code n/2} components.
+ * A singleton-edge {@link Graph} contains one or more isolated two-paths.
  */
 public class SingletonEdgeGraph
 extends AbstractGraphGenerator<LongValue, NullValue, NullValue> {
@@ -46,7 +45,9 @@ public class SingletonEdgeGraph
 	private long vertexPairCount;
 
 	/**
-	 * An undirected {@link Graph} containing isolated two-paths.
+	 * An undirected {@link Graph} containing one or more isolated two-paths.
+	 * The in- and out-degree of every vertex is 1. For {@code n} vertices
+	 * there are {@code n/2} components.
 	 *
 	 * @param env the Flink execution environment
 	 * @param vertexPairCount number of pairs of vertices
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java
index 316d749ee9671..f0f4e5ad32d3a 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/StarGraph.java
@@ -31,7 +31,7 @@
 import org.apache.flink.util.LongValueSequenceIterator;
 import org.apache.flink.util.Preconditions;
 
-/*
+/**
  * @see <a href="http://mathworld.wolfram.com/StarGraph.html">Star Graph at Wolfram MathWorld</a>
  */
 public class StarGraph
@@ -46,7 +46,9 @@ public class StarGraph
 	private long vertexCount;
 
 	/**
-	 * An undirected {@link Graph} containing a single central {@link Vertex} connected to all other leaf vertices.
+	 * An undirected {@Graph} with {@code n} vertices where the single central
+	 * node has degree {@code n-1}, connecting to the other {@code n-1}
+	 * vertices which have degree {@code 1}.
 	 *
 	 * @param env the Flink execution environment
 	 * @param vertexCount number of vertices

From 790e9d4b8ba4e79e0a430da9a18a83043854f305 Mon Sep 17 00:00:00 2001
From: FlorianFan <fanzhidongyzby@163.com>
Date: Thu, 27 Apr 2017 20:41:53 +0800
Subject: [PATCH 228/253] [FLINK-6393] [gelly] Add Circulant and Echo graph
 generators

This closes #3802
---
 docs/dev/libs/gelly/graph_generators.md       |  91 ++++++-
 .../java/org/apache/flink/graph/Runner.java   |  11 +-
 .../graph/drivers/input/CirculantGraph.java   | 129 +++++++++
 .../flink/graph/drivers/input/EchoGraph.java  |  67 +++++
 .../flink/graph/drivers/input/GridGraph.java  |   3 +-
 .../flink/graph/drivers/EdgeListITCase.java   |  90 ++++++-
 .../flink/graph/generator/CirculantGraph.java | 246 ++++++++++++++++++
 .../flink/graph/generator/CompleteGraph.java  |  57 +---
 .../flink/graph/generator/EchoGraph.java      |  82 ++++++
 .../flink/graph/generator/GridGraph.java      |   4 +-
 .../graph/generator/CirculantGraphTest.java   |  88 +++++++
 .../flink/graph/generator/EchoGraphTest.java  | 128 +++++++++
 12 files changed, 926 insertions(+), 70 deletions(-)
 create mode 100644 flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CirculantGraph.java
 create mode 100644 flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EchoGraph.java
 create mode 100644 flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CirculantGraph.java
 create mode 100644 flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EchoGraph.java
 create mode 100644 flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/CirculantGraphTest.java
 create mode 100644 flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/EchoGraphTest.java

diff --git a/docs/dev/libs/gelly/graph_generators.md b/docs/dev/libs/gelly/graph_generators.md
index 2532ee49a94f8..d4ad22931a1ce 100644
--- a/docs/dev/libs/gelly/graph_generators.md
+++ b/docs/dev/libs/gelly/graph_generators.md
@@ -72,6 +72,42 @@ val graph = new GridGraph(env.getJavaEnv).addDimension(2, wrapEndpoints).addDime
 </div>
 </div>
 
+## Circulant Graph
+
+A [circulant graph](http://mathworld.wolfram.com/CirculantGraph.html) is an
+[oriented graph](http://mathworld.wolfram.com/OrientedGraph.html) configured
+with one or more contiguous ranges of offsets. Edges connect integer vertex IDs
+whose difference equals a configured offset. The circulant graph with no offsets
+is the [empty graph](#empty-graph) and the graph with the maximum range is the
+[complete graph](#complete-graph).
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+long vertexCount = 5;
+
+Graph<LongValue, NullValue, NullValue> graph = new CirculantGraph(env, vertexCount)
+    .addRange(1, 2)
+    .generate();
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+import org.apache.flink.api.scala._
+import org.apache.flink.graph.generator.CirculantGraph
+
+val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+
+val vertexCount = 5
+
+val graph = new CirculantGraph(env.getJavaEnv, vertexCount).addRange(1, 2).generate()
+{% endhighlight %}
+</div>
+</div>
+
 ## Complete Graph
 
 An undirected graph connecting every distinct pair of vertices.
@@ -83,7 +119,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
 long vertexCount = 5;
 
-Graph<LongValue,NullValue,NullValue> graph = new CompleteGraph(env, vertexCount)
+Graph<LongValue, NullValue, NullValue> graph = new CompleteGraph(env, vertexCount)
     .generate();
 {% endhighlight %}
 </div>
@@ -148,7 +184,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
 long vertexCount = 5;
 
-Graph<LongValue,NullValue,NullValue> graph = new CycleGraph(env, vertexCount)
+Graph<LongValue, NullValue, NullValue> graph = new CycleGraph(env, vertexCount)
     .generate();
 {% endhighlight %}
 </div>
@@ -193,6 +229,41 @@ val graph = new CycleGraph(env.getJavaEnv, vertexCount).generate()
     <text x="51" y="199">4</text>
 </svg>
 
+## Echo Graph
+
+An [echo graph](http://mathworld.wolfram.com/EchoGraph.html) is a
+[circulant graph](#circulant-graph) with `n` vertices defined by the width of a
+single range of offsets centered at `n/2`. A vertex is connected to 'far'
+vertices, which connect to 'near' vertices, which connect to 'far' vertices, ....
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+long vertexCount = 5;
+long vertexDegree = 2;
+
+Graph<LongValue, NullValue, NullValue> graph = new EchoGraph(env, vertexCount, vertexDegree)
+    .generate();
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+import org.apache.flink.api.scala._
+import org.apache.flink.graph.generator.EchoGraph
+
+val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+
+val vertexCount = 5
+val vertexDegree = 2
+
+val graph = new EchoGraph(env.getJavaEnv, vertexCount, vertexDegree).generate()
+{% endhighlight %}
+</div>
+</div>
+
 ## Empty Graph
 
 A graph containing no edges.
@@ -204,7 +275,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
 long vertexCount = 5;
 
-Graph<LongValue,NullValue,NullValue> graph = new EmptyGraph(env, vertexCount)
+Graph<LongValue, NullValue, NullValue> graph = new EmptyGraph(env, vertexCount)
     .generate();
 {% endhighlight %}
 </div>
@@ -257,7 +328,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
 boolean wrapEndpoints = false;
 
-Graph<LongValue,NullValue,NullValue> graph = new GridGraph(env)
+Graph<LongValue, NullValue, NullValue> graph = new GridGraph(env)
     .addDimension(2, wrapEndpoints)
     .addDimension(4, wrapEndpoints)
     .generate();
@@ -327,7 +398,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
 long dimensions = 3;
 
-Graph<LongValue,NullValue,NullValue> graph = new HypercubeGraph(env, dimensions)
+Graph<LongValue, NullValue, NullValue> graph = new HypercubeGraph(env, dimensions)
     .generate();
 {% endhighlight %}
 </div>
@@ -403,7 +474,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
 long vertexCount = 5
 
-Graph<LongValue,NullValue,NullValue> graph = new PathGraph(env, vertexCount)
+Graph<LongValue, NullValue, NullValue> graph = new PathGraph(env, vertexCount)
     .generate();
 {% endhighlight %}
 </div>
@@ -464,7 +535,7 @@ RandomGenerableFactory<JDKRandomGenerator> rnd = new JDKRandomGeneratorFactory()
 int vertexCount = 1 << scale;
 int edgeCount = edgeFactor * vertexCount;
 
-Graph<LongValue,NullValue,NullValue> graph = new RMatGraph<>(env, rnd, vertexCount, edgeCount)
+Graph<LongValue, NullValue, NullValue> graph = new RMatGraph<>(env, rnd, vertexCount, edgeCount)
     .generate();
 {% endhighlight %}
 </div>
@@ -505,7 +576,7 @@ int edgeCount = edgeFactor * vertexCount;
 
 boolean clipAndFlip = false;
 
-Graph<LongValue,NullValue,NullValue> graph = new RMatGraph<>(env, rnd, vertexCount, edgeCount)
+Graph<LongValue, NullValue, NullValue> graph = new RMatGraph<>(env, rnd, vertexCount, edgeCount)
     .setConstants(0.57f, 0.19f, 0.19f)
     .setNoise(true, 0.10f)
     .generate();
@@ -542,7 +613,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 long vertexPairCount = 4
 
 // note: configured with the number of vertex pairs
-Graph<LongValue,NullValue,NullValue> graph = new SingletonEdgeGraph(env, vertexPairCount)
+Graph<LongValue, NullValue, NullValue> graph = new SingletonEdgeGraph(env, vertexPairCount)
     .generate();
 {% endhighlight %}
 </div>
@@ -607,7 +678,7 @@ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
 long vertexCount = 6;
 
-Graph<LongValue,NullValue,NullValue> graph = new StarGraph(env, vertexCount)
+Graph<LongValue, NullValue, NullValue> graph = new StarGraph(env, vertexCount)
     .generate();
 {% endhighlight %}
 </div>
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java
index 5ffe681cd0fc8..07cad1f36e408 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Runner.java
@@ -35,8 +35,10 @@
 import org.apache.flink.graph.drivers.JaccardIndex;
 import org.apache.flink.graph.drivers.PageRank;
 import org.apache.flink.graph.drivers.TriangleListing;
+import org.apache.flink.graph.drivers.input.CirculantGraph;
 import org.apache.flink.graph.drivers.input.CompleteGraph;
 import org.apache.flink.graph.drivers.input.CycleGraph;
+import org.apache.flink.graph.drivers.input.EchoGraph;
 import org.apache.flink.graph.drivers.input.EmptyGraph;
 import org.apache.flink.graph.drivers.input.GridGraph;
 import org.apache.flink.graph.drivers.input.HypercubeGraph;
@@ -76,9 +78,11 @@ public class Runner {
 	private static final String OUTPUT = "output";
 
 	private static ParameterizedFactory<Input> inputFactory = new ParameterizedFactory<Input>()
+		.addClass(CirculantGraph.class)
 		.addClass(CompleteGraph.class)
 		.addClass(org.apache.flink.graph.drivers.input.CSV.class)
 		.addClass(CycleGraph.class)
+		.addClass(EchoGraph.class)
 		.addClass(EmptyGraph.class)
 		.addClass(GridGraph.class)
 		.addClass(HypercubeGraph.class)
@@ -236,7 +240,12 @@ public static void main(String[] args) throws Exception {
 
 		// Input
 
-		input.configure(parameters);
+		try {
+			input.configure(parameters);
+		} catch (RuntimeException ex) {
+			throw new ProgramParametrizationException(ex.getMessage());
+		}
+
 		Graph graph = input.create(env);
 
 		// Algorithm
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CirculantGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CirculantGraph.java
new file mode 100644
index 0000000000000..14ee816a4e726
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CirculantGraph.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.drivers.input;
+
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.drivers.parameter.LongParameter;
+import org.apache.flink.graph.generator.CirculantGraph.OffsetRange;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.apache.flink.api.common.ExecutionConfig.PARALLELISM_DEFAULT;
+import static org.apache.flink.graph.generator.CirculantGraph.MINIMUM_VERTEX_COUNT;
+
+/**
+ * Generate a {@link org.apache.flink.graph.generator.CirculantGraph}.
+ */
+public class CirculantGraph
+extends GeneratedGraph<LongValue> {
+
+	private static final String PREFIX = "range";
+
+	private LongParameter vertexCount = new LongParameter(this, "vertex_count")
+		.setMinimumValue(MINIMUM_VERTEX_COUNT);
+
+	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
+		.setDefaultValue(PARALLELISM_DEFAULT);
+
+	private List<OffsetRange> offsetRanges = new ArrayList<>();
+
+	@Override
+	public String getName() {
+		return CirculantGraph.class.getSimpleName();
+	}
+
+	@Override
+	public String getUsage() {
+		return "--" + PREFIX + "0 offset:length [--" + PREFIX + "1 offset:length [--" + PREFIX + "2 ...]]"
+			+ super.getUsage();
+	}
+
+	@Override
+	public void configure(ParameterTool parameterTool) throws ProgramParametrizationException {
+		super.configure(parameterTool);
+
+		// add offset ranges as ordered by offset ID (range0, range1, range2, ...)
+
+		Map<Integer, String> offsetRangeMap = new TreeMap<>();
+
+		// first parse all offset ranges into a sorted map
+		for (String key : parameterTool.toMap().keySet()) {
+			if (key.startsWith(PREFIX)) {
+				int offsetId = Integer.parseInt(key.substring(PREFIX.length()));
+				offsetRangeMap.put(offsetId, parameterTool.get(key));
+			}
+		}
+
+		// then store offset ranges in order
+		for (String field : offsetRangeMap.values()) {
+			ProgramParametrizationException exception = new ProgramParametrizationException("Circulant offset range" +
+				" must use a colon to separate the integer offset and integer length:" + field + "'");
+
+			if (! field.contains(":")) {
+				throw exception;
+			}
+
+			String[] parts = field.split(":");
+
+			if (parts.length != 2) {
+				throw exception;
+			}
+
+			try {
+				long offset = Long.parseLong(parts[0]);
+				long length = Long.parseLong(parts[1]);
+				offsetRanges.add(new OffsetRange(offset, length));
+			} catch (NumberFormatException ex) {
+				throw exception;
+			}
+		}
+	}
+
+	@Override
+	public String getIdentity() {
+		return getTypeName() + " " + getName() + " (" + offsetRanges + ")";
+	}
+
+	@Override
+	protected long vertexCount() {
+		return vertexCount.getValue();
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) {
+		org.apache.flink.graph.generator.CirculantGraph graph = new org.apache.flink.graph.generator.CirculantGraph(env,
+			vertexCount.getValue());
+
+		for (OffsetRange offsetRange : offsetRanges) {
+			graph.addRange(offsetRange.getOffset(), offsetRange.getLength());
+		}
+
+		return graph
+			.setParallelism(littleParallelism.getValue().intValue())
+			.generate();
+	}
+}
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EchoGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EchoGraph.java
new file mode 100644
index 0000000000000..c9b0874562da5
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/EchoGraph.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.drivers.input;
+
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.drivers.parameter.LongParameter;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+
+import static org.apache.flink.api.common.ExecutionConfig.PARALLELISM_DEFAULT;
+import static org.apache.flink.graph.generator.EchoGraph.MINIMUM_VERTEX_COUNT;
+import static org.apache.flink.graph.generator.EchoGraph.MINIMUM_VERTEX_DEGREE;
+
+/**
+ * Generate an {@link org.apache.flink.graph.generator.EchoGraph}.
+ */
+public class EchoGraph
+extends GeneratedGraph<LongValue> {
+
+	private LongParameter vertexCount = new LongParameter(this, "vertex_count")
+		.setMinimumValue(MINIMUM_VERTEX_COUNT);
+
+	private LongParameter vertexDegree = new LongParameter(this, "vertex_degree")
+		.setMinimumValue(MINIMUM_VERTEX_DEGREE);
+
+	private LongParameter littleParallelism = new LongParameter(this, "little_parallelism")
+		.setDefaultValue(PARALLELISM_DEFAULT);
+
+	@Override
+	public String getName() {
+		return EchoGraph.class.getSimpleName();
+	}
+
+	@Override
+	public String getIdentity() {
+		return getTypeName() + " " + getName() + " (" + vertexCount.getValue() + ":" + vertexDegree.getValue() + ")";
+	}
+
+	@Override
+	protected long vertexCount() {
+		return vertexCount.getValue();
+	}
+
+	@Override
+	protected Graph<LongValue, NullValue, NullValue> generate(ExecutionEnvironment env) throws Exception {
+		return new org.apache.flink.graph.generator.EchoGraph(env, vertexCount.getValue(), vertexDegree.getValue())
+			.setParallelism(littleParallelism.getValue().intValue())
+			.generate();
+	}
+}
diff --git a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java
index b41b86e33f12e..2ce3c77d61ad2 100644
--- a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java
+++ b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/GridGraph.java
@@ -54,7 +54,8 @@ public String getName() {
 
 	@Override
 	public String getUsage() {
-		return "--dim0 size:wrap_endpoints [--dim1 size:wrap_endpoints [--dim2 ...]]" + super.getUsage();
+		return "--" + PREFIX + "0 size:wrap_endpoints [--" + PREFIX + " size:wrap_endpoints [--" + PREFIX + " ...]]"
+			+ super.getUsage();
 	}
 
 	@Override
diff --git a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java
index f5662180a5d64..d3ba4fb7dcfd4 100644
--- a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java
+++ b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/EdgeListITCase.java
@@ -43,6 +43,50 @@ private String[] parameters(String input, String output, String... additionalPar
 		return ArrayUtils.addAll(parameters, additionalParameters);
 	}
 
+	@Test
+	public void testHashWithCirculantGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x0000000000344448L;
+				break;
+
+			case "long":
+				checksum = 0x0000000000a19d48L;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x000000000c47ca48L;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("CirculantGraph", "hash", "--vertex_count", "42", "--range0", "13:4"),
+			168, checksum);
+	}
+
+	@Test
+	public void testPrintWithCirculantGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		expectedOutputChecksum(
+			parameters("CirculantGraph", "print", "--vertex_count", "42", "--range0", "13:4"),
+			new Checksum(168, 0x0000004bdcc52cbcL));
+	}
+
 	@Test
 	public void testLongDescription() throws Exception {
 		String expected = regexSubstring(new EdgeList().getLongDescription());
@@ -141,11 +185,55 @@ public void testPrintWithCycleGraph() throws Exception {
 			new Checksum(84, 0x000000272a136fcaL));
 	}
 
+	@Test
+	public void testHashWithEchoGraph() throws Exception {
+		long checksum;
+		switch (idType) {
+			case "byte":
+			case "nativeByte":
+			case "short":
+			case "nativeShort":
+			case "char":
+			case "nativeChar":
+			case "integer":
+			case "nativeInteger":
+			case "nativeLong":
+				checksum = 0x0000000000a9ddeaL;
+				break;
+
+			case "long":
+				checksum = 0x00000000020d3f2aL;
+				break;
+
+			case "string":
+			case "nativeString":
+				checksum = 0x0000000027e9516aL;
+				break;
+
+			default:
+				throw new IllegalArgumentException("Unknown type: " + idType);
+		}
+
+		expectedChecksum(
+			parameters("EchoGraph", "hash", "--vertex_count", "42", "--vertex_degree", "13"),
+			546, checksum);
+	}
+
+	@Test
+	public void testPrintWithEchoGraph() throws Exception {
+		// skip 'char' since it is not printed as a number
+		Assume.assumeFalse(idType.equals("char") || idType.equals("nativeChar"));
+
+		expectedOutputChecksum(
+			parameters("EchoGraph", "print", "--vertex_count", "42", "--vertex_degree", "13"),
+			new Checksum(546, 0x000000f7190b8fcaL));
+	}
+
 	@Test
 	public void testHashWithEmptyGraph() throws Exception {
 		expectedChecksum(
 			parameters("EmptyGraph", "hash", "--vertex_count", "42"),
-			0, 0x0000000000000000);
+			0, 0x0000000000000000L);
 	}
 
 	@Test
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CirculantGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CirculantGraph.java
new file mode 100644
index 0000000000000..9569b740d61a7
--- /dev/null
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CirculantGraph.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.generator;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.FunctionAnnotation;
+import org.apache.flink.graph.Edge;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.Vertex;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.LongValueSequenceIterator;
+import org.apache.flink.util.Preconditions;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @see <a href="http://mathworld.wolfram.com/CirculantGraph.html">Circulant Graph at Wolfram MathWorld</a>
+ */
+public class CirculantGraph
+extends AbstractGraphGenerator<LongValue, NullValue, NullValue> {
+
+	public static final int MINIMUM_VERTEX_COUNT = 2;
+
+	public static final int MINIMUM_OFFSET = 1;
+
+	// Required to create the DataSource
+	private final ExecutionEnvironment env;
+
+	// Required configuration
+	private long vertexCount;
+
+	private List<OffsetRange> offsetRanges = new ArrayList<>();
+
+	/**
+	 * An oriented {@link Graph} with {@code n} vertices where each vertex
+	 * v<sub>i</sub> is connected to vertex v<sub>(i+j)%n</sub> for each
+	 * configured offset {@code j}.
+	 *
+	 * @param env the Flink execution environment
+	 * @param vertexCount number of vertices
+	 */
+	public CirculantGraph(ExecutionEnvironment env, long vertexCount) {
+		Preconditions.checkArgument(vertexCount >= MINIMUM_VERTEX_COUNT,
+			"Vertex count must be at least " + MINIMUM_VERTEX_COUNT);
+
+		this.env = env;
+		this.vertexCount = vertexCount;
+	}
+
+	/**
+	 * Required configuration for each range of offsets in the graph.
+	 *
+	 * @param offset first offset appointing the vertices' position
+	 * @param length number of contiguous offsets in range
+	 * @return this
+	 */
+	public CirculantGraph addRange(long offset, long length) {
+		Preconditions.checkArgument(offset >= MINIMUM_OFFSET,
+			"Range offset must be at least " + MINIMUM_OFFSET);
+		Preconditions.checkArgument(length <= vertexCount - offset,
+			"Range length must not be greater than the vertex count minus the range offset.");
+
+		offsetRanges.add(new OffsetRange(offset, length));
+
+		return this;
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate() {
+		// Vertices
+		DataSet<Vertex<LongValue, NullValue>> vertices = GraphGeneratorUtils.vertexSequence(env, parallelism, vertexCount);
+
+		// Edges
+		LongValueSequenceIterator iterator = new LongValueSequenceIterator(0, this.vertexCount - 1);
+
+		// Validate ranges
+		Collections.sort(offsetRanges);
+		Iterator<OffsetRange> iter = offsetRanges.iterator();
+		OffsetRange lastRange = iter.next();
+
+		while (iter.hasNext()) {
+			OffsetRange nextRange = iter.next();
+
+			if (lastRange.overlaps(nextRange)) {
+				throw new IllegalArgumentException("Overlapping ranges " + lastRange + " and " + nextRange);
+			}
+
+			lastRange = nextRange;
+		}
+
+		DataSet<Edge<LongValue, NullValue>> edges = env
+			.fromParallelCollection(iterator, LongValue.class)
+				.setParallelism(parallelism)
+				.name("Edge iterators")
+			.flatMap(new LinkVertexToOffsets(vertexCount, offsetRanges))
+				.setParallelism(parallelism)
+				.name("Circulant graph edges");
+
+		// Graph
+		return Graph.fromDataSet(vertices, edges, env);
+	}
+
+	@FunctionAnnotation.ForwardedFields("*->f0")
+	private static class LinkVertexToOffsets
+	implements FlatMapFunction<LongValue, Edge<LongValue, NullValue>> {
+		private final long vertexCount;
+
+		private final List<OffsetRange> offsetRanges;
+
+		private LongValue target = new LongValue();
+
+		private Edge<LongValue, NullValue> edge = new Edge<>(null, target, NullValue.getInstance());
+
+		public LinkVertexToOffsets(long vertexCount, List<OffsetRange> offsetRanges) {
+			this.vertexCount = vertexCount;
+			this.offsetRanges = offsetRanges;
+		}
+
+		@Override
+		public void flatMap(LongValue source, Collector<Edge<LongValue, NullValue>> out)
+				throws Exception {
+			edge.f0 = source;
+			long sourceID = source.getValue();
+
+			for (OffsetRange offsetRange : offsetRanges) {
+				long targetID = sourceID + offsetRange.getOffset();
+
+				for (long i = offsetRange.getLength(); i > 0; i--) {
+					// add positive offset
+					target.setValue(targetID++ % vertexCount);
+					out.collect(edge);
+				}
+			}
+		}
+	}
+
+	/**
+	 * Stores the start offset and length configuration for an offset range.
+	 */
+	public static class OffsetRange implements Serializable, Comparable<OffsetRange> {
+		private long offset;
+
+		private long length;
+
+		/**
+		 * Construct a range with the given offset and length.
+		 *
+		 * @param offset the range offset
+		 * @param length the range length
+		 */
+		public OffsetRange(long offset, long length) {
+			this.offset = offset;
+			this.length = length;
+		}
+
+		/**
+		 * Get the range offset
+		 *
+		 * @return the offset
+		 */
+		public long getOffset() {
+			return offset;
+		}
+
+		/**
+		 * Get the range length
+		 *
+		 * @return the length
+		 */
+		public long getLength() {
+			return length;
+		}
+
+		/**
+		 * Get the offset of the last index in the range
+		 *
+		 * @return last offset
+		 */
+		public long getLastOffset() {
+			return offset + length - 1;
+		}
+
+		/**
+		 * Return true if and only if the other range and this range share a
+		 * common offset ID.
+		 *
+		 * @param other other range
+		 * @return whether ranges are overlapping
+		 */
+		public boolean overlaps(OffsetRange other) {
+			boolean overlapping = false;
+
+			long lastOffset = getLastOffset();
+			long otherLastOffset = other.getLastOffset();
+
+			// check whether this range contains other
+			overlapping |= (offset <= other.offset && other.offset <= lastOffset);
+			overlapping |= (offset <= otherLastOffset && otherLastOffset <= lastOffset);
+
+			// check whether other contains this range
+			overlapping |= (other.offset <= offset && offset <= otherLastOffset);
+			overlapping |= (other.offset <= lastOffset && lastOffset <= otherLastOffset);
+
+			return overlapping;
+		}
+
+		@Override
+		public String toString() {
+			return Long.toString(offset) + ":" + Long.toString(length);
+		}
+
+		@Override
+		public int compareTo(OffsetRange o) {
+			int cmp = Long.compare(offset, o.offset);
+			if (cmp != 0) {
+				return cmp;
+			}
+
+			return Long.compare(length, o.length);
+		}
+	}
+}
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
index 11c0bb0bcfc47..9dabe565aacf6 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/CompleteGraph.java
@@ -18,17 +18,10 @@
 
 package org.apache.flink.graph.generator;
 
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
-import org.apache.flink.graph.Edge;
 import org.apache.flink.graph.Graph;
-import org.apache.flink.graph.Vertex;
 import org.apache.flink.types.LongValue;
 import org.apache.flink.types.NullValue;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.LongValueSequenceIterator;
 import org.apache.flink.util.Preconditions;
 
 /**
@@ -61,53 +54,9 @@ public CompleteGraph(ExecutionEnvironment env, long vertexCount) {
 
 	@Override
 	public Graph<LongValue, NullValue, NullValue> generate() {
-		// Vertices
-		DataSet<Vertex<LongValue, NullValue>> vertices = GraphGeneratorUtils.vertexSequence(env, parallelism, vertexCount);
-
-		// Edges
-		LongValueSequenceIterator iterator = new LongValueSequenceIterator(0, this.vertexCount - 1);
-
-		DataSet<Edge<LongValue, NullValue>> edges = env
-			.fromParallelCollection(iterator, LongValue.class)
+		return new CirculantGraph(env, vertexCount)
+			.addRange(1, vertexCount - 1)
 				.setParallelism(parallelism)
-				.name("Edge iterators")
-			.flatMap(new LinkVertexToAll(vertexCount))
-				.setParallelism(parallelism)
-				.name("Complete graph edges");
-
-		// Graph
-		return Graph.fromDataSet(vertices, edges, env);
-	}
-
-	@ForwardedFields("*->f0")
-	private static class LinkVertexToAll
-	implements FlatMapFunction<LongValue, Edge<LongValue, NullValue>> {
-		private final long vertexCount;
-
-		private LongValue target = new LongValue();
-
-		private Edge<LongValue, NullValue> edge = new Edge<>(null, target, NullValue.getInstance());
-
-		public LinkVertexToAll(long vertexCount) {
-			this.vertexCount = vertexCount;
-		}
-
-		@Override
-		public void flatMap(LongValue source, Collector<Edge<LongValue, NullValue>> out)
-				throws Exception {
-			edge.f0 = source;
-
-			long s = source.getValue();
-			long t = (s + 1) % vertexCount;
-
-			while (s != t) {
-				target.setValue(t);
-				out.collect(edge);
-
-				if (++t == vertexCount) {
-					t = 0;
-				}
-			}
-		}
+				.generate();
 	}
 }
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EchoGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EchoGraph.java
new file mode 100644
index 0000000000000..c15cdca9b50c9
--- /dev/null
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/EchoGraph.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.generator;
+
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.Vertex;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * A {@link CirculantGraph} with {@code n} vertices defined by the width of a
+ * single range of offsets centered at {@code n/2}. A vertex is connected to
+ * 'far' vertices, which connect to 'near' vertices, which connect to 'far'
+ * vertices, ....
+ * <p>
+ * Every {@link Vertex} in the {@link EchoGraph} has the same degree.
+ * and vertices as far as possible are chose to be linked.
+ * {@link EchoGraph} is a specific case of {@link CirculantGraph}.
+ */
+public class EchoGraph
+extends AbstractGraphGenerator<LongValue, NullValue, NullValue> {
+
+	public static final int MINIMUM_VERTEX_COUNT = 2;
+
+	public static final int MINIMUM_VERTEX_DEGREE = 1;
+
+	// Required to create the DataSource
+	private final ExecutionEnvironment env;
+
+	// Required configuration
+	private long vertexCount;
+
+	private long vertexDegree;
+
+	/**
+	 * An undirected {@link Graph} whose vertices have the same degree.
+	 *
+	 * @param env the Flink execution environment
+	 * @param vertexCount number of vertices
+	 * @param vertexDegree degree of vertices
+	 */
+	public EchoGraph(ExecutionEnvironment env, long vertexCount, long vertexDegree) {
+		Preconditions.checkArgument(vertexCount >= MINIMUM_VERTEX_COUNT,
+			"Vertex count must be at least " + MINIMUM_VERTEX_COUNT);
+		Preconditions.checkArgument(vertexDegree >= MINIMUM_VERTEX_DEGREE,
+			"Vertex degree must be at least " + MINIMUM_VERTEX_DEGREE);
+		Preconditions.checkArgument(vertexDegree < vertexCount,
+			"Vertex degree must be less than the vertex count.");
+		Preconditions.checkArgument(vertexCount % 2 == 0 ^ vertexDegree % 2 == 0,
+			"Vertex count or vertex degree must be an even number but not both.");
+
+		this.env = env;
+		this.vertexCount = vertexCount;
+		this.vertexDegree = vertexDegree;
+	}
+
+	@Override
+	public Graph<LongValue, NullValue, NullValue> generate() {
+		return new CirculantGraph(env, vertexCount)
+			.addRange((vertexCount - vertexDegree + 1) / 2, vertexDegree)
+			.setParallelism(parallelism)
+			.generate();
+	}
+}
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
index 23a6f98bafb9b..0570dd2667f51 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/generator/GridGraph.java
@@ -85,9 +85,7 @@ public GridGraph addDimension(long size, boolean wrapEndpoints) {
 
 	@Override
 	public Graph<LongValue, NullValue, NullValue> generate() {
-		if (dimensions.isEmpty()) {
-			throw new RuntimeException("No dimensions added to GridGraph");
-		}
+		Preconditions.checkState(!dimensions.isEmpty(), "No dimensions added to GridGraph");
 
 		// Vertices
 		DataSet<Vertex<LongValue, NullValue>> vertices = GraphGeneratorUtils.vertexSequence(env, parallelism, vertexCount);
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/CirculantGraphTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/CirculantGraphTest.java
new file mode 100644
index 0000000000000..aae88ca5bf665
--- /dev/null
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/CirculantGraphTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.generator;
+
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.graph.Edge;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.Vertex;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.assertEquals;
+
+public class CirculantGraphTest
+extends AbstractGraphTest {
+
+	@Test
+	public void testGraph()
+			throws Exception {
+		Graph<LongValue, NullValue, NullValue> graph = new CirculantGraph(env, 10)
+			.addRange(4, 3)
+			.generate();
+
+		String vertices = "0; 1; 2; 3; 4; 5; 6; 7; 8; 9";
+		String edges = "0,4; 0,5; 0,6; 1,5; 1,6; 1,7; 2,6;" +
+			"2,7; 2,8; 3,7; 3,8; 3,9; 4,0; 4,8; 4,9;" +
+			"5,0; 5,1; 5,9; 6,0; 6,1; 6,2; 7,1; 7,2; 7,3;" +
+			"8,2; 8,3; 8,4; 9,3; 9,4; 9,5";
+
+		TestUtils.compareGraph(graph, vertices, edges);
+	}
+
+	@Test
+	public void testGraphMetrics()
+			throws Exception {
+		int vertexCount = 10;
+		int offset = 4;
+		int length = 2;
+
+		Graph<LongValue, NullValue, NullValue> graph = new CirculantGraph(env, 10)
+			.addRange(offset, length)
+			.generate();
+
+		assertEquals(vertexCount, graph.numberOfVertices());
+		assertEquals(vertexCount * length, graph.numberOfEdges());
+
+		long maxInDegree = graph.inDegrees().max(1).collect().get(0).f1.getValue();
+		long maxOutDegree = graph.outDegrees().max(1).collect().get(0).f1.getValue();
+
+		assertEquals(length, maxInDegree);
+		assertEquals(length, maxOutDegree);
+	}
+
+	@Test
+	public void testParallelism()
+			throws Exception {
+		int parallelism = 2;
+
+		Graph<LongValue, NullValue, NullValue> graph = new CirculantGraph(env, 10)
+			.addRange(4, 2)
+			.setParallelism(parallelism)
+			.generate();
+
+		graph.getVertices().output(new DiscardingOutputFormat<Vertex<LongValue, NullValue>>());
+		graph.getEdges().output(new DiscardingOutputFormat<Edge<LongValue, NullValue>>());
+
+		TestUtils.verifyParallelism(env, parallelism);
+	}
+}
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/EchoGraphTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/EchoGraphTest.java
new file mode 100644
index 0000000000000..777b576b03f6b
--- /dev/null
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/generator/EchoGraphTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.generator;
+
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.graph.Edge;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.Vertex;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.assertEquals;
+
+public class EchoGraphTest
+extends AbstractGraphTest {
+
+	@Rule
+	public ExpectedException thrown = ExpectedException.none();
+
+	@Test
+	public void testGraphWithEvenVertexCountWithOddVertexDegree()
+			throws Exception {
+		Graph<LongValue, NullValue, NullValue> graph = new EchoGraph(env, 10, 3)
+			.generate();
+
+		String vertices = "0; 1; 2; 3; 4; 5; 6; 7; 8; 9";
+		String edges = "0,4; 0,5; 0,6; 1,5; 1,6; 1,7; 2,6;" +
+			"2,7; 2,8; 3,7; 3,8; 3,9; 4,0; 4,8; 4,9;" +
+			"5,0; 5,1; 5,9; 6,0; 6,1; 6,2; 7,1; 7,2; 7,3;" +
+			"8,2; 8,3; 8,4; 9,3; 9,4; 9,5";
+
+		TestUtils.compareGraph(graph, vertices, edges);
+	}
+
+	@Test
+	public void testGraphWithOddVertexCountWithEvenVertexDegree()
+			throws Exception {
+		Graph<LongValue, NullValue, NullValue> graph = new EchoGraph(env, 9, 2)
+			.generate();
+
+		String vertices = "0; 1; 2; 3; 4; 5; 6; 7; 8";
+		String edges = "0,4; 0,5; 1,5; 1,6; 2,6; 2,7;" +
+			"3,7; 3,8; 4,8; 4,0; 5,0; 5,1;" +
+			"6,1; 6,2; 7,2; 7,3; 8,3; 8,4";
+
+		TestUtils.compareGraph(graph, vertices, edges);
+	}
+
+	@Test
+	public void testGraphWithOddVertexCountWithOddVertexDegree()
+			throws Exception {
+		thrown.expect(IllegalArgumentException.class);
+		thrown.expectMessage("Vertex count or vertex degree must be an even number but not both.");
+
+		new EchoGraph(env, 5, 3).generate();
+	}
+
+	@Test
+	public void testGraphWithEvenVertexCountWithEvenVertexDegree()
+			throws Exception {
+		thrown.expect(IllegalArgumentException.class);
+		thrown.expectMessage("Vertex count or vertex degree must be an even number but not both.");
+
+		new EchoGraph(env, 6, 2).generate();
+	}
+
+	@Test
+	public void testGraphWithVertexDegreeTooLarge()
+			throws Exception {
+		thrown.expect(IllegalArgumentException.class);
+		thrown.expectMessage("Vertex degree must be less than the vertex count.");
+
+		new EchoGraph(env, 8, 8).generate();
+	}
+
+	@Test
+	public void testGraphMetrics()
+			throws Exception {
+		int vertexCount = 10;
+		int vertexDegree = 3;
+
+		Graph<LongValue, NullValue, NullValue> graph = new EchoGraph(env, vertexCount, vertexDegree)
+			.generate();
+
+		assertEquals(vertexCount, graph.numberOfVertices());
+		assertEquals(vertexCount * vertexDegree, graph.numberOfEdges());
+
+		long maxInDegree = graph.inDegrees().max(1).collect().get(0).f1.getValue();
+		long maxOutDegree = graph.outDegrees().max(1).collect().get(0).f1.getValue();
+
+		assertEquals(vertexDegree, maxInDegree);
+		assertEquals(vertexDegree, maxOutDegree);
+	}
+
+	@Test
+	public void testParallelism()
+			throws Exception {
+		int parallelism = 2;
+
+		Graph<LongValue, NullValue, NullValue> graph = new EchoGraph(env, 10, 3)
+			.setParallelism(parallelism)
+			.generate();
+
+		graph.getVertices().output(new DiscardingOutputFormat<Vertex<LongValue, NullValue>>());
+		graph.getEdges().output(new DiscardingOutputFormat<Edge<LongValue, NullValue>>());
+
+		TestUtils.verifyParallelism(env, parallelism);
+	}
+}

From 61977ba041b3cd58b7f8ecf03c09e5b1ca5f77cb Mon Sep 17 00:00:00 2001
From: Rodrigo Bonifacio <rbonifacio123@gmail.com>
Date: Wed, 10 May 2017 11:54:44 -0300
Subject: [PATCH 229/253] [hotfix] Refactor to use multi-catch

This closes #3866
---
 .../org/apache/flink/api/java/io/AvroOutputFormat.java    | 4 +---
 .../flink/api/java/typeutils/runtime/PojoComparator.java  | 4 +---
 .../runtime/iterative/concurrent/BlockingBackChannel.java | 4 +---
 .../flink/runtime/operators/hash/CompactingHashTable.java | 8 ++------
 .../apache/flink/runtime/operators/util/TaskConfig.java   | 4 +---
 5 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/flink-connectors/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java b/flink-connectors/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
index 1db45a5619015..ae9036256360f 100644
--- a/flink-connectors/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
+++ b/flink-connectors/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
@@ -125,9 +125,7 @@ public void open(int taskNumber, int numTasks) throws IOException {
 			datumWriter = new SpecificDatumWriter<E>(avroValueType);
 			try {
 				schema = ((org.apache.avro.specific.SpecificRecordBase)avroValueType.newInstance()).getSchema();
-			} catch (InstantiationException e) {
-				throw new RuntimeException(e.getMessage());
-			} catch (IllegalAccessException e) {
+			} catch (InstantiationException | IllegalAccessException e) {
 				throw new RuntimeException(e.getMessage());
 			}
 		} else {
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
index 945abc81bd1ba..a3f4280c9b9a0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
@@ -131,9 +131,7 @@ private PojoComparator(PojoComparator<T> toClone) {
 		try {
 			this.serializer = (TypeSerializer<T>) InstantiationUtil.deserializeObject(
 					InstantiationUtil.serializeObject(toClone.serializer), Thread.currentThread().getContextClassLoader());
-		} catch (IOException e) {
-			throw new RuntimeException("Cannot copy serializer", e);
-		} catch (ClassNotFoundException e) {
+		} catch (IOException | ClassNotFoundException e) {
 			throw new RuntimeException("Cannot copy serializer", e);
 		}
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/BlockingBackChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/BlockingBackChannel.java
index 8631c992a1143..067bbfe0707de 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/BlockingBackChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/BlockingBackChannel.java
@@ -51,9 +51,7 @@ public BlockingBackChannel(SerializedUpdateBuffer buffer) {
 	public DataInputView getReadEndAfterSuperstepEnded() {
 		try {
 			return queue.take().switchBuffers();
-		} catch (InterruptedException e) {
-			throw new RuntimeException(e);
-		} catch (IOException e) {
+		} catch (InterruptedException | IOException e) {
 			throw new RuntimeException(e);
 		}
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java
index eacb02c7b2f3d..67ca342b0a255 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java
@@ -451,12 +451,8 @@ private long insertRecordIntoPartition(T record, InMemoryPartition<T> partition,
 						this.compactionMemory.allocateSegments((int)(newPointer >> this.pageSizeInBits));
 					}
 					return newPointer;
-				}
-				catch (EOFException ex) {
-					throw new RuntimeException("Memory ran out. Compaction failed. " +
-							getMemoryConsumptionString() + " Message: " + ex.getMessage());
-				}
-				catch (IndexOutOfBoundsException ex) {
+				} 
+				catch (EOFException | IndexOutOfBoundsException ex) {
 					throw new RuntimeException("Memory ran out. Compaction failed. " +
 							getMemoryConsumptionString() + " Message: " + ex.getMessage());
 				}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/TaskConfig.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/TaskConfig.java
index 71c040512ef02..a5decf49c0194 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/TaskConfig.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/TaskConfig.java
@@ -286,9 +286,7 @@ public void setStubWrapper(UserCodeWrapper<?> wrapper) {
 	public <T> UserCodeWrapper<T> getStubWrapper(ClassLoader cl) {
 		try {
 			return (UserCodeWrapper<T>) InstantiationUtil.readObjectFromConfig(this.config, STUB_OBJECT, cl);
-		} catch (ClassNotFoundException e) {
-			throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
-		} catch (IOException e) {
+		} catch (ClassNotFoundException | IOException e) {
 			throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
 		}
 	}

From 55eea6cfa7d1de19ee2d1a847d067d8c87c2f59a Mon Sep 17 00:00:00 2001
From: Fabian Hueske <fhueske@apache.org>
Date: Mon, 8 May 2017 18:41:37 +0200
Subject: [PATCH 230/253] [FLINK-6491] [table] Add QueryConfig and state clean
 up for non-windowed aggregates.

---
 .../table/api/BatchTableEnvironment.scala     |  17 ++-
 .../apache/flink/table/api/QueryConfig.scala  | 102 ++++++++++++++++
 .../table/api/StreamTableEnvironment.scala    |  51 ++++++--
 .../flink/table/api/TableEnvironment.scala    |   2 +-
 .../api/java/StreamTableEnvironment.scala     | 115 ++++++++++++++++--
 .../api/scala/StreamTableEnvironment.scala    |  46 ++++++-
 .../table/api/scala/TableConversions.scala    |  40 +++++-
 .../org/apache/flink/table/api/table.scala    |  26 +++-
 .../nodes/datastream/DataStreamCalc.scala     |   8 +-
 .../datastream/DataStreamCorrelate.scala      |   8 +-
 .../datastream/DataStreamGroupAggregate.scala |  20 ++-
 .../DataStreamGroupWindowAggregate.scala      |   8 +-
 .../datastream/DataStreamOverAggregate.scala  |   9 +-
 .../plan/nodes/datastream/DataStreamRel.scala |   7 +-
 .../nodes/datastream/DataStreamScan.scala     |   7 +-
 .../nodes/datastream/DataStreamUnion.scala    |  10 +-
 .../nodes/datastream/DataStreamValues.scala   |   6 +-
 .../datastream/StreamTableSourceScan.scala    |   7 +-
 .../runtime/aggregate/AggregateUtil.scala     |   6 +-
 .../aggregate/GroupAggProcessFunction.scala   |  54 +++++++-
 .../table/utils/MockTableEnvironment.scala    |   9 +-
 21 files changed, 494 insertions(+), 64 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/QueryConfig.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index 2a3cedf0979a6..f33c187fa04b1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -32,7 +32,7 @@ import org.apache.flink.api.java.io.DiscardingOutputFormat
 import org.apache.flink.api.java.typeutils.GenericTypeInfo
 import org.apache.flink.api.java.{DataSet, ExecutionEnvironment}
 import org.apache.flink.table.explain.PlanJsonParser
-import org.apache.flink.table.expressions.{Expression, RowtimeAttribute, TimeAttribute}
+import org.apache.flink.table.expressions.{Expression, TimeAttribute}
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.nodes.dataset.DataSetRel
 import org.apache.flink.table.plan.rules.FlinkRuleSets
@@ -113,9 +113,20 @@ abstract class BatchTableEnvironment(
     *
     * @param table The [[Table]] to write.
     * @param sink The [[TableSink]] to write the [[Table]] to.
+    * @param qConfig The configuration for the query to generate.
     * @tparam T The expected type of the [[DataSet]] which represents the [[Table]].
     */
-  override private[flink] def writeToSink[T](table: Table, sink: TableSink[T]): Unit = {
+  override private[flink] def writeToSink[T](
+      table: Table,
+      sink: TableSink[T],
+      qConfig: QueryConfig): Unit = {
+
+    // We do not pass the configuration on, because there is nothing to configure for batch queries.
+    val bQConfig = qConfig match {
+      case batchConfig: BatchQueryConfig => batchConfig
+      case _ =>
+        throw new TableException("BatchQueryConfig required to configure batch query.")
+    }
 
     sink match {
       case batchSink: BatchTableSink[T] =>
@@ -125,7 +136,7 @@ abstract class BatchTableEnvironment(
         // Give the DataSet to the TableSink to emit it.
         batchSink.emitDataSet(result)
       case _ =>
-        throw new TableException("BatchTableSink required to emit batch Table")
+        throw new TableException("BatchTableSink required to emit batch Table.")
     }
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/QueryConfig.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/QueryConfig.scala
new file mode 100644
index 0000000000000..8e8b5ac73c94e
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/QueryConfig.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api
+
+import _root_.java.io.Serializable
+import org.apache.flink.api.common.time.Time
+
+class QueryConfig private[table] extends Serializable {}
+
+/**
+  * The [[BatchQueryConfig]] holds parameters to configure the behavior of batch queries.
+  */
+class BatchQueryConfig private[table] extends QueryConfig
+
+/**
+  * The [[StreamQueryConfig]] holds parameters to configure the behavior of streaming queries.
+  *
+  * An empty [[StreamQueryConfig]] can be generated using the [[StreamTableEnvironment.qConf]]
+  * method.
+  */
+class StreamQueryConfig private[table] extends QueryConfig {
+
+  /**
+    * The minimum time until state which was not updated will be retained.
+    * State might be cleared and removed if it was not updated for the defined period of time.
+    */
+  private var minIdleStateRetentionTime: Long = Long.MinValue
+
+  /**
+    * The maximum time until state which was not updated will be retained.
+    * State will be cleared and removed if it was not updated for the defined period of time.
+    */
+  private var maxIdleStateRetentionTime: Long = Long.MinValue
+
+  /**
+    * Specifies the time interval for how long idle state, i.e., state which was not updated, will
+    * be retained. When state was not updated for the specified interval of time, it will be cleared
+    * and removed.
+    *
+    * When new data arrives for previously cleaned-up state, the new data will be handled as if it
+    * was the first data. This can result in previous results being overwritten.
+    *
+    * Note: [[setIdleStateRetentionTime(minTime: Time, maxTime: Time)]] allows to set a minimum and
+    * maximum time for state to be retained. This method is more efficient, because the system has
+    * to do less bookkeeping to identify the time at which state must be cleared.
+    *
+    * @param time The time interval for how long idle state is retained. Set to 0 (zero) to never
+    *             clean-up the state.
+    */
+  def setIdleStateRetentionTime(time: Time): StreamQueryConfig = {
+    setIdleStateRetentionTime(time, time)
+  }
+
+  /**
+    * Specifies a minimum and a maximum time interval for how long idle state, i.e., state which
+    * was not updated, will be retained.
+    * State will never be cleared until it was idle for less than the minimum time and will never
+    * be kept if it was idle for more than the maximum time.
+    *
+    * When new data arrives for previously cleaned-up state, the new data will be handled as if it
+    * was the first data. This can result in previous results being overwritten.
+    *
+    * Set to 0 (zero) to never clean-up the state.
+    *
+    * @param minTime The minimum time interval for which idle state is retained. Set to 0 (zero) to
+    *                never clean-up the state.
+    * @param maxTime The maximum time interval for which idle state is retained. May not be smaller
+    *                than than minTime. Set to 0 (zero) to never clean-up the state.
+    */
+  def setIdleStateRetentionTime(minTime: Time, maxTime: Time): StreamQueryConfig = {
+    if (maxTime.toMilliseconds < minTime.toMilliseconds) {
+      throw new IllegalArgumentException("maxTime may not be smaller than minTime.")
+    }
+    minIdleStateRetentionTime = minTime.toMilliseconds
+    maxIdleStateRetentionTime = maxTime.toMilliseconds
+    this
+  }
+
+  def getMinIdleStateRetentionTime: Long = {
+    minIdleStateRetentionTime
+  }
+
+  def getMaxIdleStateRetentionTime: Long = {
+    maxIdleStateRetentionTime
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index aef2b1b5a946c..c594d4c300a71 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -81,6 +81,8 @@ abstract class StreamTableEnvironment(
   // the naming pattern for internally registered tables.
   private val internalNamePattern = "^_DataStreamTable_[0-9]+$".r
 
+  def qConf: StreamQueryConfig = new StreamQueryConfig
+
   /**
     * Checks if the chosen table name is valid.
     *
@@ -126,9 +128,20 @@ abstract class StreamTableEnvironment(
     *
     * @param table The [[Table]] to write.
     * @param sink The [[TableSink]] to write the [[Table]] to.
+    * @param qConfig The configuration for the query to generate.
     * @tparam T The expected type of the [[DataStream]] which represents the [[Table]].
     */
-  override private[flink] def writeToSink[T](table: Table, sink: TableSink[T]): Unit = {
+  override private[flink] def writeToSink[T](
+      table: Table,
+      sink: TableSink[T],
+      qConfig: QueryConfig): Unit = {
+
+    // Check query configuration
+    val sQConf = qConfig match {
+      case streamConfig: StreamQueryConfig => streamConfig
+      case _ =>
+        throw new TableException("StreamQueryConfig required to configure stream query.")
+    }
 
     sink match {
 
@@ -137,7 +150,7 @@ abstract class StreamTableEnvironment(
         val outputType = sink.getOutputType
         // translate the Table into a DataStream and provide the type that the TableSink expects.
         val result: DataStream[T] =
-          translate(table, updatesAsRetraction = true, withChangeFlag = true)(outputType)
+          translate(table, sQConf, updatesAsRetraction = true, withChangeFlag = true)(outputType)
         // Give the DataStream to the TableSink to emit it.
         retractSink.asInstanceOf[RetractStreamTableSink[Any]]
           .emitDataStream(result.asInstanceOf[DataStream[JTuple2[JBool, Any]]])
@@ -160,7 +173,11 @@ abstract class StreamTableEnvironment(
         val outputType = sink.getOutputType
         // translate the Table into a DataStream and provide the type that the TableSink expects.
         val result: DataStream[T] =
-          translate(optimizedPlan, table.getRelNode.getRowType, withChangeFlag = true)(outputType)
+          translate(
+            optimizedPlan,
+            table.getRelNode.getRowType,
+            sQConf,
+            withChangeFlag = true)(outputType)
         // Give the DataStream to the TableSink to emit it.
         upsertSink.asInstanceOf[UpsertStreamTableSink[Any]]
           .emitDataStream(result.asInstanceOf[DataStream[JTuple2[JBool, Any]]])
@@ -176,7 +193,11 @@ abstract class StreamTableEnvironment(
         val outputType = sink.getOutputType
         // translate the Table into a DataStream and provide the type that the TableSink expects.
         val result: DataStream[T] =
-          translate(optimizedPlan, table.getRelNode.getRowType, withChangeFlag = false)(outputType)
+          translate(
+            optimizedPlan,
+            table.getRelNode.getRowType,
+            sQConf,
+            withChangeFlag = false)(outputType)
         // Give the DataStream to the TableSink to emit it.
         appendSink.asInstanceOf[AppendStreamTableSink[T]].emitDataStream(result)
 
@@ -545,17 +566,21 @@ abstract class StreamTableEnvironment(
     * Table API calls and / or SQL queries and generating corresponding [[DataStream]] operators.
     *
     * @param table The root node of the relational expression tree.
+    * @param qConfig The configuration for the query to generate.
     * @param updatesAsRetraction Set to true to encode updates as retraction messages.
     * @param withChangeFlag Set to true to emit records with change flags.
     * @param tpe The [[TypeInformation]] of the resulting [[DataStream]].
     * @tparam A The type of the resulting [[DataStream]].
     * @return The [[DataStream]] that corresponds to the translated [[Table]].
     */
-  protected def translate[A](table: Table, updatesAsRetraction: Boolean, withChangeFlag: Boolean)
-      (implicit tpe: TypeInformation[A]): DataStream[A] = {
+  protected def translate[A](
+      table: Table,
+      qConfig: StreamQueryConfig,
+      updatesAsRetraction: Boolean,
+      withChangeFlag: Boolean)(implicit tpe: TypeInformation[A]): DataStream[A] = {
     val relNode = table.getRelNode
     val dataStreamPlan = optimize(relNode, updatesAsRetraction)
-    translate(dataStreamPlan, relNode.getRowType, withChangeFlag)
+    translate(dataStreamPlan, relNode.getRowType, qConfig, withChangeFlag)
   }
 
   /**
@@ -564,6 +589,7 @@ abstract class StreamTableEnvironment(
     * @param logicalPlan The root node of the relational expression tree.
     * @param logicalType The row type of the result. Since the logicalPlan can lose the
     *                    field naming during optimization we pass the row type separately.
+    * @param qConfig     The configuration for the query to generate.
     * @param withChangeFlag Set to true to emit records with change flags.
     * @param tpe         The [[TypeInformation]] of the resulting [[DataStream]].
     * @tparam A The type of the resulting [[DataStream]].
@@ -572,6 +598,7 @@ abstract class StreamTableEnvironment(
   protected def translate[A](
       logicalPlan: RelNode,
       logicalType: RelDataType,
+      qConfig: StreamQueryConfig,
       withChangeFlag: Boolean)
       (implicit tpe: TypeInformation[A]): DataStream[A] = {
 
@@ -583,7 +610,7 @@ abstract class StreamTableEnvironment(
     }
 
     // get CRow plan
-    val plan: DataStream[CRow] = translateToCRow(logicalPlan)
+    val plan: DataStream[CRow] = translateToCRow(logicalPlan, qConfig)
 
     // convert CRow to output type
     val conversion = if (withChangeFlag) {
@@ -615,14 +642,16 @@ abstract class StreamTableEnvironment(
     * Translates a logical [[RelNode]] plan into a [[DataStream]] of type [[CRow]].
     *
     * @param logicalPlan The logical plan to translate.
+    * @param qConfig The configuration for the query to generate.
     * @return The [[DataStream]] of type [[CRow]].
     */
   protected def translateToCRow(
-    logicalPlan: RelNode): DataStream[CRow] = {
+    logicalPlan: RelNode,
+    qConfig: StreamQueryConfig): DataStream[CRow] = {
 
     logicalPlan match {
       case node: DataStreamRel =>
-        node.translateToPlan(this)
+        node.translateToPlan(this, qConfig)
       case _ =>
         throw TableException("Cannot generate DataStream due to an invalid logical plan. " +
           "This is a bug and should not happen. Please file an issue.")
@@ -638,7 +667,7 @@ abstract class StreamTableEnvironment(
   def explain(table: Table): String = {
     val ast = table.getRelNode
     val optimizedPlan = optimize(ast, updatesAsRetraction = false)
-    val dataStream = translateToCRow(optimizedPlan)
+    val dataStream = translateToCRow(optimizedPlan, qConf)
 
     val env = dataStream.getExecutionEnvironment
     val jsonSqlPlan = env.getExecutionPlan
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index bf4a8e072037a..9f50f0c806a10 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -510,7 +510,7 @@ abstract class TableEnvironment(val config: TableConfig) {
     * @param sink The [[TableSink]] to write the [[Table]] to.
     * @tparam T The data type that the [[TableSink]] expects.
     */
-  private[flink] def writeToSink[T](table: Table, sink: TableSink[T]): Unit
+  private[flink] def writeToSink[T](table: Table, sink: TableSink[T], conf: QueryConfig): Unit
 
   /**
     * Registers a Calcite [[AbstractTable]] in the TableEnvironment's catalog.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
index a70bcca79c36f..c3b59511d2954 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
@@ -150,9 +150,50 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T](table: Table, clazz: Class[T]): DataStream[T] = {
+    toDataStream(table, clazz, qConf)
+  }
+
+  /**
+    * Converts the given [[Table]] into an append [[DataStream]] of a specified type.
+    *
+    * The [[Table]] must only have insert (append) changes. If the [[Table]] is also modified
+    * by update or delete changes, the conversion will fail.
+    *
+    * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
+    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+    * types: Fields are mapped by position, field types must match.
+    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+    *
+    * @param table The [[Table]] to convert.
+    * @param typeInfo The [[TypeInformation]] that specifies the type of the [[DataStream]].
+    * @tparam T The type of the resulting [[DataStream]].
+    * @return The converted [[DataStream]].
+    */
+  def toDataStream[T](table: Table, typeInfo: TypeInformation[T]): DataStream[T] = {
+    toDataStream(table, typeInfo, qConf)
+  }
+
+  /**
+    * Converts the given [[Table]] into an append [[DataStream]] of a specified type.
+    *
+    * The [[Table]] must only have insert (append) changes. If the [[Table]] is also modified
+    * by update or delete changes, the conversion will fail.
+    *
+    * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
+    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+    * types: Fields are mapped by position, field types must match.
+    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+    *
+    * @param table The [[Table]] to convert.
+    * @param clazz The class of the type of the resulting [[DataStream]].
+    * @param qConfig The configuration of the query to generate.
+    * @tparam T The type of the resulting [[DataStream]].
+    * @return The converted [[DataStream]].
+    */
+  def toDataStream[T](table: Table, clazz: Class[T], qConfig: StreamQueryConfig): DataStream[T] = {
     val typeInfo = TypeExtractor.createTypeInfo(clazz)
     TableEnvironment.validateType(typeInfo)
-    translate[T](table, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
+    translate[T](table, qConfig, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
   }
 
   /**
@@ -168,12 +209,64 @@ class StreamTableEnvironment(
     *
     * @param table The [[Table]] to convert.
     * @param typeInfo The [[TypeInformation]] that specifies the type of the [[DataStream]].
+    * @param qConfig The configuration of the query to generate.
     * @tparam T The type of the resulting [[DataStream]].
     * @return The converted [[DataStream]].
     */
-  def toDataStream[T](table: Table, typeInfo: TypeInformation[T]): DataStream[T] = {
+  def toDataStream[T](
+      table: Table,
+      typeInfo: TypeInformation[T],
+      qConfig: StreamQueryConfig): DataStream[T] = {
     TableEnvironment.validateType(typeInfo)
-    translate[T](table, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
+    translate[T](table, qConfig, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
+  }
+
+  /**
+    * Converts the given [[Table]] into a [[DataStream]] of add and retract messages.
+    * The message will be encoded as [[JTuple2]]. The first field is a [[JBool]] flag,
+    * the second field holds the record of the specified type [[T]].
+    *
+    * A true [[JBool]] flag indicates an add message, a false flag indicates a retract message.
+    *
+    * The fields of the [[Table]] are mapped to the requested type as follows:
+    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+    * types: Fields are mapped by position, field types must match.
+    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+    *
+    * @param table The [[Table]] to convert.
+    * @param clazz The class of the requested record type.
+    * @tparam T The type of the requested record type.
+    * @return The converted [[DataStream]].
+    */
+  def toRetractStream[T](
+      table: Table,
+      clazz: Class[T]): DataStream[JTuple2[JBool, T]] = {
+
+    toRetractStream(table, clazz, qConf)
+  }
+
+  /**
+    * Converts the given [[Table]] into a [[DataStream]] of add and retract messages.
+    * The message will be encoded as [[JTuple2]]. The first field is a [[JBool]] flag,
+    * the second field holds the record of the specified type [[T]].
+    *
+    * A true [[JBool]] flag indicates an add message, a false flag indicates a retract message.
+    *
+    * The fields of the [[Table]] are mapped to the requested type as follows:
+    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+    * types: Fields are mapped by position, field types must match.
+    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+    *
+    * @param table The [[Table]] to convert.
+    * @param typeInfo The [[TypeInformation]] of the requested record type.
+    * @tparam T The type of the requested record type.
+    * @return The converted [[DataStream]].
+    */
+  def toRetractStream[T](
+      table: Table,
+      typeInfo: TypeInformation[T]): DataStream[JTuple2[JBool, T]] = {
+
+    toRetractStream(table, typeInfo, qConf)
   }
 
   /**
@@ -190,17 +283,21 @@ class StreamTableEnvironment(
     *
     * @param table The [[Table]] to convert.
     * @param clazz The class of the requested record type.
+    * @param qConfig The configuration of the query to generate.
     * @tparam T The type of the requested record type.
     * @return The converted [[DataStream]].
     */
-  def toRetractStream[T](table: Table, clazz: Class[T]):
-    DataStream[JTuple2[JBool, T]] = {
+  def toRetractStream[T](
+      table: Table,
+      clazz: Class[T],
+      qConfig: StreamQueryConfig): DataStream[JTuple2[JBool, T]] = {
 
     val typeInfo = TypeExtractor.createTypeInfo(clazz)
     TableEnvironment.validateType(typeInfo)
     val resultType = new TupleTypeInfo[JTuple2[JBool, T]](Types.BOOLEAN, typeInfo)
     translate[JTuple2[JBool, T]](
       table,
+      qConfig,
       updatesAsRetraction = true,
       withChangeFlag = true)(resultType)
   }
@@ -219,11 +316,14 @@ class StreamTableEnvironment(
     *
     * @param table The [[Table]] to convert.
     * @param typeInfo The [[TypeInformation]] of the requested record type.
+    * @param qConfig The configuration of the query to generate.
     * @tparam T The type of the requested record type.
     * @return The converted [[DataStream]].
     */
-  def toRetractStream[T](table: Table, typeInfo: TypeInformation[T]):
-    DataStream[JTuple2[JBool, T]] = {
+  def toRetractStream[T](
+      table: Table,
+      typeInfo: TypeInformation[T],
+      qConfig: StreamQueryConfig): DataStream[JTuple2[JBool, T]] = {
 
     TableEnvironment.validateType(typeInfo)
     val resultTypeInfo = new TupleTypeInfo[JTuple2[JBool, T]](
@@ -232,6 +332,7 @@ class StreamTableEnvironment(
     )
     translate[JTuple2[JBool, T]](
       table,
+      qConfig,
       updatesAsRetraction = true,
       withChangeFlag = true)(resultTypeInfo)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
index e5ad6c23e35c5..56f7d55744cf9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.api.scala
 
 import org.apache.flink.api.scala._
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.table.api.{Table, TableConfig, TableEnvironment}
+import org.apache.flink.table.api.{StreamQueryConfig, Table, TableConfig, TableEnvironment}
 import org.apache.flink.table.expressions.Expression
 import org.apache.flink.table.functions.{AggregateFunction, TableFunction}
 import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
@@ -143,8 +143,29 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T: TypeInformation](table: Table): DataStream[T] = {
+    toDataStream(table, qConf)
+  }
+
+  /**
+    * Converts the given [[Table]] into an append [[DataStream]] of a specified type.
+    *
+    * The [[Table]] must only have insert (append) changes. If the [[Table]] is also modified
+    * by update or delete changes, the conversion will fail.
+    *
+    * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
+    * - [[org.apache.flink.types.Row]] and Scala Tuple types: Fields are mapped by position, field
+    * types must match.
+    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+    *
+    * @param table The [[Table]] to convert.
+    * @param qConfig The configuration of the query to generate.
+    * @tparam T The type of the resulting [[DataStream]].
+    * @return The converted [[DataStream]].
+    */
+  def toDataStream[T: TypeInformation](table: Table, qConfig: StreamQueryConfig): DataStream[T] = {
     val returnType = createTypeInformation[T]
-    asScalaStream(translate(table, updatesAsRetraction = false, withChangeFlag = false)(returnType))
+    asScalaStream(
+      translate(table, qConfig, updatesAsRetraction = false, withChangeFlag = false)(returnType))
   }
 
 /**
@@ -159,8 +180,27 @@ class StreamTableEnvironment(
   * @return The converted [[DataStream]].
   */
   def toRetractStream[T: TypeInformation](table: Table): DataStream[(Boolean, T)] = {
+    toRetractStream(table, qConf)
+  }
+
+  /**
+    * Converts the given [[Table]] into a [[DataStream]] of add and retract messages.
+    * The message will be encoded as [[Tuple2]]. The first field is a [[Boolean]] flag,
+    * the second field holds the record of the specified type [[T]].
+    *
+    * A true [[Boolean]] flag indicates an add message, a false flag indicates a retract message.
+    *
+    * @param table The [[Table]] to convert.
+    * @param qConfig The configuration of the query to generate.
+    * @tparam T The type of the requested data type.
+    * @return The converted [[DataStream]].
+    */
+  def toRetractStream[T: TypeInformation](
+      table: Table,
+      qConfig: StreamQueryConfig): DataStream[(Boolean, T)] = {
     val returnType = createTypeInformation[(Boolean, T)]
-    asScalaStream(translate(table, updatesAsRetraction = true, withChangeFlag = true)(returnType))
+    asScalaStream(
+      translate(table, qConfig, updatesAsRetraction = true, withChangeFlag = true)(returnType))
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
index 5efff62599185..966b42f19756d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
@@ -21,7 +21,7 @@ package org.apache.flink.table.api.scala
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.scala._
 import org.apache.flink.streaming.api.scala.DataStream
-import org.apache.flink.table.api.{Table, TableException}
+import org.apache.flink.table.api.{StreamQueryConfig, Table, TableException}
 import org.apache.flink.table.api.scala.{BatchTableEnvironment => ScalaBatchTableEnv}
 import org.apache.flink.table.api.scala.{StreamTableEnvironment => ScalaStreamTableEnv}
 
@@ -57,6 +57,21 @@ class TableConversions(table: Table) {
     }
   }
 
+  /** Converts the [[Table]] to a [[DataStream]] of the specified type.
+    *
+    * @param qConfig The configuration for the generated query.
+    */
+  def toDataStream[T: TypeInformation](qConfig: StreamQueryConfig): DataStream[T] = {
+    table.tableEnv match {
+      case tEnv: ScalaStreamTableEnv =>
+        tEnv.toDataStream(table, qConfig)
+      case _ =>
+        throw new TableException(
+          "Only tables that originate from Scala DataStreams " +
+            "can be converted to Scala DataStreams.")
+    }
+  }
+
   /** Converts the [[Table]] to a [[DataStream]] of add and retract messages.
     * The message will be encoded as [[Tuple2]]. The first field is a [[Boolean]] flag,
     * the second field holds the record of the specified type [[T]].
@@ -76,5 +91,28 @@ class TableConversions(table: Table) {
     }
   }
 
+  /** Converts the [[Table]] to a [[DataStream]] of add and retract messages.
+    * The message will be encoded as [[Tuple2]]. The first field is a [[Boolean]] flag,
+    * the second field holds the record of the specified type [[T]].
+    *
+    * A true [[Boolean]] flag indicates an add message, a false flag indicates a retract message.
+    *
+    * @param qConfig The configuration for the generated query.
+    *
+    */
+  def toRetractStream[T: TypeInformation](qConfig: StreamQueryConfig): DataStream[(Boolean, T)] = {
+
+    table.tableEnv match {
+      case tEnv: ScalaStreamTableEnv =>
+        tEnv.toRetractStream(table, qConfig)
+      case _ =>
+        throw new TableException(
+          "Only tables that originate from Scala DataStreams " +
+            "can be converted to Scala DataStreams.")
+    }
+  }
+
+
+
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 310a75f3a98a3..5a2eb1c01a849 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -763,6 +763,30 @@ class Table(
     * @tparam T The data type that the [[TableSink]] expects.
     */
   def writeToSink[T](sink: TableSink[T]): Unit = {
+
+    def qConfig = this.tableEnv match {
+      case s: StreamTableEnvironment => s.qConf
+      case b: BatchTableEnvironment => new BatchQueryConfig
+      case _ => null
+    }
+
+    writeToSink(sink, qConfig)
+  }
+
+  /**
+    * Writes the [[Table]] to a [[TableSink]]. A [[TableSink]] defines an external storage location.
+    *
+    * A batch [[Table]] can only be written to a
+    * [[org.apache.flink.table.sinks.BatchTableSink]], a streaming [[Table]] requires a
+    * [[org.apache.flink.table.sinks.AppendStreamTableSink]], a
+    * [[org.apache.flink.table.sinks.RetractStreamTableSink]], or an
+    * [[org.apache.flink.table.sinks.UpsertStreamTableSink]].
+    *
+    * @param sink The [[TableSink]] to which the [[Table]] is written.
+    * @param conf The configuration for the query that writes to the sink.
+    * @tparam T The data type that the [[TableSink]] expects.
+    */
+  def writeToSink[T](sink: TableSink[T], conf: QueryConfig): Unit = {
     // get schema information of table
     val rowType = getRelNode.getRowType
     val fieldNames: Array[String] = rowType.getFieldNames.asScala.toArray
@@ -773,7 +797,7 @@ class Table(
     val configuredSink = sink.configure(fieldNames, fieldTypes)
 
     // emit the table to the configured table sink
-    tableEnv.writeToSink(this, configuredSink)
+    tableEnv.writeToSink(this, configuredSink, conf)
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
index ce0f966e9c179..0e377b510aea6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
@@ -25,7 +25,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{RelNode, RelWriter}
 import org.apache.calcite.rex.RexProgram
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.CommonCalc
@@ -83,11 +83,13 @@ class DataStreamCalc(
     estimateRowCount(calcProgram, rowCnt)
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
-    val inputDataStream = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputDataStream = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
     val inputRowType = inputDataStream.getType.asInstanceOf[CRowTypeInfo].rowType
 
     val generator = new CodeGenerator(config, false, inputRowType)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
index 19ad89b387abd..cbd818a5ecce1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
@@ -23,7 +23,7 @@ import org.apache.calcite.rex.{RexCall, RexNode}
 import org.apache.calcite.sql.SemiJoinType
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.functions.utils.TableSqlFunction
 import org.apache.flink.table.plan.nodes.CommonCorrelate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalTableFunctionScan
@@ -82,12 +82,14 @@ class DataStreamCorrelate(
       .itemIf("condition", condition.orNull, condition.isDefined)
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
     // we do not need to specify input type
-    val inputDS = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputDS = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
     val inputType = inputDS.getType.asInstanceOf[CRowTypeInfo]
 
     val funcRel = scan.asInstanceOf[FlinkLogicalTableFunctionScan]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index 506c0cb930c86..f01b24aba5ddf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -21,9 +21,10 @@ import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.core.AggregateCall
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
+import org.apache.flink.api.common.time.Time
 import org.apache.flink.api.java.functions.NullByteKeySelector
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.runtime.aggregate._
 import org.apache.flink.table.plan.nodes.CommonAggregate
@@ -31,6 +32,7 @@ import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
 import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.slf4j.LoggerFactory
 
 /**
   *
@@ -59,6 +61,8 @@ class DataStreamGroupAggregate(
     with CommonAggregate
     with DataStreamRel {
 
+  private val LOG = LoggerFactory.getLogger(this.getClass)
+
   override def deriveRowType() = schema.logicalType
 
   override def needsUpdatesAsRetraction = true
@@ -100,9 +104,18 @@ class DataStreamGroupAggregate(
         inputSchema.logicalType, groupings, getRowType, namedAggregates, Nil))
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
+
+    if (qConfig.getMinIdleStateRetentionTime < 0 || qConfig.getMaxIdleStateRetentionTime < 0) {
+      LOG.warn(
+        "No state retention interval configured for a query which accumulates state. " +
+        "Please provide a query configuration with valid retention interval to prevent excessive " +
+          "state size. You may specify a retention time of 0 to not clean up the state.")
+    }
 
-    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
 
     val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
       new CalcitePair[AggregateCall, String](
@@ -136,6 +149,7 @@ class DataStreamGroupAggregate(
       inputSchema.logicalType,
       inputSchema.physicalFieldTypeInfo,
       groupings,
+      qConfig,
       DataStreamRetractionRules.isAccRetract(this),
       DataStreamRetractionRules.isAccRetract(getInput))
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index ef207b0adba25..d2aaad03e0450 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -26,7 +26,7 @@ import org.apache.flink.api.java.tuple.Tuple
 import org.apache.flink.streaming.api.datastream.{AllWindowedStream, DataStream, KeyedStream, WindowedStream}
 import org.apache.flink.streaming.api.windowing.assigners._
 import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
-import org.apache.flink.table.api.{StreamTableEnvironment, TableException}
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment, TableException}
 import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.expressions.ExpressionUtils._
@@ -107,9 +107,11 @@ class DataStreamGroupWindowAggregate(
           namedProperties))
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
 
-    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
 
     val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
       new CalcitePair[AggregateCall, String](
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 40612421a65fb..8e9788425b07d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -26,7 +26,7 @@ import org.apache.calcite.rel.core.{AggregateCall, Window}
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
 import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.{StreamTableEnvironment, TableException}
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment, TableException}
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.OverAggregate
 import org.apache.flink.table.plan.schema.RowSchema
@@ -88,7 +88,10 @@ class DataStreamOverAggregate(
           namedAggregates))
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
+
     if (logicWindow.groups.size > 1) {
       throw new TableException(
         "Unsupported use of OVER windows. All aggregates must be computed on the same window.")
@@ -109,7 +112,7 @@ class DataStreamOverAggregate(
         "Unsupported use of OVER windows. The window can only be ordered in ASCENDING mode.")
     }
 
-    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
 
     val consumeRetraction = DataStreamRetractionRules.isAccRetract(input)
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
index 9754de4461df8..6f6edf7b5de7f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
@@ -19,7 +19,7 @@
 package org.apache.flink.table.plan.nodes.datastream
 
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.plan.nodes.FlinkRelNode
 import org.apache.flink.table.runtime.types.CRow
 
@@ -29,9 +29,12 @@ trait DataStreamRel extends FlinkRelNode {
     * Translates the FlinkRelNode into a Flink operator.
     *
     * @param tableEnv The [[StreamTableEnvironment]] of the translated Table.
+    * @param qConfig The configuration for the query to generate.
     * @return DataStream of type [[CRow]]
     */
-  def translateToPlan(tableEnv: StreamTableEnvironment) : DataStream[CRow]
+  def translateToPlan(
+    tableEnv: StreamTableEnvironment,
+    qConfig: StreamQueryConfig): DataStream[CRow]
 
   /**
     * Whether the [[DataStreamRel]] requires that update and delete changes are sent with retraction
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
index c613646f9cff2..e64bf0fe1c389 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
@@ -23,7 +23,7 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rel.core.TableScan
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.plan.schema.DataStreamTable
 import org.apache.flink.table.runtime.types.CRow
@@ -54,7 +54,10 @@ class DataStreamScan(
     )
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
+
     val config = tableEnv.getConfig
     val inputDataStream: DataStream[Any] = dataStreamTable.dataStream
     convertToInternalRow(schema, inputDataStream, dataStreamTable, config)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
index 654c259f637c9..6cc739603f43f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
@@ -21,7 +21,7 @@ package org.apache.flink.table.plan.nodes.datastream
 import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
 import org.apache.calcite.rel.{BiRel, RelNode, RelWriter}
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.runtime.types.CRow
 
@@ -58,10 +58,12 @@ class DataStreamUnion(
     s"Union All(union: (${schema.logicalFieldNames.mkString(", ")}))"
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
 
-    val leftDataSet = left.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
-    val rightDataSet = right.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+    val leftDataSet = left.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
+    val rightDataSet = right.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
     leftDataSet.union(rightDataSet)
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
index 32c9aaf0e3d20..ba6b0257a8812 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
@@ -24,7 +24,7 @@ import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.core.Values
 import org.apache.calcite.rex.RexLiteral
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.StreamTableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.runtime.io.CRowValuesInputFormat
@@ -56,7 +56,9 @@ class DataStreamValues(
     )
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
index b2d7019ae0fd6..225f23f172c91 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
@@ -22,7 +22,7 @@ import org.apache.calcite.plan._
 import org.apache.calcite.rel.RelNode
 import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.flink.streaming.api.datastream.DataStream
-import org.apache.flink.table.api.{StreamTableEnvironment, TableEnvironment}
+import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment, TableEnvironment}
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.PhysicalTableSourceScan
 import org.apache.flink.table.plan.schema.RowSchema
@@ -98,7 +98,10 @@ class StreamTableSourceScan(
     )
   }
 
-  override def translateToPlan(tableEnv: StreamTableEnvironment): DataStream[CRow] = {
+  override def translateToPlan(
+      tableEnv: StreamTableEnvironment,
+      qConfig: StreamQueryConfig): DataStream[CRow] = {
+
     val config = tableEnv.getConfig
     val inputDataStream = tableSource.getDataStream(tableEnv.execEnv).asInstanceOf[DataStream[Any]]
     convertToInternalRow(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 768c9cbb482f0..27392c72b8d98 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -33,7 +33,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction, WindowFunction}
 import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
-import org.apache.flink.table.api.TableException
+import org.apache.flink.table.api.{StreamQueryConfig, TableException}
 import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenerator
@@ -155,6 +155,7 @@ object AggregateUtil {
       inputRowType: RelDataType,
       inputFieldTypes: Seq[TypeInformation[_]],
       groupings: Array[Int],
+      qConfig: StreamQueryConfig,
       generateRetraction: Boolean,
       consumeRetraction: Boolean): ProcessFunction[CRow, CRow] = {
 
@@ -190,7 +191,8 @@ object AggregateUtil {
     new GroupAggProcessFunction(
       genFunction,
       aggregationStateType,
-      generateRetraction)
+      generateRetraction,
+      qConfig)
 
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index 6ee37e62f1655..84fee8759f4a8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -26,9 +26,9 @@ import org.apache.flink.util.Collector
 import org.apache.flink.api.common.state.ValueStateDescriptor
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.api.common.state.ValueState
-import org.apache.flink.table.api.Types
+import org.apache.flink.table.api.{StreamQueryConfig, Types}
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 import org.apache.flink.table.runtime.types.CRow
 
 /**
@@ -40,13 +40,20 @@ import org.apache.flink.table.runtime.types.CRow
 class GroupAggProcessFunction(
     private val genAggregations: GeneratedAggregationsFunction,
     private val aggregationStateType: RowTypeInfo,
-    private val generateRetraction: Boolean)
+    private val generateRetraction: Boolean,
+    private val qConfig: StreamQueryConfig)
   extends ProcessFunction[CRow, CRow]
     with Compiler[GeneratedAggregations] {
 
-  val LOG = LoggerFactory.getLogger(this.getClass)
+  val LOG: Logger = LoggerFactory.getLogger(this.getClass)
   private var function: GeneratedAggregations = _
 
+  private val minRetentionTime = qConfig.getMinIdleStateRetentionTime
+  private val maxRetentionTime = qConfig.getMaxIdleStateRetentionTime
+  private val stateCleaningEnabled = minRetentionTime > 1 && maxRetentionTime > 1
+  // interval in which clean-up timers are registered
+  private val cleanupTimerInterval = maxRetentionTime - minRetentionTime
+
   private var newRow: CRow = _
   private var prevRow: CRow = _
   private var firstRow: Boolean = _
@@ -54,6 +61,8 @@ class GroupAggProcessFunction(
   private var state: ValueState[Row] = _
   // counts the number of added and retracted input records
   private var cntState: ValueState[JLong] = _
+  // holds the latest registered cleanup timer
+  private var cleanupTimeState: ValueState[JLong] = _
 
   override def open(config: Configuration) {
     LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
@@ -74,6 +83,12 @@ class GroupAggProcessFunction(
     val inputCntDescriptor: ValueStateDescriptor[JLong] =
       new ValueStateDescriptor[JLong]("GroupAggregateInputCounter", Types.LONG)
     cntState = getRuntimeContext.getState(inputCntDescriptor)
+
+    if (stateCleaningEnabled) {
+      val inputCntDescriptor: ValueStateDescriptor[JLong] =
+        new ValueStateDescriptor[JLong]("GroupAggregateCleanupTime", Types.LONG)
+      cleanupTimeState = getRuntimeContext.getState(inputCntDescriptor)
+    }
   }
 
   override def processElement(
@@ -81,6 +96,23 @@ class GroupAggProcessFunction(
       ctx: ProcessFunction[CRow, CRow]#Context,
       out: Collector[CRow]): Unit = {
 
+    if (stateCleaningEnabled) {
+
+      val currentTime = ctx.timerService().currentProcessingTime()
+      val earliestCleanup = currentTime + minRetentionTime
+
+      // last registered timer
+      val lastCleanupTime = cleanupTimeState.value()
+
+      if (lastCleanupTime == null || earliestCleanup >= lastCleanupTime + cleanupTimerInterval) {
+        // we need to register a new timer
+        val cleanupTime = earliestCleanup + cleanupTimerInterval
+        // register timer and remember clean-up time
+        ctx.timerService().registerProcessingTimeTimer(cleanupTime)
+        cleanupTimeState.update(cleanupTime)
+      }
+    }
+
     val input = inputC.row
 
     // get accumulators and input counter
@@ -144,4 +176,18 @@ class GroupAggProcessFunction(
       cntState.clear()
     }
   }
+
+  override def onTimer(
+      timestamp: Long,
+      ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+      out: Collector[CRow]): Unit = {
+
+    if (timestamp == cleanupTimeState.value()) {
+      // clear all state
+      this.state.clear()
+      this.cntState.clear()
+      this.cleanupTimeState.clear()
+    }
+  }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
index 8626b07410a1d..3d79e2284cdb8 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
@@ -18,16 +18,17 @@
 
 package org.apache.flink.table.utils
 
-import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.tools.RuleSet
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.table.api.{Table, TableConfig, TableEnvironment}
+import org.apache.flink.table.api.{QueryConfig, Table, TableConfig, TableEnvironment}
 import org.apache.flink.table.sinks.TableSink
 import org.apache.flink.table.sources.TableSource
 
 class MockTableEnvironment extends TableEnvironment(new TableConfig) {
 
-  override private[flink] def writeToSink[T](table: Table, sink: TableSink[T]): Unit = ???
+  override private[flink] def writeToSink[T](
+      table: Table,
+      sink: TableSink[T],
+      qConfig: QueryConfig): Unit = ???
 
   override protected def checkValidTableName(name: String): Unit = ???
 

From 38bfc0906a511565942f9a426f3ae15b75e8732d Mon Sep 17 00:00:00 2001
From: sunjincheng121 <sunjincheng121@gmail.com>
Date: Tue, 9 May 2017 14:36:42 +0800
Subject: [PATCH 231/253] [FLINK-6491] [table] Add QueryConfig and state clean
 up for over-windowed aggregates.

This closes #3863.
---
 .../table/api/BatchTableEnvironment.scala     |   8 +-
 .../table/api/StreamTableEnvironment.scala    |  38 +-
 .../api/java/StreamTableEnvironment.scala     |  35 +-
 .../{QueryConfig.scala => queryConfig.scala}  |  10 +-
 .../api/scala/StreamTableEnvironment.scala    |  20 +-
 .../table/api/scala/TableConversions.scala    |  13 +-
 .../org/apache/flink/table/api/table.scala    |   6 +-
 .../nodes/datastream/DataStreamCalc.scala     |   5 +-
 .../datastream/DataStreamCorrelate.scala      |   4 +-
 .../datastream/DataStreamGroupAggregate.scala |  10 +-
 .../DataStreamGroupWindowAggregate.scala      |   4 +-
 .../datastream/DataStreamOverAggregate.scala  |  34 +-
 .../plan/nodes/datastream/DataStreamRel.scala |   4 +-
 .../nodes/datastream/DataStreamScan.scala     |   2 +-
 .../nodes/datastream/DataStreamUnion.scala    |   6 +-
 .../nodes/datastream/DataStreamValues.scala   |   2 +-
 .../datastream/StreamTableSourceScan.scala    |   2 +-
 .../runtime/aggregate/AggregateUtil.scala     |  32 +-
 .../aggregate/GroupAggProcessFunction.scala   |  44 +-
 .../aggregate/ProcTimeBoundedRangeOver.scala  |  26 +-
 .../aggregate/ProcTimeBoundedRowsOver.scala   |  20 +-
 .../ProcTimeUnboundedNonPartitionedOver.scala |  20 +-
 .../ProcTimeUnboundedPartitionedOver.scala    |  20 +-
 .../ProcessFunctionWithCleanupState.scala     |  85 ++++
 .../aggregate/RowTimeBoundedRangeOver.scala   |  44 +-
 .../aggregate/RowTimeBoundedRowsOver.scala    |  41 +-
 .../aggregate/RowTimeUnboundedOver.scala      |  44 +-
 .../table/GroupAggregationsITCase.scala       |  13 +-
 ...ocessingOverRangeProcessFunctionTest.scala | 336 -------------
 .../runtime/harness/HarnessTestBase.scala     | 281 ++++++++++-
 .../harness/NonWindowHarnessTest.scala        | 157 ++++++
 .../harness/OverWindowHarnessTest.scala       | 458 ++++++++++--------
 .../table/utils/MockTableEnvironment.scala    |   2 +-
 33 files changed, 1112 insertions(+), 714 deletions(-)
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/{QueryConfig.scala => queryConfig.scala} (92%)
 create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
 delete mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index f33c187fa04b1..3c0f51b719b9b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -113,17 +113,17 @@ abstract class BatchTableEnvironment(
     *
     * @param table The [[Table]] to write.
     * @param sink The [[TableSink]] to write the [[Table]] to.
-    * @param qConfig The configuration for the query to generate.
+    * @param queryConfig The configuration for the query to generate.
     * @tparam T The expected type of the [[DataSet]] which represents the [[Table]].
     */
   override private[flink] def writeToSink[T](
       table: Table,
       sink: TableSink[T],
-      qConfig: QueryConfig): Unit = {
+      queryConfig: QueryConfig): Unit = {
 
     // We do not pass the configuration on, because there is nothing to configure for batch queries.
-    val bQConfig = qConfig match {
-      case batchConfig: BatchQueryConfig => batchConfig
+    queryConfig match {
+      case _: BatchQueryConfig =>
       case _ =>
         throw new TableException("BatchQueryConfig required to configure batch query.")
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index c594d4c300a71..d68da04799717 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -81,7 +81,7 @@ abstract class StreamTableEnvironment(
   // the naming pattern for internally registered tables.
   private val internalNamePattern = "^_DataStreamTable_[0-9]+$".r
 
-  def qConf: StreamQueryConfig = new StreamQueryConfig
+  def queryConfig: StreamQueryConfig = new StreamQueryConfig
 
   /**
     * Checks if the chosen table name is valid.
@@ -128,16 +128,16 @@ abstract class StreamTableEnvironment(
     *
     * @param table The [[Table]] to write.
     * @param sink The [[TableSink]] to write the [[Table]] to.
-    * @param qConfig The configuration for the query to generate.
+    * @param queryConfig The configuration for the query to generate.
     * @tparam T The expected type of the [[DataStream]] which represents the [[Table]].
     */
   override private[flink] def writeToSink[T](
       table: Table,
       sink: TableSink[T],
-      qConfig: QueryConfig): Unit = {
+      queryConfig: QueryConfig): Unit = {
 
     // Check query configuration
-    val sQConf = qConfig match {
+    val streamQueryConfig = queryConfig match {
       case streamConfig: StreamQueryConfig => streamConfig
       case _ =>
         throw new TableException("StreamQueryConfig required to configure stream query.")
@@ -150,7 +150,11 @@ abstract class StreamTableEnvironment(
         val outputType = sink.getOutputType
         // translate the Table into a DataStream and provide the type that the TableSink expects.
         val result: DataStream[T] =
-          translate(table, sQConf, updatesAsRetraction = true, withChangeFlag = true)(outputType)
+          translate(
+            table,
+            streamQueryConfig,
+            updatesAsRetraction = true,
+            withChangeFlag = true)(outputType)
         // Give the DataStream to the TableSink to emit it.
         retractSink.asInstanceOf[RetractStreamTableSink[Any]]
           .emitDataStream(result.asInstanceOf[DataStream[JTuple2[JBool, Any]]])
@@ -176,7 +180,7 @@ abstract class StreamTableEnvironment(
           translate(
             optimizedPlan,
             table.getRelNode.getRowType,
-            sQConf,
+            streamQueryConfig,
             withChangeFlag = true)(outputType)
         // Give the DataStream to the TableSink to emit it.
         upsertSink.asInstanceOf[UpsertStreamTableSink[Any]]
@@ -196,7 +200,7 @@ abstract class StreamTableEnvironment(
           translate(
             optimizedPlan,
             table.getRelNode.getRowType,
-            sQConf,
+            streamQueryConfig,
             withChangeFlag = false)(outputType)
         // Give the DataStream to the TableSink to emit it.
         appendSink.asInstanceOf[AppendStreamTableSink[T]].emitDataStream(result)
@@ -566,7 +570,7 @@ abstract class StreamTableEnvironment(
     * Table API calls and / or SQL queries and generating corresponding [[DataStream]] operators.
     *
     * @param table The root node of the relational expression tree.
-    * @param qConfig The configuration for the query to generate.
+    * @param queryConfig The configuration for the query to generate.
     * @param updatesAsRetraction Set to true to encode updates as retraction messages.
     * @param withChangeFlag Set to true to emit records with change flags.
     * @param tpe The [[TypeInformation]] of the resulting [[DataStream]].
@@ -575,12 +579,12 @@ abstract class StreamTableEnvironment(
     */
   protected def translate[A](
       table: Table,
-      qConfig: StreamQueryConfig,
+      queryConfig: StreamQueryConfig,
       updatesAsRetraction: Boolean,
       withChangeFlag: Boolean)(implicit tpe: TypeInformation[A]): DataStream[A] = {
     val relNode = table.getRelNode
     val dataStreamPlan = optimize(relNode, updatesAsRetraction)
-    translate(dataStreamPlan, relNode.getRowType, qConfig, withChangeFlag)
+    translate(dataStreamPlan, relNode.getRowType, queryConfig, withChangeFlag)
   }
 
   /**
@@ -589,7 +593,7 @@ abstract class StreamTableEnvironment(
     * @param logicalPlan The root node of the relational expression tree.
     * @param logicalType The row type of the result. Since the logicalPlan can lose the
     *                    field naming during optimization we pass the row type separately.
-    * @param qConfig     The configuration for the query to generate.
+    * @param queryConfig     The configuration for the query to generate.
     * @param withChangeFlag Set to true to emit records with change flags.
     * @param tpe         The [[TypeInformation]] of the resulting [[DataStream]].
     * @tparam A The type of the resulting [[DataStream]].
@@ -598,7 +602,7 @@ abstract class StreamTableEnvironment(
   protected def translate[A](
       logicalPlan: RelNode,
       logicalType: RelDataType,
-      qConfig: StreamQueryConfig,
+      queryConfig: StreamQueryConfig,
       withChangeFlag: Boolean)
       (implicit tpe: TypeInformation[A]): DataStream[A] = {
 
@@ -610,7 +614,7 @@ abstract class StreamTableEnvironment(
     }
 
     // get CRow plan
-    val plan: DataStream[CRow] = translateToCRow(logicalPlan, qConfig)
+    val plan: DataStream[CRow] = translateToCRow(logicalPlan, queryConfig)
 
     // convert CRow to output type
     val conversion = if (withChangeFlag) {
@@ -642,16 +646,16 @@ abstract class StreamTableEnvironment(
     * Translates a logical [[RelNode]] plan into a [[DataStream]] of type [[CRow]].
     *
     * @param logicalPlan The logical plan to translate.
-    * @param qConfig The configuration for the query to generate.
+    * @param queryConfig  The configuration for the query to generate.
     * @return The [[DataStream]] of type [[CRow]].
     */
   protected def translateToCRow(
     logicalPlan: RelNode,
-    qConfig: StreamQueryConfig): DataStream[CRow] = {
+    queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
     logicalPlan match {
       case node: DataStreamRel =>
-        node.translateToPlan(this, qConfig)
+        node.translateToPlan(this, queryConfig)
       case _ =>
         throw TableException("Cannot generate DataStream due to an invalid logical plan. " +
           "This is a bug and should not happen. Please file an issue.")
@@ -667,7 +671,7 @@ abstract class StreamTableEnvironment(
   def explain(table: Table): String = {
     val ast = table.getRelNode
     val optimizedPlan = optimize(ast, updatesAsRetraction = false)
-    val dataStream = translateToCRow(optimizedPlan, qConf)
+    val dataStream = translateToCRow(optimizedPlan, queryConfig)
 
     val env = dataStream.getExecutionEnvironment
     val jsonSqlPlan = env.getExecutionPlan
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
index c3b59511d2954..311986c603012 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/StreamTableEnvironment.scala
@@ -150,7 +150,7 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T](table: Table, clazz: Class[T]): DataStream[T] = {
-    toDataStream(table, clazz, qConf)
+    toDataStream(table, clazz, queryConfig)
   }
 
   /**
@@ -170,7 +170,7 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T](table: Table, typeInfo: TypeInformation[T]): DataStream[T] = {
-    toDataStream(table, typeInfo, qConf)
+    toDataStream(table, typeInfo, queryConfig)
   }
 
   /**
@@ -186,14 +186,17 @@ class StreamTableEnvironment(
     *
     * @param table The [[Table]] to convert.
     * @param clazz The class of the type of the resulting [[DataStream]].
-    * @param qConfig The configuration of the query to generate.
+    * @param queryConfig The configuration of the query to generate.
     * @tparam T The type of the resulting [[DataStream]].
     * @return The converted [[DataStream]].
     */
-  def toDataStream[T](table: Table, clazz: Class[T], qConfig: StreamQueryConfig): DataStream[T] = {
+  def toDataStream[T](
+      table: Table,
+      clazz: Class[T],
+      queryConfig: StreamQueryConfig): DataStream[T] = {
     val typeInfo = TypeExtractor.createTypeInfo(clazz)
     TableEnvironment.validateType(typeInfo)
-    translate[T](table, qConfig, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
+    translate[T](table, queryConfig, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
   }
 
   /**
@@ -209,16 +212,16 @@ class StreamTableEnvironment(
     *
     * @param table The [[Table]] to convert.
     * @param typeInfo The [[TypeInformation]] that specifies the type of the [[DataStream]].
-    * @param qConfig The configuration of the query to generate.
+    * @param queryConfig The configuration of the query to generate.
     * @tparam T The type of the resulting [[DataStream]].
     * @return The converted [[DataStream]].
     */
   def toDataStream[T](
       table: Table,
       typeInfo: TypeInformation[T],
-      qConfig: StreamQueryConfig): DataStream[T] = {
+      queryConfig: StreamQueryConfig): DataStream[T] = {
     TableEnvironment.validateType(typeInfo)
-    translate[T](table, qConfig, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
+    translate[T](table, queryConfig, updatesAsRetraction = false, withChangeFlag = false)(typeInfo)
   }
 
   /**
@@ -242,7 +245,7 @@ class StreamTableEnvironment(
       table: Table,
       clazz: Class[T]): DataStream[JTuple2[JBool, T]] = {
 
-    toRetractStream(table, clazz, qConf)
+    toRetractStream(table, clazz, queryConfig)
   }
 
   /**
@@ -266,7 +269,7 @@ class StreamTableEnvironment(
       table: Table,
       typeInfo: TypeInformation[T]): DataStream[JTuple2[JBool, T]] = {
 
-    toRetractStream(table, typeInfo, qConf)
+    toRetractStream(table, typeInfo, queryConfig)
   }
 
   /**
@@ -283,21 +286,21 @@ class StreamTableEnvironment(
     *
     * @param table The [[Table]] to convert.
     * @param clazz The class of the requested record type.
-    * @param qConfig The configuration of the query to generate.
+    * @param queryConfig The configuration of the query to generate.
     * @tparam T The type of the requested record type.
     * @return The converted [[DataStream]].
     */
   def toRetractStream[T](
       table: Table,
       clazz: Class[T],
-      qConfig: StreamQueryConfig): DataStream[JTuple2[JBool, T]] = {
+      queryConfig: StreamQueryConfig): DataStream[JTuple2[JBool, T]] = {
 
     val typeInfo = TypeExtractor.createTypeInfo(clazz)
     TableEnvironment.validateType(typeInfo)
     val resultType = new TupleTypeInfo[JTuple2[JBool, T]](Types.BOOLEAN, typeInfo)
     translate[JTuple2[JBool, T]](
       table,
-      qConfig,
+      queryConfig,
       updatesAsRetraction = true,
       withChangeFlag = true)(resultType)
   }
@@ -316,14 +319,14 @@ class StreamTableEnvironment(
     *
     * @param table The [[Table]] to convert.
     * @param typeInfo The [[TypeInformation]] of the requested record type.
-    * @param qConfig The configuration of the query to generate.
+    * @param queryConfig The configuration of the query to generate.
     * @tparam T The type of the requested record type.
     * @return The converted [[DataStream]].
     */
   def toRetractStream[T](
       table: Table,
       typeInfo: TypeInformation[T],
-      qConfig: StreamQueryConfig): DataStream[JTuple2[JBool, T]] = {
+      queryConfig: StreamQueryConfig): DataStream[JTuple2[JBool, T]] = {
 
     TableEnvironment.validateType(typeInfo)
     val resultTypeInfo = new TupleTypeInfo[JTuple2[JBool, T]](
@@ -332,7 +335,7 @@ class StreamTableEnvironment(
     )
     translate[JTuple2[JBool, T]](
       table,
-      qConfig,
+      queryConfig,
       updatesAsRetraction = true,
       withChangeFlag = true)(resultTypeInfo)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/QueryConfig.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/queryConfig.scala
similarity index 92%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/QueryConfig.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/queryConfig.scala
index 8e8b5ac73c94e..c8fbab7e05ddb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/QueryConfig.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/queryConfig.scala
@@ -31,7 +31,7 @@ class BatchQueryConfig private[table] extends QueryConfig
 /**
   * The [[StreamQueryConfig]] holds parameters to configure the behavior of streaming queries.
   *
-  * An empty [[StreamQueryConfig]] can be generated using the [[StreamTableEnvironment.qConf]]
+  * An empty [[StreamQueryConfig]] can be generated using the [[StreamTableEnvironment.queryConfig]]
   * method.
   */
 class StreamQueryConfig private[table] extends QueryConfig {
@@ -56,15 +56,15 @@ class StreamQueryConfig private[table] extends QueryConfig {
     * When new data arrives for previously cleaned-up state, the new data will be handled as if it
     * was the first data. This can result in previous results being overwritten.
     *
-    * Note: [[setIdleStateRetentionTime(minTime: Time, maxTime: Time)]] allows to set a minimum and
+    * Note: [[withIdleStateRetentionTime(minTime: Time, maxTime: Time)]] allows to set a minimum and
     * maximum time for state to be retained. This method is more efficient, because the system has
     * to do less bookkeeping to identify the time at which state must be cleared.
     *
     * @param time The time interval for how long idle state is retained. Set to 0 (zero) to never
     *             clean-up the state.
     */
-  def setIdleStateRetentionTime(time: Time): StreamQueryConfig = {
-    setIdleStateRetentionTime(time, time)
+  def withIdleStateRetentionTime(time: Time): StreamQueryConfig = {
+    withIdleStateRetentionTime(time, time)
   }
 
   /**
@@ -83,7 +83,7 @@ class StreamQueryConfig private[table] extends QueryConfig {
     * @param maxTime The maximum time interval for which idle state is retained. May not be smaller
     *                than than minTime. Set to 0 (zero) to never clean-up the state.
     */
-  def setIdleStateRetentionTime(minTime: Time, maxTime: Time): StreamQueryConfig = {
+  def withIdleStateRetentionTime(minTime: Time, maxTime: Time): StreamQueryConfig = {
     if (maxTime.toMilliseconds < minTime.toMilliseconds) {
       throw new IllegalArgumentException("maxTime may not be smaller than minTime.")
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
index 56f7d55744cf9..8c6b27397e770 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala
@@ -143,7 +143,7 @@ class StreamTableEnvironment(
     * @return The converted [[DataStream]].
     */
   def toDataStream[T: TypeInformation](table: Table): DataStream[T] = {
-    toDataStream(table, qConf)
+    toDataStream(table, queryConfig)
   }
 
   /**
@@ -158,14 +158,16 @@ class StreamTableEnvironment(
     * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
     *
     * @param table The [[Table]] to convert.
-    * @param qConfig The configuration of the query to generate.
+    * @param queryConfig The configuration of the query to generate.
     * @tparam T The type of the resulting [[DataStream]].
     * @return The converted [[DataStream]].
     */
-  def toDataStream[T: TypeInformation](table: Table, qConfig: StreamQueryConfig): DataStream[T] = {
+  def toDataStream[T: TypeInformation](
+    table: Table,
+    queryConfig: StreamQueryConfig): DataStream[T] = {
     val returnType = createTypeInformation[T]
-    asScalaStream(
-      translate(table, qConfig, updatesAsRetraction = false, withChangeFlag = false)(returnType))
+    asScalaStream(translate(
+      table, queryConfig, updatesAsRetraction = false, withChangeFlag = false)(returnType))
   }
 
 /**
@@ -180,7 +182,7 @@ class StreamTableEnvironment(
   * @return The converted [[DataStream]].
   */
   def toRetractStream[T: TypeInformation](table: Table): DataStream[(Boolean, T)] = {
-    toRetractStream(table, qConf)
+    toRetractStream(table, queryConfig)
   }
 
   /**
@@ -191,16 +193,16 @@ class StreamTableEnvironment(
     * A true [[Boolean]] flag indicates an add message, a false flag indicates a retract message.
     *
     * @param table The [[Table]] to convert.
-    * @param qConfig The configuration of the query to generate.
+    * @param queryConfig The configuration of the query to generate.
     * @tparam T The type of the requested data type.
     * @return The converted [[DataStream]].
     */
   def toRetractStream[T: TypeInformation](
       table: Table,
-      qConfig: StreamQueryConfig): DataStream[(Boolean, T)] = {
+      queryConfig: StreamQueryConfig): DataStream[(Boolean, T)] = {
     val returnType = createTypeInformation[(Boolean, T)]
     asScalaStream(
-      translate(table, qConfig, updatesAsRetraction = true, withChangeFlag = true)(returnType))
+      translate(table, queryConfig, updatesAsRetraction = true, withChangeFlag = true)(returnType))
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
index 966b42f19756d..9874a9ea52e79 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala
@@ -59,12 +59,12 @@ class TableConversions(table: Table) {
 
   /** Converts the [[Table]] to a [[DataStream]] of the specified type.
     *
-    * @param qConfig The configuration for the generated query.
+    * @param queryConfig The configuration for the generated query.
     */
-  def toDataStream[T: TypeInformation](qConfig: StreamQueryConfig): DataStream[T] = {
+  def toDataStream[T: TypeInformation](queryConfig: StreamQueryConfig): DataStream[T] = {
     table.tableEnv match {
       case tEnv: ScalaStreamTableEnv =>
-        tEnv.toDataStream(table, qConfig)
+        tEnv.toDataStream(table, queryConfig)
       case _ =>
         throw new TableException(
           "Only tables that originate from Scala DataStreams " +
@@ -97,14 +97,15 @@ class TableConversions(table: Table) {
     *
     * A true [[Boolean]] flag indicates an add message, a false flag indicates a retract message.
     *
-    * @param qConfig The configuration for the generated query.
+    * @param queryConfig The configuration for the generated query.
     *
     */
-  def toRetractStream[T: TypeInformation](qConfig: StreamQueryConfig): DataStream[(Boolean, T)] = {
+  def toRetractStream[T: TypeInformation](
+      queryConfig: StreamQueryConfig): DataStream[(Boolean, T)] = {
 
     table.tableEnv match {
       case tEnv: ScalaStreamTableEnv =>
-        tEnv.toRetractStream(table, qConfig)
+        tEnv.toRetractStream(table, queryConfig)
       case _ =>
         throw new TableException(
           "Only tables that originate from Scala DataStreams " +
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 5a2eb1c01a849..ca61c651f2b09 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -764,13 +764,13 @@ class Table(
     */
   def writeToSink[T](sink: TableSink[T]): Unit = {
 
-    def qConfig = this.tableEnv match {
-      case s: StreamTableEnvironment => s.qConf
+    def queryConfig = this.tableEnv match {
+      case s: StreamTableEnvironment => s.queryConfig
       case b: BatchTableEnvironment => new BatchQueryConfig
       case _ => null
     }
 
-    writeToSink(sink, qConfig)
+    writeToSink(sink, queryConfig)
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
index 0e377b510aea6..5f270f6986c32 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
@@ -85,11 +85,12 @@ class DataStreamCalc(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
-    val inputDataStream = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
+    val inputDataStream =
+      getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
     val inputRowType = inputDataStream.getType.asInstanceOf[CRowTypeInfo].rowType
 
     val generator = new CodeGenerator(config, false, inputRowType)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
index cbd818a5ecce1..5b32b1053a292 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
@@ -84,12 +84,12 @@ class DataStreamCorrelate(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
     // we do not need to specify input type
-    val inputDS = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
+    val inputDS = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
     val inputType = inputDS.getType.asInstanceOf[CRowTypeInfo]
 
     val funcRel = scan.asInstanceOf[FlinkLogicalTableFunctionScan]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index f01b24aba5ddf..e5d80889ec703 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -106,16 +106,16 @@ class DataStreamGroupAggregate(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
-    if (qConfig.getMinIdleStateRetentionTime < 0 || qConfig.getMaxIdleStateRetentionTime < 0) {
+    if (groupings.length > 0 && queryConfig.getMinIdleStateRetentionTime < 0) {
       LOG.warn(
         "No state retention interval configured for a query which accumulates state. " +
         "Please provide a query configuration with valid retention interval to prevent excessive " +
-          "state size. You may specify a retention time of 0 to not clean up the state.")
+        "state size. You may specify a retention time of 0 to not clean up the state.")
     }
 
-    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
+    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
 
     val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
       new CalcitePair[AggregateCall, String](
@@ -149,7 +149,7 @@ class DataStreamGroupAggregate(
       inputSchema.logicalType,
       inputSchema.physicalFieldTypeInfo,
       groupings,
-      qConfig,
+      queryConfig,
       DataStreamRetractionRules.isAccRetract(this),
       DataStreamRetractionRules.isAccRetract(getInput))
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index d2aaad03e0450..2a71592e3fa98 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -109,9 +109,9 @@ class DataStreamGroupWindowAggregate(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
-    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
+    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
 
     val physicalNamedAggregates = namedAggregates.map { namedAggregate =>
       new CalcitePair[AggregateCall, String](
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 8e9788425b07d..a9fbf02576a8b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -36,6 +36,7 @@ import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
 import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.slf4j.LoggerFactory
 
 class DataStreamOverAggregate(
     logicWindow: Window,
@@ -47,6 +48,7 @@ class DataStreamOverAggregate(
   extends SingleRel(cluster, traitSet, inputNode)
   with OverAggregate
   with DataStreamRel {
+  private val LOG = LoggerFactory.getLogger(this.getClass)
 
   override def deriveRowType(): RelDataType = schema.logicalType
 
@@ -90,7 +92,7 @@ class DataStreamOverAggregate(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
     if (logicWindow.groups.size > 1) {
       throw new TableException(
@@ -112,10 +114,23 @@ class DataStreamOverAggregate(
         "Unsupported use of OVER windows. The window can only be ordered in ASCENDING mode.")
     }
 
-    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
+    val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
 
     val consumeRetraction = DataStreamRetractionRules.isAccRetract(input)
 
+    if (consumeRetraction) {
+      throw new TableException(
+        "Retraction on Over window aggregation is not supported yet. " +
+        "Note: Over window aggregation should not follow a non-windowed GroupBy aggregation.")
+    }
+
+    if (!logicWindow.groups.get(0).keys.isEmpty && queryConfig.getMinIdleStateRetentionTime < 0) {
+      LOG.warn(
+        "No state retention interval configured for a query which accumulates state. " +
+        "Please provide a query configuration with valid retention interval to prevent " +
+        "excessive state size. You may specify a retention time of 0 to not clean up the state.")
+    }
+
     val generator = new CodeGenerator(
       tableEnv.getConfig,
       false,
@@ -126,18 +141,13 @@ class DataStreamOverAggregate(
       .get(orderKey.getFieldIndex)
       .getType
 
-    if (consumeRetraction) {
-      throw new TableException(
-        "Retraction on Over window aggregation is not supported yet. " +
-          "Note: Over window aggregation should not follow a non-windowed GroupBy aggregation.")
-    }
-
     timeType match {
       case _ if FlinkTypeFactory.isProctimeIndicatorType(timeType)  =>
         // proc-time OVER window
         if (overWindow.lowerBound.isUnbounded && overWindow.upperBound.isCurrentRow) {
           // unbounded OVER window
           createUnboundedAndCurrentRowOverWindow(
+            queryConfig,
             generator,
             inputDS,
             isRowTimeType = false,
@@ -145,8 +155,10 @@ class DataStreamOverAggregate(
         } else if (
           overWindow.lowerBound.isPreceding && !overWindow.lowerBound.isUnbounded &&
             overWindow.upperBound.isCurrentRow) {
+
           // bounded OVER window
           createBoundedAndCurrentRowOverWindow(
+            queryConfig,
             generator,
             inputDS,
             isRowTimeType = false,
@@ -162,6 +174,7 @@ class DataStreamOverAggregate(
           overWindow.lowerBound.isUnbounded && overWindow.upperBound.isCurrentRow) {
           // unbounded OVER window
           createUnboundedAndCurrentRowOverWindow(
+            queryConfig,
             generator,
             inputDS,
             isRowTimeType = true,
@@ -169,6 +182,7 @@ class DataStreamOverAggregate(
         } else if (overWindow.lowerBound.isPreceding && overWindow.upperBound.isCurrentRow) {
           // bounded OVER window
           createBoundedAndCurrentRowOverWindow(
+            queryConfig,
             generator,
             inputDS,
             isRowTimeType = true,
@@ -185,6 +199,7 @@ class DataStreamOverAggregate(
   }
 
   def createUnboundedAndCurrentRowOverWindow(
+    queryConfig: StreamQueryConfig,
     generator: CodeGenerator,
     inputDS: DataStream[CRow],
     isRowTimeType: Boolean,
@@ -210,6 +225,7 @@ class DataStreamOverAggregate(
       inputSchema.physicalType,
       inputSchema.physicalTypeInfo,
       inputSchema.physicalFieldTypeInfo,
+      queryConfig,
       isRowTimeType,
       partitionKeys.nonEmpty,
       isRowsClause)
@@ -242,6 +258,7 @@ class DataStreamOverAggregate(
   }
 
   def createBoundedAndCurrentRowOverWindow(
+    queryConfig: StreamQueryConfig,
     generator: CodeGenerator,
     inputDS: DataStream[CRow],
     isRowTimeType: Boolean,
@@ -269,6 +286,7 @@ class DataStreamOverAggregate(
       inputSchema.physicalTypeInfo,
       inputSchema.physicalFieldTypeInfo,
       precedingOffset,
+      queryConfig,
       isRowsClause,
       isRowTimeType
     )
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
index 6f6edf7b5de7f..65d336fcdf582 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamRel.scala
@@ -29,12 +29,12 @@ trait DataStreamRel extends FlinkRelNode {
     * Translates the FlinkRelNode into a Flink operator.
     *
     * @param tableEnv The [[StreamTableEnvironment]] of the translated Table.
-    * @param qConfig The configuration for the query to generate.
+    * @param queryConfig The configuration for the query to generate.
     * @return DataStream of type [[CRow]]
     */
   def translateToPlan(
     tableEnv: StreamTableEnvironment,
-    qConfig: StreamQueryConfig): DataStream[CRow]
+    queryConfig: StreamQueryConfig): DataStream[CRow]
 
   /**
     * Whether the [[DataStreamRel]] requires that update and delete changes are sent with retraction
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
index e64bf0fe1c389..424c6a26633e5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamScan.scala
@@ -56,7 +56,7 @@ class DataStreamScan(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
     val inputDataStream: DataStream[Any] = dataStreamTable.dataStream
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
index 6cc739603f43f..6f4980aa70dab 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamUnion.scala
@@ -60,10 +60,10 @@ class DataStreamUnion(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
-    val leftDataSet = left.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
-    val rightDataSet = right.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, qConfig)
+    val leftDataSet = left.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
+    val rightDataSet = right.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
     leftDataSet.union(rightDataSet)
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
index ba6b0257a8812..d7c490f40055b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamValues.scala
@@ -58,7 +58,7 @@ class DataStreamValues(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
index 225f23f172c91..51e609f4fe3be 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
@@ -100,7 +100,7 @@ class StreamTableSourceScan(
 
   override def translateToPlan(
       tableEnv: StreamTableEnvironment,
-      qConfig: StreamQueryConfig): DataStream[CRow] = {
+      queryConfig: StreamQueryConfig): DataStream[CRow] = {
 
     val config = tableEnv.getConfig
     val inputDataStream = tableSource.getDataStream(tableEnv.execEnv).asInstanceOf[DataStream[Any]]
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 27392c72b8d98..8073959ae0bd5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -77,6 +77,7 @@ object AggregateUtil {
       inputType: RelDataType,
       inputTypeInfo: TypeInformation[Row],
       inputFieldTypeInfo: Seq[TypeInformation[_]],
+      queryConfig: StreamQueryConfig,
       isRowTimeType: Boolean,
       isPartitioned: Boolean,
       isRowsClause: Boolean)
@@ -117,23 +118,27 @@ object AggregateUtil {
         new RowTimeUnboundedRowsOver(
           genFunction,
           aggregationStateType,
-          CRowTypeInfo(inputTypeInfo))
+          CRowTypeInfo(inputTypeInfo),
+          queryConfig)
       } else {
         // RANGE unbounded over process function
         new RowTimeUnboundedRangeOver(
           genFunction,
           aggregationStateType,
-          CRowTypeInfo(inputTypeInfo))
+          CRowTypeInfo(inputTypeInfo),
+          queryConfig)
       }
     } else {
       if (isPartitioned) {
         new ProcTimeUnboundedPartitionedOver(
           genFunction,
-          aggregationStateType)
+          aggregationStateType,
+          queryConfig)
       } else {
         new ProcTimeUnboundedNonPartitionedOver(
           genFunction,
-          aggregationStateType)
+          aggregationStateType,
+          queryConfig)
       }
     }
   }
@@ -155,7 +160,7 @@ object AggregateUtil {
       inputRowType: RelDataType,
       inputFieldTypes: Seq[TypeInformation[_]],
       groupings: Array[Int],
-      qConfig: StreamQueryConfig,
+      queryConfig: StreamQueryConfig,
       generateRetraction: Boolean,
       consumeRetraction: Boolean): ProcessFunction[CRow, CRow] = {
 
@@ -192,7 +197,7 @@ object AggregateUtil {
       genFunction,
       aggregationStateType,
       generateRetraction,
-      qConfig)
+      queryConfig)
 
   }
 
@@ -217,6 +222,7 @@ object AggregateUtil {
       inputTypeInfo: TypeInformation[Row],
       inputFieldTypeInfo: Seq[TypeInformation[_]],
       precedingOffset: Long,
+      queryConfig: StreamQueryConfig,
       isRowsClause: Boolean,
       isRowTimeType: Boolean)
     : ProcessFunction[CRow, CRow] = {
@@ -258,15 +264,15 @@ object AggregateUtil {
           genFunction,
           aggregationStateType,
           inputRowType,
-          precedingOffset
-        )
+          precedingOffset,
+          queryConfig)
       } else {
         new RowTimeBoundedRangeOver(
           genFunction,
           aggregationStateType,
           inputRowType,
-          precedingOffset
-        )
+          precedingOffset,
+          queryConfig)
       }
     } else {
       if (isRowsClause) {
@@ -274,13 +280,15 @@ object AggregateUtil {
           genFunction,
           precedingOffset,
           aggregationStateType,
-          inputRowType)
+          inputRowType,
+          queryConfig)
       } else {
         new ProcTimeBoundedRangeOver(
           genFunction,
           precedingOffset,
           aggregationStateType,
-          inputRowType)
+          inputRowType,
+          queryConfig)
       }
     }
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index 84fee8759f4a8..57ea86e70cbef 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -41,19 +41,13 @@ class GroupAggProcessFunction(
     private val genAggregations: GeneratedAggregationsFunction,
     private val aggregationStateType: RowTypeInfo,
     private val generateRetraction: Boolean,
-    private val qConfig: StreamQueryConfig)
-  extends ProcessFunction[CRow, CRow]
+    private val queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with Compiler[GeneratedAggregations] {
 
   val LOG: Logger = LoggerFactory.getLogger(this.getClass)
   private var function: GeneratedAggregations = _
 
-  private val minRetentionTime = qConfig.getMinIdleStateRetentionTime
-  private val maxRetentionTime = qConfig.getMaxIdleStateRetentionTime
-  private val stateCleaningEnabled = minRetentionTime > 1 && maxRetentionTime > 1
-  // interval in which clean-up timers are registered
-  private val cleanupTimerInterval = maxRetentionTime - minRetentionTime
-
   private var newRow: CRow = _
   private var prevRow: CRow = _
   private var firstRow: Boolean = _
@@ -61,8 +55,6 @@ class GroupAggProcessFunction(
   private var state: ValueState[Row] = _
   // counts the number of added and retracted input records
   private var cntState: ValueState[JLong] = _
-  // holds the latest registered cleanup timer
-  private var cleanupTimeState: ValueState[JLong] = _
 
   override def open(config: Configuration) {
     LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
@@ -84,11 +76,7 @@ class GroupAggProcessFunction(
       new ValueStateDescriptor[JLong]("GroupAggregateInputCounter", Types.LONG)
     cntState = getRuntimeContext.getState(inputCntDescriptor)
 
-    if (stateCleaningEnabled) {
-      val inputCntDescriptor: ValueStateDescriptor[JLong] =
-        new ValueStateDescriptor[JLong]("GroupAggregateCleanupTime", Types.LONG)
-      cleanupTimeState = getRuntimeContext.getState(inputCntDescriptor)
-    }
+    initCleanupTimeState("GroupAggregateCleanupTime")
   }
 
   override def processElement(
@@ -96,22 +84,9 @@ class GroupAggProcessFunction(
       ctx: ProcessFunction[CRow, CRow]#Context,
       out: Collector[CRow]): Unit = {
 
-    if (stateCleaningEnabled) {
-
-      val currentTime = ctx.timerService().currentProcessingTime()
-      val earliestCleanup = currentTime + minRetentionTime
-
-      // last registered timer
-      val lastCleanupTime = cleanupTimeState.value()
-
-      if (lastCleanupTime == null || earliestCleanup >= lastCleanupTime + cleanupTimerInterval) {
-        // we need to register a new timer
-        val cleanupTime = earliestCleanup + cleanupTimerInterval
-        // register timer and remember clean-up time
-        ctx.timerService().registerProcessingTimeTimer(cleanupTime)
-        cleanupTimeState.update(cleanupTime)
-      }
-    }
+    val currentTime = ctx.timerService().currentProcessingTime()
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, currentTime)
 
     val input = inputC.row
 
@@ -182,11 +157,8 @@ class GroupAggProcessFunction(
       ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
       out: Collector[CRow]): Unit = {
 
-    if (timestamp == cleanupTimeState.value()) {
-      // clear all state
-      this.state.clear()
-      this.cntState.clear()
-      this.cleanupTimeState.clear()
+    if (needToCleanupState(timestamp)) {
+      cleanupState(state, cntState)
     }
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index 3fb506fd1333c..d50912ccf9021 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -31,6 +31,7 @@ import org.apache.flink.api.java.typeutils.ListTypeInfo
 import java.util.{ArrayList, List => JList}
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.slf4j.LoggerFactory
@@ -48,9 +49,11 @@ class ProcTimeBoundedRangeOver(
     genAggregations: GeneratedAggregationsFunction,
     precedingTimeBoundary: Long,
     aggregatesTypeInfo: RowTypeInfo,
-    inputType: TypeInformation[CRow])
-  extends ProcessFunction[CRow, CRow]
+    inputType: TypeInformation[CRow],
+    queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with Compiler[GeneratedAggregations] {
+
   private var output: CRow = _
   private var accumulatorState: ValueState[Row] = _
   private var rowMapState: MapState[Long, JList[Row]] = _
@@ -81,6 +84,8 @@ class ProcTimeBoundedRangeOver(
     val stateDescriptor: ValueStateDescriptor[Row] =
       new ValueStateDescriptor[Row]("overState", aggregatesTypeInfo)
     accumulatorState = getRuntimeContext.getState(stateDescriptor)
+
+    initCleanupTimeState("ProcTimeBoundedRangeOverCleanupTime")
   }
 
   override def processElement(
@@ -89,6 +94,9 @@ class ProcTimeBoundedRangeOver(
     out: Collector[CRow]): Unit = {
 
     val currentTime = ctx.timerService.currentProcessingTime
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, currentTime)
+
     // buffer the event incoming event
 
     // add current element to the window list of elements with corresponding timestamp
@@ -109,7 +117,15 @@ class ProcTimeBoundedRangeOver(
     ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
     out: Collector[CRow]): Unit = {
 
-    // we consider the original timestamp of events that have registered this time trigger 1 ms ago
+    if (needToCleanupState(timestamp)) {
+      // clean up and return
+      cleanupState(rowMapState, accumulatorState)
+      return
+    }
+
+    // we consider the original timestamp of events
+    // that have registered this time trigger 1 ms ago
+
     val currentTime = timestamp - 1
     var i = 0
 
@@ -153,7 +169,8 @@ class ProcTimeBoundedRangeOver(
 
     // get the list of elements of current proctime
     val currentElements = rowMapState.get(currentTime)
-    // add current elements to aggregator. Multiple elements might have arrived in the same proctime
+    // add current elements to aggregator. Multiple elements might
+    // have arrived in the same proctime
     // the same accumulator value will be computed for all elements
     var iElemenets = 0
     while (iElemenets < currentElements.size()) {
@@ -178,7 +195,6 @@ class ProcTimeBoundedRangeOver(
 
     // update the value of accumulators for future incremental computation
     accumulatorState.update(accumulators)
-
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
index 0c7f44ed1531b..e388c93bf6526 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
@@ -33,6 +33,7 @@ import org.apache.flink.api.java.typeutils.ListTypeInfo
 import java.util.{List => JList}
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.slf4j.LoggerFactory
@@ -49,8 +50,9 @@ class ProcTimeBoundedRowsOver(
     genAggregations: GeneratedAggregationsFunction,
     precedingOffset: Long,
     aggregatesTypeInfo: RowTypeInfo,
-    inputType: TypeInformation[CRow])
-  extends ProcessFunction[CRow, CRow]
+    inputType: TypeInformation[CRow],
+    queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with Compiler[GeneratedAggregations] {
 
   Preconditions.checkArgument(precedingOffset > 0)
@@ -99,6 +101,8 @@ class ProcTimeBoundedRowsOver(
     val smallestTimestampDescriptor : ValueStateDescriptor[Long] =
        new ValueStateDescriptor[Long]("smallestTSState", classOf[Long])
     smallestTsState = getRuntimeContext.getState(smallestTimestampDescriptor)
+
+    initCleanupTimeState("ProcTimeBoundedRowsOverCleanupTime")
   }
 
   override def processElement(
@@ -110,6 +114,9 @@ class ProcTimeBoundedRowsOver(
 
     val currentTime = ctx.timerService.currentProcessingTime
 
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, currentTime)
+
     // initialize state for the processed element
     var accumulators = accumulatorState.value
     if (accumulators == null) {
@@ -180,4 +187,13 @@ class ProcTimeBoundedRowsOver(
     out.collect(output)
   }
 
+  override def onTimer(
+    timestamp: Long,
+    ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+    out: Collector[CRow]): Unit = {
+
+    if (needToCleanupState(timestamp)) {
+      cleanupState(rowMapState, accumulatorState, counterState, smallestTsState)
+    }
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala
index 8a231327e01c2..2a6c9c85f95ba 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedNonPartitionedOver.scala
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration
 import org.apache.flink.runtime.state.{FunctionInitializationContext, FunctionSnapshotContext}
 import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction
 import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.util.Collector
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
@@ -37,8 +38,9 @@ import org.slf4j.LoggerFactory
   */
 class ProcTimeUnboundedNonPartitionedOver(
     genAggregations: GeneratedAggregationsFunction,
-    aggregationStateType: RowTypeInfo)
-  extends ProcessFunction[CRow, CRow]
+    aggregationStateType: RowTypeInfo,
+    queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with CheckpointedFunction
     with Compiler[GeneratedAggregations] {
 
@@ -68,12 +70,16 @@ class ProcTimeUnboundedNonPartitionedOver(
         accumulators = function.createAccumulators()
       }
     }
+    initCleanupTimeState("ProcTimeUnboundedNonPartitionedOverCleanupTime")
   }
 
   override def processElement(
       inputC: CRow,
       ctx: ProcessFunction[CRow, CRow]#Context,
       out: Collector[CRow]): Unit = {
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+
 
     val input = inputC.row
 
@@ -85,6 +91,16 @@ class ProcTimeUnboundedNonPartitionedOver(
     out.collect(output)
   }
 
+  override def onTimer(
+    timestamp: Long,
+    ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+    out: Collector[CRow]): Unit = {
+
+    if (needToCleanupState(timestamp)) {
+      cleanupState(state)
+    }
+  }
+
   override def snapshotState(context: FunctionSnapshotContext): Unit = {
     state.clear()
     if (null != accumulators) {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala
index 847c1bfb142f5..97f0ad78fdc79 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedPartitionedOver.scala
@@ -24,6 +24,7 @@ import org.apache.flink.util.Collector
 import org.apache.flink.api.common.state.ValueStateDescriptor
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.api.common.state.ValueState
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.table.runtime.types.CRow
 import org.slf4j.LoggerFactory
@@ -36,8 +37,9 @@ import org.slf4j.LoggerFactory
   */
 class ProcTimeUnboundedPartitionedOver(
     genAggregations: GeneratedAggregationsFunction,
-    aggregationStateType: RowTypeInfo)
-  extends ProcessFunction[CRow, CRow]
+    aggregationStateType: RowTypeInfo,
+    queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with Compiler[GeneratedAggregations] {
 
   private var output: CRow = _
@@ -59,6 +61,8 @@ class ProcTimeUnboundedPartitionedOver(
     val stateDescriptor: ValueStateDescriptor[Row] =
       new ValueStateDescriptor[Row]("overState", aggregationStateType)
     state = getRuntimeContext.getState(stateDescriptor)
+
+    initCleanupTimeState("ProcTimeUnboundedPartitionedOverCleanupTime")
   }
 
   override def processElement(
@@ -66,6 +70,9 @@ class ProcTimeUnboundedPartitionedOver(
     ctx: ProcessFunction[CRow, CRow]#Context,
     out: Collector[CRow]): Unit = {
 
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+
     val input = inputC.row
 
     var accumulators = state.value()
@@ -83,4 +90,13 @@ class ProcTimeUnboundedPartitionedOver(
     out.collect(output)
   }
 
+  override def onTimer(
+    timestamp: Long,
+    ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
+    out: Collector[CRow]): Unit = {
+
+    if (needToCleanupState(timestamp)) {
+      cleanupState(state)
+    }
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
new file mode 100644
index 0000000000000..292fd3bdf1e27
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.aggregate
+
+import java.lang.{Long => JLong}
+
+import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
+import org.apache.flink.api.common.state.State
+import org.apache.flink.streaming.api.TimeDomain
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.table.api.{StreamQueryConfig, Types}
+
+abstract class ProcessFunctionWithCleanupState[IN,OUT](queryConfig: StreamQueryConfig)
+  extends ProcessFunction[IN, OUT]{
+
+  protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
+  protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
+  protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
+
+  // holds the latest registered cleanup timer
+  private var cleanupTimeState: ValueState[JLong] = _
+
+  protected def initCleanupTimeState(stateName: String) {
+    if (stateCleaningEnabled) {
+      val inputCntDescriptor: ValueStateDescriptor[JLong] =
+        new ValueStateDescriptor[JLong](stateName, Types.LONG)
+      cleanupTimeState = getRuntimeContext.getState(inputCntDescriptor)
+    }
+  }
+
+  protected def registerProcessingCleanupTimer(
+    ctx: ProcessFunction[IN, OUT]#Context,
+    currentTime: Long): Unit = {
+    if (stateCleaningEnabled) {
+
+      // last registered timer
+      val curCleanupTime = cleanupTimeState.value()
+
+      // check if a cleanup timer is registered and
+      // that the current cleanup timer won't delete state we need to keep
+      if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
+        // we need to register a new (later) timer
+        val cleanupTime = currentTime + maxRetentionTime
+        // register timer and remember clean-up time
+        ctx.timerService().registerProcessingTimeTimer(cleanupTime)
+        cleanupTimeState.update(cleanupTime)
+      }
+    }
+  }
+
+  protected def isProcessingTimeTimer(ctx: OnTimerContext): Boolean = {
+    ctx.timeDomain() == TimeDomain.PROCESSING_TIME
+  }
+
+  protected def needToCleanupState(timestamp: Long): Boolean = {
+    if (stateCleaningEnabled) {
+      val cleanupTime = cleanupTimeState.value()
+      // check that the triggered timer is the last registered processing time timer.
+      null != cleanupTime && timestamp == cleanupTime
+    } else {
+      false
+    }
+  }
+
+  protected def cleanupState(states: State*): Unit = {
+    // clear all state
+    states.foreach(_.clear())
+    this.cleanupTimeState.clear()
+  }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index 4020d440a78fa..65edf6d662fd8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -24,6 +24,7 @@ import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.typeutils.{ListTypeInfo, RowTypeInfo}
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.apache.flink.types.Row
@@ -42,8 +43,9 @@ class RowTimeBoundedRangeOver(
     genAggregations: GeneratedAggregationsFunction,
     aggregationStateType: RowTypeInfo,
     inputRowType: CRowTypeInfo,
-    precedingOffset: Long)
-  extends ProcessFunction[CRow, CRow]
+    precedingOffset: Long,
+    queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with Compiler[GeneratedAggregations] {
   Preconditions.checkNotNull(aggregationStateType)
   Preconditions.checkNotNull(precedingOffset)
@@ -97,6 +99,8 @@ class RowTimeBoundedRangeOver(
         valueTypeInformation)
 
     dataState = getRuntimeContext.getMapState(mapStateDescriptor)
+
+    initCleanupTimeState("RowTimeBoundedRangeOverCleanupTime")
   }
 
   override def processElement(
@@ -106,6 +110,9 @@ class RowTimeBoundedRangeOver(
 
     val input = inputC.row
 
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+
     // triggering timestamp for trigger calculation
     val triggeringTs = ctx.timestamp
 
@@ -131,6 +138,34 @@ class RowTimeBoundedRangeOver(
     timestamp: Long,
     ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
     out: Collector[CRow]): Unit = {
+
+    if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
+      if (needToCleanupState(timestamp)) {
+
+        val keysIt = dataState.keys.iterator()
+        val lastProcessedTime = lastTriggeringTsState.value
+
+        // is data left which has not been processed yet?
+        var noRecordsToProcess = true
+        while (keysIt.hasNext && noRecordsToProcess) {
+          if (keysIt.next() > lastProcessedTime) {
+            noRecordsToProcess = false
+          }
+        }
+
+        if (noRecordsToProcess) {
+          // we clean the state
+          cleanupState(dataState, accumulatorState, lastTriggeringTsState)
+        } else {
+          // There are records left to process because a watermark has not been received yet.
+          // This would only happen if the input stream has stopped. So we don't need to clean up.
+          // We leave the state as it is and schedule a new cleanup timer
+          registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+        }
+      }
+      return
+    }
+
     // gets all window data from state for the calculation
     val inputs: JList[Row] = dataState.get(timestamp)
 
@@ -196,8 +231,11 @@ class RowTimeBoundedRangeOver(
 
       // update state
       accumulatorState.update(accumulators)
-      lastTriggeringTsState.update(timestamp)
     }
+    lastTriggeringTsState.update(timestamp)
+
+    // update cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
   }
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index 5ec6ec7ab1ead..395ae3986076a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.typeutils.{ListTypeInfo, RowTypeInfo}
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.types.Row
 import org.apache.flink.util.{Collector, Preconditions}
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
@@ -43,8 +44,9 @@ class RowTimeBoundedRowsOver(
     genAggregations: GeneratedAggregationsFunction,
     aggregationStateType: RowTypeInfo,
     inputRowType: CRowTypeInfo,
-    precedingOffset: Long)
-  extends ProcessFunction[CRow, CRow]
+    precedingOffset: Long,
+    queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with Compiler[GeneratedAggregations] {
 
   Preconditions.checkNotNull(aggregationStateType)
@@ -106,6 +108,8 @@ class RowTimeBoundedRowsOver(
         valueTypeInformation)
 
     dataState = getRuntimeContext.getMapState(mapStateDescriptor)
+
+    initCleanupTimeState("RowTimeBoundedRowsOverCleanupTime")
   }
 
   override def processElement(
@@ -115,6 +119,9 @@ class RowTimeBoundedRowsOver(
 
     val input = inputC.row
 
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+
     // triggering timestamp for trigger calculation
     val triggeringTs = ctx.timestamp
 
@@ -141,6 +148,33 @@ class RowTimeBoundedRowsOver(
     ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
     out: Collector[CRow]): Unit = {
 
+    if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
+      if (needToCleanupState(timestamp)) {
+
+        val keysIt = dataState.keys.iterator()
+        val lastProcessedTime = lastTriggeringTsState.value
+
+        // is data left which has not been processed yet?
+        var noRecordsToProcess = true
+        while (keysIt.hasNext && noRecordsToProcess) {
+          if (keysIt.next() > lastProcessedTime) {
+            noRecordsToProcess = false
+          }
+        }
+
+        if (noRecordsToProcess) {
+          // We clean the state
+          cleanupState(dataState, accumulatorState, dataCountState, lastTriggeringTsState)
+        } else {
+          // There are records left to process because a watermark has not been received yet.
+          // This would only happen if the input stream has stopped. So we don't need to clean up.
+          // We leave the state as it is and schedule a new cleanup timer
+          registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+        }
+      }
+      return
+    }
+
     // gets all window data from state for the calculation
     val inputs: JList[Row] = dataState.get(timestamp)
 
@@ -220,6 +254,9 @@ class RowTimeBoundedRowsOver(
     }
 
     lastTriggeringTsState.update(timestamp)
+
+    // update cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
   }
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
index 3e2a8117b56e5..741d2b48fea53 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -28,6 +28,7 @@ import org.apache.flink.util.{Collector, Preconditions}
 import org.apache.flink.api.common.state._
 import org.apache.flink.api.java.typeutils.ListTypeInfo
 import org.apache.flink.streaming.api.operators.TimestampedCollector
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 import org.slf4j.LoggerFactory
@@ -43,8 +44,9 @@ import org.slf4j.LoggerFactory
 abstract class RowTimeUnboundedOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType: TypeInformation[Row],
-    inputType: TypeInformation[CRow])
-  extends ProcessFunction[CRow, CRow]
+    inputType: TypeInformation[CRow],
+    queryConfig: StreamQueryConfig)
+  extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
     with Compiler[GeneratedAggregations] {
 
   protected var output: CRow = _
@@ -83,6 +85,8 @@ abstract class RowTimeUnboundedOver(
       new MapStateDescriptor[Long, JList[Row]]("rowmapstate",
         BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]], rowListTypeInfo)
     rowMapState = getRuntimeContext.getMapState(mapStateDescriptor)
+
+    initCleanupTimeState("RowTimeUnboundedOverCleanupTime")
   }
 
   /**
@@ -101,6 +105,9 @@ abstract class RowTimeUnboundedOver(
 
     val input = inputC.row
 
+    // register state-cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+
     val timestamp = ctx.timestamp()
     val curWatermark = ctx.timerService().currentWatermark()
 
@@ -133,6 +140,24 @@ abstract class RowTimeUnboundedOver(
       ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
       out: Collector[CRow]): Unit = {
 
+    if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
+      if (needToCleanupState(timestamp)) {
+
+        // we check whether there are still records which have not been processed yet
+        val noRecordsToProcess = !rowMapState.keys.iterator().hasNext
+        if (noRecordsToProcess) {
+          // we clean the state
+          cleanupState(rowMapState, accumulatorState)
+        } else {
+          // There are records left to process because a watermark has not been received yet.
+          // This would only happen if the input stream has stopped. So we don't need to clean up.
+          // We leave the state as it is and schedule a new cleanup timer
+          registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+        }
+      }
+      return
+    }
+
     Preconditions.checkArgument(out.isInstanceOf[TimestampedCollector[CRow]])
     val collector = out.asInstanceOf[TimestampedCollector[CRow]]
 
@@ -178,6 +203,9 @@ abstract class RowTimeUnboundedOver(
         ctx.timerService.registerEventTimeTimer(curWatermark + 1)
       }
     }
+
+    // update cleanup timer
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
   }
 
   /**
@@ -221,11 +249,13 @@ abstract class RowTimeUnboundedOver(
 class RowTimeUnboundedRowsOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType: TypeInformation[Row],
-    inputType: TypeInformation[CRow])
+    inputType: TypeInformation[CRow],
+    queryConfig: StreamQueryConfig)
   extends RowTimeUnboundedOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType,
-    inputType) {
+    inputType,
+    queryConfig) {
 
   override def processElementsWithSameTimestamp(
     curRowList: JList[Row],
@@ -259,11 +289,13 @@ class RowTimeUnboundedRowsOver(
 class RowTimeUnboundedRangeOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType: TypeInformation[Row],
-    inputType: TypeInformation[CRow])
+    inputType: TypeInformation[CRow],
+    queryConfig: StreamQueryConfig)
   extends RowTimeUnboundedOver(
     genAggregations: GeneratedAggregationsFunction,
     intermediateType,
-    inputType) {
+    inputType,
+    queryConfig) {
 
   override def processElementsWithSameTimestamp(
     curRowList: JList[Row],
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
index 910cbf267dbc5..9da2c445a9d06 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupAggregationsITCase.scala
@@ -18,11 +18,12 @@
 
 package org.apache.flink.table.api.scala.stream.table
 
+import org.apache.flink.api.common.time.Time
 import org.apache.flink.api.scala._
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 import org.apache.flink.table.api.scala._
 import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
-import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.{StreamQueryConfig, TableEnvironment}
 import org.apache.flink.table.api.scala.stream.utils.StreamITCase.RetractingSink
 import org.apache.flink.types.Row
 import org.junit.Assert.assertEquals
@@ -34,6 +35,8 @@ import scala.collection.mutable
   * Tests of groupby (without window) aggregations
   */
 class GroupAggregationsITCase extends StreamingWithStateTestBase {
+  private val queryConfig = new StreamQueryConfig()
+  queryConfig.withIdleStateRetentionTime(Time.hours(1), Time.hours(2))
 
   @Test
   def testNonKeyedGroupAggregate(): Unit = {
@@ -45,7 +48,7 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
     val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
             .select('a.sum, 'b.sum)
 
-    val results = t.toRetractStream[Row]
+    val results = t.toRetractStream[Row](queryConfig)
     results.addSink(new StreamITCase.RetractingSink).setParallelism(1)
     env.execute()
 
@@ -64,7 +67,7 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
       .groupBy('b)
       .select('b, 'a.sum)
 
-    val results = t.toRetractStream[Row]
+    val results = t.toRetractStream[Row](queryConfig)
     results.addSink(new StreamITCase.RetractingSink)
     env.execute()
 
@@ -85,7 +88,7 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
       .groupBy('cnt)
       .select('cnt, 'b.count as 'freq)
 
-    val results = t.toRetractStream[Row]
+    val results = t.toRetractStream[Row](queryConfig)
 
     results.addSink(new RetractingSink)
     env.execute()
@@ -104,7 +107,7 @@ class GroupAggregationsITCase extends StreamingWithStateTestBase {
       .groupBy('e, 'b % 3)
       .select('c.min, 'e, 'a.avg, 'd.count)
 
-    val results = t.toRetractStream[Row]
+    val results = t.toRetractStream[Row](queryConfig)
     results.addSink(new RetractingSink)
     env.execute()
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
deleted file mode 100644
index eadcfc8c6d774..0000000000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggregate/BoundedProcessingOverRangeProcessFunctionTest.scala
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.runtime.aggregate
-
-import java.util.Comparator
-import java.util.concurrent.ConcurrentLinkedQueue
-import java.lang.{Integer => JInt, Long => JLong}
-
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.streaming.api.operators.KeyedProcessOperator
-import org.apache.flink.streaming.api.watermark.Watermark
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
-import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil}
-import org.apache.flink.table.codegen.GeneratedAggregationsFunction
-import org.apache.flink.table.functions.AggregateFunction
-import org.apache.flink.table.functions.aggfunctions.{LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction}
-import org.apache.flink.table.runtime.aggregate.BoundedProcessingOverRangeProcessFunctionTest._
-import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.apache.flink.types.Row
-import org.junit.Test
-
-class BoundedProcessingOverRangeProcessFunctionTest {
-
-  @Test
-  def testProcTimePartitionedOverRange(): Unit = {
-
-    val rT =  new CRowTypeInfo(new RowTypeInfo(Array[TypeInformation[_]](
-      INT_TYPE_INFO,
-      LONG_TYPE_INFO,
-      INT_TYPE_INFO,
-      STRING_TYPE_INFO,
-      LONG_TYPE_INFO),
-      Array("a", "b", "c", "d", "e")))
-
-    val aggregates =
-      Array(new LongMinWithRetractAggFunction,
-            new LongMaxWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
-    val aggregationStateType: RowTypeInfo = AggregateUtil.createAccumulatorRowType(aggregates)
-
-    val funcCode =
-      """
-        |public class BoundedOverAggregateHelper$33
-        |  extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations {
-        |
-        |  transient org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction
-        |    fmin = null;
-        |
-        |  transient org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction
-        |    fmax = null;
-        |
-        |  public BoundedOverAggregateHelper$33() throws Exception {
-        |
-        |    fmin = (org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction)
-        |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
-        |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
-        |    "MuTG9uZ01pbldpdGhSZXRyYWN0QWdnRnVuY3Rpb26oIdX_DaMPxQIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
-        |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWluV2l0aFJldHJhY3RBZ2dGdW5jdGlvbq_ZGuzxtA_S" +
-        |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
-        |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
-        |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
-        |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
-        |
-        |    fmax = (org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction)
-        |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
-        |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
-        |    "MuTG9uZ01heFdpdGhSZXRyYWN0QWdnRnVuY3Rpb25RmsI8azNGXwIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
-        |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWF4V2l0aFJldHJhY3RBZ2dGdW5jdGlvbvnwowlX0_Qf" +
-        |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
-        |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
-        |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
-        |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
-        |  }
-        |
-        |  public void setAggregationResults(
-        |    org.apache.flink.types.Row accs,
-        |    org.apache.flink.types.Row output) {
-        |
-        |    org.apache.flink.table.functions.AggregateFunction baseClass0 =
-        |      (org.apache.flink.table.functions.AggregateFunction) fmin;
-        |    output.setField(5, baseClass0.getValue(
-        |      (org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
-        |      accs.getField(0)));
-        |
-        |    org.apache.flink.table.functions.AggregateFunction baseClass1 =
-        |      (org.apache.flink.table.functions.AggregateFunction) fmax;
-        |    output.setField(6, baseClass1.getValue(
-        |      (org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
-        |      accs.getField(1)));
-        |  }
-        |
-        |  public void accumulate(
-        |    org.apache.flink.types.Row accs,
-        |    org.apache.flink.types.Row input) {
-        |
-        |    fmin.accumulate(
-        |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
-        |      accs.getField(0)),
-        |      (java.lang.Long) input.getField(4));
-        |
-        |    fmax.accumulate(
-        |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
-        |      accs.getField(1)),
-        |      (java.lang.Long) input.getField(4));
-        |  }
-        |
-        |  public void retract(
-        |    org.apache.flink.types.Row accs,
-        |    org.apache.flink.types.Row input) {
-        |
-        |    fmin.retract(
-        |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
-        |      accs.getField(0)),
-        |      (java.lang.Long) input.getField(4));
-        |
-        |    fmax.retract(
-        |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
-        |      accs.getField(1)),
-        |      (java.lang.Long) input.getField(4));
-        |  }
-        |
-        |  public org.apache.flink.types.Row createAccumulators() {
-        |
-        |    org.apache.flink.types.Row accs = new org.apache.flink.types.Row(2);
-        |
-        |    accs.setField(
-        |      0,
-        |      fmin.createAccumulator());
-        |
-        |    accs.setField(
-        |      1,
-        |      fmax.createAccumulator());
-        |
-        |      return accs;
-        |  }
-        |
-        |  public void setForwardedFields(
-        |    org.apache.flink.types.Row input,
-        |    org.apache.flink.types.Row output) {
-        |
-        |    output.setField(0, input.getField(0));
-        |    output.setField(1, input.getField(1));
-        |    output.setField(2, input.getField(2));
-        |    output.setField(3, input.getField(3));
-        |    output.setField(4, input.getField(4));
-        |  }
-        |
-        |  public org.apache.flink.types.Row createOutputRow() {
-        |    return new org.apache.flink.types.Row(7);
-        |  }
-        |
-        |/*******  This test does not use the following methods  *******/
-        |  public org.apache.flink.types.Row mergeAccumulatorsPair(
-        |    org.apache.flink.types.Row a,
-        |    org.apache.flink.types.Row b) {
-        |    return null;
-        |  }
-        |
-        |  public void resetAccumulator(org.apache.flink.types.Row accs) {
-        |  }
-        |
-        |  public void setConstantFlags(org.apache.flink.types.Row output) {
-        |  }
-        |}
-      """.stripMargin
-
-    val funcName = "BoundedOverAggregateHelper$33"
-
-    val genAggFunction = GeneratedAggregationsFunction(funcName, funcCode)
-    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
-      new ProcTimeBoundedRangeOver(
-        genAggFunction,
-        1000,
-        aggregationStateType,
-        rT))
-
-    val testHarness = new KeyedOneInputStreamOperatorTestHarness[JInt, CRow, CRow](
-      processFunction,
-      new TupleRowSelector(0),
-      BasicTypeInfo.INT_TYPE_INFO)
-
-    testHarness.open()
-
-    // Time = 3
-    testHarness.setProcessingTime(3)
-    // key = 1
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 0))
-    // key = 2
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong), true), 0))
-
-    // Time = 4
-    testHarness.setProcessingTime(4)
-    // key = 1
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 0))
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 0))
-    // key = 2
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 0))
-
-    // Time = 5
-    testHarness.setProcessingTime(5)
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 0))
-
-    // Time = 6
-    testHarness.setProcessingTime(6)
-
-    // Time = 1002
-    testHarness.setProcessingTime(1002)
-    // key = 1
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 0))
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 0))
-    // key = 2
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 0))
-
-    // Time = 1003
-    testHarness.setProcessingTime(1003)
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 0))
-
-    // Time = 1004
-    testHarness.setProcessingTime(1004)
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 0))
-
-    // Time = 1005
-    testHarness.setProcessingTime(1005)
-    // key = 1
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 0))
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 0))
-    // key = 2
-    testHarness.processElement(new StreamRecord(
-      new CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 0))
-
-    testHarness.setProcessingTime(1006)
-
-    val result = testHarness.getOutput
-
-    val expectedOutput = new ConcurrentLinkedQueue[Object]()
-
-    // all elements at the same proc timestamp have the same value
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong, 1L: JLong, 1L: JLong), true), 4))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 4))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 3L: JLong), true), 5))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 5))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 5))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), true), 6))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 6L: JLong), true), 1003))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), true), 1003))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 1003))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 1004))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 2L: JLong, 8L: JLong), true), 1005))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 4L: JLong, 10L: JLong), true), 1006))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 4L: JLong, 10L: JLong), true), 1006))
-    expectedOutput.add(new StreamRecord(new CRow(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 30L: JLong, 40L: JLong), true), 1006))
-
-    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.",
-        expectedOutput, result, new RowResultSortComparator(6))
-
-    testHarness.close()
-
-  }
-}
-
-object BoundedProcessingOverRangeProcessFunctionTest {
-
-/**
- * Return 0 for equal CRows and non zero for different CRows
- */
-class RowResultSortComparator(indexCounter: Int) extends Comparator[Object] with Serializable {
-
-    override def compare(o1: Object, o2: Object):Int = {
-
-      if (o1.isInstanceOf[Watermark] || o2.isInstanceOf[Watermark]) {
-        // watermark is not expected
-         -1
-       } else {
-        val row1 = o1.asInstanceOf[StreamRecord[CRow]].getValue
-        val row2 = o2.asInstanceOf[StreamRecord[CRow]].getValue
-        row1.toString.compareTo(row2.toString)
-      }
-   }
-}
-
-/**
- * Simple test class that returns a specified field as the selector function
- */
-class TupleRowSelector(
-    private val selectorField:Int) extends KeySelector[CRow, Integer] {
-
-  override def getKey(value: CRow): Integer = {
-    value.row.getField(selectorField).asInstanceOf[Integer]
-  }
-}
-
-}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
index eb5acd5b9a3c9..77798f9ddcebb 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
@@ -19,15 +19,294 @@ package org.apache.flink.table.runtime.harness
 
 import java.util.{Comparator, Queue => JQueue}
 
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{INT_TYPE_INFO, LONG_TYPE_INFO, STRING_TYPE_INFO}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator
 import org.apache.flink.streaming.api.watermark.Watermark
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
 import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil}
-import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.codegen.GeneratedAggregationsFunction
+import org.apache.flink.table.functions.AggregateFunction
+import org.apache.flink.table.functions.aggfunctions.{LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction, IntSumWithRetractAggFunction}
+import org.apache.flink.table.runtime.aggregate.AggregateUtil
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 class HarnessTestBase {
+
+  protected val MinMaxRowType = new RowTypeInfo(Array[TypeInformation[_]](
+    INT_TYPE_INFO,
+    LONG_TYPE_INFO,
+    INT_TYPE_INFO,
+    STRING_TYPE_INFO,
+    LONG_TYPE_INFO),
+    Array("a", "b", "c", "d", "e"))
+
+  protected val SumRowType = new RowTypeInfo(Array[TypeInformation[_]](
+    LONG_TYPE_INFO,
+    INT_TYPE_INFO,
+    STRING_TYPE_INFO),
+    Array("a", "b", "c"))
+
+  protected val minMaxCRowType = new CRowTypeInfo(MinMaxRowType)
+  protected val sumCRowType = new CRowTypeInfo(SumRowType)
+
+  protected val minMaxAggregates =
+    Array(new LongMinWithRetractAggFunction,
+          new LongMaxWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
+
+  protected val sumAggregates =
+    Array(new IntSumWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
+
+  protected val minMaxAggregationStateType: RowTypeInfo =
+    AggregateUtil.createAccumulatorRowType(minMaxAggregates)
+
+  protected val sumAggregationStateType: RowTypeInfo =
+    AggregateUtil.createAccumulatorRowType(sumAggregates)
+
+  val minMaxCode: String =
+    """
+      |public class MinMaxAggregateHelper
+      |  extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations {
+      |
+      |  transient org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction
+      |    fmin = null;
+      |
+      |  transient org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction
+      |    fmax = null;
+      |
+      |  public MinMaxAggregateHelper() throws Exception {
+      |
+      |    fmin = (org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction)
+      |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
+      |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
+      |    "MuTG9uZ01pbldpdGhSZXRyYWN0QWdnRnVuY3Rpb26oIdX_DaMPxQIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
+      |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWluV2l0aFJldHJhY3RBZ2dGdW5jdGlvbq_ZGuzxtA_S" +
+      |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
+      |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
+      |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
+      |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
+      |
+      |    fmax = (org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction)
+      |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
+      |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
+      |    "MuTG9uZ01heFdpdGhSZXRyYWN0QWdnRnVuY3Rpb25RmsI8azNGXwIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
+      |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWF4V2l0aFJldHJhY3RBZ2dGdW5jdGlvbvnwowlX0_Qf" +
+      |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
+      |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
+      |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
+      |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
+      |  }
+      |
+      |  public void setAggregationResults(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row output) {
+      |
+      |    org.apache.flink.table.functions.AggregateFunction baseClass0 =
+      |      (org.apache.flink.table.functions.AggregateFunction) fmin;
+      |    output.setField(5, baseClass0.getValue(
+      |      (org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+      |      accs.getField(0)));
+      |
+      |    org.apache.flink.table.functions.AggregateFunction baseClass1 =
+      |      (org.apache.flink.table.functions.AggregateFunction) fmax;
+      |    output.setField(6, baseClass1.getValue(
+      |      (org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+      |      accs.getField(1)));
+      |  }
+      |
+      |  public void accumulate(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row input) {
+      |
+      |    fmin.accumulate(
+      |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+      |      accs.getField(0)),
+      |      (java.lang.Long) input.getField(4));
+      |
+      |    fmax.accumulate(
+      |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+      |      accs.getField(1)),
+      |      (java.lang.Long) input.getField(4));
+      |  }
+      |
+      |  public void retract(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row input) {
+      |
+      |    fmin.retract(
+      |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
+      |      accs.getField(0)),
+      |      (java.lang.Long) input.getField(4));
+      |
+      |    fmax.retract(
+      |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
+      |      accs.getField(1)),
+      |      (java.lang.Long) input.getField(4));
+      |  }
+      |
+      |  public org.apache.flink.types.Row createAccumulators() {
+      |
+      |    org.apache.flink.types.Row accs = new org.apache.flink.types.Row(2);
+      |
+      |    accs.setField(
+      |      0,
+      |      fmin.createAccumulator());
+      |
+      |    accs.setField(
+      |      1,
+      |      fmax.createAccumulator());
+      |
+      |      return accs;
+      |  }
+      |
+      |  public void setForwardedFields(
+      |    org.apache.flink.types.Row input,
+      |    org.apache.flink.types.Row output) {
+      |
+      |    output.setField(0, input.getField(0));
+      |    output.setField(1, input.getField(1));
+      |    output.setField(2, input.getField(2));
+      |    output.setField(3, input.getField(3));
+      |    output.setField(4, input.getField(4));
+      |  }
+      |
+      |  public org.apache.flink.types.Row createOutputRow() {
+      |    return new org.apache.flink.types.Row(7);
+      |  }
+      |
+      |/*******  This test does not use the following methods  *******/
+      |  public org.apache.flink.types.Row mergeAccumulatorsPair(
+      |    org.apache.flink.types.Row a,
+      |    org.apache.flink.types.Row b) {
+      |    return null;
+      |  }
+      |
+      |  public void resetAccumulator(org.apache.flink.types.Row accs) {
+      |  }
+      |
+      |  public void setConstantFlags(org.apache.flink.types.Row output) {
+      |  }
+      |}
+    """.stripMargin
+
+  val sumAggCode: String =
+    """
+      |public final class SumAggregationHelper
+      |  extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations {
+      |
+      |
+      |transient org.apache.flink.table.functions.aggfunctions.IntSumWithRetractAggFunction
+      |sum = null;
+      |private final org.apache.flink.table.runtime.aggregate.SingleElementIterable<org.apache
+      |    .flink.table.functions.aggfunctions.SumWithRetractAccumulator> accIt0 =
+      |      new org.apache.flink.table.runtime.aggregate.SingleElementIterable<org.apache.flink
+      |      .table
+      |      .functions.aggfunctions.SumWithRetractAccumulator>();
+      |
+      |  public SumAggregationHelper() throws Exception {
+      |
+      |sum = (org.apache.flink.table.functions.aggfunctions.IntSumWithRetractAggFunction)
+      |org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
+      |.deserialize
+      |("rO0ABXNyAEpvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuSW50U3VtV2l0a" +
+      |"FJldHJhY3RBZ2dGdW5jdGlvblkfWkeNZDeDAgAAeHIAR29yZy5hcGFjaGUuZmxpbmsudGFibGUuZnVuY3Rpb25" +
+      |"zLmFnZ2Z1bmN0aW9ucy5TdW1XaXRoUmV0cmFjdEFnZ0Z1bmN0aW9ut2oWrOsLrs0CAAFMAAdudW1lcmljdAAUT" +
+      |"HNjYWxhL21hdGgvTnVtZXJpYzt4cgAyb3JnLmFwYWNoZS5mbGluay50YWJsZS5mdW5jdGlvbnMuQWdncmVnYXR" +
+      |"lRnVuY3Rpb25NxhU-0mM1_AIAAHhyADRvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5Vc2VyRGVma" +
+      |"W5lZEZ1bmN0aW9uLQH3VDG4DJMCAAB4cHNyACFzY2FsYS5tYXRoLk51bWVyaWMkSW50SXNJbnRlZ3JhbCTw6XA" +
+      |"59sPAzAIAAHhw");
+      |
+      |
+      |  }
+      |
+      |  public final void setAggregationResults(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row output) {
+      |
+      |    org.apache.flink.table.functions.AggregateFunction baseClass0 =
+      |      (org.apache.flink.table.functions.AggregateFunction)
+      |      sum;
+      |
+      |    output.setField(
+      |      1,
+      |      baseClass0.getValue((org.apache.flink.table.functions.aggfunctions
+      |      .SumWithRetractAccumulator) accs.getField(0)));
+      |  }
+      |
+      |  public final void accumulate(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row input) {
+      |
+      |    sum.accumulate(
+      |      ((org.apache.flink.table.functions.aggfunctions.SumWithRetractAccumulator) accs
+      |      .getField
+      |      (0)),
+      |      (java.lang.Integer) input.getField(1));
+      |  }
+      |
+      |
+      |  public final void retract(
+      |    org.apache.flink.types.Row accs,
+      |    org.apache.flink.types.Row input) {
+      |  }
+      |
+      |  public final org.apache.flink.types.Row createAccumulators()
+      |     {
+      |
+      |      org.apache.flink.types.Row accs =
+      |          new org.apache.flink.types.Row(1);
+      |
+      |    accs.setField(
+      |      0,
+      |      sum.createAccumulator());
+      |
+      |      return accs;
+      |  }
+      |
+      |  public final void setForwardedFields(
+      |    org.apache.flink.types.Row input,
+      |    org.apache.flink.types.Row output)
+      |     {
+      |
+      |    output.setField(
+      |      0,
+      |      input.getField(0));
+      |  }
+      |
+      |  public final void setConstantFlags(org.apache.flink.types.Row output)
+      |     {
+      |
+      |  }
+      |
+      |  public final org.apache.flink.types.Row createOutputRow() {
+      |    return new org.apache.flink.types.Row(2);
+      |  }
+      |
+      |
+      |  public final org.apache.flink.types.Row mergeAccumulatorsPair(
+      |    org.apache.flink.types.Row a,
+      |    org.apache.flink.types.Row b)
+      |            {
+      |
+      |      return a;
+      |
+      |  }
+      |
+      |  public final void resetAccumulator(
+      |    org.apache.flink.types.Row accs) {
+      |  }
+      |}
+      |""".stripMargin
+
+
+  protected val minMaxFuncName = "MinMaxAggregateHelper"
+  protected val sumFuncName = "SumAggregationHelper"
+
+  protected val genMinMaxAggFunction = GeneratedAggregationsFunction(minMaxFuncName, minMaxCode)
+  protected val genSumAggFunction = GeneratedAggregationsFunction(sumFuncName, sumAggCode)
+
   def createHarnessTester[IN, OUT, KEY](
     operator: OneInputStreamOperator[IN, OUT],
     keySelector: KeySelector[IN, KEY],
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala
new file mode 100644
index 0000000000000..04214f9a178ef
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.harness
+
+import java.lang.{Integer => JInt, Long => JLong}
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import org.apache.flink.api.common.time.Time
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.streaming.api.operators.KeyedProcessOperator
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
+import org.apache.flink.table.api.StreamQueryConfig
+import org.apache.flink.table.runtime.aggregate._
+import org.apache.flink.table.runtime.harness.HarnessTestBase._
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.junit.Test
+
+class NonWindowHarnessTest extends HarnessTestBase {
+
+  protected var queryConfig =
+    new StreamQueryConfig().withIdleStateRetentionTime(Time.seconds(2), Time.seconds(3))
+
+  @Test
+  def testProcTimeNonWindow(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new GroupAggProcessFunction(
+        genSumAggFunction,
+        sumAggregationStateType,
+        false,
+        queryConfig))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[String](2),
+        BasicTypeInfo.STRING_TYPE_INFO)
+
+    testHarness.open()
+
+    // register cleanup timer with 3001
+    testHarness.setProcessingTime(1)
+
+    testHarness.processElement(new StreamRecord(CRow(Row.of(1L: JLong, 1: JInt, "aaa"), true), 1))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(2L: JLong, 1: JInt, "bbb"), true), 1))
+    // reuse timer 3001
+    testHarness.setProcessingTime(1000)
+    testHarness.processElement(new StreamRecord(CRow(Row.of(3L: JLong, 2: JInt, "aaa"), true), 1))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(4L: JLong, 3: JInt, "aaa"), true), 1))
+
+    // register cleanup timer with 4002
+    testHarness.setProcessingTime(1002)
+    testHarness.processElement(new StreamRecord(CRow(Row.of(5L: JLong, 4: JInt, "aaa"), true), 1))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(6L: JLong, 2: JInt, "bbb"), true), 1))
+
+    // trigger cleanup timer and register cleanup timer with 7003
+    testHarness.setProcessingTime(4003)
+    testHarness.processElement(new StreamRecord(CRow(Row.of(7L: JLong, 5: JInt, "aaa"), true), 1))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(8L: JLong, 6: JInt, "aaa"), true), 1))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(9L: JLong, 7: JInt, "aaa"), true), 1))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(10L: JLong, 3: JInt, "bbb"), true), 1))
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    expectedOutput.add(new StreamRecord(CRow(Row.of(1L: JLong, 1: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(2L: JLong, 1: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(3L: JLong, 3: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(4L: JLong, 6: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(5L: JLong, 10: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(6L: JLong, 3: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(7L: JLong, 5: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(8L: JLong, 11: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(9L: JLong, 18: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(10L: JLong, 3: JInt), true), 1))
+
+    verify(expectedOutput, result, new RowResultSortComparator(6))
+
+    testHarness.close()
+  }
+
+  @Test
+  def testProcTimeNonWindowWithRetract(): Unit = {
+
+    val processFunction = new KeyedProcessOperator[String, CRow, CRow](
+      new GroupAggProcessFunction(
+        genSumAggFunction,
+        sumAggregationStateType,
+        true,
+        queryConfig))
+
+    val testHarness =
+      createHarnessTester(
+        processFunction,
+        new TupleRowKeySelector[String](2),
+        BasicTypeInfo.STRING_TYPE_INFO)
+
+    testHarness.open()
+
+    // register cleanup timer with 3001
+    testHarness.setProcessingTime(1)
+
+    testHarness.processElement(new StreamRecord(CRow(Row.of(1L: JLong, 1: JInt, "aaa"), true), 1))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(2L: JLong, 1: JInt, "bbb"), true), 2))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(3L: JLong, 2: JInt, "aaa"), true), 3))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(4L: JLong, 3: JInt, "ccc"), true), 4))
+
+    // trigger cleanup timer and register cleanup timer with 6002
+    testHarness.setProcessingTime(3002)
+    testHarness.processElement(new StreamRecord(CRow(Row.of(5L: JLong, 4: JInt, "aaa"), true), 5))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(6L: JLong, 2: JInt, "bbb"), true), 6))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(7L: JLong, 5: JInt, "aaa"), true), 7))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(8L: JLong, 6: JInt, "eee"), true), 8))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(9L: JLong, 7: JInt, "aaa"), true), 9))
+    testHarness.processElement(new StreamRecord(CRow(Row.of(10L: JLong, 3: JInt, "bbb"), true), 10))
+
+    val result = testHarness.getOutput
+
+    val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+    expectedOutput.add(new StreamRecord(CRow(Row.of(1L: JLong, 1: JInt), true), 1))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(2L: JLong, 1: JInt), true), 2))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(3L: JLong, 1: JInt), false), 3))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(3L: JLong, 3: JInt), true), 3))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(4L: JLong, 3: JInt), true), 4))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(5L: JLong, 4: JInt), true), 5))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(6L: JLong, 2: JInt), true), 6))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(7L: JLong, 4: JInt), false), 7))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(7L: JLong, 9: JInt), true), 7))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(8L: JLong, 6: JInt), true), 8))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(9L: JLong, 9: JInt), false), 9))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(9L: JLong, 16: JInt), true), 9))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(10L: JLong, 2: JInt), false), 10))
+    expectedOutput.add(new StreamRecord(CRow(Row.of(10L: JLong, 5: JInt), true), 10))
+
+    verify(expectedOutput, result, new RowResultSortComparator(0))
+
+    testHarness.close()
+  }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala
index 56ca85c18e29d..786a843075412 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala
@@ -18,180 +18,34 @@
 package org.apache.flink.table.runtime.harness
 
 import java.lang.{Integer => JInt, Long => JLong}
-import java.util.concurrent.ConcurrentLinkedQueue
+import java.util.concurrent.{ConcurrentLinkedQueue}
 
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
-import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.common.time.Time
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
 import org.apache.flink.streaming.api.operators.KeyedProcessOperator
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
-import org.apache.flink.table.codegen.GeneratedAggregationsFunction
-import org.apache.flink.table.functions.AggregateFunction
-import org.apache.flink.table.functions.aggfunctions.{LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction}
+import org.apache.flink.table.api.StreamQueryConfig
 import org.apache.flink.table.runtime.aggregate._
 import org.apache.flink.table.runtime.harness.HarnessTestBase._
-import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.types.Row
 import org.junit.Test
 
 class OverWindowHarnessTest extends HarnessTestBase{
 
-  private val rT = new RowTypeInfo(Array[TypeInformation[_]](
-    INT_TYPE_INFO,
-    LONG_TYPE_INFO,
-    INT_TYPE_INFO,
-    STRING_TYPE_INFO,
-    LONG_TYPE_INFO),
-    Array("a", "b", "c", "d", "e"))
-
-  private val cRT = new CRowTypeInfo(rT)
-
-  private val aggregates =
-    Array(new LongMinWithRetractAggFunction,
-      new LongMaxWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
-  private val aggregationStateType: RowTypeInfo = AggregateUtil.createAccumulatorRowType(aggregates)
-
-  val funcCode: String =
-    """
-      |public class BoundedOverAggregateHelper
-      |  extends org.apache.flink.table.runtime.aggregate.GeneratedAggregations {
-      |
-      |  transient org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction
-      |    fmin = null;
-      |
-      |  transient org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction
-      |    fmax = null;
-      |
-      |  public BoundedOverAggregateHelper() throws Exception {
-      |
-      |    fmin = (org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction)
-      |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
-      |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
-      |    "MuTG9uZ01pbldpdGhSZXRyYWN0QWdnRnVuY3Rpb26oIdX_DaMPxQIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
-      |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWluV2l0aFJldHJhY3RBZ2dGdW5jdGlvbq_ZGuzxtA_S" +
-      |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
-      |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
-      |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
-      |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
-      |
-      |    fmax = (org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction)
-      |    org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
-      |    .deserialize("rO0ABXNyAEtvcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbn" +
-      |    "MuTG9uZ01heFdpdGhSZXRyYWN0QWdnRnVuY3Rpb25RmsI8azNGXwIAAHhyAEdvcmcuYXBhY2hlLmZsaW5rL" +
-      |    "nRhYmxlLmZ1bmN0aW9ucy5hZ2dmdW5jdGlvbnMuTWF4V2l0aFJldHJhY3RBZ2dGdW5jdGlvbvnwowlX0_Qf" +
-      |    "AgABTAADb3JkdAAVTHNjYWxhL21hdGgvT3JkZXJpbmc7eHIAMm9yZy5hcGFjaGUuZmxpbmsudGFibGUuZnV" +
-      |    "uY3Rpb25zLkFnZ3JlZ2F0ZUZ1bmN0aW9uTcYVPtJjNfwCAAB4cgA0b3JnLmFwYWNoZS5mbGluay50YWJsZS" +
-      |    "5mdW5jdGlvbnMuVXNlckRlZmluZWRGdW5jdGlvbi0B91QxuAyTAgAAeHBzcgAZc2NhbGEubWF0aC5PcmRlc" +
-      |    "mluZyRMb25nJOda0iCPo2ukAgAAeHA");
-      |  }
-      |
-      |  public void setAggregationResults(
-      |    org.apache.flink.types.Row accs,
-      |    org.apache.flink.types.Row output) {
-      |
-      |    org.apache.flink.table.functions.AggregateFunction baseClass0 =
-      |      (org.apache.flink.table.functions.AggregateFunction) fmin;
-      |    output.setField(5, baseClass0.getValue(
-      |      (org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
-      |      accs.getField(0)));
-      |
-      |    org.apache.flink.table.functions.AggregateFunction baseClass1 =
-      |      (org.apache.flink.table.functions.AggregateFunction) fmax;
-      |    output.setField(6, baseClass1.getValue(
-      |      (org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
-      |      accs.getField(1)));
-      |  }
-      |
-      |  public void accumulate(
-      |    org.apache.flink.types.Row accs,
-      |    org.apache.flink.types.Row input) {
-      |
-      |    fmin.accumulate(
-      |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
-      |      accs.getField(0)),
-      |      (java.lang.Long) input.getField(4));
-      |
-      |    fmax.accumulate(
-      |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
-      |      accs.getField(1)),
-      |      (java.lang.Long) input.getField(4));
-      |  }
-      |
-      |  public void retract(
-      |    org.apache.flink.types.Row accs,
-      |    org.apache.flink.types.Row input) {
-      |
-      |    fmin.retract(
-      |      ((org.apache.flink.table.functions.aggfunctions.MinWithRetractAccumulator)
-      |      accs.getField(0)),
-      |      (java.lang.Long) input.getField(4));
-      |
-      |    fmax.retract(
-      |      ((org.apache.flink.table.functions.aggfunctions.MaxWithRetractAccumulator)
-      |      accs.getField(1)),
-      |      (java.lang.Long) input.getField(4));
-      |  }
-      |
-      |  public org.apache.flink.types.Row createAccumulators() {
-      |
-      |    org.apache.flink.types.Row accs = new org.apache.flink.types.Row(2);
-      |
-      |    accs.setField(
-      |      0,
-      |      fmin.createAccumulator());
-      |
-      |    accs.setField(
-      |      1,
-      |      fmax.createAccumulator());
-      |
-      |      return accs;
-      |  }
-      |
-      |  public void setForwardedFields(
-      |    org.apache.flink.types.Row input,
-      |    org.apache.flink.types.Row output) {
-      |
-      |    output.setField(0, input.getField(0));
-      |    output.setField(1, input.getField(1));
-      |    output.setField(2, input.getField(2));
-      |    output.setField(3, input.getField(3));
-      |    output.setField(4, input.getField(4));
-      |  }
-      |
-      |  public org.apache.flink.types.Row createOutputRow() {
-      |    return new org.apache.flink.types.Row(7);
-      |  }
-      |
-      |/*******  This test does not use the following methods  *******/
-      |  public org.apache.flink.types.Row mergeAccumulatorsPair(
-      |    org.apache.flink.types.Row a,
-      |    org.apache.flink.types.Row b) {
-      |    return null;
-      |  }
-      |
-      |  public void resetAccumulator(org.apache.flink.types.Row accs) {
-      |  }
-      |
-      |  public void setConstantFlags(org.apache.flink.types.Row output) {
-      |  }
-      |}
-    """.stripMargin
-
-
-  private val funcName = "BoundedOverAggregateHelper"
-
-  private val genAggFunction = GeneratedAggregationsFunction(funcName, funcCode)
-
+  protected var queryConfig =
+    new StreamQueryConfig().withIdleStateRetentionTime(Time.seconds(2), Time.seconds(3))
 
   @Test
   def testProcTimeBoundedRowsOver(): Unit = {
 
     val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new ProcTimeBoundedRowsOver(
-        genAggFunction,
+        genMinMaxAggFunction,
         2,
-        aggregationStateType,
-        cRT))
+        minMaxAggregationStateType,
+        minMaxCRowType,
+        queryConfig))
 
     val testHarness =
       createHarnessTester(processFunction,new TupleRowKeySelector[Integer](0),BasicTypeInfo
@@ -199,6 +53,7 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.open()
 
+    // register cleanup timer with 3001
     testHarness.setProcessingTime(1)
 
     testHarness.processElement(new StreamRecord(
@@ -209,6 +64,9 @@ class OverWindowHarnessTest extends HarnessTestBase{
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 1))
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 1))
+
+    // register cleanup timer with 4100
+    testHarness.setProcessingTime(1100)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong), true), 1))
     testHarness.processElement(new StreamRecord(
@@ -220,15 +78,19 @@ class OverWindowHarnessTest extends HarnessTestBase{
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 1))
 
-    testHarness.setProcessingTime(2)
+    // register cleanup timer with 6001
+    testHarness.setProcessingTime(3001)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 2))
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 2))
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 2))
+
+    // trigger cleanup timer and register cleanup timer with 9002
+    testHarness.setProcessingTime(6002)
     testHarness.processElement(new StreamRecord(
-      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 2))
+        CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 2))
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 2))
 
@@ -274,10 +136,10 @@ class OverWindowHarnessTest extends HarnessTestBase{
         Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 8L: JLong, 9L: JLong), true), 2))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 9L: JLong, 10L: JLong), true), 2))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 10L: JLong, 10L: JLong), true), 2))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 30L: JLong, 40L: JLong), true), 2))
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 40L: JLong, 40L: JLong), true), 2))
 
     verify(expectedOutput, result, new RowResultSortComparator(6))
 
@@ -292,10 +154,11 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new ProcTimeBoundedRangeOver(
-        genAggFunction,
-        1000,
-        aggregationStateType,
-        cRT))
+        genMinMaxAggFunction,
+        4000,
+        minMaxAggregationStateType,
+        minMaxCRowType,
+        queryConfig))
 
     val testHarness =
       createHarnessTester(
@@ -305,6 +168,7 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.open()
 
+    // register cleanup timer with 3003
     testHarness.setProcessingTime(3)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 0))
@@ -314,6 +178,9 @@ class OverWindowHarnessTest extends HarnessTestBase{
     testHarness.setProcessingTime(4)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong), true), 0))
+
+    // trigger cleanup timer and register cleanup timer with 6003
+    testHarness.setProcessingTime(3003)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
@@ -323,9 +190,10 @@ class OverWindowHarnessTest extends HarnessTestBase{
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong), true), 0))
 
-    testHarness.setProcessingTime(6)
+    // register cleanup timer with 9002
+    testHarness.setProcessingTime(6002)
 
-    testHarness.setProcessingTime(1002)
+    testHarness.setProcessingTime(7002)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
@@ -333,15 +201,15 @@ class OverWindowHarnessTest extends HarnessTestBase{
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 0))
 
-    testHarness.setProcessingTime(1003)
+    // register cleanup timer with 14002
+    testHarness.setProcessingTime(11002)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 0))
 
-    testHarness.setProcessingTime(1004)
+    testHarness.setProcessingTime(11004)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 0))
 
-    testHarness.setProcessingTime(1005)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
@@ -349,7 +217,7 @@ class OverWindowHarnessTest extends HarnessTestBase{
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 0))
 
-    testHarness.setProcessingTime(1006)
+    testHarness.setProcessingTime(11006)
 
     val result = testHarness.getOutput
 
@@ -364,40 +232,40 @@ class OverWindowHarnessTest extends HarnessTestBase{
         Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 10L: JLong, 10L: JLong, 10L: JLong), true), 4))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 3L: JLong), true), 5))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 2L: JLong, 1L: JLong, 2L: JLong), true), 5))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 1L: JLong, 3L: JLong), true), 5))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 3L: JLong, 3L: JLong, 4L: JLong), true), 3004))
     expectedOutput.add(new StreamRecord(
-      CRow(
-        Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 10L: JLong, 20L: JLong), true), 5))
+      CRow(Row.of(
+        2: JInt, 0L: JLong, 0: JInt, "bbb", 20L: JLong, 20L: JLong, 20L: JLong), true), 3004))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 1L: JLong, 4L: JLong), true), 6))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 4L: JLong, 4L: JLong, 4L: JLong), true), 6))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 1L: JLong, 6L: JLong), true), 1003))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 5L: JLong, 5L: JLong, 6L: JLong), true), 7003))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 1L: JLong, 6L: JLong), true), 1003))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong, 5L: JLong, 6L: JLong), true), 7003))
     expectedOutput.add(new StreamRecord(
       CRow(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 1003))
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 30L: JLong, 30L: JLong), true), 7003))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 1004))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 7L: JLong, 7L: JLong), true), 11003))
     expectedOutput.add(new StreamRecord(
-      CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 2L: JLong, 8L: JLong), true), 1005))
+      CRow(Row.of(
+        1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 7L: JLong, 10L: JLong), true), 11005))
     expectedOutput.add(new StreamRecord(
-      CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 4L: JLong, 10L: JLong), true), 1006))
+      CRow(Row.of(
+        1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 7L: JLong, 10L: JLong), true), 11005))
     expectedOutput.add(new StreamRecord(
       CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 4L: JLong, 10L: JLong), true), 1006))
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 7L: JLong, 10L: JLong), true), 11005))
     expectedOutput.add(new StreamRecord(
       CRow(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 30L: JLong, 40L: JLong), true), 1006))
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 40L: JLong, 40L: JLong), true), 11005))
 
     verify(expectedOutput, result, new RowResultSortComparator(6))
 
@@ -409,8 +277,9 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new ProcTimeUnboundedPartitionedOver(
-        genAggFunction,
-        aggregationStateType))
+        genMinMaxAggFunction,
+        minMaxAggregationStateType,
+        queryConfig))
 
     val testHarness =
       createHarnessTester(
@@ -420,6 +289,9 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.open()
 
+    // register cleanup timer with 4003
+    testHarness.setProcessingTime(1003)
+
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
@@ -438,18 +310,19 @@ class OverWindowHarnessTest extends HarnessTestBase{
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 6L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong), true), 0))
-
-    testHarness.setProcessingTime(1003)
     testHarness.processElement(new StreamRecord(
-      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 1003))
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong), true), 0))
     testHarness.processElement(new StreamRecord(
-      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 1003))
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong), true), 0))
+
+    // trigger cleanup timer and register cleanup timer with 8003
+    testHarness.setProcessingTime(5003)
     testHarness.processElement(new StreamRecord(
-      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 1003))
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong), true), 5003))
     testHarness.processElement(new StreamRecord(
-      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 1003))
+      CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong), true), 5003))
     testHarness.processElement(new StreamRecord(
-      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 1003))
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong), true), 5003))
 
     val result = testHarness.getOutput
 
@@ -484,19 +357,19 @@ class OverWindowHarnessTest extends HarnessTestBase{
         Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 30L: JLong, 10L: JLong, 30L: JLong), true), 0))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 1003))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 7L: JLong, 1L: JLong, 7L: JLong), true), 0))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 1L: JLong, 8L: JLong), true), 1003))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 8L: JLong, 1L: JLong, 8L: JLong), true), 0))
     expectedOutput.add(new StreamRecord(
       CRow(
-        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 1L: JLong, 9L: JLong), true), 1003))
+        Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 9L: JLong, 9L: JLong, 9L: JLong), true), 5003))
     expectedOutput.add(new StreamRecord(
       CRow(
-      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 1L: JLong, 10L: JLong), true), 1003))
+      Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 10L: JLong, 9L: JLong, 10L: JLong), true), 5003))
     expectedOutput.add(new StreamRecord(
       CRow(
-      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 10L: JLong, 40L: JLong), true), 1003))
+      Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 40L: JLong, 40L: JLong), true), 5003))
 
     verify(expectedOutput, result, new RowResultSortComparator(6))
     testHarness.close()
@@ -510,10 +383,11 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new RowTimeBoundedRangeOver(
-        genAggFunction,
-        aggregationStateType,
-        cRT,
-        4000))
+        genMinMaxAggFunction,
+        minMaxAggregationStateType,
+        minMaxCRowType,
+        4000,
+        new StreamQueryConfig().withIdleStateRetentionTime(Time.seconds(1), Time.seconds(2))))
 
     val testHarness =
       createHarnessTester(
@@ -573,6 +447,40 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.processWatermark(19000)
 
+    // test cleanup
+    testHarness.setProcessingTime(1000)
+    testHarness.processWatermark(20000)
+
+    // check that state is removed after max retention time
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong), true), 20001)) // clean-up 3000
+    testHarness.setProcessingTime(2500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong), true), 20002)) // clean-up 4500
+    testHarness.processWatermark(20010) // compute output
+
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(4499)
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(4500)
+    assert(testHarness.numKeyedStateEntries() == 0) // check that all state is gone
+
+    // check that state is only removed if all data was processed
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(3: JInt, 0L: JLong, 0: JInt, "ccc", 3L: JLong), true), 20011)) // clean-up 6500
+
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(6500) // clean-up attempt but rescheduled to 8500
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+
+    testHarness.processWatermark(20020) // schedule emission
+
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(8499) // clean-up
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(8500) // clean-up
+    assert(testHarness.numKeyedStateEntries() == 0) // check that all state is gone
+
     val result = testHarness.getOutput
 
     val expectedOutput = new ConcurrentLinkedQueue[Object]()
@@ -621,6 +529,16 @@ class OverWindowHarnessTest extends HarnessTestBase{
       CRow(
       Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 40L: JLong, 40L: JLong), true), 12001))
 
+    expectedOutput.add(new StreamRecord(
+      CRow(
+      Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong, 1L: JLong, 1L: JLong), true), 20001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong, 1L: JLong, 2L: JLong), true), 20002))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(3: JInt, 0L: JLong, 0: JInt, "ccc", 3L: JLong, 3L: JLong, 3L: JLong), true), 20011))
+
     verify(expectedOutput, result, new RowResultSortComparator(6))
     testHarness.close()
   }
@@ -630,10 +548,11 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new RowTimeBoundedRowsOver(
-        genAggFunction,
-        aggregationStateType,
-        cRT,
-        3))
+        genMinMaxAggFunction,
+        minMaxAggregationStateType,
+        minMaxCRowType,
+        3,
+        new StreamQueryConfig().withIdleStateRetentionTime(Time.seconds(1), Time.seconds(2))))
 
     val testHarness =
       createHarnessTester(
@@ -689,6 +608,41 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.processWatermark(19000)
 
+    // test cleanup
+    testHarness.setProcessingTime(1000)
+    testHarness.processWatermark(20000)
+
+    // check that state is removed after max retention time
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong), true), 20001)) // clean-up 3000
+    testHarness.setProcessingTime(2500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong), true), 20002)) // clean-up 4500
+    testHarness.processWatermark(20010) // compute output
+
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(4499)
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(4500)
+    assert(testHarness.numKeyedStateEntries() == 0) // check that all state is gone
+
+    // check that state is only removed if all data was processed
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(3: JInt, 0L: JLong, 0: JInt, "ccc", 3L: JLong), true), 20011)) // clean-up 6500
+
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(6500) // clean-up attempt but rescheduled to 8500
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+
+    testHarness.processWatermark(20020) // schedule emission
+
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(8499) // clean-up
+    assert(testHarness.numKeyedStateEntries() > 0) // check that we have state
+    testHarness.setProcessingTime(8500) // clean-up
+    assert(testHarness.numKeyedStateEntries() == 0) // check that all state is gone
+
+
     val result = testHarness.getOutput
 
     val expectedOutput = new ConcurrentLinkedQueue[Object]()
@@ -736,6 +690,16 @@ class OverWindowHarnessTest extends HarnessTestBase{
       CRow(
       Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 20L: JLong, 40L: JLong), true), 12001))
 
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong, 1L: JLong, 1L: JLong), true), 20001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong, 1L: JLong, 2L: JLong), true), 20002))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(3: JInt, 0L: JLong, 0: JInt, "ccc", 3L: JLong, 3L: JLong, 3L: JLong), true), 20011))
+
     verify(expectedOutput, result, new RowResultSortComparator(6))
     testHarness.close()
   }
@@ -748,9 +712,10 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new RowTimeUnboundedRangeOver(
-        genAggFunction,
-        aggregationStateType,
-        cRT))
+        genMinMaxAggFunction,
+        minMaxAggregationStateType,
+        minMaxCRowType,
+        new StreamQueryConfig().withIdleStateRetentionTime(Time.seconds(1), Time.seconds(2))))
 
     val testHarness =
       createHarnessTester(
@@ -760,6 +725,7 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.open()
 
+    testHarness.setProcessingTime(1000)
     testHarness.processWatermark(800)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 801))
@@ -806,6 +772,30 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.processWatermark(19000)
 
+    // test cleanup
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(2999) // clean up timer is 3000, so nothing should happen
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(3000) // clean up is triggered
+    assert(testHarness.numKeyedStateEntries() == 0)
+
+    testHarness.processWatermark(20000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong), true), 20001)) // clean-up 5000
+    testHarness.setProcessingTime(2500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong), true), 20002)) // clean-up 5000
+
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(5000) // does not clean up, because data left. New timer 7000
+    testHarness.processWatermark(20010) // compute output
+
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(6999) // clean up timer is 3000, so nothing should happen
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(7000) // clean up is triggered
+    assert(testHarness.numKeyedStateEntries() == 0)
+
     val result = testHarness.getOutput
 
     val expectedOutput = new ConcurrentLinkedQueue[Object]()
@@ -854,6 +844,13 @@ class OverWindowHarnessTest extends HarnessTestBase{
       CRow(
       Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 10L: JLong, 40L: JLong), true), 12001))
 
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong, 1L: JLong, 1L: JLong), true), 20001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong, 1L: JLong, 2L: JLong), true), 20002))
+
     verify(expectedOutput, result, new RowResultSortComparator(6))
     testHarness.close()
   }
@@ -863,9 +860,10 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     val processFunction = new KeyedProcessOperator[String, CRow, CRow](
       new RowTimeUnboundedRowsOver(
-        genAggFunction,
-        aggregationStateType,
-        cRT))
+        genMinMaxAggFunction,
+        minMaxAggregationStateType,
+        minMaxCRowType,
+        new StreamQueryConfig().withIdleStateRetentionTime(Time.seconds(1), Time.seconds(2))))
 
     val testHarness =
       createHarnessTester(
@@ -875,6 +873,7 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.open()
 
+    testHarness.setProcessingTime(1000)
     testHarness.processWatermark(800)
     testHarness.processElement(new StreamRecord(
       CRow(Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 1L: JLong), true), 801))
@@ -921,6 +920,30 @@ class OverWindowHarnessTest extends HarnessTestBase{
 
     testHarness.processWatermark(19000)
 
+    // test cleanup
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(2999) // clean up timer is 3000, so nothing should happen
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(3000) // clean up is triggered
+    assert(testHarness.numKeyedStateEntries() == 0)
+
+    testHarness.processWatermark(20000)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong), true), 20001)) // clean-up 5000
+    testHarness.setProcessingTime(2500)
+    testHarness.processElement(new StreamRecord(
+      CRow(Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong), true), 20002)) // clean-up 5000
+
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(5000) // does not clean up, because data left. New timer 7000
+    testHarness.processWatermark(20010) // compute output
+
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(6999) // clean up timer is 3000, so nothing should happen
+    assert(testHarness.numKeyedStateEntries() > 0)
+    testHarness.setProcessingTime(7000) // clean up is triggered
+    assert(testHarness.numKeyedStateEntries() == 0)
+
     val result = testHarness.getOutput
 
     val expectedOutput = new ConcurrentLinkedQueue[Object]()
@@ -968,6 +991,13 @@ class OverWindowHarnessTest extends HarnessTestBase{
       CRow(
       Row.of(2: JInt, 0L: JLong, 0: JInt, "bbb", 40L: JLong, 10L: JLong, 40L: JLong), true), 12001))
 
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(1: JInt, 0L: JLong, 0: JInt, "ccc", 1L: JLong, 1L: JLong, 1L: JLong), true), 20001))
+    expectedOutput.add(new StreamRecord(
+      CRow(
+        Row.of(2: JInt, 0L: JLong, 0: JInt, "ccc", 2L: JLong, 1L: JLong, 2L: JLong), true), 20002))
+
     verify(expectedOutput, result, new RowResultSortComparator(6))
     testHarness.close()
   }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
index 3d79e2284cdb8..c4e2433833793 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/MockTableEnvironment.scala
@@ -28,7 +28,7 @@ class MockTableEnvironment extends TableEnvironment(new TableConfig) {
   override private[flink] def writeToSink[T](
       table: Table,
       sink: TableSink[T],
-      qConfig: QueryConfig): Unit = ???
+      queryConfig: QueryConfig): Unit = ???
 
   override protected def checkValidTableName(name: String): Unit = ???
 

From 9c7fd39e64cf8a0e9f0b9408276f48d5e58be81a Mon Sep 17 00:00:00 2001
From: twalthr <twalthr@apache.org>
Date: Wed, 10 May 2017 10:11:34 +0200
Subject: [PATCH 232/253] [FLINK-6483] [table] Add materialization of time
 indicators.

This closes #3862.
---
 .../table/api/StreamTableEnvironment.scala    |  39 +-
 .../calcite/RelTimeIndicatorConverter.scala   | 404 ++++++++++++------
 .../flink/table/codegen/CodeGenerator.scala   |  47 +-
 .../flink/table/plan/nodes/CommonCalc.scala   |  13 +-
 .../table/plan/nodes/CommonCorrelate.scala    |  19 +-
 .../plan/nodes/dataset/DataSetCalc.scala      |   6 +-
 .../plan/nodes/dataset/DataSetCorrelate.scala |   8 +-
 .../nodes/datastream/DataStreamCalc.scala     |  14 +-
 .../datastream/DataStreamCorrelate.scala      |  23 +-
 .../datastream/StreamTableSourceScan.scala    |   4 +-
 .../plan/nodes/logical/FlinkLogicalCalc.scala |   2 +-
 .../logical/FlinkLogicalTableSourceScan.scala |   6 +-
 ...DataStreamLogicalWindowAggregateRule.scala |  14 +-
 .../flink/table/plan/schema/RowSchema.scala   |  11 +-
 .../plan/schema/StreamTableSourceTable.scala  |   8 +-
 ...scala => CRowCorrelateProcessRunner.scala} |  36 +-
 ...apRunner.scala => CRowProcessRunner.scala} |  28 +-
 ...utes.scala => definedTimeAttributes.scala} |   4 +-
 .../stream/StreamTableEnvironmentTest.scala   |  10 +-
 .../api/scala/stream/TableSourceTest.scala    |  22 +-
 .../RelTimeIndicatorConverterTest.scala       | 351 +++++++++++++++
 .../datastream/TimeAttributesITCase.scala     | 237 ++++++++++
 .../flink/table/utils/TableTestBase.scala     |   5 +
 23 files changed, 1078 insertions(+), 233 deletions(-)
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/{CRowCorrelateFlatMapRunner.scala => CRowCorrelateProcessRunner.scala} (72%)
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/{CRowFlatMapRunner.scala => CRowProcessRunner.scala} (76%)
 rename flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/{DefinedTimeAttributes.scala => definedTimeAttributes.scala} (97%)
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverterTest.scala
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/TimeAttributesITCase.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index d68da04799717..994ac8034e953 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -18,38 +18,37 @@
 
 package org.apache.flink.table.api
 
-import _root_.java.util.concurrent.atomic.AtomicInteger
 import _root_.java.lang.{Boolean => JBool}
+import _root_.java.util.concurrent.atomic.AtomicInteger
 
 import org.apache.calcite.plan.RelOptUtil
 import org.apache.calcite.plan.hep.HepMatchOrder
-import org.apache.calcite.rel.{RelNode, RelVisitor}
 import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.{RelNode, RelVisitor}
 import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode}
 import org.apache.calcite.sql.SqlKind
 import org.apache.calcite.sql2rel.RelDecorrelator
 import org.apache.calcite.tools.{RuleSet, RuleSets}
-import org.apache.flink.api.common.typeinfo.AtomicType
-import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.api.common.functions.MapFunction
-import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
+import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
 import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.TimeCharacteristic
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.apache.flink.table.calcite.RelTimeIndicatorConverter
 import org.apache.flink.table.explain.PlanJsonParser
 import org.apache.flink.table.expressions.{Expression, ProctimeAttribute, RowtimeAttribute, UnresolvedFieldReference}
 import org.apache.flink.table.plan.nodes.FlinkConventions
-import org.apache.flink.table.plan.nodes.datastream.{DataStreamRel, UpdateAsRetractionTrait}
-import org.apache.flink.table.plan.nodes.datastream._
+import org.apache.flink.table.plan.nodes.datastream.{DataStreamRel, UpdateAsRetractionTrait, _}
 import org.apache.flink.table.plan.rules.FlinkRuleSets
 import org.apache.flink.table.plan.schema.{DataStreamTable, RowSchema, StreamTableSourceTable}
-import org.apache.flink.table.runtime.{CRowInputJavaTupleOutputMapRunner, CRowInputMapRunner, CRowInputScalaTupleOutputMapRunner}
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.table.runtime.{CRowInputJavaTupleOutputMapRunner, CRowInputMapRunner, CRowInputScalaTupleOutputMapRunner}
 import org.apache.flink.table.sinks.{AppendStreamTableSink, RetractStreamTableSink, TableSink, UpsertStreamTableSink}
-import org.apache.flink.table.sources.{StreamTableSource, TableSource}
+import org.apache.flink.table.sources.{DefinedRowtimeAttribute, StreamTableSource, TableSource}
 import org.apache.flink.table.typeutils.TypeCheckUtils
 import org.apache.flink.types.Row
 
@@ -111,6 +110,17 @@ abstract class StreamTableEnvironment(
   override def registerTableSource(name: String, tableSource: TableSource[_]): Unit = {
     checkValidTableName(name)
 
+    // check if event-time is enabled
+    tableSource match {
+      case dra: DefinedRowtimeAttribute if
+          execEnv.getStreamTimeCharacteristic != TimeCharacteristic.EventTime =>
+
+        throw TableException(
+          s"A rowtime attribute requires an EventTime time characteristic in stream environment. " +
+            s"But is: ${execEnv.getStreamTimeCharacteristic}")
+      case _ => // ok
+    }
+
     tableSource match {
       case streamTableSource: StreamTableSource[_] =>
         registerTableInternal(name, new StreamTableSourceTable(streamTableSource))
@@ -390,6 +400,13 @@ abstract class StreamTableEnvironment(
     // validate and extract time attributes
     val (rowtime, proctime) = validateAndExtractTimeAttributes(streamType, fields)
 
+    // check if event-time is enabled
+    if (rowtime.isDefined && execEnv.getStreamTimeCharacteristic != TimeCharacteristic.EventTime) {
+      throw TableException(
+        s"A rowtime attribute requires an EventTime time characteristic in stream environment. " +
+          s"But is: ${execEnv.getStreamTimeCharacteristic}")
+    }
+
     val dataStreamTable = new DataStreamTable[T](
       dataStream,
       fieldIndexes,
@@ -518,9 +535,9 @@ abstract class StreamTableEnvironment(
     // 3. normalize the logical plan
     val normRuleSet = getNormRuleSet
     val normalizedPlan = if (normRuleSet.iterator().hasNext) {
-      runHepPlanner(HepMatchOrder.BOTTOM_UP, normRuleSet, decorPlan, decorPlan.getTraitSet)
+      runHepPlanner(HepMatchOrder.BOTTOM_UP, normRuleSet, convPlan, convPlan.getTraitSet)
     } else {
-      decorPlan
+      convPlan
     }
 
     // 4. optimize the logical Flink plan
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala
index fa2e3eeb0a01d..7ceb3979972cc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala
@@ -18,73 +18,43 @@
 
 package org.apache.flink.table.calcite
 
-import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFieldImpl, RelRecordType, StructKind}
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.core._
 import org.apache.calcite.rel.logical._
-import org.apache.calcite.rel.{RelNode, RelShuttleImpl}
+import org.apache.calcite.rel.{RelNode, RelShuttle}
 import org.apache.calcite.rex._
 import org.apache.calcite.sql.fun.SqlStdOperatorTable
 import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo
-import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.calcite.FlinkTypeFactory.isTimeIndicatorType
+import org.apache.flink.table.api.{TableException, ValidationException}
 import org.apache.flink.table.functions.TimeMaterializationSqlFunction
 import org.apache.flink.table.plan.schema.TimeIndicatorRelDataType
+import org.apache.flink.table.calcite.FlinkTypeFactory.isTimeIndicatorType
+import org.apache.flink.table.plan.logical.rel.LogicalWindowAggregate
 
 import scala.collection.JavaConversions._
+import scala.collection.mutable
 
 /**
   * Traverses a [[RelNode]] tree and converts fields with [[TimeIndicatorRelDataType]] type. If a
   * time attribute is accessed for a calculation, it will be materialized. Forwarding is allowed in
   * some cases, but not all.
   */
-class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttleImpl {
+class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle {
 
-  override def visit(project: LogicalProject): RelNode = {
-    // visit children and update inputs
-    val updatedProject = super.visit(project).asInstanceOf[LogicalProject]
+  private val timestamp = rexBuilder
+      .getTypeFactory
+      .asInstanceOf[FlinkTypeFactory]
+      .createTypeFromTypeInfo(SqlTimeTypeInfo.TIMESTAMP)
 
-    // check if input field contains time indicator type
-    // materialize field if no time indicator is present anymore
-    // if input field is already materialized, change to timestamp type
-    val materializer = new RexTimeIndicatorMaterializer(
-      rexBuilder,
-      updatedProject.getInput.getRowType.getFieldList.map(_.getType))
-    val newProjects = updatedProject.getProjects.map(_.accept(materializer))
-
-    // copy project
-    updatedProject.copy(
-      updatedProject.getTraitSet,
-      updatedProject.getInput,
-      newProjects,
-      buildRowType(updatedProject.getRowType.getFieldNames, newProjects.map(_.getType))
-    )
-  }
-
-  override def visit(filter: LogicalFilter): RelNode = {
-    // visit children and update inputs
-    val updatedFilter = super.visit(filter).asInstanceOf[LogicalFilter]
-
-    // check if input field contains time indicator type
-    // materialize field if no time indicator is present anymore
-    // if input field is already materialized, change to timestamp type
-    val materializer = new RexTimeIndicatorMaterializer(
-      rexBuilder,
-      updatedFilter.getInput.getRowType.getFieldList.map(_.getType))
-    val newCondition = updatedFilter.getCondition.accept(materializer)
-
-    // copy filter
-    updatedFilter.copy(
-      updatedFilter.getTraitSet,
-      updatedFilter.getInput,
-      newCondition
-    )
-  }
+  override def visit(intersect: LogicalIntersect): RelNode =
+    throw new TableException("Logical intersect in a stream environment is not supported yet.")
 
   override def visit(union: LogicalUnion): RelNode = {
     // visit children and update inputs
-    val updatedUnion = super.visit(union).asInstanceOf[LogicalUnion]
+    val inputs = union.getInputs.map(_.accept(this))
 
     // make sure that time indicator types match
-    val inputTypes = updatedUnion.getInputs.map(_.getRowType)
+    val inputTypes = inputs.map(_.getRowType)
 
     val head = inputTypes.head.getFieldList.map(_.getType)
 
@@ -114,101 +84,269 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttleImpl {
         "Union fields with time attributes have different types.")
     }
 
-    updatedUnion
+    LogicalUnion.create(inputs, union.all)
   }
 
+  override def visit(aggregate: LogicalAggregate): RelNode = convertAggregate(aggregate)
+
+  override def visit(minus: LogicalMinus): RelNode =
+    throw new TableException("Logical minus in a stream environment is not supported yet.")
+
+  override def visit(sort: LogicalSort): RelNode =
+    throw new TableException("Logical sort in a stream environment is not supported yet.")
+
+  override def visit(`match`: LogicalMatch): RelNode =
+    throw new TableException("Logical match in a stream environment is not supported yet.")
+
   override def visit(other: RelNode): RelNode = other match {
-    case scan: LogicalTableFunctionScan if
-        stack.size() > 0 && stack.peek().isInstanceOf[LogicalCorrelate] =>
+
+    case uncollect: Uncollect =>
       // visit children and update inputs
-      val updatedScan = super.visit(scan).asInstanceOf[LogicalTableFunctionScan]
-
-      val correlate = stack.peek().asInstanceOf[LogicalCorrelate]
-
-      // check if input field contains time indicator type
-      // materialize field if no time indicator is present anymore
-      // if input field is already materialized, change to timestamp type
-      val materializer = new RexTimeIndicatorMaterializer(
-        rexBuilder,
-        correlate.getInputs.get(0).getRowType.getFieldList.map(_.getType))
-      val newCall = updatedScan.getCall.accept(materializer)
-
-      // copy scan
-      updatedScan.copy(
-        updatedScan.getTraitSet,
-        updatedScan.getInputs,
-        newCall,
-        updatedScan.getElementType,
-        updatedScan.getRowType,
-        updatedScan.getColumnMappings
-      )
+      val input = uncollect.getInput.accept(this)
+      Uncollect.create(uncollect.getTraitSet, input, uncollect.withOrdinality)
+
+    case scan: LogicalTableFunctionScan =>
+      scan
+
+    case aggregate: LogicalWindowAggregate =>
+      val convAggregate = convertAggregate(aggregate)
+
+      LogicalWindowAggregate.create(
+        aggregate.getWindow,
+        aggregate.getNamedProperties,
+        convAggregate)
 
     case _ =>
-      super.visit(other)
+      throw new TableException(s"Unsupported logical operator: ${other.getClass.getSimpleName}")
   }
 
-  private def buildRowType(names: Seq[String], types: Seq[RelDataType]): RelDataType = {
-    val fields = names.zipWithIndex.map { case (name, idx) =>
-      new RelDataTypeFieldImpl(name, idx, types(idx))
-    }
-    new RelRecordType(StructKind.FULLY_QUALIFIED, fields)
+
+  override def visit(exchange: LogicalExchange): RelNode =
+    throw new TableException("Logical exchange in a stream environment is not supported yet.")
+
+  override def visit(scan: TableScan): RelNode = scan
+
+  override def visit(scan: TableFunctionScan): RelNode =
+    throw new TableException("Table function scan in a stream environment is not supported yet.")
+
+  override def visit(values: LogicalValues): RelNode = values
+
+  override def visit(filter: LogicalFilter): RelNode = {
+    // visit children and update inputs
+    val input = filter.getInput.accept(this)
+
+    // check if input field contains time indicator type
+    // materialize field if no time indicator is present anymore
+    // if input field is already materialized, change to timestamp type
+    val materializer = new RexTimeIndicatorMaterializer(
+      rexBuilder,
+      input.getRowType.getFieldList.map(_.getType))
+
+    val condition = filter.getCondition.accept(materializer)
+    LogicalFilter.create(input, condition)
   }
-}
 
-class RexTimeIndicatorMaterializer(
-    private val rexBuilder: RexBuilder,
-    private val input: Seq[RelDataType])
-  extends RexShuttle {
-
-  val timestamp = rexBuilder
-    .getTypeFactory
-    .asInstanceOf[FlinkTypeFactory]
-    .createTypeFromTypeInfo(SqlTimeTypeInfo.TIMESTAMP)
-
-  override def visitInputRef(inputRef: RexInputRef): RexNode = {
-    // reference is interesting
-    if (isTimeIndicatorType(inputRef.getType)) {
-      val resolvedRefType = input(inputRef.getIndex)
-      // input is a valid time indicator
-      if (isTimeIndicatorType(resolvedRefType)) {
-        inputRef
-      }
-      // input has been materialized
-      else {
-        new RexInputRef(inputRef.getIndex, resolvedRefType)
-      }
-    }
-    // reference is a regular field
-    else {
-      super.visitInputRef(inputRef)
+  override def visit(project: LogicalProject): RelNode = {
+    // visit children and update inputs
+    val input = project.getInput.accept(this)
+
+    // check if input field contains time indicator type
+    // materialize field if no time indicator is present anymore
+    // if input field is already materialized, change to timestamp type
+    val materializer = new RexTimeIndicatorMaterializer(
+      rexBuilder,
+      input.getRowType.getFieldList.map(_.getType))
+
+    val projects = project.getProjects.map(_.accept(materializer))
+    val fieldNames = project.getRowType.getFieldNames
+    LogicalProject.create(input, projects, fieldNames)
+  }
+
+  override def visit(join: LogicalJoin): RelNode =
+    throw new TableException("Logical join in a stream environment is not supported yet.")
+
+  override def visit(correlate: LogicalCorrelate): RelNode = {
+    // visit children and update inputs
+    val inputs = correlate.getInputs.map(_.accept(this))
+
+    val right = inputs(1) match {
+      case scan: LogicalTableFunctionScan =>
+        // visit children and update inputs
+        val scanInputs = scan.getInputs.map(_.accept(this))
+
+        // check if input field contains time indicator type
+        // materialize field if no time indicator is present anymore
+        // if input field is already materialized, change to timestamp type
+        val materializer = new RexTimeIndicatorMaterializer(
+          rexBuilder,
+          inputs.head.getRowType.getFieldList.map(_.getType))
+
+        val call = scan.getCall.accept(materializer)
+        LogicalTableFunctionScan.create(
+          scan.getCluster,
+          scanInputs,
+          call,
+          scan.getElementType,
+          scan.getRowType,
+          scan.getColumnMappings)
+
+      case _ =>
+        inputs(1)
     }
+
+    LogicalCorrelate.create(
+      inputs.head,
+      right,
+      correlate.getCorrelationId,
+      correlate.getRequiredColumns,
+      correlate.getJoinType)
   }
 
-  override def visitCall(call: RexCall): RexNode = {
-    val updatedCall = super.visitCall(call).asInstanceOf[RexCall]
+  private def convertAggregate(aggregate: Aggregate): LogicalAggregate = {
+    // visit children and update inputs
+    val input = aggregate.getInput.accept(this)
+
+    // add a project to materialize aggregation arguments/grouping keys
+
+    val refIndices = mutable.Set[Int]()
+
+    // check arguments of agg calls
+    aggregate.getAggCallList.foreach(call => if (call.getArgList.size() == 0) {
+        // count(*) has an empty argument list
+        (0 until input.getRowType.getFieldCount).foreach(refIndices.add)
+      } else {
+        // for other aggregations
+        call.getArgList.map(_.asInstanceOf[Int]).foreach(refIndices.add)
+      })
 
-    // skip materialization for special operators
-    updatedCall.getOperator match {
-      case SqlStdOperatorTable.SESSION | SqlStdOperatorTable.HOP | SqlStdOperatorTable.TUMBLE =>
-        return updatedCall
+    // check grouping sets
+    aggregate.getGroupSets.foreach(set =>
+      set.asList().map(_.asInstanceOf[Int]).foreach(refIndices.add)
+    )
 
-      case _ => // do nothing
+    val needsMaterialization = refIndices.exists(idx =>
+      isTimeIndicatorType(input.getRowType.getFieldList.get(idx).getType))
+
+    // create project if necessary
+    val projectedInput = if (needsMaterialization) {
+
+      // insert or merge with input project if
+      // a time attribute is accessed and needs to be materialized
+      input match {
+
+        // merge
+        case lp: LogicalProject =>
+          val projects = lp.getProjects.zipWithIndex.map { case (expr, idx) =>
+            if (isTimeIndicatorType(expr.getType) && refIndices.contains(idx)) {
+              rexBuilder.makeCall(
+                TimeMaterializationSqlFunction,
+                expr)
+            } else {
+              expr
+            }
+          }
+
+          LogicalProject.create(
+            lp.getInput,
+            projects,
+            input.getRowType.getFieldNames)
+
+        // new project
+        case _ =>
+          val projects = input.getRowType.getFieldList.map { field =>
+            if (isTimeIndicatorType(field.getType) && refIndices.contains(field.getIndex)) {
+              rexBuilder.makeCall(
+                TimeMaterializationSqlFunction,
+                new RexInputRef(field.getIndex, field.getType))
+            } else {
+              new RexInputRef(field.getIndex, field.getType)
+            }
+          }
+
+          LogicalProject.create(
+            input,
+            projects,
+            input.getRowType.getFieldNames)
+      }
+    } else {
+      // no project necessary
+      input
     }
 
-    // materialize operands with time indicators
-    val materializedOperands = updatedCall.getOperands.map { o =>
-      if (isTimeIndicatorType(o.getType)) {
-        rexBuilder.makeCall(TimeMaterializationSqlFunction, o)
+    // remove time indicator type as agg call return type
+    val updatedAggCalls = aggregate.getAggCallList.map { call =>
+      val callType = if (isTimeIndicatorType(call.getType)) {
+        timestamp
       } else {
-        o
+        call.getType
       }
+      AggregateCall.create(
+        call.getAggregation,
+        call.isDistinct,
+        call.getArgList,
+        call.filterArg,
+        callType,
+        call.name)
     }
 
-    // remove time indicator return type
-    if (isTimeIndicatorType(updatedCall.getType)) {
-      updatedCall.clone(timestamp, materializedOperands)
-    } else {
-      updatedCall.clone(updatedCall.getType, materializedOperands)
+    LogicalAggregate.create(
+      projectedInput,
+      aggregate.indicator,
+      aggregate.getGroupSet,
+      aggregate.getGroupSets,
+      updatedAggCalls)
+  }
+
+  class RexTimeIndicatorMaterializer(
+      private val rexBuilder: RexBuilder,
+      private val input: Seq[RelDataType])
+    extends RexShuttle {
+
+    override def visitInputRef(inputRef: RexInputRef): RexNode = {
+      // reference is interesting
+      if (isTimeIndicatorType(inputRef.getType)) {
+        val resolvedRefType = input(inputRef.getIndex)
+        // input is a valid time indicator
+        if (isTimeIndicatorType(resolvedRefType)) {
+          inputRef
+        }
+        // input has been materialized
+        else {
+          new RexInputRef(inputRef.getIndex, resolvedRefType)
+        }
+      }
+      // reference is a regular field
+      else {
+        super.visitInputRef(inputRef)
+      }
+    }
+
+    override def visitCall(call: RexCall): RexNode = {
+      val updatedCall = super.visitCall(call).asInstanceOf[RexCall]
+
+      // materialize operands with time indicators
+      val materializedOperands = updatedCall.getOperator match {
+
+        // skip materialization for special operators
+        case SqlStdOperatorTable.SESSION | SqlStdOperatorTable.HOP | SqlStdOperatorTable.TUMBLE =>
+          updatedCall.getOperands.toList
+
+        case _ =>
+          updatedCall.getOperands.map { o =>
+            if (isTimeIndicatorType(o.getType)) {
+              rexBuilder.makeCall(TimeMaterializationSqlFunction, o)
+            } else {
+              o
+            }
+          }
+      }
+
+      // remove time indicator return type
+      if (isTimeIndicatorType(updatedCall.getType)) {
+        updatedCall.clone(timestamp, materializedOperands)
+      } else {
+        updatedCall.clone(updatedCall.getType, materializedOperands)
+      }
     }
   }
 }
@@ -217,6 +355,30 @@ object RelTimeIndicatorConverter {
 
   def convert(rootRel: RelNode, rexBuilder: RexBuilder): RelNode = {
     val converter = new RelTimeIndicatorConverter(rexBuilder)
-    rootRel.accept(converter)
+    val convertedRoot = rootRel.accept(converter)
+
+    var needsConversion = false
+
+    // materialize all remaining time indicators
+    val projects = convertedRoot.getRowType.getFieldList.map(field =>
+      if (isTimeIndicatorType(field.getType)) {
+        needsConversion = true
+        rexBuilder.makeCall(
+          TimeMaterializationSqlFunction,
+          new RexInputRef(field.getIndex, field.getType))
+      } else {
+        new RexInputRef(field.getIndex, field.getType)
+      }
+    )
+
+    // add final conversion
+    if (needsConversion) {
+      LogicalProject.create(
+      convertedRoot,
+      projects,
+      convertedRoot.getRowType.getFieldNames)
+    } else {
+      convertedRoot
+    }
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 25addbc5e923b..036889ff78128 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -34,6 +34,7 @@ import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.api.java.typeutils._
 import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
 import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.table.api.TableConfig
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.CodeGenUtils._
@@ -237,6 +238,11 @@ class CodeGenerator(
     */
   var outRecordTerm = "out"
 
+  /**
+    * @return term of the [[ProcessFunction]]'s context
+    */
+  var contextTerm = "ctx"
+
   /**
     * @return returns if null checking is enabled
     */
@@ -699,6 +705,17 @@ class CodeGenerator(
           List(s"$inputTypeTerm1 $input1Term = ($inputTypeTerm1) _in1;",
                s"$inputTypeTerm2 $input2Term = ($inputTypeTerm2) _in2;"))
       }
+
+      // ProcessFunction
+      else if (clazz == classOf[ProcessFunction[_, _]]) {
+        val baseClass = classOf[ProcessFunction[_, _]]
+        val inputTypeTerm = boxedTypeTermForTypeInfo(input1)
+        (baseClass,
+          s"void processElement(Object _in1, " +
+            s"org.apache.flink.streaming.api.functions.ProcessFunction.Context $contextTerm," +
+            s"org.apache.flink.util.Collector $collectorTerm)",
+          List(s"$inputTypeTerm $input1Term = ($inputTypeTerm) _in1;"))
+      }
       else {
         // TODO more functions
         throw new CodeGenException("Unsupported Function.")
@@ -1312,9 +1329,11 @@ class CodeGenerator(
     throw new CodeGenException("Dynamic parameter references are not supported yet.")
 
   override def visitCall(call: RexCall): GeneratedExpression = {
-    // time materialization is not implemented yet
+    // special case: time materialization
     if (call.getOperator == TimeMaterializationSqlFunction) {
-      throw new CodeGenException("Access to time attributes is not possible yet.")
+      return generateRecordTimestamp(
+        FlinkTypeFactory.isRowtimeIndicatorType(call.getOperands.get(0).getType)
+      )
     }
 
     val operands = call.getOperands.map(_.accept(this))
@@ -1840,6 +1859,30 @@ class CodeGenerator(
     }
   }
 
+  private[flink] def generateRecordTimestamp(isEventTime: Boolean): GeneratedExpression = {
+    val resultTerm = newName("result")
+    val resultTypeTerm = primitiveTypeTermForTypeInfo(SqlTimeTypeInfo.TIMESTAMP)
+
+    val resultCode = if (isEventTime) {
+      s"""
+        |$resultTypeTerm $resultTerm;
+        |if ($contextTerm.timestamp() == null) {
+        |  throw new RuntimeException("Rowtime timestamp is null. Please make sure that a proper " +
+        |    "TimestampAssigner is defined and the stream environment uses the EventTime time " +
+        |    "characteristic.");
+        |}
+        |else {
+        |  $resultTerm = $contextTerm.timestamp();
+        |}
+        |""".stripMargin
+    } else {
+      s"""
+        |$resultTypeTerm $resultTerm = $contextTerm.timerService().currentProcessingTime();
+        |""".stripMargin
+    }
+    GeneratedExpression(resultTerm, NEVER_NULL, resultCode, SqlTimeTypeInfo.TIMESTAMP)
+  }
+
   // ----------------------------------------------------------------------------------------------
   // Reusable code snippets
   // ----------------------------------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
index e8755878a3997..9b486e4e90ed5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala
@@ -20,7 +20,7 @@ package org.apache.flink.table.plan.nodes
 
 import org.apache.calcite.plan.{RelOptCost, RelOptPlanner}
 import org.apache.calcite.rex._
-import org.apache.flink.api.common.functions.FlatMapFunction
+import org.apache.flink.api.common.functions.Function
 import org.apache.flink.table.api.TableConfig
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.codegen.{CodeGenerator, GeneratedFunction}
@@ -30,16 +30,17 @@ import org.apache.flink.types.Row
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 
-trait CommonCalc[T] {
+trait CommonCalc {
 
-  private[flink] def generateFunction(
+  private[flink] def generateFunction[T <: Function](
       generator: CodeGenerator,
       ruleDescription: String,
       inputSchema: RowSchema,
       returnSchema: RowSchema,
       calcProgram: RexProgram,
-      config: TableConfig):
-    GeneratedFunction[FlatMapFunction[Row, Row], Row] = {
+      config: TableConfig,
+      functionClass: Class[T]):
+    GeneratedFunction[T, Row] = {
 
     val expandedExpressions = calcProgram
       .getProjectList
@@ -92,7 +93,7 @@ trait CommonCalc[T] {
 
     generator.generateFunction(
       ruleDescription,
-      classOf[FlatMapFunction[Row, Row]],
+      functionClass,
       body,
       returnSchema.physicalTypeInfo)
   }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
index c95f2f78d29c1..874bea28289c2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
@@ -20,7 +20,7 @@ package org.apache.flink.table.plan.nodes
 import org.apache.calcite.rel.`type`.RelDataType
 import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode, RexShuttle}
 import org.apache.calcite.sql.SemiJoinType
-import org.apache.flink.api.common.functions.FlatMapFunction
+import org.apache.flink.api.common.functions.{FlatMapFunction, Function}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.table.api.{TableConfig, TableException}
 import org.apache.flink.table.codegen.CodeGenUtils.primitiveDefaultValue
@@ -36,22 +36,22 @@ import scala.collection.JavaConverters._
 /**
   * Join a user-defined table function
   */
-trait CommonCorrelate[T] {
+trait CommonCorrelate {
 
   /**
     * Generates the flat map function to run the user-defined table function.
     */
-  private[flink] def generateFunction(
+  private[flink] def generateFunction[T <: Function](
     config: TableConfig,
     inputSchema: RowSchema,
     udtfTypeInfo: TypeInformation[Any],
     returnSchema: RowSchema,
-    rowType: RelDataType,
     joinType: SemiJoinType,
     rexCall: RexCall,
     pojoFieldMapping: Option[Array[Int]],
-    ruleDescription: String):
-  GeneratedFunction[FlatMapFunction[Row, Row], Row] = {
+    ruleDescription: String,
+    functionClass: Class[T]):
+  GeneratedFunction[T, Row] = {
 
     val functionGenerator = new CodeGenerator(
       config,
@@ -89,7 +89,7 @@ trait CommonCorrelate[T] {
       val outerResultExpr = functionGenerator.generateResultExpression(
         input1AccessExprs ++ input2NullExprs,
         returnSchema.physicalTypeInfo,
-        rowType.getFieldNames.asScala)
+        returnSchema.physicalFieldNames)
       body +=
         s"""
            |boolean hasOutput = $collectorTerm.isCollected();
@@ -104,7 +104,7 @@ trait CommonCorrelate[T] {
 
     functionGenerator.generateFunction(
       ruleDescription,
-      classOf[FlatMapFunction[Row, Row]],
+      functionClass,
       body,
       returnSchema.physicalTypeInfo)
   }
@@ -117,7 +117,6 @@ trait CommonCorrelate[T] {
     inputSchema: RowSchema,
     udtfTypeInfo: TypeInformation[Any],
     returnSchema: RowSchema,
-    rowType: RelDataType,
     condition: Option[RexNode],
     pojoFieldMapping: Option[Array[Int]])
   : GeneratedCollector = {
@@ -135,7 +134,7 @@ trait CommonCorrelate[T] {
     val crossResultExpr = generator.generateResultExpression(
       input1AccessExprs ++ input2AccessExprs,
       returnSchema.physicalTypeInfo,
-      rowType.getFieldNames.asScala)
+      returnSchema.physicalFieldNames)
 
     val collectorCode = if (condition.isEmpty) {
       s"""
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
index e340a8cce1f28..9a9f738ab7610 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala
@@ -24,6 +24,7 @@ import org.apache.calcite.rel.core.Calc
 import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{RelNode, RelWriter}
 import org.apache.calcite.rex._
+import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.api.java.DataSet
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.table.api.BatchTableEnvironment
@@ -46,7 +47,7 @@ class DataSetCalc(
     calcProgram: RexProgram,
     ruleDescription: String)
   extends Calc(cluster, traitSet, input, calcProgram)
-  with CommonCalc[Row]
+  with CommonCalc
   with DataSetRel {
 
   override def deriveRowType(): RelDataType = rowRelDataType
@@ -95,7 +96,8 @@ class DataSetCalc(
       new RowSchema(getInput.getRowType),
       new RowSchema(getRowType),
       calcProgram,
-      config)
+      config,
+      classOf[FlatMapFunction[Row, Row]])
 
     val runner = new FlatMapRunner(genFunction.name, genFunction.code, returnType)
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
index 49ead26a155d6..731d2e5bb6203 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCorrelate.scala
@@ -23,6 +23,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
 import org.apache.calcite.rex.{RexCall, RexNode}
 import org.apache.calcite.sql.SemiJoinType
+import org.apache.flink.api.common.functions.FlatMapFunction
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.DataSet
 import org.apache.flink.api.java.typeutils.RowTypeInfo
@@ -49,7 +50,7 @@ class DataSetCorrelate(
     joinType: SemiJoinType,
     ruleDescription: String)
   extends SingleRel(cluster, traitSet, inputNode)
-  with CommonCorrelate[Row]
+  with CommonCorrelate
   with DataSetRel {
 
   override def deriveRowType() = relRowType
@@ -109,18 +110,17 @@ class DataSetCorrelate(
       new RowSchema(getInput.getRowType),
       udtfTypeInfo,
       new RowSchema(getRowType),
-      rowType,
       joinType,
       rexCall,
       pojoFieldMapping,
-      ruleDescription)
+      ruleDescription,
+      classOf[FlatMapFunction[Row, Row]])
 
     val collector = generateCollector(
       config,
       new RowSchema(getInput.getRowType),
       udtfTypeInfo,
       new RowSchema(getRowType),
-      rowType,
       condition,
       pojoFieldMapping)
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
index 5f270f6986c32..f75efc80f074a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala
@@ -25,11 +25,12 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{RelNode, RelWriter}
 import org.apache.calcite.rex.RexProgram
 import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
-import org.apache.flink.table.plan.schema.RowSchema
 import org.apache.flink.table.codegen.CodeGenerator
 import org.apache.flink.table.plan.nodes.CommonCalc
-import org.apache.flink.table.runtime.CRowFlatMapRunner
+import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.runtime.CRowProcessRunner
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 /**
@@ -45,7 +46,7 @@ class DataStreamCalc(
     calcProgram: RexProgram,
     ruleDescription: String)
   extends Calc(cluster, traitSet, input, calcProgram)
-  with CommonCalc[CRow]
+  with CommonCalc
   with DataStreamRel {
 
   override def deriveRowType(): RelDataType = schema.logicalType
@@ -101,17 +102,18 @@ class DataStreamCalc(
       inputSchema,
       schema,
       calcProgram,
-      config)
+      config,
+      classOf[ProcessFunction[CRow, CRow]])
 
     val inputParallelism = inputDataStream.getParallelism
 
-    val mapFunc = new CRowFlatMapRunner(
+    val processFunc = new CRowProcessRunner(
       genFunction.name,
       genFunction.code,
       CRowTypeInfo(schema.physicalTypeInfo))
 
     inputDataStream
-      .flatMap(mapFunc)
+      .process(processFunc)
       .name(calcOpName(calcProgram, getExpressionString))
       // keep parallelism to ensure order of accumulate and retract messages
       .setParallelism(inputParallelism)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
index 5b32b1053a292..b7165cd190183 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
@@ -23,12 +23,13 @@ import org.apache.calcite.rex.{RexCall, RexNode}
 import org.apache.calcite.sql.SemiJoinType
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
 import org.apache.flink.table.functions.utils.TableSqlFunction
 import org.apache.flink.table.plan.nodes.CommonCorrelate
 import org.apache.flink.table.plan.nodes.logical.FlinkLogicalTableFunctionScan
 import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.table.runtime.CRowCorrelateFlatMapRunner
+import org.apache.flink.table.runtime.CRowCorrelateProcessRunner
 import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
 
 /**
@@ -46,7 +47,7 @@ class DataStreamCorrelate(
     joinType: SemiJoinType,
     ruleDescription: String)
   extends SingleRel(cluster, traitSet, input)
-  with CommonCorrelate[CRow]
+  with CommonCorrelate
   with DataStreamRel {
 
   override def deriveRowType() = schema.logicalType
@@ -90,7 +91,6 @@ class DataStreamCorrelate(
 
     // we do not need to specify input type
     val inputDS = getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv, queryConfig)
-    val inputType = inputDS.getType.asInstanceOf[CRowTypeInfo]
 
     val funcRel = scan.asInstanceOf[FlinkLogicalTableFunctionScan]
     val rexCall = funcRel.getCall.asInstanceOf[RexCall]
@@ -98,37 +98,36 @@ class DataStreamCorrelate(
     val pojoFieldMapping = Some(sqlFunction.getPojoFieldMapping)
     val udtfTypeInfo = sqlFunction.getRowTypeInfo.asInstanceOf[TypeInformation[Any]]
 
-    val flatMap = generateFunction(
+    val process = generateFunction(
       config,
       inputSchema,
       udtfTypeInfo,
       schema,
-      getRowType,
       joinType,
       rexCall,
       pojoFieldMapping,
-      ruleDescription)
+      ruleDescription,
+      classOf[ProcessFunction[CRow, CRow]])
 
     val collector = generateCollector(
       config,
       inputSchema,
       udtfTypeInfo,
       schema,
-      getRowType,
       condition,
       pojoFieldMapping)
 
-    val mapFunc = new CRowCorrelateFlatMapRunner(
-      flatMap.name,
-      flatMap.code,
+    val processFunc = new CRowCorrelateProcessRunner(
+      process.name,
+      process.code,
       collector.name,
       collector.code,
-      CRowTypeInfo(flatMap.returnType))
+      CRowTypeInfo(process.returnType))
 
     val inputParallelism = inputDS.getParallelism
 
     inputDS
-      .flatMap(mapFunc)
+      .process(processFunc)
       // preserve input parallelism to ensure that acc and retract messages remain in order
       .setParallelism(inputParallelism)
       .name(correlateOpName(rexCall, sqlFunction, schema.logicalType))
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
index 51e609f4fe3be..72ecac58243c4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/StreamTableSourceScan.scala
@@ -49,7 +49,7 @@ class StreamTableSourceScan(
     val fieldCnt = fieldNames.length
 
     val rowtime = tableSource match {
-      case timeSource: DefinedRowTimeAttribute if timeSource.getRowtimeAttribute != null =>
+      case timeSource: DefinedRowtimeAttribute if timeSource.getRowtimeAttribute != null =>
         val rowtimeAttribute = timeSource.getRowtimeAttribute
         Some((fieldCnt, rowtimeAttribute))
       case _ =>
@@ -57,7 +57,7 @@ class StreamTableSourceScan(
     }
 
     val proctime = tableSource match {
-      case timeSource: DefinedProcTimeAttribute if timeSource.getProctimeAttribute != null =>
+      case timeSource: DefinedProctimeAttribute if timeSource.getProctimeAttribute != null =>
         val proctimeAttribute = timeSource.getProctimeAttribute
         Some((fieldCnt + (if (rowtime.isDefined) 1 else 0), proctimeAttribute))
       case _ =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala
index 0ca079e7f03ef..ec903926ff747 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalCalc.scala
@@ -34,7 +34,7 @@ class FlinkLogicalCalc(
     calcProgram: RexProgram)
   extends Calc(cluster, traitSet, input, calcProgram)
   with FlinkLogicalRel
-  with CommonCalc[Any] {
+  with CommonCalc {
 
   override def copy(traitSet: RelTraitSet, child: RelNode, program: RexProgram): Calc = {
     new FlinkLogicalCalc(cluster, traitSet, child, program)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
index a2777ecde9b52..3ae949ef57516 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
@@ -29,7 +29,7 @@ import org.apache.flink.table.api.TableEnvironment
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.nodes.FlinkConventions
 import org.apache.flink.table.plan.schema.TableSourceTable
-import org.apache.flink.table.sources.{DefinedProcTimeAttribute, DefinedRowTimeAttribute, TableSource}
+import org.apache.flink.table.sources.{DefinedProctimeAttribute, DefinedRowtimeAttribute, TableSource}
 
 import scala.collection.JavaConverters._
 
@@ -54,7 +54,7 @@ class FlinkLogicalTableSourceScan(
     val fieldCnt = fieldNames.length
 
     val rowtime = tableSource match {
-      case timeSource: DefinedRowTimeAttribute if timeSource.getRowtimeAttribute != null =>
+      case timeSource: DefinedRowtimeAttribute if timeSource.getRowtimeAttribute != null =>
         val rowtimeAttribute = timeSource.getRowtimeAttribute
         Some((fieldCnt, rowtimeAttribute))
       case _ =>
@@ -62,7 +62,7 @@ class FlinkLogicalTableSourceScan(
     }
 
     val proctime = tableSource match {
-      case timeSource: DefinedProcTimeAttribute if timeSource.getProctimeAttribute != null =>
+      case timeSource: DefinedProctimeAttribute if timeSource.getProctimeAttribute != null =>
         val proctimeAttribute = timeSource.getProctimeAttribute
         Some((fieldCnt + (if (rowtime.isDefined) 1 else 0), proctimeAttribute))
       case _ =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
index 28efcf53152c0..d57d4cc0b75d7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
@@ -26,7 +26,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable
 import org.apache.flink.table.api.{TableException, Window}
 import org.apache.flink.table.api.scala.{Session, Slide, Tumble}
 import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.table.expressions.{Literal, UnresolvedFieldReference}
+import org.apache.flink.table.expressions.{Literal, ResolvedFieldReference, UnresolvedFieldReference}
 import org.apache.flink.table.plan.rules.common.LogicalWindowAggregateRule
 import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
 
@@ -68,10 +68,12 @@ class DataStreamLogicalWindowAggregateRule
         case _ => throw new TableException("Only constant window descriptors are supported.")
       }
 
-    def getOperandAsTimeIndicator(call: RexCall, idx: Int): String =
+    def getOperandAsTimeIndicator(call: RexCall, idx: Int): ResolvedFieldReference =
       call.getOperands.get(idx) match {
         case v: RexInputRef if FlinkTypeFactory.isTimeIndicatorType(v.getType) =>
-          rowType.getFieldList.get(v.getIndex).getName
+          ResolvedFieldReference(
+            rowType.getFieldList.get(v.getIndex).getName,
+            FlinkTypeFactory.toTypeInfo(v.getType))
         case _ =>
           throw new TableException("Window can only be defined over a time attribute column.")
       }
@@ -82,7 +84,7 @@ class DataStreamLogicalWindowAggregateRule
         val interval = getOperandAsLong(windowExpr, 1)
         val w = Tumble.over(Literal(interval, TimeIntervalTypeInfo.INTERVAL_MILLIS))
 
-        w.on(UnresolvedFieldReference(time)).as("w$")
+        w.on(time).as("w$")
 
       case SqlStdOperatorTable.HOP =>
         val time = getOperandAsTimeIndicator(windowExpr, 0)
@@ -91,14 +93,14 @@ class DataStreamLogicalWindowAggregateRule
           .over(Literal(size, TimeIntervalTypeInfo.INTERVAL_MILLIS))
           .every(Literal(slide, TimeIntervalTypeInfo.INTERVAL_MILLIS))
 
-        w.on(UnresolvedFieldReference(time)).as("w$")
+        w.on(time).as("w$")
 
       case SqlStdOperatorTable.SESSION =>
         val time = getOperandAsTimeIndicator(windowExpr, 0)
         val gap = getOperandAsLong(windowExpr, 1)
         val w = Session.withGap(Literal(gap, TimeIntervalTypeInfo.INTERVAL_MILLIS))
 
-        w.on(UnresolvedFieldReference(time)).as("w$")
+        w.on(time).as("w$")
     }
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala
index b42be827da99b..ccbe44d3af0ec 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/RowSchema.scala
@@ -20,11 +20,12 @@ package org.apache.flink.table.plan.schema
 
 import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeField, RelRecordType}
 import org.apache.calcite.rel.core.AggregateCall
-import org.apache.calcite.rex.{RexInputRef, RexNode, RexShuttle}
+import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode, RexShuttle}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.table.api.TableException
 import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.functions.TimeMaterializationSqlFunction
 import org.apache.flink.types.Row
 
 import scala.collection.JavaConversions._
@@ -76,6 +77,14 @@ class RowSchema(private val logicalRowType: RelDataType) {
     override def visitInputRef(inputRef: RexInputRef): RexNode = {
       new RexInputRef(mapIndex(inputRef.getIndex), inputRef.getType)
     }
+
+    override def visitCall(call: RexCall): RexNode = call.getOperator match {
+      // we leave time indicators unchanged yet
+      // the index becomes invalid but right now we are only
+      // interested in the type of the input reference
+      case TimeMaterializationSqlFunction => call
+      case _ => super.visitCall(call)
+    }
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala
index 75deca5d544ad..fa1528865112e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/StreamTableSourceTable.scala
@@ -19,10 +19,10 @@
 package org.apache.flink.table.plan.schema
 
 import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFactory}
-import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.{TableEnvironment, TableException}
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.stats.FlinkStatistic
-import org.apache.flink.table.sources.{DefinedProcTimeAttribute, DefinedRowTimeAttribute, TableSource}
+import org.apache.flink.table.sources.{DefinedProctimeAttribute, DefinedRowtimeAttribute, TableSource}
 
 class StreamTableSourceTable[T](
     override val tableSource: TableSource[T],
@@ -39,7 +39,7 @@ class StreamTableSourceTable[T](
     val fieldCnt = fieldNames.length
 
     val rowtime = tableSource match {
-      case timeSource: DefinedRowTimeAttribute if timeSource.getRowtimeAttribute != null =>
+      case timeSource: DefinedRowtimeAttribute if timeSource.getRowtimeAttribute != null =>
         val rowtimeAttribute = timeSource.getRowtimeAttribute
         Some((fieldCnt, rowtimeAttribute))
       case _ =>
@@ -47,7 +47,7 @@ class StreamTableSourceTable[T](
     }
 
     val proctime = tableSource match {
-      case timeSource: DefinedProcTimeAttribute if timeSource.getProctimeAttribute != null =>
+      case timeSource: DefinedProctimeAttribute if timeSource.getProctimeAttribute != null =>
         val proctimeAttribute = timeSource.getProctimeAttribute
         Some((fieldCnt + (if (rowtime.isDefined) 1 else 0), proctimeAttribute))
       case _ =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala
similarity index 72%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala
index ff3821afde318..4f0a78550794a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateFlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala
@@ -18,11 +18,11 @@
 
 package org.apache.flink.table.runtime
 
-import org.apache.flink.api.common.functions.{FlatMapFunction, RichFlatMapFunction}
 import org.apache.flink.api.common.functions.util.FunctionUtils
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.table.codegen.Compiler
 import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.types.Row
@@ -30,21 +30,21 @@ import org.apache.flink.util.Collector
 import org.slf4j.{Logger, LoggerFactory}
 
 /**
-  * A CorrelateFlatMapRunner with [[CRow]] input and [[CRow]] output.
+  * A CorrelateProcessRunner with [[CRow]] input and [[CRow]] output.
   */
-class CRowCorrelateFlatMapRunner(
-    flatMapName: String,
-    flatMapCode: String,
+class CRowCorrelateProcessRunner(
+    processName: String,
+    processCode: String,
     collectorName: String,
     collectorCode: String,
     @transient var returnType: TypeInformation[CRow])
-  extends RichFlatMapFunction[CRow, CRow]
+  extends ProcessFunction[CRow, CRow]
   with ResultTypeQueryable[CRow]
   with Compiler[Any] {
 
   val LOG: Logger = LoggerFactory.getLogger(this.getClass)
 
-  private var function: FlatMapFunction[Row, Row] = _
+  private var function: ProcessFunction[Row, Row] = _
   private var collector: TableFunctionCollector[_] = _
   private var cRowWrapper: CRowWrappingCollector = _
 
@@ -55,16 +55,21 @@ class CRowCorrelateFlatMapRunner(
     collector = clazz.newInstance().asInstanceOf[TableFunctionCollector[_]]
     this.cRowWrapper = new CRowWrappingCollector()
 
-    LOG.debug(s"Compiling FlatMapFunction: $flatMapName \n\n Code:\n$flatMapCode")
-    val flatMapClazz = compile(getRuntimeContext.getUserCodeClassLoader, flatMapName, flatMapCode)
-    val constructor = flatMapClazz.getConstructor(classOf[TableFunctionCollector[_]])
-    LOG.debug("Instantiating FlatMapFunction.")
-    function = constructor.newInstance(collector).asInstanceOf[FlatMapFunction[Row, Row]]
+    LOG.debug(s"Compiling ProcessFunction: $processName \n\n Code:\n$processCode")
+    val processClazz = compile(getRuntimeContext.getUserCodeClassLoader, processName, processCode)
+    val constructor = processClazz.getConstructor(classOf[TableFunctionCollector[_]])
+    LOG.debug("Instantiating ProcessFunction.")
+    function = constructor.newInstance(collector).asInstanceOf[ProcessFunction[Row, Row]]
     FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext)
     FunctionUtils.openFunction(function, parameters)
   }
 
-  override def flatMap(in: CRow, out: Collector[CRow]): Unit = {
+  override def processElement(
+      in: CRow,
+      ctx: ProcessFunction[CRow, CRow]#Context,
+      out: Collector[CRow])
+    : Unit = {
+
     cRowWrapper.out = out
     cRowWrapper.setChange(in.change)
 
@@ -72,7 +77,10 @@ class CRowCorrelateFlatMapRunner(
     collector.setInput(in.row)
     collector.reset()
 
-    function.flatMap(in.row, cRowWrapper)
+    function.processElement(
+      in.row,
+      ctx.asInstanceOf[ProcessFunction[Row, Row]#Context],
+      cRowWrapper)
   }
 
   override def getProducedType: TypeInformation[CRow] = returnType
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala
similarity index 76%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala
index 9701cb9b37813..cef62a517be2f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowFlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala
@@ -19,10 +19,10 @@
 package org.apache.flink.table.runtime
 
 import org.apache.flink.api.common.functions.util.FunctionUtils
-import org.apache.flink.api.common.functions.{FlatMapFunction, RichFlatMapFunction}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable
 import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.table.codegen.Compiler
 import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.types.Row
@@ -30,25 +30,25 @@ import org.apache.flink.util.Collector
 import org.slf4j.LoggerFactory
 
 /**
-  * FlatMapRunner with [[CRow]] input and [[CRow]] output.
+  * ProcessRunner with [[CRow]] input and [[CRow]] output.
   */
-class CRowFlatMapRunner(
+class CRowProcessRunner(
     name: String,
     code: String,
     @transient var returnType: TypeInformation[CRow])
-  extends RichFlatMapFunction[CRow, CRow]
+  extends ProcessFunction[CRow, CRow]
   with ResultTypeQueryable[CRow]
-  with Compiler[FlatMapFunction[Row, Row]] {
+  with Compiler[ProcessFunction[Row, Row]] {
 
   val LOG = LoggerFactory.getLogger(this.getClass)
 
-  private var function: FlatMapFunction[Row, Row] = _
+  private var function: ProcessFunction[Row, Row] = _
   private var cRowWrapper: CRowWrappingCollector = _
 
   override def open(parameters: Configuration): Unit = {
-    LOG.debug(s"Compiling FlatMapFunction: $name \n\n Code:\n$code")
+    LOG.debug(s"Compiling ProcessFunction: $name \n\n Code:\n$code")
     val clazz = compile(getRuntimeContext.getUserCodeClassLoader, name, code)
-    LOG.debug("Instantiating FlatMapFunction.")
+    LOG.debug("Instantiating ProcessFunction.")
     function = clazz.newInstance()
     FunctionUtils.setFunctionRuntimeContext(function, getRuntimeContext)
     FunctionUtils.openFunction(function, parameters)
@@ -56,10 +56,18 @@ class CRowFlatMapRunner(
     this.cRowWrapper = new CRowWrappingCollector()
   }
 
-  override def flatMap(in: CRow, out: Collector[CRow]): Unit = {
+  override def processElement(
+      in: CRow,
+      ctx: ProcessFunction[CRow, CRow]#Context,
+      out: Collector[CRow])
+    : Unit = {
+
     cRowWrapper.out = out
     cRowWrapper.setChange(in.change)
-    function.flatMap(in.row, cRowWrapper)
+    function.processElement(
+      in.row,
+      ctx.asInstanceOf[ProcessFunction[Row, Row]#Context],
+      cRowWrapper)
   }
 
   override def getProducedType: TypeInformation[CRow] = returnType
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/definedTimeAttributes.scala
similarity index 97%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
rename to flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/definedTimeAttributes.scala
index 6d87663416aa2..d38111517e706 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/DefinedTimeAttributes.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/definedTimeAttributes.scala
@@ -27,7 +27,7 @@ package org.apache.flink.table.sources
   * the event-time attribute. The attribute will be added to the schema of the
   * [[org.apache.flink.table.api.Table]] produced by the [[TableSource]].
   */
-trait DefinedRowTimeAttribute {
+trait DefinedRowtimeAttribute {
 
   /**
     * Defines a name of the event-time attribute that represents Flink's
@@ -47,7 +47,7 @@ trait DefinedRowTimeAttribute {
   * the processing-time attribute. The attribute will be added to the schema of the
   * [[org.apache.flink.table.api.Table]] produced by the [[TableSource]].
   */
-trait DefinedProcTimeAttribute {
+trait DefinedProctimeAttribute {
 
   /**
     * Defines a name of the processing-time attribute that represents Flink's
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala
index e9384c7ad6a36..7797f223d1ba9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/StreamTableEnvironmentTest.scala
@@ -19,18 +19,16 @@
 package org.apache.flink.table.api.scala.stream
 
 import java.lang.{Integer => JInt, Long => JLong}
-import java.util.Collections
-import java.util.{List => JList}
 
 import org.apache.flink.api.java.tuple.{Tuple5 => JTuple5}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
 import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
 import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JStreamExecEnv}
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
-import org.apache.flink.table.api.{TableEnvironment, TableException, Types}
 import org.apache.flink.table.api.java.{StreamTableEnvironment => JStreamTableEnv}
 import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.{TableEnvironment, TableException, Types}
 import org.apache.flink.table.utils.TableTestBase
 import org.junit.Test
 import org.mockito.Mockito.{mock, when}
@@ -151,7 +149,9 @@ class StreamTableEnvironmentTest extends TableTestBase {
   private def prepareSchemaExpressionParser:
     (JStreamTableEnv, DataStream[JTuple5[JLong, JInt, String, JInt, JLong]]) = {
 
-    val jTEnv = TableEnvironment.getTableEnvironment(mock(classOf[JStreamExecEnv]))
+    val jStreamExecEnv = mock(classOf[JStreamExecEnv])
+    when(jStreamExecEnv.getStreamTimeCharacteristic).thenReturn(TimeCharacteristic.EventTime)
+    val jTEnv = TableEnvironment.getTableEnvironment(jStreamExecEnv)
 
     val sType = new TupleTypeInfo(Types.LONG, Types.INT, Types.STRING, Types.INT, Types.LONG)
       .asInstanceOf[TupleTypeInfo[JTuple5[JLong, JInt, String, JInt, JLong]]]
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
index 18066c95cc2b9..cda90f732882e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/TableSourceTest.scala
@@ -24,7 +24,7 @@ import org.apache.flink.streaming.api.datastream.DataStream
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.apache.flink.table.api.Types
 import org.apache.flink.table.api.scala._
-import org.apache.flink.table.sources.{DefinedProcTimeAttribute, DefinedRowTimeAttribute, StreamTableSource}
+import org.apache.flink.table.sources.{DefinedProctimeAttribute, DefinedRowtimeAttribute, StreamTableSource}
 import org.apache.flink.table.utils.TableTestBase
 import org.apache.flink.table.utils.TableTestUtil.{term, unaryNode}
 import org.apache.flink.types.Row
@@ -35,7 +35,7 @@ class TableSourceTest extends TableTestBase {
   @Test
   def testRowTimeTableSourceSimple(): Unit = {
     val util = streamTestUtil()
-    util.tEnv.registerTableSource("rowTimeT", new TestRowTimeSource("addTime"))
+    util.tEnv.registerTableSource("rowTimeT", new TestRowtimeSource("addTime"))
 
     val t = util.tEnv.scan("rowTimeT").select("addTime, id, name, val")
 
@@ -43,7 +43,7 @@ class TableSourceTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         "StreamTableSourceScan(table=[[rowTimeT]], fields=[id, val, name, addTime])",
-        term("select", "addTime", "id", "name", "val")
+        term("select", "TIME_MATERIALIZATION(addTime) AS addTime", "id", "name", "val")
       )
     util.verifyTable(t, expected)
   }
@@ -51,7 +51,7 @@ class TableSourceTest extends TableTestBase {
   @Test
   def testRowTimeTableSourceGroupWindow(): Unit = {
     val util = streamTestUtil()
-    util.tEnv.registerTableSource("rowTimeT", new TestRowTimeSource("addTime"))
+    util.tEnv.registerTableSource("rowTimeT", new TestRowtimeSource("addTime"))
 
     val t = util.tEnv.scan("rowTimeT")
       .filter("val > 100")
@@ -82,7 +82,7 @@ class TableSourceTest extends TableTestBase {
   @Test
   def testProcTimeTableSourceSimple(): Unit = {
     val util = streamTestUtil()
-    util.tEnv.registerTableSource("procTimeT", new TestProcTimeSource("pTime"))
+    util.tEnv.registerTableSource("procTimeT", new TestProctimeSource("pTime"))
 
     val t = util.tEnv.scan("procTimeT").select("pTime, id, name, val")
 
@@ -90,7 +90,7 @@ class TableSourceTest extends TableTestBase {
       unaryNode(
         "DataStreamCalc",
         "StreamTableSourceScan(table=[[procTimeT]], fields=[id, val, name, pTime])",
-        term("select", "pTime", "id", "name", "val")
+        term("select", "TIME_MATERIALIZATION(pTime) AS pTime", "id", "name", "val")
       )
     util.verifyTable(t, expected)
   }
@@ -98,7 +98,7 @@ class TableSourceTest extends TableTestBase {
   @Test
   def testProcTimeTableSourceOverWindow(): Unit = {
     val util = streamTestUtil()
-    util.tEnv.registerTableSource("procTimeT", new TestProcTimeSource("pTime"))
+    util.tEnv.registerTableSource("procTimeT", new TestProctimeSource("pTime"))
 
     val t = util.tEnv.scan("procTimeT")
       .window(Over partitionBy 'id orderBy 'pTime preceding 2.hours as 'w)
@@ -123,8 +123,8 @@ class TableSourceTest extends TableTestBase {
   }
 }
 
-class TestRowTimeSource(timeField: String)
-    extends StreamTableSource[Row] with DefinedRowTimeAttribute {
+class TestRowtimeSource(timeField: String)
+    extends StreamTableSource[Row] with DefinedRowtimeAttribute {
 
   override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = ???
 
@@ -137,8 +137,8 @@ class TestRowTimeSource(timeField: String)
   }
 }
 
-class TestProcTimeSource(timeField: String)
-    extends StreamTableSource[Row] with DefinedProcTimeAttribute {
+class TestProctimeSource(timeField: String)
+    extends StreamTableSource[Row] with DefinedProctimeAttribute {
 
   override def getDataStream(execEnv: StreamExecutionEnvironment): DataStream[Row] = ???
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverterTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverterTest.scala
new file mode 100644
index 0000000000000..7ac08740d0e28
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverterTest.scala
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.calcite
+
+import java.sql.Timestamp
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.calcite.RelTimeIndicatorConverterTest.TableFunc
+import org.apache.flink.table.expressions.{TimeIntervalUnit, WindowReference}
+import org.apache.flink.table.functions.TableFunction
+import org.apache.flink.table.plan.logical.TumblingGroupWindow
+import org.apache.flink.table.utils.TableTestBase
+import org.apache.flink.table.utils.TableTestUtil._
+import org.junit.Test
+
+/**
+  * Tests for [[RelTimeIndicatorConverter]].
+  */
+class RelTimeIndicatorConverterTest extends TableTestBase {
+
+  @Test
+  def testSimpleMaterialization(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]('rowtime.rowtime, 'long, 'int, 'proctime.proctime)
+
+    val result = t
+      .select('rowtime.floor(TimeIntervalUnit.DAY) as 'rowtime, 'long)
+      .filter('long > 0)
+      .select('rowtime)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      streamTableNode(0),
+      term("select", "FLOOR(TIME_MATERIALIZATION(rowtime)", "FLAG(DAY)) AS rowtime"),
+      term("where", ">(long, 0)")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testSelectAll(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]('rowtime.rowtime, 'long, 'int, 'proctime.proctime)
+
+    val result = t.select('*)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      streamTableNode(0),
+      term("select", "TIME_MATERIALIZATION(rowtime) AS rowtime", "long", "int",
+        "TIME_MATERIALIZATION(proctime) AS proctime")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testFilteringOnRowtime(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]('rowtime.rowtime, 'long, 'int)
+
+    val result = t
+      .filter('rowtime > "1990-12-02 12:11:11".toTimestamp)
+      .select('rowtime)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      streamTableNode(0),
+      term("select", "TIME_MATERIALIZATION(rowtime) AS rowtime"),
+      term("where", ">(TIME_MATERIALIZATION(rowtime), 1990-12-02 12:11:11)")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testGroupingOnRowtime(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]('rowtime.rowtime, 'long, 'int, 'proctime.proctime)
+
+    val result = t
+      .groupBy('rowtime)
+      .select('long.count)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamGroupAggregate",
+        unaryNode(
+          "DataStreamCalc",
+          streamTableNode(0),
+          term("select", "long", "TIME_MATERIALIZATION(rowtime) AS rowtime")
+        ),
+        term("groupBy", "rowtime"),
+        term("select", "rowtime", "COUNT(long) AS TMP_0")
+      ),
+      term("select", "TMP_0")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testGroupingOnProctimeSql(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int)]("MyTable" , 'long, 'int, 'proctime.proctime)
+
+    val result = util.tEnv.sql("SELECT COUNT(long) FROM MyTable GROUP BY proctime")
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamGroupAggregate",
+        unaryNode(
+          "DataStreamCalc",
+          streamTableNode(0),
+          term("select", "TIME_MATERIALIZATION(proctime) AS proctime", "long")
+        ),
+        term("groupBy", "proctime"),
+        term("select", "proctime", "COUNT(long) AS EXPR$0")
+      ),
+      term("select", "EXPR$0")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testAggregationOnRowtime(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]('rowtime.rowtime, 'long, 'int)
+
+    val result = t
+      .groupBy('long)
+      .select('rowtime.min)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamGroupAggregate",
+        unaryNode(
+          "DataStreamCalc",
+          streamTableNode(0),
+          term("select", "TIME_MATERIALIZATION(rowtime) AS rowtime", "long")
+        ),
+        term("groupBy", "long"),
+        term("select", "long", "MIN(rowtime) AS TMP_0")
+      ),
+      term("select", "TMP_0")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testAggregationOnProctimeSql(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Int)]("MyTable" , 'long, 'int, 'proctime.proctime)
+
+    val result = util.tEnv.sql("SELECT MIN(proctime) FROM MyTable GROUP BY long")
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamGroupAggregate",
+        unaryNode(
+          "DataStreamCalc",
+          streamTableNode(0),
+          term("select", "long", "TIME_MATERIALIZATION(proctime) AS proctime")
+        ),
+        term("groupBy", "long"),
+        term("select", "long", "MIN(proctime) AS EXPR$0")
+      ),
+      term("select", "EXPR$0")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testTableFunction(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]('rowtime.rowtime, 'long, 'int, 'proctime.proctime)
+    val func = new TableFunc
+
+    val result = t.join(func('rowtime, 'proctime) as 's).select('rowtime, 'proctime, 's)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamCorrelate",
+        streamTableNode(0),
+        term("invocation",
+          s"${func.functionIdentifier}(TIME_MATERIALIZATION($$0), TIME_MATERIALIZATION($$3))"),
+        term("function", func),
+        term("rowType", "RecordType(TIMESTAMP(3) rowtime, BIGINT long, INTEGER int, " +
+          "TIMESTAMP(3) proctime, VARCHAR(2147483647) s)"),
+        term("joinType", "INNER")
+      ),
+      term("select",
+        "TIME_MATERIALIZATION(rowtime) AS rowtime",
+        "TIME_MATERIALIZATION(proctime) AS proctime",
+        "s")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testWindow(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]('rowtime.rowtime, 'long, 'int)
+
+    val result = t
+      .window(Tumble over 100.millis on 'rowtime as 'w)
+      .groupBy('w, 'long)
+      .select('w.end as 'rowtime, 'long, 'int.sum)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamGroupWindowAggregate",
+        streamTableNode(0),
+        term("groupBy", "long"),
+        term(
+          "window",
+          TumblingGroupWindow(
+            WindowReference("w"),
+            'rowtime,
+            100.millis)),
+        term("select", "long", "SUM(int) AS TMP_1", "end(WindowReference(w)) AS TMP_0")
+      ),
+      term("select", "TMP_0 AS rowtime", "long", "TMP_1")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testWindowSql(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Long, Int)]("MyTable", 'rowtime.rowtime, 'long, 'int)
+
+    val result = util.tEnv.sql(
+      "SELECT TUMBLE_END(rowtime, INTERVAL '0.1' SECOND) AS `rowtime`, `long`, " +
+        "SUM(`int`) FROM MyTable " +
+        "GROUP BY `long`, TUMBLE(rowtime, INTERVAL '0.1' SECOND)")
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamGroupWindowAggregate",
+        streamTableNode(0),
+        term("groupBy", "long"),
+        term(
+          "window",
+          TumblingGroupWindow(
+            'w$,
+            'rowtime,
+            100.millis)),
+        term("select", "long", "SUM(int) AS EXPR$2", "start('w$) AS w$start", "end('w$) AS w$end")
+      ),
+      term("select", "w$end", "long", "EXPR$2")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testWindowWithAggregationOnRowtimeSql(): Unit = {
+    val util = streamTestUtil()
+    util.addTable[(Long, Long, Int)]("MyTable", 'rowtime.rowtime, 'long, 'int)
+
+    val result = util.tEnv.sql("SELECT MIN(rowtime), long FROM MyTable " +
+      "GROUP BY long, TUMBLE(rowtime, INTERVAL '0.1' SECOND)")
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      unaryNode(
+        "DataStreamGroupWindowAggregate",
+        unaryNode(
+          "DataStreamCalc",
+          streamTableNode(0),
+          term("select", "long", "1970-01-01 00:00:00 AS $f1",
+            "TIME_MATERIALIZATION(rowtime) AS $f2")
+        ),
+        term("groupBy", "long"),
+        term(
+          "window",
+          TumblingGroupWindow(
+            'w$,
+            'rowtime,
+            100.millis)),
+        term("select", "long", "MIN($f2) AS EXPR$0")
+      ),
+      term("select", "EXPR$0", "long")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+  @Test
+  def testUnion(): Unit = {
+    val util = streamTestUtil()
+    val t = util.addTable[(Long, Long, Int)]("MyTable", 'rowtime.rowtime, 'long, 'int)
+
+    val result = t.unionAll(t).select('rowtime)
+
+    val expected = unaryNode(
+      "DataStreamCalc",
+      binaryNode(
+        "DataStreamUnion",
+        streamTableNode(0),
+        streamTableNode(0),
+        term("union all", "rowtime", "long", "int")
+      ),
+      term("select", "TIME_MATERIALIZATION(rowtime) AS rowtime")
+    )
+
+    util.verifyTable(result, expected)
+  }
+
+}
+
+object RelTimeIndicatorConverterTest {
+
+  class TableFunc extends TableFunction[String] {
+    val t = new Timestamp(0L)
+    def eval(time1: Long, time2: Timestamp): Unit = {
+      collect(time1.toString + time2.after(t))
+    }
+  }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/TimeAttributesITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/TimeAttributesITCase.scala
new file mode 100644
index 0000000000000..7d7088e7018e0
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/TimeAttributesITCase.scala
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.datastream
+
+import java.math.BigDecimal
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.scala.stream.utils.StreamITCase
+import org.apache.flink.table.api.{TableEnvironment, TableException, Types}
+import org.apache.flink.table.calcite.RelTimeIndicatorConverterTest.TableFunc
+import org.apache.flink.table.expressions.TimeIntervalUnit
+import org.apache.flink.table.runtime.datastream.TimeAttributesITCase.TimestampWithEqualWatermark
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+/**
+  * Tests for access and materialization of time attributes.
+  */
+class TimeAttributesITCase extends StreamingMultipleProgramsTestBase {
+
+  val data = List(
+    (1L, 1, 1d, 1f, new BigDecimal("1"), "Hi"),
+    (2L, 2, 2d, 2f, new BigDecimal("2"), "Hallo"),
+    (3L, 2, 2d, 2f, new BigDecimal("2"), "Hello"),
+    (4L, 5, 5d, 5f, new BigDecimal("5"), "Hello"),
+    (7L, 3, 3d, 3f, new BigDecimal("3"), "Hello"),
+    (8L, 3, 3d, 3f, new BigDecimal("3"), "Hello world"),
+    (16L, 4, 4d, 4f, new BigDecimal("4"), "Hello world"))
+
+  @Test(expected = classOf[TableException])
+  def testInvalidTimeCharacteristic(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    val stream = env
+      .fromCollection(data)
+      .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+    stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+  }
+
+  @Test
+  def testCalcMaterialization(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env
+      .fromCollection(data)
+      .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+    val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+    val t = table.select('rowtime.cast(Types.STRING))
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq(
+      "1970-01-01 00:00:00.001",
+      "1970-01-01 00:00:00.002",
+      "1970-01-01 00:00:00.003",
+      "1970-01-01 00:00:00.004",
+      "1970-01-01 00:00:00.007",
+      "1970-01-01 00:00:00.008",
+      "1970-01-01 00:00:00.016")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testCalcMaterialization2(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env
+      .fromCollection(data)
+      .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+    val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+    val t = table
+      .filter('rowtime.cast(Types.LONG) > 4)
+      .select('rowtime, 'rowtime.floor(TimeIntervalUnit.DAY), 'rowtime.ceil(TimeIntervalUnit.DAY))
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq(
+      "1970-01-01 00:00:00.007,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0",
+      "1970-01-01 00:00:00.008,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0",
+      "1970-01-01 00:00:00.016,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testTableFunction(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env
+      .fromCollection(data)
+      .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+    val table = stream.toTable(
+      tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string, 'proctime.proctime)
+    val func = new TableFunc
+
+    val t = table.join(func('rowtime, 'proctime) as 's).select('rowtime, 's)
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq(
+      "1970-01-01 00:00:00.001,1true",
+      "1970-01-01 00:00:00.002,2true",
+      "1970-01-01 00:00:00.003,3true",
+      "1970-01-01 00:00:00.004,4true",
+      "1970-01-01 00:00:00.007,7true",
+      "1970-01-01 00:00:00.008,8true",
+      "1970-01-01 00:00:00.016,16true")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testUnion(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env
+      .fromCollection(data)
+      .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+    val table = stream.toTable(
+      tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+    val t = table.unionAll(table).select('rowtime)
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq(
+      "1970-01-01 00:00:00.001",
+      "1970-01-01 00:00:00.001",
+      "1970-01-01 00:00:00.002",
+      "1970-01-01 00:00:00.002",
+      "1970-01-01 00:00:00.003",
+      "1970-01-01 00:00:00.003",
+      "1970-01-01 00:00:00.004",
+      "1970-01-01 00:00:00.004",
+      "1970-01-01 00:00:00.007",
+      "1970-01-01 00:00:00.007",
+      "1970-01-01 00:00:00.008",
+      "1970-01-01 00:00:00.008",
+      "1970-01-01 00:00:00.016",
+      "1970-01-01 00:00:00.016")
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+  @Test
+  def testWindowWithAggregationOnRowtimeSql(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = TableEnvironment.getTableEnvironment(env)
+    StreamITCase.testResults = mutable.MutableList()
+
+    val stream = env
+      .fromCollection(data)
+      .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+    val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+    tEnv.registerTable("MyTable", table)
+
+    val t = tEnv.sql("SELECT COUNT(`rowtime`) FROM MyTable " +
+      "GROUP BY TUMBLE(rowtime, INTERVAL '0.003' SECOND)")
+
+    val results = t.toDataStream[Row]
+    results.addSink(new StreamITCase.StringSink)
+    env.execute()
+
+    val expected = Seq(
+      "1",
+      "2",
+      "2",
+      "2"
+    )
+    assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+  }
+
+}
+
+object TimeAttributesITCase {
+  class TimestampWithEqualWatermark
+  extends AssignerWithPunctuatedWatermarks[(Long, Int, Double, Float, BigDecimal, String)] {
+
+    override def checkAndGetNextWatermark(
+        lastElement: (Long, Int, Double, Float, BigDecimal, String),
+        extractedTimestamp: Long)
+      : Watermark = {
+      new Watermark(extractedTimestamp)
+    }
+
+    override def extractTimestamp(
+        element: (Long, Int, Double, Float, BigDecimal, String),
+        previousElementTimestamp: Long): Long = {
+      element._1
+    }
+  }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
index 0e6d4617b874a..65014cd8ee474 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala
@@ -24,9 +24,11 @@ import org.apache.flink.api.java.{DataSet => JDataSet}
 import org.apache.flink.table.api.{Table, TableEnvironment}
 import org.apache.flink.table.api.scala._
 import org.apache.flink.api.scala.{DataSet, ExecutionEnvironment}
+import org.apache.flink.streaming.api.TimeCharacteristic
 import org.apache.flink.table.expressions.Expression
 import org.apache.flink.table.functions.{ScalarFunction, TableFunction}
 import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream}
+import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JStreamExecutionEnvironment}
 import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
 import org.apache.flink.table.api.scala.batch.utils.LogicalPlanFormatUtils
 import org.junit.Assert.assertEquals
@@ -174,7 +176,10 @@ case class BatchTableTestUtil() extends TableTestUtil {
 
 case class StreamTableTestUtil() extends TableTestUtil {
 
+  val javaEnv = mock(classOf[JStreamExecutionEnvironment])
+  when(javaEnv.getStreamTimeCharacteristic).thenReturn(TimeCharacteristic.EventTime)
   val env = mock(classOf[StreamExecutionEnvironment])
+  when(env.getWrappedStreamExecutionEnvironment).thenReturn(javaEnv)
   val tEnv = TableEnvironment.getTableEnvironment(env)
 
   def addTable[T: TypeInformation](

From 178296f2613547f2ac8198cc37f2e6f91352db09 Mon Sep 17 00:00:00 2001
From: Fabian Hueske <fhueske@apache.org>
Date: Fri, 12 May 2017 14:25:48 +0200
Subject: [PATCH 233/253] [FLINK-6564] [build] Fix copying of license files.

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 4cc38956385dc..db19b9d710912 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1132,7 +1132,7 @@ under the License.
 							<goal>copy-resources</goal>
 						</goals>
 						<configuration>
-							<outputDirectory>${basedir}/target/classes/META-INF/license/</outputDirectory>
+							<outputDirectory>${basedir}/target/classes/META-INF/licenses/</outputDirectory>
 							<resources>
 								<resource>
 									<directory>${basedir}/packaged_licenses</directory>

From 5c1e075e1e0627bea857bf6b1fa55f8d2c5dd968 Mon Sep 17 00:00:00 2001
From: Greg Hogan <code@greghogan.com>
Date: Thu, 2 Feb 2017 13:23:04 -0500
Subject: [PATCH 234/253] [hotfix] Minutiae

---
 .../org/apache/flink/client/CliFrontend.java  |  2 +-
 .../io/avro/AvroRecordInputFormatTest.java    | 26 ++++++----
 .../connectors/fs/SequenceFileWriter.java     |  2 +-
 .../accumulators/AccumulatorHelper.java       |  2 +-
 .../api/java/typeutils/TypeExtractor.java     |  2 +-
 .../AbstractGenericArraySerializerTest.java   | 16 +++----
 .../AbstractGenericTypeComparatorTest.java    |  6 +--
 .../AbstractGenericTypeSerializerTest.java    |  2 +-
 .../runtime/kryo/SerializersTest.java         |  2 +-
 .../apache/flink/types/BasicTypeInfoTest.java |  9 ++--
 .../flink/api/java/operators/Grouping.java    |  2 +-
 .../java/functions/SelectByFunctionsTest.java | 18 +++----
 .../java/functions/SemanticPropUtilTest.java  | 32 ++++++-------
 .../traversals/GraphCreatingVisitor.java      |  4 +-
 .../runtime/util/SerializedThrowable.java     | 10 ++--
 .../scheduler/SchedulerSlotSharingTest.java   |  2 +-
 .../api/operator/SelectByFunctionTest.scala   | 18 +++----
 .../operators/AbstractStreamOperatorTest.java | 47 ++++++++++---------
 .../runtime/io/BarrierTrackerTest.java        |  2 +-
 .../test/accumulators/AccumulatorITCase.java  |  2 +-
 20 files changed, 107 insertions(+), 99 deletions(-)

diff --git a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java
index 74d5f5de0b28d..62fa402e03956 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java
@@ -860,7 +860,7 @@ protected int executeProgram(PackagedProgram program, ClusterClient client, int
 			Map<String, Object> accumulatorsResult = execResult.getAllAccumulatorResults();
 			if (accumulatorsResult.size() > 0) {
 				System.out.println("Accumulator Results: ");
-				System.out.println(AccumulatorHelper.getResultsFormated(accumulatorsResult));
+				System.out.println(AccumulatorHelper.getResultsFormatted(accumulatorsResult));
 			}
 		} else {
 			logAndSysout("Job has been submitted with JobID " + result.getJobID());
diff --git a/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
index 91a9612361f7e..3b6ad63d13983 100644
--- a/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
+++ b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
@@ -18,14 +18,6 @@
 
 package org.apache.flink.api.io.avro;
 
-import static org.junit.Assert.*;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-
 import org.apache.avro.Schema;
 import org.apache.avro.file.DataFileReader;
 import org.apache.avro.file.DataFileWriter;
@@ -59,6 +51,22 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test the avro input format.
  * (The testcase is mostly the getting started tutorial of avro)
@@ -273,7 +281,7 @@ public void testDeserializeToGenericType() throws IOException {
 		DatumReader<GenericData.Record> datumReader = new GenericDatumReader<>(userSchema);
 
 		try (FileReader<GenericData.Record> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
-			// initialize Record by reading it from disk (thats easier than creating it by hand)
+			// initialize Record by reading it from disk (that's easier than creating it by hand)
 			GenericData.Record rec = new GenericData.Record(userSchema);
 			dataFileReader.next(rec);
 			
diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/SequenceFileWriter.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/SequenceFileWriter.java
index 08c0d0afcf22d..32cadecdb6fb2 100644
--- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/SequenceFileWriter.java
+++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/SequenceFileWriter.java
@@ -38,7 +38,7 @@
 /**
  * A {@link Writer} that writes the bucket files as Hadoop {@link SequenceFile SequenceFiles}.
  * The input to the {@link BucketingSink} must
- * be a {@link org.apache.flink.api.java.tuple.Tuple2} of two Hadopo
+ * be a {@link org.apache.flink.api.java.tuple.Tuple2} of two Hadoop
  * {@link org.apache.hadoop.io.Writable Writables}.
  *
  * @param <K> The type of the first tuple field.
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
index 1a87235f442fc..3282302446e3a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
@@ -112,7 +112,7 @@ public static Map<String, Object> toResultMap(Map<String, Accumulator<?, ?>> acc
 		return resultMap;
 	}
 
-	public static String getResultsFormated(Map<String, Object> map) {
+	public static String getResultsFormatted(Map<String, Object> map) {
 		StringBuilder builder = new StringBuilder();
 		for (Map.Entry<String, Object> entry : map.entrySet()) {
 			builder
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index f1bf9575bb41f..112ca38c8a2ce 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -394,7 +394,7 @@ public static <IN, OUT> TypeInformation<OUT> getUnaryOperatorReturnType(
 	 * @param inputTypeArgumentIndex Index of the type argument of function's first parameter
 	 *                               specifying the input type if it is wrapped (Iterable, Map,
 	 *                               etc.). Otherwise -1.
-	 * @param outputTypeArgumentIndex Index of the type argument of functions second parameter
+	 * @param outputTypeArgumentIndex Index of the type argument of function's second parameter
 	 *                                specifying the output type if it is wrapped in a Collector.
 	 *                                Otherwise -1.
 	 * @param inType Type of the input elements (In case of an iterable, it is the element type)
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericArraySerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericArraySerializerTest.java
index 307a3d2170c81..7386e6ef65273 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericArraySerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericArraySerializerTest.java
@@ -18,12 +18,6 @@
 
 package org.apache.flink.api.java.typeutils.runtime;
 
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.Random;
-
-import org.junit.Assert;
-import org.junit.Test;
 import org.apache.flink.api.common.typeutils.SerializerTestInstance;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.base.GenericArraySerializer;
@@ -34,6 +28,12 @@
 import org.apache.flink.api.java.typeutils.runtime.AbstractGenericTypeSerializerTest.ComplexNestedObject2;
 import org.apache.flink.api.java.typeutils.runtime.AbstractGenericTypeSerializerTest.SimpleTypes;
 import org.apache.flink.util.StringUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Random;
 
 public abstract class AbstractGenericArraySerializerTest {
 	
@@ -110,11 +110,11 @@ public void testNestedObjects() {
 	@Test
 	public void testBeanStyleObjects() {
 		{
-			Book b1 = new Book(976243875L, "The Serialization Odysse", 42);
+			Book b1 = new Book(976243875L, "The Serialization Odyssey", 42);
 			Book b2 = new Book(0L, "Debugging byte streams", 1337);
 			Book b3 = new Book(-1L, "Low level interfaces", 0xC0FFEE);
 			Book b4 = new Book(Long.MAX_VALUE, "The joy of bits and bytes", 0xDEADBEEF);
-			Book b5 = new Book(Long.MIN_VALUE, "Winnign a prize for creative test strings", 0xBADF00);
+			Book b5 = new Book(Long.MIN_VALUE, "Winning a prize for creative test strings", 0xBADF00);
 			Book b6 = new Book(-2L, "Distributed Systems", 0xABCDEF0123456789L);
 			
 			runTests(	new Book[] {b1, b2},
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeComparatorTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeComparatorTest.java
index ef168d8ddde55..44464a33d638e 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeComparatorTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeComparatorTest.java
@@ -30,16 +30,14 @@ abstract public class AbstractGenericTypeComparatorTest {
 
 	@Test
 	public void testString() {
-		runTests(new String[]{
-				"",
+		runTests("",
 				"Lorem Ipsum Dolor Omit Longer",
 				"aaaa",
 				"abcd",
 				"abce",
 				"abdd",
 				"accd",
-				"bbcd"
-		});
+				"bbcd");
 	}
 
 	@Test
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeSerializerTest.java
index 059c78d6d2810..a75208a4da078 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/AbstractGenericTypeSerializerTest.java
@@ -88,7 +88,7 @@ public void testNestedObjects() {
 	@Test
 	public void testBeanStyleObjects() {
 		{
-			Book b1 = new Book(976243875L, "The Serialization Odysse", 42);
+			Book b1 = new Book(976243875L, "The Serialization Odyssey", 42);
 			Book b2 = new Book(0L, "Debugging byte streams", 1337);
 			Book b3 = new Book(-1L, "Low level interfaces", 0xC0FFEE);
 
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/SerializersTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/SerializersTest.java
index 7c6d02354a6a6..4a39f8f6d2acf 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/SerializersTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/SerializersTest.java
@@ -46,7 +46,7 @@ public static class FromGeneric2 {}
 	
 	public static class Nested1 {
 		private FromNested fromNested;
-		private Path yodaIntervall;
+		private Path yodaInterval;
 	}
 
 	public static class ClassWithNested {
diff --git a/flink-core/src/test/java/org/apache/flink/types/BasicTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/types/BasicTypeInfoTest.java
index c090b76161ba5..5707701e070d0 100644
--- a/flink-core/src/test/java/org/apache/flink/types/BasicTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/types/BasicTypeInfoTest.java
@@ -11,22 +11,23 @@
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WNTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 
 package org.apache.flink.types;
 
-import java.math.BigDecimal;
-import java.math.BigInteger;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.util.TestLogger;
 import org.junit.Test;
 
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.util.Date;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 
 public class BasicTypeInfoTest extends TestLogger {
 
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
index 8601c335cb512..dbaaa9d52c426 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
@@ -29,7 +29,7 @@
  * Grouping is an intermediate step for a transformation on a grouped DataSet.<br>
  * The following transformation can be applied on Grouping:
  * <ul>
- * 	<li>{@link UnsortedGrouping#reduce(org.apache.flink.api.common.functions.ReduceFunction)},</li>
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.common.functions.ReduceFunction)},</li>
  * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.common.functions.GroupReduceFunction)}, and</li>
  * <li>{@link UnsortedGrouping#aggregate(org.apache.flink.api.java.aggregation.Aggregations, int)}.</li>
  * </ul>
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/functions/SelectByFunctionsTest.java b/flink-java/src/test/java/org/apache/flink/api/java/functions/SelectByFunctionsTest.java
index f40c0cacb0f7e..52b59ec97a50f 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/functions/SelectByFunctionsTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/functions/SelectByFunctionsTest.java
@@ -49,7 +49,7 @@ public void testMaxByComparison() {
 			Assert.assertSame("SelectByMax must return bigger tuple", bigger, maxByTuple.reduce(smaller, bigger));
 			Assert.assertSame("SelectByMax must return bigger tuple", bigger, maxByTuple.reduce(bigger, smaller));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -67,7 +67,7 @@ public void testMaxByComparisonSpecialCase1() {
 			Assert.assertSame("SelectByMax must return the first given tuple", specialCaseBigger, maxByTuple.reduce(specialCaseBigger, bigger));
 			Assert.assertSame("SelectByMax must return the first given tuple", bigger, maxByTuple.reduce(bigger, specialCaseBigger));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -82,7 +82,7 @@ public void testMaxByComparisonSpecialCase2() {
 			Assert.assertSame("SelectByMax must return bigger tuple", bigger, maxByTuple.reduce(specialCaseBigger, bigger));
 			Assert.assertSame("SelectByMax must return bigger tuple", bigger, maxByTuple.reduce(bigger, specialCaseBigger));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -97,7 +97,7 @@ public void testMaxByComparisonMultiple() {
 			Assert.assertSame("SelectByMax must return bigger tuple", bigger, maxByTuple.reduce(smaller, bigger));
 			Assert.assertSame("SelectByMax must return bigger tuple", bigger, maxByTuple.reduce(bigger, smaller));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -112,7 +112,7 @@ public void testMaxByComparisonMustReturnATuple() {
 			Assert.assertSame("SelectByMax must return bigger tuple", bigger, maxByTuple.reduce(bigger, bigger));
 			Assert.assertSame("SelectByMax must return smaller tuple", smaller, maxByTuple.reduce(smaller, smaller));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -129,7 +129,7 @@ public void testMinByComparison() {
 			Assert.assertSame("SelectByMin must return smaller tuple", smaller, minByTuple.reduce(smaller, bigger));
 			Assert.assertSame("SelectByMin must return smaller tuple", smaller, minByTuple.reduce(bigger, smaller));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -145,7 +145,7 @@ public void testMinByComparisonSpecialCase1() {
 			Assert.assertSame("SelectByMin must return the first given tuple", specialCaseBigger, minByTuple.reduce(specialCaseBigger, bigger));
 			Assert.assertSame("SelectByMin must return the first given tuple", bigger, minByTuple.reduce(bigger, specialCaseBigger));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -161,7 +161,7 @@ public void testMinByComparisonSpecialCase2() {
 			Assert.assertSame("SelectByMin must return smaller tuple", smaller, minByTuple.reduce(specialCaseSmaller, smaller));
 			Assert.assertSame("SelectByMin must return smaller tuple", smaller, minByTuple.reduce(smaller, specialCaseSmaller));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
@@ -176,7 +176,7 @@ public void testMinByComparisonMultiple() {
 			Assert.assertSame("SelectByMin must return smaller tuple", smaller, minByTuple.reduce(smaller, bigger));
 			Assert.assertSame("SelectByMin must return smaller tuple", smaller, minByTuple.reduce(bigger, smaller));
 		} catch (Exception e) {
-			Assert.fail("No exception should be thrown while comapring both tuples");
+			Assert.fail("No exception should be thrown while comparing both tuples");
 		}
 	}
 	
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java b/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java
index b845e738b26c0..453c0227fd4ac 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java
@@ -324,7 +324,7 @@ public void testAddSourceFieldOffsets() {
 	// --------------------------------------------------------------------------------------------
 
 	@Test
-	public void testForwardedNoArrrowIndividualStrings() {
+	public void testForwardedNoArrowIndividualStrings() {
 		String[] forwardedFields = {"f2","f3","f0"};
 		SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsSingleFromString(sp, forwardedFields, null, null, fiveIntTupleType, fiveIntTupleType);
@@ -335,7 +335,7 @@ public void testForwardedNoArrrowIndividualStrings() {
 	}
 
 	@Test
-	public void testForwardedNoArrrowOneString() {
+	public void testForwardedNoArrowOneString() {
 		String[] forwardedFields = {"f2;f3;f0"};
 		SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsSingleFromString(sp, forwardedFields, null, null, fiveIntTupleType, fiveIntTupleType);
@@ -360,7 +360,7 @@ public void testForwardedNoArrrowOneString() {
 	}
 
 	@Test
-	public void testForwardedNoArrrowSpaces() {
+	public void testForwardedNoArrowSpaces() {
 		String[] forwardedFields = {"  f2  ;   f3  ;  f0   "};
 		SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsSingleFromString(sp, forwardedFields, null, null, fiveIntTupleType, fiveIntTupleType);
@@ -1219,11 +1219,11 @@ public void testForwardedDual() {
 
 	@Test
 	public void testNonForwardedDual() {
-		String[] nonNorwardedFieldsFirst = { "f1;f2" };
-		String[] nonNorwardedFieldsSecond = { "f0" };
+		String[] nonForwardedFieldsFirst = { "f1;f2" };
+		String[] nonForwardedFieldsSecond = { "f0" };
 		DualInputSemanticProperties dsp = new DualInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
-				nonNorwardedFieldsFirst, nonNorwardedFieldsSecond, null, null, threeIntTupleType, threeIntTupleType, threeIntTupleType);
+				nonForwardedFieldsFirst, nonForwardedFieldsSecond, null, null, threeIntTupleType, threeIntTupleType, threeIntTupleType);
 
 		assertTrue(dsp.getForwardingTargetFields(0, 0).contains(0));
 		assertTrue(dsp.getForwardingTargetFields(0, 1).size() == 0);
@@ -1232,11 +1232,11 @@ public void testNonForwardedDual() {
 		assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1));
 		assertTrue(dsp.getForwardingTargetFields(1, 2).contains(2));
 
-		nonNorwardedFieldsFirst[0] = "f1";
-		nonNorwardedFieldsSecond[0] = "";
+		nonForwardedFieldsFirst[0] = "f1";
+		nonForwardedFieldsSecond[0] = "";
 		dsp = new DualInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
-				nonNorwardedFieldsFirst, null, null, null, threeIntTupleType, fiveIntTupleType, threeIntTupleType);
+				nonForwardedFieldsFirst, null, null, null, threeIntTupleType, fiveIntTupleType, threeIntTupleType);
 
 		assertTrue(dsp.getForwardingTargetFields(0, 0).contains(0));
 		assertTrue(dsp.getForwardingTargetFields(0, 1).size() == 0);
@@ -1245,11 +1245,11 @@ public void testNonForwardedDual() {
 		assertTrue(dsp.getForwardingTargetFields(1, 1).size() == 0);
 		assertTrue(dsp.getForwardingTargetFields(1, 2).size() == 0);
 
-		nonNorwardedFieldsFirst[0] = "";
-		nonNorwardedFieldsSecond[0] = "f2;f0";
+		nonForwardedFieldsFirst[0] = "";
+		nonForwardedFieldsSecond[0] = "f2;f0";
 		dsp = new DualInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
-				null, nonNorwardedFieldsSecond, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType);
+				null, nonForwardedFieldsSecond, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType);
 
 		assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
 		assertTrue(dsp.getForwardingTargetFields(0, 1).size() == 0);
@@ -1283,19 +1283,19 @@ public void testNonForwardedDual() {
 	@Test(expected = InvalidSemanticAnnotationException.class)
 	public void testNonForwardedDualInvalidTypes1() {
 
-		String[] nonNorwardedFieldsFirst = { "f1" };
+		String[] nonForwardedFieldsFirst = { "f1" };
 		DualInputSemanticProperties dsp = new DualInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
-				nonNorwardedFieldsFirst, null, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType);
+				nonForwardedFieldsFirst, null, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType);
 	}
 
 	@Test(expected = InvalidSemanticAnnotationException.class)
 	public void testNonForwardedDualInvalidTypes2() {
 
-		String[] nonNorwardedFieldsSecond = { "f1" };
+		String[] nonForwardedFieldsSecond = { "f1" };
 		DualInputSemanticProperties dsp = new DualInputSemanticProperties();
 		SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
-				null, nonNorwardedFieldsSecond, null, null, threeIntTupleType, pojoInTupleType, threeIntTupleType);
+				null, nonForwardedFieldsSecond, null, null, threeIntTupleType, pojoInTupleType, threeIntTupleType);
 	}
 
 	@Test
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
index 3f3eae17fce19..92217b58cf19d 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
@@ -76,8 +76,8 @@
 /**
  * This traversal creates the optimizer DAG from a program.
  * It works as a visitor that walks the program's flow in a depth-first fashion, starting from the data sinks.
- * During the descend, it creates an optimizer node for each operator, respectively data source or -sink.
- * During the ascend, it connects the nodes to the full graph.
+ * During the descent it creates an optimizer node for each operator, respectively data source or sink.
+ * During the ascent it connects the nodes to the full graph.
  */
 public class GraphCreatingVisitor implements Visitor<Operator<?>> {
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
index f6f08e429a5d0..63f4363e13e82 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedThrowable.java
@@ -48,7 +48,7 @@ public class SerializedThrowable extends Exception implements Serializable {
 	private final String originalErrorClassName;
 	
 	/** The original stack trace, to be printed */
-	private final String fullStingifiedStackTrace;
+	private final String fullStringifiedStackTrace;
 
 	/** The original exception, not transported via serialization, 
 	 * because the class may not be part of the system class loader.
@@ -83,7 +83,7 @@ private SerializedThrowable(Throwable exception, Set<Throwable> alreadySeen) {
 
 			// record the original exception's properties (name, stack prints)
 			this.originalErrorClassName = exception.getClass().getName();
-			this.fullStingifiedStackTrace = ExceptionUtils.stringifyException(exception);
+			this.fullStringifiedStackTrace = ExceptionUtils.stringifyException(exception);
 
 			// mimic the original exception's stack trace
 			setStackTrace(exception.getStackTrace());
@@ -106,7 +106,7 @@ private SerializedThrowable(Throwable exception, Set<Throwable> alreadySeen) {
 			SerializedThrowable other = (SerializedThrowable) exception;
 			this.serializedException = other.serializedException;
 			this.originalErrorClassName = other.originalErrorClassName;
-			this.fullStingifiedStackTrace = other.fullStingifiedStackTrace;
+			this.fullStringifiedStackTrace = other.fullStringifiedStackTrace;
 			this.cachedException = other.cachedException;
 			this.setStackTrace(other.getStackTrace());
 			this.initCause(other.getCause());
@@ -145,13 +145,13 @@ public String getOriginalErrorClassName() {
 
 	@Override
 	public void printStackTrace(PrintStream s) {
-		s.print(fullStingifiedStackTrace);
+		s.print(fullStringifiedStackTrace);
 		s.flush();
 	}
 	
 	@Override
 	public void printStackTrace(PrintWriter s) {
-		s.print(fullStingifiedStackTrace);
+		s.print(fullStringifiedStackTrace);
 		s.flush();
 	}
 	
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java
index 5238e95602dc2..c049593262b47 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java
@@ -320,7 +320,7 @@ public void allocateSlotWithIntermediateTotallyEmptySharingGroup() {
 	}
 	
 	@Test
-	public void allocateSlotWithTemprarilyEmptyVertexGroup() {
+	public void allocateSlotWithTemporarilyEmptyVertexGroup() {
 		try {
 			JobVertexID jid1 = new JobVertexID();
 			JobVertexID jid2 = new JobVertexID();
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/operator/SelectByFunctionTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/operator/SelectByFunctionTest.scala
index 291df79c32e15..d6af6bd11f3fd 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/operator/SelectByFunctionTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/operator/SelectByFunctionTest.scala
@@ -54,7 +54,7 @@ class SelectByFunctionTest {
           bigger, maxByTuple.reduce(bigger, smaller))
       } catch {
         case e : Exception =>
-          Assert.fail("No exception should be thrown while comapring both tuples")
+          Assert.fail("No exception should be thrown while comparing both tuples")
       }
   }
 
@@ -77,7 +77,7 @@ class SelectByFunctionTest {
         bigger, maxByTuple.reduce(bigger, specialCaseBigger))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown " +
-        "while comapring both tuples")
+        "while comparing both tuples")
     }
   }
 
@@ -95,7 +95,7 @@ class SelectByFunctionTest {
         bigger, maxByTuple.reduce(bigger, specialCaseBigger))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown" +
-        " while comapring both tuples")
+        " while comparing both tuples")
     }
   }
 
@@ -113,7 +113,7 @@ class SelectByFunctionTest {
         bigger, maxByTuple.reduce(bigger, smaller))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown " +
-        "while comapring both tuples")
+        "while comparing both tuples")
     }
   }
 
@@ -132,7 +132,7 @@ class SelectByFunctionTest {
         smaller, maxByTuple.reduce(smaller, smaller))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown" +
-        " while comapring both tuples")
+        " while comparing both tuples")
     }
   }
 
@@ -153,7 +153,7 @@ class SelectByFunctionTest {
         smaller, minByTuple.reduce(bigger, smaller))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown " +
-        "while comapring both tuples")
+        "while comparing both tuples")
     }
   }
 
@@ -174,7 +174,7 @@ class SelectByFunctionTest {
         bigger, minByTuple.reduce(bigger, specialCaseBigger))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown " +
-        "while comapring both tuples")
+        "while comparing both tuples")
     }
   }
 
@@ -195,7 +195,7 @@ class SelectByFunctionTest {
         smaller, minByTuple.reduce(smaller, specialCaseSmaller))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown" +
-        " while comapring both tuples")
+        " while comparing both tuples")
     }
   }
 
@@ -213,7 +213,7 @@ class SelectByFunctionTest {
         smaller, minByTuple.reduce(bigger, smaller))
     } catch {
       case e : Exception => Assert.fail("No exception should be thrown" +
-        " while comapring both tuples")
+        " while comparing both tuples")
     }
   }
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
index 774ab4a893cd5..67004ea03b6cd 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java
@@ -17,28 +17,6 @@
  */
 package org.apache.flink.streaming.api.operators;
 
-import static junit.framework.TestCase.assertTrue;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.contains;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.doReturn;
-import static org.powermock.api.mockito.PowerMockito.mock;
-import static org.powermock.api.mockito.PowerMockito.spy;
-import static org.powermock.api.mockito.PowerMockito.when;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.RunnableFuture;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeutils.base.IntSerializer;
@@ -71,6 +49,29 @@
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.RunnableFuture;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.doReturn;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.spy;
+import static org.powermock.api.mockito.PowerMockito.when;
+import static org.powermock.api.mockito.PowerMockito.whenNew;
+
 /**
  * Tests for the facilities provided by {@link AbstractStreamOperator}. This mostly
  * tests timers and state and whether they are correctly checkpointed/restored
@@ -510,7 +511,7 @@ public void testSnapshotMethod() throws Exception {
 
 	/**
 	 * Tests that the created StateSnapshotContextSynchronousImpl is closed in case of a failing
-	 * Operator#snapshotState(StaetSnapshotContextSynchronousImpl) call.
+	 * Operator#snapshotState(StateSnapshotContextSynchronousImpl) call.
 	 */
 	@Test
 	public void testFailingSnapshotMethod() throws Exception {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java
index cbcf45dbc06a7..8c6620550e0f8 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierTrackerTest.java
@@ -526,7 +526,7 @@ public void abortCheckpointOnBarrier(long checkpointId, Throwable cause) {
 
 			final long expectedId = checkpointIDs[i++];
 			if (expectedId < 0) {
-				assertEquals("wrong checkpoint id for checkoint abort", -expectedId, checkpointId);
+				assertEquals("wrong checkpoint id for checkpoint abort", -expectedId, checkpointId);
 			} else {
 				fail("got 'abortCheckpointOnBarrier()' when expecting an 'triggerCheckpointOnBarrier()'");
 			}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java
index b4015e5994c34..5f2b0a95dbb00 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java
@@ -74,7 +74,7 @@ protected void postSubmit() throws Exception {
 		// Test accumulator results
 		System.out.println("Accumulator results:");
 		JobExecutionResult res = this.result;
-		System.out.println(AccumulatorHelper.getResultsFormated(res.getAllAccumulatorResults()));
+		System.out.println(AccumulatorHelper.getResultsFormatted(res.getAllAccumulatorResults()));
 
 		Assert.assertEquals(Integer.valueOf(3), (Integer) res.getAccumulatorResult("num-lines"));
 

From 9d3f1a229a4c46af7e3c39518621d3b4f39bdcaf Mon Sep 17 00:00:00 2001
From: Haohui Mai <wheat9@apache.org>
Date: Thu, 11 May 2017 15:35:39 -0700
Subject: [PATCH 235/253] [FLINK-6562] [table] Support implicit table
 references for nested fields in SQL.

This closes #3879.
---
 .../plan/schema/CompositeRelDataType.scala    |  4 ++--
 .../expressions/CompositeAccessTest.scala     | 22 ++++++++++++++++++-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/CompositeRelDataType.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/CompositeRelDataType.scala
index 92f9199b0f831..a60514bc2f65b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/CompositeRelDataType.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/schema/CompositeRelDataType.scala
@@ -20,7 +20,7 @@ package org.apache.flink.table.plan.schema
 
 import java.util
 
-import org.apache.calcite.rel.`type`.{RelDataTypeField, RelDataTypeFieldImpl, RelRecordType}
+import org.apache.calcite.rel.`type`.{RelDataTypeField, RelDataTypeFieldImpl, RelRecordType, StructKind}
 import org.apache.flink.api.common.typeutils.CompositeType
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.schema.CompositeRelDataType.createFieldList
@@ -36,7 +36,7 @@ import scala.collection.JavaConverters._
 class CompositeRelDataType(
     val compositeType: CompositeType[_],
     typeFactory: FlinkTypeFactory)
-  extends RelRecordType(createFieldList(compositeType, typeFactory)) {
+  extends RelRecordType(StructKind.PEEK_FIELDS, createFieldList(compositeType, typeFactory)) {
 
   override def toString = s"COMPOSITE($compositeType)"
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala
index 202588003ac52..91e06c5cfec2f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala
@@ -41,10 +41,13 @@ class CompositeAccessTest extends ExpressionTestBase {
       "f0.get('intField')",
       "testTable.f0.intField",
       "42")
+    testSqlApi("f0.intField", "42")
 
     testSqlApi("testTable.f0.stringField", "Bob")
+    testSqlApi("f0.stringField", "Bob")
 
     testSqlApi("testTable.f0.booleanField", "true")
+    testSqlApi("f0.booleanField", "true")
 
     // single field by int key
     testTableApi(
@@ -58,22 +61,29 @@ class CompositeAccessTest extends ExpressionTestBase {
       "f1.get('objectField').get('intField')",
       "testTable.f1.objectField.intField",
       "25")
+    testSqlApi("f1.objectField.intField", "25")
 
     testSqlApi("testTable.f1.objectField.stringField", "Timo")
+    testSqlApi("f1.objectField.stringField", "Timo")
 
     testSqlApi("testTable.f1.objectField.booleanField", "false")
+    testSqlApi("f1.objectField.booleanField", "false")
 
     testAllApis(
       'f2.get(0),
       "f2.get(0)",
       "testTable.f2._1",
       "a")
+    testSqlApi("f2._1", "a")
 
     testSqlApi("testTable.f3.f1", "b")
+    testSqlApi("f3.f1", "b")
 
     testSqlApi("testTable.f4.myString", "Hello")
+    testSqlApi("f4.myString", "Hello")
 
     testSqlApi("testTable.f5", "13")
+    testSqlApi("f5", "13")
 
     testAllApis(
       'f7.get("_1"),
@@ -83,18 +93,21 @@ class CompositeAccessTest extends ExpressionTestBase {
 
     // composite field return type
     testSqlApi("testTable.f6", "MyCaseClass2(null)")
+    testSqlApi("f6", "MyCaseClass2(null)")
 
     testAllApis(
       'f1.get("objectField"),
       "f1.get('objectField')",
       "testTable.f1.objectField",
       "MyCaseClass(25,Timo,false)")
+    testSqlApi("f1.objectField", "MyCaseClass(25,Timo,false)")
 
     testAllApis(
       'f0,
       "f0",
       "testTable.f0",
       "MyCaseClass(42,Bob,true)")
+    testSqlApi("f0", "MyCaseClass(42,Bob,true)")
 
     // flattening (test base only returns first column)
     testAllApis(
@@ -102,12 +115,14 @@ class CompositeAccessTest extends ExpressionTestBase {
       "f1.get('objectField').flatten()",
       "testTable.f1.objectField.*",
       "25")
+    testSqlApi("f1.objectField.*", "25")
 
     testAllApis(
       'f0.flatten(),
       "flatten(f0)",
       "testTable.f0.*",
       "42")
+    testSqlApi("f0.*", "42")
 
     testTableApi(12.flatten(), "12.flatten()", "12")
 
@@ -115,10 +130,15 @@ class CompositeAccessTest extends ExpressionTestBase {
   }
 
   @Test(expected = classOf[ValidationException])
-  def testWrongSqlField(): Unit = {
+  def testWrongSqlFieldFull(): Unit = {
     testSqlApi("testTable.f5.test", "13")
   }
 
+  @Test(expected = classOf[ValidationException])
+  def testWrongSqlField(): Unit = {
+    testSqlApi("f5.test", "13")
+  }
+
   @Test(expected = classOf[ValidationException])
   def testWrongIntKeyField(): Unit = {
     testTableApi('f0.get(555), "'fail'", "fail")

From 4d65d537974f52dc934972ea3d578e9d6ee8793e Mon Sep 17 00:00:00 2001
From: DmytroShkvyra <dshkvyra@gmail.com>
Date: Wed, 5 Apr 2017 12:24:30 +0300
Subject: [PATCH 236/253] [FLINK-5256] [table] Extend DataSetSingleRowJoin to
 support Left and Right joins.

This closes #3673.
---
 .../nodes/dataset/DataSetSingleRowJoin.scala  |  57 ++-
 .../plan/nodes/logical/FlinkLogicalJoin.scala |  13 +-
 .../dataSet/DataSetSingleRowJoinRule.scala    |  10 +-
 .../table/runtime/MapJoinLeftRunner.scala     |   4 +
 .../table/runtime/MapJoinRightRunner.scala    |   4 +
 .../batch/sql/DataSetSingleRowJoinTest.scala  | 428 ++++++++++++++++++
 .../batch/sql/DistinctAggregateTest.scala     |   6 +-
 .../api/scala/batch/sql/JoinITCase.scala      | 181 +++++++-
 .../scala/batch/sql/SingleRowJoinTest.scala   | 195 --------
 9 files changed, 671 insertions(+), 227 deletions(-)
 create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DataSetSingleRowJoinTest.scala
 delete mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SingleRowJoinTest.scala

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSingleRowJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSingleRowJoin.scala
index b7d1a4bfb60c6..e92dec59397e7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSingleRowJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSingleRowJoin.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.plan.nodes.dataset
 
 import org.apache.calcite.plan._
 import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.core.JoinRelType
 import org.apache.calcite.rel.metadata.RelMetadataQuery
 import org.apache.calcite.rel.{BiRel, RelNode, RelWriter}
 import org.apache.calcite.rex.RexNode
@@ -47,6 +48,7 @@ class DataSetSingleRowJoin(
     rowRelDataType: RelDataType,
     joinCondition: RexNode,
     joinRowType: RelDataType,
+    joinType: JoinRelType,
     ruleDescription: String)
   extends BiRel(cluster, traitSet, leftNode, rightNode)
   with DataSetRel {
@@ -63,6 +65,7 @@ class DataSetSingleRowJoin(
       getRowType,
       joinCondition,
       joinRowType,
+      joinType,
       ruleDescription)
   }
 
@@ -97,7 +100,6 @@ class DataSetSingleRowJoin(
       tableEnv.getConfig,
       leftDataSet.getType,
       rightDataSet.getType,
-      leftIsSingle,
       joinCondition,
       broadcastSetName)
 
@@ -118,14 +120,18 @@ class DataSetSingleRowJoin(
       config: TableConfig,
       inputType1: TypeInformation[Row],
       inputType2: TypeInformation[Row],
-      firstIsSingle: Boolean,
       joinCondition: RexNode,
       broadcastInputSetName: String)
     : FlatMapFunction[Row, Row] = {
 
+    val isOuterJoin = joinType match {
+      case JoinRelType.LEFT | JoinRelType.RIGHT => true
+      case _ => false
+    }    
+    
     val codeGenerator = new CodeGenerator(
       config,
-      false,
+      isOuterJoin,
       inputType1,
       Some(inputType2))
 
@@ -138,13 +144,38 @@ class DataSetSingleRowJoin(
     val condition = codeGenerator.generateExpression(joinCondition)
 
     val joinMethodBody =
-      s"""
-        |${condition.code}
-        |if (${condition.resultTerm}) {
-        |  ${conversion.code}
-        |  ${codeGenerator.collectorTerm}.collect(${conversion.resultTerm});
-        |}
-        |""".stripMargin
+      if (joinType == JoinRelType.INNER) {
+        s"""
+         |${condition.code}
+         |if (${condition.resultTerm}) {
+         |  ${conversion.code}
+         |  ${codeGenerator.collectorTerm}.collect(${conversion.resultTerm});
+         |}
+         |""".stripMargin
+    } else {
+      val singleNode =
+        if (!leftIsSingle) {
+          rightNode
+        }
+        else {
+          leftNode
+        }
+
+      val notSuitedToCondition = singleNode
+        .getRowType
+        .getFieldList
+        .map(field => getRowType.getFieldNames.indexOf(field.getName))
+        .map(i => s"${conversion.resultTerm}.setField($i,null);")
+
+        s"""
+           |${condition.code}
+           |${conversion.code}
+           |if(!${condition.resultTerm}){
+           |${notSuitedToCondition.mkString("\n")}
+           |}
+           |${codeGenerator.collectorTerm}.collect(${conversion.resultTerm});
+           |""".stripMargin
+    }
 
     val genFunction = codeGenerator.generateFunction(
       ruleDescription,
@@ -152,16 +183,18 @@ class DataSetSingleRowJoin(
       joinMethodBody,
       returnType)
 
-    if (firstIsSingle) {
+    if (leftIsSingle) {
       new MapJoinRightRunner[Row, Row, Row](
         genFunction.name,
         genFunction.code,
+        isOuterJoin,
         genFunction.returnType,
         broadcastInputSetName)
     } else {
       new MapJoinLeftRunner[Row, Row, Row](
         genFunction.name,
         genFunction.code,
+        isOuterJoin,
         genFunction.returnType,
         broadcastInputSetName)
     }
@@ -181,7 +214,7 @@ class DataSetSingleRowJoin(
   }
 
   private def joinTypeToString: String = {
-    "NestedLoopJoin"
+    "NestedLoop" + joinType.toString.toLowerCase.capitalize + "Join"
   }
 
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalJoin.scala
index 8df0b59c828c6..33c4caf2a8780 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalJoin.scala
@@ -77,7 +77,7 @@ private class FlinkLogicalJoinConverter
     val join: LogicalJoin = call.rel(0).asInstanceOf[LogicalJoin]
     val joinInfo = join.analyzeCondition
 
-    hasEqualityPredicates(join, joinInfo) || isSingleRowInnerJoin(join)
+    hasEqualityPredicates(join, joinInfo) || isSingleRowJoin(join)
   }
 
   override def convert(rel: RelNode): RelNode = {
@@ -101,11 +101,12 @@ private class FlinkLogicalJoinConverter
     !joinInfo.pairs().isEmpty && (joinInfo.isEqui || join.getJoinType == JoinRelType.INNER)
   }
 
-  private def isSingleRowInnerJoin(join: LogicalJoin): Boolean = {
-    if (join.getJoinType == JoinRelType.INNER) {
-      isSingleRow(join.getRight) || isSingleRow(join.getLeft)
-    } else {
-      false
+  private def isSingleRowJoin(join: LogicalJoin): Boolean = {
+    join.getJoinType match {
+      case JoinRelType.INNER if isSingleRow(join.getRight) || isSingleRow(join.getLeft) => true
+      case JoinRelType.LEFT if isSingleRow(join.getRight) => true
+      case JoinRelType.RIGHT if isSingleRow(join.getLeft) => true
+      case _ => false
     }
   }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala
index b61573c53965d..f964a95f3d521 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala
@@ -37,10 +37,11 @@ class DataSetSingleRowJoinRule
   override def matches(call: RelOptRuleCall): Boolean = {
     val join = call.rel(0).asInstanceOf[FlinkLogicalJoin]
 
-    if (isInnerJoin(join)) {
-      isSingleRow(join.getRight) || isSingleRow(join.getLeft)
-    } else {
-      false
+    join.getJoinType match {
+      case JoinRelType.INNER if isSingleRow(join.getLeft) || isSingleRow(join.getRight) => true
+      case JoinRelType.LEFT if isSingleRow(join.getRight) => true
+      case JoinRelType.RIGHT if isSingleRow(join.getLeft) => true
+      case _ => false
     }
   }
 
@@ -79,6 +80,7 @@ class DataSetSingleRowJoinRule
       rel.getRowType,
       join.getCondition,
       join.getRowType,
+      join.getJoinType,
       description)
   }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala
index 5f3dbb4cbaf98..c906b63d6acea 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinLeftRunner.scala
@@ -19,11 +19,13 @@
 package org.apache.flink.table.runtime
 
 import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.types.Row
 import org.apache.flink.util.Collector
 
 class MapJoinLeftRunner[IN1, IN2, OUT](
     name: String,
     code: String,
+    outerJoin: Boolean,
     returnType: TypeInformation[OUT],
     broadcastSetName: String)
   extends MapSideJoinRunner[IN1, IN2, IN2, IN1, OUT](name, code, returnType, broadcastSetName) {
@@ -31,7 +33,9 @@ class MapJoinLeftRunner[IN1, IN2, OUT](
   override def flatMap(multiInput: IN1, out: Collector[OUT]): Unit = {
     broadcastSet match {
       case Some(singleInput) => function.join(multiInput, singleInput, out)
+      case None if outerJoin => function.join(multiInput, null.asInstanceOf[IN2], out)
       case None =>
     }
   }
+
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala
index e2d9331187623..0cb4edadb26e3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapJoinRightRunner.scala
@@ -19,11 +19,13 @@
 package org.apache.flink.table.runtime
 
 import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.types.Row
 import org.apache.flink.util.Collector
 
 class MapJoinRightRunner[IN1, IN2, OUT](
     name: String,
     code: String,
+    outerJoin: Boolean,
     returnType: TypeInformation[OUT],
     broadcastSetName: String)
   extends MapSideJoinRunner[IN1, IN2, IN1, IN2, OUT](name, code, returnType, broadcastSetName) {
@@ -31,7 +33,9 @@ class MapJoinRightRunner[IN1, IN2, OUT](
   override def flatMap(multiInput: IN2, out: Collector[OUT]): Unit = {
     broadcastSet match {
       case Some(singleInput) => function.join(singleInput, multiInput, out)
+      case None if outerJoin => function.join(null.asInstanceOf[IN1], multiInput, out)
       case None =>
     }
   }
+
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DataSetSingleRowJoinTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DataSetSingleRowJoinTest.scala
new file mode 100644
index 0000000000000..1f7ce84abae70
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DataSetSingleRowJoinTest.scala
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.batch.sql
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.utils.TableTestBase
+import org.apache.flink.table.utils.TableTestUtil._
+import org.junit.Test
+
+class DataSetSingleRowJoinTest extends TableTestBase {
+
+  @Test
+  def testSingleRowCrossJoin(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Int, Int)]("A", 'a1, 'a2)
+
+    val query =
+      "SELECT a1, asum " +
+      "FROM A, (SELECT sum(a1) + sum(a2) AS asum FROM A)"
+
+    val expected =
+      binaryNode(
+        "DataSetSingleRowJoin",
+        unaryNode(
+          "DataSetCalc",
+          batchTableNode(0),
+          term("select", "a1")
+        ),
+        unaryNode(
+          "DataSetCalc",
+          unaryNode(
+            "DataSetAggregate",
+            unaryNode(
+              "DataSetUnion",
+              unaryNode(
+                "DataSetValues",
+                batchTableNode(0),
+                tuples(List(null, null)),
+                term("values", "a1", "a2")
+              ),
+              term("union","a1","a2")
+            ),
+            term("select", "SUM(a1) AS $f0", "SUM(a2) AS $f1")
+          ),
+          term("select", "+($f0, $f1) AS asum")
+        ),
+        term("where", "true"),
+        term("join", "a1", "asum"),
+        term("joinType", "NestedLoopInnerJoin")
+      )
+
+    util.verifySql(query, expected)
+  }
+
+  @Test
+  def testSingleRowEquiJoin(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Int, String)]("A", 'a1, 'a2)
+
+    val query =
+      "SELECT a1, a2 " +
+      "FROM A, (SELECT count(a1) AS cnt FROM A) " +
+      "WHERE a1 = cnt"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        binaryNode(
+          "DataSetSingleRowJoin",
+          batchTableNode(0),
+          unaryNode(
+            "DataSetAggregate",
+            unaryNode(
+              "DataSetUnion",
+              unaryNode(
+                "DataSetValues",
+                unaryNode(
+                  "DataSetCalc",
+                  batchTableNode(0),
+                  term("select", "a1")
+                ),
+                tuples(List(null)),
+                term("values", "a1")
+              ),
+              term("union","a1")
+            ),
+            term("select", "COUNT(a1) AS cnt")
+          ),
+          term("where", "=(CAST(a1), cnt)"),
+          term("join", "a1", "a2", "cnt"),
+          term("joinType", "NestedLoopInnerJoin")
+        ),
+        term("select", "a1", "a2")
+      )
+
+    util.verifySql(query, expected)
+  }
+
+  @Test
+  def testSingleRowNotEquiJoin(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Int, String)]("A", 'a1, 'a2)
+
+    val query =
+      "SELECT a1, a2 " +
+      "FROM A, (SELECT count(a1) AS cnt FROM A) " +
+      "WHERE a1 < cnt"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        binaryNode(
+          "DataSetSingleRowJoin",
+          batchTableNode(0),
+          unaryNode(
+            "DataSetAggregate",
+            unaryNode(
+              "DataSetUnion",
+              unaryNode(
+                "DataSetValues",
+                unaryNode(
+                  "DataSetCalc",
+                  batchTableNode(0),
+                  term("select", "a1")
+                ),
+                tuples(List(null)),
+                term("values", "a1")
+              ),
+              term("union", "a1")
+            ),
+            term("select", "COUNT(a1) AS cnt")
+          ),
+          term("where", "<(a1, cnt)"),
+          term("join", "a1", "a2", "cnt"),
+          term("joinType", "NestedLoopInnerJoin")
+        ),
+        term("select", "a1", "a2")
+      )
+
+    util.verifySql(query, expected)
+  }
+
+  @Test
+  def testSingleRowJoinWithComplexPredicate(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Int, Long)]("A", 'a1, 'a2)
+    util.addTable[(Int, Long)]("B", 'b1, 'b2)
+
+    val query =
+      "SELECT a1, a2, b1, b2 " +
+        "FROM A, (SELECT min(b1) AS b1, max(b2) AS b2 FROM B) " +
+        "WHERE a1 < b1 AND a2 = b2"
+
+    val expected = binaryNode(
+      "DataSetSingleRowJoin",
+      batchTableNode(0),
+      unaryNode(
+        "DataSetAggregate",
+        unaryNode(
+          "DataSetUnion",
+          unaryNode(
+            "DataSetValues",
+            batchTableNode(1),
+            tuples(List(null, null)),
+            term("values", "b1", "b2")
+          ),
+          term("union","b1","b2")
+        ),
+        term("select", "MIN(b1) AS b1", "MAX(b2) AS b2")
+      ),
+      term("where", "AND(<(a1, b1)", "=(a2, b2))"),
+      term("join", "a1", "a2", "b1", "b2"),
+      term("joinType", "NestedLoopInnerJoin")
+    )
+
+    util.verifySql(query, expected)
+  }
+
+  @Test
+  def testRightSingleLeftJoinEqualPredicate(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Long, Int)]("A", 'a1, 'a2)
+    util.addTable[(Int, Int)]("B", 'b1, 'b2)
+
+    val queryLeftJoin =
+      "SELECT a2 " +
+        "FROM A " +
+        "  LEFT JOIN " +
+        "(SELECT COUNT(*) AS cnt FROM B) AS x " +
+        "  ON a1 = cnt"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        unaryNode(
+          "DataSetSingleRowJoin",
+          batchTableNode(0),
+          term("where", "=(a1, cnt)"),
+          term("join", "a1", "a2", "cnt"),
+          term("joinType", "NestedLoopLeftJoin")
+        ),
+        term("select", "a2")
+      ) + "\n" +
+        unaryNode(
+          "DataSetAggregate",
+          unaryNode(
+            "DataSetUnion",
+            unaryNode(
+              "DataSetValues",
+              unaryNode(
+                "DataSetCalc",
+                batchTableNode(1),
+                term("select", "0 AS $f0")),
+              tuples(List(null)), term("values", "$f0")
+            ),
+            term("union", "$f0")
+          ),
+          term("select", "COUNT(*) AS cnt")
+        )
+
+    util.verifySql(queryLeftJoin, expected)
+  }
+
+  @Test
+  def testRightSingleLeftJoinNotEqualPredicate(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Long, Int)]("A", 'a1, 'a2)
+    util.addTable[(Int, Int)]("B", 'b1, 'b2)
+
+    val queryLeftJoin =
+      "SELECT a2 " +
+        "FROM A " +
+        "  LEFT JOIN " +
+        "(SELECT COUNT(*) AS cnt FROM B) AS x " +
+        "  ON a1 > cnt"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        unaryNode(
+          "DataSetSingleRowJoin",
+          batchTableNode(0),
+          term("where", ">(a1, cnt)"),
+          term("join", "a1", "a2", "cnt"),
+          term("joinType", "NestedLoopLeftJoin")
+        ),
+        term("select", "a2")
+      ) + "\n" +
+        unaryNode(
+          "DataSetAggregate",
+          unaryNode(
+            "DataSetUnion",
+            unaryNode(
+              "DataSetValues",
+              unaryNode(
+                "DataSetCalc",
+                batchTableNode(1),
+                term("select", "0 AS $f0")),
+              tuples(List(null)), term("values", "$f0")
+            ),
+            term("union", "$f0")
+          ),
+          term("select", "COUNT(*) AS cnt")
+        )
+
+    util.verifySql(queryLeftJoin, expected)
+  }
+
+  @Test
+  def testLeftSingleRightJoinEqualPredicate(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Long, Long)]("A", 'a1, 'a2)
+    util.addTable[(Long, Long)]("B", 'b1, 'b2)
+
+    val queryRightJoin =
+      "SELECT a1 " +
+        "FROM (SELECT COUNT(*) AS cnt FROM B) " +
+        "  RIGHT JOIN " +
+        "A " +
+        "  ON cnt = a2"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        unaryNode(
+          "DataSetSingleRowJoin",
+          "",
+          term("where", "=(cnt, a2)"),
+          term("join", "cnt", "a1", "a2"),
+          term("joinType", "NestedLoopRightJoin")
+        ),
+        term("select", "a1")
+      ) + unaryNode(
+          "DataSetAggregate",
+          unaryNode(
+            "DataSetUnion",
+            unaryNode(
+              "DataSetValues",
+              unaryNode(
+                "DataSetCalc",
+                batchTableNode(1),
+                term("select", "0 AS $f0")),
+              tuples(List(null)), term("values", "$f0")
+            ),
+            term("union", "$f0")
+          ),
+          term("select", "COUNT(*) AS cnt")
+        ) + "\n" +
+        batchTableNode(0)
+
+    util.verifySql(queryRightJoin, expected)
+  }
+
+  @Test
+  def testLeftSingleRightJoinNotEqualPredicate(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Long, Long)]("A", 'a1, 'a2)
+    util.addTable[(Long, Long)]("B", 'b1, 'b2)
+
+    val queryRightJoin =
+      "SELECT a1 " +
+        "FROM (SELECT COUNT(*) AS cnt FROM B) " +
+        "  RIGHT JOIN " +
+        "A " +
+        "  ON cnt < a2"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        unaryNode(
+          "DataSetSingleRowJoin",
+          "",
+          term("where", "<(cnt, a2)"),
+          term("join", "cnt", "a1", "a2"),
+          term("joinType", "NestedLoopRightJoin")
+        ),
+        term("select", "a1")
+      ) +
+        unaryNode(
+          "DataSetAggregate",
+          unaryNode(
+            "DataSetUnion",
+            unaryNode(
+              "DataSetValues",
+              unaryNode(
+                "DataSetCalc",
+                batchTableNode(1),
+                term("select", "0 AS $f0")),
+              tuples(List(null)), term("values", "$f0")
+            ),
+            term("union", "$f0")
+          ),
+          term("select", "COUNT(*) AS cnt")
+        ) + "\n" +
+        batchTableNode(0)
+
+    util.verifySql(queryRightJoin, expected)
+  }
+
+  @Test
+  def testSingleRowJoinInnerJoin(): Unit = {
+    val util = batchTestUtil()
+    util.addTable[(Int, Int)]("A", 'a1, 'a2)
+    val query =
+      "SELECT a2, sum(a1) " +
+        "FROM A " +
+        "GROUP BY a2 " +
+        "HAVING sum(a1) > (SELECT sum(a1) * 0.1 FROM A)"
+
+    val expected =
+      unaryNode(
+        "DataSetCalc",
+        unaryNode(
+          "DataSetSingleRowJoin",
+          unaryNode(
+            "DataSetAggregate",
+            batchTableNode(0),
+            term("groupBy", "a2"),
+            term("select", "a2", "SUM(a1) AS EXPR$1")
+          ),
+          term("where", ">(EXPR$1, EXPR$0)"),
+          term("join", "a2", "EXPR$1", "EXPR$0"),
+          term("joinType", "NestedLoopInnerJoin")
+        ),
+        term("select", "a2", "EXPR$1")
+      ) + "\n" +
+        unaryNode(
+          "DataSetCalc",
+          unaryNode(
+            "DataSetAggregate",
+            unaryNode(
+              "DataSetUnion",
+              unaryNode(
+                "DataSetValues",
+                unaryNode(
+                  "DataSetCalc",
+                  batchTableNode(0),
+                  term("select", "a1")
+                ),
+                tuples(List(null)), term("values", "a1")
+              ),
+              term("union", "a1")
+            ),
+            term("select", "SUM(a1) AS $f0")
+          ),
+          term("select", "*($f0, 0.1) AS EXPR$0")
+        )
+
+    util.verifySql(query, expected)
+  }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DistinctAggregateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DistinctAggregateTest.scala
index 54b4d24a767ae..85cfb18f521a5 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DistinctAggregateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/DistinctAggregateTest.scala
@@ -211,7 +211,7 @@ class DistinctAggregateTest extends TableTestBase {
       ),
       term("where", "true"),
       term("join", "EXPR$0", "EXPR$1"),
-      term("joinType", "NestedLoopJoin")
+      term("joinType", "NestedLoopInnerJoin")
     )
 
     util.verifySql(sqlQuery, expected)
@@ -268,7 +268,7 @@ class DistinctAggregateTest extends TableTestBase {
           ),
           term("where", "true"),
           term("join", "EXPR$2, EXPR$0"),
-          term("joinType", "NestedLoopJoin")
+          term("joinType", "NestedLoopInnerJoin")
         ),
         unaryNode(
           "DataSetAggregate",
@@ -294,7 +294,7 @@ class DistinctAggregateTest extends TableTestBase {
         ),
         term("where", "true"),
         term("join", "EXPR$2", "EXPR$0, EXPR$1"),
-        term("joinType", "NestedLoopJoin")
+        term("joinType", "NestedLoopInnerJoin")
       ),
       term("select", "EXPR$0, EXPR$1, EXPR$2")
     )
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala
index 8a8c0ce82fc56..3ed44a32b54aa 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/JoinITCase.scala
@@ -40,10 +40,8 @@ class JoinITCase(
 
   @Test
   def testJoin(): Unit = {
-
     val env = ExecutionEnvironment.getExecutionEnvironment
     val tEnv = TableEnvironment.getTableEnvironment(env, config)
-
     val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE b = e"
 
     val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
@@ -280,8 +278,6 @@ class JoinITCase(
     tEnv.registerTable("Table3", ds1)
     tEnv.registerTable("Table5", ds2)
 
-    tEnv.sql(sqlQuery).toDataSet[Row].collect()
-
     val expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n" +
       "null,Hallo Welt wie\n" + "null,Hallo Welt wie gehts?\n" + "null,ABC\n" + "null,BCD\n" +
       "null,CDE\n" + "null,DEF\n" + "null,EFG\n" + "null,FGH\n" + "null,GHI\n" + "null,HIJ\n" +
@@ -304,8 +300,6 @@ class JoinITCase(
     tEnv.registerTable("Table3", ds1)
     tEnv.registerTable("Table5", ds2)
 
-    tEnv.sql(sqlQuery).toDataSet[Row].collect()
-
     val expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n" +
       "null,Hallo Welt wie\n" + "null,Hallo Welt wie gehts?\n" + "null,ABC\n" + "null,BCD\n" +
       "null,CDE\n" + "null,DEF\n" + "null,EFG\n" + "null,FGH\n" + "null,GHI\n" + "null,HIJ\n" +
@@ -372,10 +366,183 @@ class JoinITCase(
 
     val sqlQuery1 = "SELECT * FROM A CROSS JOIN (SELECT count(*) FROM A HAVING count(*) < 0)"
     val result = tEnv.sql(sqlQuery1).count()
-
     Assert.assertEquals(0, result)
   }
 
+  @Test
+  def testLeftNullRightJoin(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val sqlQuery =
+      "SELECT a, cnt " +
+      "FROM (SELECT cnt FROM (SELECT COUNT(*) AS cnt FROM B) WHERE cnt < 0) RIGHT JOIN A ON a < cnt"
+
+    val ds1 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv)
+    tEnv.registerTable("A", ds1)
+    tEnv.registerTable("B", ds2)
+
+
+    val result = tEnv.sql(sqlQuery)
+    val expected = Seq(
+          "1,null",
+          "2,null", "2,null",
+          "3,null", "3,null", "3,null",
+          "4,null", "4,null", "4,null", "4,null",
+          "5,null", "5,null", "5,null", "5,null", "5,null").mkString("\n")
+
+    val results = result.toDataSet[Row].collect()
+
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+
+  @Test
+  def testLeftSingleRightJoinEqualPredicate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val sqlQuery =
+      "SELECT a, cnt FROM (SELECT COUNT(*) AS cnt FROM B) RIGHT JOIN A ON cnt = a"
+
+    val ds1 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv)
+    tEnv.registerTable("A", ds1)
+    tEnv.registerTable("B", ds2)
+
+    val result = tEnv.sql(sqlQuery)
+    val expected = Seq(
+      "1,null", "2,null", "2,null", "3,3", "3,3",
+      "3,3", "4,null", "4,null", "4,null",
+      "4,null", "5,null", "5,null", "5,null",
+      "5,null", "5,null").mkString("\n")
+
+    val results = result.toDataSet[Row].collect()
+
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testLeftSingleRightJoinNotEqualPredicate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val sqlQuery =
+      "SELECT a, cnt FROM (SELECT COUNT(*) AS cnt FROM B) RIGHT JOIN A ON cnt > a"
+
+    val ds1 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv)
+    tEnv.registerTable("A", ds1)
+    tEnv.registerTable("B", ds2)
+
+    val result = tEnv.sql(sqlQuery)
+    val expected = Seq(
+      "1,3", "2,3", "2,3", "3,null", "3,null",
+      "3,null", "4,null", "4,null", "4,null",
+      "4,null", "5,null", "5,null", "5,null",
+      "5,null", "5,null").mkString("\n")
+
+    val results = result.toDataSet[Row].collect()
+
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testRightNullLeftJoin(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val sqlQuery =
+      "SELECT a, cnt " +
+      "FROM A LEFT JOIN (SELECT cnt FROM (SELECT COUNT(*) AS cnt FROM B) WHERE cnt < 0) ON cnt > a"
+
+    val ds1 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv)
+    val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+    tEnv.registerTable("A", ds2)
+    tEnv.registerTable("B", ds1)
+
+    val result = tEnv.sql(sqlQuery)
+
+    val expected = Seq(
+      "2,null", "3,null", "1,null").mkString("\n")
+
+    val results = result.toDataSet[Row].collect()
+
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testRightSingleLeftJoinEqualPredicate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val sqlQuery =
+      "SELECT a, cnt FROM A LEFT JOIN (SELECT COUNT(*) AS cnt FROM B) ON cnt = a"
+
+    val ds1 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv)
+    tEnv.registerTable("A", ds1)
+    tEnv.registerTable("B", ds2)
+
+    val result = tEnv.sql(sqlQuery)
+
+    val expected = Seq(
+      "1,null", "2,null", "2,null", "3,3", "3,3",
+      "3,3", "4,null", "4,null", "4,null",
+      "4,null", "5,null", "5,null", "5,null",
+      "5,null", "5,null").mkString("\n")
+
+    val results = result.toDataSet[Row].collect()
+
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testRightSingleLeftJoinNotEqualPredicate(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val sqlQuery =
+      "SELECT a, cnt FROM A LEFT JOIN (SELECT COUNT(*) AS cnt FROM B) ON cnt < a"
+
+    val ds1 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv)
+    tEnv.registerTable("A", ds1)
+    tEnv.registerTable("B", ds2)
+
+    val result = tEnv.sql(sqlQuery)
+
+    val expected = Seq(
+      "1,null", "2,null", "2,null", "3,null", "3,null",
+      "3,null", "4,3", "4,3", "4,3",
+      "4,3", "5,3", "5,3", "5,3",
+      "5,3", "5,3").mkString("\n")
+
+    val results = result.toDataSet[Row].collect()
+
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
+  @Test
+  def testRightSingleLeftJoinTwoFields(): Unit = {
+    val env = ExecutionEnvironment.getExecutionEnvironment
+    val tEnv = TableEnvironment.getTableEnvironment(env, config)
+    val sqlQuery =
+      "SELECT a, cnt, cnt2 " +
+      "FROM t1 LEFT JOIN (SELECT COUNT(*) AS cnt,COUNT(*) AS cnt2 FROM t2 ) AS x ON a = cnt"
+
+    val ds1 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c, 'd, 'e)
+    val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv)
+    tEnv.registerTable("t1", ds1)
+    tEnv.registerTable("t2", ds2)
+
+    val result = tEnv.sql(sqlQuery)
+    val expected = Seq(
+      "1,null,null",
+      "2,null,null", "2,null,null",
+      "3,3,3", "3,3,3", "3,3,3",
+      "4,null,null", "4,null,null", "4,null,null", "4,null,null",
+      "5,null,null", "5,null,null", "5,null,null", "5,null,null", "5,null,null").mkString("\n")
+
+    val results = result.toDataSet[Row].collect()
+    TestBaseUtils.compareResultAsText(results.asJava, expected)
+  }
+
   @Test
   def testCrossWithUnnest(): Unit = {
 
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SingleRowJoinTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SingleRowJoinTest.scala
deleted file mode 100644
index 27e385304528b..0000000000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/SingleRowJoinTest.scala
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.api.scala.batch.sql
-
-import org.apache.flink.api.scala._
-import org.apache.flink.table.api.scala._
-import org.apache.flink.table.utils.TableTestBase
-import org.apache.flink.table.utils.TableTestUtil._
-import org.junit.Test
-
-class SingleRowJoinTest extends TableTestBase {
-
-  @Test
-  def testSingleRowJoinWithCalcInput(): Unit = {
-    val util = batchTestUtil()
-    util.addTable[(Int, Int)]("A", 'a1, 'a2)
-
-    val query =
-      "SELECT a1, asum " +
-      "FROM A, (SELECT sum(a1) + sum(a2) AS asum FROM A)"
-
-    val expected =
-      binaryNode(
-        "DataSetSingleRowJoin",
-        unaryNode(
-          "DataSetCalc",
-          batchTableNode(0),
-          term("select", "a1")
-        ),
-        unaryNode(
-          "DataSetCalc",
-          unaryNode(
-            "DataSetAggregate",
-            unaryNode(
-              "DataSetUnion",
-              unaryNode(
-                "DataSetValues",
-                batchTableNode(0),
-                tuples(List(null, null)),
-                term("values", "a1", "a2")
-              ),
-              term("union","a1","a2")
-            ),
-            term("select", "SUM(a1) AS $f0", "SUM(a2) AS $f1")
-          ),
-          term("select", "+($f0, $f1) AS asum")
-        ),
-        term("where", "true"),
-        term("join", "a1", "asum"),
-        term("joinType", "NestedLoopJoin")
-      )
-
-    util.verifySql(query, expected)
-  }
-
-  @Test
-  def testSingleRowEquiJoin(): Unit = {
-    val util = batchTestUtil()
-    util.addTable[(Int, String)]("A", 'a1, 'a2)
-
-    val query =
-      "SELECT a1, a2 " +
-      "FROM A, (SELECT count(a1) AS cnt FROM A) " +
-      "WHERE a1 = cnt"
-
-    val expected =
-      unaryNode(
-        "DataSetCalc",
-        binaryNode(
-          "DataSetSingleRowJoin",
-          batchTableNode(0),
-          unaryNode(
-            "DataSetAggregate",
-            unaryNode(
-              "DataSetUnion",
-              unaryNode(
-                "DataSetValues",
-                unaryNode(
-                  "DataSetCalc",
-                  batchTableNode(0),
-                  term("select", "a1")
-                ),
-                tuples(List(null)),
-                term("values", "a1")
-              ),
-              term("union","a1")
-            ),
-            term("select", "COUNT(a1) AS cnt")
-          ),
-          term("where", "=(CAST(a1), cnt)"),
-          term("join", "a1", "a2", "cnt"),
-          term("joinType", "NestedLoopJoin")
-        ),
-        term("select", "a1", "a2")
-      )
-
-    util.verifySql(query, expected)
-  }
-
-  @Test
-  def testSingleRowNotEquiJoin(): Unit = {
-    val util = batchTestUtil()
-    util.addTable[(Int, String)]("A", 'a1, 'a2)
-
-    val query =
-      "SELECT a1, a2 " +
-      "FROM A, (SELECT count(a1) AS cnt FROM A) " +
-      "WHERE a1 < cnt"
-
-    val expected =
-      unaryNode(
-        "DataSetCalc",
-        binaryNode(
-          "DataSetSingleRowJoin",
-          batchTableNode(0),
-          unaryNode(
-            "DataSetAggregate",
-            unaryNode(
-              "DataSetUnion",
-              unaryNode(
-                "DataSetValues",
-                unaryNode(
-                  "DataSetCalc",
-                  batchTableNode(0),
-                  term("select", "a1")
-                ),
-                tuples(List(null)),
-                term("values", "a1")
-              ),
-              term("union", "a1")
-            ),
-            term("select", "COUNT(a1) AS cnt")
-          ),
-          term("where", "<(a1, cnt)"),
-          term("join", "a1", "a2", "cnt"),
-          term("joinType", "NestedLoopJoin")
-        ),
-        term("select", "a1", "a2")
-      )
-
-    util.verifySql(query, expected)
-  }
-
-  @Test
-  def testSingleRowJoinWithComplexPredicate(): Unit = {
-    val util = batchTestUtil()
-    util.addTable[(Int, Long)]("A", 'a1, 'a2)
-    util.addTable[(Int, Long)]("B", 'b1, 'b2)
-
-    val query =
-      "SELECT a1, a2, b1, b2 " +
-        "FROM A, (SELECT min(b1) AS b1, max(b2) AS b2 FROM B) " +
-        "WHERE a1 < b1 AND a2 = b2"
-
-    val expected = binaryNode(
-      "DataSetSingleRowJoin",
-      batchTableNode(0),
-      unaryNode(
-        "DataSetAggregate",
-        unaryNode(
-          "DataSetUnion",
-          unaryNode(
-            "DataSetValues",
-            batchTableNode(1),
-            tuples(List(null, null)),
-            term("values", "b1", "b2")
-          ),
-          term("union","b1","b2")
-        ),
-        term("select", "MIN(b1) AS b1", "MAX(b2) AS b2")
-      ),
-      term("where", "AND(<(a1, b1)", "=(a2, b2))"),
-      term("join", "a1", "a2", "b1", "b2"),
-      term("joinType", "NestedLoopJoin")
-    )
-
-    util.verifySql(query, expected)
-  }
-}

From 3e9cf94b288d217ed4d235e07319bf82485d450e Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Wed, 23 Nov 2016 16:59:22 +0100
Subject: [PATCH 237/253] [FLINK-5101] Track pending records in
 CassandraSinkBase

---
 .../cassandra/CassandraPojoSink.java          |  7 ++-
 .../cassandra/CassandraSinkBase.java          | 56 ++++++++++++++-----
 2 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraPojoSink.java b/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraPojoSink.java
index 650c481cb851c..9cfb2f8934348 100644
--- a/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraPojoSink.java
+++ b/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraPojoSink.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.connectors.cassandra;
 
+import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.mapping.Mapper;
 import com.datastax.driver.mapping.MappingManager;
 import com.google.common.util.concurrent.ListenableFuture;
@@ -31,7 +32,7 @@
  *
  * @param <IN> Type of the elements emitted by this sink
  */
-public class CassandraPojoSink<IN> extends CassandraSinkBase<IN, Void> {
+public class CassandraPojoSink<IN> extends CassandraSinkBase<IN, ResultSet> {
 
 	private static final long serialVersionUID = 1L;
 
@@ -61,7 +62,7 @@ public void open(Configuration configuration) {
 	}
 
 	@Override
-	public ListenableFuture<Void> send(IN value) {
-		return mapper.saveAsync(value);
+	public ListenableFuture<ResultSet> send(IN value) {
+		return session.executeAsync(mapper.saveQuery(value));
 	}
 }
diff --git a/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraSinkBase.java b/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraSinkBase.java
index 49b1efacbe576..b281525b27fd3 100644
--- a/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraSinkBase.java
+++ b/flink-connectors/flink-connector-cassandra/src/main/java/org/apache/flink/streaming/connectors/cassandra/CassandraSinkBase.java
@@ -29,6 +29,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * CassandraSinkBase is the common abstract class of {@link CassandraPojoSink} and {@link CassandraTupleSink}.
@@ -40,11 +41,13 @@ public abstract class CassandraSinkBase<IN, V> extends RichSinkFunction<IN> {
 	protected transient Cluster cluster;
 	protected transient Session session;
 
-	protected transient Throwable exception = null;
+	protected transient volatile Throwable exception;
 	protected transient FutureCallback<V> callback;
 
 	private final ClusterBuilder builder;
 
+	private final AtomicInteger updatesPending = new AtomicInteger();
+
 	protected CassandraSinkBase(ClusterBuilder builder) {
 		this.builder = builder;
 		ClosureCleaner.clean(builder, true);
@@ -55,11 +58,24 @@ public void open(Configuration configuration) {
 		this.callback = new FutureCallback<V>() {
 			@Override
 			public void onSuccess(V ignored) {
+				int pending = updatesPending.decrementAndGet();
+				if (pending == 0) {
+					synchronized (updatesPending) {
+						updatesPending.notifyAll();
+					}
+				}
 			}
 
 			@Override
 			public void onFailure(Throwable t) {
+				int pending = updatesPending.decrementAndGet();
+				if (pending == 0) {
+					synchronized (updatesPending) {
+						updatesPending.notifyAll();
+					}
+				}
 				exception = t;
+				
 				LOG.error("Error while sending value.", t);
 			}
 		};
@@ -70,29 +86,43 @@ public void onFailure(Throwable t) {
 	@Override
 	public void invoke(IN value) throws Exception {
 		if (exception != null) {
-			throw new IOException("invoke() failed", exception);
+			throw new IOException("Error while sending value.", exception);
 		}
 		ListenableFuture<V> result = send(value);
+		updatesPending.incrementAndGet();
 		Futures.addCallback(result, callback);
 	}
 
 	public abstract ListenableFuture<V> send(IN value);
 
 	@Override
-	public void close() {
+	public void close() throws Exception {
 		try {
-			if (session != null) {
-				session.close();
+			if (exception != null) {
+				throw new IOException("Error while sending value.", exception);
 			}
-		} catch (Exception e) {
-			LOG.error("Error while closing session.", e);
-		}
-		try {
-			if (cluster != null) {
-				cluster.close();
+
+			while (updatesPending.get() > 0) {
+				synchronized (updatesPending) {
+					updatesPending.wait();
+				}
+			}
+			
+		} finally {
+			try {
+				if (session != null) {
+					session.close();
+				}
+			} catch (Exception e) {
+				LOG.error("Error while closing session.", e);
+			}
+			try {
+				if (cluster != null) {
+					cluster.close();
+				}
+			} catch (Exception e) {
+				LOG.error("Error while closing cluster.", e);
 			}
-		} catch (Exception e) {
-			LOG.error("Error while closing cluster.", e);
 		}
 	}
 }

From 12614517ad153e49665bee2f437c46d3fd882c58 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Wed, 10 May 2017 12:08:11 +0200
Subject: [PATCH 238/253] [FLINK-5101] Refactor CassandraConnectorITCase

This closes #2866.
---
 .../cassandra/CassandraConnectorITCase.java   | 213 ++++++++----------
 1 file changed, 89 insertions(+), 124 deletions(-)

diff --git a/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java b/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java
index f2e8f8bd433a7..06f3c352da850 100644
--- a/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java
+++ b/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java
@@ -28,39 +28,24 @@
 import org.apache.cassandra.service.CassandraDaemon;
 
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeinfo.TypeHint;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.OutputFormat;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.batch.connectors.cassandra.CassandraInputFormat;
 import org.apache.flink.batch.connectors.cassandra.CassandraOutputFormat;
-import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
-import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.core.io.InputSplit;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
 import org.apache.flink.streaming.runtime.operators.WriteAheadSinkTestBase;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.apache.flink.test.util.TestEnvironment;
 
-import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -71,6 +56,8 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
+import java.util.Random;
 import java.util.Scanner;
 import java.util.UUID;
 
@@ -100,12 +87,15 @@ protected Cluster buildCluster(Cluster.Builder builder) {
 	private static Cluster cluster;
 	private static Session session;
 
+	private static final String TABLE_NAME_PREFIX = "flink_";
+	private static final String TABLE_NAME_VARIABLE = "$TABLE";
 	private static final String CREATE_KEYSPACE_QUERY = "CREATE KEYSPACE flink WITH replication= {'class':'SimpleStrategy', 'replication_factor':1};";
-	private static final String DROP_KEYSPACE_QUERY = "DROP KEYSPACE flink;";
-	private static final String CREATE_TABLE_QUERY = "CREATE TABLE flink.test (id text PRIMARY KEY, counter int, batch_id int);";
-	private static final String CLEAR_TABLE_QUERY = "TRUNCATE flink.test;";
-	private static final String INSERT_DATA_QUERY = "INSERT INTO flink.test (id, counter, batch_id) VALUES (?, ?, ?)";
-	private static final String SELECT_DATA_QUERY = "SELECT * FROM flink.test;";
+	private static final String CREATE_TABLE_QUERY = "CREATE TABLE flink." + TABLE_NAME_VARIABLE + " (id text PRIMARY KEY, counter int, batch_id int);";
+	private static final String INSERT_DATA_QUERY = "INSERT INTO flink." + TABLE_NAME_VARIABLE + " (id, counter, batch_id) VALUES (?, ?, ?)";
+	private static final String SELECT_DATA_QUERY = "SELECT * FROM flink." + TABLE_NAME_VARIABLE + ';';
+
+	private static final Random random = new Random();
+	private int tableID;
 
 	private static final ArrayList<Tuple3<String, Integer, Integer>> collection = new ArrayList<>(20);
 
@@ -129,12 +119,6 @@ public void stop() {
 		}
 	}
 
-	private static LocalFlinkMiniCluster flinkCluster;
-
-	// ------------------------------------------------------------------------
-	//  Cluster Setup (Cassandra & Flink)
-	// ------------------------------------------------------------------------
-
 	@BeforeClass
 	public static void startCassandra() throws IOException {
 
@@ -173,39 +157,39 @@ public static void startCassandra() throws IOException {
 			cassandra.start();
 		}
 
-		try {
-			Thread.sleep(1000 * 10);
-		} catch (InterruptedException e) { //give cassandra a few seconds to start up
+		// start establishing a connection within 30 seconds
+		long start = System.nanoTime();
+		long deadline = start + 30_000_000_000L;
+		while (true) {
+			try {
+				cluster = builder.getCluster();
+				session = cluster.connect();
+				break;
+			} catch (Exception e) {
+				if (System.nanoTime() > deadline) {
+					throw e;
+				}
+				try {
+					Thread.sleep(500);
+				} catch (InterruptedException ignored) {
+				}
+			}
 		}
-
-		cluster = builder.getCluster();
-		session = cluster.connect();
+		LOG.debug("Connection established after {}ms.", System.currentTimeMillis() - start);
 
 		session.execute(CREATE_KEYSPACE_QUERY);
-		session.execute(CREATE_TABLE_QUERY);
-	}
-
-	@BeforeClass
-	public static void startFlink() throws Exception {
-		Configuration config = new Configuration();
-		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
-
-		flinkCluster = new LocalFlinkMiniCluster(config);
-		flinkCluster.start();
+		session.execute(CREATE_TABLE_QUERY.replace(TABLE_NAME_VARIABLE, TABLE_NAME_PREFIX + "initial"));
 	}
 
-	@AfterClass
-	public static void stopFlink() {
-		if (flinkCluster != null) {
-			flinkCluster.stop();
-			flinkCluster = null;
-		}
+	@Before
+	public void createTable() {
+		tableID = random.nextInt(Integer.MAX_VALUE);
+		session.execute(injectTableName(CREATE_TABLE_QUERY));
 	}
 
 	@AfterClass
 	public static void closeCassandra() {
 		if (session != null) {
-			session.executeAsync(DROP_KEYSPACE_QUERY);
 			session.close();
 		}
 
@@ -223,21 +207,6 @@ public static void closeCassandra() {
 		}
 	}
 
-	// ------------------------------------------------------------------------
-	//  Test preparation & cleanup
-	// ------------------------------------------------------------------------
-
-	@Before
-	public void initializeExecutionEnvironment() {
-		TestStreamEnvironment.setAsContext(flinkCluster, 4);
-		new TestEnvironment(flinkCluster, 4, false).setAsContext();
-	}
-
-	@After
-	public void deleteSchema() throws Exception {
-		session.executeAsync(CLEAR_TABLE_QUERY);
-	}
-
 	// ------------------------------------------------------------------------
 	//  Exactly-once Tests
 	// ------------------------------------------------------------------------
@@ -245,7 +214,7 @@ public void deleteSchema() throws Exception {
 	@Override
 	protected CassandraTupleWriteAheadSink<Tuple3<String, Integer, Integer>> createSink() throws Exception {
 		return new CassandraTupleWriteAheadSink<>(
-			INSERT_DATA_QUERY,
+			injectTableName(INSERT_DATA_QUERY),
 			TypeExtractor.getForObject(new Tuple3<>("", 0, 0)).createSerializer(new ExecutionConfig()),
 			builder,
 			new CassandraCommitter(builder));
@@ -264,7 +233,7 @@ protected Tuple3<String, Integer, Integer> generateValue(int counter, int checkp
 	@Override
 	protected void verifyResultsIdealCircumstances(CassandraTupleWriteAheadSink<Tuple3<String, Integer, Integer>> sink) {
 
-		ResultSet result = session.execute(SELECT_DATA_QUERY);
+		ResultSet result = session.execute(injectTableName(SELECT_DATA_QUERY));
 		ArrayList<Integer> list = new ArrayList<>();
 		for (int x = 1; x <= 60; x++) {
 			list.add(x);
@@ -279,7 +248,7 @@ protected void verifyResultsIdealCircumstances(CassandraTupleWriteAheadSink<Tupl
 	@Override
 	protected void verifyResultsDataPersistenceUponMissedNotify(CassandraTupleWriteAheadSink<Tuple3<String, Integer, Integer>> sink) {
 
-		ResultSet result = session.execute(SELECT_DATA_QUERY);
+		ResultSet result = session.execute(injectTableName(SELECT_DATA_QUERY));
 		ArrayList<Integer> list = new ArrayList<>();
 		for (int x = 1; x <= 60; x++) {
 			list.add(x);
@@ -294,7 +263,7 @@ protected void verifyResultsDataPersistenceUponMissedNotify(CassandraTupleWriteA
 	@Override
 	protected void verifyResultsDataDiscardingUponRestore(CassandraTupleWriteAheadSink<Tuple3<String, Integer, Integer>> sink) {
 
-		ResultSet result = session.execute(SELECT_DATA_QUERY);
+		ResultSet result = session.execute(injectTableName(SELECT_DATA_QUERY));
 		ArrayList<Integer> list = new ArrayList<>();
 		for (int x = 1; x <= 20; x++) {
 			list.add(x);
@@ -324,7 +293,7 @@ protected void verifyResultsWhenReScaling(
 		}
 
 		ArrayList<Integer> actual = new ArrayList<>();
-		ResultSet result = session.execute(SELECT_DATA_QUERY);
+		ResultSet result = session.execute(injectTableName(SELECT_DATA_QUERY));
 		for (Row s : result) {
 			actual.add(s.getInt("counter"));
 		}
@@ -335,16 +304,17 @@ protected void verifyResultsWhenReScaling(
 
 	@Test
 	public void testCassandraCommitter() throws Exception {
-		CassandraCommitter cc1 = new CassandraCommitter(builder);
-		cc1.setJobId("job");
+		String jobID = new JobID().toString();
+		CassandraCommitter cc1 = new CassandraCommitter(builder, "flink_auxiliary_cc");
+		cc1.setJobId(jobID);
 		cc1.setOperatorId("operator");
 
-		CassandraCommitter cc2 = new CassandraCommitter(builder);
-		cc2.setJobId("job");
+		CassandraCommitter cc2 = new CassandraCommitter(builder, "flink_auxiliary_cc");
+		cc2.setJobId(jobID);
 		cc2.setOperatorId("operator");
 
-		CassandraCommitter cc3 = new CassandraCommitter(builder);
-		cc3.setJobId("job");
+		CassandraCommitter cc3 = new CassandraCommitter(builder, "flink_auxiliary_cc");
+		cc3.setJobId(jobID);
 		cc3.setOperatorId("operator1");
 
 		cc1.createResource();
@@ -370,8 +340,8 @@ public void testCassandraCommitter() throws Exception {
 		cc2.close();
 		cc3.close();
 
-		cc1 = new CassandraCommitter(builder);
-		cc1.setJobId("job");
+		cc1 = new CassandraCommitter(builder, "flink_auxiliary_cc");
+		cc1.setJobId(jobID);
 		cc1.setOperatorId("operator");
 
 		cc1.open();
@@ -389,70 +359,65 @@ public void testCassandraCommitter() throws Exception {
 
 	@Test
 	public void testCassandraTupleAtLeastOnceSink() throws Exception {
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-		env.setParallelism(1);
+		CassandraTupleSink<Tuple3<String, Integer, Integer>> sink = new CassandraTupleSink<>(injectTableName(INSERT_DATA_QUERY), builder);
 
-		DataStream<Tuple3<String, Integer, Integer>> source = env.fromCollection(collection);
-		source.addSink(new CassandraTupleSink<Tuple3<String, Integer, Integer>>(INSERT_DATA_QUERY, builder));
+		sink.open(new Configuration());
+
+		for (Tuple3<String, Integer, Integer> value : collection) {
+			sink.send(value);
+		}
 
-		env.execute();
+		sink.close();
 
-		ResultSet rs = session.execute(SELECT_DATA_QUERY);
+		ResultSet rs = session.execute(injectTableName(SELECT_DATA_QUERY));
 		Assert.assertEquals(20, rs.all().size());
 	}
 
 	@Test
 	public void testCassandraPojoAtLeastOnceSink() throws Exception {
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-		env.setParallelism(1);
-
-		DataStreamSource<Pojo> source = env
-			.addSource(new SourceFunction<Pojo>() {
-
-				private boolean running = true;
-				private volatile int cnt = 0;
-
-				@Override
-				public void run(SourceContext<Pojo> ctx) throws Exception {
-					while (running) {
-						ctx.collect(new Pojo(UUID.randomUUID().toString(), cnt, 0));
-						cnt++;
-						if (cnt == 20) {
-							cancel();
-						}
-					}
-				}
+		session.execute(CREATE_TABLE_QUERY.replace(TABLE_NAME_VARIABLE, "test"));
 
-				@Override
-				public void cancel() {
-					running = false;
-				}
-			});
+		CassandraPojoSink<Pojo> sink = new CassandraPojoSink<>(Pojo.class, builder);
 
-		source.addSink(new CassandraPojoSink<>(Pojo.class, builder));
+		sink.open(new Configuration());
 
-		env.execute();
+		for (int x = 0; x < 20; x++) {
+			sink.send(new Pojo(UUID.randomUUID().toString(), x, 0));
+		}
 
-		ResultSet rs = session.execute(SELECT_DATA_QUERY);
+		sink.close();
+
+		ResultSet rs = session.execute(SELECT_DATA_QUERY.replace(TABLE_NAME_VARIABLE, "test"));
 		Assert.assertEquals(20, rs.all().size());
 	}
 
 	@Test
 	public void testCassandraBatchFormats() throws Exception {
-		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-		env.setParallelism(1);
+		OutputFormat<Tuple3<String, Integer, Integer>> sink = new CassandraOutputFormat<>(injectTableName(INSERT_DATA_QUERY), builder);
+		sink.configure(new Configuration());
+		sink.open(0, 1);
 
-		DataSet<Tuple3<String, Integer, Integer>> dataSet = env.fromCollection(collection);
-		dataSet.output(new CassandraOutputFormat<Tuple3<String, Integer, Integer>>(INSERT_DATA_QUERY, builder));
+		for (Tuple3<String, Integer, Integer> value : collection) {
+			sink.writeRecord(value);
+		}
 
-		env.execute("Write data");
+		sink.close();
 
-		DataSet<Tuple3<String, Integer, Integer>> inputDS = env.createInput(
-			new CassandraInputFormat<Tuple3<String, Integer, Integer>>(SELECT_DATA_QUERY, builder),
-			TypeInformation.of(new TypeHint<Tuple3<String, Integer, Integer>>(){}));
+		InputFormat<Tuple3<String, Integer, Integer>, InputSplit> source = new CassandraInputFormat<>(injectTableName(SELECT_DATA_QUERY), builder);
+		source.configure(new Configuration());
+		source.open(null);
 
+		List<Tuple3<String, Integer, Integer>> result = new ArrayList<>();
+
+		while (!source.reachedEnd()) {
+			result.add(source.nextRecord(new Tuple3<String, Integer, Integer>()));
+		}
+
+		source.close();
+		Assert.assertEquals(20, result.size());
+	}
 
-		long count = inputDS.count();
-		Assert.assertEquals(count, 20L);
+	private String injectTableName(String target) {
+		return target.replace(TABLE_NAME_VARIABLE, TABLE_NAME_PREFIX + tableID);
 	}
 }

From 2f3645770ca6e81b9e8c3787a3b3c841a6d4bd73 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 11 May 2017 16:39:12 +0200
Subject: [PATCH 239/253] [FLINK-6548] Fix AvroOutputFormatTest#testCompression
 on Windows

---
 .../apache/flink/api/java/io/AvroOutputFormatTest.java   | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroOutputFormatTest.java b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroOutputFormatTest.java
index 4d6c6b768d2ee..f843d3b97d970 100644
--- a/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroOutputFormatTest.java
+++ b/flink-connectors/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroOutputFormatTest.java
@@ -28,8 +28,6 @@
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 
 import org.apache.avro.Schema;
 import org.apache.flink.api.io.avro.example.User;
@@ -135,12 +133,13 @@ public void testCompression() throws Exception {
 		assertTrue(fileSize(outputPath) > fileSize(compressedOutputPath));
 
 		// cleanup
-		Files.delete(Paths.get(outputPath.getPath()));
-		Files.delete(Paths.get(compressedOutputPath.getPath()));
+		FileSystem fs = FileSystem.getLocalFileSystem();
+		fs.delete(outputPath, false);
+		fs.delete(compressedOutputPath, false);
 	}
 
 	private long fileSize(Path path) throws IOException {
-		return Files.size(Paths.get(path.getPath()));
+		return path.getFileSystem().getFileStatus(path).getLen();
 	}
 
 	private void output(final AvroOutputFormat<User> outputFormat) throws IOException {

From 64b2fa5cb9fc273d05947f70c95c6bf49780c7f0 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 11 May 2017 16:55:11 +0200
Subject: [PATCH 240/253] [FLINK-6558] Disable yarn tests on Windows

---
 .../flink/yarn/YarnApplicationMasterRunnerTest.java       | 8 ++++++++
 .../YarnIntraNonHaMasterServicesTest.java                 | 4 ++++
 .../YarnPreConfiguredMasterHaServicesTest.java            | 4 ++++
 3 files changed, 16 insertions(+)

diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnApplicationMasterRunnerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnApplicationMasterRunnerTest.java
index f8748967affb5..4884dd06f9b7f 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnApplicationMasterRunnerTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnApplicationMasterRunnerTest.java
@@ -21,8 +21,11 @@
 import com.google.common.collect.ImmutableMap;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.util.OperatingSystem;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -49,6 +52,11 @@ public class YarnApplicationMasterRunnerTest {
 	@Rule
 	public TemporaryFolder folder = new TemporaryFolder();
 
+	@BeforeClass
+	public static void checkOS() {
+		Assume.assumeTrue(!OperatingSystem.isWindows());
+	}
+
 	@Test
 	public void testCreateTaskExecutorContext() throws Exception {
 		File root = folder.getRoot();
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServicesTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServicesTest.java
index 64c22d20b3788..b4d2ba85722d5 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServicesTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServicesTest.java
@@ -24,12 +24,14 @@
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.util.OperatingSystem;
 import org.apache.flink.util.StringUtils;
 
 import org.apache.flink.util.TestLogger;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 
 import org.junit.AfterClass;
+import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -70,6 +72,8 @@ public class YarnIntraNonHaMasterServicesTest extends TestLogger {
 
 	@BeforeClass
 	public static void createHDFS() throws Exception {
+		Assume.assumeTrue(!OperatingSystem.isWindows());
+
 		final File tempDir = TEMP_DIR.newFolder();
 
 		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
index 83ec37da98028..07c952aab63fa 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java
@@ -24,12 +24,14 @@
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.util.OperatingSystem;
 import org.apache.flink.util.TestLogger;
 import org.apache.flink.yarn.configuration.YarnConfigOptions;
 
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 
 import org.junit.AfterClass;
+import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -59,6 +61,8 @@ public class YarnPreConfiguredMasterHaServicesTest extends TestLogger {
 
 	@BeforeClass
 	public static void createHDFS() throws Exception {
+		Assume.assumeTrue(!OperatingSystem.isWindows());
+
 		final File tempDir = TEMP_DIR.newFolder();
 
 		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();

From 20634821e3ab10783c82a6a5918a27ff152c88ba Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Thu, 11 May 2017 21:24:02 +0200
Subject: [PATCH 241/253] [FLINK-6561] Disable glob test on Windows

---
 .../apache/flink/api/common/io/GlobFilePathFilterTest.java    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/GlobFilePathFilterTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/GlobFilePathFilterTest.java
index c9f8da4921ec1..bcdb5c334d5c5 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/GlobFilePathFilterTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/GlobFilePathFilterTest.java
@@ -19,6 +19,8 @@
 
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.util.OperatingSystem;
+import org.junit.Assume;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -110,6 +112,8 @@ public void testExcludeHDFSFile() {
 
 	@Test
 	public void testExcludeFilenameWithStart() {
+		Assume.assumeTrue("Windows does not allow asterisks in file names.", !OperatingSystem.isWindows());
+
 		GlobFilePathFilter matcher = new GlobFilePathFilter(
 			Collections.singletonList("**"),
 			Collections.singletonList("\\*"));

From fe243d7205f7770f8c7958cfaafb7d537c264f2d Mon Sep 17 00:00:00 2001
From: "Tzu-Li (Gordon) Tai" <tzulitai@apache.org>
Date: Fri, 12 May 2017 19:11:25 +0800
Subject: [PATCH 242/253] [FLINK-6565] Fail memory-backed state restores with
 meaningful message if previous serializer is unavailable

This closes #3882.
---
 .../state/DefaultOperatorStateBackend.java    |  17 +++
 ...endStateMetaInfoSnapshotReaderWriters.java |   8 ++
 ...endStateMetaInfoSnapshotReaderWriters.java |   6 +
 .../state/heap/HeapKeyedStateBackend.java     |  17 +++
 .../runtime/state/MemoryStateBackendTest.java | 135 ++++++++++++++++++
 .../state/OperatorStateBackendTest.java       |  70 ++++++++-
 .../runtime/state/StateBackendTestBase.java   |   2 +-
 7 files changed, 251 insertions(+), 4 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
index ab0c1f0bf3ed2..1d3af7272dcff 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
@@ -24,6 +24,7 @@
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
 import org.apache.flink.core.fs.CloseableRegistry;
 import org.apache.flink.core.fs.FSDataInputStream;
 import org.apache.flink.core.fs.FSDataOutputStream;
@@ -293,6 +294,22 @@ public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exc
 
 				// Recreate all PartitionableListStates from the meta info
 				for (RegisteredOperatorBackendStateMetaInfo.Snapshot<?> restoredMetaInfo : restoredMetaInfoSnapshots) {
+
+					if (restoredMetaInfo.getPartitionStateSerializer() == null ||
+							restoredMetaInfo.getPartitionStateSerializer()
+								instanceof TypeSerializerSerializationProxy.ClassNotFoundDummyTypeSerializer) {
+
+						// must fail now if the previous serializer cannot be restored because there is no serializer
+						// capable of reading previous state
+						// TODO when eager state registration is in place, we can try to get a convert deserializer
+						// TODO from the newly registered serializer instead of simply failing here
+
+						throw new IOException("Unable to restore operator state [" + restoredMetaInfo.getName() + "]." +
+							" The previous serializer of the operator state must be present; the serializer could" +
+							" have been removed from the classpath, or its implementation have changed and could" +
+							" not be loaded. This is a temporary restriction that will be fixed in future versions.");
+					}
+
 					PartitionableListState<?> listState = registeredStates.get(restoredMetaInfo.getName());
 
 					if (null == listState) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java
index 83aa33576aded..ac81e8637184c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java
@@ -28,6 +28,8 @@
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
@@ -37,6 +39,8 @@
  */
 public class KeyedBackendStateMetaInfoSnapshotReaderWriters {
 
+	private static final Logger LOG = LoggerFactory.getLogger(KeyedBackendStateMetaInfoSnapshotReaderWriters.class);
+
 	// -------------------------------------------------------------------------------
 	//  Writers
 	//   - v1: Flink 1.2.x
@@ -230,6 +234,8 @@ public RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> readStateMetaInfo(Data
 				namespaceSerializerProxy.read(inViewWrapper);
 				metaInfo.setNamespaceSerializer(namespaceSerializerProxy.getTypeSerializer());
 			} catch (IOException e) {
+				LOG.warn("Deserialization of previous namespace serializer errored; setting serializer to null. ", e);
+
 				metaInfo.setNamespaceSerializer(null);
 			}
 
@@ -241,6 +247,8 @@ public RegisteredKeyedBackendStateMetaInfo.Snapshot<N, S> readStateMetaInfo(Data
 				stateSerializerProxy.read(inViewWrapper);
 				metaInfo.setStateSerializer(stateSerializerProxy.getTypeSerializer());
 			} catch (IOException e) {
+				LOG.warn("Deserialization of previous state serializer errored; setting serializer to null. ", e);
+
 				metaInfo.setStateSerializer(null);
 			}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
index 9ab106bc492db..4f151c9bd3e55 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
@@ -30,6 +30,8 @@
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
@@ -39,6 +41,8 @@
  */
 public class OperatorBackendStateMetaInfoSnapshotReaderWriters {
 
+	private static final Logger LOG = LoggerFactory.getLogger(OperatorBackendStateMetaInfoSnapshotReaderWriters.class);
+
 	// -------------------------------------------------------------------------------
 	//  Writers
 	//   - v1: Flink 1.2.x
@@ -219,6 +223,8 @@ public RegisteredOperatorBackendStateMetaInfo.Snapshot<S> readStateMetaInfo(Data
 				partitionStateSerializerProxy.read(inViewWrapper);
 				stateMetaInfo.setPartitionStateSerializer(partitionStateSerializerProxy.getTypeSerializer());
 			} catch (IOException e) {
+				LOG.warn("Deserialization of previous serializer errored; setting serializer to null. ", e);
+
 				stateMetaInfo.setPartitionStateSerializer(null);
 			}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index 866ed2830cd26..bc314df73e1bd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -30,6 +30,7 @@
 import org.apache.flink.api.common.state.StateDescriptor;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.core.fs.FSDataInputStream;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
@@ -389,6 +390,22 @@ private void restorePartitionedState(Collection<KeyedStateHandle> state) throws
 
 				for (RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> restoredMetaInfo : restoredMetaInfos) {
 
+					if (restoredMetaInfo.getStateSerializer() == null ||
+							restoredMetaInfo.getStateSerializer()
+								instanceof TypeSerializerSerializationProxy.ClassNotFoundDummyTypeSerializer) {
+
+						// must fail now if the previous serializer cannot be restored because there is no serializer
+						// capable of reading previous state
+						// TODO when eager state registration is in place, we can try to get a convert deserializer
+						// TODO from the newly registered serializer instead of simply failing here
+
+						throw new IOException("Unable to restore keyed state [" + restoredMetaInfo.getName() + "]." +
+							" For memory-backed keyed state, the previous serializer of the keyed state must be" +
+							" present; the serializer could have been removed from the classpath, or its implementation" +
+							" have changed and could not be loaded. This is a temporary restriction that will be fixed" +
+							" in future versions.");
+					}
+
 					StateTable<K, ?, ?> stateTable = stateTables.get(restoredMetaInfo.getName());
 
 					//important: only create a new table we did not already create it previously
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java
index 48d56e26f2e82..fee97f4eacdc0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/MemoryStateBackendTest.java
@@ -20,27 +20,48 @@
 
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
 import org.apache.flink.api.common.typeutils.base.IntSerializer;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.runtime.checkpoint.CheckpointOptions;
+import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.state.heap.HeapKeyedStateBackend;
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.util.FutureUtil;
+import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.concurrent.RunnableFuture;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Tests for the {@link org.apache.flink.runtime.state.memory.MemoryStateBackend}.
  */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({KeyedBackendStateMetaInfoSnapshotReaderWriters.class, OperatorBackendStateMetaInfoSnapshotReaderWriters.class})
 public class MemoryStateBackendTest extends StateBackendTestBase<MemoryStateBackend> {
 
 	@Override
@@ -198,6 +219,120 @@ public void testOversizedStateStream() {
 		}
 	}
 
+	/**
+	 * Verifies that the operator state backend fails with appropriate error and message if
+	 * previous serializer can not be restored.
+	 */
+	@Test
+	public void testOperatorStateRestoreFailsIfSerializerDeserializationFails() throws Exception {
+		AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
+
+		Environment env = mock(Environment.class);
+		when(env.getExecutionConfig()).thenReturn(new ExecutionConfig());
+		when(env.getUserClassLoader()).thenReturn(OperatorStateBackendTest.class.getClassLoader());
+
+		OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(env, "test-op-name");
+
+		// write some state
+		ListStateDescriptor<Serializable> stateDescriptor1 = new ListStateDescriptor<>("test1", new JavaSerializer<>());
+		ListStateDescriptor<Serializable> stateDescriptor2 = new ListStateDescriptor<>("test2", new JavaSerializer<>());
+		ListStateDescriptor<Serializable> stateDescriptor3 = new ListStateDescriptor<>("test3", new JavaSerializer<>());
+		ListState<Serializable> listState1 = operatorStateBackend.getListState(stateDescriptor1);
+		ListState<Serializable> listState2 = operatorStateBackend.getListState(stateDescriptor2);
+		ListState<Serializable> listState3 = operatorStateBackend.getUnionListState(stateDescriptor3);
+
+		listState1.add(42);
+		listState1.add(4711);
+
+		listState2.add(7);
+		listState2.add(13);
+		listState2.add(23);
+
+		listState3.add(17);
+		listState3.add(18);
+		listState3.add(19);
+		listState3.add(20);
+
+		CheckpointStreamFactory streamFactory = abstractStateBackend.createStreamFactory(new JobID(), "testOperator");
+		RunnableFuture<OperatorStateHandle> runnableFuture =
+			operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint());
+		OperatorStateHandle stateHandle = FutureUtil.runIfNotDoneAndGet(runnableFuture);
+
+		try {
+
+			operatorStateBackend.close();
+			operatorStateBackend.dispose();
+
+			operatorStateBackend = abstractStateBackend.createOperatorStateBackend(
+				env,
+				"testOperator");
+
+			// mock failure when deserializing serializer
+			TypeSerializerSerializationProxy<?> mockProxy = mock(TypeSerializerSerializationProxy.class);
+			doThrow(new IOException()).when(mockProxy).read(any(DataInputViewStreamWrapper.class));
+			PowerMockito.whenNew(TypeSerializerSerializationProxy.class).withAnyArguments().thenReturn(mockProxy);
+
+			operatorStateBackend.restore(Collections.singletonList(stateHandle));
+
+			fail("The operator state restore should have failed if the previous state serializer could not be loaded.");
+		} catch (IOException expected) {
+			Assert.assertTrue(expected.getMessage().contains("Unable to restore operator state"));
+		} finally {
+			stateHandle.discardState();
+		}
+	}
+
+	/**
+	 * Verifies that memory-backed keyed state backend fails with appropriate error and message if
+	 * previous serializer can not be restored.
+	 */
+	@Test
+	public void testKeyedStateRestoreFailsIfSerializerDeserializationFails() throws Exception {
+		CheckpointStreamFactory streamFactory = createStreamFactory();
+		KeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
+
+		ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
+		kvId.initializeSerializerUnlessSet(new ExecutionConfig());
+
+		HeapKeyedStateBackend<Integer> heapBackend = (HeapKeyedStateBackend<Integer>) backend;
+
+		assertEquals(0, heapBackend.numStateEntries());
+
+		ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
+
+		// write some state
+		backend.setCurrentKey(0);
+		state.update("hello");
+		state.update("ciao");
+
+		KeyedStateHandle snapshot = runSnapshot(((HeapKeyedStateBackend<Integer>) backend).snapshot(
+			682375462378L,
+			2,
+			streamFactory,
+			CheckpointOptions.forFullCheckpoint()));
+
+		backend.dispose();
+
+		// ========== restore snapshot ==========
+
+		Environment env = mock(Environment.class);
+		when(env.getExecutionConfig()).thenReturn(new ExecutionConfig());
+		when(env.getUserClassLoader()).thenReturn(OperatorStateBackendTest.class.getClassLoader());
+
+		// mock failure when deserializing serializer
+		TypeSerializerSerializationProxy<?> mockProxy = mock(TypeSerializerSerializationProxy.class);
+		doThrow(new IOException()).when(mockProxy).read(any(DataInputViewStreamWrapper.class));
+		PowerMockito.whenNew(TypeSerializerSerializationProxy.class).withAnyArguments().thenReturn(mockProxy);
+
+		try {
+			restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
+
+			fail("The keyed state restore should have failed if the previous state serializer could not be loaded.");
+		} catch (IOException expected) {
+			Assert.assertTrue(expected.getMessage().contains("Unable to restore keyed state"));
+		}
+	}
+
 	@Ignore
 	@Test
 	public void testConcurrentMapIfQueryable() throws Exception {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
index 85b9eaf5f8cc3..af5f0b2cabeae 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java
@@ -22,7 +22,9 @@
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
 import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.testutils.OneShotLatch;
 import org.apache.flink.runtime.checkpoint.CheckpointOptions;
 import org.apache.flink.runtime.execution.Environment;
@@ -32,6 +34,10 @@
 import org.apache.flink.util.FutureUtil;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
 import java.io.File;
 import java.io.IOException;
@@ -52,9 +58,13 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(OperatorBackendStateMetaInfoSnapshotReaderWriters.class)
 public class OperatorStateBackendTest {
 
 	private final ClassLoader classLoader = getClass().getClassLoader();
@@ -290,7 +300,7 @@ public void testSnapshotRestoreSync() throws Exception {
 
 	@Test
 	public void testSnapshotRestoreAsync() throws Exception {
-		DefaultOperatorStateBackend operatorStateBackend =
+		OperatorStateBackend operatorStateBackend =
 				new DefaultOperatorStateBackend(OperatorStateBackendTest.class.getClassLoader(), new ExecutionConfig(), true);
 
 		ListStateDescriptor<MutableType> stateDescriptor1 =
@@ -362,8 +372,7 @@ public void testSnapshotRestoreAsync() throws Exception {
 
 			AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
 
-			//TODO this is temporarily casted to test already functionality that we do not yet expose through public API
-			operatorStateBackend = (DefaultOperatorStateBackend) abstractStateBackend.createOperatorStateBackend(
+			operatorStateBackend = abstractStateBackend.createOperatorStateBackend(
 					createMockEnvironment(),
 					"testOperator");
 
@@ -494,6 +503,61 @@ public void testSnapshotAsyncCancel() throws Exception {
 		}
 	}
 
+	@Test
+	public void testRestoreFailsIfSerializerDeserializationFails() throws Exception {
+		AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
+
+		OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "test-op-name");
+
+		// write some state
+		ListStateDescriptor<Serializable> stateDescriptor1 = new ListStateDescriptor<>("test1", new JavaSerializer<>());
+		ListStateDescriptor<Serializable> stateDescriptor2 = new ListStateDescriptor<>("test2", new JavaSerializer<>());
+		ListStateDescriptor<Serializable> stateDescriptor3 = new ListStateDescriptor<>("test3", new JavaSerializer<>());
+		ListState<Serializable> listState1 = operatorStateBackend.getListState(stateDescriptor1);
+		ListState<Serializable> listState2 = operatorStateBackend.getListState(stateDescriptor2);
+		ListState<Serializable> listState3 = operatorStateBackend.getUnionListState(stateDescriptor3);
+
+		listState1.add(42);
+		listState1.add(4711);
+
+		listState2.add(7);
+		listState2.add(13);
+		listState2.add(23);
+
+		listState3.add(17);
+		listState3.add(18);
+		listState3.add(19);
+		listState3.add(20);
+
+		CheckpointStreamFactory streamFactory = abstractStateBackend.createStreamFactory(new JobID(), "testOperator");
+		RunnableFuture<OperatorStateHandle> runnableFuture =
+			operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forFullCheckpoint());
+		OperatorStateHandle stateHandle = FutureUtil.runIfNotDoneAndGet(runnableFuture);
+
+		try {
+
+			operatorStateBackend.close();
+			operatorStateBackend.dispose();
+
+			operatorStateBackend = abstractStateBackend.createOperatorStateBackend(
+				createMockEnvironment(),
+				"testOperator");
+
+			// mock failure when deserializing serializer
+			TypeSerializerSerializationProxy<?> mockProxy = mock(TypeSerializerSerializationProxy.class);
+			doThrow(new IOException()).when(mockProxy).read(any(DataInputViewStreamWrapper.class));
+			PowerMockito.whenNew(TypeSerializerSerializationProxy.class).withAnyArguments().thenReturn(mockProxy);
+
+			operatorStateBackend.restore(Collections.singletonList(stateHandle));
+
+			fail("The operator state restore should have failed if the previous state serializer could not be loaded.");
+		} catch (IOException expected) {
+			Assert.assertTrue(expected.getMessage().contains("Unable to restore operator state"));
+		} finally {
+			stateHandle.discardState();
+		}
+	}
+
 	static final class MutableType implements Serializable {
 
 		private static final long serialVersionUID = 1L;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
index 96025fe16552b..658ccde8c14cc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
@@ -2508,7 +2508,7 @@ private static <UK, UV, K, N> Map<UK, UV> getSerializedMap(
 		}
 	}
 
-	private KeyedStateHandle runSnapshot(RunnableFuture<KeyedStateHandle> snapshotRunnableFuture) throws Exception {
+	protected KeyedStateHandle runSnapshot(RunnableFuture<KeyedStateHandle> snapshotRunnableFuture) throws Exception {
 		if(!snapshotRunnableFuture.isDone()) {
 			Thread runner = new Thread(snapshotRunnableFuture);
 			runner.start();

From 1971adb9e4617fe0436f2c37d642989bc2dabad9 Mon Sep 17 00:00:00 2001
From: "Tzu-Li (Gordon) Tai" <tzulitai@apache.org>
Date: Fri, 12 May 2017 20:06:01 +0800
Subject: [PATCH 243/253] [FLINK-6566] [core] More restricted interface for
 VersionedIOReadableWritable hooks

This commit makes the method hooks for defining compatibile
serialization versions of VersionedIOReadableWritables more restricted.

Functionally everything remains the same, but with lesser space for
error-prone user implementations. It also allows for a better error
message to indicate version mismatch.

This closes #3883.
---
 .../TypeSerializerConfigSnapshot.java         | 23 --------
 .../core/io/VersionedIOReadableWritable.java  | 53 +++++++++++--------
 .../core/io/VersionedIOWriteableTest.java     | 16 ++----
 .../state/KeyedBackendSerializationProxy.java | 20 ++-----
 .../OperatorBackendSerializationProxy.java    | 17 ++----
 .../state/heap/HeapKeyedStateBackend.java     |  2 +-
 6 files changed, 42 insertions(+), 89 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java
index 27369b9245469..389d141fff0e8 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java
@@ -20,7 +20,6 @@
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.core.io.VersionMismatchException;
 import org.apache.flink.core.io.VersionedIOReadableWritable;
 import org.apache.flink.util.Preconditions;
 
@@ -51,22 +50,6 @@ public abstract class TypeSerializerConfigSnapshot extends VersionedIOReadableWr
 	/** The user code class loader; only relevant if this configuration instance was deserialized from binary form. */
 	private ClassLoader userCodeClassLoader;
 
-	/** The snapshot version of this configuration. */
-	private Integer snapshotVersion;
-
-	/**
-	 * Returns the version of the configuration at the time its snapshot was taken.
-	 *
-	 * @return the snapshot configuration's version.
-	 */
-	public int getSnapshotVersion() {
-		if (snapshotVersion == null) {
-			return getVersion();
-		} else {
-			return snapshotVersion;
-		}
-	}
-
 	/**
 	 * Set the user code class loader.
 	 * Only relevant if this configuration instance was deserialized from binary form.
@@ -91,12 +74,6 @@ public final ClassLoader getUserCodeClassLoader() {
 		return userCodeClassLoader;
 	}
 
-	@Override
-	protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
-		super.resolveVersionRead(foundVersion);
-		this.snapshotVersion = foundVersion;
-	}
-
 	public abstract boolean equals(Object obj);
 
 	public abstract int hashCode();
diff --git a/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java b/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java
index bad9cef48de0c..b4a0b2fe1e8b2 100644
--- a/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java
+++ b/flink-core/src/main/java/org/apache/flink/core/io/VersionedIOReadableWritable.java
@@ -18,20 +18,23 @@
 
 package org.apache.flink.core.io;
 
-import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.annotation.Internal;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * This is the abstract base class for {@link IOReadableWritable} which allows to differentiate between serialization
  * versions. Concrete subclasses should typically override the {@link #write(DataOutputView)} and
  * {@link #read(DataInputView)}, thereby calling super to ensure version checking.
  */
-@PublicEvolving
+@Internal
 public abstract class VersionedIOReadableWritable implements IOReadableWritable, Versioned {
 
+	private int readVersion = Integer.MIN_VALUE;
+
 	@Override
 	public void write(DataOutputView out) throws IOException {
 		out.writeInt(getVersion());
@@ -39,34 +42,42 @@ public void write(DataOutputView out) throws IOException {
 
 	@Override
 	public void read(DataInputView in) throws IOException {
-		int foundVersion = in.readInt();
-		resolveVersionRead(foundVersion);
+		this.readVersion = in.readInt();
+		resolveVersionRead(readVersion);
 	}
 
 	/**
-	 * This method is a hook to react on the version tag that we find during read. This can also be used to initialize
-	 * further read logic w.r.t. the version at hand.
-	 * Default implementation of this method just checks the compatibility of a version number against the own version.
+	 * Returns the found serialization version. If this instance was not read from serialized bytes
+	 * but simply instantiated, then the current version is returned.
 	 *
-	 * @param foundVersion the version found from reading the input stream
-	 * @throws VersionMismatchException thrown when serialization versions mismatch
+	 * @return the read serialization version, or the current version if the instance was not read from bytes.
 	 */
-	protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
-		if (!isCompatibleVersion(foundVersion)) {
-			int expectedVersion = getVersion();
-			throw new VersionMismatchException(
-					"Incompatible version: found " + foundVersion + ", required " + expectedVersion);
-		}
+	public int getReadVersion() {
+		return (readVersion == Integer.MIN_VALUE) ? getVersion() : readVersion;
 	}
 
 	/**
-	 * Checks for compatibility between this and the found version. Subclasses can override this methods in case of
-	 * intended backwards backwards compatibility.
+	 * Returns the compatible version values.
 	 *
-	 * @param version version number to compare against.
-	 * @return true, iff this is compatible to the passed version.
+	 * <p>By default, the base implementation recognizes only the current version (identified by {@link #getVersion()})
+	 * as compatible. This method can be used as a hook and may be overridden to identify more compatible versions.
+	 *
+	 * @return an array of integers representing the compatible version values.
 	 */
-	public boolean isCompatibleVersion(int version) {
-		return getVersion() == version;
+	public int[] getCompatibleVersions() {
+		return new int[] {getVersion()};
+	}
+
+	private void resolveVersionRead(int readVersion) throws VersionMismatchException {
+
+		int[] compatibleVersions = getCompatibleVersions();
+		for (int compatibleVersion : compatibleVersions) {
+			if (compatibleVersion == readVersion) {
+				return;
+			}
+		}
+
+		throw new VersionMismatchException(
+			"Incompatible version: found " + readVersion + ", compatible versions are " + Arrays.toString(compatibleVersions));
 	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/core/io/VersionedIOWriteableTest.java b/flink-core/src/test/java/org/apache/flink/core/io/VersionedIOWriteableTest.java
index b7b6d6fd93597..ec5f79207b290 100644
--- a/flink-core/src/test/java/org/apache/flink/core/io/VersionedIOWriteableTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/io/VersionedIOWriteableTest.java
@@ -65,8 +65,8 @@ public void testReadCompatibleVersion() throws Exception {
 
 		testWriteable = new TestWriteable(2) {
 			@Override
-			public boolean isCompatibleVersion(int version) {
-				return getVersion() >= version;
+			public int[] getCompatibleVersions() {
+				return new int[] {1, 2};
 			}
 		};
 		try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
@@ -130,19 +130,9 @@ public void read(DataInputView in) throws IOException {
 			this.data = in.readUTF();
 		}
 
-		@Override
-		protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
-			super.resolveVersionRead(foundVersion);
-		}
-
-		@Override
-		public boolean isCompatibleVersion(int version) {
-			return super.isCompatibleVersion(version);
-		}
-
 		public String getData() {
 			return data;
 		}
 	}
 
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java
index a389c4f7ef7f8..a20628c517fec 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java
@@ -20,7 +20,6 @@
 
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerSerializationProxy;
-import org.apache.flink.core.io.VersionMismatchException;
 import org.apache.flink.core.io.VersionedIOReadableWritable;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
@@ -41,7 +40,6 @@ public class KeyedBackendSerializationProxy extends VersionedIOReadableWritable
 	private TypeSerializer<?> keySerializer;
 	private List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots;
 
-	private int restoredVersion;
 	private ClassLoader userCodeClassLoader;
 
 	public KeyedBackendSerializationProxy(ClassLoader userCodeClassLoader) {
@@ -57,8 +55,6 @@ public KeyedBackendSerializationProxy(
 		Preconditions.checkNotNull(stateMetaInfoSnapshots);
 		Preconditions.checkArgument(stateMetaInfoSnapshots.size() <= Short.MAX_VALUE);
 		this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
-
-		this.restoredVersion = VERSION;
 	}
 
 	public List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> getStateMetaInfoSnapshots() {
@@ -74,20 +70,10 @@ public int getVersion() {
 		return VERSION;
 	}
 
-	public int getRestoredVersion() {
-		return restoredVersion;
-	}
-
-	@Override
-	protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
-		super.resolveVersionRead(foundVersion);
-		this.restoredVersion = foundVersion;
-	}
-
 	@Override
-	public boolean isCompatibleVersion(int version) {
+	public int[] getCompatibleVersions() {
 		// we are compatible with version 3 (Flink 1.3.x) and version 1 & 2 (Flink 1.2.x)
-		return super.isCompatibleVersion(version) || version == 2 || version == 1;
+		return new int[] {VERSION, 2, 1};
 	}
 
 	@Override
@@ -119,7 +105,7 @@ public void read(DataInputView in) throws IOException {
 		for (int i = 0; i < numKvStates; i++) {
 			stateMetaInfoSnapshots.add(
 				KeyedBackendStateMetaInfoSnapshotReaderWriters
-					.getReaderForVersion(restoredVersion, userCodeClassLoader)
+					.getReaderForVersion(getReadVersion(), userCodeClassLoader)
 					.readStateMetaInfo(in));
 		}
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java
index 91d7aab10f5db..074d84e6b4c2c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.runtime.state;
 
-import org.apache.flink.core.io.VersionMismatchException;
 import org.apache.flink.core.io.VersionedIOReadableWritable;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
@@ -39,8 +38,6 @@ public class OperatorBackendSerializationProxy extends VersionedIOReadableWritab
 	private List<RegisteredOperatorBackendStateMetaInfo.Snapshot<?>> stateMetaInfoSnapshots;
 	private ClassLoader userCodeClassLoader;
 
-	private int restoredVersion;
-
 	public OperatorBackendSerializationProxy(ClassLoader userCodeClassLoader) {
 		this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
 	}
@@ -50,8 +47,6 @@ public OperatorBackendSerializationProxy(
 
 		this.stateMetaInfoSnapshots = Preconditions.checkNotNull(stateMetaInfoSnapshots);
 		Preconditions.checkArgument(stateMetaInfoSnapshots.size() <= Short.MAX_VALUE);
-
-		this.restoredVersion = VERSION;
 	}
 
 	@Override
@@ -60,15 +55,9 @@ public int getVersion() {
 	}
 
 	@Override
-	protected void resolveVersionRead(int foundVersion) throws VersionMismatchException {
-		super.resolveVersionRead(foundVersion);
-		this.restoredVersion = foundVersion;
-	}
-
-	@Override
-	public boolean isCompatibleVersion(int version) {
+	public int[] getCompatibleVersions() {
 		// we are compatible with version 2 (Flink 1.3.x) and version 1 (Flink 1.2.x)
-		return super.isCompatibleVersion(version) || version == 1;
+		return new int[] {VERSION, 1};
 	}
 
 	@Override
@@ -92,7 +81,7 @@ public void read(DataInputView in) throws IOException {
 		for (int i = 0; i < numKvStates; i++) {
 			stateMetaInfoSnapshots.add(
 				OperatorBackendStateMetaInfoSnapshotReaderWriters
-					.getReaderForVersion(restoredVersion, userCodeClassLoader)
+					.getReaderForVersion(getReadVersion(), userCodeClassLoader)
 					.readStateMetaInfo(in));
 		}
 	}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index bc314df73e1bd..11e7760fa3d56 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -448,7 +448,7 @@ private void restorePartitionedState(Collection<KeyedStateHandle> state) throws
 						StateTableByKeyGroupReader keyGroupReader =
 								StateTableByKeyGroupReaders.readerForVersion(
 										stateTable,
-										serializationProxy.getRestoredVersion());
+										serializationProxy.getReadVersion());
 
 						keyGroupReader.readMappingsInKeyGroup(inView, keyGroupIndex);
 					}

From 09a57cf666bd1359f301f9edb11391c2f3144846 Mon Sep 17 00:00:00 2001
From: "Tzu-Li (Gordon) Tai" <tzulitai@apache.org>
Date: Fri, 12 May 2017 21:00:51 +0800
Subject: [PATCH 244/253] [FLINK-6554] [core] Make CompatibilityResult options
 more explicitly defined

Previously, if a serializer determines that state migration needs to be
performed but could not provide a fallback convert deserializer, it
would use CompatibilityResult.requiresMigration(null).

This commit makes this option more explicit by having a
CompatibilityResult.requiresMigration() option that takes no parameters.
This improves how the user perceives the API without having to rely on
the Javadoc that it is allowed to have no fallback convert deserializer.

Consequently, when using
CompatibilityResult.requiresMigration(TypeDeserializer), the provided
argument cannot be null.

This closes #3886.
---
 .../typeutils/runtime/WritableSerializer.java |  2 +-
 .../state/RocksDBKeyedStateBackend.java       |  2 +-
 .../common/typeutils/CompatibilityResult.java | 36 +++++++++++++------
 .../api/common/typeutils/TypeSerializer.java  |  9 +++--
 .../common/typeutils/base/EnumSerializer.java |  2 +-
 .../base/GenericArraySerializer.java          |  6 ++--
 .../common/typeutils/base/ListSerializer.java |  4 +--
 .../common/typeutils/base/MapSerializer.java  |  4 +--
 .../base/TypeSerializerSingleton.java         |  2 +-
 .../typeutils/runtime/AvroSerializer.java     |  4 +--
 .../runtime/CopyableValueSerializer.java      |  2 +-
 .../typeutils/runtime/EitherSerializer.java   |  4 +--
 .../typeutils/runtime/PojoSerializer.java     | 16 ++++-----
 .../java/typeutils/runtime/RowSerializer.java |  6 ++--
 .../runtime/TupleSerializerBase.java          |  6 ++--
 .../typeutils/runtime/ValueSerializer.java    |  2 +-
 .../runtime/kryo/KryoSerializer.java          |  4 +--
 .../common/typeutils/SerializerTestBase.java  |  4 +--
 .../typeutils/base/EnumSerializerTest.java    |  6 ++--
 .../typeutils/runtime/PojoSerializerTest.java | 10 +++---
 .../kryo/KryoSerializerCompatibilityTest.java |  4 +--
 .../AbstractKeyedCEPPatternOperator.java      |  4 +--
 .../table/runtime/types/CRowSerializer.scala  |  6 ++--
 .../runtime/state/ArrayListSerializer.java    |  4 +--
 .../runtime/state/HashMapSerializer.java      |  4 +--
 .../runtime/state/StateMigrationUtil.java     |  4 +--
 .../scala/typeutils/EitherSerializer.scala    |  8 +++--
 .../scala/typeutils/EnumValueSerializer.scala |  6 ++--
 .../scala/typeutils/OptionSerializer.scala    |  6 ++--
 .../api/scala/typeutils/TrySerializer.scala   |  6 ++--
 .../MultiplexingStreamRecordSerializer.java   |  4 +--
 .../streamrecord/StreamRecordSerializer.java  |  6 ++--
 .../streamrecord/StreamElementSerializer.java |  4 +--
 33 files changed, 109 insertions(+), 88 deletions(-)

diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
index 1a02e7be6f6f8..421d7a3ed69a7 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
@@ -171,7 +171,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 
 			return CompatibilityResult.compatible();
 		} else {
-			return CompatibilityResult.requiresMigration(null);
+			return CompatibilityResult.requiresMigration();
 		}
 	}
 
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index f5dddd6f2f0e4..6af53c3e978ff 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -1520,7 +1520,7 @@ protected <N, S> ColumnFamilyHandle getColumnFamily(
 					restoredMetaInfo.getStateSerializerConfigSnapshot(),
 					newMetaInfo.getStateSerializer());
 
-			if (!namespaceCompatibility.requiresMigration() && !stateCompatibility.requiresMigration()) {
+			if (!namespaceCompatibility.isRequiresMigration() && !stateCompatibility.isRequiresMigration()) {
 				stateInfo.f1 = newMetaInfo;
 				return stateInfo.f0;
 			} else {
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
index 891cfe0eebd9d..5ad0b5e0a9273 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java
@@ -19,6 +19,7 @@
 package org.apache.flink.api.common.typeutils;
 
 import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.util.Preconditions;
 
 /**
  * A {@code CompatibilityResult} contains information about whether or not data migration
@@ -41,7 +42,7 @@ public final class CompatibilityResult<T> {
 	private final TypeDeserializer<T> convertDeserializer;
 
 	/**
-	 * Returns a strategy that signals that the new serializer is compatible and no migration is required.
+	 * Returns a result that signals that the new serializer is compatible and no migration is required.
 	 *
 	 * @return a result that signals migration is not required for the new serializer
 	 */
@@ -50,19 +51,32 @@ public static <T> CompatibilityResult<T> compatible() {
 	}
 
 	/**
-	 * Returns a strategy that signals migration to be performed.
+	 * Returns a result that signals migration to be performed, and in the case that the preceding serializer
+	 * cannot be found or restored to read the previous data during migration, a provided convert deserializer
+	 * can be used.
 	 *
-	 * <p>Furthermore, in the case that the preceding serializer cannot be found or restored to read the
-	 * previous data during migration, a provided convert deserializer can be used (may be {@code null}
-	 * if one cannot be provided).
+	 * @param convertDeserializer the convert deserializer to use, in the case that the preceding serializer
+	 *                            cannot be found.
 	 *
-	 * <p>In the case that the preceding serializer cannot be found and a convert deserializer is not
-	 * provided, the migration will fail due to the incapability of reading previous data.
-	 *
-	 * @return a result that signals migration is necessary, possibly providing a convert deserializer.
+	 * @return a result that signals migration is necessary, also providing a convert deserializer.
 	 */
 	public static <T> CompatibilityResult<T> requiresMigration(TypeDeserializer<T> convertDeserializer) {
-		return new CompatibilityResult<>(true, convertDeserializer);
+		Preconditions.checkNotNull(convertDeserializer, "Convert deserializer cannot be null.");
+
+		return new CompatibilityResult<>(true, Preconditions.checkNotNull(convertDeserializer));
+	}
+
+	/**
+	 * Returns a result that signals migration to be performed. The migration will fail if the preceding
+	 * serializer for the previous data cannot be found.
+	 *
+	 * <p>You can also provide a convert deserializer using {@link #requiresMigration(TypeDeserializer)},
+	 * which will be used as a fallback resort in such cases.
+	 *
+	 * @return a result that signals migration is necessary, without providing a convert deserializer.
+	 */
+	public static <T> CompatibilityResult<T> requiresMigration() {
+		return new CompatibilityResult<>(true, null);
 	}
 
 	private CompatibilityResult(boolean requiresMigration, TypeDeserializer<T> convertDeserializer) {
@@ -74,7 +88,7 @@ public TypeDeserializer<T> getConvertDeserializer() {
 		return convertDeserializer;
 	}
 
-	public boolean requiresMigration() {
+	public boolean isRequiresMigration() {
 		return requiresMigration;
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
index 0b5a08a5203ee..85cbfdbbd1515 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java
@@ -201,14 +201,19 @@ public abstract class TypeSerializer<T> implements TypeDeserializer<T>, Serializ
 	 *     migration needs to be performed, because this serializer is not compatible, or cannot be reconfigured to be
 	 *     compatible, for previous data. Furthermore, in the case that the preceding serializer cannot be found or
 	 *     restored to read the previous data to perform the migration, the provided convert deserializer can be
-	 *     used (may be {@code null} if one cannot be provided).</li>
+	 *     used as a fallback resort.</li>
+	 *
+	 *     <li>{@link CompatibilityResult#requiresMigration()}: this signals Flink that migration needs to be
+	 *     performed, because this serializer is not compatible, or cannot be reconfigured to be compatible, for
+	 *     previous data. If the preceding serializer cannot be found (either its implementation changed or it was
+	 *     removed from the classpath) then the migration will fail due to incapability to read previous data.</li>
 	 * </ul>
 	 *
 	 * @see CompatibilityResult
 	 *
 	 * @param configSnapshot configuration snapshot of a preceding serializer for the same managed state
 	 *
-	 * @return the determined compatibility result.
+	 * @return the determined compatibility result (cannot be {@code null}).
 	 */
 	public abstract CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot);
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
index 2f74d84fc60bf..d9246ae0bb791 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
@@ -205,7 +205,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	/**
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
index 3e592b48a4b52..54c604cffa8d1 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
@@ -208,8 +208,8 @@ public CompatibilityResult<C[]> ensureCompatibility(TypeSerializerConfigSnapshot
 				CompatibilityResult<C> compatResult = componentSerializer.ensureCompatibility(
 					config.getSingleNestedSerializerConfigSnapshot());
 
-				if (!compatResult.requiresMigration()) {
-					return CompatibilityResult.requiresMigration(null);
+				if (!compatResult.isRequiresMigration()) {
+					return CompatibilityResult.compatible();
 				} else if (compatResult.getConvertDeserializer() != null) {
 					return CompatibilityResult.requiresMigration(
 						new GenericArraySerializer<>(
@@ -219,6 +219,6 @@ public CompatibilityResult<C[]> ensureCompatibility(TypeSerializerConfigSnapshot
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
index 1b6540c37ecd6..aa9808efe2353 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java
@@ -185,7 +185,7 @@ public CompatibilityResult<List<T>> ensureCompatibility(TypeSerializerConfigSnap
 			CompatibilityResult<T> compatResult = elementSerializer.ensureCompatibility(
 				((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
 
-			if (!compatResult.requiresMigration()) {
+			if (!compatResult.isRequiresMigration()) {
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
@@ -193,6 +193,6 @@ public CompatibilityResult<List<T>> ensureCompatibility(TypeSerializerConfigSnap
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
index 182fff6929665..d5d6ec8b74a9c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java
@@ -216,7 +216,7 @@ public CompatibilityResult<Map<K, V>> ensureCompatibility(TypeSerializerConfigSn
 			CompatibilityResult<K> keyCompatResult = keySerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[0]);
 			CompatibilityResult<V> valueCompatResult = valueSerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[1]);
 
-			if (!keyCompatResult.requiresMigration() && !valueCompatResult.requiresMigration()) {
+			if (!keyCompatResult.isRequiresMigration() && !valueCompatResult.isRequiresMigration()) {
 				return CompatibilityResult.compatible();
 			} else if (keyCompatResult.getConvertDeserializer() != null && valueCompatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
@@ -226,6 +226,6 @@ public CompatibilityResult<Map<K, V>> ensureCompatibility(TypeSerializerConfigSn
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java
index c5decc5c2658c..9354af07e18a4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java
@@ -66,7 +66,7 @@ && isCompatibleSerializationFormatIdentifier(
 
 			return CompatibilityResult.compatible();
 		} else {
-			return CompatibilityResult.requiresMigration(null);
+			return CompatibilityResult.requiresMigration();
 		}
 	}
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
index c9eeb342c65a0..565bd4d654b80 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
@@ -238,7 +238,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 
 				for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : kryoRegistrations.entrySet()) {
 					if (reconfiguredRegistrationEntry.getValue().isDummy()) {
-						return CompatibilityResult.requiresMigration(null);
+						return CompatibilityResult.requiresMigration();
 					}
 				}
 
@@ -249,7 +249,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 
 		// ends up here if the preceding serializer is not
 		// the ValueSerializer, or serialized data type has changed
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	public static class AvroSerializerConfigSnapshot<T> extends KryoRegistrationSerializerConfigSnapshot<T> {
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
index 46b93c2000fca..b9039690f9ff3 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
@@ -147,7 +147,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 				&& valueClass.equals(((CopyableValueSerializerConfigSnapshot) configSnapshot).getTypeClass())) {
 			return CompatibilityResult.compatible();
 		} else {
-			return CompatibilityResult.requiresMigration(null);
+			return CompatibilityResult.requiresMigration();
 		}
 	}
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
index 461dd8721ad8e..4373ee06145c8 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
@@ -207,7 +207,7 @@ public CompatibilityResult<Either<L, R>> ensureCompatibility(TypeSerializerConfi
 			CompatibilityResult<L> leftCompatResult = leftSerializer.ensureCompatibility(leftRightSerializerConfigSnapshots[0]);
 			CompatibilityResult<R> rightCompatResult = rightSerializer.ensureCompatibility(leftRightSerializerConfigSnapshots[1]);
 
-			if (!leftCompatResult.requiresMigration() && !rightCompatResult.requiresMigration()) {
+			if (!leftCompatResult.isRequiresMigration() && !rightCompatResult.isRequiresMigration()) {
 				return CompatibilityResult.compatible();
 			} else {
 				if (leftCompatResult.getConvertDeserializer() != null && rightCompatResult.getConvertDeserializer() != null) {
@@ -219,6 +219,6 @@ public CompatibilityResult<Either<L, R>> ensureCompatibility(TypeSerializerConfi
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index 08da49eb91d89..a8368c4462c18 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -580,13 +580,13 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 							reorderedFields[i] = fieldToConfigSnapshotEntry.getKey();
 
 							compatResult = fieldSerializers[fieldIndex].ensureCompatibility(fieldToConfigSnapshotEntry.getValue());
-							if (compatResult.requiresMigration()) {
-								return CompatibilityResult.requiresMigration(null);
+							if (compatResult.isRequiresMigration()) {
+								return CompatibilityResult.requiresMigration();
 							} else {
 								reorderedFieldSerializers[i] = fieldSerializers[fieldIndex];
 							}
 						} else {
-							return CompatibilityResult.requiresMigration(null);
+							return CompatibilityResult.requiresMigration();
 						}
 
 						i++;
@@ -618,8 +618,8 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 					for (TypeSerializerConfigSnapshot previousRegisteredSerializerConfig : previousRegistrations.values()) {
 						// check compatibility of subclass serializer
 						compatResult = reorderedRegisteredSubclassSerializers[i].ensureCompatibility(previousRegisteredSerializerConfig);
-						if (compatResult.requiresMigration()) {
-							return CompatibilityResult.requiresMigration(null);
+						if (compatResult.isRequiresMigration()) {
+							return CompatibilityResult.requiresMigration();
 						}
 
 						i++;
@@ -638,8 +638,8 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 
 						// check compatibility of cached subclass serializer
 						compatResult = cachedSerializer.ensureCompatibility(previousCachedEntry.getValue());
-						if (compatResult.requiresMigration()) {
-							return CompatibilityResult.requiresMigration(null);
+						if (compatResult.isRequiresMigration()) {
+							return CompatibilityResult.requiresMigration();
 						} else {
 							rebuiltCache.put(previousCachedEntry.getKey(), cachedSerializer);
 						}
@@ -661,7 +661,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	public static final class PojoSerializerConfigSnapshot<T> extends GenericTypeSerializerConfigSnapshot<T> {
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
index 075c9d3483dcb..ba41d4bc38749 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java
@@ -270,12 +270,12 @@ public CompatibilityResult<Row> ensureCompatibility(TypeSerializerConfigSnapshot
 				CompatibilityResult<?> compatResult;
 				for (int i = 0; i < fieldSerializers.length; i++) {
 					compatResult = fieldSerializers[i].ensureCompatibility(fieldSerializerConfigSnapshots[i]);
-					if (compatResult.requiresMigration()) {
+					if (compatResult.isRequiresMigration()) {
 						requireMigration = true;
 
 						if (compatResult.getConvertDeserializer() == null) {
 							// one of the field serializers cannot provide a fallback deserializer
-							return CompatibilityResult.requiresMigration(null);
+							return CompatibilityResult.requiresMigration();
 						} else {
 							convertDeserializers[i] =
 								new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer());
@@ -291,7 +291,7 @@ public CompatibilityResult<Row> ensureCompatibility(TypeSerializerConfigSnapshot
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	public static final class RowSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot {
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
index 68d5aa8f73f3b..032c3f12ab777 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
@@ -145,8 +145,8 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 					CompatibilityResult compatResult;
 					for (int i = 0; i < fieldSerializers.length; i++) {
 						compatResult = fieldSerializers[i].ensureCompatibility(fieldSerializerConfigSnapshots[i]);
-						if (compatResult.requiresMigration()) {
-							return CompatibilityResult.requiresMigration(null);
+						if (compatResult.isRequiresMigration()) {
+							return CompatibilityResult.requiresMigration();
 						}
 					}
 
@@ -155,6 +155,6 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
index 10e233037b61e..0a028ebd79d66 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
@@ -193,7 +193,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	public static class ValueSerializerConfigSnapshot<T extends Value> extends KryoRegistrationSerializerConfigSnapshot<T> {
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
index a172b722f2a6b..655de7682a608 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
@@ -399,7 +399,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 							"proper serializer, because its previous serializer cannot be loaded or is no " +
 							"longer valid but a new serializer is not available", reconfiguredRegistrationEntry.getKey());
 
-						return CompatibilityResult.requiresMigration(null);
+						return CompatibilityResult.requiresMigration();
 					}
 				}
 
@@ -410,7 +410,7 @@ public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot c
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	public static final class KryoSerializerConfigSnapshot<T> extends KryoRegistrationSerializerConfigSnapshot<T> {
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
index f2879ac7f783e..73c43795dfe97 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
@@ -121,11 +121,11 @@ public void testSnapshotConfigurationAndReconfigure() throws Exception {
 		}
 
 		CompatibilityResult strategy = getSerializer().ensureCompatibility(restoredConfig);
-		assertFalse(strategy.requiresMigration());
+		assertFalse(strategy.isRequiresMigration());
 
 		// also verify that the serializer's reconfigure implementation detects incompatibility
 		strategy = getSerializer().ensureCompatibility(new TestIncompatibleSerializerConfigSnapshot());
-		assertTrue(strategy.requiresMigration());
+		assertTrue(strategy.isRequiresMigration());
 	}
 	
 	@Test
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java
index 5c615de87c8e9..16ea945e6ed8e 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java
@@ -74,7 +74,7 @@ public void testReconfiguration() {
 		// reconfigure and verify compatibility
 		CompatibilityResult<PublicEnum> compatResult = serializer.ensureCompatibility(
 			new EnumSerializer.EnumSerializerConfigSnapshot<>(PublicEnum.class, mockPreviousOrder));
-		assertFalse(compatResult.requiresMigration());
+		assertFalse(compatResult.isRequiresMigration());
 
 		// after reconfiguration, the order should be first the original BAR, PAULA, NATHANIEL,
 		// followed by the "new enum constants" FOO, PETER, EMMA
@@ -107,7 +107,7 @@ public void testConfigurationSnapshotSerialization() throws Exception {
 		}
 
 		CompatibilityResult<PublicEnum> compatResult = serializer.ensureCompatibility(restoredConfig);
-		assertFalse(compatResult.requiresMigration());
+		assertFalse(compatResult.isRequiresMigration());
 
 		assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
 		assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
@@ -163,7 +163,7 @@ public void testSerializeReconfiguredEnumSerializer() throws Exception {
 		// reconfigure and verify compatibility
 		CompatibilityResult<PublicEnum> compatResult = serializer.ensureCompatibility(
 			new EnumSerializer.EnumSerializerConfigSnapshot<>(PublicEnum.class, mockPreviousOrder));
-		assertFalse(compatResult.requiresMigration());
+		assertFalse(compatResult.isRequiresMigration());
 
 		// serialize and deserialize again the serializer
 		byte[] serializedSerializer = InstantiationUtil.serializeObject(serializer);
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
index 5459d538d7053..c77ffccc91944 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
@@ -300,7 +300,7 @@ public void testReconfigureWithDifferentPojoType() throws Exception {
 		}
 
 		CompatibilityResult<SubTestUserClassA> compatResult = pojoSerializer2.ensureCompatibility(pojoSerializerConfigSnapshot);
-		assertTrue(compatResult.requiresMigration());
+		assertTrue(compatResult.isRequiresMigration());
 	}
 
 	/**
@@ -340,7 +340,7 @@ public void testReconfigureDifferentSubclassRegistrationOrder() throws Exception
 		}
 
 		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot);
-		assertTrue(!compatResult.requiresMigration());
+		assertTrue(!compatResult.isRequiresMigration());
 
 		// reconfigure - check reconfiguration result and that registration ids remains the same
 		//assertEquals(ReconfigureResult.COMPATIBLE, pojoSerializer.reconfigure(pojoSerializerConfigSnapshot));
@@ -384,7 +384,7 @@ public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() thro
 
 		// reconfigure - check reconfiguration result and that subclass serializer cache is repopulated
 		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot);
-		assertFalse(compatResult.requiresMigration());
+		assertFalse(compatResult.isRequiresMigration());
 		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
 		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
 		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
@@ -446,7 +446,7 @@ public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Except
 		// 1) subclass serializer cache is repopulated
 		// 2) registrations also contain the now registered subclasses
 		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot);
-		assertFalse(compatResult.requiresMigration());
+		assertFalse(compatResult.isRequiresMigration());
 		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
 		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
 		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
@@ -501,7 +501,7 @@ public void testReconfigureWithDifferentFieldOrder() throws Exception {
 		CompatibilityResult<TestUserClass> compatResult = pojoSerializer.ensureCompatibility(
 
 			mockPreviousConfigSnapshot);
-		assertFalse(compatResult.requiresMigration());
+		assertFalse(compatResult.isRequiresMigration());
 		int i = 0;
 		for (Field field : mockOriginalFieldOrder) {
 			assertEquals(field, pojoSerializer.getFields()[i]);
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
index 60c4dc45cfe7d..860c56029a267 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
@@ -66,7 +66,7 @@ public void testMigrationStrategyWithDifferentKryoType() throws Exception {
 		}
 
 		CompatibilityResult<TestClassB> compatResult = kryoSerializerForB.ensureCompatibility(kryoSerializerConfigSnapshot);
-		assertTrue(compatResult.requiresMigration());
+		assertTrue(compatResult.isRequiresMigration());
 	}
 
 	/**
@@ -110,7 +110,7 @@ public void testMigrationStrategyForDifferentRegistrationOrder() throws Exceptio
 
 		// reconfigure - check reconfiguration result and that registration id remains the same
 		CompatibilityResult<TestClass> compatResult = kryoSerializer.ensureCompatibility(kryoSerializerConfigSnapshot);
-		assertFalse(compatResult.requiresMigration());
+		assertFalse(compatResult.isRequiresMigration());
 		assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId());
 		assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId());
 		assertEquals(testClassBId, kryoSerializer.getKryo().getRegistration(TestClassB.class).getId());
diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
index 140e091382a6a..3afe397ecbe3c 100644
--- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
+++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/AbstractKeyedCEPPatternOperator.java
@@ -513,7 +513,7 @@ public CompatibilityResult<PriorityQueue<T>> ensureCompatibility(TypeSerializerC
 				CompatibilityResult<T> compatResult = elementSerializer.ensureCompatibility(
 						((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
 
-				if (!compatResult.requiresMigration()) {
+				if (!compatResult.isRequiresMigration()) {
 					return CompatibilityResult.compatible();
 				} else if (compatResult.getConvertDeserializer() != null) {
 					return CompatibilityResult.requiresMigration(
@@ -522,7 +522,7 @@ public CompatibilityResult<PriorityQueue<T>> ensureCompatibility(TypeSerializerC
 				}
 			}
 
-			return CompatibilityResult.requiresMigration(null);
+			return CompatibilityResult.requiresMigration();
 		}
 	}
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
index 122f4fba1b493..0fd3680d17953 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
@@ -93,20 +93,20 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali
         val compatResult = rowSerializer.ensureCompatibility(
             crowSerializerConfigSnapshot.getSingleNestedSerializerConfigSnapshot)
 
-        if (compatResult.requiresMigration()) {
+        if (compatResult.isRequiresMigration) {
           if (compatResult.getConvertDeserializer != null) {
             CompatibilityResult.requiresMigration(
               new CRowSerializer(
                 new TypeDeserializerAdapter(compatResult.getConvertDeserializer))
             )
           } else {
-            CompatibilityResult.requiresMigration(null)
+            CompatibilityResult.requiresMigration()
           }
         } else {
           CompatibilityResult.compatible()
         }
 
-      case _ => CompatibilityResult.requiresMigration(null)
+      case _ => CompatibilityResult.requiresMigration()
     }
   }
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
index c39cb9bb3d4aa..56eb7ea2d188a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java
@@ -152,7 +152,7 @@ public CompatibilityResult<ArrayList<T>> ensureCompatibility(TypeSerializerConfi
 			CompatibilityResult<T> compatResult = elementSerializer.ensureCompatibility(
 				((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
 
-			if (!compatResult.requiresMigration()) {
+			if (!compatResult.isRequiresMigration()) {
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
@@ -160,6 +160,6 @@ public CompatibilityResult<ArrayList<T>> ensureCompatibility(TypeSerializerConfi
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
index 925fe7883aa96..b93c9e0c5000d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/HashMapSerializer.java
@@ -217,7 +217,7 @@ public CompatibilityResult<HashMap<K, V>> ensureCompatibility(TypeSerializerConf
 			CompatibilityResult<K> keyCompatResult = keySerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[0]);
 			CompatibilityResult<V> valueCompatResult = valueSerializer.ensureCompatibility(keyValueSerializerConfigSnapshots[1]);
 
-			if (!keyCompatResult.requiresMigration() && !valueCompatResult.requiresMigration()) {
+			if (!keyCompatResult.isRequiresMigration() && !valueCompatResult.isRequiresMigration()) {
 				return CompatibilityResult.compatible();
 			} else if (keyCompatResult.getConvertDeserializer() != null && valueCompatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
@@ -227,6 +227,6 @@ public CompatibilityResult<HashMap<K, V>> ensureCompatibility(TypeSerializerConf
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java
index 978f28d15291d..39bb743a47b6e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateMigrationUtil.java
@@ -48,7 +48,7 @@ public class StateMigrationUtil {
 	 *
 	 * @param <T> Type of the data handled by the serializers
 	 *
-	 * @return the final resolved compatiblity result
+	 * @return the final resolved compatibility result
 	 */
 	public static <T> CompatibilityResult<T> resolveCompatibilityResult(
 			TypeSerializer<T> precedingSerializer,
@@ -59,7 +59,7 @@ public static <T> CompatibilityResult<T> resolveCompatibilityResult(
 		if (precedingSerializerConfigSnapshot != null) {
 			CompatibilityResult<T> initialResult = newSerializer.ensureCompatibility(precedingSerializerConfigSnapshot);
 
-			if (!initialResult.requiresMigration()) {
+			if (!initialResult.isRequiresMigration()) {
 				return initialResult;
 			} else {
 				if (precedingSerializer != null && !(precedingSerializer.getClass().equals(dummySerializerClassTag))) {
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
index 88b2041462e44..6c4378a249392 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala
@@ -127,7 +127,9 @@ class EitherSerializer[A, B, T <: Either[A, B]](
         val leftCompatResult = leftSerializer.ensureCompatibility(leftRightConfigs(0))
         val rightCompatResult = rightSerializer.ensureCompatibility(leftRightConfigs(1))
 
-        if (leftCompatResult.requiresMigration || rightCompatResult.requiresMigration) {
+        if (leftCompatResult.isRequiresMigration
+            || rightCompatResult.isRequiresMigration) {
+
           if (leftCompatResult.getConvertDeserializer != null
               && rightCompatResult.getConvertDeserializer != null) {
 
@@ -139,13 +141,13 @@ class EitherSerializer[A, B, T <: Either[A, B]](
             )
 
           } else {
-            CompatibilityResult.requiresMigration(null)
+            CompatibilityResult.requiresMigration()
           }
         } else {
           CompatibilityResult.compatible()
         }
 
-      case _ => CompatibilityResult.requiresMigration(null)
+      case _ => CompatibilityResult.requiresMigration()
     }
   }
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala
index dc96c9899cb9d..843079a5a05e8 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala
@@ -95,16 +95,16 @@ class EnumValueSerializer[E <: Enumeration](val enum: E) extends TypeSerializer[
             // and original constants must be in the exact same order
 
             if (currentEnumConstants(i) != enumSerializerConfigSnapshot.getEnumConstants(i)) {
-              CompatibilityResult.requiresMigration(null)
+              return CompatibilityResult.requiresMigration()
             }
           }
 
           CompatibilityResult.compatible()
         } else {
-          CompatibilityResult.requiresMigration(null)
+          CompatibilityResult.requiresMigration()
         }
 
-      case _ => CompatibilityResult.requiresMigration(null)
+      case _ => CompatibilityResult.requiresMigration()
     }
   }
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
index 81b3bccf9f213..4b56059546a52 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala
@@ -111,19 +111,19 @@ class OptionSerializer[A](val elemSerializer: TypeSerializer[A])
         val compatResult = elemSerializer.ensureCompatibility(
           optionSerializerConfigSnapshot.getSingleNestedSerializerConfigSnapshot)
 
-        if (compatResult.requiresMigration()) {
+        if (compatResult.isRequiresMigration) {
           if (compatResult.getConvertDeserializer != null) {
             CompatibilityResult.requiresMigration(
               new OptionSerializer[A](
                 new TypeDeserializerAdapter(compatResult.getConvertDeserializer)))
           } else {
-            CompatibilityResult.requiresMigration(null)
+            CompatibilityResult.requiresMigration()
           }
         } else {
           CompatibilityResult.compatible()
         }
 
-      case _ => CompatibilityResult.requiresMigration(null)
+      case _ => CompatibilityResult.requiresMigration()
     }
   }
 }
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
index c864dc784c9e6..5de76ca88dc0e 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
@@ -123,13 +123,13 @@ class TrySerializer[A](
         val throwableCompatRes =
           throwableSerializer.ensureCompatibility(serializerConfigSnapshots(1))
 
-        if (elemCompatRes.requiresMigration() || throwableCompatRes.requiresMigration()) {
-          CompatibilityResult.requiresMigration(null)
+        if (elemCompatRes.isRequiresMigration || throwableCompatRes.isRequiresMigration) {
+          CompatibilityResult.requiresMigration()
         } else {
           CompatibilityResult.compatible()
         }
 
-      case _ => CompatibilityResult.requiresMigration(null)
+      case _ => CompatibilityResult.requiresMigration()
     }
   }
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
index 552ffd032534c..dc23b8d0affc6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/MultiplexingStreamRecordSerializer.java
@@ -225,7 +225,7 @@ public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConf
 			CompatibilityResult<T> compatResult = typeSerializer.ensureCompatibility(
 				((MultiplexingStreamRecordSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
 
-			if (!compatResult.requiresMigration()) {
+			if (!compatResult.isRequiresMigration()) {
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
@@ -234,7 +234,7 @@ public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConf
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
index f7a661e32e883..7b0390d9bc93e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/migration/streaming/runtime/streamrecord/StreamRecordSerializer.java
@@ -164,8 +164,8 @@ public CompatibilityResult<StreamRecord<T>> ensureCompatibility(TypeSerializerCo
 			CompatibilityResult<T> compatResult = typeSerializer.ensureCompatibility(
 				((StreamRecordSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
 
-			if (!compatResult.requiresMigration()) {
-				return CompatibilityResult.requiresMigration(null);
+			if (!compatResult.isRequiresMigration()) {
+				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
 					new StreamRecordSerializer<>(
@@ -173,7 +173,7 @@ public CompatibilityResult<StreamRecord<T>> ensureCompatibility(TypeSerializerCo
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	/**
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
index e444ced44fd66..ba69fed6eb32a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java
@@ -286,7 +286,7 @@ public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConf
 			CompatibilityResult<T> compatResult = typeSerializer.ensureCompatibility(
 				((StreamElementSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerConfigSnapshot());
 
-			if (!compatResult.requiresMigration()) {
+			if (!compatResult.isRequiresMigration()) {
 				return CompatibilityResult.compatible();
 			} else if (compatResult.getConvertDeserializer() != null) {
 				return CompatibilityResult.requiresMigration(
@@ -295,7 +295,7 @@ public CompatibilityResult<StreamElement> ensureCompatibility(TypeSerializerConf
 			}
 		}
 
-		return CompatibilityResult.requiresMigration(null);
+		return CompatibilityResult.requiresMigration();
 	}
 
 	/**

From 3bbca7721ac94f709aaeacd9c47e2a46b8996e6d Mon Sep 17 00:00:00 2001
From: "biao.liub" <biao.liub@alibaba-inc.com>
Date: Tue, 2 May 2017 16:07:34 +0800
Subject: [PATCH 245/253] [FLINK-6397] [tests] Reset context environments after
 test execution

This closes #3810.
---
 .../apache/flink/ml/util/FlinkTestBase.scala  |  2 ++
 .../test/util/CollectionTestEnvironment.java  |  4 +++
 .../flink/test/util/JavaProgramTestBase.java  |  9 +++++--
 .../test/util/MultipleProgramsTestBase.java   | 25 +++++++++++++++++--
 .../CustomDistributionITCase.java             |  6 +++++
 5 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
index 3ee7a99ab6662..c27a2b574555b 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
@@ -70,6 +70,8 @@ trait FlinkTestBase extends BeforeAndAfter {
 
   after {
     cluster.foreach(c => TestBaseUtils.stopCluster(c, TestBaseUtils.DEFAULT_TIMEOUT))
+
+    TestEnvironment.unsetAsContext()
   }
 
 }
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java
index e56c7e8c2f0cd..fd047e4ea14fb 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java
@@ -60,4 +60,8 @@ public ExecutionEnvironment createExecutionEnvironment() {
 
 		initializeContextEnvironment(factory);
 	}
+
+	protected static void unsetAsContext() {
+		resetContextEnvironment();
+	}
 }
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
index f06cdb7ded786..a17b68f1745e0 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
@@ -97,8 +97,9 @@ protected boolean skipCollectionExecution() {
 	@Test
 	public void testJobWithObjectReuse() throws Exception {
 		isCollectionExecution = false;
-		
+
 		startCluster();
+
 		try {
 			// pre-submit
 			try {
@@ -109,7 +110,7 @@ public void testJobWithObjectReuse() throws Exception {
 				e.printStackTrace();
 				Assert.fail("Pre-submit work caused an error: " + e.getMessage());
 			}
-			
+
 			// prepare the test environment
 			TestEnvironment env = new TestEnvironment(this.executor, this.parallelism, false);
 			env.getConfig().enableObjectReuse();
@@ -143,6 +144,7 @@ public void testJobWithObjectReuse() throws Exception {
 			}
 		} finally {
 			stopCluster();
+			TestEnvironment.unsetAsContext();
 		}
 	}
 
@@ -195,6 +197,7 @@ public void testJobWithoutObjectReuse() throws Exception {
 			}
 		} finally {
 			stopCluster();
+			TestEnvironment.unsetAsContext();
 		}
 	}
 	
@@ -231,6 +234,8 @@ public void testJobCollectionExecution() throws Exception {
 			System.err.println(e.getMessage());
 			e.printStackTrace();
 			Assert.fail("Error while calling the test program: " + e.getMessage());
+		} finally {
+			CollectionTestEnvironment.unsetAsContext();
 		}
 		
 		Assert.assertNotNull("The test program never triggered an execution.", this.latestExecutionResult);
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
index ed141d12ebc98..468f8c785eefd 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
@@ -19,7 +19,9 @@
 package org.apache.flink.test.util;
 
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.runners.Parameterized;
 
@@ -80,10 +82,16 @@ public enum TestExecutionMode {
 	
 	protected final TestExecutionMode mode;
 
-	
 	public MultipleProgramsTestBase(TestExecutionMode mode) {
 		this.mode = mode;
-		
+	}
+
+	// ------------------------------------------------------------------------
+	//  Environment setup & teardown
+	// ------------------------------------------------------------------------
+
+	@Before
+	public void setupEnvironment() {
 		switch(mode){
 			case CLUSTER:
 				new TestEnvironment(cluster, 4, false).setAsContext();
@@ -97,6 +105,19 @@ public MultipleProgramsTestBase(TestExecutionMode mode) {
 		}
 	}
 
+	@After
+	public void teardownEnvironment() {
+		switch(mode) {
+			case CLUSTER:
+			case CLUSTER_OBJECT_REUSE:
+				TestEnvironment.unsetAsContext();
+				break;
+			case COLLECTION:
+				CollectionTestEnvironment.unsetAsContext();
+				break;
+		}
+	}
+
 	// ------------------------------------------------------------------------
 	//  Cluster setup & teardown
 	// ------------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
index da92c05ab02a3..24a09ba29045f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
@@ -36,6 +36,7 @@
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.TestLogger;
 
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -70,6 +71,11 @@ public void prepare() {
 		clusterEnv.setAsContext();
 	}
 
+	@After
+	public void cleanup() {
+		TestEnvironment.unsetAsContext();
+	}
+
 	// ------------------------------------------------------------------------
 
 	/**

From d1abeeaca3a3941d9833c8600cdb58a1e2a37b3f Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Wed, 10 May 2017 10:26:19 +0200
Subject: [PATCH 246/253] [FLINK-6518] Port blobserver config parameters to
 ConfigOptions

This closes #3865.
---
 .../configuration/BlobServerOptions.java      | 76 +++++++++++++++++++
 .../flink/configuration/ConfigConstants.java  | 40 ++++++----
 .../apache/flink/runtime/blob/BlobCache.java  |  9 +--
 .../apache/flink/runtime/blob/BlobClient.java |  5 +-
 .../apache/flink/runtime/blob/BlobServer.java | 22 +++---
 .../flink/runtime/blob/BlobClientSslTest.java |  5 +-
 .../runtime/blob/BlobServerRangeTest.java     |  8 +-
 .../jobmanager/JobManagerStartupTest.java     |  4 +-
 8 files changed, 125 insertions(+), 44 deletions(-)
 create mode 100644 flink-core/src/main/java/org/apache/flink/configuration/BlobServerOptions.java

diff --git a/flink-core/src/main/java/org/apache/flink/configuration/BlobServerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/BlobServerOptions.java
new file mode 100644
index 0000000000000..e27c29f2e0b46
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/BlobServerOptions.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import static org.apache.flink.configuration.ConfigOptions.key;
+
+/**
+ * Configuration options for the BlobServer.
+ */
+@PublicEvolving
+public class BlobServerOptions {
+
+	/**
+	 * The config parameter defining the storage directory to be used by the blob server.
+	 */
+	public static final ConfigOption<String> STORAGE_DIRECTORY =
+		key("blob.storage.directory")
+			.noDefaultValue();
+
+	/**
+	 * The config parameter defining number of retires for failed BLOB fetches.
+	 */
+	public static final ConfigOption<Integer> FETCH_RETRIES =
+		key("blob.fetch.retries")
+			.defaultValue(5);
+
+	/**
+	 * The config parameter defining the maximum number of concurrent BLOB fetches that the JobManager serves.
+	 */
+	public static final ConfigOption<Integer> FETCH_CONCURRENT =
+		key("blob.fetch.num-concurrent")
+			.defaultValue(50);
+
+	/**
+	 * The config parameter defining the backlog of BLOB fetches on the JobManager.
+	 */
+	public static final ConfigOption<Integer> FETCH_BACKLOG =
+		key("blob.fetch.backlog")
+			.defaultValue(1000);
+
+	/**
+	 * The config parameter defining the server port of the blob service.
+	 * The port can either be a port, such as "9123",
+	 * a range of ports: "50100-50200"
+	 * or a list of ranges and or points: "50100-50200,50300-50400,51234"
+	 *
+	 * Setting the port to 0 will let the OS choose an available port.
+	 */
+	public static final ConfigOption<String> PORT =
+		key("blob.server.port")
+			.defaultValue("0");
+
+	/**
+	 * Flag to override ssl support for the blob service transport.
+	 */
+	public static final ConfigOption<Boolean> SSL_ENABLED =
+		key("blob.service.ssl.enabled")
+			.defaultValue(true);
+}
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index c3704be43e29f..b5b5486b350ed 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -139,36 +139,39 @@ public final class ConfigConstants {
 	public static final String RESOURCE_MANAGER_IPC_PORT_KEY = "resourcemanager.rpc.port";
 
 	/**
-	 * The config parameter defining the storage directory to be used by the blob server.
+	 * @deprecated use {@link BlobServerOptions#STORAGE_DIRECTORY} instead
 	 */
+	@Deprecated
 	public static final String BLOB_STORAGE_DIRECTORY_KEY = "blob.storage.directory";
 
 	/**
-	 * The config parameter defining number of retires for failed BLOB fetches.
+	 * @deprecated use {@link BlobServerOptions#FETCH_RETRIES} instead
 	 */
+	@Deprecated
 	public static final String BLOB_FETCH_RETRIES_KEY = "blob.fetch.retries";
 
 	/**
-	 * The config parameter defining the maximum number of concurrent BLOB fetches that the JobManager serves.
+	 * @deprecated use {@link BlobServerOptions#FETCH_CONCURRENT} instead
 	 */
+	@Deprecated
 	public static final String BLOB_FETCH_CONCURRENT_KEY = "blob.fetch.num-concurrent";
 
 	/**
-	 * The config parameter defining the backlog of BLOB fetches on the JobManager
+	 * @deprecated use {@link BlobServerOptions#FETCH_BACKLOG} instead
 	 */
+	@Deprecated
 	public static final String BLOB_FETCH_BACKLOG_KEY = "blob.fetch.backlog";
 
 	/**
-	 * The config parameter defining the server port of the blob service.
-	 * The port can either be a port, such as "9123",
-	 * a range of ports: "50100-50200"
-	 * or a list of ranges and or points: "50100-50200,50300-50400,51234"
-	 *
-	 * Setting the port to 0 will let the OS choose an available port.
+	 * @deprecated use {@link BlobServerOptions#PORT} instead
 	 */
+	@Deprecated
 	public static final String BLOB_SERVER_PORT = "blob.server.port";
 
-	/** Flag to override ssl support for the blob service transport */
+	/**
+	 * @deprecated use {@link BlobServerOptions#SSL_ENABLED} instead
+	 */
+	@Deprecated
 	public static final String BLOB_SERVICE_SSL_ENABLED = "blob.service.ssl.enabled";
 
 	/**
@@ -1094,28 +1097,33 @@ public final class ConfigConstants {
 	public static final int DEFAULT_RESOURCE_MANAGER_IPC_PORT = 0;
 
 	/**
-	 * The default value to override ssl support for blob service transport
+	 * @deprecated use {@link BlobServerOptions#SSL_ENABLED} instead
 	 */
+	@Deprecated
 	public static final boolean DEFAULT_BLOB_SERVICE_SSL_ENABLED = true;
 
 	/**
-	 * Default number of retries for failed BLOB fetches.
+	 * @deprecated use {@link BlobServerOptions#FETCH_RETRIES} instead
 	 */
+	@Deprecated
 	public static final int DEFAULT_BLOB_FETCH_RETRIES = 5;
 
 	/**
-	 * Default number of concurrent BLOB fetch operations.
+	 * @deprecated use {@link BlobServerOptions#FETCH_CONCURRENT} instead
 	 */
+	@Deprecated
 	public static final int DEFAULT_BLOB_FETCH_CONCURRENT = 50;
 
 	/**
-	 * Default BLOB fetch connection backlog.
+	 * @deprecated use {@link BlobServerOptions#FETCH_BACKLOG} instead
 	 */
+	@Deprecated
 	public static final int DEFAULT_BLOB_FETCH_BACKLOG = 1000;
 
 	/**
-	 * Default BLOB server port. 0 means ephemeral port.
+	 * @deprecated use {@link BlobServerOptions#PORT} instead
 	 */
+	@Deprecated
 	public static final String DEFAULT_BLOB_SERVER_PORT = "0";
 
 	/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
index 2587b155d607d..23c7e63b403a5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
@@ -18,7 +18,7 @@
 
 package org.apache.flink.runtime.blob;
 
-import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.BlobServerOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.util.FileUtils;
@@ -129,19 +129,18 @@ private BlobCache(
 		this.blobStore = blobStore;
 
 		// configure and create the storage directory
-		String storageDirectory = blobClientConfig.getString(ConfigConstants.BLOB_STORAGE_DIRECTORY_KEY, null);
+		String storageDirectory = blobClientConfig.getString(BlobServerOptions.STORAGE_DIRECTORY);
 		this.storageDir = BlobUtils.initStorageDirectory(storageDirectory);
 		LOG.info("Created BLOB cache storage directory " + storageDir);
 
 		// configure the number of fetch retries
-		final int fetchRetries = blobClientConfig.getInteger(
-			ConfigConstants.BLOB_FETCH_RETRIES_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_RETRIES);
+		final int fetchRetries = blobClientConfig.getInteger(BlobServerOptions.FETCH_RETRIES);
 		if (fetchRetries >= 0) {
 			this.numFetchRetries = fetchRetries;
 		}
 		else {
 			LOG.warn("Invalid value for {}. System will attempt no retires on failed fetches of BLOBs.",
-				ConfigConstants.BLOB_FETCH_RETRIES_KEY);
+				BlobServerOptions.FETCH_RETRIES.key());
 			this.numFetchRetries = 0;
 		}
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobClient.java
index ea90f5441f517..49e54a16f3618 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobClient.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobClient.java
@@ -19,7 +19,7 @@
 package org.apache.flink.runtime.blob;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.BlobServerOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.fs.FSDataInputStream;
 import org.apache.flink.core.fs.FileSystem;
@@ -92,8 +92,7 @@ public BlobClient(InetSocketAddress serverAddress, Configuration clientConfig) t
 			// Check if ssl is enabled
 			SSLContext clientSSLContext = null;
 			if (clientConfig != null &&
-				clientConfig.getBoolean(ConfigConstants.BLOB_SERVICE_SSL_ENABLED,
-						ConfigConstants.DEFAULT_BLOB_SERVICE_SSL_ENABLED)) {
+				clientConfig.getBoolean(BlobServerOptions.SSL_ENABLED)) {
 
 				clientSSLContext = SSLUtils.createSSLClientContext(clientConfig);
 			}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
index 8a70559b589ec..0e157772defa2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
@@ -19,7 +19,7 @@
 package org.apache.flink.runtime.blob;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.BlobServerOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
@@ -111,34 +111,32 @@ private BlobServer(Configuration config, BlobStore blobStore) throws IOException
 		this.blobStore = checkNotNull(blobStore);
 
 		// configure and create the storage directory
-		String storageDirectory = config.getString(ConfigConstants.BLOB_STORAGE_DIRECTORY_KEY, null);
+		String storageDirectory = config.getString(BlobServerOptions.STORAGE_DIRECTORY);
 		this.storageDir = BlobUtils.initStorageDirectory(storageDirectory);
 		LOG.info("Created BLOB server storage directory {}", storageDir);
 
 		// configure the maximum number of concurrent connections
-		final int maxConnections = config.getInteger(
-				ConfigConstants.BLOB_FETCH_CONCURRENT_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_CONCURRENT);
+		final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
 		if (maxConnections >= 1) {
 			this.maxConnections = maxConnections;
 		}
 		else {
 			LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
-					maxConnections, ConfigConstants.DEFAULT_BLOB_FETCH_CONCURRENT);
-			this.maxConnections = ConfigConstants.DEFAULT_BLOB_FETCH_CONCURRENT;
+					maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
+			this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
 		}
 
 		// configure the backlog of connections
-		int backlog = config.getInteger(ConfigConstants.BLOB_FETCH_BACKLOG_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_BACKLOG);
+		int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
 		if (backlog < 1) {
 			LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
-					backlog, ConfigConstants.DEFAULT_BLOB_FETCH_BACKLOG);
-			backlog = ConfigConstants.DEFAULT_BLOB_FETCH_BACKLOG;
+					backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
+			backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
 		}
 
 		this.shutdownHook = BlobUtils.addShutdownHook(this, LOG);
 
-		if (config.getBoolean(ConfigConstants.BLOB_SERVICE_SSL_ENABLED,
-				ConfigConstants.DEFAULT_BLOB_SERVICE_SSL_ENABLED)) {
+		if (config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
 			try {
 				serverSSLContext = SSLUtils.createSSLServerContext(config);
 			} catch (Exception e) {
@@ -148,7 +146,7 @@ private BlobServer(Configuration config, BlobStore blobStore) throws IOException
 
 		//  ----------------------- start the server -------------------
 
-		String serverPortRange = config.getString(ConfigConstants.BLOB_SERVER_PORT, ConfigConstants.DEFAULT_BLOB_SERVER_PORT);
+		String serverPortRange = config.getString(BlobServerOptions.PORT);
 
 		Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobClientSslTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobClientSslTest.java
index 5054107ce6691..27603d0586940 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobClientSslTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobClientSslTest.java
@@ -31,6 +31,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.flink.configuration.BlobServerOptions;
 import org.apache.flink.configuration.ConfigConstants;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.api.common.JobID;
@@ -91,7 +92,7 @@ public static void startNonSSLServer() {
 		try {
 			Configuration config = new Configuration();
 			config.setBoolean(ConfigConstants.SECURITY_SSL_ENABLED, true);
-			config.setBoolean(ConfigConstants.BLOB_SERVICE_SSL_ENABLED, false);
+			config.setBoolean(BlobServerOptions.SSL_ENABLED, false);
 			config.setString(ConfigConstants.SECURITY_SSL_KEYSTORE, "src/test/resources/local127.keystore");
 			config.setString(ConfigConstants.SECURITY_SSL_KEYSTORE_PASSWORD, "password");
 			config.setString(ConfigConstants.SECURITY_SSL_KEY_PASSWORD, "password");
@@ -104,7 +105,7 @@ public static void startNonSSLServer() {
 
 		clientConfig = new Configuration();
 		clientConfig.setBoolean(ConfigConstants.SECURITY_SSL_ENABLED, true);
-		clientConfig.setBoolean(ConfigConstants.BLOB_SERVICE_SSL_ENABLED, false);
+		clientConfig.setBoolean(BlobServerOptions.SSL_ENABLED, false);
 		clientConfig.setString(ConfigConstants.SECURITY_SSL_TRUSTSTORE, "src/test/resources/local127.truststore");
 		clientConfig.setString(ConfigConstants.SECURITY_SSL_TRUSTSTORE_PASSWORD, "password");
 	}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerRangeTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerRangeTest.java
index ea0eb94721aae..c3762aa49f68a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerRangeTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerRangeTest.java
@@ -18,7 +18,7 @@
 
 package org.apache.flink.runtime.blob;
 
-import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.BlobServerOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.util.NetUtils;
 import org.apache.flink.util.TestLogger;
@@ -38,7 +38,7 @@ public class BlobServerRangeTest extends TestLogger {
 	@Test
 	public void testOnEphemeralPort() throws IOException {
 		Configuration conf = new Configuration();
-		conf.setString(ConfigConstants.BLOB_SERVER_PORT, "0");
+		conf.setString(BlobServerOptions.PORT, "0");
 		BlobServer srv = new BlobServer(conf);
 		srv.shutdown();
 	}
@@ -59,7 +59,7 @@ public void testPortUnavailable() throws IOException {
 		}
 
 		Configuration conf = new Configuration();
-		conf.setString(ConfigConstants.BLOB_SERVER_PORT, String.valueOf(socket.getLocalPort()));
+		conf.setString(BlobServerOptions.PORT, String.valueOf(socket.getLocalPort()));
 
 		// this thing is going to throw an exception
 		try {
@@ -88,7 +88,7 @@ public void testOnePortAvailable() throws IOException {
 		}
 		int availablePort = NetUtils.getAvailablePort();
 		Configuration conf = new Configuration();
-		conf.setString(ConfigConstants.BLOB_SERVER_PORT, sockets[0].getLocalPort() + "," + sockets[1].getLocalPort() + "," + availablePort);
+		conf.setString(BlobServerOptions.PORT, sockets[0].getLocalPort() + "," + sockets[1].getLocalPort() + "," + availablePort);
 
 		// this thing is going to throw an exception
 		try {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java
index 9ac6873767026..a906d9b44df12 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerStartupTest.java
@@ -30,7 +30,7 @@
 
 import com.google.common.io.Files;
 
-import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.BlobServerOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.util.StartupUtils;
 import org.apache.flink.util.NetUtils;
@@ -130,7 +130,7 @@ public void testJobManagerStartupFails() {
 		}
 		Configuration failConfig = new Configuration();
 		String nonExistDirectory = new File(blobStorageDirectory, DOES_NOT_EXISTS_NO_SIR).getAbsolutePath();
-		failConfig.setString(ConfigConstants.BLOB_STORAGE_DIRECTORY_KEY, nonExistDirectory);
+		failConfig.setString(BlobServerOptions.STORAGE_DIRECTORY, nonExistDirectory);
 
 		try {
 			JobManager.runJobManager(failConfig, JobManagerMode.CLUSTER, "localhost", portNum);

From dfcff484228f9ed3ebc1c62cffba4b25c3a89954 Mon Sep 17 00:00:00 2001
From: zentol <chesnay@apache.org>
Date: Sat, 13 May 2017 17:45:46 +0200
Subject: [PATCH 247/253] [FLINK-6530] Close response in DatadogHttpClient

This closes #3892.
---
 .../org/apache/flink/metrics/datadog/DatadogHttpClient.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java
index dfbcee138ce53..c94a2b7ea13ab 100644
--- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpClient.java
@@ -64,8 +64,7 @@ public DatadogHttpClient(String dgApiKey) {
 	private void validateApiKey() {
 		Request r = new Request.Builder().url(validateUrl).get().build();
 
-		try {
-			Response response = client.newCall(r).execute();
+		try (Response response = client.newCall(r).execute()) {
 			if (!response.isSuccessful()) {
 				throw new IllegalArgumentException(
 					String.format("API key: %s is invalid", apiKey));

From d24045d8284a6cead8271512dd42405905808bf6 Mon Sep 17 00:00:00 2001
From: Fabian Hueske <fhueske@apache.org>
Date: Sat, 13 May 2017 23:37:21 +0200
Subject: [PATCH 248/253] [hotfix] [table] Change scala.Boolean to
 java.lang.Boolean in UpsertStreamTableSink interface.

---
 .../org/apache/flink/table/sinks/UpsertStreamTableSink.scala    | 2 +-
 .../org/apache/flink/table/sinks/StreamTableSinksITCase.scala   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala
index 2ae3406a72d47..9c735d0658230 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/UpsertStreamTableSink.scala
@@ -67,7 +67,7 @@ trait UpsertStreamTableSink[T] extends TableSink[JTuple2[JBool, T]] {
     *
     * @param isAppendOnly true if the table is append-only, false otherwise.
     */
-  def setIsAppendOnly(isAppendOnly: Boolean): Unit
+  def setIsAppendOnly(isAppendOnly: JBool): Unit
 
   /** Returns the requested record type */
   def getRecordType: TypeInformation[T]
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala
index 2dfb658320555..47c55f176f900 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sinks/StreamTableSinksITCase.scala
@@ -462,7 +462,7 @@ private class TestUpsertSink(
       assertNull("Provided key fields should not be null.", expectedKeys)
     }
 
-  override def setIsAppendOnly(isAppendOnly: Boolean): Unit =
+  override def setIsAppendOnly(isAppendOnly: JBool): Unit =
     assertEquals(
       "Provided isAppendOnly does not match expected isAppendOnly",
       expectedIsAppendOnly,

From e4e393499d130ca36abf488f799366b59d30e70a Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Wed, 10 May 2017 17:59:39 +0200
Subject: [PATCH 249/253] [FLINK-6537] [checkpoint] First set of fixes for
 (de)registration of shared state in incremental checkpoints

---
 .../RocksDBIncrementalKeyedStateHandle.java   | 123 ++++++-----
 .../state/RocksDBKeyedStateBackend.java       |  64 +++++-
 .../runtime/state/SharedStateRegistry.java    | 196 +++++++++++++-----
 .../runtime/state/SharedStateRegistryKey.java |  68 ++++++
 .../state/SharedStateRegistryTest.java        |  85 +++++---
 .../runtime/state/StateBackendTestBase.java   |  18 +-
 6 files changed, 397 insertions(+), 157 deletions(-)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
index 5ac9e4618c0b1..961182dd19f78 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
@@ -18,12 +18,11 @@
 
 package org.apache.flink.contrib.streaming.state;
 
-import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.state.CompositeStateHandle;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.SharedStateHandle;
 import org.apache.flink.runtime.state.SharedStateRegistry;
+import org.apache.flink.runtime.state.SharedStateRegistryKey;
 import org.apache.flink.runtime.state.StateUtil;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.util.Preconditions;
@@ -54,17 +53,15 @@ public class RocksDBIncrementalKeyedStateHandle implements KeyedStateHandle, Com
 
 	private static final long serialVersionUID = -8328808513197388231L;
 
-	private final JobID jobId;
-
 	private final String operatorIdentifier;
 
 	private final KeyGroupRange keyGroupRange;
 
 	private final long checkpointId;
 
-	private final Map<String, StreamStateHandle> newSstFiles;
+	private final Map<String, StreamStateHandle> unregisteredSstFiles;
 
-	private final Map<String, StreamStateHandle> oldSstFiles;
+	private final Map<String, StreamStateHandle> registeredSstFiles;
 
 	private final Map<String, StreamStateHandle> miscFiles;
 
@@ -81,21 +78,19 @@ public class RocksDBIncrementalKeyedStateHandle implements KeyedStateHandle, Com
 	private boolean registered;
 
 	RocksDBIncrementalKeyedStateHandle(
-			JobID jobId,
 			String operatorIdentifier,
 			KeyGroupRange keyGroupRange,
 			long checkpointId,
-			Map<String, StreamStateHandle> newSstFiles,
-			Map<String, StreamStateHandle> oldSstFiles,
+			Map<String, StreamStateHandle> unregisteredSstFiles,
+			Map<String, StreamStateHandle> registeredSstFiles,
 			Map<String, StreamStateHandle> miscFiles,
 			StreamStateHandle metaStateHandle) {
 
-		this.jobId = Preconditions.checkNotNull(jobId);
 		this.operatorIdentifier = Preconditions.checkNotNull(operatorIdentifier);
 		this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange);
 		this.checkpointId = checkpointId;
-		this.newSstFiles = Preconditions.checkNotNull(newSstFiles);
-		this.oldSstFiles = Preconditions.checkNotNull(oldSstFiles);
+		this.unregisteredSstFiles = Preconditions.checkNotNull(unregisteredSstFiles);
+		this.registeredSstFiles = Preconditions.checkNotNull(registeredSstFiles);
 		this.miscFiles = Preconditions.checkNotNull(miscFiles);
 		this.metaStateHandle = Preconditions.checkNotNull(metaStateHandle);
 		this.registered = false;
@@ -110,12 +105,12 @@ long getCheckpointId() {
 		return checkpointId;
 	}
 
-	Map<String, StreamStateHandle> getNewSstFiles() {
-		return newSstFiles;
+	Map<String, StreamStateHandle> getUnregisteredSstFiles() {
+		return unregisteredSstFiles;
 	}
 
-	Map<String, StreamStateHandle> getOldSstFiles() {
-		return oldSstFiles;
+	Map<String, StreamStateHandle> getRegisteredSstFiles() {
+		return registeredSstFiles;
 	}
 
 	Map<String, StreamStateHandle> getMiscFiles() {
@@ -138,6 +133,8 @@ public KeyedStateHandle getIntersection(KeyGroupRange keyGroupRange) {
 	@Override
 	public void discardState() throws Exception {
 
+		Preconditions.checkState(!registered, "Attempt to dispose a registered composite state with registered shared state. Must unregister first.");
+
 		try {
 			metaStateHandle.discardState();
 		} catch (Exception e) {
@@ -150,24 +147,23 @@ public void discardState() throws Exception {
 			LOG.warn("Could not properly discard misc file states.", e);
 		}
 
-		if (!registered) {
-			try {
-				StateUtil.bestEffortDiscardAllStateObjects(newSstFiles.values());
-			} catch (Exception e) {
-				LOG.warn("Could not properly discard new sst file states.", e);
-			}
+		try {
+			StateUtil.bestEffortDiscardAllStateObjects(unregisteredSstFiles.values());
+		} catch (Exception e) {
+			LOG.warn("Could not properly discard new sst file states.", e);
 		}
+
 	}
 
 	@Override
 	public long getStateSize() {
 		long size = StateUtil.getStateSize(metaStateHandle);
 
-		for (StreamStateHandle newSstFileHandle : newSstFiles.values()) {
+		for (StreamStateHandle newSstFileHandle : unregisteredSstFiles.values()) {
 			size += newSstFileHandle.getStateSize();
 		}
 
-		for (StreamStateHandle oldSstFileHandle : oldSstFiles.values()) {
+		for (StreamStateHandle oldSstFileHandle : registeredSstFiles.values()) {
 			size += oldSstFileHandle.getStateSize();
 		}
 
@@ -180,69 +176,66 @@ public long getStateSize() {
 
 	@Override
 	public void registerSharedStates(SharedStateRegistry stateRegistry) {
+
 		Preconditions.checkState(!registered, "The state handle has already registered its shared states.");
 
-		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : newSstFiles.entrySet()) {
-			SstFileStateHandle stateHandle = new SstFileStateHandle(newSstFileEntry.getKey(), newSstFileEntry.getValue());
+		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : unregisteredSstFiles.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(newSstFileEntry.getKey());
 
-			int referenceCount = stateRegistry.register(stateHandle);
-			Preconditions.checkState(referenceCount == 1);
+			SharedStateRegistry.Result result =
+				stateRegistry.registerNewReference(registryKey, newSstFileEntry.getValue());
+
+			// We update our reference with the result from the registry, to prevent the following
+			// problem:
+			// A previous checkpoint n has already registered the state. This can happen if a
+			// following checkpoint (n + x) wants to reference the same state before the backend got
+			// notified that checkpoint n completed. In this case, the shared registry did
+			// deduplication and returns the previous reference.
+			newSstFileEntry.setValue(result.getReference());
 		}
 
-		for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : oldSstFiles.entrySet()) {
-			SstFileStateHandle stateHandle = new SstFileStateHandle(oldSstFileEntry.getKey(), oldSstFileEntry.getValue());
+		for (Map.Entry<String, StreamStateHandle> oldSstFileName : registeredSstFiles.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(oldSstFileName.getKey());
+
+			SharedStateRegistry.Result result = stateRegistry.obtainReference(registryKey);
 
-			int referenceCount = stateRegistry.register(stateHandle);
-			Preconditions.checkState(referenceCount > 1);
+			// Again we update our state handle with the result from the registry, thus replacing
+			// placeholder state handles with the originals.
+			oldSstFileName.setValue(result.getReference());
 		}
 
+		// Migrate state from unregistered to registered, so that it will not count as private state
+		// for #discardState() from now.
+		registeredSstFiles.putAll(unregisteredSstFiles);
+		unregisteredSstFiles.clear();
+
 		registered = true;
 	}
 
 	@Override
 	public void unregisterSharedStates(SharedStateRegistry stateRegistry) {
+
 		Preconditions.checkState(registered, "The state handle has not registered its shared states yet.");
 
-		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : newSstFiles.entrySet()) {
-			stateRegistry.unregister(new SstFileStateHandle(newSstFileEntry.getKey(), newSstFileEntry.getValue()));
+		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : unregisteredSstFiles.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(newSstFileEntry.getKey());
+			stateRegistry.releaseReference(registryKey);
 		}
 
-		for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : oldSstFiles.entrySet()) {
-			stateRegistry.unregister(new SstFileStateHandle(oldSstFileEntry.getKey(), oldSstFileEntry.getValue()));
+		for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : registeredSstFiles.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(oldSstFileEntry.getKey());
+			stateRegistry.releaseReference(registryKey);
 		}
 
 		registered = false;
 	}
 
-	private class SstFileStateHandle implements SharedStateHandle {
-
-		private static final long serialVersionUID = 9092049285789170669L;
-
-		private final String fileName;
-
-		private final StreamStateHandle delegateStateHandle;
-
-		private SstFileStateHandle(
-				String fileName,
-				StreamStateHandle delegateStateHandle) {
-			this.fileName = fileName;
-			this.delegateStateHandle = delegateStateHandle;
-		}
-
-		@Override
-		public String getRegistrationKey() {
-			return jobId + "-" + operatorIdentifier + "-" + keyGroupRange + "-" + fileName;
-		}
-
-		@Override
-		public void discardState() throws Exception {
-			delegateStateHandle.discardState();
-		}
-
-		@Override
-		public long getStateSize() {
-			return delegateStateHandle.getStateSize();
-		}
+	private SharedStateRegistryKey createSharedStateRegistryKeyFromFileName(String fileName) {
+		return new SharedStateRegistryKey(operatorIdentifier + "-" + keyGroupRange + "-" + fileName);
 	}
 }
 
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 6af53c3e978ff..1080e59050568 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -54,7 +54,6 @@
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
-import org.apache.flink.runtime.state.StateMigrationUtil;
 import org.apache.flink.runtime.state.DoneFuture;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
@@ -62,9 +61,10 @@
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
 import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
+import org.apache.flink.runtime.state.StateMigrationUtil;
 import org.apache.flink.runtime.state.StateObject;
 import org.apache.flink.runtime.state.StateUtil;
-import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.internal.InternalAggregatingState;
 import org.apache.flink.runtime.state.internal.InternalFoldingState;
@@ -72,6 +72,7 @@
 import org.apache.flink.runtime.state.internal.InternalMapState;
 import org.apache.flink.runtime.state.internal.InternalReducingState;
 import org.apache.flink.runtime.state.internal.InternalValueState;
+import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
 import org.apache.flink.runtime.util.SerializableObject;
 import org.apache.flink.util.FileUtils;
 import org.apache.flink.util.IOUtils;
@@ -709,16 +710,22 @@ private static void checkInterrupted() throws InterruptedException {
 
 	private static final class RocksDBIncrementalSnapshotOperation {
 
+		/** The backend which we snapshot */
 		private final RocksDBKeyedStateBackend<?> stateBackend;
 
+		/** Stream factory that creates the outpus streams to DFS */
 		private final CheckpointStreamFactory checkpointStreamFactory;
 
+		/** Id for the current checkpoint */
 		private final long checkpointId;
 
+		/** Timestamp for the current checkpoint */
 		private final long checkpointTimestamp;
 
+		/** All sst files that were part of the last previously completed checkpoint */
 		private Map<String, StreamStateHandle> baseSstFiles;
 
+		/** The state meta data */
 		private final List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots = new ArrayList<>();
 
 		private FileSystem backupFileSystem;
@@ -864,10 +871,11 @@ KeyedStateHandle materializeSnapshot() throws Exception {
 
 						if (fileHandle == null) {
 							fileHandle = materializeStateData(filePath);
-
 							newSstFiles.put(fileName, fileHandle);
 						} else {
-							oldSstFiles.put(fileName, fileHandle);
+							// we introduce a placeholder state handle, that is replaced with the
+							// original from the shared state registry (created from a previous checkpoint)
+							oldSstFiles.put(fileName, new PlaceholderStreamStateHandle(fileHandle.getStateSize()));
 						}
 					} else {
 						StreamStateHandle fileHandle = materializeStateData(filePath);
@@ -882,9 +890,14 @@ KeyedStateHandle materializeSnapshot() throws Exception {
 
 			stateBackend.materializedSstFiles.put(checkpointId, sstFiles);
 
-			return new RocksDBIncrementalKeyedStateHandle(stateBackend.jobId,
-				stateBackend.operatorIdentifier, stateBackend.keyGroupRange,
-				checkpointId, newSstFiles, oldSstFiles, miscFiles, metaStateHandle);
+			return new RocksDBIncrementalKeyedStateHandle(
+				stateBackend.operatorIdentifier,
+				stateBackend.keyGroupRange,
+				checkpointId,
+				newSstFiles,
+				oldSstFiles,
+				miscFiles,
+				metaStateHandle);
 		}
 
 		void stop() {
@@ -922,6 +935,39 @@ void releaseResources(boolean canceled) {
 				}
 			}
 		}
+
+		/**
+		 * A placeholder state handle for shared state that will replaced by an original that was
+		 * created in a previous checkpoint. So we don't have to send the handle twice, e.g. in
+		 * case of {@link ByteStreamStateHandle}.
+		 */
+		private static final class PlaceholderStreamStateHandle implements StreamStateHandle {
+
+			private static final long serialVersionUID = 1L;
+
+			/** We remember the size of the original file for which this is a placeholder */
+			private final long originalSize;
+
+			public PlaceholderStreamStateHandle(long originalSize) {
+				this.originalSize = originalSize;
+			}
+
+			@Override
+			public FSDataInputStream openInputStream() {
+				throw new UnsupportedOperationException(
+					"This is only a placeholder to be replaced by a real StreamStateHandle in the checkpoint coordinator.");
+			}
+
+			@Override
+			public void discardState() throws Exception {
+				// nothing to do.
+			}
+
+			@Override
+			public long getStateSize() {
+				return originalSize;
+			}
+		}
 	}
 
 	@Override
@@ -1260,7 +1306,7 @@ private void restoreInstance(
 				UUID.randomUUID().toString());
 
 			try {
-				Map<String, StreamStateHandle> newSstFiles = restoreStateHandle.getNewSstFiles();
+				Map<String, StreamStateHandle> newSstFiles = restoreStateHandle.getUnregisteredSstFiles();
 				for (Map.Entry<String, StreamStateHandle> newSstFileEntry : newSstFiles.entrySet()) {
 					String fileName = newSstFileEntry.getKey();
 					StreamStateHandle remoteFileHandle = newSstFileEntry.getValue();
@@ -1268,7 +1314,7 @@ private void restoreInstance(
 					readStateData(new Path(restoreInstancePath, fileName), remoteFileHandle);
 				}
 
-				Map<String, StreamStateHandle> oldSstFiles = restoreStateHandle.getOldSstFiles();
+				Map<String, StreamStateHandle> oldSstFiles = restoreStateHandle.getRegisteredSstFiles();
 				for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : oldSstFiles.entrySet()) {
 					String fileName = oldSstFileEntry.getKey();
 					StreamStateHandle remoteFileHandle = oldSstFileEntry.getValue();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
index 2cb43accfa40b..dbf464238c430 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
@@ -18,91 +18,137 @@
 
 package org.apache.flink.runtime.state;
 
+import org.apache.flink.runtime.concurrent.Executors;
 import org.apache.flink.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.Executor;
 
 /**
  * A {@code SharedStateRegistry} will be deployed in the 
- * {@link org.apache.flink.runtime.checkpoint.CheckpointCoordinator} to 
+ * {@link org.apache.flink.runtime.checkpoint.CompletedCheckpointStore} to
  * maintain the reference count of {@link SharedStateHandle}s which are shared
- * among different checkpoints.
- *
+ * among different incremental checkpoints.
  */
 public class SharedStateRegistry {
 
 	private static final Logger LOG = LoggerFactory.getLogger(SharedStateRegistry.class);
 
 	/** All registered state objects by an artificial key */
-	private final Map<String, SharedStateRegistry.SharedStateEntry> registeredStates;
+	private final Map<SharedStateRegistryKey, SharedStateRegistry.SharedStateEntry> registeredStates;
+
+	/** Executor for async state deletion */
+	private final Executor asyncDisposalExecutor;
 
 	public SharedStateRegistry() {
 		this.registeredStates = new HashMap<>();
+		this.asyncDisposalExecutor = Executors.directExecutor(); //TODO: FLINK-6534
 	}
 
 	/**
-	 * Register a reference to the given shared state in the registry. This increases the reference
-	 * count for the this shared state by one. Returns the reference count after the update.
+	 * Register a reference to the given (supposedly new) shared state in the registry.
+	 * This does the following: We check if the state handle is actually new by the
+	 * registrationKey. If it is new, we register it with a reference count of 1. If there is
+	 * already a state handle registered under the given key, we dispose the given "new" state
+	 * handle, uptick the reference count of the previously existing state handle and return it as
+	 * a replacement with the result.
+	 *
+	 * <p>IMPORTANT: caller should check the state handle returned by the result, because the
+	 * registry is performing deduplication and could potentially return a handle that is supposed
+	 * to replace the one from the registration request.
 	 *
 	 * @param state the shared state for which we register a reference.
-	 * @return the updated reference count for the given shared state.
+	 * @return the result of this registration request, consisting of the state handle that is
+	 * registered under the key by the end of the oepration and its current reference count.
 	 */
-	public int register(SharedStateHandle state) {
-		if (state == null) {
-			return 0;
-		}
+	public Result registerNewReference(SharedStateRegistryKey registrationKey, StreamStateHandle state) {
+
+		Preconditions.checkNotNull(state);
+
+		StreamStateHandle scheduledStateDeletion = null;
+		SharedStateRegistry.SharedStateEntry entry;
 
 		synchronized (registeredStates) {
-			SharedStateRegistry.SharedStateEntry entry =
-				registeredStates.get(state.getRegistrationKey());
+			entry = registeredStates.get(registrationKey);
 
 			if (entry == null) {
-				SharedStateRegistry.SharedStateEntry stateEntry =
-					new SharedStateRegistry.SharedStateEntry(state);
-				registeredStates.put(state.getRegistrationKey(), stateEntry);
-				return 1;
+				entry = new SharedStateRegistry.SharedStateEntry(state);
+				registeredStates.put(registrationKey, entry);
 			} else {
+				// delete if this is a real duplicate
+				if (!Objects.equals(state, entry.state)) {
+					scheduledStateDeletion = state;
+				}
 				entry.increaseReferenceCount();
-				return entry.getReferenceCount();
 			}
 		}
+
+		scheduleAsyncDelete(scheduledStateDeletion);
+		return new Result(entry);
 	}
 
 	/**
-	 * Unregister one reference to the given shared state in the registry. This decreases the
-	 * reference count by one. Once the count reaches zero, the shared state is deleted.
+	 * Obtains one reference to the given shared state in the registry. This increases the
+	 * reference count by one.
 	 *
-	 * @param state the shared state for which we unregister a reference.
-	 * @return the reference count for the shared state after the update.
+	 * @param registrationKey the shared state for which we obtain a reference.
+	 * @return the shared state for which we release a reference.
+	 * @return the result of the request, consisting of the reference count after this operation
+	 * and the state handle.
 	 */
-	public int unregister(SharedStateHandle state) {
-		if (state == null) {
-			return 0;
+	public Result obtainReference(SharedStateRegistryKey registrationKey) {
+
+		Preconditions.checkNotNull(registrationKey);
+
+		synchronized (registeredStates) {
+			SharedStateRegistry.SharedStateEntry entry =
+				Preconditions.checkNotNull(registeredStates.get(registrationKey),
+					"Could not find a state for the given registration key!");
+			entry.increaseReferenceCount();
+			return new Result(entry);
 		}
+	}
+
+	/**
+	 * Releases one reference to the given shared state in the registry. This decreases the
+	 * reference count by one. Once the count reaches zero, the shared state is deleted.
+	 *
+	 * @param registrationKey the shared state for which we release a reference.
+	 * @return the result of the request, consisting of the reference count after this operation
+	 * and the state handle, or null if the state handle was deleted through this request.
+	 */
+	public Result releaseReference(SharedStateRegistryKey registrationKey) {
+
+		Preconditions.checkNotNull(registrationKey);
+
+		final Result result;
+		final StreamStateHandle scheduledStateDeletion;
 
 		synchronized (registeredStates) {
-			SharedStateRegistry.SharedStateEntry entry = registeredStates.get(state.getRegistrationKey());
+			SharedStateRegistry.SharedStateEntry entry = registeredStates.get(registrationKey);
 
-			Preconditions.checkState(entry != null, "Cannot unregister a state that is not registered.");
+			Preconditions.checkState(entry != null,
+				"Cannot unregister a state that is not registered.");
 
 			entry.decreaseReferenceCount();
 
-			final int newReferenceCount = entry.getReferenceCount();
-
 			// Remove the state from the registry when it's not referenced any more.
-			if (newReferenceCount <= 0) {
-				registeredStates.remove(state.getRegistrationKey());
-				try {
-					entry.getState().discardState();
-				} catch (Exception e) {
-					LOG.warn("Cannot properly discard the state {}.", entry.getState(), e);
-				}
+			if (entry.getReferenceCount() <= 0) {
+				registeredStates.remove(registrationKey);
+				scheduledStateDeletion = entry.getState();
+				result = new Result(null, 0);
+			} else {
+				scheduledStateDeletion = null;
+				result = new Result(entry);
 			}
-			return newReferenceCount;
 		}
+
+		scheduleAsyncDelete(scheduledStateDeletion);
+		return result;
 	}
 
 	/**
@@ -122,8 +168,6 @@ public void registerAll(Iterable<? extends CompositeStateHandle> stateHandles) {
 		}
 	}
 
-
-
 	/**
 	 * Unregister all the shared states referenced by the given.
 	 *
@@ -141,20 +185,30 @@ public void unregisterAll(Iterable<? extends CompositeStateHandle> stateHandles)
 		}
 	}
 
+	private void scheduleAsyncDelete(StreamStateHandle streamStateHandle) {
+		if (streamStateHandle != null) {
+			asyncDisposalExecutor.execute(
+				new SharedStateRegistry.AsyncDisposalRunnable(streamStateHandle));
+		}
+	}
+
+	/**
+	 * An entry in the registry, tracking the handle and the corresponding reference count.
+	 */
 	private static class SharedStateEntry {
 
-		/** The shared object */
-		private final SharedStateHandle state;
+		/** The shared state handle */
+		private final StreamStateHandle state;
 
-		/** The reference count of the object */
+		/** The current reference count of the state handle */
 		private int referenceCount;
 
-		SharedStateEntry(SharedStateHandle value) {
+		SharedStateEntry(StreamStateHandle value) {
 			this.state = value;
 			this.referenceCount = 1;
 		}
 
-		SharedStateHandle getState() {
+		StreamStateHandle getState() {
 			return state;
 		}
 
@@ -171,14 +225,56 @@ void decreaseReferenceCount() {
 		}
 	}
 
-	public int getReferenceCount(SharedStateHandle state) {
-		if (state == null) {
-			return 0;
+	/**
+	 * The result of an attempt to (un)/reference state
+	 */
+	public static class Result {
+
+		/** The (un)registered state handle from the request */
+		private final StreamStateHandle reference;
+
+		/** The reference count to the state handle after the request to (un)register */
+		private final int referenceCount;
+
+		private Result(SharedStateEntry sharedStateEntry) {
+			this.reference = sharedStateEntry.getState();
+			this.referenceCount = sharedStateEntry.getReferenceCount();
 		}
 
-		SharedStateRegistry.SharedStateEntry entry =
-			registeredStates.get(state.getRegistrationKey());
+		public Result(StreamStateHandle reference, int referenceCount) {
+			Preconditions.checkArgument(referenceCount >= 0);
 
-		return entry == null ? 0 : entry.getReferenceCount();
+			this.reference = reference;
+			this.referenceCount = referenceCount;
+		}
+
+		public StreamStateHandle getReference() {
+			return reference;
+		}
+
+		public int getReferenceCount() {
+			return referenceCount;
+		}
+	}
+
+	/**
+	 * Encapsulates the operation the delete state handles asynchronously.
+	 */
+	private static final class AsyncDisposalRunnable implements Runnable {
+
+		private final StateObject toDispose;
+
+		public AsyncDisposalRunnable(StateObject toDispose) {
+			this.toDispose = Preconditions.checkNotNull(toDispose);
+		}
+
+		@Override
+		public void run() {
+			try {
+				toDispose.discardState();
+			} catch (Exception e) {
+				LOG.warn("A problem occurred during asynchronous disposal of a shared state object: {}", toDispose, e);
+			}
+		}
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java
new file mode 100644
index 0000000000000..9e5935957a4cf
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.util.Preconditions;
+
+import java.io.Serializable;
+
+/**
+ * This class represents a key that uniquely identifies (on a logical level) state handles for
+ * registration in the {@link SharedStateRegistry}. Two files which should logically
+ * be the same should have the same {@link SharedStateRegistryKey}. The meaning of logical
+ * equivalence is up to the application.
+ */
+public class SharedStateRegistryKey implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	/** Uses a String as internal representation */
+	private final String keyString;
+
+	public SharedStateRegistryKey(String keyString) {
+		this.keyString = Preconditions.checkNotNull(keyString);
+	}
+
+	public String getKeyString() {
+		return keyString;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+
+		SharedStateRegistryKey that = (SharedStateRegistryKey) o;
+		return keyString.equals(that.keyString);
+	}
+
+	@Override
+	public int hashCode() {
+		return keyString.hashCode();
+	}
+
+	@Override
+	public String toString() {
+		return keyString;
+	}
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
index 821bb69912262..03e2a1392be84 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SharedStateRegistryTest.java
@@ -19,9 +19,14 @@
 
 package org.apache.flink.runtime.state;
 
+import org.apache.flink.core.fs.FSDataInputStream;
 import org.junit.Test;
 
+import java.io.IOException;
+
+import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class SharedStateRegistryTest {
 
@@ -30,24 +35,50 @@ public class SharedStateRegistryTest {
 	 */
 	@Test
 	public void testRegistryNormal() {
+
 		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 
 		// register one state
 		TestSharedState firstState = new TestSharedState("first");
-		assertEquals(1, sharedStateRegistry.register(firstState));
+		SharedStateRegistry.Result result = sharedStateRegistry.registerNewReference(firstState.getRegistrationKey(), firstState);
+		assertEquals(1, result.getReferenceCount());
+		assertTrue(firstState == result.getReference());
+		assertFalse(firstState.isDiscarded());
 
 		// register another state
 		TestSharedState secondState = new TestSharedState("second");
-		assertEquals(1, sharedStateRegistry.register(secondState));
-
-		// register the first state again
-		assertEquals(2, sharedStateRegistry.register(firstState));
+		result = sharedStateRegistry.registerNewReference(secondState.getRegistrationKey(), secondState);
+		assertEquals(1, result.getReferenceCount());
+		assertTrue(secondState == result.getReference());
+		assertFalse(firstState.isDiscarded());
+		assertFalse(secondState.isDiscarded());
+
+		// attempt to register state under an existing key
+		TestSharedState firstStatePrime = new TestSharedState(firstState.getRegistrationKey().getKeyString());
+		result = sharedStateRegistry.registerNewReference(firstState.getRegistrationKey(), firstStatePrime);
+		assertEquals(2, result.getReferenceCount());
+		assertFalse(firstStatePrime == result.getReference());
+		assertTrue(firstState == result.getReference());
+		assertTrue(firstStatePrime.isDiscarded());
+		assertFalse(firstState.isDiscarded());
+
+		// reference the first state again
+		result = sharedStateRegistry.obtainReference(firstState.getRegistrationKey());
+		assertEquals(3, result.getReferenceCount());
+		assertTrue(firstState == result.getReference());
+		assertFalse(firstState.isDiscarded());
 
 		// unregister the second state
-		assertEquals(0, sharedStateRegistry.unregister(secondState));
+		result = sharedStateRegistry.releaseReference(secondState.getRegistrationKey());
+		assertEquals(0, result.getReferenceCount());
+		assertTrue(result.getReference() == null);
+		assertTrue(secondState.isDiscarded());
 
 		// unregister the first state
-		assertEquals(1, sharedStateRegistry.unregister(firstState));
+		result = sharedStateRegistry.releaseReference(firstState.getRegistrationKey());
+		assertEquals(2, result.getReferenceCount());
+		assertTrue(firstState == result.getReference());
+		assertFalse(firstState.isDiscarded());
 	}
 
 	/**
@@ -56,51 +87,47 @@ public void testRegistryNormal() {
 	@Test(expected = IllegalStateException.class)
 	public void testUnregisterWithUnexistedKey() {
 		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
-
-		sharedStateRegistry.unregister(new TestSharedState("unexisted"));
+		sharedStateRegistry.releaseReference(new SharedStateRegistryKey("non-existent"));
 	}
 
-	private static class TestSharedState implements SharedStateHandle {
+	private static class TestSharedState implements StreamStateHandle {
 		private static final long serialVersionUID = 4468635881465159780L;
 
-		private String key;
+		private SharedStateRegistryKey key;
+
+		private boolean discarded;
 
 		TestSharedState(String key) {
-			this.key = key;
+			this.key = new SharedStateRegistryKey(key);
+			this.discarded = false;
 		}
 
-		@Override
-		public String getRegistrationKey() {
+		public SharedStateRegistryKey getRegistrationKey() {
 			return key;
 		}
 
 		@Override
 		public void discardState() throws Exception {
-			// nothing to do
+			this.discarded = true;
 		}
 
 		@Override
 		public long getStateSize() {
-			return key.length();
+			return key.toString().length();
 		}
 
 		@Override
-		public boolean equals(Object o) {
-			if (this == o) {
-				return true;
-			}
-			if (o == null || getClass() != o.getClass()) {
-				return false;
-			}
-
-			TestSharedState testState = (TestSharedState) o;
-
-			return key.equals(testState.key);
+		public int hashCode() {
+			return key.hashCode();
 		}
 
 		@Override
-		public int hashCode() {
-			return key.hashCode();
+		public FSDataInputStream openInputStream() throws IOException {
+			throw new UnsupportedOperationException();
+		}
+
+		public boolean isDiscarded() {
+			return discarded;
 		}
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
index 658ccde8c14cc..ca66ffb944ba7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
@@ -482,6 +482,7 @@ public void testKryoRegisteringRestoreResilienceWithRegisteredType() throws Exce
 	@SuppressWarnings("unchecked")
 	public void testKryoRegisteringRestoreResilienceWithDefaultSerializer() throws Exception {
 		CheckpointStreamFactory streamFactory = createStreamFactory();
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		Environment env = new DummyEnvironment("test", 1, 0);
 		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
 
@@ -509,6 +510,7 @@ public void testKryoRegisteringRestoreResilienceWithDefaultSerializer() throws E
 				streamFactory,
 				CheckpointOptions.forFullCheckpoint()));
 
+		snapshot.registerSharedStates(sharedStateRegistry);
 		backend.dispose();
 
 		// ========== restore snapshot - should use default serializer (ONLY SERIALIZATION) ==========
@@ -518,8 +520,6 @@ public void testKryoRegisteringRestoreResilienceWithDefaultSerializer() throws E
 
 		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
 
-		snapshot.discardState();
-
 		// re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
 		// initializeSerializerUnlessSet would not pick up our new config
 		kvId = new ValueStateDescriptor<>("id", pojoType);
@@ -536,6 +536,11 @@ public void testKryoRegisteringRestoreResilienceWithDefaultSerializer() throws E
 				streamFactory,
 				CheckpointOptions.forFullCheckpoint()));
 
+		snapshot2.registerSharedStates(sharedStateRegistry);
+
+		snapshot.unregisterSharedStates(sharedStateRegistry);
+		snapshot.discardState();
+
 		backend.dispose();
 
 		// ========= restore snapshot - should use default serializer (FAIL ON DESERIALIZATION) =========
@@ -570,6 +575,7 @@ public void testKryoRegisteringRestoreResilienceWithDefaultSerializer() throws E
 	@Test
 	public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throws Exception {
 		CheckpointStreamFactory streamFactory = createStreamFactory();
+		SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
 		Environment env = new DummyEnvironment("test", 1, 0);
 
 		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
@@ -597,6 +603,7 @@ public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throw
 				streamFactory,
 				CheckpointOptions.forFullCheckpoint()));
 
+		snapshot.registerSharedStates(sharedStateRegistry);
 		backend.dispose();
 
 		// ========== restore snapshot - should use specific serializer (ONLY SERIALIZATION) ==========
@@ -605,8 +612,6 @@ public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throw
 
 		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
 
-		snapshot.discardState();
-
 		// re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
 		// initializeSerializerUnlessSet would not pick up our new config
 		kvId = new ValueStateDescriptor<>("id", pojoType);
@@ -623,6 +628,11 @@ public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throw
 				streamFactory,
 				CheckpointOptions.forFullCheckpoint()));
 
+		snapshot2.registerSharedStates(sharedStateRegistry);
+
+		snapshot.unregisterSharedStates(sharedStateRegistry);
+		snapshot.discardState();
+
 		backend.dispose();
 
 		// ========= restore snapshot - should use specific serializer (FAIL ON DESERIALIZATION) =========

From df798a1c24766c8cecbf321b1e07169ddf899e64 Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Thu, 11 May 2017 11:59:47 +0200
Subject: [PATCH 250/253] [FLINK-6504] [checkpoint] Fix synchronization on
 materializedSstFiles in RocksDBKeyedStateBackend

---
 .../contrib/streaming/state/RocksDBKeyedStateBackend.java  | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 1080e59050568..b9468f7371821 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -828,9 +828,12 @@ private StreamStateHandle materializeMetaData() throws Exception {
 		}
 
 		void takeSnapshot() throws Exception {
+			assert (Thread.holdsLock(stateBackend.asyncSnapshotLock));
+
 			// use the last completed checkpoint as the comparison base.
 			baseSstFiles = stateBackend.materializedSstFiles.get(stateBackend.lastCompletedCheckpointId);
 
+
 			// save meta data
 			for (Map.Entry<String, Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>> stateMetaInfoEntry
 					: stateBackend.kvStateInformation.entrySet()) {
@@ -888,7 +891,9 @@ KeyedStateHandle materializeSnapshot() throws Exception {
 			sstFiles.putAll(newSstFiles);
 			sstFiles.putAll(oldSstFiles);
 
-			stateBackend.materializedSstFiles.put(checkpointId, sstFiles);
+			synchronized (stateBackend.asyncSnapshotLock) {
+				stateBackend.materializedSstFiles.put(checkpointId, sstFiles);
+			}
 
 			return new RocksDBIncrementalKeyedStateHandle(
 				stateBackend.operatorIdentifier,

From 9d5053f3296eec91cb9d83bb8732b89b9130539d Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Wed, 10 May 2017 14:57:55 +0200
Subject: [PATCH 251/253] [FLINK-6527] [checkpoint] OperatorSubtaskState has
 empty implementations of (un)/registerSharedStates

---
 .../runtime/checkpoint/OperatorSubtaskState.java | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java
index 863816a2b2353..49ef863c37cc8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorSubtaskState.java
@@ -148,12 +148,24 @@ public void discardState() {
 
 	@Override
 	public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
-		// No shared states
+		if (managedKeyedState != null) {
+			managedKeyedState.registerSharedStates(sharedStateRegistry);
+		}
+
+		if (rawKeyedState != null) {
+			rawKeyedState.registerSharedStates(sharedStateRegistry);
+		}
 	}
 
 	@Override
 	public void unregisterSharedStates(SharedStateRegistry sharedStateRegistry) {
-		// No shared states
+		if (managedKeyedState != null) {
+			managedKeyedState.unregisterSharedStates(sharedStateRegistry);
+		}
+
+		if (rawKeyedState != null) {
+			rawKeyedState.unregisterSharedStates(sharedStateRegistry);
+		}
 	}
 
 	@Override

From 856bebfe871816c10c9dd73cd68c17c6e9815fa6 Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Thu, 11 May 2017 21:04:29 +0200
Subject: [PATCH 252/253] [FLINK-6545] [checkpoint] Make incremental
 checkpoints externalizable

---
 .../RocksDBIncrementalKeyedStateHandle.java   | 241 -------------
 .../state/RocksDBKeyedStateBackend.java       | 155 ++++-----
 .../org/apache/flink/util/StringBasedID.java  |  69 ++++
 .../savepoint/SavepointV2Serializer.java      |  83 ++++-
 .../state/IncrementalKeyedStateHandle.java    | 324 ++++++++++++++++++
 .../state/PlaceholderStreamStateHandle.java   |  88 +++++
 .../runtime/state/SharedStateHandle.java      |  39 ---
 .../runtime/state/SharedStateRegistry.java    |   2 +-
 .../runtime/state/SharedStateRegistryKey.java |  42 +--
 .../flink/runtime/state/StateHandle.java      |  37 --
 .../flink/runtime/state/StateHandleID.java}   |  25 +-
 .../savepoint/CheckpointTestUtils.java        |  87 ++++-
 .../savepoint/SavepointV2SerializerTest.java  |   1 -
 13 files changed, 712 insertions(+), 481 deletions(-)
 delete mode 100644 flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/util/StringBasedID.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/IncrementalKeyedStateHandle.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/PlaceholderStreamStateHandle.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
 delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java
 rename flink-runtime/src/{test/java/org/apache/flink/runtime/state/StateUtilTest.java => main/java/org/apache/flink/runtime/state/StateHandleID.java} (56%)

diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
deleted file mode 100644
index 961182dd19f78..0000000000000
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBIncrementalKeyedStateHandle.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.contrib.streaming.state;
-
-import org.apache.flink.runtime.state.CompositeStateHandle;
-import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.SharedStateRegistry;
-import org.apache.flink.runtime.state.SharedStateRegistryKey;
-import org.apache.flink.runtime.state.StateUtil;
-import org.apache.flink.runtime.state.StreamStateHandle;
-import org.apache.flink.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-/**
- * The handle to states in incremental snapshots taken by {@link RocksDBKeyedStateBackend}.
- *
- * The states contained in an incremental snapshot include
- * <ul>
- * <li> New SST state which includes the sst files produced since the last completed
- *   checkpoint. These files can be referenced by succeeding checkpoints if the
- *   checkpoint succeeds to complete. </li>
- * <li> Old SST state which includes the sst files materialized in previous
- *   checkpoints. </li>
- * <li> MISC state which include the other files in the RocksDB instance, e.g. the
- *   LOG and MANIFEST files. These files are mutable, hence cannot be shared by
- *   other checkpoints. </li>
- * <li> Meta state which includes the information of existing states. </li>
- * </ul>
- */
-public class RocksDBIncrementalKeyedStateHandle implements KeyedStateHandle, CompositeStateHandle {
-
-	private static final Logger LOG = LoggerFactory.getLogger(RocksDBIncrementalKeyedStateHandle.class);
-
-	private static final long serialVersionUID = -8328808513197388231L;
-
-	private final String operatorIdentifier;
-
-	private final KeyGroupRange keyGroupRange;
-
-	private final long checkpointId;
-
-	private final Map<String, StreamStateHandle> unregisteredSstFiles;
-
-	private final Map<String, StreamStateHandle> registeredSstFiles;
-
-	private final Map<String, StreamStateHandle> miscFiles;
-
-	private final StreamStateHandle metaStateHandle;
-
-	/**
-	 * True if the state handle has already registered shared states.
-	 *
-	 * Once the shared states are registered, it's the {@link SharedStateRegistry}'s
-	 * responsibility to maintain the shared states. But in the cases where the
-	 * state handle is discarded before performing the registration, the handle
-	 * should delete all the shared states created by it.
-	 */
-	private boolean registered;
-
-	RocksDBIncrementalKeyedStateHandle(
-			String operatorIdentifier,
-			KeyGroupRange keyGroupRange,
-			long checkpointId,
-			Map<String, StreamStateHandle> unregisteredSstFiles,
-			Map<String, StreamStateHandle> registeredSstFiles,
-			Map<String, StreamStateHandle> miscFiles,
-			StreamStateHandle metaStateHandle) {
-
-		this.operatorIdentifier = Preconditions.checkNotNull(operatorIdentifier);
-		this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange);
-		this.checkpointId = checkpointId;
-		this.unregisteredSstFiles = Preconditions.checkNotNull(unregisteredSstFiles);
-		this.registeredSstFiles = Preconditions.checkNotNull(registeredSstFiles);
-		this.miscFiles = Preconditions.checkNotNull(miscFiles);
-		this.metaStateHandle = Preconditions.checkNotNull(metaStateHandle);
-		this.registered = false;
-	}
-
-	@Override
-	public KeyGroupRange getKeyGroupRange() {
-		return keyGroupRange;
-	}
-
-	long getCheckpointId() {
-		return checkpointId;
-	}
-
-	Map<String, StreamStateHandle> getUnregisteredSstFiles() {
-		return unregisteredSstFiles;
-	}
-
-	Map<String, StreamStateHandle> getRegisteredSstFiles() {
-		return registeredSstFiles;
-	}
-
-	Map<String, StreamStateHandle> getMiscFiles() {
-		return miscFiles;
-	}
-
-	StreamStateHandle getMetaStateHandle() {
-		return metaStateHandle;
-	}
-
-	@Override
-	public KeyedStateHandle getIntersection(KeyGroupRange keyGroupRange) {
-		if (this.keyGroupRange.getIntersection(keyGroupRange) != KeyGroupRange.EMPTY_KEY_GROUP_RANGE) {
-			return this;
-		} else {
-			return null;
-		}
-	}
-
-	@Override
-	public void discardState() throws Exception {
-
-		Preconditions.checkState(!registered, "Attempt to dispose a registered composite state with registered shared state. Must unregister first.");
-
-		try {
-			metaStateHandle.discardState();
-		} catch (Exception e) {
-			LOG.warn("Could not properly discard meta data.", e);
-		}
-
-		try {
-			StateUtil.bestEffortDiscardAllStateObjects(miscFiles.values());
-		} catch (Exception e) {
-			LOG.warn("Could not properly discard misc file states.", e);
-		}
-
-		try {
-			StateUtil.bestEffortDiscardAllStateObjects(unregisteredSstFiles.values());
-		} catch (Exception e) {
-			LOG.warn("Could not properly discard new sst file states.", e);
-		}
-
-	}
-
-	@Override
-	public long getStateSize() {
-		long size = StateUtil.getStateSize(metaStateHandle);
-
-		for (StreamStateHandle newSstFileHandle : unregisteredSstFiles.values()) {
-			size += newSstFileHandle.getStateSize();
-		}
-
-		for (StreamStateHandle oldSstFileHandle : registeredSstFiles.values()) {
-			size += oldSstFileHandle.getStateSize();
-		}
-
-		for (StreamStateHandle miscFileHandle : miscFiles.values()) {
-			size += miscFileHandle.getStateSize();
-		}
-
-		return size;
-	}
-
-	@Override
-	public void registerSharedStates(SharedStateRegistry stateRegistry) {
-
-		Preconditions.checkState(!registered, "The state handle has already registered its shared states.");
-
-		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : unregisteredSstFiles.entrySet()) {
-			SharedStateRegistryKey registryKey =
-				createSharedStateRegistryKeyFromFileName(newSstFileEntry.getKey());
-
-			SharedStateRegistry.Result result =
-				stateRegistry.registerNewReference(registryKey, newSstFileEntry.getValue());
-
-			// We update our reference with the result from the registry, to prevent the following
-			// problem:
-			// A previous checkpoint n has already registered the state. This can happen if a
-			// following checkpoint (n + x) wants to reference the same state before the backend got
-			// notified that checkpoint n completed. In this case, the shared registry did
-			// deduplication and returns the previous reference.
-			newSstFileEntry.setValue(result.getReference());
-		}
-
-		for (Map.Entry<String, StreamStateHandle> oldSstFileName : registeredSstFiles.entrySet()) {
-			SharedStateRegistryKey registryKey =
-				createSharedStateRegistryKeyFromFileName(oldSstFileName.getKey());
-
-			SharedStateRegistry.Result result = stateRegistry.obtainReference(registryKey);
-
-			// Again we update our state handle with the result from the registry, thus replacing
-			// placeholder state handles with the originals.
-			oldSstFileName.setValue(result.getReference());
-		}
-
-		// Migrate state from unregistered to registered, so that it will not count as private state
-		// for #discardState() from now.
-		registeredSstFiles.putAll(unregisteredSstFiles);
-		unregisteredSstFiles.clear();
-
-		registered = true;
-	}
-
-	@Override
-	public void unregisterSharedStates(SharedStateRegistry stateRegistry) {
-
-		Preconditions.checkState(registered, "The state handle has not registered its shared states yet.");
-
-		for (Map.Entry<String, StreamStateHandle> newSstFileEntry : unregisteredSstFiles.entrySet()) {
-			SharedStateRegistryKey registryKey =
-				createSharedStateRegistryKeyFromFileName(newSstFileEntry.getKey());
-			stateRegistry.releaseReference(registryKey);
-		}
-
-		for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : registeredSstFiles.entrySet()) {
-			SharedStateRegistryKey registryKey =
-				createSharedStateRegistryKeyFromFileName(oldSstFileEntry.getKey());
-			stateRegistry.releaseReference(registryKey);
-		}
-
-		registered = false;
-	}
-
-	private SharedStateRegistryKey createSharedStateRegistryKeyFromFileName(String fileName) {
-		return new SharedStateRegistryKey(operatorIdentifier + "-" + keyGroupRange + "-" + fileName);
-	}
-}
-
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index b9468f7371821..4bd94fd21b4e5 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -55,13 +55,16 @@
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.CheckpointStreamFactory;
 import org.apache.flink.runtime.state.DoneFuture;
+import org.apache.flink.runtime.state.IncrementalKeyedStateHandle;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
 import org.apache.flink.runtime.state.KeyGroupRangeOffsets;
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedBackendSerializationProxy;
 import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.PlaceholderStreamStateHandle;
 import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo;
+import org.apache.flink.runtime.state.StateHandleID;
 import org.apache.flink.runtime.state.StateMigrationUtil;
 import org.apache.flink.runtime.state.StateObject;
 import org.apache.flink.runtime.state.StateUtil;
@@ -72,7 +75,6 @@
 import org.apache.flink.runtime.state.internal.InternalMapState;
 import org.apache.flink.runtime.state.internal.InternalReducingState;
 import org.apache.flink.runtime.state.internal.InternalValueState;
-import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
 import org.apache.flink.runtime.util.SerializableObject;
 import org.apache.flink.util.FileUtils;
 import org.apache.flink.util.IOUtils;
@@ -172,7 +174,7 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
 	private final boolean enableIncrementalCheckpointing;
 
 	/** The sst files materialized in pending checkpoints */
-	private final SortedMap<Long, Map<String, StreamStateHandle>> materializedSstFiles = new TreeMap<>();
+	private final SortedMap<Long, Map<StateHandleID, StreamStateHandle>> materializedSstFiles = new TreeMap<>();
 
 	/** The identifier of the last completed checkpoint */
 	private long lastCompletedCheckpointId = -1;
@@ -723,7 +725,7 @@ private static final class RocksDBIncrementalSnapshotOperation {
 		private final long checkpointTimestamp;
 
 		/** All sst files that were part of the last previously completed checkpoint */
-		private Map<String, StreamStateHandle> baseSstFiles;
+		private Map<StateHandleID, StreamStateHandle> baseSstFiles;
 
 		/** The state meta data */
 		private final List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots = new ArrayList<>();
@@ -735,13 +737,13 @@ private static final class RocksDBIncrementalSnapshotOperation {
 		private final CloseableRegistry closeableRegistry = new CloseableRegistry();
 
 		// new sst files since the last completed checkpoint
-		private final Map<String, StreamStateHandle> newSstFiles = new HashMap<>();
+		private final Map<StateHandleID, StreamStateHandle> newSstFiles = new HashMap<>();
 
 		// old sst files which have been materialized in previous completed checkpoints
-		private final Map<String, StreamStateHandle> oldSstFiles = new HashMap<>();
+		private final Map<StateHandleID, StreamStateHandle> oldSstFiles = new HashMap<>();
 
 		// handles to the misc files in the current snapshot
-		private final Map<String, StreamStateHandle> miscFiles = new HashMap<>();
+		private final Map<StateHandleID, StreamStateHandle> miscFiles = new HashMap<>();
 
 		private StreamStateHandle metaStateHandle = null;
 
@@ -865,8 +867,9 @@ KeyedStateHandle materializeSnapshot() throws Exception {
 			FileStatus[] fileStatuses = backupFileSystem.listStatus(backupPath);
 			if (fileStatuses != null) {
 				for (FileStatus fileStatus : fileStatuses) {
-					Path filePath = fileStatus.getPath();
-					String fileName = filePath.getName();
+					final Path filePath = fileStatus.getPath();
+					final String fileName = filePath.getName();
+					final StateHandleID stateHandleID = new StateHandleID(fileName);
 
 					if (fileName.endsWith(SST_FILE_SUFFIX)) {
 						StreamStateHandle fileHandle =
@@ -874,20 +877,24 @@ KeyedStateHandle materializeSnapshot() throws Exception {
 
 						if (fileHandle == null) {
 							fileHandle = materializeStateData(filePath);
-							newSstFiles.put(fileName, fileHandle);
+							newSstFiles.put(stateHandleID, fileHandle);
 						} else {
 							// we introduce a placeholder state handle, that is replaced with the
 							// original from the shared state registry (created from a previous checkpoint)
-							oldSstFiles.put(fileName, new PlaceholderStreamStateHandle(fileHandle.getStateSize()));
+							oldSstFiles.put(
+								stateHandleID,
+								new PlaceholderStreamStateHandle(fileHandle.getStateSize()));
 						}
 					} else {
 						StreamStateHandle fileHandle = materializeStateData(filePath);
-						miscFiles.put(fileName, fileHandle);
+						miscFiles.put(stateHandleID, fileHandle);
 					}
 				}
 			}
 
-			Map<String, StreamStateHandle> sstFiles = new HashMap<>(newSstFiles.size() + oldSstFiles.size());
+			Map<StateHandleID, StreamStateHandle> sstFiles =
+				new HashMap<>(newSstFiles.size() + oldSstFiles.size());
+
 			sstFiles.putAll(newSstFiles);
 			sstFiles.putAll(oldSstFiles);
 
@@ -895,7 +902,7 @@ KeyedStateHandle materializeSnapshot() throws Exception {
 				stateBackend.materializedSstFiles.put(checkpointId, sstFiles);
 			}
 
-			return new RocksDBIncrementalKeyedStateHandle(
+			return new IncrementalKeyedStateHandle(
 				stateBackend.operatorIdentifier,
 				stateBackend.keyGroupRange,
 				checkpointId,
@@ -940,39 +947,6 @@ void releaseResources(boolean canceled) {
 				}
 			}
 		}
-
-		/**
-		 * A placeholder state handle for shared state that will replaced by an original that was
-		 * created in a previous checkpoint. So we don't have to send the handle twice, e.g. in
-		 * case of {@link ByteStreamStateHandle}.
-		 */
-		private static final class PlaceholderStreamStateHandle implements StreamStateHandle {
-
-			private static final long serialVersionUID = 1L;
-
-			/** We remember the size of the original file for which this is a placeholder */
-			private final long originalSize;
-
-			public PlaceholderStreamStateHandle(long originalSize) {
-				this.originalSize = originalSize;
-			}
-
-			@Override
-			public FSDataInputStream openInputStream() {
-				throw new UnsupportedOperationException(
-					"This is only a placeholder to be replaced by a real StreamStateHandle in the checkpoint coordinator.");
-			}
-
-			@Override
-			public void discardState() throws Exception {
-				// nothing to do.
-			}
-
-			@Override
-			public long getStateSize() {
-				return originalSize;
-			}
-		}
 	}
 
 	@Override
@@ -989,7 +963,7 @@ public void restore(Collection<KeyedStateHandle> restoreState) throws Exception
 			} else if (MigrationUtil.isOldSavepointKeyedState(restoreState)) {
 				LOG.info("Converting RocksDB state from old savepoint.");
 				restoreOldSavepointKeyedState(restoreState);
-			} else if (restoreState.iterator().next() instanceof RocksDBIncrementalKeyedStateHandle) {
+			} else if (restoreState.iterator().next() instanceof IncrementalKeyedStateHandle) {
 				RocksDBIncrementalRestoreOperation restoreOperation = new RocksDBIncrementalRestoreOperation(this);
 				restoreOperation.restore(restoreState);
 			} else {
@@ -1302,7 +1276,7 @@ private void readStateData(
 		}
 
 		private void restoreInstance(
-				RocksDBIncrementalKeyedStateHandle restoreStateHandle,
+				IncrementalKeyedStateHandle restoreStateHandle,
 				boolean hasExtraKeys) throws Exception {
 
 			// read state data
@@ -1311,29 +1285,16 @@ private void restoreInstance(
 				UUID.randomUUID().toString());
 
 			try {
-				Map<String, StreamStateHandle> newSstFiles = restoreStateHandle.getUnregisteredSstFiles();
-				for (Map.Entry<String, StreamStateHandle> newSstFileEntry : newSstFiles.entrySet()) {
-					String fileName = newSstFileEntry.getKey();
-					StreamStateHandle remoteFileHandle = newSstFileEntry.getValue();
+				final Map<StateHandleID, StreamStateHandle> newSstFiles =
+					restoreStateHandle.getCreatedSharedState();
+				final Map<StateHandleID, StreamStateHandle> oldSstFiles =
+					restoreStateHandle.getReferencedSharedState();
+				final Map<StateHandleID, StreamStateHandle> miscFiles =
+					restoreStateHandle.getPrivateState();
 
-					readStateData(new Path(restoreInstancePath, fileName), remoteFileHandle);
-				}
-
-				Map<String, StreamStateHandle> oldSstFiles = restoreStateHandle.getRegisteredSstFiles();
-				for (Map.Entry<String, StreamStateHandle> oldSstFileEntry : oldSstFiles.entrySet()) {
-					String fileName = oldSstFileEntry.getKey();
-					StreamStateHandle remoteFileHandle = oldSstFileEntry.getValue();
-
-					readStateData(new Path(restoreInstancePath, fileName), remoteFileHandle);
-				}
-
-				Map<String, StreamStateHandle> miscFiles = restoreStateHandle.getMiscFiles();
-				for (Map.Entry<String, StreamStateHandle> miscFileEntry : miscFiles.entrySet()) {
-					String fileName = miscFileEntry.getKey();
-					StreamStateHandle remoteFileHandle = miscFileEntry.getValue();
-
-					readStateData(new Path(restoreInstancePath, fileName), remoteFileHandle);
-				}
+				readAllStateData(newSstFiles, restoreInstancePath);
+				readAllStateData(oldSstFiles, restoreInstancePath);
+				readAllStateData(miscFiles, restoreInstancePath);
 
 				// read meta data
 				List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> stateMetaInfoSnapshots =
@@ -1425,26 +1386,9 @@ private void restoreInstance(
 						throw new IOException("Could not create RocksDB data directory.");
 					}
 
-					for (String newSstFileName : newSstFiles.keySet()) {
-						File restoreFile = new File(restoreInstancePath.getPath(), newSstFileName);
-						File targetFile = new File(stateBackend.instanceRocksDBPath, newSstFileName);
-
-						Files.createLink(targetFile.toPath(), restoreFile.toPath());
-					}
-
-					for (String oldSstFileName : oldSstFiles.keySet()) {
-						File restoreFile = new File(restoreInstancePath.getPath(), oldSstFileName);
-						File targetFile = new File(stateBackend.instanceRocksDBPath, oldSstFileName);
-
-						Files.createLink(targetFile.toPath(), restoreFile.toPath());
-					}
-
-					for (String miscFileName : miscFiles.keySet()) {
-						File restoreFile = new File(restoreInstancePath.getPath(), miscFileName);
-						File targetFile = new File(stateBackend.instanceRocksDBPath, miscFileName);
-
-						Files.createLink(targetFile.toPath(), restoreFile.toPath());
-					}
+					createFileHardLinksInRestorePath(newSstFiles, restoreInstancePath);
+					createFileHardLinksInRestorePath(oldSstFiles, restoreInstancePath);
+					createFileHardLinksInRestorePath(miscFiles, restoreInstancePath);
 
 					List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
 					stateBackend.db = stateBackend.openDB(
@@ -1470,7 +1414,7 @@ private void restoreInstance(
 
 
 					// use the restore sst files as the base for succeeding checkpoints
-					Map<String, StreamStateHandle> sstFiles = new HashMap<>();
+					Map<StateHandleID, StreamStateHandle> sstFiles = new HashMap<>();
 					sstFiles.putAll(newSstFiles);
 					sstFiles.putAll(oldSstFiles);
 					stateBackend.materializedSstFiles.put(restoreStateHandle.getCheckpointId(), sstFiles);
@@ -1485,6 +1429,29 @@ private void restoreInstance(
 			}
 		}
 
+		private void readAllStateData(
+			Map<StateHandleID, StreamStateHandle> stateHandleMap,
+			Path restoreInstancePath) throws IOException {
+
+			for (Map.Entry<StateHandleID, StreamStateHandle> entry : stateHandleMap.entrySet()) {
+				StateHandleID stateHandleID = entry.getKey();
+				StreamStateHandle remoteFileHandle = entry.getValue();
+				readStateData(new Path(restoreInstancePath, stateHandleID.toString()), remoteFileHandle);
+			}
+		}
+
+		private void createFileHardLinksInRestorePath(
+			Map<StateHandleID, StreamStateHandle> stateHandleMap,
+			Path restoreInstancePath) throws IOException {
+
+			for (StateHandleID stateHandleID : stateHandleMap.keySet()) {
+				String newSstFileName = stateHandleID.toString();
+				File restoreFile = new File(restoreInstancePath.getPath(), newSstFileName);
+				File targetFile = new File(stateBackend.instanceRocksDBPath, newSstFileName);
+				Files.createLink(targetFile.toPath(), restoreFile.toPath());
+			}
+		}
+
 		void restore(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {
 
 			boolean hasExtraKeys = (restoreStateHandles.size() > 1 ||
@@ -1496,13 +1463,13 @@ void restore(Collection<KeyedStateHandle> restoreStateHandles) throws Exception
 
 			for (KeyedStateHandle rawStateHandle : restoreStateHandles) {
 
-				if (! (rawStateHandle instanceof RocksDBIncrementalKeyedStateHandle)) {
+				if (! (rawStateHandle instanceof IncrementalKeyedStateHandle)) {
 					throw new IllegalStateException("Unexpected state handle type, " +
-						"expected " + RocksDBIncrementalKeyedStateHandle.class +
+						"expected " + IncrementalKeyedStateHandle.class +
 						", but found " + rawStateHandle.getClass());
 				}
 
-				RocksDBIncrementalKeyedStateHandle keyedStateHandle = (RocksDBIncrementalKeyedStateHandle) rawStateHandle;
+				IncrementalKeyedStateHandle keyedStateHandle = (IncrementalKeyedStateHandle) rawStateHandle;
 
 				restoreInstance(keyedStateHandle, hasExtraKeys);
 			}
diff --git a/flink-core/src/main/java/org/apache/flink/util/StringBasedID.java b/flink-core/src/main/java/org/apache/flink/util/StringBasedID.java
new file mode 100644
index 0000000000000..7245e612090fe
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/StringBasedID.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import java.io.Serializable;
+
+/**
+ * Base class for typed IDs that are internally represented by a string. This class is not intended
+ * for direct use, but should be subclassed for type-safety.
+ */
+public class StringBasedID implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	/**
+	 * Uses a String as internal representation
+	 */
+	private final String keyString;
+
+	/**
+	 * Protected constructor to enfore that subclassing.
+	 */
+	protected StringBasedID(String keyString) {
+		this.keyString = Preconditions.checkNotNull(keyString);
+	}
+
+	public String getKeyString() {
+		return keyString;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+
+		StringBasedID that = (StringBasedID) o;
+		return keyString.equals(that.keyString);
+	}
+
+	@Override
+	public int hashCode() {
+		return keyString.hashCode();
+	}
+
+	@Override
+	public String toString() {
+		return keyString;
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
index 1b5f2c6f12aa9..b71418b41611b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2Serializer.java
@@ -20,14 +20,17 @@
 
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.checkpoint.MasterState;
-import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.checkpoint.OperatorState;
 import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.state.IncrementalKeyedStateHandle;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyGroupRangeOffsets;
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
 import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.PlaceholderStreamStateHandle;
+import org.apache.flink.runtime.state.StateHandleID;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
 import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
@@ -71,6 +74,8 @@ class SavepointV2Serializer implements SavepointSerializer<SavepointV2> {
 	private static final byte FILE_STREAM_STATE_HANDLE = 2;
 	private static final byte KEY_GROUPS_HANDLE = 3;
 	private static final byte PARTITIONABLE_OPERATOR_STATE_HANDLE = 4;
+	private static final byte INCREMENTAL_KEY_GROUPS_HANDLE = 5;
+	private static final byte PLACEHOLDER_STREAM_STATE_HANDLE = 6;
 
 	/** The singleton instance of the serializer */
 	public static final SavepointV2Serializer INSTANCE = new SavepointV2Serializer();
@@ -287,7 +292,6 @@ private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis)
 
 		KeyedStateHandle keyedStateStream = deserializeKeyedStateHandle(dis);
 
-
 		return new OperatorSubtaskState(
 				nonPartitionableState,
 				operatorStateBackend,
@@ -311,19 +315,63 @@ private static void serializeKeyedStateHandle(
 				dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
 			}
 			serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
+		} else if (stateHandle instanceof IncrementalKeyedStateHandle) {
+			IncrementalKeyedStateHandle incrementalKeyedStateHandle =
+				(IncrementalKeyedStateHandle) stateHandle;
+
+			dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);
+
+			dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
+			dos.writeUTF(incrementalKeyedStateHandle.getOperatorIdentifier());
+			dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
+			dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
+
+			serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);
+
+			serializeStreamStateHandleMap(incrementalKeyedStateHandle.getCreatedSharedState(), dos);
+			serializeStreamStateHandleMap(incrementalKeyedStateHandle.getReferencedSharedState(), dos);
+			serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
 		} else {
 			throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
 		}
 	}
 
+	private static void serializeStreamStateHandleMap(
+		Map<StateHandleID, StreamStateHandle> map,
+		DataOutputStream dos) throws IOException {
+		dos.writeInt(map.size());
+		for (Map.Entry<StateHandleID, StreamStateHandle> entry : map.entrySet()) {
+			dos.writeUTF(entry.getKey().toString());
+			serializeStreamStateHandle(entry.getValue(), dos);
+		}
+	}
+
+	private static Map<StateHandleID, StreamStateHandle> deserializeStreamStateHandleMap(
+		DataInputStream dis) throws IOException {
+
+		final int size = dis.readInt();
+		Map<StateHandleID, StreamStateHandle> result = new HashMap<>(size);
+
+		for (int i = 0; i < size; ++i) {
+			StateHandleID stateHandleID = new StateHandleID(dis.readUTF());
+			StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
+			result.put(stateHandleID, stateHandle);
+		}
+
+		return result;
+	}
+
 	private static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
 		final int type = dis.readByte();
 		if (NULL_HANDLE == type) {
+
 			return null;
 		} else if (KEY_GROUPS_HANDLE == type) {
+
 			int startKeyGroup = dis.readInt();
 			int numKeyGroups = dis.readInt();
-			KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
+			KeyGroupRange keyGroupRange =
+				KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
 			long[] offsets = new long[numKeyGroups];
 			for (int i = 0; i < numKeyGroups; ++i) {
 				offsets[i] = dis.readLong();
@@ -332,6 +380,28 @@ private static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis)
 				keyGroupRange, offsets);
 			StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
 			return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
+		} else if (INCREMENTAL_KEY_GROUPS_HANDLE == type) {
+
+			long checkpointId = dis.readLong();
+			String operatorId = dis.readUTF();
+			int startKeyGroup = dis.readInt();
+			int numKeyGroups = dis.readInt();
+			KeyGroupRange keyGroupRange =
+				KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
+
+			StreamStateHandle metaDataStateHandle = deserializeStreamStateHandle(dis);
+			Map<StateHandleID, StreamStateHandle> createdStates = deserializeStreamStateHandleMap(dis);
+			Map<StateHandleID, StreamStateHandle> referencedStates = deserializeStreamStateHandleMap(dis);
+			Map<StateHandleID, StreamStateHandle> privateStates = deserializeStreamStateHandleMap(dis);
+
+			return new IncrementalKeyedStateHandle(
+				operatorId,
+				keyGroupRange,
+				checkpointId,
+				createdStates,
+				referencedStates,
+				privateStates,
+				metaDataStateHandle);
 		} else {
 			throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
 		}
@@ -415,7 +485,10 @@ private static void serializeStreamStateHandle(
 			byte[] internalData = byteStreamStateHandle.getData();
 			dos.writeInt(internalData.length);
 			dos.write(byteStreamStateHandle.getData());
-
+		} else if (stateHandle instanceof PlaceholderStreamStateHandle) {
+			PlaceholderStreamStateHandle placeholder = (PlaceholderStreamStateHandle) stateHandle;
+			dos.writeByte(PLACEHOLDER_STREAM_STATE_HANDLE);
+			dos.writeLong(placeholder.getStateSize());
 		} else {
 			throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
 		}
@@ -437,6 +510,8 @@ private static StreamStateHandle deserializeStreamStateHandle(DataInputStream di
 			byte[] data = new byte[numBytes];
 			dis.readFully(data);
 			return new ByteStreamStateHandle(handleName, data);
+		} else if (PLACEHOLDER_STREAM_STATE_HANDLE == type) {
+			return new PlaceholderStreamStateHandle(dis.readLong());
 		} else {
 			throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
 		}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/IncrementalKeyedStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/IncrementalKeyedStateHandle.java
new file mode 100644
index 0000000000000..706e219a028d0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/IncrementalKeyedStateHandle.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * The handle to states of an incremental snapshot.
+ * <p>
+ * The states contained in an incremental snapshot include
+ * <ul>
+ * <li> Created shared state which includes (the supposed to be) shared files produced since the last
+ * completed checkpoint. These files can be referenced by succeeding checkpoints if the
+ * checkpoint succeeds to complete. </li>
+ * <li> Referenced shared state which includes the shared files materialized in previous
+ * checkpoints. </li>
+ * <li> Private state which includes all other files, typically mutable, that cannot be shared by
+ * other checkpoints. </li>
+ * <li> Backend meta state which includes the information of existing states. </li>
+ * </ul>
+ *
+ * IMPORTANT: This class currently overrides equals and hash code only for testing purposes. They
+ * should not be called from production code. This means this class is also not suited to serve as
+ * a key, e.g. in hash maps.
+ */
+public class IncrementalKeyedStateHandle implements KeyedStateHandle {
+
+	private static final Logger LOG = LoggerFactory.getLogger(IncrementalKeyedStateHandle.class);
+
+	private static final long serialVersionUID = -8328808513197388231L;
+
+	/**
+	 * The operator instance identifier for this handle
+	 */
+	private final String operatorIdentifier;
+
+	/**
+	 * The key-group range covered by this state handle
+	 */
+	private final KeyGroupRange keyGroupRange;
+
+	/**
+	 * The checkpoint Id
+	 */
+	private final long checkpointId;
+
+	/**
+	 * State that the incremental checkpoint created new
+	 */
+	private final Map<StateHandleID, StreamStateHandle> createdSharedState;
+
+	/**
+	 * State that the incremental checkpoint references from previous checkpoints
+	 */
+	private final Map<StateHandleID, StreamStateHandle> referencedSharedState;
+
+	/**
+	 * Private state in the incremental checkpoint
+	 */
+	private final Map<StateHandleID, StreamStateHandle> privateState;
+
+	/**
+	 * Primary meta data state of the incremental checkpoint
+	 */
+	private final StreamStateHandle metaStateHandle;
+
+	/**
+	 * True if the state handle has already registered shared states.
+	 * <p>
+	 * Once the shared states are registered, it's the {@link SharedStateRegistry}'s
+	 * responsibility to maintain the shared states. But in the cases where the
+	 * state handle is discarded before performing the registration, the handle
+	 * should delete all the shared states created by it.
+	 */
+	private boolean registered;
+
+	public IncrementalKeyedStateHandle(
+		String operatorIdentifier,
+		KeyGroupRange keyGroupRange,
+		long checkpointId,
+		Map<StateHandleID, StreamStateHandle> createdSharedState,
+		Map<StateHandleID, StreamStateHandle> referencedSharedState,
+		Map<StateHandleID, StreamStateHandle> privateState,
+		StreamStateHandle metaStateHandle) {
+
+		this.operatorIdentifier = Preconditions.checkNotNull(operatorIdentifier);
+		this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange);
+		this.checkpointId = checkpointId;
+		this.createdSharedState = Preconditions.checkNotNull(createdSharedState);
+		this.referencedSharedState = Preconditions.checkNotNull(referencedSharedState);
+		this.privateState = Preconditions.checkNotNull(privateState);
+		this.metaStateHandle = Preconditions.checkNotNull(metaStateHandle);
+		this.registered = false;
+	}
+
+	@Override
+	public KeyGroupRange getKeyGroupRange() {
+		return keyGroupRange;
+	}
+
+	public long getCheckpointId() {
+		return checkpointId;
+	}
+
+	public Map<StateHandleID, StreamStateHandle> getCreatedSharedState() {
+		return createdSharedState;
+	}
+
+	public Map<StateHandleID, StreamStateHandle> getReferencedSharedState() {
+		return referencedSharedState;
+	}
+
+	public Map<StateHandleID, StreamStateHandle> getPrivateState() {
+		return privateState;
+	}
+
+	public StreamStateHandle getMetaStateHandle() {
+		return metaStateHandle;
+	}
+
+	public String getOperatorIdentifier() {
+		return operatorIdentifier;
+	}
+
+	@Override
+	public KeyedStateHandle getIntersection(KeyGroupRange keyGroupRange) {
+		if (this.keyGroupRange.getIntersection(keyGroupRange) != KeyGroupRange.EMPTY_KEY_GROUP_RANGE) {
+			return this;
+		} else {
+			return null;
+		}
+	}
+
+	@Override
+	public void discardState() throws Exception {
+
+		Preconditions.checkState(!registered, "Attempt to dispose a registered composite state with registered shared state. Must unregister first.");
+
+		try {
+			metaStateHandle.discardState();
+		} catch (Exception e) {
+			LOG.warn("Could not properly discard meta data.", e);
+		}
+
+		try {
+			StateUtil.bestEffortDiscardAllStateObjects(privateState.values());
+		} catch (Exception e) {
+			LOG.warn("Could not properly discard misc file states.", e);
+		}
+
+		try {
+			StateUtil.bestEffortDiscardAllStateObjects(createdSharedState.values());
+		} catch (Exception e) {
+			LOG.warn("Could not properly discard new sst file states.", e);
+		}
+
+	}
+
+	@Override
+	public long getStateSize() {
+		long size = getPrivateStateSize();
+
+		for (StreamStateHandle oldSstFileHandle : referencedSharedState.values()) {
+			size += oldSstFileHandle.getStateSize();
+		}
+
+		return size;
+	}
+
+	/**
+	 * Returns the size of the state that is privately owned by this handle.
+	 */
+	public long getPrivateStateSize() {
+		long size = StateUtil.getStateSize(metaStateHandle);
+
+		for (StreamStateHandle newSstFileHandle : createdSharedState.values()) {
+			size += newSstFileHandle.getStateSize();
+		}
+
+		for (StreamStateHandle miscFileHandle : privateState.values()) {
+			size += miscFileHandle.getStateSize();
+		}
+
+		return size;
+	}
+
+	@Override
+	public void registerSharedStates(SharedStateRegistry stateRegistry) {
+
+		Preconditions.checkState(!registered, "The state handle has already registered its shared states.");
+
+		for (Map.Entry<StateHandleID, StreamStateHandle> newSstFileEntry : createdSharedState.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(newSstFileEntry.getKey());
+
+			SharedStateRegistry.Result result =
+				stateRegistry.registerNewReference(registryKey, newSstFileEntry.getValue());
+
+			// We update our reference with the result from the registry, to prevent the following
+			// problem:
+			// A previous checkpoint n has already registered the state. This can happen if a
+			// following checkpoint (n + x) wants to reference the same state before the backend got
+			// notified that checkpoint n completed. In this case, the shared registry did
+			// deduplication and returns the previous reference.
+			newSstFileEntry.setValue(result.getReference());
+		}
+
+		for (Map.Entry<StateHandleID, StreamStateHandle> oldSstFileName : referencedSharedState.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(oldSstFileName.getKey());
+
+			SharedStateRegistry.Result result = stateRegistry.obtainReference(registryKey);
+
+			// Again we update our state handle with the result from the registry, thus replacing
+			// placeholder state handles with the originals.
+			oldSstFileName.setValue(result.getReference());
+		}
+
+		// Migrate state from unregistered to registered, so that it will not count as private state
+		// for #discardState() from now.
+		referencedSharedState.putAll(createdSharedState);
+		createdSharedState.clear();
+
+		registered = true;
+	}
+
+	@Override
+	public void unregisterSharedStates(SharedStateRegistry stateRegistry) {
+
+		Preconditions.checkState(registered, "The state handle has not registered its shared states yet.");
+
+		for (Map.Entry<StateHandleID, StreamStateHandle> newSstFileEntry : createdSharedState.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(newSstFileEntry.getKey());
+			stateRegistry.releaseReference(registryKey);
+		}
+
+		for (Map.Entry<StateHandleID, StreamStateHandle> oldSstFileEntry : referencedSharedState.entrySet()) {
+			SharedStateRegistryKey registryKey =
+				createSharedStateRegistryKeyFromFileName(oldSstFileEntry.getKey());
+			stateRegistry.releaseReference(registryKey);
+		}
+
+		registered = false;
+	}
+
+	private SharedStateRegistryKey createSharedStateRegistryKeyFromFileName(StateHandleID shId) {
+		return new SharedStateRegistryKey(operatorIdentifier + '-' + keyGroupRange, shId);
+	}
+
+	/**
+	 * This method is should only be called in tests! This should never serve as key in a hash map.
+	 */
+	@VisibleForTesting
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+
+		IncrementalKeyedStateHandle that = (IncrementalKeyedStateHandle) o;
+
+		if (getCheckpointId() != that.getCheckpointId()) {
+			return false;
+		}
+		if (!getOperatorIdentifier().equals(that.getOperatorIdentifier())) {
+			return false;
+		}
+		if (!getKeyGroupRange().equals(that.getKeyGroupRange())) {
+			return false;
+		}
+		if (!getCreatedSharedState().equals(that.getCreatedSharedState())) {
+			return false;
+		}
+		if (!getReferencedSharedState().equals(that.getReferencedSharedState())) {
+			return false;
+		}
+		if (!getPrivateState().equals(that.getPrivateState())) {
+			return false;
+		}
+		return getMetaStateHandle().equals(that.getMetaStateHandle());
+	}
+
+	/**
+	 * This method should only be called in tests! This should never serve as key in a hash map.
+	 */
+	@VisibleForTesting
+	@Override
+	public int hashCode() {
+		int result = getOperatorIdentifier().hashCode();
+		result = 31 * result + getKeyGroupRange().hashCode();
+		result = 31 * result + (int) (getCheckpointId() ^ (getCheckpointId() >>> 32));
+		result = 31 * result + getCreatedSharedState().hashCode();
+		result = 31 * result + getReferencedSharedState().hashCode();
+		result = 31 * result + getPrivateState().hashCode();
+		result = 31 * result + getMetaStateHandle().hashCode();
+		return result;
+	}
+}
+
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/PlaceholderStreamStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/PlaceholderStreamStateHandle.java
new file mode 100644
index 0000000000000..2136061a4e2fd
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/PlaceholderStreamStateHandle.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
+
+/**
+ * A placeholder state handle for shared state that will replaced by an original that was
+ * created in a previous checkpoint. So we don't have to send the handle twice, e.g. in
+ * case of {@link ByteStreamStateHandle}. To be used in the referenced states of
+ * {@link IncrementalKeyedStateHandle}.
+ * <p>
+ * IMPORTANT: This class currently overrides equals and hash code only for testing purposes. They
+ * should not be called from production code. This means this class is also not suited to serve as
+ * a key, e.g. in hash maps.
+ */
+public class PlaceholderStreamStateHandle implements StreamStateHandle {
+
+	private static final long serialVersionUID = 1L;
+
+	/** We remember the size of the original file for which this is a placeholder */
+	private final long originalSize;
+
+	public PlaceholderStreamStateHandle(long originalSize) {
+		this.originalSize = originalSize;
+	}
+
+	@Override
+	public FSDataInputStream openInputStream() {
+		throw new UnsupportedOperationException(
+			"This is only a placeholder to be replaced by a real StreamStateHandle in the checkpoint coordinator.");
+	}
+
+	@Override
+	public void discardState() throws Exception {
+		// nothing to do.
+	}
+
+	@Override
+	public long getStateSize() {
+		return originalSize;
+	}
+
+	/**
+	 * This method is should only be called in tests! This should never serve as key in a hash map.
+	 */
+	@VisibleForTesting
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+
+		PlaceholderStreamStateHandle that = (PlaceholderStreamStateHandle) o;
+
+		return originalSize == that.originalSize;
+	}
+
+	/**
+	 * This method is should only be called in tests! This should never serve as key in a hash map.
+	 */
+	@VisibleForTesting
+	@Override
+	public int hashCode() {
+		return (int) (originalSize ^ (originalSize >>> 32));
+	}
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
deleted file mode 100644
index c8c404672b23f..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateHandle.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.state;
-
-/**
- * A handle to those states that are referenced by different checkpoints.
- *
- * <p> Each shared state handle is identified by a unique key. Two shared states
- * are considered equal if their keys are identical.
- *
- * <p> All shared states are registered at the {@link SharedStateRegistry} once
- * they are received by the {@link org.apache.flink.runtime.checkpoint.CheckpointCoordinator}
- * and will be unregistered when the checkpoints are discarded. A shared state
- * will be discarded once it is not referenced by any checkpoint. A shared state
- * should not be referenced any more if it has been discarded.
- */
-public interface SharedStateHandle extends StateObject {
-
-	/**
-	 * Return the identifier of the shared state.
-	 */
-	String getRegistrationKey();
-}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
index dbf464238c430..9cfdec7671c2a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
@@ -31,7 +31,7 @@
 /**
  * A {@code SharedStateRegistry} will be deployed in the 
  * {@link org.apache.flink.runtime.checkpoint.CompletedCheckpointStore} to
- * maintain the reference count of {@link SharedStateHandle}s which are shared
+ * maintain the reference count of {@link StreamStateHandle}s which are shared
  * among different incremental checkpoints.
  */
 public class SharedStateRegistry {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java
index 9e5935957a4cf..58262ca8e2a97 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistryKey.java
@@ -18,9 +18,8 @@
 
 package org.apache.flink.runtime.state;
 
-import org.apache.flink.util.Preconditions;
-
-import java.io.Serializable;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.util.StringBasedID;
 
 /**
  * This class represents a key that uniquely identifies (on a logical level) state handles for
@@ -28,41 +27,16 @@
  * be the same should have the same {@link SharedStateRegistryKey}. The meaning of logical
  * equivalence is up to the application.
  */
-public class SharedStateRegistryKey implements Serializable {
+public class SharedStateRegistryKey extends StringBasedID {
 
 	private static final long serialVersionUID = 1L;
 
-	/** Uses a String as internal representation */
-	private final String keyString;
-
-	public SharedStateRegistryKey(String keyString) {
-		this.keyString = Preconditions.checkNotNull(keyString);
-	}
-
-	public String getKeyString() {
-		return keyString;
+	public SharedStateRegistryKey(String prefix, StateHandleID stateHandleID) {
+		super(prefix + '-' + stateHandleID);
 	}
 
-	@Override
-	public boolean equals(Object o) {
-		if (this == o) {
-			return true;
-		}
-		if (o == null || getClass() != o.getClass()) {
-			return false;
-		}
-
-		SharedStateRegistryKey that = (SharedStateRegistryKey) o;
-		return keyString.equals(that.keyString);
-	}
-
-	@Override
-	public int hashCode() {
-		return keyString.hashCode();
-	}
-
-	@Override
-	public String toString() {
-		return keyString;
+	@VisibleForTesting
+	public SharedStateRegistryKey(String keyString) {
+		super(keyString);
 	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java
deleted file mode 100644
index b7362528b683f..0000000000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.state;
-
-/**
- * StateHandle is a general handle interface meant to abstract operator state fetching. 
- * A StateHandle implementation can for example include the state itself in cases where the state 
- * is lightweight or fetching it lazily from some external storage when the state is too large.
- */
-public interface StateHandle<T> extends StateObject {
-
-	/**
-	 * This retrieves and return the state represented by the handle.
-	 *
-	 * @param userCodeClassLoader Class loader for deserializing user code specific classes
-	 *
-	 * @return The state represented by the handle.
-	 * @throws java.lang.Exception Thrown, if the state cannot be fetched.
-	 */
-	T getState(ClassLoader userCodeClassLoader) throws Exception;
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateUtilTest.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandleID.java
similarity index 56%
rename from flink-runtime/src/test/java/org/apache/flink/runtime/state/StateUtilTest.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandleID.java
index d6966d0a039bb..5e95cff5543cf 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateUtilTest.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandleID.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,19 +18,20 @@
 
 package org.apache.flink.runtime.state;
 
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
+import org.apache.flink.util.StringBasedID;
 
-import java.util.concurrent.RunnableFuture;
+/**
+ * Unique ID that allows for logical comparison between state handles.
+ * <p>
+ * Two state handles that are considered as logically equal should always return the same ID
+ * (whatever logically equal means is up to the implementation). For example, this could be based
+ * on the string representation of the full filepath for a state that is based on a file.
+ */
+public class StateHandleID extends StringBasedID {
 
-public class StateUtilTest extends TestLogger {
+	private static final long serialVersionUID = 1L;
 
-	/**
-	 * Tests that {@link StateUtil#discardStateFuture} can handle state futures with null value.
-	 */
-	@Test
-	public void testDiscardRunnableFutureWithNullValue() throws Exception {
-		RunnableFuture<StateHandle<?>> stateFuture = DoneFuture.nullValue();
-		StateUtil.discardStateFuture(stateFuture);
+	public StateHandleID(String keyString) {
+		super(keyString);
 	}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
index ba77dbc0b6ca4..b63782d4b41c8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/CheckpointTestUtils.java
@@ -27,10 +27,15 @@
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.IncrementalKeyedStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.KeyGroupRangeOffsets;
 import org.apache.flink.runtime.state.KeyGroupsStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle.StateMetaInfo;
+import org.apache.flink.runtime.state.PlaceholderStreamStateHandle;
+import org.apache.flink.runtime.state.StateHandleID;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.util.TestByteStreamStateHandleDeepCompare;
 import org.apache.flink.util.StringUtils;
@@ -41,6 +46,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.UUID;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -78,6 +84,7 @@ public static Collection<OperatorState> createOperatorStates(
 
 			boolean hasKeyedBackend = random.nextInt(4) != 0;
 			boolean hasKeyedStream = random.nextInt(4) != 0;
+			boolean isIncremental = random.nextInt(3) == 0;
 
 			for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {
 
@@ -108,21 +115,19 @@ public static Collection<OperatorState> createOperatorStates(
 					operatorStateHandleStream = new OperatorStateHandle(offsetsMap, operatorStateStream);
 				}
 
-				KeyGroupsStateHandle keyedStateBackend = null;
-				KeyGroupsStateHandle keyedStateStream = null;
+				KeyedStateHandle keyedStateBackend = null;
+				KeyedStateHandle keyedStateStream = null;
 
 				if (hasKeyedBackend) {
-					keyedStateBackend = new KeyGroupsStateHandle(
-							new KeyGroupRangeOffsets(1, 1, new long[]{42}),
-							new TestByteStreamStateHandleDeepCompare("c", "Hello"
-									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+					if (isIncremental) {
+						keyedStateBackend = createDummyIncrementalKeyedStateHandle(random);
+					} else {
+						keyedStateBackend = createDummyKeyGroupStateHandle(random);
+					}
 				}
 
 				if (hasKeyedStream) {
-					keyedStateStream = new KeyGroupsStateHandle(
-							new KeyGroupRangeOffsets(1, 1, new long[]{23}),
-							new TestByteStreamStateHandleDeepCompare("d", "World"
-									.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+					keyedStateStream = createDummyKeyGroupStateHandle(random);
 				}
 
 				taskState.putState(subtaskIdx, new OperatorSubtaskState(
@@ -210,17 +215,11 @@ public static Collection<TaskState> createTaskStates(
 				KeyGroupsStateHandle keyedStateStream = null;
 
 				if (hasKeyedBackend) {
-					keyedStateBackend = new KeyGroupsStateHandle(
-							new KeyGroupRangeOffsets(1, 1, new long[]{42}),
-							new TestByteStreamStateHandleDeepCompare("c", "Hello"
-								.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+					keyedStateBackend = createDummyKeyGroupStateHandle(random);
 				}
 
 				if (hasKeyedStream) {
-					keyedStateStream = new KeyGroupsStateHandle(
-							new KeyGroupRangeOffsets(1, 1, new long[]{23}),
-							new TestByteStreamStateHandleDeepCompare("d", "World"
-								.getBytes(ConfigConstants.DEFAULT_CHARSET)));
+					keyedStateStream = createDummyKeyGroupStateHandle(random);
 				}
 
 				taskState.putState(subtaskIdx, new SubtaskState(
@@ -272,4 +271,56 @@ public static void assertMasterStateEquality(MasterState a, MasterState b) {
 
 	/** utility class, not meant to be instantiated */
 	private CheckpointTestUtils() {}
+
+
+	private static IncrementalKeyedStateHandle createDummyIncrementalKeyedStateHandle(Random rnd) {
+		return new IncrementalKeyedStateHandle(
+			createRandomUUID(rnd).toString(),
+			new KeyGroupRange(1, 1),
+			42L,
+			createRandomOwnedHandleMap(rnd),
+			createRandomReferencedHandleMap(rnd),
+			createRandomOwnedHandleMap(rnd),
+			createDummyStreamStateHandle(rnd));
+	}
+
+	private static Map<StateHandleID, StreamStateHandle> createRandomOwnedHandleMap(Random rnd) {
+		final int size = rnd.nextInt(4);
+		Map<StateHandleID, StreamStateHandle> result = new HashMap<>(size);
+		for (int i = 0; i < size; ++i) {
+			StateHandleID randomId = new StateHandleID(createRandomUUID(rnd).toString());
+			StreamStateHandle stateHandle = createDummyStreamStateHandle(rnd);
+			result.put(randomId, stateHandle);
+		}
+
+		return result;
+	}
+
+	private static Map<StateHandleID, StreamStateHandle> createRandomReferencedHandleMap(Random rnd) {
+		final int size = rnd.nextInt(4);
+		Map<StateHandleID, StreamStateHandle> result = new HashMap<>(size);
+		for (int i = 0; i < size; ++i) {
+			StateHandleID randomId = new StateHandleID(createRandomUUID(rnd).toString());
+			result.put(randomId, new PlaceholderStreamStateHandle(rnd.nextInt(1024)));
+		}
+
+		return result;
+	}
+
+	private static KeyGroupsStateHandle createDummyKeyGroupStateHandle(Random rnd) {
+		return new KeyGroupsStateHandle(
+			new KeyGroupRangeOffsets(1, 1, new long[]{rnd.nextInt(1024)}),
+			createDummyStreamStateHandle(rnd));
+	}
+
+	private static StreamStateHandle createDummyStreamStateHandle(Random rnd) {
+		return new TestByteStreamStateHandleDeepCompare(
+			String.valueOf(createRandomUUID(rnd)),
+			String.valueOf(createRandomUUID(rnd)).getBytes(ConfigConstants.DEFAULT_CHARSET));
+	}
+
+	private static UUID createRandomUUID(Random rnd) {
+		return new UUID(rnd.nextLong(), rnd.nextLong());
+	}
+
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
index 154d761dd025f..602390babffd5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV2SerializerTest.java
@@ -24,7 +24,6 @@
 import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
 import org.apache.flink.runtime.checkpoint.MasterState;
 import org.apache.flink.runtime.checkpoint.OperatorState;
-
 import org.junit.Test;
 
 import java.io.DataInputStream;

From ae2dad96dc8bc2e9e6843a43df8b19c7e6ed8367 Mon Sep 17 00:00:00 2001
From: Stefan Richter <s.richter@data-artisans.com>
Date: Fri, 12 May 2017 16:01:05 +0200
Subject: [PATCH 253/253] [FLINK-6534] [checkpoint] Use async IO to dispose
 state in SharedStateRegistry

---
 .../checkpoint/AbstractCompletedCheckpointStore.java        | 6 ++++++
 .../checkpoint/ZooKeeperCompletedCheckpointStore.java       | 2 ++
 .../org/apache/flink/runtime/state/SharedStateRegistry.java | 6 +++++-
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java
index f42fd06f3bfed..bf70501de271f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/AbstractCompletedCheckpointStore.java
@@ -20,6 +20,8 @@
 
 import org.apache.flink.runtime.state.SharedStateRegistry;
 
+import java.util.concurrent.Executor;
+
 /**
  * This is the base class that provides implementation of some aspects common for all
  * {@link CompletedCheckpointStore}s.
@@ -34,4 +36,8 @@ public abstract class AbstractCompletedCheckpointStore implements CompletedCheck
 	public AbstractCompletedCheckpointStore() {
 		this.sharedStateRegistry = new SharedStateRegistry();
 	}
+
+	public AbstractCompletedCheckpointStore(Executor asyncIOExecutor) {
+		this.sharedStateRegistry = new SharedStateRegistry(asyncIOExecutor);
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
index 52a4eeaeeaa6a..c8c68bceaa126 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java
@@ -106,6 +106,8 @@ public ZooKeeperCompletedCheckpointStore(
 			RetrievableStateStorageHelper<CompletedCheckpoint> stateStorage,
 			Executor executor) throws Exception {
 
+		super(executor);
+
 		checkArgument(maxNumberOfCheckpointsToRetain >= 1, "Must retain at least one checkpoint.");
 		checkNotNull(stateStorage, "State storage");
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
index 9cfdec7671c2a..f9161b0924960 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
@@ -45,8 +45,12 @@ public class SharedStateRegistry {
 	private final Executor asyncDisposalExecutor;
 
 	public SharedStateRegistry() {
+		this(Executors.directExecutor());
+	}
+
+	public SharedStateRegistry(Executor asyncDisposalExecutor) {
 		this.registeredStates = new HashMap<>();
-		this.asyncDisposalExecutor = Executors.directExecutor(); //TODO: FLINK-6534
+		this.asyncDisposalExecutor = Preconditions.checkNotNull(asyncDisposalExecutor);
 	}
 
 	/**